diff -Nru youtube-dl-2017.10.15.1/ChangeLog youtube-dl-2017.11.15/ChangeLog --- youtube-dl-2017.10.15.1/ChangeLog 2017-10-14 23:16:36.000000000 +0000 +++ youtube-dl-2017.11.15/ChangeLog 2017-11-14 17:15:15.000000000 +0000 @@ -1,3 +1,88 @@ +version 2017.11.15 + +Core +* [common] Skip Apple FairPlay m3u8 manifests (#14741) +* [YoutubeDL] Fix playlist range optimization for --playlist-items (#14740) + +Extractors +* [vshare] Capture and output error message +* [vshare] Fix extraction (#14473) +* [crunchyroll] Extract old RTMP formats +* [tva] Fix extraction (#14736) +* [gamespot] Lower preference of HTTP formats (#14652) +* [instagram:user] Fix extraction (#14699) +* [ccma] Fix typo (#14730) +- Remove sensitive data from logging in messages +* [instagram:user] Fix extraction (#14699) ++ [gamespot] Add support for article URLs (#14652) +* [gamespot] Skip Brightcove Once HTTP formats (#14652) +* [cartoonnetwork] Update tokenizer_src (#14666) ++ [wsj] Recognize another URL pattern (#14704) +* [pandatv] Update API URL and sign format URLs (#14693) +* [crunchyroll] Use old login method (#11572) + + +version 2017.11.06 + +Core ++ [extractor/common] Add protocol for f4m formats +* [f4m] Prefer baseURL for relative URLs (#14660) +* [extractor/common] Respect URL query in _extract_wowza_formats (14645) + +Extractors ++ [hotstar:playlist] Add support for playlists (#12465) +* [hotstar] Bypass geo restriction (#14672) +- [22tracks] Remove extractor (#11024, #14628) ++ [skysport] Sdd support ooyala videos protected with embed_token (#14641) +* [gamespot] Extract formats referenced with new data fields (#14652) +* [spankbang] Detect unavailable videos (#14644) + + +version 2017.10.29 + +Core +* [extractor/common] Prefix format id for audio only HLS formats ++ [utils] Add support for zero years and months in parse_duration + +Extractors +* [egghead] Fix extraction (#14388) ++ [fxnetworks] Extract series metadata (#14603) ++ [younow] Add support for younow.com (#9255, #9432, #12436) +* [dctptv] Fix extraction (#14599) +* [youtube] Restrict embed regex (#14600) +* [vimeo] Restrict iframe embed regex (#14600) +* [soundgasm] Improve extraction (#14588) +- [myvideo] Remove extractor (#8557) ++ [nbc] Add support for classic-tv videos (#14575) ++ [vrtnu] Add support for cookies authentication and simplify (#11873) ++ [canvas] Add support for vrt.be/vrtnu (#11873) +* [twitch:clips] Fix title extraction (#14566) ++ [ndtv] Add support for sub-sites (#14534) +* [dramafever] Fix login error message extraction ++ [nick] Add support for more nickelodeon sites (no, dk, se, ch, fr, es, pt, + ro, hu) (#14553) + + +version 2017.10.20 + +Core +* [downloader/fragment] Report warning instead of error on inconsistent + download state +* [downloader/hls] Fix total fragments count when ad fragments exist + +Extractors +* [parliamentliveuk] Fix extraction (#14524) +* [soundcloud] Update client id (#14546) ++ [servus] Add support for servus.com (#14362) ++ [unity] Add support for unity3d.com (#14528) +* [youtube] Replace youtube redirect URLs in description (#14517) +* [pbs] Restrict direct video URL regular expression (#14519) +* [drtv] Respect preference for direct HTTP formats (#14509) ++ [eporner] Add support for embed URLs (#14507) +* [arte] Capture and output error message +* [niconico] Improve uploader metadata extraction robustness (#14135) + + version 2017.10.15.1 Core @@ -834,7 +919,7 @@ Core + [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955) -+ [adobepass] Improve Comcast and Verison login code (#10803) ++ [adobepass] Improve Comcast and Verizon login code (#10803) + [adobepass] Add support for Verizon (#10803) Extractors diff -Nru youtube-dl-2017.10.15.1/debian/changelog youtube-dl-2017.11.15/debian/changelog --- youtube-dl-2017.10.15.1/debian/changelog 2017-10-17 09:14:35.000000000 +0000 +++ youtube-dl-2017.11.15/debian/changelog 2017-11-21 10:56:55.000000000 +0000 @@ -1,8 +1,14 @@ -youtube-dl (1:2017.10.15.1-1~webupd8~xenial0) xenial; urgency=medium +youtube-dl (1:2017.11.15-1~webupd8~xenial0) xenial; urgency=medium * New upstream release (automated upload) - -- Alin Andrei Tue, 17 Oct 2017 12:14:35 +0300 + -- Alin Andrei Tue, 21 Nov 2017 12:56:55 +0200 + +youtube-dl (1:2017.10.15.1-1~webupd8~trusty0) trusty; urgency=medium + + * New upstream release (automated upload) + + -- Alin Andrei Tue, 17 Oct 2017 12:14:50 +0300 youtube-dl (1:2017.09.24-1~webupd8~trusty0) trusty; urgency=medium diff -Nru youtube-dl-2017.10.15.1/docs/supportedsites.md youtube-dl-2017.11.15/docs/supportedsites.md --- youtube-dl-2017.10.15.1/docs/supportedsites.md 2017-10-14 23:16:41.000000000 +0000 +++ youtube-dl-2017.11.15/docs/supportedsites.md 2017-11-14 17:15:40.000000000 +0000 @@ -3,8 +3,6 @@ - **1up.com** - **20min** - **220.ro** - - **22tracks:genre** - - **22tracks:track** - **24video** - **3qsdn**: 3Q SDN - **3sat** @@ -342,6 +340,7 @@ - **HornBunny** - **HotNewHipHop** - **HotStar** + - **hotstar:playlist** - **Howcast** - **HowStuffWorks** - **HRTi** @@ -498,7 +497,6 @@ - **MySpace:album** - **MySpass** - **Myvi** - - **myvideo** (Currently broken) - **MyVidster** - **n-tv.de** - **natgeo** @@ -728,6 +726,7 @@ - **SenateISVP** - **SendtoNews** - **ServingSys** + - **Servus** - **Sexu** - **Shahid** - **Shared**: shared.sx @@ -887,6 +886,7 @@ - **UDNEmbed**: 聯合影音 - **UKTVPlay** - **Unistra** + - **Unity** - **uol.com.br** - **uplynk** - **uplynk:preplay** @@ -975,6 +975,7 @@ - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **Vrak** - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be + - **VrtNU**: VrtNU.be - **vrv** - **vrv:series** - **VShare** @@ -1033,6 +1034,9 @@ - **YouJizz** - **youku**: 优酷 - **youku:show** + - **YouNowChannel** + - **YouNowLive** + - **YouNowMoment** - **YouPorn** - **YourUpload** - **youtube**: YouTube.com diff -Nru youtube-dl-2017.10.15.1/README.md youtube-dl-2017.11.15/README.md --- youtube-dl-2017.10.15.1/README.md 2017-10-14 23:16:39.000000000 +0000 +++ youtube-dl-2017.11.15/README.md 2017-11-14 17:15:34.000000000 +0000 @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.org/rg3/youtube-dl.svg?branch=master)](https://travis-ci.org/rg3/youtube-dl) + youtube-dl - download videos from youtube.com or other video platforms - [INSTALLATION](#installation) diff -Nru youtube-dl-2017.10.15.1/README.txt youtube-dl-2017.11.15/README.txt --- youtube-dl-2017.10.15.1/README.txt 2017-10-14 23:17:11.000000000 +0000 +++ youtube-dl-2017.11.15/README.txt 2017-11-14 17:16:20.000000000 +0000 @@ -1,3 +1,5 @@ +[Build Status] + youtube-dl - download videos from youtube.com or other video platforms - INSTALLATION diff -Nru youtube-dl-2017.10.15.1/test/test_InfoExtractor.py youtube-dl-2017.11.15/test/test_InfoExtractor.py --- youtube-dl-2017.10.15.1/test/test_InfoExtractor.py 2017-10-14 23:16:11.000000000 +0000 +++ youtube-dl-2017.11.15/test/test_InfoExtractor.py 2017-11-14 17:14:40.000000000 +0000 @@ -574,6 +574,32 @@ self.ie._sort_formats(formats) expect_value(self, formats, expected_formats, None) + def test_parse_f4m_formats(self): + _TEST_CASES = [ + ( + # https://github.com/rg3/youtube-dl/issues/14660 + 'custom_base_url', + 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m', + [{ + 'manifest_url': 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m', + 'ext': 'flv', + 'format_id': '2148', + 'protocol': 'f4m', + 'tbr': 2148, + 'width': 1280, + 'height': 720, + }] + ), + ] + + for f4m_file, f4m_url, expected_formats in _TEST_CASES: + with io.open('./test/testdata/f4m/%s.f4m' % f4m_file, + mode='r', encoding='utf-8') as f: + formats = self.ie._parse_f4m_formats( + compat_etree_fromstring(f.read().encode('utf-8')), + f4m_url, None) + self.ie._sort_formats(formats) + expect_value(self, formats, expected_formats, None) if __name__ == '__main__': unittest.main() diff -Nru youtube-dl-2017.10.15.1/test/test_utils.py youtube-dl-2017.11.15/test/test_utils.py --- youtube-dl-2017.10.15.1/test/test_utils.py 2017-10-14 23:16:11.000000000 +0000 +++ youtube-dl-2017.11.15/test/test_utils.py 2017-11-14 17:14:40.000000000 +0000 @@ -540,6 +540,7 @@ self.assertEqual(parse_duration('87 Min.'), 5220) self.assertEqual(parse_duration('PT1H0.040S'), 3600.04) self.assertEqual(parse_duration('PT00H03M30SZ'), 210) + self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88) def test_fix_xml_ampersands(self): self.assertEqual( diff -Nru youtube-dl-2017.10.15.1/youtube_dl/downloader/f4m.py youtube-dl-2017.11.15/youtube_dl/downloader/f4m.py --- youtube-dl-2017.10.15.1/youtube_dl/downloader/f4m.py 2017-10-14 23:16:11.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/downloader/f4m.py 2017-11-14 17:14:40.000000000 +0000 @@ -243,8 +243,17 @@ media)) -def _add_ns(prop): - return '{http://ns.adobe.com/f4m/1.0}%s' % prop +def _add_ns(prop, ver=1): + return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop) + + +def get_base_url(manifest): + base_url = xpath_text( + manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)], + 'base URL', default=None) + if base_url: + base_url = base_url.strip() + return base_url class F4mFD(FragmentFD): @@ -330,13 +339,13 @@ rate, media = list(filter( lambda f: int(f[0]) == requested_bitrate, formats))[0] - base_url = compat_urlparse.urljoin(man_url, media.attrib['url']) + # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec. + man_base_url = get_base_url(doc) or man_url + + base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url']) bootstrap_node = doc.find(_add_ns('bootstrapInfo')) - # From Adobe F4M 3.0 spec: - # The element SHALL be the base URL for all relative - # (HTTP-based) URLs in the manifest. If is not present, said - # URLs should be relative to the location of the containing document. - boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url) + boot_info, bootstrap_url = self._parse_bootstrap_node( + bootstrap_node, man_base_url) live = boot_info['live'] metadata_node = media.find(_add_ns('metadata')) if metadata_node is not None: diff -Nru youtube-dl-2017.10.15.1/youtube_dl/downloader/fragment.py youtube-dl-2017.11.15/youtube_dl/downloader/fragment.py --- youtube-dl-2017.10.15.1/youtube_dl/downloader/fragment.py 2017-10-14 23:16:24.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/downloader/fragment.py 2017-11-14 17:14:40.000000000 +0000 @@ -158,7 +158,7 @@ if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))): self._read_ytdl_file(ctx) if ctx['fragment_index'] > 0 and resume_len == 0: - self.report_error( + self.report_warning( 'Inconsistent state of incomplete fragment download. ' 'Restarting from the beginning...') ctx['fragment_index'] = resume_len = 0 diff -Nru youtube-dl-2017.10.15.1/youtube_dl/downloader/hls.py youtube-dl-2017.11.15/youtube_dl/downloader/hls.py --- youtube-dl-2017.10.15.1/youtube_dl/downloader/hls.py 2017-10-14 23:16:24.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/downloader/hls.py 2017-11-14 17:14:40.000000000 +0000 @@ -88,6 +88,7 @@ if line.startswith('#'): if anvato_ad(line): ad_frags += 1 + ad_frag_next = True continue if ad_frag_next: ad_frag_next = False diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/animeondemand.py youtube-dl-2017.11.15/youtube_dl/extractor/animeondemand.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/animeondemand.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/animeondemand.py 2017-11-14 17:14:53.000000000 +0000 @@ -78,7 +78,7 @@ post_url = urljoin(self._LOGIN_URL, post_url) response = self._download_webpage( - post_url, None, 'Logging in as %s' % username, + post_url, None, 'Logging in', data=urlencode_postdata(login_form), headers={ 'Referer': self._LOGIN_URL, }) diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/arte.py youtube-dl-2017.11.15/youtube_dl/extractor/arte.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/arte.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/arte.py 2017-11-14 17:14:40.000000000 +0000 @@ -6,6 +6,7 @@ from .common import InfoExtractor from ..compat import ( compat_parse_qs, + compat_str, compat_urllib_parse_urlparse, ) from ..utils import ( @@ -15,6 +16,7 @@ int_or_none, NO_DEFAULT, qualities, + try_get, unified_strdate, ) @@ -80,12 +82,15 @@ info = self._download_json(json_url, video_id) player_info = info['videoJsonPlayer'] - vsr = player_info['VSR'] - + vsr = try_get(player_info, lambda x: x['VSR'], dict) if not vsr: - raise ExtractorError( - 'Video %s is not available' % player_info.get('VID') or video_id, - expected=True) + error = None + if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error': + error = try_get( + player_info, lambda x: x['custom_msg']['msg'], compat_str) + if not error: + error = 'Video %s is not available' % player_info.get('VID') or video_id + raise ExtractorError(error, expected=True) upload_date_str = player_info.get('shootingDate') if not upload_date_str: diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/atresplayer.py youtube-dl-2017.11.15/youtube_dl/extractor/atresplayer.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/atresplayer.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/atresplayer.py 2017-11-14 17:14:53.000000000 +0000 @@ -87,7 +87,7 @@ self._LOGIN_URL, urlencode_postdata(login_form)) request.add_header('Content-Type', 'application/x-www-form-urlencoded') response = self._download_webpage( - request, None, 'Logging in as %s' % username) + request, None, 'Logging in') error = self._html_search_regex( r'(?s)]+class="[^"]*\blist_error\b[^"]*">(.+?)', diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/azmedien.py youtube-dl-2017.11.15/youtube_dl/extractor/azmedien.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/azmedien.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/azmedien.py 2017-11-14 17:14:40.000000000 +0000 @@ -47,7 +47,7 @@ 'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom', 'info_dict': { 'id': '1_2444peh4', - 'ext': 'mov', + 'ext': 'mp4', 'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom', 'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8', 'uploader_id': 'TeleZ?ri', diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/bambuser.py youtube-dl-2017.11.15/youtube_dl/extractor/bambuser.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/bambuser.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/bambuser.py 2017-11-14 17:14:53.000000000 +0000 @@ -59,7 +59,7 @@ self._LOGIN_URL, urlencode_postdata(login_form)) request.add_header('Referer', self._LOGIN_URL) response = self._download_webpage( - request, None, 'Logging in as %s' % username) + request, None, 'Logging in') login_error = self._html_search_regex( r'(?s)
(.+?)
', diff -Nru youtube-dl-2017.10.15.1/youtube_dl/extractor/canvas.py youtube-dl-2017.11.15/youtube_dl/extractor/canvas.py --- youtube-dl-2017.10.15.1/youtube_dl/extractor/canvas.py 2017-10-14 23:16:12.000000000 +0000 +++ youtube-dl-2017.11.15/youtube_dl/extractor/canvas.py 2017-11-14 17:14:40.000000000 +0000 @@ -1,16 +1,22 @@ from __future__ import unicode_literals import re +import json from .common import InfoExtractor +from .gigya import GigyaBaseIE +from ..compat import compat_HTTPError from ..utils import ( - float_or_none, + ExtractorError, strip_or_none, + float_or_none, + int_or_none, + parse_iso8601, ) class CanvasIE(InfoExtractor): - _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet)/assets/(?Pm[dz]-ast-[^/?#&]+)' + _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet|vrtvideo)/assets/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', 'md5': '90139b746a0a9bd7bb631283f6e2a64e', @@ -166,3 +172,139 @@ 'title': title, 'description': self._og_search_description(webpage), } + + +class VrtNUIE(GigyaBaseIE): + IE_DESC = 'VrtNU.be' + _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?Pvrtnu)/(?:[^/]+/)*(?P[^/?#&]+)' + _TESTS = [{ + 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/', + 'info_dict': { + 'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de', + 'ext': 'flv', + 'title': 'De zwarte weduwe', + 'description': 'md5:d90c21dced7db869a85db89a623998d4', + 'duration': 1457.04, + 'thumbnail': r're:^https?://.*\.jpg$', + 'season': '1', + 'season_number': 1, + 'episode_number': 1, + }, + 'skip': 'This video is only available for registered users' + }] + _NETRC_MACHINE = 'vrtnu' + _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' + _CONTEXT_ID = 'R3595707040' + + def _real_initialize(self): + self._login() + + def _login(self): + username, password = self._get_login_info() + if username is None: + return + + auth_data = { + 'APIKey': self._APIKEY, + 'targetEnv': 'jssdk', + 'loginID': username, + 'password': password, + 'authMode': 'cookie', + } + + auth_info = self._gigya_login(auth_data) + + # Sometimes authentication fails for no good reason, retry + login_attempt = 1 + while login_attempt <= 3: + try: + # When requesting a token, no actual token is returned, but the + # necessary cookies are set. + self._request_webpage( + 'https://token.vrt.be', + None, note='Requesting a token', errnote='Could not get a token', + headers={ + 'Content-Type': 'application/json', + 'Referer': 'https://www.vrt.be/vrtnu/', + }, + data=json.dumps({ + 'uid': auth_info['UID'], + 'uidsig': auth_info['UIDSignature'], + 'ts': auth_info['signatureTimestamp'], + 'email': auth_info['profile']['email'], + }).encode('utf-8')) + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: + login_attempt += 1 + self.report_warning('Authentication failed') + self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again') + else: + raise e + else: + break + + def _real_extract(self, url): + display_id = self._match_id(url) + + webpage = self._download_webpage(url, display_id) + + title = self._html_search_regex( + r'(?ms)

(.+?)

', + webpage, 'title').strip() + + description = self._html_search_regex( + r'(?ms)
(.+?)
', + webpage, 'description', default=None) + + season = self._html_search_regex( + [r'''(?xms)\s* + seizoen\ (.+?)\s* + ''', + r'