diff -Nru youtube-dl-2017.04.11/ChangeLog youtube-dl-2017.04.17/ChangeLog --- youtube-dl-2017.04.11/ChangeLog 2017-04-10 19:17:50.000000000 +0000 +++ youtube-dl-2017.04.17/ChangeLog 2017-04-16 17:46:27.000000000 +0000 @@ -1,3 +1,58 @@ +version 2017.04.17 + +Extractors +* [limelight] Improve extraction LimelightEmbeddedPlayerFlash media embeds and + add support for channel and channelList embeds +* [generic] Extract multiple Limelight embeds (#12761) ++ [itv] Extract series metadata +* [itv] Fix RTMP formats downloading (#12759) +* [itv] Use native HLS downloader by default ++ [go90] Extract subtitles (#12752) ++ [go90] Extract series metadata (#12752) + + +version 2017.04.16 + +Core +* [YoutubeDL] Apply expand_path after output template substitution ++ [YoutubeDL] Propagate overridden meta fields to extraction results of type + url (#11163) + +Extractors ++ [generic] Extract RSS entries as url_transparent (#11163) ++ [streamango] Add support for streamango.com (#12643) ++ [wsj:article] Add support for articles (#12558) +* [brightcove] Relax video tag embeds extraction and validate ambiguous embeds' + URLs (#9163, #12005, #12178, #12480) ++ [udemy] Add support for react rendition (#12744) + + +version 2017.04.15 + +Extractors +* [youku] Fix fileid extraction (#12741, #12743) + + +version 2017.04.14 + +Core ++ [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955) ++ [adobepass] Improve Comcast and Verison login code (#10803) ++ [adobepass] Add support for Verizon (#10803) + +Extractors ++ [aenetworks] Add support for specials (#12723) ++ [hbo] Extract HLS formats ++ [go90] Add support for go90.com (#10127) ++ [tv2hu] Add support for tv2.hu (#10509) ++ [generic] Exclude URLs with xml ext from valid video URLs (#10768, #11654) +* [youtube] Improve HLS formats extraction +* [afreecatv] Fix extraction for videos with different key layout (#12718) +- [youtube] Remove explicit preference for audio-only and video-only formats in + order not to break sorting when new formats appear +* [canalplus] Bypass geo restriction + + version 2017.04.11 Extractors diff -Nru youtube-dl-2017.04.11/debian/changelog youtube-dl-2017.04.17/debian/changelog --- youtube-dl-2017.04.11/debian/changelog 2017-04-11 09:24:12.000000000 +0000 +++ youtube-dl-2017.04.17/debian/changelog 2017-04-18 09:16:37.000000000 +0000 @@ -1,8 +1,14 @@ -youtube-dl (1:2017.04.11-1~webupd8~yakkety0) yakkety; urgency=medium +youtube-dl (1:2017.04.17-1~webupd8~yakkety0) yakkety; urgency=medium * New upstream release (automated upload) - -- Alin Andrei Tue, 11 Apr 2017 12:24:12 +0300 + -- Alin Andrei Tue, 18 Apr 2017 12:16:37 +0300 + +youtube-dl (1:2017.04.11-1~webupd8~trusty0) trusty; urgency=medium + + * New upstream release (automated upload) + + -- Alin Andrei Tue, 11 Apr 2017 12:24:38 +0300 youtube-dl (1:2017.04.09-1~webupd8~trusty0) trusty; urgency=medium diff -Nru youtube-dl-2017.04.11/docs/supportedsites.md youtube-dl-2017.04.17/docs/supportedsites.md --- youtube-dl-2017.04.11/docs/supportedsites.md 2017-04-10 19:17:53.000000000 +0000 +++ youtube-dl-2017.04.17/docs/supportedsites.md 2017-04-16 17:46:32.000000000 +0000 @@ -308,6 +308,7 @@ - **Globo** - **GloboArticle** - **Go** + - **Go90** - **GodTube** - **GodTV** - **Golem** @@ -744,6 +745,7 @@ - **Steam** - **Stitcher** - **Streamable** + - **Streamango** - **streamcloud.eu** - **StreamCZ** - **StreetVoice** @@ -818,6 +820,7 @@ - **Tutv** - **tv.dfb.de** - **TV2** + - **tv2.hu** - **TV2Article** - **TV3** - **TV4**: tv4.se and tv4play.se @@ -964,6 +967,7 @@ - **wrzuta.pl** - **wrzuta.pl:playlist** - **WSJ**: Wall Street Journal + - **WSJArticle** - **XBef** - **XboxClips** - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE, Vid ABC, VidBom, vidlo diff -Nru youtube-dl-2017.04.11/test/test_YoutubeDL.py youtube-dl-2017.04.17/test/test_YoutubeDL.py --- youtube-dl-2017.04.11/test/test_YoutubeDL.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/test/test_YoutubeDL.py 2017-04-16 05:41:06.000000000 +0000 @@ -755,6 +755,7 @@ '_type': 'url_transparent', 'url': 'foo2:', 'ie_key': 'Foo2', + 'title': 'foo1 title' } class Foo2IE(InfoExtractor): @@ -771,7 +772,7 @@ _VALID_URL = r'foo3:' def _real_extract(self, url): - return _make_result([{'url': TEST_URL}]) + return _make_result([{'url': TEST_URL}], title='foo3 title') ydl.add_info_extractor(Foo1IE(ydl)) ydl.add_info_extractor(Foo2IE(ydl)) @@ -779,6 +780,7 @@ ydl.extract_info('foo1:') downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['url'], TEST_URL) + self.assertEqual(downloaded['title'], 'foo1 title') if __name__ == '__main__': diff -Nru youtube-dl-2017.04.11/youtube_dl/compat.py youtube-dl-2017.04.17/youtube_dl/compat.py --- youtube-dl-2017.04.11/youtube_dl/compat.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/compat.py 2017-04-16 05:41:06.000000000 +0000 @@ -2692,7 +2692,7 @@ userhome = pwent.pw_dir userhome = userhome.rstrip('/') return (userhome + path[i:]) or '/' - elif compat_os_name == 'nt' or compat_os_name == 'ce': + elif compat_os_name in ('nt', 'ce'): def compat_expanduser(path): """Expand ~ and ~user constructs. diff -Nru youtube-dl-2017.04.11/youtube_dl/downloader/hls.py youtube-dl-2017.04.17/youtube_dl/downloader/hls.py --- youtube-dl-2017.04.11/youtube_dl/downloader/hls.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/downloader/hls.py 2017-04-16 05:41:06.000000000 +0000 @@ -34,7 +34,7 @@ def can_download(manifest, info_dict): UNSUPPORTED_FEATURES = ( r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1] - r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2] + # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2] # Live streams heuristic does not always work (e.g. geo restricted to Germany # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0) @@ -52,7 +52,9 @@ # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5 ) check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES] - check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest) + is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest + check_results.append(can_decrypt_frag or not is_aes128_enc) + check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest)) check_results.append(not info_dict.get('is_live')) return all(check_results) @@ -100,6 +102,7 @@ i = 0 media_sequence = 0 decrypt_info = {'METHOD': 'NONE'} + byte_range = {} frags_filenames = [] for line in s.splitlines(): line = line.strip() @@ -114,11 +117,14 @@ if extra_query: frag_url = update_url_query(frag_url, extra_query) count = 0 + headers = info_dict.get('http_headers', {}) + if byte_range: + headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end']) while count <= fragment_retries: try: success = ctx['dl'].download(frag_filename, { 'url': frag_url, - 'http_headers': info_dict.get('http_headers'), + 'http_headers': headers, }) if not success: return False @@ -167,6 +173,13 @@ decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read() elif line.startswith('#EXT-X-MEDIA-SEQUENCE'): media_sequence = int(line[22:]) + elif line.startswith('#EXT-X-BYTERANGE'): + splitted_byte_range = line[17:].split('@') + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] + byte_range = { + 'start': sub_range_start, + 'end': sub_range_start + int(splitted_byte_range[0]), + } self._finish_frag_download(ctx) diff -Nru youtube-dl-2017.04.11/youtube_dl/downloader/rtmp.py youtube-dl-2017.04.17/youtube_dl/downloader/rtmp.py --- youtube-dl-2017.04.11/youtube_dl/downloader/rtmp.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/downloader/rtmp.py 2017-04-16 05:41:06.000000000 +0000 @@ -169,7 +169,7 @@ self.report_error('[rtmpdump] Could not connect to RTMP server.') return False - while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live: + while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live: prevsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen('[rtmpdump] %s bytes' % prevsize) time.sleep(5.0) # This seems to be needed diff -Nru youtube-dl-2017.04.11/youtube_dl/extractor/adobepass.py youtube-dl-2017.04.17/youtube_dl/extractor/adobepass.py --- youtube-dl-2017.04.11/youtube_dl/extractor/adobepass.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/extractor/adobepass.py 2017-04-16 05:41:06.000000000 +0000 @@ -41,6 +41,11 @@ 'username_field': 'IDToken1', 'password_field': 'IDToken2', }, + 'Verizon': { + 'name': 'Verizon FiOS', + 'username_field': 'IDToken1', + 'password_field': 'IDToken2', + }, 'thr030': { 'name': '3 Rivers Communications' }, @@ -1384,40 +1389,72 @@ # Comcast page flow varies by video site and whether you # are on Comcast's network. provider_redirect_page, urlh = provider_redirect_page_res - # Check for Comcast auto login if 'automatically signing you in' in provider_redirect_page: oauth_redirect_url = self._html_search_regex( r'window\.location\s*=\s*[\'"]([^\'"]+)', provider_redirect_page, 'oauth redirect') - # Just need to process the request. No useful data comes back self._download_webpage( oauth_redirect_url, video_id, 'Confirming auto login') else: if '
Resume' in mvpd_confirm_page: post_form(mvpd_confirm_page_res, 'Confirming Login') - + elif mso_id == 'Verizon': + # In general, if you're connecting from a Verizon-assigned IP, + # you will not actually pass your credentials. + provider_redirect_page, urlh = provider_redirect_page_res + if 'Please wait ...' in provider_redirect_page: + saml_redirect_url = self._html_search_regex( + r'self\.parent\.location=(["\'])(?P.+?)\1', + provider_redirect_page, + 'SAML Redirect URL', group='url') + saml_login_page = self._download_webpage( + saml_redirect_url, video_id, + 'Downloading SAML Login Page') + else: + saml_login_page_res = post_form( + provider_redirect_page_res, 'Logging in', { + mso_info['username_field']: username, + mso_info['password_field']: password, + }) + saml_login_page, urlh = saml_login_page_res + if 'Please try again.' in saml_login_page: + raise ExtractorError( + 'We\'re sorry, but either the User ID or Password entered is not correct.') + saml_login_url = self._search_regex( + r'xmlHttp\.open\("POST"\s*,\s*(["\'])(?P.+?)\1', + saml_login_page, 'SAML Login URL', group='url') + saml_response_json = self._download_json( + saml_login_url, video_id, 'Downloading SAML Response', + headers={'Content-Type': 'text/xml'}) + self._download_webpage( + saml_response_json['targetValue'], video_id, + 'Confirming Login', data=urlencode_postdata({ + 'SAMLResponse': saml_response_json['SAMLResponse'], + 'RelayState': saml_response_json['RelayState'] + }), headers={ + 'Content-Type': 'application/x-www-form-urlencoded' + }) else: - # Normal, non-Comcast flow provider_login_page_res = post_form( provider_redirect_page_res, 'Downloading Provider Login Page') mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', { diff -Nru youtube-dl-2017.04.11/youtube_dl/extractor/aenetworks.py youtube-dl-2017.04.17/youtube_dl/extractor/aenetworks.py --- youtube-dl-2017.04.11/youtube_dl/extractor/aenetworks.py 2017-04-10 19:17:19.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/extractor/aenetworks.py 2017-04-16 05:41:06.000000000 +0000 @@ -23,7 +23,19 @@ class AENetworksIE(AENetworksBaseIE): IE_NAME = 'aenetworks' IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network' - _VALID_URL = r'https?://(?:www\.)?(?P(?:history|aetv|mylifetime|lifetimemovieclub)\.com|fyi\.tv)/(?:shows/(?P[^/]+(?:/[^/]+){0,2})|movies/(?P[^/]+)(?:/full-movie)?)' + _VALID_URL = r'''(?x) + https?:// + (?:www\.)? + (?P + (?:history|aetv|mylifetime|lifetimemovieclub)\.com| + fyi\.tv + )/ + (?: + shows/(?P[^/]+(?:/[^/]+){0,2})| + movies/(?P[^/]+)(?:/full-movie)?| + specials/(?P[^/]+)/full-special + ) + ''' _TESTS = [{ 'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1', 'md5': 'a97a65f7e823ae10e9244bc5433d5fe6', @@ -65,6 +77,9 @@ }, { 'url': 'https://www.lifetimemovieclub.com/movies/a-killer-among-us', 'only_matching': True + }, { + 'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special', + 'only_matching': True }] _DOMAIN_TO_REQUESTOR_ID = { 'history.com': 'HISTORY', @@ -75,8 +90,8 @@ } def _real_extract(self, url): - domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups() - display_id = show_path or movie_display_id + domain, show_path, movie_display_id, special_display_id = re.match(self._VALID_URL, url).groups() + display_id = show_path or movie_display_id or special_display_id webpage = self._download_webpage(url, display_id) if show_path: url_parts = show_path.split('/') diff -Nru youtube-dl-2017.04.11/youtube_dl/extractor/afreecatv.py youtube-dl-2017.04.17/youtube_dl/extractor/afreecatv.py --- youtube-dl-2017.04.11/youtube_dl/extractor/afreecatv.py 2017-04-10 19:17:33.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/extractor/afreecatv.py 2017-04-16 05:41:06.000000000 +0000 @@ -97,7 +97,7 @@ 'playlist': [{ 'md5': 'd8b7c174568da61d774ef0203159bf97', 'info_dict': { - 'id': '10481652_1', + 'id': '20160502_c4c62b9d_174361386_1', 'ext': 'mp4', 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)", 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', @@ -109,7 +109,7 @@ }, { 'md5': '58f2ce7f6044e34439ab2d50612ab02b', 'info_dict': { - 'id': '10481652_2', + 'id': '20160502_39e739bb_174361386_2', 'ext': 'mp4', 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)", 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', @@ -123,6 +123,22 @@ 'skip_download': True, }, }, { + # non standard key + 'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605', + 'info_dict': { + 'id': '20170411_BE689A0E_190960999_1_2_h', + 'ext': 'mp4', + 'title': '혼자사는여자집', + 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$', + 'uploader': '♥이슬이', + 'uploader_id': 'dasl8121', + 'upload_date': '20170411', + 'duration': 213, + }, + 'params': { + 'skip_download': True, + }, + }, { 'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652', 'only_matching': True, }, { @@ -176,26 +192,27 @@ if not video_url: entries = [] - for file_num, file_element in enumerate( - video_element.findall(compat_xpath('./file')), start=1): + file_elements = video_element.findall(compat_xpath('./file')) + one = len(file_elements) == 1 + for file_num, file_element in enumerate(file_elements, start=1): file_url = file_element.text if not file_url: continue - video_key = self.parse_video_key(file_element.get('key', '')) - if not video_key: - continue + key = file_element.get('key', '') + upload_date = self._search_regex( + r'^(\d{8})_', key, 'upload date', default=None) file_duration = int_or_none(file_element.get('duration')) - part = video_key.get('part', file_num) - format_id = '%s_%s' % (video_id, part) + format_id = key if key else '%s_%s' % (video_id, file_num) formats = self._extract_m3u8_formats( file_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', note='Downloading part %d m3u8 information' % file_num) + title = title if one else '%s (part %d)' % (title, file_num) file_info = common_entry.copy() file_info.update({ 'id': format_id, - 'title': '%s (part %d)' % (title, part), - 'upload_date': video_key.get('upload_date'), + 'title': title, + 'upload_date': upload_date, 'duration': file_duration, 'formats': formats, }) diff -Nru youtube-dl-2017.04.11/youtube_dl/extractor/bbc.py youtube-dl-2017.04.17/youtube_dl/extractor/bbc.py --- youtube-dl-2017.04.11/youtube_dl/extractor/bbc.py 2017-04-10 19:17:33.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/extractor/bbc.py 2017-04-16 05:41:06.000000000 +0000 @@ -409,7 +409,7 @@ description = smp_config['summary'] for item in smp_config['items']: kind = item['kind'] - if kind != 'programme' and kind != 'radioProgramme': + if kind not in ('programme', 'radioProgramme'): continue programme_id = item.get('vpid') duration = int_or_none(item.get('duration')) @@ -450,7 +450,7 @@ for item in self._extract_items(playlist): kind = item.get('kind') - if kind != 'programme' and kind != 'radioProgramme': + if kind not in ('programme', 'radioProgramme'): continue title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS) diff -Nru youtube-dl-2017.04.11/youtube_dl/extractor/brightcove.py youtube-dl-2017.04.17/youtube_dl/extractor/brightcove.py --- youtube-dl-2017.04.11/youtube_dl/extractor/brightcove.py 2017-04-10 19:17:20.000000000 +0000 +++ youtube-dl-2017.04.17/youtube_dl/extractor/brightcove.py 2017-04-16 05:41:06.000000000 +0000 @@ -17,6 +17,7 @@ from ..utils import ( determine_ext, ExtractorError, + extract_attributes, find_xpath_attr, fix_xml_ampersands, float_or_none, @@ -109,6 +110,7 @@ 'upload_date': '20140827', 'uploader_id': '710858724001', }, + 'skip': 'Video gone', }, { # playlist with 'videoList' @@ -487,12 +489,13 @@ return urls[0] if urls else None @staticmethod - def _extract_urls(webpage): + def _extract_urls(ie, webpage): # Reference: # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe - # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript - # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html - # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player + # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag + # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript + # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html + # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player entries = [] @@ -501,22 +504,48 @@ r']+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage): entries.append(url if url.startswith('http') else 'http:' + url) - # Look for embed_in_page embeds [2] - for video_id, account_id, player_id, embed in re.findall( - # According to examples from [3] it's unclear whether video id - # may be optional and what to do when it is - # According to [4] data-video-id may be prefixed with ref: - r'''(?sx) - ]+ - data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*? - .*? - ]+ - src=["\'](?:https?:)?//players\.brightcove\.net/ - (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js + # Look for