]+id=(["\'])defchat\1[^>]*>\s*(?P[^<]+)<'],
webpage, 'error', group='error', default=None)
if not error:
- if any(p not in webpage for p in (
+ if any(p in webpage for p in (
self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
error = self._ROOM_OFFLINE
if error:
raise ExtractorError(error, expected=True)
raise ExtractorError('Unable to find stream URL')
- formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+ formats = []
+ for m3u8_url in m3u8_urls:
+ m3u8_id = 'fast' if '_fast' in m3u8_url else 'slow'
+ formats.extend(self._extract_m3u8_formats(
+ m3u8_url, video_id, ext='mp4',
+ # ffmpeg skips segments for fast m3u8
+ preference=-10 if m3u8_id == 'fast' else None,
+ m3u8_id=m3u8_id, fatal=False, live=True))
+ self._sort_formats(formats)
return {
'id': video_id,
'title': self._live_title(video_id),
- 'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+ 'thumbnail': 'https://roomimg.stream.highwebmedia.com/ri/%s.jpg' % video_id,
'age_limit': self._rta_search(webpage),
'is_live': True,
'formats': formats,
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/chilloutzone.py youtube-dl-2019.05.20/youtube_dl/extractor/chilloutzone.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/chilloutzone.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/chilloutzone.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,10 +1,11 @@
from __future__ import unicode_literals
import re
-import base64
import json
from .common import InfoExtractor
+from .youtube import YoutubeIE
+from ..compat import compat_b64decode
from ..utils import (
clean_html,
ExtractorError
@@ -57,7 +58,7 @@
base64_video_info = self._html_search_regex(
r'var cozVidData = "(.+?)";', webpage, 'video data')
- decoded_video_info = base64.b64decode(base64_video_info.encode('utf-8')).decode('utf-8')
+ decoded_video_info = compat_b64decode(base64_video_info).decode('utf-8')
video_info_dict = json.loads(decoded_video_info)
# get video information from dict
@@ -70,11 +71,9 @@
# If nativePlatform is None a fallback mechanism is used (i.e. youtube embed)
if native_platform is None:
- youtube_url = self._html_search_regex(
- r'[\da-zA-Z]+)'
_TESTS = [{
- 'url': 'http://chirb.it/PrIPv5',
- 'md5': '9847b0dad6ac3e074568bf2cfb197de8',
+ 'url': 'http://chirb.it/be2abG',
'info_dict': {
- 'id': 'PrIPv5',
+ 'id': 'be2abG',
'ext': 'mp3',
- 'title': 'Фасадстрой',
- 'duration': 52,
- 'view_count': int,
- 'comment_count': int,
+ 'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
+ 'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
+ 'duration': 306,
+ 'uploader': 'Gerryaudio',
+ },
+ 'params': {
+ 'skip_download': True,
}
}, {
'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
'only_matching': True,
+ }, {
+ 'url': 'https://chirb.it/wp/MN58c2',
+ 'only_matching': True,
}]
def _real_extract(self, url):
@@ -33,38 +38,43 @@
webpage = self._download_webpage(
'http://chirb.it/%s' % audio_id, audio_id)
- audio_url = self._search_regex(
- r'"setFile"\s*,\s*"([^"]+)"', webpage, 'audio url')
+ data_fd = self._search_regex(
+ r'data-fd=(["\'])(?P(?:(?!\1).)+)\1',
+ webpage, 'data fd', group='url')
+
+ # Reverse engineered from https://chirb.it/js/chirbit.player.js (look
+ # for soundURL)
+ audio_url = compat_b64decode(data_fd[::-1]).decode('utf-8')
title = self._search_regex(
- r'itemprop="name">([^<]+)', webpage, 'title')
- duration = parse_duration(self._html_search_meta(
- 'duration', webpage, 'duration', fatal=False))
- view_count = int_or_none(self._search_regex(
- r'itemprop="playCount"\s*>(\d+)', webpage,
- 'listen count', fatal=False))
- comment_count = int_or_none(self._search_regex(
- r'>(\d+) Comments?:', webpage,
- 'comment count', fatal=False))
+ r'class=["\']chirbit-title["\'][^>]*>([^<]+)', webpage, 'title')
+ description = self._search_regex(
+ r'Description
\s*]*>([^<]+)',
+ webpage, 'description', default=None)
+ duration = parse_duration(self._search_regex(
+ r'class=["\']c-length["\'][^>]*>([^<]+)',
+ webpage, 'duration', fatal=False))
+ uploader = self._search_regex(
+ r'id=["\']chirbit-username["\'][^>]*>([^<]+)',
+ webpage, 'uploader', fatal=False)
return {
'id': audio_id,
'url': audio_url,
'title': title,
+ 'description': description,
'duration': duration,
- 'view_count': view_count,
- 'comment_count': comment_count,
+ 'uploader': uploader,
}
class ChirbitProfileIE(InfoExtractor):
IE_NAME = 'chirbit:profile'
- _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P[^/]+)'
+ _VALID_URL = r'https?://(?:www\.)?chirbit\.com/(?:rss/)?(?P[^/]+)'
_TEST = {
'url': 'http://chirbit.com/ScarletBeauty',
'info_dict': {
'id': 'ScarletBeauty',
- 'title': 'Chirbits by ScarletBeauty',
},
'playlist_mincount': 3,
}
@@ -72,13 +82,10 @@
def _real_extract(self, url):
profile_id = self._match_id(url)
- rss = self._download_xml(
- 'http://chirbit.com/rss/%s' % profile_id, profile_id)
+ webpage = self._download_webpage(url, profile_id)
entries = [
- self.url_result(audio_url.text, 'Chirbit')
- for audio_url in rss.findall('./channel/item/link')]
-
- title = rss.find('./channel/title').text
+ self.url_result(self._proto_relative_url('//chirb.it/' + video_id))
+ for _, video_id in re.findall(r']+id=([\'"])copy-btn-(?P[0-9a-zA-Z]+)\1', webpage)]
- return self.playlist_result(entries, profile_id, title)
+ return self.playlist_result(entries, profile_id)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cinchcast.py youtube-dl-2019.05.20/youtube_dl/extractor/cinchcast.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cinchcast.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cinchcast.py 2019-06-07 18:49:07.000000000 +0000
@@ -9,12 +9,20 @@
class CinchcastIE(InfoExtractor):
- _VALID_URL = r'https?://player\.cinchcast\.com/.*?assetId=(?P[0-9]+)'
- _TEST = {
+ _VALID_URL = r'https?://player\.cinchcast\.com/.*?(?:assetId|show_id)=(?P[0-9]+)'
+ _TESTS = [{
+ 'url': 'http://player.cinchcast.com/?show_id=5258197&platformId=1&assetType=single',
+ 'info_dict': {
+ 'id': '5258197',
+ 'ext': 'mp3',
+ 'title': 'Train Your Brain to Up Your Game with Coach Mandy',
+ 'upload_date': '20130816',
+ },
+ }, {
# Actual test is run in generic, look for undergroundwellness
'url': 'http://player.cinchcast.com/?platformId=1&assetType=single&assetId=7141703',
'only_matching': True,
- }
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cinemassacre.py youtube-dl-2019.05.20/youtube_dl/extractor/cinemassacre.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cinemassacre.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cinemassacre.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,107 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-from .screenwavemedia import ScreenwaveMediaIE
-
-
-class CinemassacreIE(InfoExtractor):
- _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P[0-9]{4})/(?P[0-9]{2})/(?P[0-9]{2})/(?P[^?#/]+)'
- _TESTS = [
- {
- 'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
- 'md5': 'fde81fbafaee331785f58cd6c0d46190',
- 'info_dict': {
- 'id': 'Cinemassacre-19911',
- 'ext': 'mp4',
- 'upload_date': '20121110',
- 'title': '“Angry Video Game Nerd: The Movie” – Trailer',
- 'description': 'md5:fb87405fcb42a331742a0dce2708560b',
- },
- },
- {
- 'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
- 'md5': 'd72f10cd39eac4215048f62ab477a511',
- 'info_dict': {
- 'id': 'Cinemassacre-521be8ef82b16',
- 'ext': 'mp4',
- 'upload_date': '20131002',
- 'title': 'The Mummy’s Hand (1940)',
- },
- },
- {
- # Youtube embedded video
- 'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
- 'md5': 'df4cf8a1dcedaec79a73d96d83b99023',
- 'info_dict': {
- 'id': 'OEVzPCY2T-g',
- 'ext': 'mp4',
- 'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
- 'upload_date': '20061207',
- 'uploader': 'Cinemassacre',
- 'uploader_id': 'JamesNintendoNerd',
- 'description': 'md5:784734696c2b8b7f4b8625cc799e07f6',
- }
- },
- {
- # Youtube embedded video
- 'url': 'http://cinemassacre.com/2006/09/01/mckids/',
- 'md5': '6eb30961fa795fedc750eac4881ad2e1',
- 'info_dict': {
- 'id': 'FnxsNhuikpo',
- 'ext': 'mp4',
- 'upload_date': '20060901',
- 'uploader': 'Cinemassacre Extras',
- 'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
- 'uploader_id': 'Cinemassacre',
- 'title': 'AVGN: McKids',
- }
- },
- {
- 'url': 'http://cinemassacre.com/2015/05/25/mario-kart-64-nintendo-64-james-mike-mondays/',
- 'md5': '1376908e49572389e7b06251a53cdd08',
- 'info_dict': {
- 'id': 'Cinemassacre-555779690c440',
- 'ext': 'mp4',
- 'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
- 'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
- 'upload_date': '20150525',
- }
- }
- ]
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- display_id = mobj.group('display_id')
- video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
-
- webpage = self._download_webpage(url, display_id)
-
- playerdata_url = self._search_regex(
- [
- ScreenwaveMediaIE.EMBED_PATTERN,
- r']+src="(?P(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
- ],
- webpage, 'player data URL', default=None, group='url')
- if not playerdata_url:
- raise ExtractorError('Unable to find player data')
-
- video_title = self._html_search_regex(
- r'(?P<title>.+?)\|', webpage, 'title')
- video_description = self._html_search_regex(
- r'<div class="entry-content">(?P<description>.+?)</div>',
- webpage, 'description', flags=re.DOTALL, fatal=False)
- video_thumbnail = self._og_search_thumbnail(webpage)
-
- return {
- '_type': 'url_transparent',
- 'display_id': display_id,
- 'title': video_title,
- 'description': video_description,
- 'upload_date': video_date,
- 'thumbnail': video_thumbnail,
- 'url': playerdata_url,
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cinemax.py youtube-dl-2019.05.20/youtube_dl/extractor/cinemax.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cinemax.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cinemax.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,29 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .hbo import HBOBaseIE
+
+
+class CinemaxIE(HBOBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?cinemax\.com/(?P<path>[^/]+/video/[0-9a-z-]+-(?P<id>\d+))'
+ _TESTS = [{
+ 'url': 'https://www.cinemax.com/warrior/video/s1-ep-1-recap-20126903',
+ 'md5': '82e0734bba8aa7ef526c9dd00cf35a05',
+ 'info_dict': {
+ 'id': '20126903',
+ 'ext': 'mp4',
+ 'title': 'S1 Ep 1: Recap',
+ },
+ 'expected_warnings': ['Unknown MIME type application/mp4 in DASH manifest'],
+ }, {
+ 'url': 'https://www.cinemax.com/warrior/video/s1-ep-1-recap-20126903.embed',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ path, video_id = re.match(self._VALID_URL, url).groups()
+ info = self._extract_info('https://www.cinemax.com/%s.xml' % path, video_id)
+ info['id'] = video_id
+ return info
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/ciscolive.py youtube-dl-2019.05.20/youtube_dl/extractor/ciscolive.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/ciscolive.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/ciscolive.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_parse_qs,
+ compat_urllib_parse_urlparse,
+)
+from ..utils import (
+ clean_html,
+ float_or_none,
+ int_or_none,
+ try_get,
+ urlencode_postdata,
+)
+
+
+class CiscoLiveBaseIE(InfoExtractor):
+ # These appear to be constant across all Cisco Live presentations
+ # and are not tied to any user session or event
+ RAINFOCUS_API_URL = 'https://events.rainfocus.com/api/%s'
+ RAINFOCUS_API_PROFILE_ID = 'Na3vqYdAlJFSxhYTYQGuMbpafMqftalz'
+ RAINFOCUS_WIDGET_ID = 'n6l4Lo05R8fiy3RpUBm447dZN8uNWoye'
+ BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5647924234001/SyK2FdqjM_default/index.html?videoId=%s'
+
+ HEADERS = {
+ 'Origin': 'https://ciscolive.cisco.com',
+ 'rfApiProfileId': RAINFOCUS_API_PROFILE_ID,
+ 'rfWidgetId': RAINFOCUS_WIDGET_ID,
+ }
+
+ def _call_api(self, ep, rf_id, query, referrer, note=None):
+ headers = self.HEADERS.copy()
+ headers['Referer'] = referrer
+ return self._download_json(
+ self.RAINFOCUS_API_URL % ep, rf_id, note=note,
+ data=urlencode_postdata(query), headers=headers)
+
+ def _parse_rf_item(self, rf_item):
+ event_name = rf_item.get('eventName')
+ title = rf_item['title']
+ description = clean_html(rf_item.get('abstract'))
+ presenter_name = try_get(rf_item, lambda x: x['participants'][0]['fullName'])
+ bc_id = rf_item['videos'][0]['url']
+ bc_url = self.BRIGHTCOVE_URL_TEMPLATE % bc_id
+ duration = float_or_none(try_get(rf_item, lambda x: x['times'][0]['length']))
+ location = try_get(rf_item, lambda x: x['times'][0]['room'])
+
+ if duration:
+ duration = duration * 60
+
+ return {
+ '_type': 'url_transparent',
+ 'url': bc_url,
+ 'ie_key': 'BrightcoveNew',
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'creator': presenter_name,
+ 'location': location,
+ 'series': event_name,
+ }
+
+
+class CiscoLiveSessionIE(CiscoLiveBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/[^#]*#/session/(?P<id>[^/?&]+)'
+ _TESTS = [{
+ 'url': 'https://ciscolive.cisco.com/on-demand-library/?#/session/1423353499155001FoSs',
+ 'md5': 'c98acf395ed9c9f766941c70f5352e22',
+ 'info_dict': {
+ 'id': '5803694304001',
+ 'ext': 'mp4',
+ 'title': '13 Smart Automations to Monitor Your Cisco IOS Network',
+ 'description': 'md5:ec4a436019e09a918dec17714803f7cc',
+ 'timestamp': 1530305395,
+ 'upload_date': '20180629',
+ 'uploader_id': '5647924234001',
+ 'location': '16B Mezz.',
+ },
+ }, {
+ 'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.event=ciscoliveemea2019#/session/15361595531500013WOU',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.ciscolive.com/global/on-demand-library.html?#/session/1490051371645001kNaS',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ rf_id = self._match_id(url)
+ rf_result = self._call_api('session', rf_id, {'id': rf_id}, url)
+ return self._parse_rf_item(rf_result['items'][0])
+
+
+class CiscoLiveSearchIE(CiscoLiveBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?ciscolive(?:\.cisco)?\.com/(?:global/)?on-demand-library(?:\.html|/)'
+ _TESTS = [{
+ 'url': 'https://ciscolive.cisco.com/on-demand-library/?search.event=ciscoliveus2018&search.technicallevel=scpsSkillLevel_aintroductory&search.focus=scpsSessionFocus_designAndDeployment#/',
+ 'info_dict': {
+ 'title': 'Search query',
+ },
+ 'playlist_count': 5,
+ }, {
+ 'url': 'https://ciscolive.cisco.com/on-demand-library/?search.technology=scpsTechnology_applicationDevelopment&search.technology=scpsTechnology_ipv6&search.focus=scpsSessionFocus_troubleshootingTroubleshooting#/',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.ciscolive.com/global/on-demand-library.html?search.technicallevel=scpsSkillLevel_aintroductory&search.event=ciscoliveemea2019&search.technology=scpsTechnology_dataCenter&search.focus=scpsSessionFocus_bestPractices#/',
+ 'only_matching': True,
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ return False if CiscoLiveSessionIE.suitable(url) else super(CiscoLiveSearchIE, cls).suitable(url)
+
+ @staticmethod
+ def _check_bc_id_exists(rf_item):
+ return int_or_none(try_get(rf_item, lambda x: x['videos'][0]['url'])) is not None
+
+ def _entries(self, query, url):
+ query['size'] = 50
+ query['from'] = 0
+ for page_num in itertools.count(1):
+ results = self._call_api(
+ 'search', None, query, url,
+ 'Downloading search JSON page %d' % page_num)
+ sl = try_get(results, lambda x: x['sectionList'][0], dict)
+ if sl:
+ results = sl
+ items = results.get('items')
+ if not items or not isinstance(items, list):
+ break
+ for item in items:
+ if not isinstance(item, dict):
+ continue
+ if not self._check_bc_id_exists(item):
+ continue
+ yield self._parse_rf_item(item)
+ size = int_or_none(results.get('size'))
+ if size is not None:
+ query['size'] = size
+ total = int_or_none(results.get('total'))
+ if total is not None and query['from'] + query['size'] > total:
+ break
+ query['from'] += query['size']
+
+ def _real_extract(self, url):
+ query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+ query['type'] = 'session'
+ return self.playlist_result(
+ self._entries(query, url), playlist_title='Search query')
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cjsw.py youtube-dl-2019.05.20/youtube_dl/extractor/cjsw.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cjsw.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cjsw.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ determine_ext,
+ unescapeHTML,
+)
+
+
+class CJSWIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?cjsw\.com/program/(?P<program>[^/]+)/episode/(?P<id>\d+)'
+ _TESTS = [{
+ 'url': 'http://cjsw.com/program/freshly-squeezed/episode/20170620',
+ 'md5': 'cee14d40f1e9433632c56e3d14977120',
+ 'info_dict': {
+ 'id': '91d9f016-a2e7-46c5-8dcb-7cbcd7437c41',
+ 'ext': 'mp3',
+ 'title': 'Freshly Squeezed – Episode June 20, 2017',
+ 'description': 'md5:c967d63366c3898a80d0c7b0ff337202',
+ 'series': 'Freshly Squeezed',
+ 'episode_id': '20170620',
+ },
+ }, {
+ # no description
+ 'url': 'http://cjsw.com/program/road-pops/episode/20170707/',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ program, episode_id = mobj.group('program', 'id')
+ audio_id = '%s/%s' % (program, episode_id)
+
+ webpage = self._download_webpage(url, episode_id)
+
+ title = unescapeHTML(self._search_regex(
+ (r'<h1[^>]+class=["\']episode-header__title["\'][^>]*>(?P<title>[^<]+)',
+ r'data-audio-title=(["\'])(?P<title>(?:(?!\1).)+)\1'),
+ webpage, 'title', group='title'))
+
+ audio_url = self._search_regex(
+ r'<button[^>]+data-audio-src=(["\'])(?P<url>(?:(?!\1).)+)\1',
+ webpage, 'audio url', group='url')
+
+ audio_id = self._search_regex(
+ r'/([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})\.mp3',
+ audio_url, 'audio id', default=audio_id)
+
+ formats = [{
+ 'url': audio_url,
+ 'ext': determine_ext(audio_url, 'mp3'),
+ 'vcodec': 'none',
+ }]
+
+ description = self._html_search_regex(
+ r'<p>(?P<description>.+?)</p>', webpage, 'description',
+ default=None)
+ series = self._search_regex(
+ r'data-showname=(["\'])(?P<name>(?:(?!\1).)+)\1', webpage,
+ 'series', default=program, group='name')
+
+ return {
+ 'id': audio_id,
+ 'title': title,
+ 'description': description,
+ 'formats': formats,
+ 'series': series,
+ 'episode_id': episode_id,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/clipfish.py youtube-dl-2019.05.20/youtube_dl/extractor/clipfish.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/clipfish.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/clipfish.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,62 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
- int_or_none,
- unified_strdate,
-)
-
-
-class ClipfishIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
- _TEST = {
- 'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
- 'md5': '79bc922f3e8a9097b3d68a93780fd475',
- 'info_dict': {
- 'id': '3966754',
- 'ext': 'mp4',
- 'title': 'FIFA 14 - E3 2013 Trailer',
- 'description': 'Video zu FIFA 14: E3 2013 Trailer',
- 'upload_date': '20130611',
- 'duration': 82,
- 'view_count': int,
- }
- }
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- video_info = self._download_json(
- 'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
- video_id)['items'][0]
-
- formats = []
-
- m3u8_url = video_info.get('media_videourl_hls')
- if m3u8_url:
- formats.append({
- 'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
- 'ext': 'mp4',
- 'format_id': 'hls',
- })
-
- mp4_url = video_info.get('media_videourl')
- if mp4_url:
- formats.append({
- 'url': mp4_url,
- 'format_id': 'mp4',
- 'width': int_or_none(video_info.get('width')),
- 'height': int_or_none(video_info.get('height')),
- 'tbr': int_or_none(video_info.get('bitrate')),
- })
-
- return {
- 'id': video_id,
- 'title': video_info['title'],
- 'description': video_info.get('descr'),
- 'formats': formats,
- 'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
- 'duration': int_or_none(video_info.get('media_length')),
- 'upload_date': unified_strdate(video_info.get('pubDate')),
- 'view_count': int_or_none(video_info.get('media_views'))
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cliphunter.py youtube-dl-2019.05.20/youtube_dl/extractor/cliphunter.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cliphunter.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cliphunter.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,39 +1,41 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-_translation_table = {
- 'a': 'h', 'd': 'e', 'e': 'v', 'f': 'o', 'g': 'f', 'i': 'd', 'l': 'n',
- 'm': 'a', 'n': 'm', 'p': 'u', 'q': 't', 'r': 's', 'v': 'p', 'x': 'r',
- 'y': 'l', 'z': 'i',
- '$': ':', '&': '.', '(': '=', '^': '&', '=': '/',
-}
-
-
-def _decode(s):
- return ''.join(_translation_table.get(c, c) for c in s)
+from ..utils import (
+ int_or_none,
+ url_or_none,
+)
class CliphunterIE(InfoExtractor):
IE_NAME = 'cliphunter'
- _VALID_URL = r'''(?x)http://(?:www\.)?cliphunter\.com/w/
+ _VALID_URL = r'''(?x)https?://(?:www\.)?cliphunter\.com/w/
(?P<id>[0-9]+)/
(?P<seo>.+?)(?:$|[#\?])
'''
- _TEST = {
+ _TESTS = [{
'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
'info_dict': {
'id': '1012420',
'ext': 'flv',
'title': 'Fun Jynx Maze solo',
- 'thumbnail': 're:^https?://.*\.jpg$',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'age_limit': 18,
+ },
+ 'skip': 'Video gone',
+ }, {
+ 'url': 'http://www.cliphunter.com/w/2019449/ShesNew__My_booty_girlfriend_Victoria_Paradices_pussy_filled_with_jizz',
+ 'md5': '55a723c67bfc6da6b0cfa00d55da8a27',
+ 'info_dict': {
+ 'id': '2019449',
+ 'ext': 'mp4',
+ 'title': 'ShesNew - My booty girlfriend, Victoria Paradice\'s pussy filled with jizz',
+ 'thumbnail': r're:^https?://.*\.jpg$',
'age_limit': 18,
- }
- }
+ },
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -49,14 +51,14 @@
formats = []
for format_id, f in gexo_files.items():
- video_url = f.get('url')
+ video_url = url_or_none(f.get('url'))
if not video_url:
continue
fmt = f.get('fmt')
height = f.get('h')
format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id
formats.append({
- 'url': _decode(video_url),
+ 'url': video_url,
'format_id': format_id,
'width': int_or_none(f.get('w')),
'height': int_or_none(height),
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/clippit.py youtube-dl-2019.05.20/youtube_dl/extractor/clippit.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/clippit.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/clippit.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,74 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_iso8601,
+ qualities,
+)
+
+import re
+
+
+class ClippitIE(InfoExtractor):
+
+ _VALID_URL = r'https?://(?:www\.)?clippituser\.tv/c/(?P<id>[a-z]+)'
+ _TEST = {
+ 'url': 'https://www.clippituser.tv/c/evmgm',
+ 'md5': '963ae7a59a2ec4572ab8bf2f2d2c5f09',
+ 'info_dict': {
+ 'id': 'evmgm',
+ 'ext': 'mp4',
+ 'title': 'Bye bye Brutus. #BattleBots - Clippit',
+ 'uploader': 'lizllove',
+ 'uploader_url': 'https://www.clippituser.tv/p/lizllove',
+ 'timestamp': 1472183818,
+ 'upload_date': '20160826',
+ 'description': 'BattleBots | ABC',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(r'<title.*>(.+?)', webpage, 'title')
+
+ FORMATS = ('sd', 'hd')
+ quality = qualities(FORMATS)
+ formats = []
+ for format_id in FORMATS:
+ url = self._html_search_regex(r'data-%s-file="(.+?)"' % format_id,
+ webpage, 'url', fatal=False)
+ if not url:
+ continue
+ match = re.search(r'/(?P\d+)\.mp4', url)
+ formats.append({
+ 'url': url,
+ 'format_id': format_id,
+ 'quality': quality(format_id),
+ 'height': int(match.group('height')) if match else None,
+ })
+
+ uploader = self._html_search_regex(r'class="username".*>\s+(.+?)\n',
+ webpage, 'uploader', fatal=False)
+ uploader_url = ('https://www.clippituser.tv/p/' + uploader
+ if uploader else None)
+
+ timestamp = self._html_search_regex(r'datetime="(.+?)"',
+ webpage, 'date', fatal=False)
+ thumbnail = self._html_search_regex(r'data-image="(.+?)"',
+ webpage, 'thumbnail', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'uploader': uploader,
+ 'uploader_url': uploader_url,
+ 'timestamp': parse_iso8601(timestamp),
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': thumbnail,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cliprs.py youtube-dl-2019.05.20/youtube_dl/extractor/cliprs.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cliprs.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cliprs.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .onet import OnetBaseIE
+
+
+class ClipRsIE(OnetBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?clip\.rs/(?P[^/]+)/\d+'
+ _TEST = {
+ 'url': 'http://www.clip.rs/premijera-frajle-predstavljaju-novi-spot-za-pesmu-moli-me-moli/3732',
+ 'md5': 'c412d57815ba07b56f9edc7b5d6a14e5',
+ 'info_dict': {
+ 'id': '1488842.1399140381',
+ 'ext': 'mp4',
+ 'title': 'PREMIJERA Frajle predstavljaju novi spot za pesmu Moli me, moli',
+ 'description': 'md5:56ce2c3b4ab31c5a2e0b17cb9a453026',
+ 'duration': 229,
+ 'timestamp': 1459850243,
+ 'upload_date': '20160405',
+ }
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ mvp_id = self._search_mvp_id(webpage)
+
+ info_dict = self._extract_from_id(mvp_id, webpage)
+ info_dict['display_id'] = display_id
+
+ return info_dict
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/clipsyndicate.py youtube-dl-2019.05.20/youtube_dl/extractor/clipsyndicate.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/clipsyndicate.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/clipsyndicate.py 2019-06-07 18:49:07.000000000 +0000
@@ -8,7 +8,7 @@
class ClipsyndicateIE(InfoExtractor):
- _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P\d+)'
+ _VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P\d+)'
_TESTS = [{
'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
@@ -18,7 +18,7 @@
'ext': 'mp4',
'title': 'Brick Briscoe',
'duration': 612,
- 'thumbnail': 're:^https?://.+\.jpg',
+ 'thumbnail': r're:^https?://.+\.jpg',
},
}, {
'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/closertotruth.py youtube-dl-2019.05.20/youtube_dl/extractor/closertotruth.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/closertotruth.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/closertotruth.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CloserToTruthIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(?:[^/]+/)*(?P[^/?#&]+)'
+ _TESTS = [{
+ 'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
+ 'info_dict': {
+ 'id': '0_zof1ktre',
+ 'display_id': 'solutions-the-mind-body-problem',
+ 'ext': 'mov',
+ 'title': 'Solutions to the Mind-Body Problem?',
+ 'upload_date': '20140221',
+ 'timestamp': 1392956007,
+ 'uploader_id': 'CTTXML'
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'http://closertotruth.com/episodes/how-do-brains-work',
+ 'info_dict': {
+ 'id': '0_iuxai6g6',
+ 'display_id': 'how-do-brains-work',
+ 'ext': 'mov',
+ 'title': 'How do Brains Work?',
+ 'upload_date': '20140221',
+ 'timestamp': 1392956024,
+ 'uploader_id': 'CTTXML'
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'http://closertotruth.com/interviews/1725',
+ 'info_dict': {
+ 'id': '1725',
+ 'title': 'AyaFr-002',
+ },
+ 'playlist_mincount': 2,
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ partner_id = self._search_regex(
+ r']+src=["\'].*?\b(?:partner_id|p)/(\d+)',
+ webpage, 'kaltura partner_id')
+
+ title = self._search_regex(
+ r'(.+?)\s*\|\s*.+?', webpage, 'video title')
+
+ select = self._search_regex(
+ r'(?s)]+id="select-version"[^>]*>(.+?)',
+ webpage, 'select version', default=None)
+ if select:
+ entry_ids = set()
+ entries = []
+ for mobj in re.finditer(
+ r']+value=(["\'])(?P[0-9a-z_]+)(?:#.+?)?\1[^>]*>(?P[^<]+)',
+ webpage):
+ entry_id = mobj.group('id')
+ if entry_id in entry_ids:
+ continue
+ entry_ids.add(entry_id)
+ entries.append({
+ '_type': 'url_transparent',
+ 'url': 'kaltura:%s:%s' % (partner_id, entry_id),
+ 'ie_key': 'Kaltura',
+ 'title': mobj.group('title'),
+ })
+ if entries:
+ return self.playlist_result(entries, display_id, title)
+
+ entry_id = self._search_regex(
+ r'<a[^>]+id=(["\'])embed-kaltura\1[^>]+data-kaltura=(["\'])(?P<id>[0-9a-z_]+)\2',
+ webpage, 'kaltura entry_id', group='id')
+
+ return {
+ '_type': 'url_transparent',
+ 'display_id': display_id,
+ 'url': 'kaltura:%s:%s' % (partner_id, entry_id),
+ 'ie_key': 'Kaltura',
+ 'title': title
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cloudflarestream.py youtube-dl-2019.05.20/youtube_dl/extractor/cloudflarestream.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cloudflarestream.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cloudflarestream.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CloudflareStreamIE(InfoExtractor):
+ _VALID_URL = r'''(?x)
+ https?://
+ (?:
+ (?:watch\.)?(?:cloudflarestream\.com|videodelivery\.net)/|
+ embed\.(?:cloudflarestream\.com|videodelivery\.net)/embed/[^/]+\.js\?.*?\bvideo=
+ )
+ (?P<id>[\da-f]+)
+ '''
+ _TESTS = [{
+ 'url': 'https://embed.cloudflarestream.com/embed/we4g.fla9.latest.js?video=31c9291ab41fac05471db4e73aa11717',
+ 'info_dict': {
+ 'id': '31c9291ab41fac05471db4e73aa11717',
+ 'ext': 'mp4',
+ 'title': '31c9291ab41fac05471db4e73aa11717',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://cloudflarestream.com/31c9291ab41fac05471db4e73aa11717/manifest/video.mpd',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://embed.videodelivery.net/embed/r4xu.fla9.latest.js?video=81d80727f3022488598f68d323c1ad5e',
+ 'only_matching': True,
+ }]
+
+ @staticmethod
+ def _extract_urls(webpage):
+ return [
+ mobj.group('url')
+ for mobj in re.finditer(
+ r'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//embed\.(?:cloudflarestream\.com|videodelivery\.net)/embed/[^/]+\.js\?.*?\bvideo=[\da-f]+?.*?)\1',
+ webpage)]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ formats = self._extract_m3u8_formats(
+ 'https://cloudflarestream.com/%s/manifest/video.m3u8' % video_id,
+ video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls',
+ fatal=False)
+ formats.extend(self._extract_mpd_formats(
+ 'https://cloudflarestream.com/%s/manifest/video.mpd' % video_id,
+ video_id, mpd_id='dash', fatal=False))
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video_id,
+ 'formats': formats,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cloudy.py youtube-dl-2019.05.20/youtube_dl/extractor/cloudy.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cloudy.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cloudy.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,111 +1,60 @@
# coding: utf-8
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
-from ..compat import (
- compat_parse_qs,
- compat_urllib_parse,
- compat_HTTPError,
-)
from ..utils import (
- ExtractorError,
- HEADRequest,
- remove_end,
+ str_to_int,
+ unified_strdate,
)
class CloudyIE(InfoExtractor):
- _IE_DESC = 'cloudy.ec and videoraj.ch'
- _VALID_URL = r'''(?x)
- https?://(?:www\.)?(?P<host>cloudy\.ec|videoraj\.ch)/
- (?:v/|embed\.php\?id=)
- (?P<id>[A-Za-z0-9]+)
- '''
- _EMBED_URL = 'http://www.%s/embed.php?id=%s'
- _API_URL = 'http://www.%s/api/player.api.php?%s'
- _MAX_TRIES = 2
- _TESTS = [
- {
- 'url': 'https://www.cloudy.ec/v/af511e2527aac',
- 'md5': '5cb253ace826a42f35b4740539bedf07',
- 'info_dict': {
- 'id': 'af511e2527aac',
- 'ext': 'flv',
- 'title': 'Funny Cats and Animals Compilation june 2013',
- }
- },
- {
- 'url': 'http://www.videoraj.ch/v/47f399fd8bb60',
- 'md5': '7d0f8799d91efd4eda26587421c3c3b0',
- 'info_dict': {
- 'id': '47f399fd8bb60',
- 'ext': 'flv',
- 'title': 'Burning a New iPhone 5 with Gasoline - Will it Survive?',
- }
+ _IE_DESC = 'cloudy.ec'
+ _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
+ _TESTS = [{
+ 'url': 'https://www.cloudy.ec/v/af511e2527aac',
+ 'md5': '29832b05028ead1b58be86bf319397ca',
+ 'info_dict': {
+ 'id': 'af511e2527aac',
+ 'ext': 'mp4',
+ 'title': 'Funny Cats and Animals Compilation june 2013',
+ 'upload_date': '20130913',
+ 'view_count': int,
}
- ]
+ }, {
+ 'url': 'http://www.cloudy.ec/embed.php?autoplay=1&id=af511e2527aac',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
- def _extract_video(self, video_host, video_id, file_key, error_url=None, try_num=0):
+ webpage = self._download_webpage(
+ 'https://www.cloudy.ec/embed.php', video_id, query={
+ 'id': video_id,
+ 'playerPage': 1,
+ 'autoplay': 1,
+ })
- if try_num > self._MAX_TRIES - 1:
- raise ExtractorError('Unable to extract video URL', expected=True)
+ info = self._parse_html5_media_entries(url, webpage, video_id)[0]
- form = {
- 'file': video_id,
- 'key': file_key,
- }
+ webpage = self._download_webpage(
+ 'https://www.cloudy.ec/v/%s' % video_id, video_id, fatal=False)
- if error_url:
- form.update({
- 'numOfErrors': try_num,
- 'errorCode': '404',
- 'errorUrl': error_url,
+ if webpage:
+ info.update({
+ 'title': self._search_regex(
+ r'<h\d[^>]*>([^<]+)<', webpage, 'title'),
+ 'upload_date': unified_strdate(self._search_regex(
+ r'>Published at (\d{4}-\d{1,2}-\d{1,2})', webpage,
+ 'upload date', fatal=False)),
+ 'view_count': str_to_int(self._search_regex(
+ r'([\d,.]+) views<', webpage, 'view count', fatal=False)),
})
- data_url = self._API_URL % (video_host, compat_urllib_parse.urlencode(form))
- player_data = self._download_webpage(
- data_url, video_id, 'Downloading player data')
- data = compat_parse_qs(player_data)
-
- try_num += 1
-
- if 'error' in data:
- raise ExtractorError(
- '%s error: %s' % (self.IE_NAME, ' '.join(data['error_msg'])),
- expected=True)
-
- title = data.get('title', [None])[0]
- if title:
- title = remove_end(title, '&asdasdas').strip()
-
- video_url = data.get('url', [None])[0]
-
- if video_url:
- try:
- self._request_webpage(HEADRequest(video_url), video_id, 'Checking video URL')
- except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]:
- self.report_warning('Invalid video URL, requesting another', video_id)
- return self._extract_video(video_host, video_id, file_key, video_url, try_num)
-
- return {
- 'id': video_id,
- 'url': video_url,
- 'title': title,
- }
+ if not info.get('title'):
+ info['title'] = video_id
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_host = mobj.group('host')
- video_id = mobj.group('id')
-
- url = self._EMBED_URL % (video_host, video_id)
- webpage = self._download_webpage(url, video_id)
-
- file_key = self._search_regex(
- [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
- webpage, 'file_key')
+ info['id'] = video_id
- return self._extract_video(video_host, video_id, file_key)
+ return info
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/clubic.py youtube-dl-2019.05.20/youtube_dl/extractor/clubic.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/clubic.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/clubic.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,9 +1,6 @@
# coding: utf-8
from __future__ import unicode_literals
-import json
-import re
-
from .common import InfoExtractor
from ..utils import (
clean_html,
@@ -12,7 +9,7 @@
class ClubicIE(InfoExtractor):
- _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
+ _VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
_TESTS = [{
'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
@@ -22,7 +19,7 @@
'ext': 'mp4',
'title': 'Clubic Week 2.0 : le FBI se lance dans la photo d\u0092identité',
'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*',
- 'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$',
+ 'thumbnail': r're:^http://img\.clubic\.com/.*\.jpg$',
}
}, {
'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html',
@@ -30,16 +27,14 @@
}]
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
+ video_id = self._match_id(url)
player_url = 'http://player.m6web.fr/v1/player/clubic/%s.html' % video_id
player_page = self._download_webpage(player_url, video_id)
- config_json = self._search_regex(
+ config = self._parse_json(self._search_regex(
r'(?m)M6\.Player\.config\s*=\s*(\{.+?\});$', player_page,
- 'configuration')
- config = json.loads(config_json)
+ 'configuration'), video_id)
video_info = config['videoInfo']
sources = config['sources']
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/clyp.py youtube-dl-2019.05.20/youtube_dl/extractor/clyp.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/clyp.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/clyp.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,15 +1,19 @@
from __future__ import unicode_literals
from .common import InfoExtractor
+from ..compat import (
+ compat_parse_qs,
+ compat_urllib_parse_urlparse,
+)
from ..utils import (
float_or_none,
- parse_iso8601,
+ unified_timestamp,
)
class ClypIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
- _TEST = {
+ _TESTS = [{
'url': 'https://clyp.it/ojz2wfah',
'md5': '1d4961036c41247ecfdcc439c0cddcbb',
'info_dict': {
@@ -21,13 +25,34 @@
'timestamp': 1443515251,
'upload_date': '20150929',
},
- }
+ }, {
+ 'url': 'https://clyp.it/b04p1odi?token=b0078e077e15835845c528a44417719d',
+ 'info_dict': {
+ 'id': 'b04p1odi',
+ 'ext': 'mp3',
+ 'title': 'GJ! (Reward Edit)',
+ 'description': 'Metal Resistance (THE ONE edition)',
+ 'duration': 177.789,
+ 'timestamp': 1528241278,
+ 'upload_date': '20180605',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
def _real_extract(self, url):
audio_id = self._match_id(url)
+ qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+ token = qs.get('token', [None])[0]
+
+ query = {}
+ if token:
+ query['token'] = token
+
metadata = self._download_json(
- 'https://api.clyp.it/%s' % audio_id, audio_id)
+ 'https://api.clyp.it/%s' % audio_id, audio_id, query=query)
formats = []
for secure in ('', 'Secure'):
@@ -45,7 +70,7 @@
title = metadata['Title']
description = metadata.get('Description')
duration = float_or_none(metadata.get('Duration'))
- timestamp = parse_iso8601(metadata.get('DateCreated'))
+ timestamp = unified_timestamp(metadata.get('DateCreated'))
return {
'id': audio_id,
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cmt.py youtube-dl-2019.05.20/youtube_dl/extractor/cmt.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cmt.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cmt.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,11 +1,11 @@
from __future__ import unicode_literals
+
from .mtv import MTVIE
class CMTIE(MTVIE):
IE_NAME = 'cmt.com'
- _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
- _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
+ _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|(?:full-)?episodes|video-clips)/(?P<id>[^/]+)'
_TESTS = [{
'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
@@ -16,7 +16,39 @@
'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
'description': 'Blame It All On My Roots',
},
+ 'skip': 'Video not available',
+ }, {
+ 'url': 'http://www.cmt.com/videos/misc/1504699/still-the-king-ep-109-in-3-minutes.jhtml#id=1739908',
+ 'md5': 'e61a801ca4a183a466c08bd98dccbb1c',
+ 'info_dict': {
+ 'id': '1504699',
+ 'ext': 'mp4',
+ 'title': 'Still The King Ep. 109 in 3 Minutes',
+ 'description': 'Relive or catch up with Still The King by watching this recap of season 1, episode 9.',
+ 'timestamp': 1469421000.0,
+ 'upload_date': '20160725',
+ },
}, {
'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
'only_matching': True,
+ }, {
+ 'url': 'http://www.cmt.com/full-episodes/537qb3/nashville-the-wayfaring-stranger-season-5-ep-501',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.cmt.com/video-clips/t9e4ci/nashville-juliette-in-2-minutes',
+ 'only_matching': True,
}]
+
+ def _extract_mgid(self, webpage):
+ mgid = self._search_regex(
+ r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1',
+ webpage, 'mgid', group='mgid', default=None)
+ if not mgid:
+ mgid = self._extract_triforce_mgid(webpage)
+ return mgid
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ mgid = self._extract_mgid(webpage)
+ return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cnbc.py youtube-dl-2019.05.20/youtube_dl/extractor/cnbc.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cnbc.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cnbc.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class CNBCIE(InfoExtractor):
+ _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://video.cnbc.com/gallery/?video=3000503714',
+ 'info_dict': {
+ 'id': '3000503714',
+ 'ext': 'mp4',
+ 'title': 'Fighting zombies is big business',
+ 'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
+ 'timestamp': 1459332000,
+ 'upload_date': '20160330',
+ 'uploader': 'NBCU-CNBC',
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ return {
+ '_type': 'url_transparent',
+ 'ie_key': 'ThePlatform',
+ 'url': smuggle_url(
+ 'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id,
+ {'force_smil_url': True}),
+ 'id': video_id,
+ }
+
+
+class CNBCVideoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?cnbc\.com/video/(?:[^/]+/)+(?P<id>[^./?#&]+)'
+ _TEST = {
+ 'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
+ 'info_dict': {
+ 'id': '7000031301',
+ 'ext': 'mp4',
+ 'title': "Trump: I don't necessarily agree with raising rates",
+ 'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3',
+ 'timestamp': 1531958400,
+ 'upload_date': '20180719',
+ 'uploader': 'NBCU-CNBC',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+ video_id = self._search_regex(
+ r'content_id["\']\s*:\s*["\'](\d+)', webpage, display_id,
+ 'video id')
+ return self.url_result(
+ 'http://video.cnbc.com/gallery/?video=%s' % video_id,
+ CNBCIE.ie_key())
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cnet.py youtube-dl-2019.05.20/youtube_dl/extractor/cnet.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cnet.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cnet.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,84 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .theplatform import ThePlatformIE
-from ..utils import int_or_none
-
-
-class CNETIE(ThePlatformIE):
- _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
- _TESTS = [{
- 'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
- 'info_dict': {
- 'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
- 'ext': 'flv',
- 'title': 'Hands-on with Microsoft Windows 8.1 Update',
- 'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
- 'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
- 'uploader': 'Sarah Mitroff',
- 'duration': 70,
- },
- }, {
- 'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
- 'info_dict': {
- 'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
- 'ext': 'flv',
- 'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
- 'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
- 'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
- 'uploader': 'Ashley Esqueda',
- 'duration': 1482,
- },
- }]
-
- def _real_extract(self, url):
- display_id = self._match_id(url)
- webpage = self._download_webpage(url, display_id)
-
- data_json = self._html_search_regex(
- r"data-cnet-video(?:-uvp)?-options='([^']+)'",
- webpage, 'data json')
- data = self._parse_json(data_json, display_id)
- vdata = data.get('video') or data['videos'][0]
-
- video_id = vdata['id']
- title = vdata['title']
- author = vdata.get('author')
- if author:
- uploader = '%s %s' % (author['firstName'], author['lastName'])
- uploader_id = author.get('id')
- else:
- uploader = None
- uploader_id = None
-
- mpx_account = data['config']['uvpConfig']['default']['mpx_account']
-
- metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id)
- description = vdata.get('description') or metadata.get('description')
- duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
-
- formats = []
- subtitles = {}
- for (fkey, vid) in vdata['files'].items():
- if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
- continue
- release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid)
- if fkey == 'hds':
- release_url += '&manifest=f4m'
- tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
- formats.extend(tp_formats)
- subtitles = self._merge_subtitles(subtitles, tp_subtitles)
- self._sort_formats(formats)
-
- return {
- 'id': video_id,
- 'display_id': display_id,
- 'title': title,
- 'description': description,
- 'thumbnail': metadata.get('thumbnail'),
- 'duration': duration,
- 'uploader': uploader,
- 'uploader_id': uploader_id,
- 'subtitles': subtitles,
- 'formats': formats,
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/cnn.py youtube-dl-2019.05.20/youtube_dl/extractor/cnn.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/cnn.py 2016-02-14 22:26:17.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/cnn.py 2019-06-07 18:49:07.000000000 +0000
@@ -3,15 +3,12 @@
import re
from .common import InfoExtractor
-from ..utils import (
- int_or_none,
- parse_duration,
- url_basename,
-)
+from .turner import TurnerBaseIE
+from ..utils import url_basename
-class CNNIE(InfoExtractor):
- _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
+class CNNIE(TurnerBaseIE):
+ _VALID_URL = r'''(?x)https?://(?:(?P<sub_domain>edition|www|money)\.)?cnn\.com/(?:video/(?:data/.+?|\?)/)?videos?/
(?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
_TESTS = [{
@@ -25,6 +22,7 @@
'duration': 135,
'upload_date': '20130609',
},
+ 'expected_warnings': ['Failed to download m3u8 information'],
}, {
'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
@@ -34,7 +32,8 @@
'title': "Student's epic speech stuns new freshmen",
'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
'upload_date': '20130821',
- }
+ },
+ 'expected_warnings': ['Failed to download m3u8 information'],
}, {
'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
'md5': 'f14d02ebd264df951feb2400e2c25a1b',
@@ -44,80 +43,61 @@
'title': 'Nashville Ep. 1: Hand crafted skateboards',
'description': 'md5:e7223a503315c9f150acac52e76de086',
'upload_date': '20141222',
- }
+ },
+ 'expected_warnings': ['Failed to download m3u8 information'],
+ }, {
+ 'url': 'http://money.cnn.com/video/news/2016/08/19/netflix-stunning-stats.cnnmoney/index.html',
+ 'md5': '52a515dc1b0f001cd82e4ceda32be9d1',
+ 'info_dict': {
+ 'id': '/video/news/2016/08/19/netflix-stunning-stats.cnnmoney',
+ 'ext': 'mp4',
+ 'title': '5 stunning stats about Netflix',
+ 'description': 'Did you know that Netflix has more than 80 million members? Here are five facts about the online video distributor that you probably didn\'t know.',
+ 'upload_date': '20160819',
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
}, {
'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
'only_matching': True,
}, {
'url': 'http://cnn.com/video/?/video/us/2015/04/06/dnt-baker-refuses-anti-gay-order.wkmg',
'only_matching': True,
+ }, {
+ 'url': 'http://edition.cnn.com/videos/arts/2016/04/21/olympic-games-cultural-a-z-brazil.cnn',
+ 'only_matching': True,
}]
+ _CONFIG = {
+ # http://edition.cnn.com/.element/apps/cvp/3.0/cfg/spider/cnn/expansion/config.xml
+ 'edition': {
+ 'data_src': 'http://edition.cnn.com/video/data/3.0/video/%s/index.xml',
+ 'media_src': 'http://pmd.cdn.turner.com/cnn/big',
+ },
+ # http://money.cnn.com/.element/apps/cvp2/cfg/config.xml
+ 'money': {
+ 'data_src': 'http://money.cnn.com/video/data/4.0/video/%s.xml',
+ 'media_src': 'http://ht3.cdn.turner.com/money/big',
+ },
+ }
+
+ def _extract_timestamp(self, video_data):
+ # TODO: fix timestamp extraction
+ return None
+
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- path = mobj.group('path')
- page_title = mobj.group('title')
- info_url = 'http://edition.cnn.com/video/data/3.0/%s/index.xml' % path
- info = self._download_xml(info_url, page_title)
-
- formats = []
- rex = re.compile(r'''(?x)
- (?P<width>[0-9]+)x(?P<height>[0-9]+)
- (?:_(?P<bitrate>[0-9]+)k)?
- ''')
- for f in info.findall('files/file'):
- video_url = 'http://ht.cdn.turner.com/cnn/big%s' % (f.text.strip())
- fdct = {
- 'format_id': f.attrib['bitrate'],
- 'url': video_url,
- }
-
- mf = rex.match(f.attrib['bitrate'])
- if mf:
- fdct['width'] = int(mf.group('width'))
- fdct['height'] = int(mf.group('height'))
- fdct['tbr'] = int_or_none(mf.group('bitrate'))
- else:
- mf = rex.search(f.text)
- if mf:
- fdct['width'] = int(mf.group('width'))
- fdct['height'] = int(mf.group('height'))
- fdct['tbr'] = int_or_none(mf.group('bitrate'))
- else:
- mi = re.match(r'ios_(audio|[0-9]+)$', f.attrib['bitrate'])
- if mi:
- if mi.group(1) == 'audio':
- fdct['vcodec'] = 'none'
- fdct['ext'] = 'm4a'
- else:
- fdct['tbr'] = int(mi.group(1))
-
- formats.append(fdct)
-
- self._sort_formats(formats)
-
- thumbnails = [{
- 'height': int(t.attrib['height']),
- 'width': int(t.attrib['width']),
- 'url': t.text,
- } for t in info.findall('images/image')]
-
- metas_el = info.find('metas')
- upload_date = (
- metas_el.attrib.get('version') if metas_el is not None else None)
-
- duration_el = info.find('length')
- duration = parse_duration(duration_el.text)
-
- return {
- 'id': info.attrib['id'],
- 'title': info.find('headline').text,
- 'formats': formats,
- 'thumbnails': thumbnails,
- 'description': info.find('description').text,
- 'duration': duration,
- 'upload_date': upload_date,
- }
+ sub_domain, path, page_title = re.match(self._VALID_URL, url).groups()
+ if sub_domain not in ('money', 'edition'):
+ sub_domain = 'edition'
+ config = self._CONFIG[sub_domain]
+ return self._extract_cvp_info(
+ config['data_src'] % path, page_title, {
+ 'default': {
+ 'media_src': config['media_src'],
+ }
+ })
class CNNBlogsIE(InfoExtractor):
@@ -132,21 +112,18 @@
'description': 'Glenn Greenwald responds to comments made this week on Capitol Hill that journalists could be criminal accessories.',
'upload_date': '20140209',
},
+ 'expected_warnings': ['Failed to download m3u8 information'],
'add_ie': ['CNN'],
}
def _real_extract(self, url):
webpage = self._download_webpage(url, url_basename(url))
cnn_url = self._html_search_regex(r'data-url="(.+?)"', webpage, 'cnn url')
- return {
- '_type': 'url',
- 'url': cnn_url,
- 'ie_key': CNNIE.ie_key(),
- }
+ return self.url_result(cnn_url, CNNIE.ie_key())
class CNNArticleIE(InfoExtractor):
- _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
+ _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!videos?/)'
_TEST = {
'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
@@ -154,17 +131,14 @@
'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
'ext': 'mp4',
'title': 'Obama: Cyberattack not an act of war',
- 'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
+ 'description': 'md5:0a802a40d2376f60e6b04c8d5bcebc4b',
'upload_date': '20141221',
},
+ 'expected_warnings': ['Failed to download m3u8 information'],
'add_ie': ['CNN'],
}
def _real_extract(self, url):
webpage = self._download_webpage(url, url_basename(url))
cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
- return {
- '_type': 'url',
- 'url': 'http://cnn.com/video/?/video/' + cnn_url,
- 'ie_key': CNNIE.ie_key(),
- }
+ return self.url_result('http://cnn.com/video/?/video/' + cnn_url, CNNIE.ie_key())
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/collegehumor.py youtube-dl-2019.05.20/youtube_dl/extractor/collegehumor.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/collegehumor.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/collegehumor.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,101 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class CollegeHumorIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
-
- _TESTS = [
- {
- 'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
- 'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
- 'info_dict': {
- 'id': '6902724',
- 'ext': 'mp4',
- 'title': 'Comic-Con Cosplay Catastrophe',
- 'description': "Fans get creative this year at San Diego. Too creative. And yes, that's really Joss Whedon.",
- 'age_limit': 13,
- 'duration': 187,
- },
- }, {
- 'url': 'http://www.collegehumor.com/video/3505939/font-conference',
- 'md5': '72fa701d8ef38664a4dbb9e2ab721816',
- 'info_dict': {
- 'id': '3505939',
- 'ext': 'mp4',
- 'title': 'Font Conference',
- 'description': "This video wasn't long enough, so we made it double-spaced.",
- 'age_limit': 10,
- 'duration': 179,
- },
- }, {
- # embedded youtube video
- 'url': 'http://www.collegehumor.com/embed/6950306',
- 'info_dict': {
- 'id': 'Z-bao9fg6Yc',
- 'ext': 'mp4',
- 'title': 'Young Americans Think President John F. Kennedy Died THIS MORNING IN A CAR ACCIDENT!!!',
- 'uploader': 'Mark Dice',
- 'uploader_id': 'MarkDice',
- 'description': 'md5:62c3dab9351fac7bb44b53b69511d87f',
- 'upload_date': '20140127',
- },
- 'params': {
- 'skip_download': True,
- },
- 'add_ie': ['Youtube'],
- },
- ]
-
- def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('videoid')
-
- jsonUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + '.json'
- data = json.loads(self._download_webpage(
- jsonUrl, video_id, 'Downloading info JSON'))
- vdata = data['video']
- if vdata.get('youtubeId') is not None:
- return {
- '_type': 'url',
- 'url': vdata['youtubeId'],
- 'ie_key': 'Youtube',
- }
-
- AGE_LIMITS = {'nc17': 18, 'r': 18, 'pg13': 13, 'pg': 10, 'g': 0}
- rating = vdata.get('rating')
- if rating:
- age_limit = AGE_LIMITS.get(rating.lower())
- else:
- age_limit = None # None = No idea
-
- PREFS = {'high_quality': 2, 'low_quality': 0}
- formats = []
- for format_key in ('mp4', 'webm'):
- for qname, qurl in vdata.get(format_key, {}).items():
- formats.append({
- 'format_id': format_key + '_' + qname,
- 'url': qurl,
- 'format': format_key,
- 'preference': PREFS.get(qname),
- })
- self._sort_formats(formats)
-
- duration = int_or_none(vdata.get('duration'), 1000)
- like_count = int_or_none(vdata.get('likes'))
-
- return {
- 'id': video_id,
- 'title': vdata['title'],
- 'description': vdata.get('description'),
- 'thumbnail': vdata.get('thumbnail'),
- 'formats': formats,
- 'age_limit': age_limit,
- 'duration': duration,
- 'like_count': like_count,
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/collegerama.py youtube-dl-2019.05.20/youtube_dl/extractor/collegerama.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/collegerama.py 2016-02-14 22:26:17.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/collegerama.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,92 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
- float_or_none,
- int_or_none,
- sanitized_Request,
-)
-
-
-class CollegeRamaIE(InfoExtractor):
- _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
- _TESTS = [
- {
- 'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
- 'md5': '481fda1c11f67588c0d9d8fbdced4e39',
- 'info_dict': {
- 'id': '585a43626e544bdd97aeb71a0ec907a01d',
- 'ext': 'mp4',
- 'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
- 'description': '',
- 'thumbnail': 're:^https?://.*\.jpg$',
- 'duration': 7713.088,
- 'timestamp': 1413309600,
- 'upload_date': '20141014',
- },
- },
- {
- 'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
- 'md5': 'ef1fdded95bdf19b12c5999949419c92',
- 'info_dict': {
- 'id': '86a9ea9f53e149079fbdb4202b521ed21d',
- 'ext': 'wmv',
- 'title': '64ste Vakantiecursus: Afvalwater',
- 'description': 'md5:7fd774865cc69d972f542b157c328305',
- 'duration': 10853,
- 'timestamp': 1326446400,
- 'upload_date': '20120113',
- },
- },
- ]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- player_options_request = {
- 'getPlayerOptionsRequest': {
- 'ResourceId': video_id,
- 'QueryString': '',
- }
- }
-
- request = sanitized_Request(
- 'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
- json.dumps(player_options_request))
- request.add_header('Content-Type', 'application/json')
-
- player_options = self._download_json(request, video_id)
-
- presentation = player_options['d']['Presentation']
- title = presentation['Title']
- description = presentation.get('Description')
- thumbnail = None
- duration = float_or_none(presentation.get('Duration'), 1000)
- timestamp = int_or_none(presentation.get('UnixTime'), 1000)
-
- formats = []
- for stream in presentation['Streams']:
- for video in stream['VideoUrls']:
- thumbnail_url = stream.get('ThumbnailUrl')
- if thumbnail_url:
- thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
- format_id = video['MediaType']
- if format_id == 'SS':
- continue
- formats.append({
- 'url': video['Location'],
- 'format_id': format_id,
- })
- self._sort_formats(formats)
-
- return {
- 'id': video_id,
- 'title': title,
- 'description': description,
- 'thumbnail': thumbnail,
- 'duration': duration,
- 'timestamp': timestamp,
- 'formats': formats,
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/comcarcoff.py youtube-dl-2019.05.20/youtube_dl/extractor/comcarcoff.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/comcarcoff.py 2016-02-12 20:57:05.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/comcarcoff.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
from __future__ import unicode_literals
from .common import InfoExtractor
@@ -11,7 +11,7 @@
class ComCarCoffIE(InfoExtractor):
- _VALID_URL = r'http://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
+ _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
_TESTS = [{
'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
'info_dict': {
@@ -41,7 +41,13 @@
display_id = full_data['activeVideo']['video']
video_data = full_data.get('videos', {}).get(display_id) or full_data['singleshots'][display_id]
+
video_id = compat_str(video_data['mediaId'])
+ title = video_data['title']
+ formats = self._extract_m3u8_formats(
+ video_data['mediaUrl'], video_id, 'mp4')
+ self._sort_formats(formats)
+
thumbnails = [{
'url': video_data['images']['thumb'],
}, {
@@ -54,15 +60,14 @@
video_data.get('duration'))
return {
- '_type': 'url_transparent',
- 'url': 'crackle:%s' % video_id,
'id': video_id,
'display_id': display_id,
- 'title': video_data['title'],
+ 'title': title,
'description': video_data.get('description'),
'timestamp': timestamp,
'duration': duration,
'thumbnails': thumbnails,
+ 'formats': formats,
'season_number': int_or_none(video_data.get('season')),
'episode_number': int_or_none(video_data.get('episode')),
'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/comedycentral.py youtube-dl-2019.05.20/youtube_dl/extractor/comedycentral.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/comedycentral.py 2016-02-14 22:26:17.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/comedycentral.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,22 +1,12 @@
from __future__ import unicode_literals
-import re
-
from .mtv import MTVServicesInfoExtractor
-from ..compat import (
- compat_str,
- compat_urllib_parse,
-)
-from ..utils import (
- ExtractorError,
- float_or_none,
- unified_strdate,
-)
+from .common import InfoExtractor
class ComedyCentralIE(MTVServicesInfoExtractor):
_VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
- (video-clips|episodes|cc-studios|video-collections|full-episodes|shows)
+ (video-clips|episodes|cc-studios|video-collections|shows(?=/[^/]+/(?!full-episodes)))
/(?P<title>.*)'''
_FEED_URL = 'http://comedycentral.com/feeds/mrss/'
@@ -26,8 +16,10 @@
'info_dict': {
'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
'ext': 'mp4',
- 'title': 'CC:Stand-Up|Greg Fitzsimmons: Life on Stage|Uncensored - Too Good of a Mother',
+ 'title': 'CC:Stand-Up|August 18, 2013|1|0101|Uncensored - Too Good of a Mother',
'description': 'After a certain point, breastfeeding becomes c**kblocking.',
+ 'timestamp': 1376798400,
+ 'upload_date': '20130818',
},
}, {
'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
@@ -35,238 +27,116 @@
}]
-class ComedyCentralShowsIE(MTVServicesInfoExtractor):
- IE_DESC = 'The Daily Show / The Colbert Report'
- # urls can be abbreviations like :thedailyshow
- # urls for episodes like:
- # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
- # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
- # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
- _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
- |https?://(:www\.)?
- (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
- ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|
- (?P<clip>
- (?:(?:guests/[^/]+|videos|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
- |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
- |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
- )|
- (?P<interview>
- extended-interviews/(?P<interID>[0-9a-z]+)/
- (?:playlist_tds_extended_)?(?P<interview_title>[^/?#]*?)
- (?:/[^/?#]?|[?#]|$))))
- '''
+class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor):
+ _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
+ (?:full-episodes|shows(?=/[^/]+/full-episodes))
+ /(?P<id>[^?]+)'''
+ _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
+
_TESTS = [{
- 'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
- 'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
+ 'url': 'http://www.cc.com/full-episodes/pv391a/the-daily-show-with-trevor-noah-november-28--2016---ryan-speedo-green-season-22-ep-22028',
'info_dict': {
- 'id': 'ab9ab3e7-5a98-4dbe-8b21-551dc0523d55',
- 'ext': 'mp4',
- 'upload_date': '20121213',
- 'description': 'Kristen Stewart learns to let loose in "On the Road."',
- 'uploader': 'thedailyshow',
- 'title': 'thedailyshow kristen-stewart part 1',
- }
+ 'description': 'Donald Trump is accused of exploiting his president-elect status for personal gain, Cuban leader Fidel Castro dies, and Ryan Speedo Green discusses "Sing for Your Life."',
+ 'title': 'November 28, 2016 - Ryan Speedo Green',
+ },
+ 'playlist_count': 4,
}, {
- 'url': 'http://thedailyshow.cc.com/extended-interviews/b6364d/sarah-chayes-extended-interview',
+ 'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ webpage = self._download_webpage(url, playlist_id)
+ mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1')
+ videos_info = self._get_videos_info(mgid)
+ return videos_info
+
+
+class ToshIE(MTVServicesInfoExtractor):
+ IE_DESC = 'Tosh.0'
+ _VALID_URL = r'^https?://tosh\.cc\.com/video-(?:clips|collections)/[^/]+/(?P<videotitle>[^/?#]+)'
+ _FEED_URL = 'http://tosh.cc.com/feeds/mrss'
+
+ _TESTS = [{
+ 'url': 'http://tosh.cc.com/video-clips/68g93d/twitter-users-share-summer-plans',
'info_dict': {
- 'id': 'sarah-chayes-extended-interview',
- 'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
- 'title': 'thedailyshow Sarah Chayes Extended Interview',
+ 'description': 'Tosh asked fans to share their summer plans.',
+ 'title': 'Twitter Users Share Summer Plans',
},
- 'playlist': [
- {
- 'info_dict': {
- 'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
- 'ext': 'mp4',
- 'upload_date': '20150129',
- 'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
- 'uploader': 'thedailyshow',
- 'title': 'thedailyshow sarah-chayes-extended-interview part 1',
- },
- },
- {
- 'info_dict': {
- 'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
- 'ext': 'mp4',
- 'upload_date': '20150129',
- 'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
- 'uploader': 'thedailyshow',
- 'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+ 'playlist': [{
+ 'md5': 'f269e88114c1805bb6d7653fecea9e06',
+ 'info_dict': {
+ 'id': '90498ec2-ed00-11e0-aca6-0026b9414f30',
+ 'ext': 'mp4',
+ 'title': 'Tosh.0|June 9, 2077|2|211|Twitter Users Share Summer Plans',
+ 'description': 'Tosh asked fans to share their summer plans.',
+ 'thumbnail': r're:^https?://.*\.jpg',
+ # It's really reported to be published on year 2077
+ 'upload_date': '20770610',
+ 'timestamp': 3390510600,
+ 'subtitles': {
+ 'en': 'mincount:3',
},
},
- ],
+ }]
+ }, {
+ 'url': 'http://tosh.cc.com/video-collections/x2iz7k/just-plain-foul/m5q4fp',
+ 'only_matching': True,
+ }]
+
+
+class ComedyCentralTVIE(MTVServicesInfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'
+ _TESTS = [{
+ 'url': 'http://www.comedycentral.tv/staffeln/7436-the-mindy-project-staffel-4',
+ 'info_dict': {
+ 'id': 'local_playlist-f99b626bdfe13568579a',
+ 'ext': 'flv',
+ 'title': 'Episode_the-mindy-project_shows_season-4_episode-3_full-episode_part1',
+ },
'params': {
+ # rtmp download
'skip_download': True,
},
}, {
- 'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
+ 'url': 'http://www.comedycentral.tv/shows/1074-workaholics',
'only_matching': True,
}, {
- 'url': 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news',
+ 'url': 'http://www.comedycentral.tv/shows/1727-the-mindy-project/bonus',
'only_matching': True,
- }, {
- 'url': 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114',
- 'only_matching': True,
- }, {
- 'url': 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3',
- 'only_matching': True,
- }, {
- 'url': 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary',
- 'only_matching': True,
- }, {
- 'url': 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall',
- 'only_matching': True,
- }, {
- 'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
- 'only_matching': True,
- }, {
- 'url': 'http://thedailyshow.cc.com/video-playlists/t6d9sg/the-daily-show-20038-highlights/be3cwo',
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+
+ mrss_url = self._search_regex(
+ r'data-mrss=(["\'])(?P<url>(?:(?!\1).)+)\1',
+ webpage, 'mrss url', group='url')
+
+ return self._get_videos_info_from_url(mrss_url, video_id)
+
+
+class ComedyCentralShortnameIE(InfoExtractor):
+ _VALID_URL = r'^:(?P<id>tds|thedailyshow|theopposition)$'
+ _TESTS = [{
+ 'url': ':tds',
'only_matching': True,
}, {
- 'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
+ 'url': ':thedailyshow',
'only_matching': True,
}, {
- 'url': 'http://thedailyshow.cc.com/news-team/michael-che/7wnfel/we-need-to-talk-about-israel',
+ 'url': ':theopposition',
'only_matching': True,
}]
- _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
-
- _video_extensions = {
- '3500': 'mp4',
- '2200': 'mp4',
- '1700': 'mp4',
- '1200': 'mp4',
- '750': 'mp4',
- '400': 'mp4',
- }
- _video_dimensions = {
- '3500': (1280, 720),
- '2200': (960, 540),
- '1700': (768, 432),
- '1200': (640, 360),
- '750': (512, 288),
- '400': (384, 216),
- }
-
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
-
- if mobj.group('shortname'):
- return self.url_result('http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes')
-
- if mobj.group('clip'):
- if mobj.group('videotitle'):
- epTitle = mobj.group('videotitle')
- elif mobj.group('showname') == 'thedailyshow':
- epTitle = mobj.group('tdstitle')
- else:
- epTitle = mobj.group('cntitle')
- dlNewest = False
- elif mobj.group('interview'):
- epTitle = mobj.group('interview_title')
- dlNewest = False
- else:
- dlNewest = not mobj.group('episode')
- if dlNewest:
- epTitle = mobj.group('showname')
- else:
- epTitle = mobj.group('episode')
- show_name = mobj.group('showname')
-
- webpage, htmlHandle = self._download_webpage_handle(url, epTitle)
- if dlNewest:
- url = htmlHandle.geturl()
- mobj = re.match(self._VALID_URL, url, re.VERBOSE)
- if mobj is None:
- raise ExtractorError('Invalid redirected URL: ' + url)
- if mobj.group('episode') == '':
- raise ExtractorError('Redirected URL is still not specific: ' + url)
- epTitle = (mobj.group('episode') or mobj.group('videotitle')).rpartition('/')[-1]
-
- mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)
- if len(mMovieParams) == 0:
- # The Colbert Report embeds the information in a without
- # a URL prefix; so extract the alternate reference
- # and then add the URL prefix manually.
-
- altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video|playlist).*?:.*?)"', webpage)
- if len(altMovieParams) == 0:
- raise ExtractorError('unable to find Flash URL in webpage ' + url)
- else:
- mMovieParams = [('http://media.mtvnservices.com/' + altMovieParams[0], altMovieParams[0])]
-
- uri = mMovieParams[0][1]
- # Correct cc.com in uri
- uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.com', uri)
-
- index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse.urlencode({'uri': uri}))
- idoc = self._download_xml(
- index_url, epTitle,
- 'Downloading show index', 'Unable to download episode index')
-
- title = idoc.find('./channel/title').text
- description = idoc.find('./channel/description').text
-
- entries = []
- item_els = idoc.findall('.//item')
- for part_num, itemEl in enumerate(item_els):
- upload_date = unified_strdate(itemEl.findall('./pubDate')[0].text)
- thumbnail = itemEl.find('.//{http://search.yahoo.com/mrss/}thumbnail').attrib.get('url')
-
- content = itemEl.find('.//{http://search.yahoo.com/mrss/}content')
- duration = float_or_none(content.attrib.get('duration'))
- mediagen_url = content.attrib['url']
- guid = itemEl.find('./guid').text.rpartition(':')[-1]
-
- cdoc = self._download_xml(
- mediagen_url, epTitle,
- 'Downloading configuration for segment %d / %d' % (part_num + 1, len(item_els)))
-
- turls = []
- for rendition in cdoc.findall('.//rendition'):
- finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
- turls.append(finfo)
-
- formats = []
- for format, rtmp_video_url in turls:
- w, h = self._video_dimensions.get(format, (None, None))
- formats.append({
- 'format_id': 'vhttp-%s' % format,
- 'url': self._transform_rtmp_url(rtmp_video_url),
- 'ext': self._video_extensions.get(format, 'mp4'),
- 'height': h,
- 'width': w,
- })
- formats.append({
- 'format_id': 'rtmp-%s' % format,
- 'url': rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm'),
- 'ext': self._video_extensions.get(format, 'mp4'),
- 'height': h,
- 'width': w,
- })
- self._sort_formats(formats)
-
- subtitles = self._extract_subtitles(cdoc, guid)
-
- virtual_id = show_name + ' ' + epTitle + ' part ' + compat_str(part_num + 1)
- entries.append({
- 'id': guid,
- 'title': virtual_id,
- 'formats': formats,
- 'uploader': show_name,
- 'upload_date': upload_date,
- 'duration': duration,
- 'thumbnail': thumbnail,
- 'description': description,
- 'subtitles': subtitles,
- })
-
- return {
- '_type': 'playlist',
- 'id': epTitle,
- 'entries': entries,
- 'title': show_name + ' ' + title,
- 'description': description,
+ video_id = self._match_id(url)
+ shortcut_map = {
+ 'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+ 'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+ 'theopposition': 'http://www.cc.com/shows/the-opposition-with-jordan-klepper/full-episodes',
}
+ return self.url_result(shortcut_map[video_id])
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/commonmistakes.py youtube-dl-2019.05.20/youtube_dl/extractor/commonmistakes.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/commonmistakes.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/commonmistakes.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,5 +1,7 @@
from __future__ import unicode_literals
+import sys
+
from .common import InfoExtractor
from ..utils import ExtractorError
@@ -7,7 +9,7 @@
class CommonMistakesIE(InfoExtractor):
IE_DESC = False # Do not list
_VALID_URL = r'''(?x)
- (?:url|URL)
+ (?:url|URL)$
'''
_TESTS = [{
@@ -30,17 +32,19 @@
class UnicodeBOMIE(InfoExtractor):
- IE_DESC = False
- _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
+ IE_DESC = False
+ _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
- _TESTS = [{
- 'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- real_url = self._match_id(url)
- self.report_warning(
- 'Your URL starts with a Byte Order Mark (BOM). '
- 'Removing the BOM and looking for "%s" ...' % real_url)
- return self.url_result(real_url)
+ # Disable test for python 3.2 since BOM is broken in re in this version
+ # (see https://github.com/ytdl-org/youtube-dl/issues/9751)
+ _TESTS = [] if (3, 0) < sys.version_info <= (3, 3) else [{
+ 'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ real_url = self._match_id(url)
+ self.report_warning(
+ 'Your URL starts with a Byte Order Mark (BOM). '
+ 'Removing the BOM and looking for "%s" ...' % real_url)
+ return self.url_result(real_url)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/commonprotocols.py youtube-dl-2019.05.20/youtube_dl/extractor/commonprotocols.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/commonprotocols.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/commonprotocols.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urlparse,
+)
+
+
+class RtmpIE(InfoExtractor):
+ IE_DESC = False # Do not list
+ _VALID_URL = r'(?i)rtmp[est]?://.+'
+
+ _TESTS = [{
+ 'url': 'rtmp://cp44293.edgefcs.net/ondemand?auth=daEcTdydfdqcsb8cZcDbAaCbhamacbbawaS-bw7dBb-bWG-GqpGFqCpNCnGoyL&aifp=v001&slist=public/unsecure/audio/2c97899446428e4301471a8cb72b4b97--audio--pmg-20110908-0900a_flv_aac_med_int.mp4',
+ 'only_matching': True,
+ }, {
+ 'url': 'rtmp://edge.live.hitbox.tv/live/dimak',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._generic_id(url)
+ title = self._generic_title(url)
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': [{
+ 'url': url,
+ 'ext': 'flv',
+ 'format_id': compat_urlparse.urlparse(url).scheme,
+ }],
+ }
+
+
+class MmsIE(InfoExtractor):
+ IE_DESC = False # Do not list
+ _VALID_URL = r'(?i)mms://.+'
+
+ _TEST = {
+ # Direct MMS link
+ 'url': 'mms://kentro.kaist.ac.kr/200907/MilesReid(0709).wmv',
+ 'info_dict': {
+ 'id': 'MilesReid(0709)',
+ 'ext': 'wmv',
+ 'title': 'MilesReid(0709)',
+ },
+ 'params': {
+ 'skip_download': True, # rtsp downloads, requiring mplayer or mpv
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._generic_id(url)
+ title = self._generic_title(url)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': url,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/common.py youtube-dl-2019.05.20/youtube_dl/extractor/common.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/common.py 2016-02-22 10:57:19.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/common.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,3 +1,4 @@
+# coding: utf-8
from __future__ import unicode_literals
import base64
@@ -6,6 +7,7 @@
import json
import netrc
import os
+import random
import re
import socket
import sys
@@ -15,38 +17,68 @@
from ..compat import (
compat_cookiejar,
compat_cookies,
+ compat_etree_Element,
+ compat_etree_fromstring,
compat_getpass,
+ compat_integer_types,
compat_http_client,
+ compat_os_name,
+ compat_str,
compat_urllib_error,
- compat_urllib_parse,
+ compat_urllib_parse_unquote,
+ compat_urllib_parse_urlencode,
+ compat_urllib_request,
compat_urlparse,
- compat_str,
- compat_etree_fromstring,
+ compat_xml_parse_error,
+)
+from ..downloader.f4m import (
+ get_base_url,
+ remove_encrypted_media,
)
from ..utils import (
NO_DEFAULT,
age_restricted,
+ base_url,
bug_reports_message,
clean_html,
compiled_regex_type,
determine_ext,
+ determine_protocol,
+ dict_get,
error_to_compat_str,
ExtractorError,
+ extract_attributes,
fix_xml_ampersands,
float_or_none,
+ GeoRestrictedError,
+ GeoUtils,
int_or_none,
+ js_to_json,
+ JSON_LD_RE,
+ mimetype2ext,
+ orderedSet,
+ parse_bitrate,
+ parse_codecs,
+ parse_duration,
parse_iso8601,
+ parse_m3u8_attributes,
+ parse_resolution,
RegexNotFoundError,
- sanitize_filename,
sanitized_Request,
+ sanitize_filename,
+ str_or_none,
+ strip_or_none,
unescapeHTML,
unified_strdate,
+ unified_timestamp,
+ update_Request,
+ update_url_query,
+ urljoin,
url_basename,
+ url_or_none,
+ xpath_element,
xpath_text,
xpath_with_ns,
- determine_protocol,
- parse_duration,
- mimetype2ext,
)
@@ -76,7 +108,26 @@
from worst to best quality.
Potential fields:
- * url Mandatory. The URL of the video file
+ * url The mandatory URL representing the media:
+ for plain file media - HTTP URL of this file,
+ for RTMP - RTMP URL,
+ for HLS - URL of the M3U8 media playlist,
+ for HDS - URL of the F4M manifest,
+ for DASH
+ - HTTP URL to plain file media (in case of
+ unfragmented media)
+ - URL of the MPD manifest or base URL
+ representing the media if MPD manifest
+ is parsed from a string (in case of
+ fragmented media)
+ for MSS - URL of the ISM manifest.
+ * manifest_url
+ The URL of the manifest file in case of
+ fragmented media:
+ for HLS - URL of the M3U8 master playlist,
+ for HDS - URL of the F4M manifest,
+ for DASH - URL of the MPD manifest,
+ for MSS - URL of the ISM manifest.
* ext Will be calculated from URL if missing
* format A human-readable description of the format
("mp4 container with h264/opus").
@@ -104,7 +155,22 @@
* protocol The protocol that will be used for the actual
download, lower-case.
"http", "https", "rtsp", "rtmp", "rtmpe",
- "m3u8", or "m3u8_native".
+ "m3u8", "m3u8_native" or "http_dash_segments".
+ * fragment_base_url
+ Base URL for fragments. Each fragment's path
+ value (if present) will be relative to
+ this URL.
+ * fragments A list of fragments of a fragmented media.
+ Each fragment entry must contain either an url
+ or a path. If an url is present it should be
+ considered by a client. Otherwise both path and
+ fragment_base_url must be present. Here is
+ the list of all potential fields:
+ * "url" - fragment's URL
+ * "path" - fragment's path relative to
+ fragment_base_url
+ * "duration" (optional, int or float)
+ * "filesize" (optional, int)
* preference Order number of this format. If this field is
present and not None, the formats get sorted
by this field, regardless of all other values.
@@ -133,6 +199,8 @@
width : height ratio as float.
* no_resume The server does not support resuming the
(HTTP or RTMP) download. Boolean.
+ * downloader_options A dictionary of downloader options as
+ described in FileDownloader
url: Final video URL.
ext: Video filename extension.
@@ -154,20 +222,29 @@
* "height" (optional, int)
* "resolution" (optional, string "{width}x{height"},
deprecated)
+ * "filesize" (optional, int)
thumbnail: Full URL to a video thumbnail image.
description: Full video description.
uploader: Full name of the video uploader.
- creator: The main artist who created the video.
+ license: License name the video is licensed under.
+ creator: The creator of the video.
release_date: The date (YYYYMMDD) when the video was released.
timestamp: UNIX timestamp of the moment the video became available.
upload_date: Video upload date (YYYYMMDD).
If not explicitly set, calculated from timestamp.
uploader_id: Nickname or id of the video uploader.
+ uploader_url: Full URL to a personal webpage of the video uploader.
+ channel: Full name of the channel the video is uploaded on.
+ Note that channel fields may or may not repeat uploader
+ fields. This depends on a particular extractor.
+ channel_id: Id of the channel.
+ channel_url: Full URL to a channel webpage.
location: Physical location where the video was filmed.
subtitles: The available subtitles as a dictionary in the format
- {language: subformats}. "subformats" is a list sorted from
- lower to higher preference, each element is a dictionary
- with the "ext" entry and one of:
+ {tag: subformats}. "tag" is usually a language code, and
+ "subformats" is a list sorted from lower to higher
+ preference, each element is a dictionary with the "ext"
+ entry and one of:
* "data": The subtitles file contents
* "url": A URL pointing to the subtitles file
"ext" will be calculated from URL if missing
@@ -204,6 +281,10 @@
specified in the URL.
end_time: Time in seconds where the reproduction should end, as
specified in the URL.
+ chapters: A list of dictionaries, with the following entries:
+ * "start_time" - The start time of the chapter in seconds
+ * "end_time" - The end time of the chapter in seconds
+ * "title" (optional, string)
The following fields should only be used when the video belongs to some logical
chapter or section:
@@ -213,7 +294,7 @@
chapter_id: Id of the chapter the video belongs to, as a unicode string.
The following fields should only be used when the video is an episode of some
- series or programme:
+ series, programme or podcast:
series: Title of the series or programme the video episode belongs to.
season: Title of the season the video episode belongs to.
@@ -225,6 +306,24 @@
episode_number: Number of the video episode within a season, as an integer.
episode_id: Id of the video episode, as a unicode string.
+ The following fields should only be used when the media is a track or a part of
+ a music album:
+
+ track: Title of the track.
+ track_number: Number of the track within an album or a disc, as an integer.
+ track_id: Id of the track (useful in case of custom indexing, e.g. 6.iii),
+ as a unicode string.
+ artist: Artist(s) of the track.
+ genre: Genre(s) of the track.
+ album: Title of the album the track belongs to.
+ album_type: Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
+ album_artist: List of all artists appeared on the album (e.g.
+ "Ash Borer / Fell Voices" or "Various Artists", useful for splits
+ and compilations).
+ disc_number: Number of the disc or other physical medium the track belongs to,
+ as an integer.
+ release_year: Year (YYYY) when the album was released.
+
Unless mentioned otherwise, the fields should be Unicode strings.
Unless mentioned otherwise, None is equivalent to absence of information.
@@ -234,8 +333,9 @@
There must be a key "entries", which is a list, an iterable, or a PagedList
object, each element of which is a valid dictionary by this specification.
- Additionally, playlists can have "title", "description" and "id" attributes
- with the same semantics as videos (see above).
+ Additionally, playlists can have "id", "title", "description", "uploader",
+ "uploader_id", "uploader_url" attributes with the same semantics as videos
+ (see above).
_type "multi_video" indicates that there are multiple videos that
@@ -266,17 +366,37 @@
_real_extract() methods and define a _VALID_URL regexp.
Probably, they should also be added to the list of extractors.
+ _GEO_BYPASS attribute may be set to False in order to disable
+ geo restriction bypass mechanisms for a particular extractor.
+ Though it won't disable explicit geo restriction bypass based on
+ country code provided with geo_bypass_country.
+
+ _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
+ countries for this extractor. One of these countries will be used by
+ geo restriction bypass mechanism right away in order to bypass
+ geo restriction, of course, if the mechanism is not disabled.
+
+ _GEO_IP_BLOCKS attribute may contain a list of presumably geo unrestricted
+ IP blocks in CIDR notation for this extractor. One of these IP blocks
+ will be used by geo restriction bypass mechanism similarly
+ to _GEO_COUNTRIES.
+
Finally, the _WORKING attribute should be set to False for broken IEs
in order to warn the users and skip the tests.
"""
_ready = False
_downloader = None
+ _x_forwarded_for_ip = None
+ _GEO_BYPASS = True
+ _GEO_COUNTRIES = None
+ _GEO_IP_BLOCKS = None
_WORKING = True
def __init__(self, downloader=None):
"""Constructor. Receives an optional downloader."""
self._ready = False
+ self._x_forwarded_for_ip = None
self.set_downloader(downloader)
@classmethod
@@ -296,7 +416,7 @@
cls._VALID_URL_RE = re.compile(cls._VALID_URL)
m = cls._VALID_URL_RE.match(url)
assert m
- return m.group('id')
+ return compat_str(m.group('id'))
@classmethod
def working(cls):
@@ -305,15 +425,116 @@
def initialize(self):
"""Initializes an instance (authentication, etc)."""
+ self._initialize_geo_bypass({
+ 'countries': self._GEO_COUNTRIES,
+ 'ip_blocks': self._GEO_IP_BLOCKS,
+ })
if not self._ready:
self._real_initialize()
self._ready = True
+ def _initialize_geo_bypass(self, geo_bypass_context):
+ """
+ Initialize geo restriction bypass mechanism.
+
+ This method is used to initialize geo bypass mechanism based on faking
+ X-Forwarded-For HTTP header. A random country from provided country list
+ is selected and a random IP belonging to this country is generated. This
+ IP will be passed as X-Forwarded-For HTTP header in all subsequent
+ HTTP requests.
+
+ This method will be used for initial geo bypass mechanism initialization
+ during the instance initialization with _GEO_COUNTRIES and
+ _GEO_IP_BLOCKS.
+
+ You may also manually call it from extractor's code if geo bypass
+ information is not available beforehand (e.g. obtained during
+ extraction) or due to some other reason. In this case you should pass
+ this information in geo bypass context passed as first argument. It may
+ contain following fields:
+
+ countries: List of geo unrestricted countries (similar
+ to _GEO_COUNTRIES)
+ ip_blocks: List of geo unrestricted IP blocks in CIDR notation
+ (similar to _GEO_IP_BLOCKS)
+
+ """
+ if not self._x_forwarded_for_ip:
+
+ # Geo bypass mechanism is explicitly disabled by user
+ if not self._downloader.params.get('geo_bypass', True):
+ return
+
+ if not geo_bypass_context:
+ geo_bypass_context = {}
+
+ # Backward compatibility: previously _initialize_geo_bypass
+ # expected a list of countries, some 3rd party code may still use
+ # it this way
+ if isinstance(geo_bypass_context, (list, tuple)):
+ geo_bypass_context = {
+ 'countries': geo_bypass_context,
+ }
+
+ # The whole point of geo bypass mechanism is to fake IP
+ # as X-Forwarded-For HTTP header based on some IP block or
+ # country code.
+
+ # Path 1: bypassing based on IP block in CIDR notation
+
+ # Explicit IP block specified by user, use it right away
+ # regardless of whether extractor is geo bypassable or not
+ ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
+
+ # Otherwise use random IP block from geo bypass context but only
+ # if extractor is known as geo bypassable
+ if not ip_block:
+ ip_blocks = geo_bypass_context.get('ip_blocks')
+ if self._GEO_BYPASS and ip_blocks:
+ ip_block = random.choice(ip_blocks)
+
+ if ip_block:
+ self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
+ if self._downloader.params.get('verbose', False):
+ self._downloader.to_screen(
+ '[debug] Using fake IP %s as X-Forwarded-For.'
+ % self._x_forwarded_for_ip)
+ return
+
+ # Path 2: bypassing based on country code
+
+ # Explicit country code specified by user, use it right away
+ # regardless of whether extractor is geo bypassable or not
+ country = self._downloader.params.get('geo_bypass_country', None)
+
+ # Otherwise use random country code from geo bypass context but
+ # only if extractor is known as geo bypassable
+ if not country:
+ countries = geo_bypass_context.get('countries')
+ if self._GEO_BYPASS and countries:
+ country = random.choice(countries)
+
+ if country:
+ self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
+ if self._downloader.params.get('verbose', False):
+ self._downloader.to_screen(
+ '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
+ % (self._x_forwarded_for_ip, country.upper()))
+
def extract(self, url):
"""Extracts URL information and returns it in list of dicts."""
try:
- self.initialize()
- return self._real_extract(url)
+ for _ in range(2):
+ try:
+ self.initialize()
+ ie_result = self._real_extract(url)
+ if self._x_forwarded_for_ip:
+ ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
+ return ie_result
+ except GeoRestrictedError as e:
+ if self.__maybe_fake_ip_and_retry(e.countries):
+ continue
+ raise
except ExtractorError:
raise
except compat_http_client.IncompleteRead as e:
@@ -321,6 +542,21 @@
except (KeyError, StopIteration) as e:
raise ExtractorError('An extractor error has occurred.', cause=e)
+ def __maybe_fake_ip_and_retry(self, countries):
+ if (not self._downloader.params.get('geo_bypass_country', None)
+ and self._GEO_BYPASS
+ and self._downloader.params.get('geo_bypass', True)
+ and not self._x_forwarded_for_ip
+ and countries):
+ country_code = random.choice(countries)
+ self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
+ if self._x_forwarded_for_ip:
+ self.report_warning(
+ 'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
+ % (self._x_forwarded_for_ip, country_code.upper()))
+ return True
+ return False
+
def set_downloader(self, downloader):
"""Sets the downloader for this IE."""
self._downloader = downloader
@@ -342,8 +578,26 @@
def IE_NAME(self):
return compat_str(type(self).__name__[:-2])
- def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
- """ Returns the response handle """
+ @staticmethod
+ def __can_accept_status_code(err, expected_status):
+ assert isinstance(err, compat_urllib_error.HTTPError)
+ if expected_status is None:
+ return False
+ if isinstance(expected_status, compat_integer_types):
+ return err.code == expected_status
+ elif isinstance(expected_status, (list, tuple)):
+ return err.code in expected_status
+ elif callable(expected_status):
+ return expected_status(err.code) is True
+ else:
+ assert False
+
+ def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
+ """
+ Return the response handle.
+
+ See _download_webpage docstring for arguments specification.
+ """
if note is None:
self.report_download_webpage(video_id)
elif note is not False:
@@ -351,9 +605,36 @@
self.to_screen('%s' % (note,))
else:
self.to_screen('%s: %s' % (video_id, note))
+
+ # Some sites check X-Forwarded-For HTTP header in order to figure out
+ # the origin of the client behind proxy. This allows bypassing geo
+ # restriction by faking this header's value to IP that belongs to some
+ # geo unrestricted country. We will do so once we encounter any
+ # geo restriction error.
+ if self._x_forwarded_for_ip:
+ if 'X-Forwarded-For' not in headers:
+ headers['X-Forwarded-For'] = self._x_forwarded_for_ip
+
+ if isinstance(url_or_request, compat_urllib_request.Request):
+ url_or_request = update_Request(
+ url_or_request, data=data, headers=headers, query=query)
+ else:
+ if query:
+ url_or_request = update_url_query(url_or_request, query)
+ if data is not None or headers:
+ url_or_request = sanitized_Request(url_or_request, data, headers)
try:
return self._downloader.urlopen(url_or_request)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ if isinstance(err, compat_urllib_error.HTTPError):
+ if self.__can_accept_status_code(err, expected_status):
+ # Retain reference to error to prevent file object from
+ # being closed before it can be read. Works around the
+ # effects of <https://bugs.python.org/issue15002>
+ # introduced in Python 3.4.1.
+ err.fp._error = err
+ return err.fp
+
if errnote is False:
return False
if errnote is None:
@@ -366,13 +647,17 @@
self._downloader.report_warning(errmsg)
return False
- def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
- """ Returns a tuple (page content as string, URL handle) """
+ def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
+ """
+ Return a tuple (page content as string, URL handle).
+
+ See _download_webpage docstring for arguments specification.
+ """
# Strip hashes from the URL (#1038)
if isinstance(url_or_request, (compat_str, str)):
url_or_request = url_or_request.partition('#')[0]
- urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
+ urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
if urlh is False:
assert not fatal
return False
@@ -396,6 +681,34 @@
return encoding
+ def __check_blocked(self, content):
+ first_block = content[:512]
+ if ('<title>Access to this site is blocked' in content
+ and 'Websense' in first_block):
+ msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
+ blocked_iframe = self._html_search_regex(
+ r'',
- start_page, 'xml root', default=None)
- if xml_root is None:
- # Probably need to authenticate
- login_res = self._login(webpage_url, display_id)
- if login_res is None:
- self.report_warning('Could not login.')
- else:
- start_page = login_res
- # Grab the url from the authenticated page
- xml_root = self._html_search_regex(
- r'',
- start_page, 'xml filename', default=None)
- if xml_name is None:
- # Fallback to the older format
- xml_name = self._html_search_regex(r''
+
+ xml_root = self._html_search_regex(
+ PLAYER_REGEX, start_page, 'xml root', default=None)
+ if xml_root is None:
+ # Probably need to authenticate
+ login_res = self._login(webpage_url, display_id)
+ if login_res is None:
+ self.report_warning('Could not login.')
+ else:
+ start_page = login_res
+ # Grab the url from the authenticated page
+ xml_root = self._html_search_regex(
+ PLAYER_REGEX, start_page, 'xml root')
+
+ xml_name = self._html_search_regex(
+ r'', webpage)
- if mobj:
- embedded_url = mobj.group(1)
- return self.url_result(embedded_url)
-
- video_title = self._html_search_regex(
- r']*>([^<]+)', webpage, 'title')
- flashvars = self._parse_json(self._search_regex(
- r'var\s+flashvars\s*=\s*([^;]+);', webpage, 'flashvars'), video_id)
+ webpage = self._download_webpage(
+ url, display_id, headers={'Cookie': 'age_verified=1'})
formats = []
- for height in (180, 240, 480):
- if flashvars.get('quality_%dp' % height):
- video_url = flashvars['quality_%dp' % height]
- a_format = {
- 'url': video_url,
- 'height': height,
- 'format_id': '%dp' % height,
- }
- filename_parts = url_basename(video_url).split('_')
- if len(filename_parts) >= 2 and re.match(r'\d+[Kk]', filename_parts[1]):
- a_format['tbr'] = int(filename_parts[1][:-1])
- formats.append(a_format)
+ format_urls = set()
- age_limit = self._rta_search(webpage)
+ title = None
+ thumbnail = None
+ duration = None
+ encrypted = False
+
+ def extract_format(format_url, height=None):
+ format_url = url_or_none(format_url)
+ if not format_url or not format_url.startswith(('http', '//')):
+ return
+ if format_url in format_urls:
+ return
+ format_urls.add(format_url)
+ tbr = int_or_none(self._search_regex(
+ r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None))
+ if not height:
+ height = int_or_none(self._search_regex(
+ r'[/_](\d+)[pP][/_]', format_url, 'height', default=None))
+ if encrypted:
+ format_url = aes_decrypt_text(
+ video_url, title, 32).decode('utf-8')
+ formats.append({
+ 'url': format_url,
+ 'format_id': '%dp' % height if height else None,
+ 'height': height,
+ 'tbr': tbr,
+ })
+
+ flashvars = self._parse_json(
+ self._search_regex(
+ r'flashvars\s*=\s*({.+?});', webpage,
+ 'flashvars', default='{}'),
+ display_id, fatal=False)
+
+ if flashvars:
+ title = flashvars.get('video_title')
+ thumbnail = flashvars.get('image_url')
+ duration = int_or_none(flashvars.get('video_duration'))
+ encrypted = flashvars.get('encrypted') is True
+ for key, value in flashvars.items():
+ mobj = re.search(r'quality_(\d+)[pP]', key)
+ if mobj:
+ extract_format(value, int(mobj.group(1)))
+ video_url = flashvars.get('video_url')
+ if video_url and determine_ext(video_url, None):
+ extract_format(video_url)
+
+ video_url = self._html_search_regex(
+ r'flashvars\.video_url\s*=\s*(["\'])(?Phttp.+?)\1',
+ webpage, 'video url', default=None, group='url')
+ if video_url:
+ extract_format(compat_urllib_parse_unquote(video_url))
+
+ if not formats:
+ if 'title="This video is no longer available"' in webpage:
+ raise ExtractorError(
+ 'Video %s is no longer available' % video_id, expected=True)
+
+ try:
+ self._sort_formats(formats)
+ except ExtractorError:
+ if fatal:
+ raise
+
+ if not title:
+ title = self._html_search_regex(
+ r']*>([^<]+)', webpage, 'title')
- return {
+ return webpage, {
'id': video_id,
- 'title': video_title,
+ 'display_id': display_id,
+ 'title': strip_or_none(title),
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'age_limit': 18,
'formats': formats,
- 'age_limit': age_limit,
- 'thumbnail': flashvars.get('image_url')
}
+
+ def _real_extract(self, url):
+ webpage, info = self._extract_info(url, fatal=False)
+ if not info['formats']:
+ return self.url_result(url, 'Generic')
+ info['view_count'] = str_to_int(self._search_regex(
+ r'([\d,.]+) Views?', webpage, 'view count', fatal=False))
+ return info
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/ketnet.py youtube-dl-2019.05.20/youtube_dl/extractor/ketnet.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/ketnet.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/ketnet.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+from .canvas import CanvasIE
+from .common import InfoExtractor
+
+
+class KetnetIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?:[^/]+/)*(?P[^/?#&]+)'
+ _TESTS = [{
+ 'url': 'https://www.ketnet.be/kijken/zomerse-filmpjes',
+ 'md5': '6bdeb65998930251bbd1c510750edba9',
+ 'info_dict': {
+ 'id': 'zomerse-filmpjes',
+ 'ext': 'mp4',
+ 'title': 'Gluur mee op de filmset en op Pennenzakkenrock',
+ 'description': 'Gluur mee met Ghost Rockers op de filmset',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ }
+ }, {
+ # mzid in playerConfig instead of sources
+ 'url': 'https://www.ketnet.be/kijken/nachtwacht/de-greystook',
+ 'md5': '90139b746a0a9bd7bb631283f6e2a64e',
+ 'info_dict': {
+ 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+ 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+ 'ext': 'flv',
+ 'title': 'Nachtwacht: De Greystook',
+ 'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'duration': 1468.03,
+ },
+ 'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
+ }, {
+ 'url': 'https://www.ketnet.be/kijken/karrewiet/uitzending-8-september-2016',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.ketnet.be/achter-de-schermen/sien-repeteert-voor-stars-for-life',
+ 'only_matching': True,
+ }, {
+ # mzsource, geo restricted to Belgium
+ 'url': 'https://www.ketnet.be/kijken/nachtwacht/de-bermadoe',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+
+ config = self._parse_json(
+ self._search_regex(
+ r'(?s)playerConfig\s*=\s*({.+?})\s*;', webpage,
+ 'player config'),
+ video_id)
+
+ mzid = config.get('mzid')
+ if mzid:
+ return self.url_result(
+ 'https://mediazone.vrt.be/api/v1/ketnet/assets/%s' % mzid,
+ CanvasIE.ie_key(), video_id=mzid)
+
+ title = config['title']
+
+ formats = []
+ for source_key in ('', 'mz'):
+ source = config.get('%ssource' % source_key)
+ if not isinstance(source, dict):
+ continue
+ for format_id, format_url in source.items():
+ if format_id == 'hls':
+ formats.extend(self._extract_m3u8_formats(
+ format_url, video_id, 'mp4',
+ entry_protocol='m3u8_native', m3u8_id=format_id,
+ fatal=False))
+ elif format_id == 'hds':
+ formats.extend(self._extract_f4m_formats(
+ format_url, video_id, f4m_id=format_id, fatal=False))
+ else:
+ formats.append({
+ 'url': format_url,
+ 'format_id': format_id,
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': config.get('description'),
+ 'thumbnail': config.get('image'),
+ 'series': config.get('program'),
+ 'episode': config.get('episode'),
+ 'formats': formats,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/khanacademy.py youtube-dl-2019.05.20/youtube_dl/extractor/khanacademy.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/khanacademy.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/khanacademy.py 2019-06-07 18:49:07.000000000 +0000
@@ -14,10 +14,10 @@
_TESTS = [{
'url': 'http://www.khanacademy.org/video/one-time-pad',
- 'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+ 'md5': '7b391cce85e758fb94f763ddc1bbb979',
'info_dict': {
'id': 'one-time-pad',
- 'ext': 'mp4',
+ 'ext': 'webm',
'title': 'The one-time pad',
'description': 'The perfect cipher',
'duration': 176,
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/kickstarter.py youtube-dl-2019.05.20/youtube_dl/extractor/kickstarter.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/kickstarter.py 2016-02-01 10:41:30.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/kickstarter.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
from __future__ import unicode_literals
from .common import InfoExtractor
@@ -6,7 +6,7 @@
class KickStarterIE(InfoExtractor):
- _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P[^/]*)/.*'
+ _VALID_URL = r'https?://(?:www\.)?kickstarter\.com/projects/(?P[^/]*)/.*'
_TESTS = [{
'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant/description',
'md5': 'c81addca81327ffa66c642b5d8b08cab',
@@ -37,7 +37,6 @@
'ext': 'mp4',
'title': 'Power Drive 2000',
},
- 'expected_warnings': ['OpenGraph description'],
}]
def _real_extract(self, url):
@@ -67,6 +66,6 @@
'id': video_id,
'url': video_url,
'title': title,
- 'description': self._og_search_description(webpage),
+ 'description': self._og_search_description(webpage, default=None),
'thumbnail': thumbnail,
}
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/kinopoisk.py youtube-dl-2019.05.20/youtube_dl/extractor/kinopoisk.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/kinopoisk.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/kinopoisk.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+ dict_get,
+ int_or_none,
+)
+
+
+class KinoPoiskIE(InfoExtractor):
+ _GEO_COUNTRIES = ['RU']
+ _VALID_URL = r'https?://(?:www\.)?kinopoisk\.ru/film/(?P\d+)'
+ _TESTS = [{
+ 'url': 'https://www.kinopoisk.ru/film/81041/watch/',
+ 'md5': '4f71c80baea10dfa54a837a46111d326',
+ 'info_dict': {
+ 'id': '81041',
+ 'ext': 'mp4',
+ 'title': 'Алеша попович и тугарин змей',
+ 'description': 'md5:43787e673d68b805d0aa1df5a5aea701',
+ 'thumbnail': r're:^https?://.*',
+ 'duration': 4533,
+ 'age_limit': 12,
+ },
+ 'params': {
+ 'format': 'bestvideo',
+ },
+ }, {
+ 'url': 'https://www.kinopoisk.ru/film/81041',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(
+ 'https://ott-widget.kinopoisk.ru/v1/kp/', video_id,
+ query={'kpId': video_id})
+
+ data = self._parse_json(
+ self._search_regex(
+ r'(?s)]+\btype=["\']application/json[^>]+>(.+?)<',
+ webpage, 'data'),
+ video_id)['models']
+
+ film = data['filmStatus']
+ title = film.get('title') or film['originalTitle']
+
+ formats = self._extract_m3u8_formats(
+ data['playlistEntity']['uri'], video_id, 'mp4',
+ entry_protocol='m3u8_native', m3u8_id='hls')
+ self._sort_formats(formats)
+
+ description = dict_get(
+ film, ('descriptscription', 'description',
+ 'shortDescriptscription', 'shortDescription'))
+ thumbnail = film.get('coverUrl') or film.get('posterUrl')
+ duration = int_or_none(film.get('duration'))
+ age_limit = int_or_none(film.get('restrictionAge'))
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'age_limit': age_limit,
+ 'formats': formats,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/konserthusetplay.py youtube-dl-2019.05.20/youtube_dl/extractor/konserthusetplay.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/konserthusetplay.py 2016-02-09 11:57:41.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/konserthusetplay.py 2019-06-07 18:49:07.000000000 +0000
@@ -3,28 +3,30 @@
from .common import InfoExtractor
from ..utils import (
+ determine_ext,
float_or_none,
int_or_none,
+ url_or_none,
)
class KonserthusetPlayIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?.*\bm=(?P[^&]+)'
- _TEST = {
+ _VALID_URL = r'https?://(?:www\.)?(?:konserthusetplay|rspoplay)\.se/\?.*\bm=(?P[^&]+)'
+ _TESTS = [{
'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
+ 'md5': 'e3fd47bf44e864bd23c08e487abe1967',
'info_dict': {
'id': 'CKDDnlCY-dhWAAqiMERd-A',
- 'ext': 'flv',
+ 'ext': 'mp4',
'title': 'Orkesterns instrument: Valthornen',
'description': 'md5:f10e1f0030202020396a4d712d2fa827',
'thumbnail': 're:^https?://.*$',
- 'duration': 398.8,
+ 'duration': 398.76,
},
- 'params': {
- # rtmp download
- 'skip_download': True,
- },
- }
+ }, {
+ 'url': 'http://rspoplay.se/?m=elWuEH34SMKvaO4wO_cHBw',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -42,12 +44,18 @@
player_config = media['playerconfig']
playlist = player_config['playlist']
- source = next(f for f in playlist if f.get('bitrates'))
+ source = next(f for f in playlist if f.get('bitrates') or f.get('provider'))
FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
formats = []
+ m3u8_url = source.get('url')
+ if m3u8_url and determine_ext(m3u8_url) == 'm3u8':
+ formats.extend(self._extract_m3u8_formats(
+ m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+ m3u8_id='hls', fatal=False))
+
fallback_url = source.get('fallbackUrl')
fallback_format_id = None
if fallback_url:
@@ -97,6 +105,14 @@
thumbnail = media.get('image')
duration = float_or_none(media.get('duration'), 1000)
+ subtitles = {}
+ captions = source.get('captionsAvailableLanguages')
+ if isinstance(captions, dict):
+ for lang, subtitle_url in captions.items():
+ subtitle_url = url_or_none(subtitle_url)
+ if lang != 'none' and subtitle_url:
+ subtitles.setdefault(lang, []).append({'url': subtitle_url})
+
return {
'id': video_id,
'title': title,
@@ -104,4 +120,5 @@
'thumbnail': thumbnail,
'duration': duration,
'formats': formats,
+ 'subtitles': subtitles,
}
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/kontrtube.py youtube-dl-2019.05.20/youtube_dl/extractor/kontrtube.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/kontrtube.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/kontrtube.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
from __future__ import unicode_literals
import re
@@ -13,7 +13,7 @@
class KontrTubeIE(InfoExtractor):
IE_NAME = 'kontrtube'
IE_DESC = 'KontrTube.ru - Труба зовёт'
- _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P\d+)/(?P[^/]+)/'
+ _VALID_URL = r'https?://(?:www\.)?kontrtube\.ru/videos/(?P\d+)/(?P[^/]+)/'
_TEST = {
'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/krasview.py youtube-dl-2019.05.20/youtube_dl/extractor/krasview.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/krasview.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/krasview.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
from __future__ import unicode_literals
import json
@@ -23,7 +23,7 @@
'title': 'Снег, лёд, заносы',
'description': 'Снято в городе Нягань, в Ханты-Мансийском автономном округе.',
'duration': 27,
- 'thumbnail': 're:^https?://.*\.jpg',
+ 'thumbnail': r're:^https?://.*\.jpg',
},
'params': {
'skip_download': 'Not accessible from Travis CI server',
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/ku6.py youtube-dl-2019.05.20/youtube_dl/extractor/ku6.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/ku6.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/ku6.py 2019-06-07 18:49:07.000000000 +0000
@@ -4,7 +4,7 @@
class Ku6IE(InfoExtractor):
- _VALID_URL = r'http://v\.ku6\.com/show/(?P[a-zA-Z0-9\-\_]+)(?:\.)*html'
+ _VALID_URL = r'https?://v\.ku6\.com/show/(?P[a-zA-Z0-9\-\_]+)(?:\.)*html'
_TEST = {
'url': 'http://v.ku6.com/show/JG-8yS14xzBr4bCn1pu0xw...html',
'md5': '01203549b9efbb45f4b87d55bdea1ed1',
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/kusi.py youtube-dl-2019.05.20/youtube_dl/extractor/kusi.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/kusi.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/kusi.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
+from ..utils import (
+ int_or_none,
+ float_or_none,
+ timeconvert,
+ update_url_query,
+ xpath_text,
+)
+
+
+class KUSIIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?Pstory/.+|video\?clipId=(?P\d+))'
+ _TESTS = [{
+ 'url': 'http://www.kusi.com/story/32849881/turko-files-refused-to-help-it-aint-right',
+ 'md5': '4e76ce8e53660ce9697d06c0ba6fc47d',
+ 'info_dict': {
+ 'id': '12689020',
+ 'ext': 'mp4',
+ 'title': "Turko Files: Refused to Help, It Ain't Right!",
+ 'duration': 223.586,
+ 'upload_date': '20160826',
+ 'timestamp': 1472233118,
+ 'thumbnail': r're:^https?://.*\.jpg$'
+ },
+ }, {
+ 'url': 'http://kusi.com/video?clipId=12203019',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ clip_id = mobj.group('clipId')
+ video_id = clip_id or mobj.group('path')
+
+ webpage = self._download_webpage(url, video_id)
+
+ if clip_id is None:
+ video_id = clip_id = self._html_search_regex(
+ r'"clipId"\s*,\s*"(\d+)"', webpage, 'clip id')
+
+ affiliate_id = self._search_regex(
+ r'affiliateId\s*:\s*\'([^\']+)\'', webpage, 'affiliate id')
+
+ # See __Packages/worldnow/model/GalleryModel.as of WNGallery.swf
+ xml_url = update_url_query('http://www.kusi.com/build.asp', {
+ 'buildtype': 'buildfeaturexmlrequest',
+ 'featureType': 'Clip',
+ 'featureid': clip_id,
+ 'affiliateno': affiliate_id,
+ 'clientgroupid': '1',
+ 'rnd': int(round(random.random() * 1000000)),
+ })
+
+ doc = self._download_xml(xml_url, video_id)
+
+ video_title = xpath_text(doc, 'HEADLINE', fatal=True)
+ duration = float_or_none(xpath_text(doc, 'DURATION'), scale=1000)
+ description = xpath_text(doc, 'ABSTRACT')
+ thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME')
+ createtion_time = timeconvert(xpath_text(doc, 'rfc822creationdate'))
+
+ quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
+ formats = []
+ for quality in quality_options:
+ formats.append({
+ 'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
+ 'height': int_or_none(quality.attrib.get('height')),
+ 'width': int_or_none(quality.attrib.get('width')),
+ 'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1000),
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'description': description,
+ 'duration': duration,
+ 'formats': formats,
+ 'thumbnail': thumbnail,
+ 'timestamp': createtion_time,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/kuwo.py youtube-dl-2019.05.20/youtube_dl/extractor/kuwo.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/kuwo.py 2016-02-04 12:39:20.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/kuwo.py 2019-06-07 18:49:07.000000000 +0000
@@ -2,13 +2,14 @@
from __future__ import unicode_literals
import re
-import itertools
from .common import InfoExtractor
+from ..compat import compat_urlparse
from ..utils import (
get_element_by_id,
clean_html,
ExtractorError,
+ InAdvancePagedList,
remove_start,
)
@@ -23,16 +24,24 @@
{'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
]
- def _get_formats(self, song_id):
+ def _get_formats(self, song_id, tolerate_ip_deny=False):
formats = []
for file_format in self._FORMATS:
+ query = {
+ 'format': file_format['ext'],
+ 'br': file_format.get('br', ''),
+ 'rid': 'MUSIC_%s' % song_id,
+ 'type': 'convert_url',
+ 'response': 'url'
+ }
+
song_url = self._download_webpage(
- 'http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url' %
- (file_format['ext'], file_format.get('br', ''), song_id),
+ 'http://antiserver.kuwo.cn/anti.s',
song_id, note='Download %s url info' % file_format['format'],
+ query=query, headers=self.geo_verification_headers(),
)
- if song_url == 'IPDeny':
+ if song_url == 'IPDeny' and not tolerate_ip_deny:
raise ExtractorError('This song is blocked in this region', expected=True)
if song_url.startswith('http://') or song_url.startswith('https://'):
@@ -43,14 +52,14 @@
'preference': file_format['preference'],
'abr': file_format.get('abr'),
})
- self._sort_formats(formats)
+
return formats
class KuwoIE(KuwoBaseIE):
IE_NAME = 'kuwo:song'
IE_DESC = '酷我音乐'
- _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P\d+?)/'
+ _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/yinyue/(?P\d+)'
_TESTS = [{
'url': 'http://www.kuwo.cn/yinyue/635632/',
'info_dict': {
@@ -68,35 +77,40 @@
'id': '6446136',
'ext': 'mp3',
'title': '心',
+ 'description': 'md5:5d0e947b242c35dc0eb1d2fce9fbf02c',
'creator': 'IU',
'upload_date': '20150518',
},
'params': {
- 'format': 'mp3-320'
+ 'format': 'mp3-320',
},
+ }, {
+ 'url': 'http://www.kuwo.cn/yinyue/3197154?catalog=yueku2016',
+ 'only_matching': True,
}]
def _real_extract(self, url):
song_id = self._match_id(url)
- webpage = self._download_webpage(
+ webpage, urlh = self._download_webpage_handle(
url, song_id, note='Download song detail info',
errnote='Unable to get song detail info')
- if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+ if song_id not in urlh.geturl() or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
raise ExtractorError('this song has been offline because of copyright issues', expected=True)
song_name = self._html_search_regex(
- r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?]+title="([^"]+)"', webpage, 'song name')
- singer_name = self._html_search_regex(
- r']+class="s_img">\s*]+title="([^>]+)"',
- webpage, 'singer name', fatal=False)
+ r']+id="lrcName">([^<]+)
', webpage, 'song name')
+ singer_name = remove_start(self._html_search_regex(
+ r']+href="http://www\.kuwo\.cn/artist/content\?name=([^"]+)">',
+ webpage, 'singer name', fatal=False), '歌手')
lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
if lrc_content == '暂无': # indicates no lyrics
lrc_content = None
formats = self._get_formats(song_id)
+ self._sort_formats(formats)
album_id = self._html_search_regex(
- r']+class="album"[^<]+]+href="http://www\.kuwo\.cn/album/(\d+)/"',
+ r']+href="http://www\.kuwo\.cn/album/(\d+)/"',
webpage, 'album id', fatal=False)
publish_time = None
@@ -125,13 +139,13 @@
class KuwoAlbumIE(InfoExtractor):
IE_NAME = 'kuwo:album'
IE_DESC = '酷我音乐 - 专辑'
- _VALID_URL = r'http://www\.kuwo\.cn/album/(?P\d+?)/'
+ _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/album/(?P\d+?)/'
_TEST = {
'url': 'http://www.kuwo.cn/album/502294/',
'info_dict': {
'id': '502294',
- 'title': 'M',
- 'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
+ 'title': 'Made\xa0Series\xa0《M》',
+ 'description': 'md5:d463f0d8a0ff3c3ea3d6ed7452a9483f',
},
'playlist_count': 2,
}
@@ -161,15 +175,13 @@
class KuwoChartIE(InfoExtractor):
IE_NAME = 'kuwo:chart'
IE_DESC = '酷我音乐 - 排行榜'
- _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P[^.]+).htm'
+ _VALID_URL = r'https?://yinyue\.kuwo\.cn/billboard_(?P[^.]+).htm'
_TEST = {
'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
'info_dict': {
'id': '香港中文龙虎榜',
- 'title': '香港中文龙虎榜',
- 'description': 're:\d{4}第\d{2}期',
},
- 'playlist_mincount': 10,
+ 'playlist_mincount': 7,
}
def _real_extract(self, url):
@@ -178,30 +190,24 @@
url, chart_id, note='Download chart info',
errnote='Unable to get chart info')
- chart_name = self._html_search_regex(
- r']+class="unDis">([^<]+)', webpage, 'chart name')
-
- chart_desc = self._html_search_regex(
- r']+class="tabDef">(\d{4}第\d{2}期)', webpage, 'chart desc')
-
entries = [
self.url_result(song_url, 'Kuwo') for song_url in re.findall(
- r']+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
+ r']+href="(http://www\.kuwo\.cn/yinyue/\d+)', webpage)
]
- return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+ return self.playlist_result(entries, chart_id)
class KuwoSingerIE(InfoExtractor):
IE_NAME = 'kuwo:singer'
IE_DESC = '酷我音乐 - 歌手'
- _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P[^/]+)'
+ _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mingxing/(?P[^/]+)'
_TESTS = [{
'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
'info_dict': {
'id': 'bruno+mars',
- 'title': 'Bruno Mars',
+ 'title': 'Bruno\xa0Mars',
},
- 'playlist_count': 10,
+ 'playlist_mincount': 329,
}, {
'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
'info_dict': {
@@ -209,9 +215,11 @@
'title': 'Ali',
},
'playlist_mincount': 95,
- 'skip': 'Regularly stalls travis build', # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
+ 'skip': 'Regularly stalls travis build', # See https://travis-ci.org/ytdl-org/youtube-dl/jobs/78878540
}]
+ PAGE_SIZE = 15
+
def _real_extract(self, url):
singer_id = self._match_id(url)
webpage = self._download_webpage(
@@ -219,25 +227,29 @@
errnote='Unable to get singer info')
singer_name = self._html_search_regex(
- r'\s*
([^<]+)([^<]+)
', webpage, 'singer name')
+
+ artist_id = self._html_search_regex(
+ r'data-artistid="(\d+)"', webpage, 'artist id')
+
+ page_count = int(self._html_search_regex(
+ r'data-page="(\d+)"', webpage, 'page count'))
- entries = []
- first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
- for page_num in itertools.count(1):
+ def page_func(page_num):
webpage = self._download_webpage(
- 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
- singer_id, note='Download song list page #%d' % page_num,
- errnote='Unable to get song list page #%d' % page_num)
-
- entries.extend([
- self.url_result(song_url, 'Kuwo') for song_url in re.findall(
- r'
]+class="m_name">]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
+ 'http://www.kuwo.cn/artist/contentMusicsAjax',
+ singer_id, note='Download song list page #%d' % (page_num + 1),
+ errnote='Unable to get song list page #%d' % (page_num + 1),
+ query={'artistId': artist_id, 'pn': page_num, 'rn': self.PAGE_SIZE})
+
+ return [
+ self.url_result(compat_urlparse.urljoin(url, song_url), 'Kuwo')
+ for song_url in re.findall(
+ r']+class="name">]+href="(/yinyue/\d+)',
webpage)
- ][:10 if first_page_only else None])
+ ]
- if first_page_only or not re.search(r']+href="[^"]+">下一页', webpage):
- break
+ entries = InAdvancePagedList(page_func, page_count, self.PAGE_SIZE)
return self.playlist_result(entries, singer_id, singer_name)
@@ -245,7 +257,7 @@
class KuwoCategoryIE(InfoExtractor):
IE_NAME = 'kuwo:category'
IE_DESC = '酷我音乐 - 分类'
- _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P\d+?).htm'
+ _VALID_URL = r'https?://yinyue\.kuwo\.cn/yy/cinfo_(?P\d+?).htm'
_TEST = {
'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
'info_dict': {
@@ -253,7 +265,7 @@
'title': '八十年代精选',
'description': '这些都是属于八十年代的回忆！',
},
- 'playlist_count': 30,
+ 'playlist_mincount': 24,
}
def _real_extract(self, url):
@@ -268,6 +280,8 @@
category_desc = remove_start(
get_element_by_id('intro', webpage).strip(),
'%s简介：' % category_name)
+ if category_desc == '暂无':
+ category_desc = None
jsonm = self._parse_json(self._html_search_regex(
r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
@@ -282,15 +296,21 @@
class KuwoMvIE(KuwoBaseIE):
IE_NAME = 'kuwo:mv'
IE_DESC = '酷我音乐 - MV'
- _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P\d+?)/'
+ _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mv/(?P\d+?)/'
_TEST = {
'url': 'http://www.kuwo.cn/mv/6480076/',
'info_dict': {
'id': '6480076',
- 'ext': 'mkv',
- 'title': '我们家MV',
+ 'ext': 'mp4',
+ 'title': 'My HouseMV',
'creator': '2PM',
},
+ # In this video, music URLs (anti.s) are blocked outside China and
+ # USA, while the MV URL (mvurl) is available globally, so force the MV
+ # URL for consistent results in different countries
+ 'params': {
+ 'format': 'mv',
+ },
}
_FORMATS = KuwoBaseIE._FORMATS + [
{'format': 'mkv', 'ext': 'mkv', 'preference': 250},
@@ -312,7 +332,17 @@
else:
raise ExtractorError('Unable to find song or singer names')
- formats = self._get_formats(song_id)
+ formats = self._get_formats(song_id, tolerate_ip_deny=True)
+
+ mv_url = self._download_webpage(
+ 'http://www.kuwo.cn/yy/st/mvurl?rid=MUSIC_%s' % song_id,
+ song_id, note='Download %s MV URL' % song_id)
+ formats.append({
+ 'url': mv_url,
+ 'format_id': 'mv',
+ })
+
+ self._sort_formats(formats)
return {
'id': song_id,
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/la7.py youtube-dl-2019.05.20/youtube_dl/extractor/la7.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/la7.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/la7.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,60 +1,67 @@
+# coding: utf-8
from __future__ import unicode_literals
from .common import InfoExtractor
from ..utils import (
- parse_duration,
+ js_to_json,
+ smuggle_url,
)
class LA7IE(InfoExtractor):
- IE_NAME = 'la7.tv'
- _VALID_URL = r'''(?x)
- https?://(?:www\.)?la7\.tv/
- (?:
- richplayer/\?assetid=|
- \?contentId=
- )
- (?P[0-9]+)'''
-
- _TEST = {
- 'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
- 'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
+ IE_NAME = 'la7.it'
+ _VALID_URL = r'''(?x)(https?://)?(?:
+ (?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video)/|
+ tg\.la7\.it/repliche-tgla7\?id=
+ )(?P.+)'''
+
+ _TESTS = [{
+ # 'src' is a plain URL
+ 'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
+ 'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
'info_dict': {
- 'id': '50355319',
+ 'id': 'inccool8-02-10-2015-163722',
'ext': 'mp4',
- 'title': 'IL DIVO',
- 'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti e Flavio Bucci',
- 'duration': 6254,
+ 'title': 'Inc.Cool8',
+ 'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
+ 'thumbnail': 're:^https?://.*',
+ 'uploader_id': 'kdla7pillole@iltrovatore.it',
+ 'timestamp': 1443814869,
+ 'upload_date': '20151002',
},
- 'skip': 'Blocked in the US',
- }
+ }, {
+ # 'src' is a dictionary
+ 'url': 'http://tg.la7.it/repliche-tgla7?id=189080',
+ 'md5': '6b0d8888d286e39870208dfeceaf456b',
+ 'info_dict': {
+ 'id': '189080',
+ 'ext': 'mp4',
+ 'title': 'TG LA7',
+ },
+ }, {
+ 'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
- xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
- doc = self._download_xml(xml_url, video_id)
- video_title = doc.find('title').text
- description = doc.find('description').text
- duration = parse_duration(doc.find('duration').text)
- thumbnail = doc.find('img').text
- view_count = int(doc.find('views').text)
-
- prefix = doc.find('.//fqdn').text.strip().replace('auto:', 'http:')
-
- formats = [{
- 'format': vnode.find('quality').text,
- 'tbr': int(vnode.find('quality').text),
- 'url': vnode.find('fms').text.strip().replace('mp4:', prefix),
- } for vnode in doc.findall('.//videos/video')]
- self._sort_formats(formats)
+ webpage = self._download_webpage(url, video_id)
+
+ player_data = self._parse_json(
+ self._search_regex(
+ [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
+ webpage, 'player data'),
+ video_id, transform_source=js_to_json)
return {
+ '_type': 'url_transparent',
+ 'url': smuggle_url('kaltura:103:%s' % player_data['vid'], {
+ 'service_url': 'http://kdam.iltrovatore.it',
+ }),
'id': video_id,
- 'title': video_title,
- 'description': description,
- 'thumbnail': thumbnail,
- 'duration': duration,
- 'formats': formats,
- 'view_count': view_count,
+ 'title': player_data['title'],
+ 'description': self._og_search_description(webpage, default=None),
+ 'thumbnail': player_data.get('poster'),
+ 'ie_key': 'Kaltura',
}
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/laola1tv.py youtube-dl-2019.05.20/youtube_dl/extractor/laola1tv.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/laola1tv.py 2016-02-14 22:26:17.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/laola1tv.py 2019-06-07 18:49:07.000000000 +0000
@@ -1,25 +1,179 @@
# coding: utf-8
from __future__ import unicode_literals
+import json
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urlparse,
-)
from ..utils import (
ExtractorError,
- sanitized_Request,
unified_strdate,
urlencode_postdata,
xpath_element,
xpath_text,
+ update_url_query,
+ js_to_json,
)
-class Laola1TvIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P[a-z]+)-(?P[a-z]+)/[^/]+/(?P[^/?#&]+)'
+class Laola1TvEmbedIE(InfoExtractor):
+ IE_NAME = 'laola1tv:embed'
+ _VALID_URL = r'https?://(?:www\.)?laola1\.tv/titanplayer\.php\?.*?\bvideoid=(?P\d+)'
+ _TESTS = [{
+ # flashvars.premium = "false";
+ 'url': 'https://www.laola1.tv/titanplayer.php?videoid=708065&type=V&lang=en&portal=int&customer=1024',
+ 'info_dict': {
+ 'id': '708065',
+ 'ext': 'mp4',
+ 'title': 'MA Long CHN - FAN Zhendong CHN',
+ 'uploader': 'ITTF - International Table Tennis Federation',
+ 'upload_date': '20161211',
+ },
+ }]
+
+ def _extract_token_url(self, stream_access_url, video_id, data):
+ return self._download_json(
+ self._proto_relative_url(stream_access_url, 'https:'), video_id,
+ headers={
+ 'Content-Type': 'application/json',
+ }, data=json.dumps(data).encode())['data']['stream-access'][0]
+
+ def _extract_formats(self, token_url, video_id):
+ token_doc = self._download_xml(
+ token_url, video_id, 'Downloading token',
+ headers=self.geo_verification_headers())
+
+ token_attrib = xpath_element(token_doc, './/token').attrib
+
+ if token_attrib['status'] != '0':
+ raise ExtractorError(
+ 'Token error: %s' % token_attrib['comment'], expected=True)
+
+ formats = self._extract_akamai_formats(
+ '%s?hdnea=%s' % (token_attrib['url'], token_attrib['auth']),
+ video_id)
+ self._sort_formats(formats)
+ return formats
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ flash_vars = self._search_regex(
+ r'(?s)flashvars\s*=\s*({.+?});', webpage, 'flash vars')
+
+ def get_flashvar(x, *args, **kwargs):
+ flash_var = self._search_regex(
+ r'%s\s*:\s*"([^"]+)"' % x,
+ flash_vars, x, default=None)
+ if not flash_var:
+ flash_var = self._search_regex([
+ r'flashvars\.%s\s*=\s*"([^"]+)"' % x,
+ r'%s\s*=\s*"([^"]+)"' % x],
+ webpage, x, *args, **kwargs)
+ return flash_var
+
+ hd_doc = self._download_xml(
+ 'http://www.laola1.tv/server/hd_video.php', video_id, query={
+ 'play': get_flashvar('streamid'),
+ 'partner': get_flashvar('partnerid'),
+ 'portal': get_flashvar('portalid'),
+ 'lang': get_flashvar('sprache'),
+ 'v5ident': '',
+ })
+
+ _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
+ title = _v('title', fatal=True)
+
+ token_url = None
+ premium = get_flashvar('premium', default=None)
+ if premium:
+ token_url = update_url_query(
+ _v('url', fatal=True), {
+ 'timestamp': get_flashvar('timestamp'),
+ 'auth': get_flashvar('auth'),
+ })
+ else:
+ data_abo = urlencode_postdata(
+ dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(','))))
+ stream_access_url = update_url_query(
+ 'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access', {
+ 'videoId': _v('id'),
+ 'target': self._search_regex(r'vs_target = (\d+);', webpage, 'vs target'),
+ 'label': _v('label'),
+ 'area': _v('area'),
+ })
+ token_url = self._extract_token_url(stream_access_url, video_id, data_abo)
+
+ formats = self._extract_formats(token_url, video_id)
+
+ categories_str = _v('meta_sports')
+ categories = categories_str.split(',') if categories_str else []
+ is_live = _v('islive') == 'true'
+
+ return {
+ 'id': video_id,
+ 'title': self._live_title(title) if is_live else title,
+ 'upload_date': unified_strdate(_v('time_date')),
+ 'uploader': _v('meta_organisation'),
+ 'categories': categories,
+ 'is_live': is_live,
+ 'formats': formats,
+ }
+
+
+class Laola1TvBaseIE(Laola1TvEmbedIE):
+ def _extract_video(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+
+ if 'Dieser Livestream ist bereits beendet.' in webpage:
+ raise ExtractorError('This live stream has already finished.', expected=True)
+
+ conf = self._parse_json(self._search_regex(
+ r'(?s)conf\s*=\s*({.+?});', webpage, 'conf'),
+ display_id,
+ transform_source=lambda s: js_to_json(re.sub(r'shareurl:.+,', '', s)))
+ video_id = conf['videoid']
+
+ config = self._download_json(conf['configUrl'], video_id, query={
+ 'videoid': video_id,
+ 'partnerid': conf['partnerid'],
+ 'language': conf.get('language', ''),
+ 'portal': conf.get('portalid', ''),
+ })
+ error = config.get('error')
+ if error:
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+ video_data = config['video']
+ title = video_data['title']
+ is_live = video_data.get('isLivestream') and video_data.get('isLive')
+ meta = video_data.get('metaInformation')
+ sports = meta.get('sports')
+ categories = sports.split(',') if sports else []
+
+ token_url = self._extract_token_url(
+ video_data['streamAccess'], video_id,
+ video_data['abo']['required'])
+
+ formats = self._extract_formats(token_url, video_id)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': self._live_title(title) if is_live else title,
+ 'description': video_data.get('description'),
+ 'thumbnail': video_data.get('image'),
+ 'categories': categories,
+ 'formats': formats,
+ 'is_live': is_live,
+ }
+
+
+class Laola1TvIE(Laola1TvBaseIE):
+ IE_NAME = 'laola1tv'
+ _VALID_URL = r'https?://(?:www\.)?laola1\.tv/[a-z]+-[a-z]+/[^/]+/(?P[^/?#&]+)'
+
_TESTS = [{
'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
'info_dict': {
@@ -33,7 +187,7 @@
},
'params': {
'skip_download': True,
- }
+ },
}, {
'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
'info_dict': {
@@ -47,79 +201,65 @@
},
'params': {
'skip_download': True,
- }
+ },
+ }, {
+ 'url': 'http://www.laola1.tv/de-de/livestream/2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+ 'info_dict': {
+ 'id': '487850',
+ 'display_id': '2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+ 'ext': 'flv',
+ 'title': 'Belogorie BELGOROD - TRENTINO Diatec',
+ 'upload_date': '20160322',
+ 'uploader': 'CEV - Europäischer Volleyball Verband',
+ 'is_live': True,
+ 'categories': ['Volleyball'],
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'This live stream has already finished.',
}]
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- display_id = mobj.group('slug')
- lang = mobj.group('lang')
- portal = mobj.group('portal')
-
- webpage = self._download_webpage(url, display_id)
-
- iframe_url = self._search_regex(
- r']*?id="videoplayer"[^>]*?src="([^"]+)"',
- webpage, 'iframe url')
-
- video_id = self._search_regex(
- r'videoid=(\d+)', iframe_url, 'video id')
-
- iframe = self._download_webpage(compat_urlparse.urljoin(
- url, iframe_url), display_id, 'Downloading iframe')
-
- partner_id = self._search_regex(
- r'partnerid\s*:\s*(["\'])(?P.+?)\1',
- iframe, 'partner id', group='partner_id')
-
- hd_doc = self._download_xml(
- 'http://www.laola1.tv/server/hd_video.php?%s'
- % compat_urllib_parse.urlencode({
- 'play': video_id,
- 'partner': partner_id,
- 'portal': portal,
- 'lang': lang,
- 'v5ident': '',
- }), display_id)
-
- _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
- title = _v('title', fatal=True)
+ return self._extract_video(url)
- req = sanitized_Request(
- 'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
- compat_urllib_parse.urlencode({
- 'videoId': video_id,
- 'target': '2',
- 'label': 'laola1tv',
- 'area': _v('area'),
- }),
- urlencode_postdata(
- dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
- token_url = self._download_json(req, display_id)['data']['stream-access'][0]
- token_doc = self._download_xml(token_url, display_id, 'Downloading token')
+class EHFTVIE(Laola1TvBaseIE):
+ IE_NAME = 'ehftv'
+ _VALID_URL = r'https?://(?:www\.)?ehftv\.com/[a-z]+(?:-[a-z]+)?/[^/]+/(?P[^/?#&]+)'
- token_attrib = xpath_element(token_doc, './/token').attrib
- token_auth = token_attrib['auth']
+ _TESTS = [{
+ 'url': 'https://www.ehftv.com/int/video/paris-saint-germain-handball-pge-vive-kielce/1166761',
+ 'info_dict': {
+ 'id': '1166761',
+ 'display_id': 'paris-saint-germain-handball-pge-vive-kielce',
+ 'ext': 'mp4',
+ 'title': 'Paris Saint-Germain Handball - PGE Vive Kielce',
+ 'is_live': False,
+ 'categories': ['Handball'],
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
- if token_auth in ('blocked', 'restricted', 'error'):
- raise ExtractorError(
- 'Token error: %s' % token_attrib['comment'], expected=True)
+ def _real_extract(self, url):
+ return self._extract_video(url)
- formats = self._extract_f4m_formats(
- '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth),
- video_id, f4m_id='hds')
- categories_str = _v('meta_sports')
- categories = categories_str.split(',') if categories_str else []
+class ITTFIE(InfoExtractor):
+ _VALID_URL = r'https?://tv\.ittf\.com/video/[^/]+/(?P\d+)'
+ _TEST = {
+ 'url': 'https://tv.ittf.com/video/peng-wang-wei-matsudaira-kenta/951802',
+ 'only_matching': True,
+ }
- return {
- 'id': video_id,
- 'display_id': display_id,
- 'title': title,
- 'upload_date': unified_strdate(_v('time_date')),
- 'uploader': _v('meta_organisation'),
- 'categories': categories,
- 'is_live': _v('islive') == 'true',
- 'formats': formats,
- }
+ def _real_extract(self, url):
+ return self.url_result(
+ update_url_query('https://www.laola1.tv/titanplayer.php', {
+ 'videoid': self._match_id(url),
+ 'type': 'V',
+ 'lang': 'en',
+ 'portal': 'int',
+ 'customer': 1024,
+ }), Laola1TvEmbedIE.ie_key())
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lci.py youtube-dl-2019.05.20/youtube_dl/extractor/lci.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lci.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lci.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,26 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LCIIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?lci\.fr/[^/]+/[\w-]+-(?P\d+)\.html'
+ _TEST = {
+ 'url': 'http://www.lci.fr/international/etats-unis-a-j-62-hillary-clinton-reste-sans-voix-2001679.html',
+ 'md5': '2fdb2538b884d4d695f9bd2bde137e6c',
+ 'info_dict': {
+ 'id': '13244802',
+ 'ext': 'mp4',
+ 'title': 'Hillary Clinton et sa quinte de toux, en plein meeting',
+ 'description': 'md5:a4363e3a960860132f8124b62f4a01c9',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ wat_id = self._search_regex(
+ (r'data-watid=[\'"](\d+)', r'idwat["\']?\s*:\s*["\']?(\d+)'),
+ webpage, 'wat id')
+ return self.url_result('wat:' + wat_id, 'Wat', wat_id)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lcp.py youtube-dl-2019.05.20/youtube_dl/extractor/lcp.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lcp.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lcp.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .arkena import ArkenaIE
+
+
+class LcpPlayIE(ArkenaIE):
+ _VALID_URL = r'https?://play\.lcp\.fr/embed/(?P[^/]+)/(?P[^/]+)/[^/]+/[^/]+'
+ _TESTS = [{
+ 'url': 'http://play.lcp.fr/embed/327336/131064/darkmatter/0',
+ 'md5': 'b8bd9298542929c06c1c15788b1f277a',
+ 'info_dict': {
+ 'id': '327336',
+ 'ext': 'mp4',
+ 'title': '327336',
+ 'timestamp': 1456391602,
+ 'upload_date': '20160225',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
+
+
+class LcpIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?lcp\.fr/(?:[^/]+/)*(?P[^/]+)'
+
+ _TESTS = [{
+ # arkena embed
+ 'url': 'http://www.lcp.fr/la-politique-en-video/schwartzenberg-prg-preconise-francois-hollande-de-participer-une-primaire',
+ 'md5': 'b8bd9298542929c06c1c15788b1f277a',
+ 'info_dict': {
+ 'id': 'd56d03e9',
+ 'ext': 'mp4',
+ 'title': 'Schwartzenberg (PRG) préconise à François Hollande de participer à une primaire à gauche',
+ 'description': 'md5:96ad55009548da9dea19f4120c6c16a8',
+ 'timestamp': 1456488895,
+ 'upload_date': '20160226',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ # dailymotion live stream
+ 'url': 'http://www.lcp.fr/le-direct',
+ 'info_dict': {
+ 'id': 'xji3qy',
+ 'ext': 'mp4',
+ 'title': 'La Chaine Parlementaire (LCP), Live TNT',
+ 'description': 'md5:5c69593f2de0f38bd9a949f2c95e870b',
+ 'uploader': 'LCP',
+ 'uploader_id': 'xbz33d',
+ 'timestamp': 1308923058,
+ 'upload_date': '20110624',
+ },
+ 'params': {
+ # m3u8 live stream
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'http://www.lcp.fr/emissions/277792-les-volontaires',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ play_url = self._search_regex(
+ r']+src=(["\'])(?P%s?(?:(?!\1).)*)\1' % LcpPlayIE._VALID_URL,
+ webpage, 'play iframe', default=None, group='url')
+
+ if not play_url:
+ return self.url_result(url, 'Generic')
+
+ title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+ 'twitter:title', webpage, fatal=True)
+ description = self._html_search_meta(
+ ('description', 'twitter:description'), webpage)
+
+ return {
+ '_type': 'url_transparent',
+ 'ie_key': LcpPlayIE.ie_key(),
+ 'url': play_url,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/learnr.py youtube-dl-2019.05.20/youtube_dl/extractor/learnr.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/learnr.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/learnr.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LearnrIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?learnr\.pro/view/video/(?P[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.learnr.pro/view/video/51624-web-development-tutorial-for-beginners-1-how-to-build-webpages-with-html-css-javascript',
+ 'md5': '3719fdf0a68397f49899e82c308a89de',
+ 'info_dict': {
+ 'id': '51624',
+ 'ext': 'mp4',
+ 'title': 'Web Development Tutorial for Beginners (#1) - How to build webpages with HTML, CSS, Javascript',
+ 'description': 'md5:b36dbfa92350176cdf12b4d388485503',
+ 'uploader': 'LearnCode.academy',
+ 'uploader_id': 'learncodeacademy',
+ 'upload_date': '20131021',
+ },
+ 'add_ie': ['Youtube'],
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ return {
+ '_type': 'url_transparent',
+ 'url': self._search_regex(
+ r"videoId\s*:\s*'([^']+)'", webpage, 'youtube id'),
+ 'id': video_id,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lecture2go.py youtube-dl-2019.05.20/youtube_dl/extractor/lecture2go.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lecture2go.py 2015-12-30 19:30:33.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lecture2go.py 2019-06-07 18:49:07.000000000 +0000
@@ -6,6 +6,7 @@
from .common import InfoExtractor
from ..utils import (
determine_ext,
+ determine_protocol,
parse_duration,
int_or_none,
)
@@ -18,10 +19,14 @@
'md5': 'ac02b570883020d208d405d5a3fd2f7f',
'info_dict': {
'id': '17473',
- 'ext': 'flv',
+ 'ext': 'mp4',
'title': '2 - Endliche Automaten und reguläre Sprachen',
'creator': 'Frank Heitmann',
'duration': 5220,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
}
}
@@ -32,14 +37,18 @@
title = self._html_search_regex(r']+class="title">(.+)', webpage, 'title')
formats = []
- for url in set(re.findall(r'"src","([^"]+)"', webpage)):
+ for url in set(re.findall(r'var\s+playerUri\d+\s*=\s*"([^"]+)"', webpage)):
ext = determine_ext(url)
+ protocol = determine_protocol({'url': url})
if ext == 'f4m':
- formats.extend(self._extract_f4m_formats(url, video_id))
+ formats.extend(self._extract_f4m_formats(url, video_id, f4m_id='hds'))
elif ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(url, video_id))
+ formats.extend(self._extract_m3u8_formats(url, video_id, ext='mp4', m3u8_id='hls'))
else:
+ if protocol == 'rtmp':
+ continue # XXX: currently broken
formats.append({
+ 'format_id': protocol,
'url': url,
})
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lecturio.py youtube-dl-2019.05.20/youtube_dl/extractor/lecturio.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lecturio.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lecturio.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,229 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+ determine_ext,
+ extract_attributes,
+ ExtractorError,
+ float_or_none,
+ int_or_none,
+ str_or_none,
+ url_or_none,
+ urlencode_postdata,
+ urljoin,
+)
+
+
+class LecturioBaseIE(InfoExtractor):
+ _LOGIN_URL = 'https://app.lecturio.com/en/login'
+ _NETRC_MACHINE = 'lecturio'
+
+ def _real_initialize(self):
+ self._login()
+
+ def _login(self):
+ username, password = self._get_login_info()
+ if username is None:
+ return
+
+ # Sets some cookies
+ _, urlh = self._download_webpage_handle(
+ self._LOGIN_URL, None, 'Downloading login popup')
+
+ def is_logged(url_handle):
+ return self._LOGIN_URL not in compat_str(url_handle.geturl())
+
+ # Already logged in
+ if is_logged(urlh):
+ return
+
+ login_form = {
+ 'signin[email]': username,
+ 'signin[password]': password,
+ 'signin[remember]': 'on',
+ }
+
+ response, urlh = self._download_webpage_handle(
+ self._LOGIN_URL, None, 'Logging in',
+ data=urlencode_postdata(login_form))
+
+ # Logged in successfully
+ if is_logged(urlh):
+ return
+
+ errors = self._html_search_regex(
+ r'(?s)]+class=["\']error_list[^>]+>(.+?)', response,
+ 'errors', default=None)
+ if errors:
+ raise ExtractorError('Unable to login: %s' % errors, expected=True)
+ raise ExtractorError('Unable to log in')
+
+
+class LecturioIE(LecturioBaseIE):
+ _VALID_URL = r'''(?x)
+ https://
+ (?:
+ app\.lecturio\.com/[^/]+/(?P[^/?#&]+)\.lecture|
+ (?:www\.)?lecturio\.de/[^/]+/(?P[^/?#&]+)\.vortrag
+ )
+ '''
+ _TESTS = [{
+ 'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
+ 'md5': 'f576a797a5b7a5e4e4bbdfc25a6a6870',
+ 'info_dict': {
+ 'id': '39634',
+ 'ext': 'mp4',
+ 'title': 'Important Concepts and Terms – Introduction to Microbiology',
+ },
+ 'skip': 'Requires lecturio account credentials',
+ }, {
+ 'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
+ 'only_matching': True,
+ }]
+
+ _CC_LANGS = {
+ 'German': 'de',
+ 'English': 'en',
+ 'Spanish': 'es',
+ 'French': 'fr',
+ 'Polish': 'pl',
+ 'Russian': 'ru',
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('id') or mobj.group('id_de')
+
+ webpage = self._download_webpage(
+ 'https://app.lecturio.com/en/lecture/%s/player.html' % display_id,
+ display_id)
+
+ lecture_id = self._search_regex(
+ r'lecture_id\s*=\s*(?:L_)?(\d+)', webpage, 'lecture id')
+
+ api_url = self._search_regex(
+ r'lectureDataLink\s*:\s*(["\'])(?P(?:(?!\1).)+)\1', webpage,
+ 'api url', group='url')
+
+ video = self._download_json(api_url, display_id)
+
+ title = video['title'].strip()
+
+ formats = []
+ for format_ in video['content']['media']:
+ if not isinstance(format_, dict):
+ continue
+ file_ = format_.get('file')
+ if not file_:
+ continue
+ ext = determine_ext(file_)
+ if ext == 'smil':
+ # smil contains only broken RTMP formats anyway
+ continue
+ file_url = url_or_none(file_)
+ if not file_url:
+ continue
+ label = str_or_none(format_.get('label'))
+ filesize = int_or_none(format_.get('fileSize'))
+ formats.append({
+ 'url': file_url,
+ 'format_id': label,
+ 'filesize': float_or_none(filesize, invscale=1000)
+ })
+ self._sort_formats(formats)
+
+ subtitles = {}
+ automatic_captions = {}
+ cc = self._parse_json(
+ self._search_regex(
+ r'subtitleUrls\s*:\s*({.+?})\s*,', webpage, 'subtitles',
+ default='{}'), display_id, fatal=False)
+ for cc_label, cc_url in cc.items():
+ cc_url = url_or_none(cc_url)
+ if not cc_url:
+ continue
+ lang = self._search_regex(
+ r'/([a-z]{2})_', cc_url, 'lang',
+ default=cc_label.split()[0] if cc_label else 'en')
+ original_lang = self._search_regex(
+ r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
+ default=None)
+ sub_dict = (automatic_captions
+ if 'auto-translated' in cc_label or original_lang
+ else subtitles)
+ sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
+ 'url': cc_url,
+ })
+
+ return {
+ 'id': lecture_id,
+ 'title': title,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ 'automatic_captions': automatic_captions,
+ }
+
+
+class LecturioCourseIE(LecturioBaseIE):
+ _VALID_URL = r'https://app\.lecturio\.com/[^/]+/(?P[^/?#&]+)\.course'
+ _TEST = {
+ 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
+ 'info_dict': {
+ 'id': 'microbiology-introduction',
+ 'title': 'Microbiology: Introduction',
+ },
+ 'playlist_count': 45,
+ 'skip': 'Requires lecturio account credentials',
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ entries = []
+ for mobj in re.finditer(
+ r'(?s)<[^>]+\bdata-url=(["\'])(?:(?!\1).)+\.lecture\b[^>]+>',
+ webpage):
+ params = extract_attributes(mobj.group(0))
+ lecture_url = urljoin(url, params.get('data-url'))
+ lecture_id = params.get('data-id')
+ entries.append(self.url_result(
+ lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
+
+ title = self._search_regex(
+ r']+class=["\']content-title[^>]+>([^<]+)', webpage,
+ 'title', default=None)
+
+ return self.playlist_result(entries, display_id, title)
+
+
+class LecturioDeCourseIE(LecturioBaseIE):
+ _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P[^/?#&]+)\.kurs'
+ _TEST = {
+ 'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
+ 'only_matching': True,
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ entries = []
+ for mobj in re.finditer(
+ r'(?s)]+\bdata-lecture-id=["\'](?P\d+).+?\bhref=(["\'])(?P(?:(?!\2).)+\.vortrag)\b[^>]+>',
+ webpage):
+ lecture_url = urljoin(url, mobj.group('url'))
+ lecture_id = mobj.group('id')
+ entries.append(self.url_result(
+ lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
+
+ title = self._search_regex(
+ r']*>([^<]+)', webpage, 'title', default=None)
+
+ return self.playlist_result(entries, display_id, title)
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/leeco.py youtube-dl-2019.05.20/youtube_dl/extractor/leeco.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/leeco.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/leeco.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,368 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+import hashlib
+import re
+import time
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_b64decode,
+ compat_ord,
+ compat_str,
+ compat_urllib_parse_urlencode,
+)
+from ..utils import (
+ determine_ext,
+ encode_data_uri,
+ ExtractorError,
+ int_or_none,
+ orderedSet,
+ parse_iso8601,
+ str_or_none,
+ url_basename,
+ urshift,
+)
+
+
+class LeIE(InfoExtractor):
+ IE_DESC = '乐视网'
+ _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|(?:sports\.le|(?:www\.)?lesports)\.com/(?:match|video))/(?P\d+)\.html'
+ _GEO_COUNTRIES = ['CN']
+ _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
+
+ _TESTS = [{
+ 'url': 'http://www.le.com/ptv/vplay/22005890.html',
+ 'md5': 'edadcfe5406976f42f9f266057ee5e40',
+ 'info_dict': {
+ 'id': '22005890',
+ 'ext': 'mp4',
+ 'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
+ 'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
+ },
+ 'params': {
+ 'hls_prefer_native': True,
+ },
+ }, {
+ 'url': 'http://www.le.com/ptv/vplay/1415246.html',
+ 'info_dict': {
+ 'id': '1415246',
+ 'ext': 'mp4',
+ 'title': '美人天下01',
+ 'description': 'md5:28942e650e82ed4fcc8e4de919ee854d',
+ },
+ 'params': {
+ 'hls_prefer_native': True,
+ },
+ }, {
+ 'note': 'This video is available only in Mainland China, thus a proxy is needed',
+ 'url': 'http://www.le.com/ptv/vplay/1118082.html',
+ 'md5': '2424c74948a62e5f31988438979c5ad1',
+ 'info_dict': {
+ 'id': '1118082',
+ 'ext': 'mp4',
+ 'title': '与龙共舞 完整版',
+ 'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
+ },
+ 'params': {
+ 'hls_prefer_native': True,
+ },
+ }, {
+ 'url': 'http://sports.le.com/video/25737697.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.lesports.com/match/1023203003.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://sports.le.com/match/1023203003.html',
+ 'only_matching': True,
+ }]
+
+ # ror() and calc_time_key() are reversed from a embedded swf file in LetvPlayer.swf
+ def ror(self, param1, param2):
+ _loc3_ = 0
+ while _loc3_ < param2:
+ param1 = urshift(param1, 1) + ((param1 & 1) << 31)
+ _loc3_ += 1
+ return param1
+
+ def calc_time_key(self, param1):
+ _loc2_ = 185025305
+ return self.ror(param1, _loc2_ % 17) ^ _loc2_
+
+ # see M3U8Encryption class in KLetvPlayer.swf
+ @staticmethod
+ def decrypt_m3u8(encrypted_data):
+ if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
+ return encrypted_data
+ encrypted_data = encrypted_data[5:]
+
+ _loc4_ = bytearray(2 * len(encrypted_data))
+ for idx, val in enumerate(encrypted_data):
+ b = compat_ord(val)
+ _loc4_[2 * idx] = b // 16
+ _loc4_[2 * idx + 1] = b % 16
+ idx = len(_loc4_) - 11
+ _loc4_ = _loc4_[idx:] + _loc4_[:idx]
+ _loc7_ = bytearray(len(encrypted_data))
+ for i in range(len(encrypted_data)):
+ _loc7_[i] = _loc4_[2 * i] * 16 + _loc4_[2 * i + 1]
+
+ return bytes(_loc7_)
+
+ def _check_errors(self, play_json):
+ # Check for errors
+ playstatus = play_json['msgs']['playstatus']
+ if playstatus['status'] == 0:
+ flag = playstatus['flag']
+ if flag == 1:
+ self.raise_geo_restricted()
+ else:
+ raise ExtractorError('Generic error. flag = %d' % flag, expected=True)
+
+ def _real_extract(self, url):
+ media_id = self._match_id(url)
+ page = self._download_webpage(url, media_id)
+
+ play_json_flash = self._download_json(
+ 'http://player-pc.le.com/mms/out/video/playJson',
+ media_id, 'Downloading flash playJson data', query={
+ 'id': media_id,
+ 'platid': 1,
+ 'splatid': 105,
+ 'format': 1,
+ 'source': 1000,
+ 'tkey': self.calc_time_key(int(time.time())),
+ 'domain': 'www.le.com',
+ 'region': 'cn',
+ },
+ headers=self.geo_verification_headers())
+ self._check_errors(play_json_flash)
+
+ def get_flash_urls(media_url, format_id):
+ nodes_data = self._download_json(
+ media_url, media_id,
+ 'Download JSON metadata for format %s' % format_id,
+ query={
+ 'm3v': 1,
+ 'format': 1,
+ 'expect': 3,
+ 'tss': 'ios',
+ })
+
+ req = self._request_webpage(
+ nodes_data['nodelist'][0]['location'], media_id,
+ note='Downloading m3u8 information for format %s' % format_id)
+
+ m3u8_data = self.decrypt_m3u8(req.read())
+
+ return {
+ 'hls': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
+ }
+
+ extracted_formats = []
+ formats = []
+ playurl = play_json_flash['msgs']['playurl']
+ play_domain = playurl['domain'][0]
+
+ for format_id, format_data in playurl.get('dispatch', []).items():
+ if format_id in extracted_formats:
+ continue
+ extracted_formats.append(format_id)
+
+ media_url = play_domain + format_data[0]
+ for protocol, format_url in get_flash_urls(media_url, format_id).items():
+ f = {
+ 'url': format_url,
+ 'ext': determine_ext(format_data[1]),
+ 'format_id': '%s-%s' % (protocol, format_id),
+ 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
+ 'quality': int_or_none(format_id),
+ }
+
+ if format_id[-1:] == 'p':
+ f['height'] = int_or_none(format_id[:-1])
+
+ formats.append(f)
+ self._sort_formats(formats, ('height', 'quality', 'format_id'))
+
+ publish_time = parse_iso8601(self._html_search_regex(
+ r'发布时间 ([^<>]+) ', page, 'publish time', default=None),
+ delimiter=' ', timezone=datetime.timedelta(hours=8))
+ description = self._html_search_meta('description', page, fatal=False)
+
+ return {
+ 'id': media_id,
+ 'formats': formats,
+ 'title': playurl['title'],
+ 'thumbnail': playurl['pic'],
+ 'description': description,
+ 'timestamp': publish_time,
+ }
+
+
+class LePlaylistIE(InfoExtractor):
+ _VALID_URL = r'https?://[a-z]+\.le\.com/(?!video)[a-z]+/(?P[a-z0-9_]+)'
+
+ _TESTS = [{
+ 'url': 'http://www.le.com/tv/46177.html',
+ 'info_dict': {
+ 'id': '46177',
+ 'title': '美人天下',
+ 'description': 'md5:395666ff41b44080396e59570dbac01c'
+ },
+ 'playlist_count': 35
+ }, {
+ 'url': 'http://tv.le.com/izt/wuzetian/index.html',
+ 'info_dict': {
+ 'id': 'wuzetian',
+ 'title': '武媚娘传奇',
+ 'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
+ },
+ # This playlist contains some extra videos other than the drama itself
+ 'playlist_mincount': 96
+ }, {
+ 'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
+ # This series is moved to http://www.le.com/tv/10005297.html
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.le.com/comic/92063.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://list.le.com/listn/c1009_sc532002_d2_p1_o1.html',
+ 'only_matching': True,
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ return False if LeIE.suitable(url) else super(LePlaylistIE, cls).suitable(url)
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ page = self._download_webpage(url, playlist_id)
+
+ # Currently old domain names are still used in playlists
+ media_ids = orderedSet(re.findall(
+ r']+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
+ entries = [self.url_result(LeIE._URL_TEMPLATE % media_id, ie='Le')
+ for media_id in media_ids]
+
+ title = self._html_search_meta('keywords', page,
+ fatal=False).split('，')[0]
+ description = self._html_search_meta('description', page, fatal=False)
+
+ return self.playlist_result(entries, playlist_id, playlist_title=title,
+ playlist_description=description)
+
+
+class LetvCloudIE(InfoExtractor):
+ # Most of *.letv.com is changed to *.le.com on 2016/01/02
+ # but yuntv.letv.com is kept, so also keep the extractor name
+ IE_DESC = '乐视云'
+ _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
+
+ _TESTS = [{
+ 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
+ 'md5': '26450599afd64c513bc77030ad15db44',
+ 'info_dict': {
+ 'id': 'p7jnfw5hw9_467623dedf',
+ 'ext': 'mp4',
+ 'title': 'Video p7jnfw5hw9_467623dedf',
+ },
+ }, {
+ 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
+ 'md5': 'e03d9cc8d9c13191e1caf277e42dbd31',
+ 'info_dict': {
+ 'id': 'p7jnfw5hw9_ec93197892',
+ 'ext': 'mp4',
+ 'title': 'Video p7jnfw5hw9_ec93197892',
+ },
+ }, {
+ 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
+ 'md5': 'cb988699a776b22d4a41b9d43acfb3ac',
+ 'info_dict': {
+ 'id': 'p7jnfw5hw9_187060b6fd',
+ 'ext': 'mp4',
+ 'title': 'Video p7jnfw5hw9_187060b6fd',
+ },
+ }]
+
+ @staticmethod
+ def sign_data(obj):
+ if obj['cf'] == 'flash':
+ salt = '2f9d6924b33a165a6d8b5d3d42f4f987'
+ items = ['cf', 'format', 'ran', 'uu', 'ver', 'vu']
+ elif obj['cf'] == 'html5':
+ salt = 'fbeh5player12c43eccf2bec3300344'
+ items = ['cf', 'ran', 'uu', 'bver', 'vu']
+ input_data = ''.join([item + obj[item] for item in items]) + salt
+ obj['sign'] = hashlib.md5(input_data.encode('utf-8')).hexdigest()
+
+ def _get_formats(self, cf, uu, vu, media_id):
+ def get_play_json(cf, timestamp):
+ data = {
+ 'cf': cf,
+ 'ver': '2.2',
+ 'bver': 'firefox44.0',
+ 'format': 'json',
+ 'uu': uu,
+ 'vu': vu,
+ 'ran': compat_str(timestamp),
+ }
+ self.sign_data(data)
+ return self._download_json(
+ 'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse_urlencode(data),
+ media_id, 'Downloading playJson data for type %s' % cf)
+
+ play_json = get_play_json(cf, time.time())
+ # The server time may be different from local time
+ if play_json.get('code') == 10071:
+ play_json = get_play_json(cf, play_json['timestamp'])
+
+ if not play_json.get('data'):
+ if play_json.get('message'):
+ raise ExtractorError('Letv cloud said: %s' % play_json['message'], expected=True)
+ elif play_json.get('code'):
+ raise ExtractorError('Letv cloud returned error %d' % play_json['code'], expected=True)
+ else:
+ raise ExtractorError('Letv cloud returned an unknwon error')
+
+ def b64decode(s):
+ return compat_b64decode(s).decode('utf-8')
+
+ formats = []
+ for media in play_json['data']['video_info']['media'].values():
+ play_url = media['play_url']
+ url = b64decode(play_url['main_url'])
+ decoded_url = b64decode(url_basename(url))
+ formats.append({
+ 'url': url,
+ 'ext': determine_ext(decoded_url),
+ 'format_id': str_or_none(play_url.get('vtype')),
+ 'format_note': str_or_none(play_url.get('definition')),
+ 'width': int_or_none(play_url.get('vwidth')),
+ 'height': int_or_none(play_url.get('vheight')),
+ })
+
+ return formats
+
+ def _real_extract(self, url):
+ uu_mobj = re.search(r'uu=([\w]+)', url)
+ vu_mobj = re.search(r'vu=([\w]+)', url)
+
+ if not uu_mobj or not vu_mobj:
+ raise ExtractorError('Invalid URL: %s' % url, expected=True)
+
+ uu = uu_mobj.group(1)
+ vu = vu_mobj.group(1)
+ media_id = uu + '_' + vu
+
+ formats = self._get_formats('flash', uu, vu, media_id) + self._get_formats('html5', uu, vu, media_id)
+ self._sort_formats(formats)
+
+ return {
+ 'id': media_id,
+ 'title': 'Video %s' % media_id,
+ 'formats': formats,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lego.py youtube-dl-2019.05.20/youtube_dl/extractor/lego.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lego.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lego.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,128 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+ unescapeHTML,
+ parse_duration,
+ get_element_by_class,
+)
+
+
+class LEGOIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?lego\.com/(?P[^/]+)/(?:[^/]+/)*videos/(?:[^/]+/)*[^/?#]+-(?P[0-9a-f]+)'
+ _TESTS = [{
+ 'url': 'http://www.lego.com/en-us/videos/themes/club/blocumentary-kawaguchi-55492d823b1b4d5e985787fa8c2973b1',
+ 'md5': 'f34468f176cfd76488767fc162c405fa',
+ 'info_dict': {
+ 'id': '55492d823b1b4d5e985787fa8c2973b1',
+ 'ext': 'mp4',
+ 'title': 'Blocumentary Great Creations: Akiyuki Kawaguchi',
+ 'description': 'Blocumentary Great Creations: Akiyuki Kawaguchi',
+ },
+ }, {
+ # geo-restricted but the contentUrl contain a valid url
+ 'url': 'http://www.lego.com/nl-nl/videos/themes/nexoknights/episode-20-kingdom-of-heroes-13bdc2299ab24d9685701a915b3d71e7##sp=399',
+ 'md5': '4c3fec48a12e40c6e5995abc3d36cc2e',
+ 'info_dict': {
+ 'id': '13bdc2299ab24d9685701a915b3d71e7',
+ 'ext': 'mp4',
+ 'title': 'Aflevering 20 - Helden van het koninkrijk',
+ 'description': 'md5:8ee499aac26d7fa8bcb0cedb7f9c3941',
+ },
+ }, {
+ # special characters in title
+ 'url': 'http://www.lego.com/en-us/starwars/videos/lego-star-wars-force-surprise-9685ee9d12e84ff38e84b4e3d0db533d',
+ 'info_dict': {
+ 'id': '9685ee9d12e84ff38e84b4e3d0db533d',
+ 'ext': 'mp4',
+ 'title': 'Force Surprise – LEGO® Star Wars™ Microfighters',
+ 'description': 'md5:9c673c96ce6f6271b88563fe9dc56de3',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
+ _BITRATES = [256, 512, 1024, 1536, 2560]
+
+ def _real_extract(self, url):
+ locale, video_id = re.match(self._VALID_URL, url).groups()
+ webpage = self._download_webpage(url, video_id)
+ title = get_element_by_class('video-header', webpage).strip()
+ progressive_base = 'https://lc-mediaplayerns-live-s.legocdn.com/'
+ streaming_base = 'http://legoprod-f.akamaihd.net/'
+ content_url = self._html_search_meta('contentUrl', webpage)
+ path = self._search_regex(
+ r'(?:https?:)?//[^/]+/(?:[iz]/s/)?public/(.+)_[0-9,]+\.(?:mp4|webm)',
+ content_url, 'video path', default=None)
+ if not path:
+ player_url = self._proto_relative_url(self._search_regex(
+ r']+src="((?:https?)?//(?:www\.)?lego\.com/[^/]+/mediaplayer/video/[^"]+)',
+ webpage, 'player url', default=None))
+ if not player_url:
+ base_url = self._proto_relative_url(self._search_regex(
+ r'data-baseurl="([^"]+)"', webpage, 'base url',
+ default='http://www.lego.com/%s/mediaplayer/video/' % locale))
+ player_url = base_url + video_id
+ player_webpage = self._download_webpage(player_url, video_id)
+ video_data = self._parse_json(unescapeHTML(self._search_regex(
+ r"video='([^']+)'", player_webpage, 'video data')), video_id)
+ progressive_base = self._search_regex(
+ r'data-video-progressive-url="([^"]+)"',
+ player_webpage, 'progressive base', default='https://lc-mediaplayerns-live-s.legocdn.com/')
+ streaming_base = self._search_regex(
+ r'data-video-streaming-url="([^"]+)"',
+ player_webpage, 'streaming base', default='http://legoprod-f.akamaihd.net/')
+ item_id = video_data['ItemId']
+
+ net_storage_path = video_data.get('NetStoragePath') or '/'.join([item_id[:2], item_id[2:4]])
+ base_path = '_'.join([item_id, video_data['VideoId'], video_data['Locale'], compat_str(video_data['VideoVersion'])])
+ path = '/'.join([net_storage_path, base_path])
+ streaming_path = ','.join(map(lambda bitrate: compat_str(bitrate), self._BITRATES))
+
+ formats = self._extract_akamai_formats(
+ '%si/s/public/%s_,%s,.mp4.csmil/master.m3u8' % (streaming_base, path, streaming_path), video_id)
+ m3u8_formats = list(filter(
+ lambda f: f.get('protocol') == 'm3u8_native' and f.get('vcodec') != 'none',
+ formats))
+ if len(m3u8_formats) == len(self._BITRATES):
+ self._sort_formats(m3u8_formats)
+ for bitrate, m3u8_format in zip(self._BITRATES, m3u8_formats):
+ progressive_base_url = '%spublic/%s_%d.' % (progressive_base, path, bitrate)
+ mp4_f = m3u8_format.copy()
+ mp4_f.update({
+ 'url': progressive_base_url + 'mp4',
+ 'format_id': m3u8_format['format_id'].replace('hls', 'mp4'),
+ 'protocol': 'http',
+ })
+ web_f = {
+ 'url': progressive_base_url + 'webm',
+ 'format_id': m3u8_format['format_id'].replace('hls', 'webm'),
+ 'width': m3u8_format['width'],
+ 'height': m3u8_format['height'],
+ 'tbr': m3u8_format.get('tbr'),
+ 'ext': 'webm',
+ }
+ formats.extend([web_f, mp4_f])
+ else:
+ for bitrate in self._BITRATES:
+ for ext in ('web', 'mp4'):
+ formats.append({
+ 'format_id': '%s-%s' % (ext, bitrate),
+ 'url': '%spublic/%s_%d.%s' % (progressive_base, path, bitrate, ext),
+ 'tbr': bitrate,
+ 'ext': ext,
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': self._html_search_meta('description', webpage),
+ 'thumbnail': self._html_search_meta('thumbnail', webpage),
+ 'duration': parse_duration(self._html_search_meta('duration', webpage)),
+ 'formats': formats,
+ }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lemonde.py youtube-dl-2019.05.20/youtube_dl/extractor/lemonde.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lemonde.py 2016-01-23 10:08:46.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lemonde.py 2019-06-07 18:49:07.000000000 +0000
@@ -7,20 +7,40 @@
_VALID_URL = r'https?://(?:.+?\.)?lemonde\.fr/(?:[^/]+/)*(?P[^/]+)\.html'
_TESTS = [{
'url': 'http://www.lemonde.fr/police-justice/video/2016/01/19/comprendre-l-affaire-bygmalion-en-cinq-minutes_4849702_1653578.html',
- 'md5': '01fb3c92de4c12c573343d63e163d302',
+ 'md5': 'da120c8722d8632eec6ced937536cc98',
'info_dict': {
'id': 'lqm3kl',
'ext': 'mp4',
'title': "Comprendre l'affaire Bygmalion en 5 minutes",
- 'thumbnail': 're:^https?://.*\.jpg',
- 'duration': 320,
+ 'thumbnail': r're:^https?://.*\.jpg',
+ 'duration': 309,
'upload_date': '20160119',
'timestamp': 1453194778,
'uploader_id': '3pmkp',
},
}, {
+ # standard iframe embed
+ 'url': 'http://www.lemonde.fr/les-decodeurs/article/2016/10/18/tout-comprendre-du-ceta-le-petit-cousin-du-traite-transatlantique_5015920_4355770.html',
+ 'info_dict': {
+ 'id': 'uzsxms',
+ 'ext': 'mp4',
+ 'title': "CETA : quelles suites pour l'accord commercial entre l'Europe et le Canada ?",
+ 'thumbnail': r're:^https?://.*\.jpg',
+ 'duration': 325,
+ 'upload_date': '20161021',
+ 'timestamp': 1477044540,
+ 'uploader_id': '3pmkp',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
'url': 'http://redaction.actu.lemonde.fr/societe/video/2016/01/18/calais-debut-des-travaux-de-defrichement-dans-la-jungle_4849233_3224.html',
'only_matching': True,
+ }, {
+ # YouTube embeds
+ 'url': 'http://www.lemonde.fr/pixels/article/2016/12/09/pourquoi-pewdiepie-superstar-de-youtube-a-menace-de-fermer-sa-chaine_5046649_4408996.html',
+ 'only_matching': True,
}]
def _real_extract(self, url):
@@ -30,5 +50,9 @@
digiteka_url = self._proto_relative_url(self._search_regex(
r'url\s*:\s*(["\'])(?P(?:https?://)?//(?:www\.)?(?:digiteka\.net|ultimedia\.com)/deliver/.+?)\1',
- webpage, 'digiteka url', group='url'))
- return self.url_result(digiteka_url, 'Digiteka')
+ webpage, 'digiteka url', group='url', default=None))
+
+ if digiteka_url:
+ return self.url_result(digiteka_url, 'Digiteka')
+
+ return self.url_result(url, 'Generic')
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/lenta.py youtube-dl-2019.05.20/youtube_dl/extractor/lenta.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/lenta.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/lenta.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LentaIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?lenta\.ru/[^/]+/\d+/\d+/\d+/(?P[^/?#&]+)'
+ _TESTS = [{
+ 'url': 'https://lenta.ru/news/2018/03/22/savshenko_go/',
+ 'info_dict': {
+ 'id': '964400',
+ 'ext': 'mp4',
+ 'title': 'Надежду Савченко задержали',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'duration': 61,
+ 'view_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ # EaglePlatform iframe embed
+ 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
+ 'info_dict': {
+ 'id': '227304',
+ 'ext': 'mp4',
+ 'title': 'Навальный вышел на свободу',
+ 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
+ 'thumbnail': r're:^https?://.*\.jpg$',
+ 'duration': 87,
+ 'view_count': int,
+ 'age_limit': 0,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_id = self._search_regex(
+ r'vid\s*:\s*["\']?(\d+)', webpage, 'eagleplatform id',
+ default=None)
+ if video_id:
+ return self.url_result(
+ 'eagleplatform:lentaru.media.eagleplatform.com:%s' % video_id,
+ ie='EaglePlatform', video_id=video_id)
+
+ return self.url_result(url, ie='Generic')
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/letv.py youtube-dl-2019.05.20/youtube_dl/extractor/letv.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/letv.py 2016-01-31 11:57:01.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/letv.py 1970-01-01 00:00:00.000000000 +0000
@@ -1,356 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import datetime
-import re
-import time
-import base64
-import hashlib
-
-from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_ord,
- compat_str,
-)
-from ..utils import (
- determine_ext,
- ExtractorError,
- parse_iso8601,
- sanitized_Request,
- int_or_none,
- str_or_none,
- encode_data_uri,
- url_basename,
-)
-
-
-class LetvIE(InfoExtractor):
- IE_DESC = '乐视网'
- _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P\d+).html'
-
- _TESTS = [{
- 'url': 'http://www.letv.com/ptv/vplay/22005890.html',
- 'md5': 'edadcfe5406976f42f9f266057ee5e40',
- 'info_dict': {
- 'id': '22005890',
- 'ext': 'mp4',
- 'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
- 'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
- },
- 'params': {
- 'hls_prefer_native': True,
- },
- }, {
- 'url': 'http://www.letv.com/ptv/vplay/1415246.html',
- 'info_dict': {
- 'id': '1415246',
- 'ext': 'mp4',
- 'title': '美人天下01',
- 'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
- },
- 'params': {
- 'hls_prefer_native': True,
- },
- }, {
- 'note': 'This video is available only in Mainland China, thus a proxy is needed',
- 'url': 'http://www.letv.com/ptv/vplay/1118082.html',
- 'md5': '2424c74948a62e5f31988438979c5ad1',
- 'info_dict': {
- 'id': '1118082',
- 'ext': 'mp4',
- 'title': '与龙共舞 完整版',
- 'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
- },
- 'params': {
- 'hls_prefer_native': True,
- },
- 'skip': 'Only available in China',
- }]
-
- @staticmethod
- def urshift(val, n):
- return val >> n if val >= 0 else (val + 0x100000000) >> n
-
- # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
- def ror(self, param1, param2):
- _loc3_ = 0
- while _loc3_ < param2:
- param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
- _loc3_ += 1
- return param1
-
- def calc_time_key(self, param1):
- _loc2_ = 773625421
- _loc3_ = self.ror(param1, _loc2_ % 13)
- _loc3_ = _loc3_ ^ _loc2_
- _loc3_ = self.ror(_loc3_, _loc2_ % 17)
- return _loc3_
-
- # see M3U8Encryption class in KLetvPlayer.swf
- @staticmethod
- def decrypt_m3u8(encrypted_data):
- if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
- return encrypted_data
- encrypted_data = encrypted_data[5:]
-
- _loc4_ = bytearray()
- while encrypted_data:
- b = compat_ord(encrypted_data[0])
- _loc4_.extend([b // 16, b & 0x0f])
- encrypted_data = encrypted_data[1:]
- idx = len(_loc4_) - 11
- _loc4_ = _loc4_[idx:] + _loc4_[:idx]
- _loc7_ = bytearray()
- while _loc4_:
- _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
- _loc4_ = _loc4_[2:]
-
- return bytes(_loc7_)
-
- def _real_extract(self, url):
- media_id = self._match_id(url)
- page = self._download_webpage(url, media_id)
- params = {
- 'id': media_id,
- 'platid': 1,
- 'splatid': 101,
- 'format': 1,
- 'tkey': self.calc_time_key(int(time.time())),
- 'domain': 'www.letv.com'
- }
- play_json_req = sanitized_Request(
- 'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
- )
- cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
- if cn_verification_proxy:
- play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
-
- play_json = self._download_json(
- play_json_req,
- media_id, 'Downloading playJson data')
-
- # Check for errors
- playstatus = play_json['playstatus']
- if playstatus['status'] == 0:
- flag = playstatus['flag']
- if flag == 1:
- msg = 'Country %s auth error' % playstatus['country']
- else:
- msg = 'Generic error. flag = %d' % flag
- raise ExtractorError(msg, expected=True)
-
- playurl = play_json['playurl']
-
- formats = ['350', '1000', '1300', '720p', '1080p']
- dispatch = playurl['dispatch']
-
- urls = []
- for format_id in formats:
- if format_id in dispatch:
- media_url = playurl['domain'][0] + dispatch[format_id][0]
- media_url += '&' + compat_urllib_parse.urlencode({
- 'm3v': 1,
- 'format': 1,
- 'expect': 3,
- 'rateid': format_id,
- })
-
- nodes_data = self._download_json(
- media_url, media_id,
- 'Download JSON metadata for format %s' % format_id)
-
- req = self._request_webpage(
- nodes_data['nodelist'][0]['location'], media_id,
- note='Downloading m3u8 information for format %s' % format_id)
-
- m3u8_data = self.decrypt_m3u8(req.read())
-
- url_info_dict = {
- 'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
- 'ext': determine_ext(dispatch[format_id][1]),
- 'format_id': format_id,
- 'protocol': 'm3u8',
- }
-
- if format_id[-1:] == 'p':
- url_info_dict['height'] = int_or_none(format_id[:-1])
-
- urls.append(url_info_dict)
-
- publish_time = parse_iso8601(self._html_search_regex(
- r'发布时间 ([^<>]+) ', page, 'publish time', default=None),
- delimiter=' ', timezone=datetime.timedelta(hours=8))
- description = self._html_search_meta('description', page, fatal=False)
-
- return {
- 'id': media_id,
- 'formats': urls,
- 'title': playurl['title'],
- 'thumbnail': playurl['pic'],
- 'description': description,
- 'timestamp': publish_time,
- }
-
-
-class LetvTvIE(InfoExtractor):
- _VALID_URL = r'http://www.letv.com/tv/(?P\d+).html'
- _TESTS = [{
- 'url': 'http://www.letv.com/tv/46177.html',
- 'info_dict': {
- 'id': '46177',
- 'title': '美人天下',
- 'description': 'md5:395666ff41b44080396e59570dbac01c'
- },
- 'playlist_count': 35
- }]
-
- def _real_extract(self, url):
- playlist_id = self._match_id(url)
- page = self._download_webpage(url, playlist_id)
-
- media_urls = list(set(re.findall(
- r'http://www.letv.com/ptv/vplay/\d+.html', page)))
- entries = [self.url_result(media_url, ie='Letv')
- for media_url in media_urls]
-
- title = self._html_search_meta('keywords', page,
- fatal=False).split('，')[0]
- description = self._html_search_meta('description', page, fatal=False)
-
- return self.playlist_result(entries, playlist_id, playlist_title=title,
- playlist_description=description)
-
-
-class LetvPlaylistIE(LetvTvIE):
- _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P[a-z]+)/index.s?html'
- _TESTS = [{
- 'url': 'http://tv.letv.com/izt/wuzetian/index.html',
- 'info_dict': {
- 'id': 'wuzetian',
- 'title': '武媚娘传奇',
- 'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
- },
- # This playlist contains some extra videos other than the drama itself
- 'playlist_mincount': 96
- }, {
- 'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
- 'info_dict': {
- 'id': 'lswjzzjc',
- # The title should be "劲舞青春", but I can't find a simple way to
- # determine the playlist title
- 'title': '乐视午间自制剧场',
- 'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
- },
- 'playlist_mincount': 7
- }]
-
-
-class LetvCloudIE(InfoExtractor):
- IE_DESC = '乐视云'
- _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
-
- _TESTS = [{
- 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
- 'md5': '26450599afd64c513bc77030ad15db44',
- 'info_dict': {
- 'id': 'p7jnfw5hw9_467623dedf',
- 'ext': 'mp4',
- 'title': 'Video p7jnfw5hw9_467623dedf',
- },
- }, {
- 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
- 'md5': 'e03d9cc8d9c13191e1caf277e42dbd31',
- 'info_dict': {
- 'id': 'p7jnfw5hw9_ec93197892',
- 'ext': 'mp4',
- 'title': 'Video p7jnfw5hw9_ec93197892',
- },
- }, {
- 'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
- 'md5': 'cb988699a776b22d4a41b9d43acfb3ac',
- 'info_dict': {
- 'id': 'p7jnfw5hw9_187060b6fd',
- 'ext': 'mp4',
- 'title': 'Video p7jnfw5hw9_187060b6fd',
- },
- }]
-
- @staticmethod
- def sign_data(obj):
- if obj['cf'] == 'flash':
- salt = '2f9d6924b33a165a6d8b5d3d42f4f987'
- items = ['cf', 'format', 'ran', 'uu', 'ver', 'vu']
- elif obj['cf'] == 'html5':
- salt = 'fbeh5player12c43eccf2bec3300344'
- items = ['cf', 'ran', 'uu', 'bver', 'vu']
- input_data = ''.join([item + obj[item] for item in items]) + salt
- obj['sign'] = hashlib.md5(input_data.encode('utf-8')).hexdigest()
-
- def _get_formats(self, cf, uu, vu, media_id):
- def get_play_json(cf, timestamp):
- data = {
- 'cf': cf,
- 'ver': '2.2',
- 'bver': 'firefox44.0',
- 'format': 'json',
- 'uu': uu,
- 'vu': vu,
- 'ran': compat_str(timestamp),
- }
- self.sign_data(data)
- return self._download_json(
- 'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse.urlencode(data),
- media_id, 'Downloading playJson data for type %s' % cf)
-
- play_json = get_play_json(cf, time.time())
- # The server time may be different from local time
- if play_json.get('code') == 10071:
- play_json = get_play_json(cf, play_json['timestamp'])
-
- if not play_json.get('data'):
- if play_json.get('message'):
- raise ExtractorError('Letv cloud said: %s' % play_json['message'], expected=True)
- elif play_json.get('code'):
- raise ExtractorError('Letv cloud returned error %d' % play_json['code'], expected=True)
- else:
- raise ExtractorError('Letv cloud returned an unknwon error')
-
- def b64decode(s):
- return base64.b64decode(s.encode('utf-8')).decode('utf-8')
-
- formats = []
- for media in play_json['data']['video_info']['media'].values():
- play_url = media['play_url']
- url = b64decode(play_url['main_url'])
- decoded_url = b64decode(url_basename(url))
- formats.append({
- 'url': url,
- 'ext': determine_ext(decoded_url),
- 'format_id': int_or_none(play_url.get('vtype')),
- 'format_note': str_or_none(play_url.get('definition')),
- 'width': int_or_none(play_url.get('vwidth')),
- 'height': int_or_none(play_url.get('vheight')),
- })
-
- return formats
-
- def _real_extract(self, url):
- uu_mobj = re.search('uu=([\w]+)', url)
- vu_mobj = re.search('vu=([\w]+)', url)
-
- if not uu_mobj or not vu_mobj:
- raise ExtractorError('Invalid URL: %s' % url, expected=True)
-
- uu = uu_mobj.group(1)
- vu = vu_mobj.group(1)
- media_id = uu + '_' + vu
-
- formats = self._get_formats('flash', uu, vu, media_id) + self._get_formats('html5', uu, vu, media_id)
- self._sort_formats(formats)
-
- return {
- 'id': media_id,
- 'title': 'Video %s' % media_id,
- 'formats': formats,
- }
diff -Nru youtube-dl-2016.02.22/youtube_dl/extractor/libraryofcongress.py youtube-dl-2019.05.20/youtube_dl/extractor/libraryofcongress.py
--- youtube-dl-2016.02.22/youtube_dl/extractor/libraryofcongress.py 1970-01-01 00:00:00.000000000 +0000
+++ youtube-dl-2019.05.20/youtube_dl/extractor/libraryofcongress.py 2019-06-07 18:49:07.000000000 +0000
@@ -0,0 +1,153 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+ determine_ext,
+ float_or_none,
+ int_or_none,
+ parse_filesize,
+)
+
+
+class LibraryOfCongressIE(InfoExtractor):
+ IE_NAME = 'loc'
+ IE_DESC = 'Library of Congress'
+ _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P[0-9a-z_.]+)'
+ _TESTS = [{
+ # embedded via