From 836a086ce9d48338444f010f690119a9a3998517 Mon Sep 17 00:00:00 2001 From: Allan Zhou Date: Mon, 19 Aug 2013 18:22:25 -0700 Subject: [PATCH 1/5] Add YouTube DASH formats to YouTubeIE --- youtube_dl/extractor/youtube.py | 96 ++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 843a973ca7..248105d7f2 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -155,11 +155,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Listed in order of quality _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13', '95', '94', '93', '92', '132', '151', - '85', '84', '102', '83', '101', '82', '100', + '85', '84', '102', '83', '101', '82', '100', # 3D + '138', '137', '136', '135', '134', '133', '160', # Dash video mp4 + '141', '140', '139', # Dash auido mp4 + '248', '247', '246', '245', '244', '243', '242', # Dash video webm + '172', '171', # Dash audio webm ] _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13', '95', '94', '93', '92', '132', '151', '85', '102', '84', '101', '83', '100', '82', + '248', '247', '246', '245', '244', '243', '242', # Dash video webm + '172', '171', # Dash audio webm + '138', '137', '136', '135', '134', '133', '160', # Dash video mp4 + '141', '140', '139', # Dash auido mp4 ] _video_extensions = { '13': '3gp', @@ -181,7 +189,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '100': 'webm', '101': 'webm', '102': 'webm', - + # videos that use m3u8 '92': 'mp4', '93': 'mp4', @@ -190,6 +198,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '96': 'mp4', '132': 'mp4', '151': 'mp4', + + # Dash mp4 + '133': 'mp4', + '134': 'mp4', + '135': 'mp4', + '136': 'mp4', + '137': 'mp4', + '138': 'mp4', + '139': 'mp4', + '140': 'mp4', + '141': 'mp4', + '160': 'mp4', + + # Dash webm + '171': 'webm', + '172': 'webm', + '242': 'webm', + '243': 'webm', + '244': 'webm', + '245': 'webm', + '246': 'webm', + '247': 'webm', + '248': 'webm', } _video_dimensions = { '5': '240x400', @@ -217,11 +248,58 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '96': '1080p', '100': '360p', '101': '480p', - '102': '720p', + '102': '720p', '132': '240p', '151': '72p', + '133': '240p', + '134': '360p', + '135': '480p', + '136': '720p', + '137': '1080p', + '138': '>1080p', + '139': '48k', + '140': '128k', + '141': '256k', + '160': '192p', + '171': '128k', + '172': '256k', + '242': '240p', + '243': '360p', + '244': '480p', + '245': '480p', + '246': '480p', + '247': '720p', + '248': '1080p', } - _3d_itags = ['85', '84', '102', '83', '101', '82', '100'] + _special_itags = { + '82': '3D', + '83': '3D', + '84': '3D', + '85': '3D', + '100': '3D', + '101': '3D', + '102': '3D', + '133': 'DASH Video', + '134': 'DASH Video', + '135': 'DASH Video', + '136': 'DASH Video', + '137': 'DASH Video', + '138': 'DASH Video', + '139': 'DASH Audio', + '140': 'DASH Audio', + '141': 'DASH Audio', + '160': 'DASH Video', + '171': 'DASH Audio', + '172': 'DASH Audio', + '242': 'DASH Video', + '243': 'DASH Video', + '244': 'DASH Video', + '245': 'DASH Video', + '246': 'DASH Video', + '247': 'DASH Video', + '248': 'DASH Video', + } + IE_NAME = u'youtube' _TESTS = [ { @@ -472,7 +550,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): for x in formats: print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'), - ' (3D)' if x in self._3d_itags else '')) + ' ('+self._special_itags[x]+')' if x in self._special_itags else '')) def _extract_id(self, url): mobj = re.match(self._VALID_URL, url, re.VERBOSE) @@ -699,6 +777,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if m_s is not None: self.to_screen(u'%s: Encrypted signatures detected.' % video_id) video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] + m_s = re.search(r'[&,]s=', args['adaptive_fmts']) + if m_s is not None: + video_info['url_encoded_fmt_stream_map'][0] += ','+args['adaptive_fmts'] + else: + video_info['url_encoded_fmt_stream_map'][0] += ','+video_info['adaptive_fmts'][0] + except ValueError: pass @@ -758,7 +842,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_format = '{0} - {1}{2}'.format(format_param if format_param else video_extension, self._video_dimensions.get(format_param, '???'), - ' (3D)' if format_param in self._3d_itags else '') + ' ('+self._special_itags[format_param]+')' if format_param in self._special_itags else '') results.append({ 'id': video_id, From 211fbc1328edda1752fce9dc5ed604b98f9dc865 Mon Sep 17 00:00:00 2001 From: Allan Zhou Date: Mon, 19 Aug 2013 18:57:55 -0700 Subject: [PATCH 2/5] fix failed tests --- youtube_dl/extractor/youtube.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 248105d7f2..bdd399d3e0 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -777,10 +777,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if m_s is not None: self.to_screen(u'%s: Encrypted signatures detected.' % video_id) video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] - m_s = re.search(r'[&,]s=', args['adaptive_fmts']) - if m_s is not None: + m_s = re.search(r'[&,]s=', args['adaptive_fmts'] if 'adaptive_fmts' in args else '') + if m_s is not None and 'adaptive_fmts' in args: video_info['url_encoded_fmt_stream_map'][0] += ','+args['adaptive_fmts'] - else: + elif 'adaptive_fmts' in video_info: video_info['url_encoded_fmt_stream_map'][0] += ','+video_info['adaptive_fmts'][0] except ValueError: From cde846b3d3f59029fc07ecd97e49cfae050af3c9 Mon Sep 17 00:00:00 2001 From: Allan Zhou Date: Tue, 20 Aug 2013 21:42:49 -0700 Subject: [PATCH 3/5] fix code style --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index bdd399d3e0..1cd2d40f18 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -777,7 +777,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if m_s is not None: self.to_screen(u'%s: Encrypted signatures detected.' % video_id) video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] - m_s = re.search(r'[&,]s=', args['adaptive_fmts'] if 'adaptive_fmts' in args else '') + m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u'')) if m_s is not None and 'adaptive_fmts' in args: video_info['url_encoded_fmt_stream_map'][0] += ','+args['adaptive_fmts'] elif 'adaptive_fmts' in video_info: From b7a68384078ec0d97fb3c8e4a3100e9c60f340d0 Mon Sep 17 00:00:00 2001 From: Allan Zhou Date: Tue, 20 Aug 2013 21:57:32 -0700 Subject: [PATCH 4/5] address review comment --- youtube_dl/extractor/youtube.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1cd2d40f18..e573b021d9 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -778,7 +778,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.to_screen(u'%s: Encrypted signatures detected.' % video_id) video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u'')) - if m_s is not None and 'adaptive_fmts' in args: + if 'url_encoded_fmt_stream_map' not in video_info or not video_info['url_encoded_fmt_stream_map']: + video_info['url_encoded_fmt_stream_map'] = [''] + if m_s is not None: video_info['url_encoded_fmt_stream_map'][0] += ','+args['adaptive_fmts'] elif 'adaptive_fmts' in video_info: video_info['url_encoded_fmt_stream_map'][0] += ','+video_info['adaptive_fmts'][0] From 37b6d5f684d409365bbac6d3f2b8074b57e643a8 Mon Sep 17 00:00:00 2001 From: Allan Zhou Date: Tue, 20 Aug 2013 23:51:05 -0700 Subject: [PATCH 5/5] fix hls test --- youtube_dl/extractor/youtube.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index e573b021d9..1599dd484c 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -778,13 +778,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.to_screen(u'%s: Encrypted signatures detected.' % video_id) video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']] m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u'')) - if 'url_encoded_fmt_stream_map' not in video_info or not video_info['url_encoded_fmt_stream_map']: - video_info['url_encoded_fmt_stream_map'] = [''] if m_s is not None: - video_info['url_encoded_fmt_stream_map'][0] += ','+args['adaptive_fmts'] + if 'url_encoded_fmt_stream_map' in video_info: + video_info['url_encoded_fmt_stream_map'][0] += ',' + args['adaptive_fmts'] + else: + video_info['url_encoded_fmt_stream_map'] = [args['adaptive_fmts']] elif 'adaptive_fmts' in video_info: - video_info['url_encoded_fmt_stream_map'][0] += ','+video_info['adaptive_fmts'][0] - + if 'url_encoded_fmt_stream_map' in video_info: + video_info['url_encoded_fmt_stream_map'][0] += ',' + video_info['adaptive_fmts'][0] + else: + video_info['url_encoded_fmt_stream_map'] = video_info['adaptive_fmts'] except ValueError: pass