From 957e0db1d2d1d4b2e9fdd1c314b0c46e68ca0cb3 Mon Sep 17 00:00:00 2001 From: remitamine Date: Fri, 30 Oct 2015 13:56:21 +0100 Subject: [PATCH 1/5] [baidu] improve info extraction --- youtube_dl/extractor/baidu.py | 45 +++++++++++++---------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index e37ee44403..84fab551bf 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -14,8 +14,8 @@ class BaiduVideoIE(InfoExtractor): 'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6', 'info_dict': { 'id': '1069', - 'title': '中华小当家 TV版 (全52集)', - 'description': 'md5:395a419e41215e531c857bb037bbaf80', + 'title': '中华小当家 TV版国语', + 'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0', }, 'playlist_count': 52, }, { @@ -25,45 +25,32 @@ class BaiduVideoIE(InfoExtractor): 'title': 're:^奔跑吧兄弟', 'description': 'md5:1bf88bad6d850930f542d51547c089b8', }, - 'playlist_mincount': 3, + 'playlist_mincount': 12, }] + def _call_api(self, path, category, playlist_id): + return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id) + def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - category = category2 = mobj.group('type') + category, playlist_id = re.match(self._VALID_URL, url).groups() if category == 'show': - category2 = 'tvshow' + category = 'tvshow' + if category == 'tv': + category = 'tvplay' - webpage = self._download_webpage(url, playlist_id) + playlist_detail = self._call_api('xqinfo', category, playlist_id) - playlist_title = self._html_search_regex( - r'title\s*:\s*(["\'])(?P[^\']+)\1', webpage, - 'playlist title', group='title') - playlist_description = self._html_search_regex( - r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage, - playlist_id, 'playlist description') + playlist_title = playlist_detail['title'] + playlist_description = playlist_detail.get('intro') - site = self._html_search_regex( - r'filterSite\s*:\s*["\']([^"]*)["\']', webpage, - 'primary provider site') - api_result = self._download_json( - 'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % ( - category, category2, playlist_id, site), - playlist_id, 'Get playlist links') + episodes_detail = self._call_api('xqsingle', category, playlist_id) entries = [] - for episode in api_result[0]['episodes']: + for episode in episodes_detail['videos']: episode_id = '%s_%s' % (playlist_id, episode['episode']) - redirect_page = self._download_webpage( - compat_urlparse.urljoin(url, episode['url']), episode_id, - note='Download Baidu redirect page') - real_url = self._html_search_regex( - r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL') - entries.append(self.url_result( - real_url, video_title=episode['single_title'])) + episode['url'], video_title=episode['title'])) return self.playlist_result( entries, playlist_id, playlist_title, playlist_description) From c579c5e967cd223baa2c1f593b6ed65ac8643e42 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:31:02 +0800 Subject: [PATCH 2/5] [baidu] Cleanups --- youtube_dl/extractor/baidu.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index 84fab551bf..734b50d1d9 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urlparse class BaiduVideoIE(InfoExtractor): @@ -47,8 +46,6 @@ class BaiduVideoIE(InfoExtractor): entries = [] for episode in episodes_detail['videos']: - episode_id = '%s_%s' % (playlist_id, episode['episode']) - entries.append(self.url_result( episode['url'], video_title=episode['title'])) From a1d9f6c5dc15f1d62e0ab8c9dd6feaed4ff83a34 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:36:35 +0800 Subject: [PATCH 3/5] [baidu] Improve playlist description --- youtube_dl/extractor/baidu.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index 734b50d1d9..d60cb0fdd3 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..utils import unescapeHTML class BaiduVideoIE(InfoExtractor): @@ -14,7 +15,7 @@ class BaiduVideoIE(InfoExtractor): 'info_dict': { 'id': '1069', 'title': '中华小当家 TV版国语', - 'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0', + 'description': 'md5:51be07afe461cf99fa61231421b5397c', }, 'playlist_count': 52, }, { @@ -40,7 +41,7 @@ class BaiduVideoIE(InfoExtractor): playlist_detail = self._call_api('xqinfo', category, playlist_id) playlist_title = playlist_detail['title'] - playlist_description = playlist_detail.get('intro') + playlist_description = unescapeHTML(playlist_detail.get('intro')) episodes_detail = self._call_api('xqsingle', category, playlist_id) From 03116772583103ba97eaf8ce2cbabba9742e1929 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:44:49 +0800 Subject: [PATCH 4/5] [baidu] Add notes for API calls --- youtube_dl/extractor/baidu.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index d60cb0fdd3..d35116a7cb 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -28,8 +28,9 @@ class BaiduVideoIE(InfoExtractor): 'playlist_mincount': 12, }] - def _call_api(self, path, category, playlist_id): - return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id) + def _call_api(self, path, category, playlist_id, note): + return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % ( + path, category, playlist_id), playlist_id, note) def _real_extract(self, url): category, playlist_id = re.match(self._VALID_URL, url).groups() @@ -38,12 +39,14 @@ class BaiduVideoIE(InfoExtractor): if category == 'tv': category = 'tvplay' - playlist_detail = self._call_api('xqinfo', category, playlist_id) + playlist_detail = self._call_api( + 'xqinfo', category, playlist_id, 'Download playlist JSON metadata') playlist_title = playlist_detail['title'] playlist_description = unescapeHTML(playlist_detail.get('intro')) - episodes_detail = self._call_api('xqsingle', category, playlist_id) + episodes_detail = self._call_api( + 'xqsingle', category, playlist_id, 'Download episodes JSON metadata') entries = [] for episode in episodes_detail['videos']: From b7546397f080e65fed42c69701f423e4f57f43f5 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan <yan12125@gmail.com> Date: Sat, 2 Jan 2016 21:46:40 +0800 Subject: [PATCH 5/5] [baidu] Use list comprehension --- youtube_dl/extractor/baidu.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py index d35116a7cb..76b21e5962 100644 --- a/youtube_dl/extractor/baidu.py +++ b/youtube_dl/extractor/baidu.py @@ -48,10 +48,9 @@ class BaiduVideoIE(InfoExtractor): episodes_detail = self._call_api( 'xqsingle', category, playlist_id, 'Download episodes JSON metadata') - entries = [] - for episode in episodes_detail['videos']: - entries.append(self.url_result( - episode['url'], video_title=episode['title'])) + entries = [self.url_result( + episode['url'], video_title=episode['title'] + ) for episode in episodes_detail['videos']] return self.playlist_result( entries, playlist_id, playlist_title, playlist_description)