From bfed4813b224f720e716de4d4f27471ca9053a0b Mon Sep 17 00:00:00 2001 From: linhua55 <5linhua5@gmail.com> Date: Sat, 18 Jul 2015 23:33:42 +0800 Subject: [PATCH 1/8] fix extraction for http://www.tudou.com/albumplay/cJAHGih4yYg.html --- youtube_dl/extractor/tudou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index c89de5ba4..8095e18d2 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -9,7 +9,7 @@ from .common import InfoExtractor class TudouIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P[^/?#]+?)(?:\.html)?/?(?:$|[?#])' + _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P[^/?#]+?)(?:\.html)?/?(?:$|[?#])' _TESTS = [{ 'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html', 'md5': '140a49ed444bd22f93330985d8475fcb', From 41ebd6530b124b9265a3df9d7d09aef02041b088 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 12 Sep 2015 22:42:57 +0800 Subject: [PATCH 2/8] [tudou] Add the test case (#6273) --- youtube_dl/extractor/tudou.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index e800477e2..950c42afb 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -27,6 +27,9 @@ class TudouIE(InfoExtractor): 'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012', 'thumbnail': 're:^https?://.*\.jpg$', } + }, { + 'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html', + 'only_matching': True, }] _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf' From 94e507aea798dac6974237cc44257dda45d5fa5a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 12 Sep 2015 22:45:09 +0800 Subject: [PATCH 3/8] [tudou] A more comprehensive _VALID_URL --- youtube_dl/extractor/tudou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 950c42afb..68712cb4a 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -9,7 +9,7 @@ from .common import InfoExtractor class TudouIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P[^/?#]+?)(?:\.html)?/?(?:$|[?#])' + _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P[^/?#]+?)(?:\.html)?/?(?:$|[?#])' _TESTS = [{ 'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html', 'md5': '140a49ed444bd22f93330985d8475fcb', From 141ba36996f77a420df69903a59792f6f93ae314 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 12 Sep 2015 22:51:49 +0800 Subject: [PATCH 4/8] [tudou] Modernize --- youtube_dl/extractor/tudou.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 68712cb4a..c9d80a7ef 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -2,9 +2,6 @@ from __future__ import unicode_literals -import re -import json - from .common import InfoExtractor @@ -46,13 +43,10 @@ class TudouIE(InfoExtractor): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage) - if m and m.group(1): - return { - '_type': 'url', - 'url': 'youku:' + m.group(1), - 'ie_key': 'Youku' - } + youku_vcode = self._search_regex( + r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None) + if youku_vcode: + return self.url_result('youku:' + youku_vcode, ie='Youku') title = self._search_regex( r",kw:\s*['\"](.+?)[\"']", webpage, 'title') @@ -63,8 +57,8 @@ class TudouIE(InfoExtractor): r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']", webpage, 'player URL', default=self._PLAYER_URL) - segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments') - segments = json.loads(segs_json) + segments = self._parse_json(self._search_regex( + r'segs: \'(.*)\'', webpage, 'segments'), video_id) # It looks like the keys are the arguments that have to be passed as # the hd field in the request url, we pick the higher # Also, filter non-number qualities (see issue #3643). From aab135516b288f24c55178b024976fd3e130c7b8 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 12 Sep 2015 22:52:51 +0800 Subject: [PATCH 5/8] [tudou] Avoid shadowing builtin names --- youtube_dl/extractor/tudou.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index c9d80a7ef..6116b209d 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -31,11 +31,11 @@ class TudouIE(InfoExtractor): _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf' - def _url_for_id(self, id, quality=None): - info_url = "http://v2.tudou.com/f?id=" + str(id) + def _url_for_id(self, video_id, quality=None): + info_url = "http://v2.tudou.com/f?id=" + str(video_id) if quality: info_url += '&hd' + quality - webpage = self._download_webpage(info_url, id, "Opening the info webpage") + webpage = self._download_webpage(info_url, video_id, "Opening the info webpage") final_url = self._html_search_regex('>(.+?)', webpage, 'video url') return final_url From 87813a857009dc3c3dfcc421679e5e806d363863 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Sep 2015 02:36:51 +0800 Subject: [PATCH 6/8] [tudou] Use _download_xml --- youtube_dl/extractor/tudou.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 6116b209d..3b993192c 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -35,8 +35,8 @@ class TudouIE(InfoExtractor): info_url = "http://v2.tudou.com/f?id=" + str(video_id) if quality: info_url += '&hd' + quality - webpage = self._download_webpage(info_url, video_id, "Opening the info webpage") - final_url = self._html_search_regex('>(.+?)', webpage, 'video url') + xml_data = self._download_xml(info_url, video_id, "Opening the info XML page") + final_url = xml_data.text return final_url def _real_extract(self, url): From 349b3a2ea0d6c264facacd92508516e8530108b2 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Sep 2015 02:51:20 +0800 Subject: [PATCH 7/8] [tudou] Improve regexs --- youtube_dl/extractor/tudou.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 3b993192c..53ba8511f 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -44,21 +44,21 @@ class TudouIE(InfoExtractor): webpage = self._download_webpage(url, video_id) youku_vcode = self._search_regex( - r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None) + r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None) if youku_vcode: return self.url_result('youku:' + youku_vcode, ie='Youku') title = self._search_regex( - r",kw:\s*['\"](.+?)[\"']", webpage, 'title') + r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title') thumbnail_url = self._search_regex( - r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False) + r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False) player_url = self._search_regex( - r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']", + r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]', webpage, 'player URL', default=self._PLAYER_URL) segments = self._parse_json(self._search_regex( - r'segs: \'(.*)\'', webpage, 'segments'), video_id) + r'segs: \'([^\']+)\'', webpage, 'segments'), video_id) # It looks like the keys are the arguments that have to be passed as # the hd field in the request url, we pick the higher # Also, filter non-number qualities (see issue #3643). From b264c2130221912adfc7cc35d73c2a88d79eafeb Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Sep 2015 02:57:14 +0800 Subject: [PATCH 8/8] [tudou] Use single quotes and compat_str --- youtube_dl/extractor/tudou.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py index 53ba8511f..5f7ac4b35 100644 --- a/youtube_dl/extractor/tudou.py +++ b/youtube_dl/extractor/tudou.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from .common import InfoExtractor +from ..compat import compat_str class TudouIE(InfoExtractor): @@ -32,7 +33,7 @@ class TudouIE(InfoExtractor): _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf' def _url_for_id(self, video_id, quality=None): - info_url = "http://v2.tudou.com/f?id=" + str(video_id) + info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id) if quality: info_url += '&hd' + quality xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")