From f870544302f75bee0d96f6a8623c8ff270beca89 Mon Sep 17 00:00:00 2001 From: fnord Date: Mon, 13 Jul 2015 07:41:38 -0500 Subject: [PATCH 1/7] Add support for democracynow.org Supports downloading clips or entire shows. Subtitle support --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/democracynow.py | 100 +++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 youtube_dl/extractor/democracynow.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index cbaa07391d..5cc03b8758 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -112,6 +112,7 @@ from .daum import DaumIE from .dbtv import DBTVIE from .dctp import DctpTvIE from .deezer import DeezerPlaylistIE +from .democracynow import DemocracynowIE from .dfb import DFBIE from .dhm import DHMIE from .dotsub import DotsubIE diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py new file mode 100644 index 0000000000..1c9b360521 --- /dev/null +++ b/youtube_dl/extractor/democracynow.py @@ -0,0 +1,100 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json +import time +import hmac +import hashlib +import itertools +import re +from ..utils import ( + ExtractorError, + int_or_none, + parse_age_limit, + parse_iso8601, +) +from ..compat import compat_urllib_request +from .common import InfoExtractor + + +class DemocracynowIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P[^\?]*)' + IE_NAME = 'democracynow' + _TESTS = [{ + 'url': 'http://www.democracynow.org/shows/2015/7/3', + 'info_dict': { + 'id': '2015-0703-001', + 'ext': 'mp4', + 'title': 'July 03, 2015 - Democracy Now!', + 'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs', + 'uploader': 'Democracy Now', + 'upload_date': None, + }, + },{ + 'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree', + 'info_dict': { + 'id': '2015-0703-001', + 'ext': 'mp4', + 'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag', + 'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21', + 'uploader': 'Democracy Now', + 'upload_date': None, + }, + + }] + + def _real_extract(self, url): + display_id = self._match_id(url) + base_host = re.search(r'^(.+?://[^/]+)', url).group(1) + if display_id == '': + display_id = 'home' + webpage = self._download_webpage(url, display_id) + re_desc = re.search(r'[^/]+)/(?:dn)?(?P[^/]+?)\.(?P[^\.\?]+)(?P\?|$)',url) + if video_id == None: + video_id = purl.group('fn') + if js.get('start') != None: + url += '&' if purl.group('hasparams') == '?' else '?' + url = url + 'start='+str(js.get('start')) + formats.append({ + 'format_id': purl.group('dir'), + 'ext': purl.group('ext'), + 'url': url, + }) + self._sort_formats(formats) + ret = { + 'id': video_id, + 'title': js.get('title'), + 'description': description, + 'uploader': 'Democracy Now', +# 'thumbnails': thumbnails, + 'subtitles': subtitles, + 'formats': formats, + } + return ret +# \ No newline at end of file From eb08081330f5ef52d66140589137ae1bb05eee5f Mon Sep 17 00:00:00 2001 From: fnord Date: Fri, 17 Jul 2015 02:57:08 -0500 Subject: [PATCH 2/7] democracynow: correct syntax --- youtube_dl/extractor/democracynow.py | 43 +++++++++------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 1c9b360521..973bb437b1 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -1,19 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals -import json -import time -import hmac -import hashlib -import itertools import re -from ..utils import ( - ExtractorError, - int_or_none, - parse_age_limit, - parse_iso8601, -) -from ..compat import compat_urllib_request from .common import InfoExtractor @@ -30,7 +18,7 @@ class DemocracynowIE(InfoExtractor): 'uploader': 'Democracy Now', 'upload_date': None, }, - },{ + }, { 'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree', 'info_dict': { 'id': '2015-0703-001', @@ -40,7 +28,6 @@ class DemocracynowIE(InfoExtractor): 'uploader': 'Democracy Now', 'upload_date': None, }, - }] def _real_extract(self, url): @@ -49,7 +36,7 @@ class DemocracynowIE(InfoExtractor): if display_id == '': display_id = 'home' webpage = self._download_webpage(url, display_id) - re_desc = re.search(r'[^/]+)/(?:dn)?(?P[^/]+?)\.(?P[^\.\?]+)(?P\?|$)',url) - if video_id == None: + purl = re.search(r'/(?P[^/]+)/(?:dn)?(?P[^/]+?)\.(?P[^\.\?]+)(?P\?|$)', url) + if video_id is None: video_id = purl.group('fn') - if js.get('start') != None: + if js.get('start') is not None: url += '&' if purl.group('hasparams') == '?' else '?' - url = url + 'start='+str(js.get('start')) + url = url + 'start=' + str(js.get('start')) formats.append({ 'format_id': purl.group('dir'), 'ext': purl.group('ext'), @@ -92,9 +79,7 @@ class DemocracynowIE(InfoExtractor): 'title': js.get('title'), 'description': description, 'uploader': 'Democracy Now', -# 'thumbnails': thumbnails, 'subtitles': subtitles, 'formats': formats, } return ret -# \ No newline at end of file From 78d7ee19dc417b16b26fe2fa1101124866727a85 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 31 Oct 2015 22:21:52 +0800 Subject: [PATCH 3/7] [democracynow] Fix _TESTS --- youtube_dl/extractor/democracynow.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 973bb437b1..05cfc7502d 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -36,10 +36,9 @@ class DemocracynowIE(InfoExtractor): if display_id == '': display_id = 'home' webpage = self._download_webpage(url, display_id) - re_desc = re.search(r']+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json') js = self._parse_json(jstr, display_id) video_id = None formats = [] @@ -56,7 +55,7 @@ class DemocracynowIE(InfoExtractor): 'ext': ext, 'url': url, }] - for key in ('file', 'audio'): + for key in ('file', 'audio', 'video'): url = js.get(key, '') if url == '' or url is None: continue From dde9fe9788f23f168e0bddaf8ab0470f469165fa Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 3 Nov 2015 18:36:54 +0800 Subject: [PATCH 4/7] [democracynow] Simplify --- youtube_dl/extractor/democracynow.py | 86 ++++++++++++++-------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 05cfc7502d..824b8e2c5c 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -2,11 +2,18 @@ from __future__ import unicode_literals import re +import os.path + from .common import InfoExtractor +from ..compat import compat_urlparse +from ..utils import ( + url_basename, + remove_start, +) class DemocracynowIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P[^\?]*)' + _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P[^\?]*)' IE_NAME = 'democracynow' _TESTS = [{ 'url': 'http://www.democracynow.org/shows/2015/7/3', @@ -14,9 +21,7 @@ class DemocracynowIE(InfoExtractor): 'id': '2015-0703-001', 'ext': 'mp4', 'title': 'July 03, 2015 - Democracy Now!', - 'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs', - 'uploader': 'Democracy Now', - 'upload_date': None, + 'description': 'A daily independent global news hour with Amy Goodman & Juan González "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs', }, }, { 'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree', @@ -25,60 +30,57 @@ class DemocracynowIE(InfoExtractor): 'ext': 'mp4', 'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag', 'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21', - 'uploader': 'Democracy Now', - 'upload_date': None, }, }] def _real_extract(self, url): display_id = self._match_id(url) - base_host = re.search(r'^(.+?://[^/]+)', url).group(1) - if display_id == '': - display_id = 'home' webpage = self._download_webpage(url, display_id) description = self._og_search_description(webpage) - jstr = self._search_regex(r']+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json') - js = self._parse_json(jstr, display_id) + js = self._parse_json(self._search_regex( + r']+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'), + display_id) video_id = None formats = [] + + default_lang = 'en' + subtitles = {} - for key in ('caption_file', '.......'): - # ....... = pending vtt support that doesn't clobber srt 'chapter_file': - url = js.get(key, '') - if url == '' or url is None: - continue - if not re.match(r'^https?://', url): - url = base_host + url - ext = re.search(r'\.([^\.]+)$', url).group(1) - subtitles['eng'] = [{ - 'ext': ext, - 'url': url, - }] - for key in ('file', 'audio', 'video'): - url = js.get(key, '') - if url == '' or url is None: - continue - if not re.match(r'^https?://', url): - url = base_host + url - purl = re.search(r'/(?P[^/]+)/(?:dn)?(?P[^/]+?)\.(?P[^\.\?]+)(?P\?|$)', url) - if video_id is None: - video_id = purl.group('fn') - if js.get('start') is not None: - url += '&' if purl.group('hasparams') == '?' else '?' - url = url + 'start=' + str(js.get('start')) - formats.append({ - 'format_id': purl.group('dir'), - 'ext': purl.group('ext'), - 'url': url, + + def add_subtitle_item(lang, info_dict): + if lang not in subtitles: + subtitles[lang] = [] + subtitles[lang].append(info_dict) + + # chapter_file are not subtitles + if 'caption_file' in js: + add_subtitle_item(default_lang, { + 'url': compat_urlparse.urljoin(url, js['caption_file']), }) + + for subtitle_item in js.get('captions', []): + lang = subtitle_item.get('language', '').lower() or default_lang + add_subtitle_item(lang, { + 'url': compat_urlparse.urljoin(url, subtitle_item['url']), + }) + + for key in ('file', 'audio', 'video'): + media_url = js.get(key, '') + if not media_url: + continue + media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url)) + video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn') + formats.append({ + 'url': media_url, + }) + self._sort_formats(formats) - ret = { + + return { 'id': video_id, 'title': js.get('title'), 'description': description, - 'uploader': 'Democracy Now', 'subtitles': subtitles, 'formats': formats, } - return ret From fc68d52bb95dc81ed3d05a5c5397cd3f35ee093a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 3 Nov 2015 21:24:10 +0800 Subject: [PATCH 5/7] [democracynow] Add MD5 sums --- youtube_dl/extractor/democracynow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 824b8e2c5c..70c364e8b0 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -17,6 +17,7 @@ class DemocracynowIE(InfoExtractor): IE_NAME = 'democracynow' _TESTS = [{ 'url': 'http://www.democracynow.org/shows/2015/7/3', + 'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d', 'info_dict': { 'id': '2015-0703-001', 'ext': 'mp4', @@ -25,6 +26,7 @@ class DemocracynowIE(InfoExtractor): }, }, { 'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree', + 'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d', 'info_dict': { 'id': '2015-0703-001', 'ext': 'mp4', From fd8102820c4d14fdb1ff7e090553211717012f67 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 4 Nov 2015 00:09:55 +0800 Subject: [PATCH 6/7] [democracynow] Rename js to json_data --- youtube_dl/extractor/democracynow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 70c364e8b0..72fc75d80c 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -40,7 +40,7 @@ class DemocracynowIE(InfoExtractor): webpage = self._download_webpage(url, display_id) description = self._og_search_description(webpage) - js = self._parse_json(self._search_regex( + json_data = self._parse_json(self._search_regex( r']+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'), display_id) video_id = None @@ -56,19 +56,19 @@ class DemocracynowIE(InfoExtractor): subtitles[lang].append(info_dict) # chapter_file are not subtitles - if 'caption_file' in js: + if 'caption_file' in json_data: add_subtitle_item(default_lang, { - 'url': compat_urlparse.urljoin(url, js['caption_file']), + 'url': compat_urlparse.urljoin(url, json_data['caption_file']), }) - for subtitle_item in js.get('captions', []): + for subtitle_item in json_data.get('captions', []): lang = subtitle_item.get('language', '').lower() or default_lang add_subtitle_item(lang, { 'url': compat_urlparse.urljoin(url, subtitle_item['url']), }) for key in ('file', 'audio', 'video'): - media_url = js.get(key, '') + media_url = json_data.get(key, '') if not media_url: continue media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url)) @@ -81,7 +81,7 @@ class DemocracynowIE(InfoExtractor): return { 'id': video_id, - 'title': js.get('title'), + 'title': json_data.get('title'), 'description': description, 'subtitles': subtitles, 'formats': formats, From 0aeb9a106e1aad37967e0ee666ed816a7d5eb7c2 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 4 Nov 2015 00:13:00 +0800 Subject: [PATCH 7/7] [democracynow] Prevent required fields to be None --- youtube_dl/extractor/democracynow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py index 72fc75d80c..6cd395e116 100644 --- a/youtube_dl/extractor/democracynow.py +++ b/youtube_dl/extractor/democracynow.py @@ -80,8 +80,8 @@ class DemocracynowIE(InfoExtractor): self._sort_formats(formats) return { - 'id': video_id, - 'title': json_data.get('title'), + 'id': video_id or display_id, + 'title': json_data['title'], 'description': description, 'subtitles': subtitles, 'formats': formats,