From a229909fa616c8b2820ea3b254861f07bedcd376 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 3 Aug 2014 21:24:44 +0300 Subject: [PATCH 1/2] [jove] Add new extractor. Closes #3177 --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/jove.py | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 youtube_dl/extractor/jove.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2bf8bc5e86..c80a1bd483 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -144,6 +144,7 @@ from .ivi import ( from .izlesene import IzleseneIE from .jadorecettepub import JadoreCettePubIE from .jeuxvideo import JeuxVideoIE +from .jove import JoveIE from .jukebox import JukeboxIE from .justintv import JustinTVIE from .jpopsukitv import JpopsukiIE diff --git a/youtube_dl/extractor/jove.py b/youtube_dl/extractor/jove.py new file mode 100644 index 0000000000..a7110b6838 --- /dev/null +++ b/youtube_dl/extractor/jove.py @@ -0,0 +1,64 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +from datetime import datetime + +from .common import InfoExtractor +from ..utils import determine_ext, ExtractorError + + +class JoveIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P[0-9]+)' + _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}' + _TEST = { + 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current', + 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b', + 'info_dict': { + 'id': '2744', + 'ext': 'mp4', + 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation', + 'description': 'Transcranial direct current stimulation (tDCS) is an established technique to modulate cortical excitability1,2. It has been ...', + 'thumbnail': 're:^https?://.*\.png$', + 'upload_date': '20110523', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + title = self._html_search_meta('citation_title', webpage, 'title') + thumbnail = self._og_search_thumbnail(webpage) + description = self._html_search_meta( + 'description', webpage, 'description', fatal=False) + publish_date = self._html_search_meta( + 'citation_publication_date', webpage, 'publish date', fatal=False) + if publish_date: + publish_date = datetime.strptime(publish_date, + '%Y/%m/%d').strftime('%Y%m%d') + + # Not the same as video_id. + chapters_id = self._html_search_regex( + r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id') + chapters_xml = self._download_xml( + self._CHAPTERS_URL.format(video_id=chapters_id), + video_id, note='Downloading chapter XML', + errnote='Failed to download chapter XML' + ) + video_url = chapters_xml.attrib.get('video') + if not video_url: + raise ExtractorError('Failed to get the video URL') + + ext = determine_ext(video_url) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'ext': ext, + 'thumbnail': thumbnail, + 'description': description, + 'upload_date': publish_date, + } From fe556f1b0cfd5782ec379a731f4b8879f2a352a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 19 Aug 2014 20:02:08 +0700 Subject: [PATCH 2/2] [jove] Simplify, extract full description and add test for video that requires subscription --- youtube_dl/extractor/jove.py | 76 ++++++++++++++++++++++-------------- youtube_dl/utils.py | 1 + 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/youtube_dl/extractor/jove.py b/youtube_dl/extractor/jove.py index a7110b6838..cf73cd7533 100644 --- a/youtube_dl/extractor/jove.py +++ b/youtube_dl/extractor/jove.py @@ -1,64 +1,80 @@ -# coding: utf-8 from __future__ import unicode_literals import re -from datetime import datetime from .common import InfoExtractor -from ..utils import determine_ext, ExtractorError +from ..utils import ( + ExtractorError, + unified_strdate +) class JoveIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P[0-9]+)' _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}' - _TEST = { - 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current', - 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b', - 'info_dict': { - 'id': '2744', - 'ext': 'mp4', - 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation', - 'description': 'Transcranial direct current stimulation (tDCS) is an established technique to modulate cortical excitability1,2. It has been ...', - 'thumbnail': 're:^https?://.*\.png$', - 'upload_date': '20110523', - } - } + _TESTS = [ + { + 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current', + 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b', + 'info_dict': { + 'id': '2744', + 'ext': 'mp4', + 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation', + 'description': 'md5:015dd4509649c0908bc27f049e0262c6', + 'thumbnail': 're:^https?://.*\.png$', + 'upload_date': '20110523', + } + }, + { + 'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation', + 'md5': '914aeb356f416811d911996434811beb', + 'info_dict': { + 'id': '51796', + 'ext': 'mp4', + 'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment', + 'description': 'md5:35ff029261900583970c4023b70f1dc9', + 'thumbnail': 're:^https?://.*\.png$', + 'upload_date': '20140802', + } + }, + + ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) - title = self._html_search_meta('citation_title', webpage, 'title') - thumbnail = self._og_search_thumbnail(webpage) - description = self._html_search_meta( - 'description', webpage, 'description', fatal=False) - publish_date = self._html_search_meta( - 'citation_publication_date', webpage, 'publish date', fatal=False) - if publish_date: - publish_date = datetime.strptime(publish_date, - '%Y/%m/%d').strftime('%Y%m%d') - # Not the same as video_id. chapters_id = self._html_search_regex( r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id') + chapters_xml = self._download_xml( self._CHAPTERS_URL.format(video_id=chapters_id), - video_id, note='Downloading chapter XML', - errnote='Failed to download chapter XML' - ) + video_id, note='Downloading chapters XML', + errnote='Failed to download chapters XML') + video_url = chapters_xml.attrib.get('video') if not video_url: raise ExtractorError('Failed to get the video URL') - ext = determine_ext(video_url) + title = self._html_search_meta('citation_title', webpage, 'title') + thumbnail = self._og_search_thumbnail(webpage) + description = self._html_search_regex( + r'

(.+?)

', + webpage, 'description', fatal=False) + publish_date = unified_strdate(self._html_search_meta( + 'citation_publication_date', webpage, 'publish date', fatal=False)) + comment_count = self._html_search_regex( + r'