From 643fe72717e6b9c45af4528e46dfc181cad7aebb Mon Sep 17 00:00:00 2001 From: Oskar Jauch Date: Sat, 28 Mar 2015 10:38:52 +0100 Subject: [PATCH 1/3] [DHM] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/dhm.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 youtube_dl/extractor/dhm.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index d56eb64488..a65c0c25b8 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -106,6 +106,7 @@ from .dbtv import DBTVIE from .dctp import DctpTvIE from .deezer import DeezerPlaylistIE from .dfb import DFBIE +from .dhm import DHMIE from .dotsub import DotsubIE from .douyutv import DouyuTVIE from .dreisat import DreiSatIE diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py new file mode 100644 index 0000000000..d379c9d532 --- /dev/null +++ b/youtube_dl/extractor/dhm.py @@ -0,0 +1,52 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + +import urllib2 +import xml.etree.ElementTree as ET +import re + + +class DHMIE(InfoExtractor): + _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P.*?)' + + _TEST = { + 'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/', + 'md5': '11c475f670209bf6acca0b2b7ef51827', + 'info_dict': { + 'id': 'marshallwg', + 'ext': 'flv', + 'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE', + 'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg', + } + } + + def _real_extract(self, url): + video_id = '' + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex( + r'dc:title=\"(.*?)\"', webpage, 'title') + + playlist_url = self._html_search_regex( + r'file: \'(.*?)\'', webpage, 'playlist URL') + + xml_file = urllib2.urlopen(playlist_url) + data = xml_file.read() + xml_file.close() + + root = ET.fromstring(data) + video_url = root[0][0][0].text + thumbnail = root[0][0][2].text + + m = re.search('video/(.+?).flv', video_url) + if m: + video_id = m.group(1) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'thumbnail': thumbnail, + } From ff79552f13efbfa87a814dd0a950aa797b08a5cd Mon Sep 17 00:00:00 2001 From: Oskar Jauch Date: Sat, 28 Mar 2015 10:42:35 +0100 Subject: [PATCH 2/3] [DHM] Add extractor description --- youtube_dl/extractor/dhm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py index d379c9d532..a0a584f6a5 100644 --- a/youtube_dl/extractor/dhm.py +++ b/youtube_dl/extractor/dhm.py @@ -9,6 +9,7 @@ import re class DHMIE(InfoExtractor): + IE_DESC = 'Deutsches Historisches Museum' _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P.*?)' _TEST = { From af8c93086c45cf61cadc8571644713927659a65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Mar 2015 22:30:13 +0600 Subject: [PATCH 3/3] [dhm] Simplify --- youtube_dl/extractor/dhm.py | 59 ++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py index a0a584f6a5..80ee400181 100644 --- a/youtube_dl/extractor/dhm.py +++ b/youtube_dl/extractor/dhm.py @@ -1,53 +1,64 @@ -# coding: utf-8 from __future__ import unicode_literals from .common import InfoExtractor - -import urllib2 -import xml.etree.ElementTree as ET -import re +from ..utils import ( + xpath_text, + parse_duration, +) class DHMIE(InfoExtractor): - IE_DESC = 'Deutsches Historisches Museum' - _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P.*?)' + IE_DESC = 'Filmarchiv - Deutsches Historisches Museum' + _VALID_URL = r'http://www\.dhm\.de/filmarchiv/die-filme/(?P[^/]+)' _TEST = { 'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/', 'md5': '11c475f670209bf6acca0b2b7ef51827', 'info_dict': { - 'id': 'marshallwg', + 'id': 'the-marshallplan-at-work-in-west-germany', 'ext': 'flv', 'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE', - 'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg', + 'description': 'md5:1fabd480c153f97b07add61c44407c82', + 'duration': 660, + 'thumbnail': 're:^https?://.*\.jpg$', } } def _real_extract(self, url): - video_id = '' + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) - title = self._html_search_regex( - r'dc:title=\"(.*?)\"', webpage, 'title') + playlist_url = self._search_regex( + r"file\s*:\s*'([^']+)'", webpage, 'playlist url') - playlist_url = self._html_search_regex( - r'file: \'(.*?)\'', webpage, 'playlist URL') + playlist = self._download_xml(playlist_url, video_id) - xml_file = urllib2.urlopen(playlist_url) - data = xml_file.read() - xml_file.close() + track = playlist.find( + './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track') - root = ET.fromstring(data) - video_url = root[0][0][0].text - thumbnail = root[0][0][2].text + video_url = xpath_text( + track, './{http://xspf.org/ns/0/}location', + 'video url', fatal=True) + thumbnail = xpath_text( + track, './{http://xspf.org/ns/0/}image', + 'thumbnail') - m = re.search('video/(.+?).flv', video_url) - if m: - video_id = m.group(1) + title = self._search_regex( + [r'dc:title="([^"]+)"', r' »([^<]+)'], + webpage, 'title').strip() + description = self._html_search_regex( + r'

Description:(.+?)

', + webpage, 'description', fatal=False) + duration = parse_duration(self._search_regex( + r'Length\s*\s*:\s*([^<]+)', + webpage, 'duration', fatal=False)) return { 'id': video_id, - 'title': title, 'url': video_url, + 'title': title, + 'description': description, + 'duration': duration, 'thumbnail': thumbnail, }