fenix
/
ytdl-nightly
mirror of https://github.com/ytdl-org/ytdl-nightly.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[stanfordoc] Modernize

This commit is contained in:
Philipp Hagemeister 2014-11-20 16:34:54 +01:00
parent c9bf41145f
commit f42c190769
1 changed files with 37 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
youtube_dl/extractor/stanfordoc.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
@ -9,24 +11,23 @@ from ..utils import (
class StanfordOpenClassroomIE(InfoExtractor): class StanfordOpenClassroomIE(InfoExtractor):
IE_NAME = u'stanfordoc' IE_NAME = 'stanfordoc'
IE_DESC = u'Stanford Open ClassRoom' IE_DESC = 'Stanford Open ClassRoom'
_VALID_URL = r'^(?:https?://)?openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' _VALID_URL = r'https?://openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
_TEST = { _TEST = {
u'url': u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100', 'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
u'file': u'PracticalUnix_intro-environment.mp4', 'md5': '544a9468546059d4e80d76265b0443b8',
u'md5': u'544a9468546059d4e80d76265b0443b8', 'info_dict': {
u'info_dict': { 'id': 'PracticalUnix_intro-environment',
u"title": u"Intro Environment" 'ext': 'mp4',
'title': 'Intro Environment',
} }
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
if mobj.group('course') and mobj.group('video'): # A specific video if mobj.group('course') and mobj.group('video'): # A specific video
course = mobj.group('course') course = mobj.group('course')
video = mobj.group('video') video = mobj.group('video')
info = { info = {
@ -35,7 +36,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
'upload_date': None, 'upload_date': None,
} }
self.report_extraction(info['id'])
baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
xmlUrl = baseUrl + video + '.xml' xmlUrl = baseUrl + video + '.xml'
mdoc = self._download_xml(xmlUrl, info['id']) mdoc = self._download_xml(xmlUrl, info['id'])
@ -43,63 +43,49 @@ class StanfordOpenClassroomIE(InfoExtractor):
info['title'] = mdoc.findall('./title')[0].text info['title'] = mdoc.findall('./title')[0].text
info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
except IndexError: except IndexError:
raise ExtractorError(u'Invalid metadata XML file') raise ExtractorError('Invalid metadata XML file')
info['ext'] = info['url'].rpartition('.')[2] return info
return [info] elif mobj.group('course'): # A course page
elif mobj.group('course'): # A course page
course = mobj.group('course') course = mobj.group('course')
info = { info = {
'id': course, 'id': course,
'type': 'playlist', '_type': 'playlist',
'uploader': None, 'uploader': None,
'upload_date': None, 'upload_date': None,
} }
coursepage = self._download_webpage(url, info['id'], coursepage = self._download_webpage(
note='Downloading course info page', url, info['id'],
errnote='Unable to download course info page') note='Downloading course info page',
errnote='Unable to download course info page')
info['title'] = self._html_search_regex('<h1>([^<]+)</h1>', coursepage, 'title', default=info['id']) info['title'] = self._html_search_regex(
r'<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
info['description'] = self._html_search_regex('<description>([^<]+)</description>', info['description'] = self._html_search_regex(
coursepage, u'description', fatal=False) r'(?s)<description>([^<]+)</description>',
coursepage, 'description', fatal=False)
links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
info['list'] = [ info['entries'] = [self.url_result(
{ 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
'type': 'reference', ) for l in links]
'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), return info
} else: # Root page
for vpage in links]
results = []
for entry in info['list']:
assert entry['type'] == 'reference'
results += self.extract(entry['url'])
return results
else: # Root page
info = { info = {
'id': 'Stanford OpenClassroom', 'id': 'Stanford OpenClassroom',
'type': 'playlist', '_type': 'playlist',
'uploader': None, 'uploader': None,
'upload_date': None, 'upload_date': None,
} }
info['title'] = info['id']
rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
rootpage = self._download_webpage(rootURL, info['id'], rootpage = self._download_webpage(rootURL, info['id'],
errnote=u'Unable to download course info page') errnote='Unable to download course info page')
info['title'] = info['id']
links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage)) links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
info['list'] = [ info['entries'] = [self.url_result(
{ 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
'type': 'reference', ) for l in links]
'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), return info
}
for cpage in links]
results = []
for entry in info['list']:
assert entry['type'] == 'reference'
results += self.extract(entry['url'])
return results