From 34e3885bc9e3aecab104b96eabce03854ac8f7a2 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Thu, 17 Oct 2019 15:55:44 +0100 Subject: [PATCH] [viewster->contv] remove viewster extractor and add support for contv.com --- youtube_dl/extractor/contv.py | 118 ++++++++++++++++ youtube_dl/extractor/extractors.py | 2 +- youtube_dl/extractor/viewster.py | 217 ----------------------------- 3 files changed, 119 insertions(+), 218 deletions(-) create mode 100644 youtube_dl/extractor/contv.py delete mode 100644 youtube_dl/extractor/viewster.py diff --git a/youtube_dl/extractor/contv.py b/youtube_dl/extractor/contv.py new file mode 100644 index 000000000..84b462d40 --- /dev/null +++ b/youtube_dl/extractor/contv.py @@ -0,0 +1,118 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + float_or_none, + int_or_none, +) + + +class CONtvIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?contv\.com/details-movie/(?P[^/]+)' + _TESTS = [{ + 'url': 'https://www.contv.com/details-movie/CEG10022949/days-of-thrills-&-laughter', + 'info_dict': { + 'id': 'CEG10022949', + 'ext': 'mp4', + 'title': 'Days Of Thrills & Laughter', + 'description': 'md5:5d6b3d0b1829bb93eb72898c734802eb', + 'upload_date': '20180703', + 'timestamp': 1530634789.61, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, { + 'url': 'https://www.contv.com/details-movie/CLIP-show_fotld_bts/fight-of-the-living-dead:-behind-the-scenes-bites', + 'info_dict': { + 'id': 'CLIP-show_fotld_bts', + 'title': 'Fight of the Living Dead: Behind the Scenes Bites', + }, + 'playlist_mincount': 7, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + details = self._download_json( + 'http://metax.contv.live.junctiontv.net/metax/2.5/details/' + video_id, + video_id, query={'device': 'web'}) + + if details.get('type') == 'episodic': + seasons = self._download_json( + 'http://metax.contv.live.junctiontv.net/metax/2.5/seriesfeed/json/' + video_id, + video_id) + entries = [] + for season in seasons: + for episode in season.get('episodes', []): + episode_id = episode.get('id') + if not episode_id: + continue + entries.append(self.url_result( + 'https://www.contv.com/details-movie/' + episode_id, + CONtvIE.ie_key(), episode_id)) + return self.playlist_result(entries, video_id, details.get('title')) + + m_details = details['details'] + title = details['title'] + + formats = [] + + media_hls_url = m_details.get('media_hls_url') + if media_hls_url: + formats.extend(self._extract_m3u8_formats( + media_hls_url, video_id, 'mp4', + m3u8_id='hls', fatal=False)) + + media_mp4_url = m_details.get('media_mp4_url') + if media_mp4_url: + formats.append({ + 'format_id': 'http', + 'url': media_mp4_url, + }) + + self._sort_formats(formats) + + subtitles = {} + captions = m_details.get('captions') or {} + for caption_url in captions.values(): + subtitles.setdefault('en', []).append({ + 'url': caption_url + }) + + thumbnails = [] + for image in m_details.get('images', []): + image_url = image.get('url') + if not image_url: + continue + thumbnails.append({ + 'url': image_url, + 'width': int_or_none(image.get('width')), + 'height': int_or_none(image.get('height')), + }) + + description = None + for p in ('large_', 'medium_', 'small_', ''): + d = m_details.get(p + 'description') + if d: + description = d + break + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'thumbnails': thumbnails, + 'description': description, + 'timestamp': float_or_none(details.get('metax_added_on'), 1000), + 'subtitles': subtitles, + 'duration': float_or_none(m_details.get('duration'), 1000), + 'view_count': int_or_none(details.get('num_watched')), + 'like_count': int_or_none(details.get('num_fav')), + 'categories': details.get('category'), + 'tags': details.get('tags'), + 'season_number': int_or_none(details.get('season')), + 'episode_number': int_or_none(details.get('episode')), + 'release_year': int_or_none(details.get('pub_year')), + } diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index ea47b99f6..1db21529f 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -231,6 +231,7 @@ from .commonprotocols import ( RtmpIE, ) from .condenast import CondeNastIE +from .contv import CONtvIE from .corus import CorusIE from .cracked import CrackedIE from .crackle import CrackleIE @@ -1322,7 +1323,6 @@ from .viewlift import ( ViewLiftIE, ViewLiftEmbedIE, ) -from .viewster import ViewsterIE from .viidea import ViideaIE from .vimeo import ( VimeoIE, diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py deleted file mode 100644 index 6e318479c..000000000 --- a/youtube_dl/extractor/viewster.py +++ /dev/null @@ -1,217 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..compat import ( - compat_HTTPError, - compat_urllib_parse_unquote, -) -from ..utils import ( - determine_ext, - ExtractorError, - int_or_none, - parse_iso8601, - sanitized_Request, - HEADRequest, - url_basename, -) - - -class ViewsterIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P\d+-\d+-\d+)' - _TESTS = [{ - # movie, Type=Movie - 'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/', - 'md5': 'e642d1b27fcf3a4ffa79f194f5adde36', - 'info_dict': { - 'id': '1140-11855-000', - 'ext': 'mp4', - 'title': 'The listening Project', - 'description': 'md5:bac720244afd1a8ea279864e67baa071', - 'timestamp': 1214870400, - 'upload_date': '20080701', - 'duration': 4680, - }, - }, { - # series episode, Type=Episode - 'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/', - 'md5': '9243079a8531809efe1b089db102c069', - 'info_dict': { - 'id': '1284-19427-001', - 'ext': 'mp4', - 'title': 'The World and a Wall', - 'description': 'md5:24814cf74d3453fdf5bfef9716d073e3', - 'timestamp': 1428192000, - 'upload_date': '20150405', - 'duration': 1500, - }, - }, { - # serie, Type=Serie - 'url': 'http://www.viewster.com/serie/1303-19426-000/', - 'info_dict': { - 'id': '1303-19426-000', - 'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?', - 'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11', - }, - 'playlist_count': 13, - }, { - # unfinished serie, no Type - 'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/', - 'info_dict': { - 'id': '1284-19427-000', - 'title': 'Baby Steps—Season 2', - 'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1', - }, - 'playlist_mincount': 16, - }, { - # geo restricted series - 'url': 'https://www.viewster.com/serie/1280-18794-002/', - 'only_matching': True, - }, { - # geo restricted video - 'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/', - 'only_matching': True, - }] - - _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01' - - def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True, query={}): - request = sanitized_Request(url) - request.add_header('Accept', self._ACCEPT_HEADER) - request.add_header('Auth-token', self._AUTH_TOKEN) - return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal, query=query) - - def _real_extract(self, url): - video_id = self._match_id(url) - # Get 'api_token' cookie - self._request_webpage( - HEADRequest('http://www.viewster.com/'), - video_id, headers=self.geo_verification_headers()) - cookies = self._get_cookies('http://www.viewster.com/') - self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value) - - info = self._download_json( - 'https://public-api.viewster.com/search/%s' % video_id, - video_id, 'Downloading entry JSON') - - entry_id = info.get('Id') or info['id'] - - # unfinished serie has no Type - if info.get('Type') in ('Serie', None): - try: - episodes = self._download_json( - 'https://public-api.viewster.com/series/%s/episodes' % entry_id, - video_id, 'Downloading series JSON') - except ExtractorError as e: - if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404: - self.raise_geo_restricted() - else: - raise - entries = [ - self.url_result( - 'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster') - for episode in episodes] - title = (info.get('Title') or info['Synopsis']['Title']).strip() - description = info.get('Synopsis', {}).get('Detailed') - return self.playlist_result(entries, video_id, title, description) - - formats = [] - for language_set in info.get('LanguageSets', []): - manifest_url = None - m3u8_formats = [] - audio = language_set.get('Audio') or '' - subtitle = language_set.get('Subtitle') or '' - base_format_id = audio - if subtitle: - base_format_id += '-%s' % subtitle - - def concat(suffix, sep='-'): - return (base_format_id + '%s%s' % (sep, suffix)) if base_format_id else suffix - - medias = self._download_json( - 'https://public-api.viewster.com/movies/%s/videos' % entry_id, - video_id, fatal=False, query={ - 'mediaTypes': ['application/f4m+xml', 'application/x-mpegURL', 'video/mp4'], - 'language': audio, - 'subtitle': subtitle, - }) - if not medias: - continue - for media in medias: - video_url = media.get('Uri') - if not video_url: - continue - ext = determine_ext(video_url) - if ext == 'f4m': - manifest_url = video_url - video_url += '&' if '?' in video_url else '?' - video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1' - formats.extend(self._extract_f4m_formats( - video_url, video_id, f4m_id=concat('hds'))) - elif ext == 'm3u8': - manifest_url = video_url - m3u8_formats = self._extract_m3u8_formats( - video_url, video_id, 'mp4', m3u8_id=concat('hls'), - fatal=False) # m3u8 sometimes fail - if m3u8_formats: - formats.extend(m3u8_formats) - else: - qualities_basename = self._search_regex( - r'/([^/]+)\.csmil/', - manifest_url, 'qualities basename', default=None) - if not qualities_basename: - continue - QUALITIES_RE = r'((,\d+k)+,?)' - qualities = self._search_regex( - QUALITIES_RE, qualities_basename, - 'qualities', default=None) - if not qualities: - continue - qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(','))) - qualities.sort() - http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename) - http_url_basename = url_basename(video_url) - if m3u8_formats: - self._sort_formats(m3u8_formats) - m3u8_formats = list(filter( - lambda f: f.get('vcodec') != 'none', m3u8_formats)) - if len(qualities) == len(m3u8_formats): - for q, m3u8_format in zip(qualities, m3u8_formats): - f = m3u8_format.copy() - f.update({ - 'url': video_url.replace(http_url_basename, http_template % q), - 'format_id': f['format_id'].replace('hls', 'http'), - 'protocol': 'http', - }) - formats.append(f) - else: - for q in qualities: - formats.append({ - 'url': video_url.replace(http_url_basename, http_template % q), - 'ext': 'mp4', - 'format_id': 'http-%d' % q, - 'tbr': q, - }) - - if not formats and not info.get('VODSettings'): - self.raise_geo_restricted() - - self._sort_formats(formats) - - synopsis = info.get('Synopsis') or {} - # Prefer title outside synopsis since it's less messy - title = (info.get('Title') or synopsis['Title']).strip() - description = synopsis.get('Detailed') or (info.get('Synopsis') or {}).get('Short') - duration = int_or_none(info.get('Duration')) - timestamp = parse_iso8601(info.get('ReleaseDate')) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'timestamp': timestamp, - 'duration': duration, - 'formats': formats, - }