yt-dlp/youtube_dl/extractor/appletrailers.py

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
)


class AppleTrailersIE(InfoExtractor):
    IE_NAME = 'appletrailers'
    _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
        'info_dict': {
            'id': 'manofsteel',
        },
        'playlist': [
            {
                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
                'info_dict': {
                    'id': 'manofsteel-trailer4',
                    'ext': 'mov',
                    'duration': 111,
                    'title': 'Trailer 4',
                    'upload_date': '20130523',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
                'info_dict': {
                    'id': 'manofsteel-trailer3',
                    'ext': 'mov',
                    'duration': 182,
                    'title': 'Trailer 3',
                    'upload_date': '20130417',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'd0f1e1150989b9924679b441f3404d48',
                'info_dict': {
                    'id': 'manofsteel-trailer',
                    'ext': 'mov',
                    'duration': 148,
                    'title': 'Trailer',
                    'upload_date': '20121212',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
                'info_dict': {
                    'id': 'manofsteel-teaser',
                    'ext': 'mov',
                    'duration': 93,
                    'title': 'Teaser',
                    'upload_date': '20120721',
                    'uploader_id': 'wb',
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
        'info_dict': {
            'id': 'blackthorn',
        },
        'playlist_mincount': 2,
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }, {
        'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            first_url = trailer_info.get('url')
            if not first_url:
                continue
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }


class AppleTrailersSectionIE(InfoExtractor):
    IE_NAME = 'appletrailers:section'
    _SECTIONS = {
        'justadded': {
            'feed_path': 'just_added',
            'title': 'Just Added',
        },
        'exclusive': {
            'feed_path': 'exclusive',
            'title': 'Exclusive',
        },
        'justhd': {
            'feed_path': 'just_hd',
            'title': 'Just HD',
        },
        'mostpopular': {
            'feed_path': 'most_pop',
            'title': 'Most Popular',
        },
        'moviestudios': {
            'feed_path': 'studios',
            'title': 'Movie Studios',
        },
    }
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
    _TESTS = [{
        'url': 'http://trailers.apple.com/#section=justadded',
        'info_dict': {
            'title': 'Just Added',
            'id': 'justadded',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=exclusive',
        'info_dict': {
            'title': 'Exclusive',
            'id': 'exclusive',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=justhd',
        'info_dict': {
            'title': 'Just HD',
            'id': 'justhd',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=mostpopular',
        'info_dict': {
            'title': 'Most Popular',
            'id': 'mostpopular',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=moviestudios',
        'info_dict': {
            'title': 'Movie Studios',
            'id': 'moviestudios',
        },
        'playlist_mincount': 80,
    }]

    def _real_extract(self, url):
        section = self._match_id(url)
        section_data = self._download_json(
            'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
            section)
        entries = [
            self.url_result('http://trailers.apple.com' + e['location'])
            for e in section_data]
        return self.playlist_result(entries, section, self._SECTIONS[section]['title'])
More unicode literals 2014-01-07 10:04:48 +01:00			`from __future__ import unicode_literals`

[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`import re`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`import json`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`from .common import InfoExtractor`
Fix imports and general cleanup · Import from compat what comes from compat. Yes, some names are available in utils too, but that's an implementation detail. · Use _match_id consistently whenever possible · Fix some outdated tests · Use consistent valid URL (always match the whole protocol, no ^ at start required) · Use modern test definitions 2014-12-13 12:24:42 +01:00			`from ..compat import compat_urlparse`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`from ..utils import (`
[appletrailers] Support height-less videos 2014-08-10 13:04:45 +02:00			`int_or_none,`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`)`


			`class AppleTrailersIE(InfoExtractor):`
[appletrailers] Add support for AppleTrailers Section 2015-12-23 10:40:45 +01:00			`IE_NAME = 'appletrailers'`
[appletrailers] Extend _VALID_URL (#8524) 2016-02-20 08:54:00 +01:00			`_VALID_URL = r'https?://(?:www\.\|movie)?trailers\.apple\.com/(?:trailers\|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'`
[appletrailers] Add test (#5027) 2015-02-22 14:58:39 +01:00			`_TESTS = [{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',`
Extend various playlist tests 2015-02-18 00:49:10 +01:00			`'info_dict': {`
			`'id': 'manofsteel',`
			`},`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'playlist': [`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer4',`
			`'ext': 'mov',`
			`'duration': 111,`
			`'title': 'Trailer 4',`
			`'upload_date': '20130523',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'b8017b7131b721fb4e8d6f49e1df908c',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer3',`
			`'ext': 'mov',`
			`'duration': 182,`
			`'title': 'Trailer 3',`
			`'upload_date': '20130417',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'd0f1e1150989b9924679b441f3404d48',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer',`
			`'ext': 'mov',`
			`'duration': 148,`
			`'title': 'Trailer',`
			`'upload_date': '20121212',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': '5fe08795b943eb2e757fa95cb6def1cb',`
			`'info_dict': {`
			`'id': 'manofsteel-teaser',`
			`'ext': 'mov',`
			`'duration': 93,`
			`'title': 'Teaser',`
			`'upload_date': '20120721',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`},`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`]`
[appletrailers] skip clips with empty url 2015-12-23 17:48:37 +01:00			`}, {`
			`'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',`
			`'info_dict': {`
			`'id': 'blackthorn',`
			`},`
			`'playlist_mincount': 2,`
[appletrailers] Add test (#5027) 2015-02-22 14:58:39 +01:00			`}, {`
			`'url': 'http://trailers.apple.com/ca/metropole/autrui/',`
			`'only_matching': True,`
[appletrailers] Extend _VALID_URL (#8524) 2016-02-20 08:54:00 +01:00			`}, {`
			`'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',`
			`'only_matching': True,`
[appletrailers] Add test (#5027) 2015-02-22 14:58:39 +01:00			`}]`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`_JSON_RE = r'iTunes.playURL\((.*?)\);'`

[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`def _real_extract(self, url):`
			`mobj = re.match(self._VALID_URL, url)`
			`movie = mobj.group('movie')`
			`uploader_id = mobj.group('company')`

[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')`
PEP8 applied 2014-11-23 20:41:03 +01:00
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`def fix_html(s):`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`s = re.sub(r'(?s)<script[^<]?>.?</script>', '', s)`
[appletrailers] Improve regex for fixing '<img>' tags (#7953) 2015-12-23 14:48:40 +01:00			`s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`# The ' in the onClick attributes are not escaped, it couldn't be parsed`
			`# like: http://trailers.apple.com/trailers/wb/gravity/`
PEP8 applied 2014-11-23 20:41:03 +01:00
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`def _clean_json(m):`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`s = re.sub(self._JSON_RE, _clean_json, s)`
[appletrailers] Modernize 2014-11-26 12:41:24 +01:00			`s = '<html>%s</html>' % s`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`return s`
			`doc = self._download_xml(playlist_url, movie, transform_source=fix_html)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`playlist = []`
			`for li in doc.findall('./div/ul/li'):`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`on_click = li.find('.//a').attrib['onClick']`
			`trailer_info_json = self._search_regex(self._JSON_RE,`
PEP8: applied even more rules 2014-11-23 21:39:15 +01:00			`on_click, 'trailer info')`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`trailer_info = json.loads(trailer_info_json)`
[appletrailers] skip clips with empty url 2015-12-23 17:48:37 +01:00			`first_url = trailer_info.get('url')`
			`if not first_url:`
			`continue`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`title = trailer_info['title']`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()`
			`thumbnail = li.find('.//img').attrib['src']`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`upload_date = trailer_info['posted'].replace('-', '')`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`runtime = trailer_info['runtime']`
			`m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`duration = None`
			`if m:`
			`duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))`

[appletrailers] The request for the settings must have the trailer name in lower case (fixes #1329) 2013-09-29 20:59:19 +02:00			`trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`formats = []`
			`for format in settings['metadata']['sizes']:`
			`# The src is a file pointing to the real video file`
			`format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])`
			`formats.append({`
			`'url': format_url,`
			`'format': format['type'],`
[appletrailers] Support height-less videos 2014-08-10 13:04:45 +02:00			`'width': int_or_none(format['width']),`
			`'height': int_or_none(format['height']),`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`})`
[appletrailers] Use centralized format selection 2013-12-25 15:24:41 +01:00
			`self._sort_formats(formats)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
Remove the compatibility code used before the new format system was implemented 2013-12-03 14:21:06 +01:00			`playlist.append({`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`'_type': 'video',`
			`'id': video_id,`
			`'formats': formats,`
			`'title': title,`
			`'duration': duration,`
			`'thumbnail': thumbnail,`
			`'upload_date': upload_date,`
			`'uploader_id': uploader_id,`
[extractors] Use http_headers for setting the User-Agent and the Referer 2015-01-24 18:19:58 +01:00			`'http_headers': {`
			`'User-Agent': 'QuickTime compatible (youtube-dl)',`
			`},`
Remove the compatibility code used before the new format system was implemented 2013-12-03 14:21:06 +01:00			`})`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`return {`
			`'_type': 'playlist',`
			`'id': movie,`
			`'entries': playlist,`
			`}`
[appletrailers] Add support for AppleTrailers Section 2015-12-23 10:40:45 +01:00

			`class AppleTrailersSectionIE(InfoExtractor):`
			`IE_NAME = 'appletrailers:section'`
			`_SECTIONS = {`
			`'justadded': {`
			`'feed_path': 'just_added',`
			`'title': 'Just Added',`
			`},`
			`'exclusive': {`
			`'feed_path': 'exclusive',`
			`'title': 'Exclusive',`
			`},`
			`'justhd': {`
			`'feed_path': 'just_hd',`
			`'title': 'Just HD',`
			`},`
			`'mostpopular': {`
			`'feed_path': 'most_pop',`
			`'title': 'Most Popular',`
			`},`
			`'moviestudios': {`
			`'feed_path': 'studios',`
			`'title': 'Movie Studios',`
			`},`
			`}`
			`_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '\|'.join(_SECTIONS)`
			`_TESTS = [{`
			`'url': 'http://trailers.apple.com/#section=justadded',`
			`'info_dict': {`
			`'title': 'Just Added',`
			`'id': 'justadded',`
			`},`
			`'playlist_mincount': 80,`
			`}, {`
			`'url': 'http://trailers.apple.com/#section=exclusive',`
			`'info_dict': {`
			`'title': 'Exclusive',`
			`'id': 'exclusive',`
			`},`
			`'playlist_mincount': 80,`
			`}, {`
			`'url': 'http://trailers.apple.com/#section=justhd',`
			`'info_dict': {`
			`'title': 'Just HD',`
			`'id': 'justhd',`
			`},`
			`'playlist_mincount': 80,`
			`}, {`
			`'url': 'http://trailers.apple.com/#section=mostpopular',`
			`'info_dict': {`
			`'title': 'Most Popular',`
			`'id': 'mostpopular',`
			`},`
			`'playlist_mincount': 80,`
			`}, {`
			`'url': 'http://trailers.apple.com/#section=moviestudios',`
			`'info_dict': {`
			`'title': 'Movie Studios',`
			`'id': 'moviestudios',`
			`},`
			`'playlist_mincount': 80,`
			`}]`

			`def _real_extract(self, url):`
			`section = self._match_id(url)`
			`section_data = self._download_json(`
			`'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],`
			`section)`
			`entries = [`
			`self.url_result('http://trailers.apple.com' + e['location'])`
			`for e in section_data]`
			`return self.playlist_result(entries, section, self._SECTIONS[section]['title'])`