mirror of https://github.com/yt-dlp/yt-dlp.git
[cleanup] Fix line endings for `nebula.py` (#1064)
:ci skip Authored by: glenn-slayden
This commit is contained in:
Glenn Slayden
GitHub
parent
388bc4a640
commit
bdc196a444
|
|
@ -1,238 +1,238 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from urllib.error import HTTPError
|
from urllib.error import HTTPError
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_str, compat_urllib_parse_unquote, compat_urllib_parse_quote
|
from ..compat import compat_str, compat_urllib_parse_unquote, compat_urllib_parse_quote
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
try_get,
|
try_get,
|
||||||
urljoin,
|
urljoin,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class NebulaIE(InfoExtractor):
|
class NebulaIE(InfoExtractor):
|
||||||
|
|
||||||
_VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/videos/(?P<id>[-\w]+)'
|
_VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/videos/(?P<id>[-\w]+)'
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',
|
'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast',
|
||||||
'md5': 'fe79c4df8b3aa2fea98a93d027465c7e',
|
'md5': 'fe79c4df8b3aa2fea98a93d027465c7e',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '5c271b40b13fd613090034fd',
|
'id': '5c271b40b13fd613090034fd',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'That Time Disney Remade Beauty and the Beast',
|
'title': 'That Time Disney Remade Beauty and the Beast',
|
||||||
'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
|
'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.',
|
||||||
'upload_date': '20180731',
|
'upload_date': '20180731',
|
||||||
'timestamp': 1533009600,
|
'timestamp': 1533009600,
|
||||||
'channel': 'Lindsay Ellis',
|
'channel': 'Lindsay Ellis',
|
||||||
'uploader': 'Lindsay Ellis',
|
'uploader': 'Lindsay Ellis',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'usenetrc': True,
|
'usenetrc': True,
|
||||||
},
|
},
|
||||||
'skip': 'All Nebula content requires authentication',
|
'skip': 'All Nebula content requires authentication',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
|
'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore',
|
||||||
'md5': '6d4edd14ce65720fa63aba5c583fb328',
|
'md5': '6d4edd14ce65720fa63aba5c583fb328',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '5e7e78171aaf320001fbd6be',
|
'id': '5e7e78171aaf320001fbd6be',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Landing Craft - How The Allies Got Ashore',
|
'title': 'Landing Craft - How The Allies Got Ashore',
|
||||||
'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
|
'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.',
|
||||||
'upload_date': '20200327',
|
'upload_date': '20200327',
|
||||||
'timestamp': 1585348140,
|
'timestamp': 1585348140,
|
||||||
'channel': 'The Logistics of D-Day',
|
'channel': 'The Logistics of D-Day',
|
||||||
'uploader': 'The Logistics of D-Day',
|
'uploader': 'The Logistics of D-Day',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'usenetrc': True,
|
'usenetrc': True,
|
||||||
},
|
},
|
||||||
'skip': 'All Nebula content requires authentication',
|
'skip': 'All Nebula content requires authentication',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'https://nebula.app/videos/money-episode-1-the-draw',
|
'url': 'https://nebula.app/videos/money-episode-1-the-draw',
|
||||||
'md5': '8c7d272910eea320f6f8e6d3084eecf5',
|
'md5': '8c7d272910eea320f6f8e6d3084eecf5',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '5e779ebdd157bc0001d1c75a',
|
'id': '5e779ebdd157bc0001d1c75a',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Episode 1: The Draw',
|
'title': 'Episode 1: The Draw',
|
||||||
'description': r'contains:There’s free money on offer… if the players can all work together.',
|
'description': r'contains:There’s free money on offer… if the players can all work together.',
|
||||||
'upload_date': '20200323',
|
'upload_date': '20200323',
|
||||||
'timestamp': 1584980400,
|
'timestamp': 1584980400,
|
||||||
'channel': 'Tom Scott Presents: Money',
|
'channel': 'Tom Scott Presents: Money',
|
||||||
'uploader': 'Tom Scott Presents: Money',
|
'uploader': 'Tom Scott Presents: Money',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'usenetrc': True,
|
'usenetrc': True,
|
||||||
},
|
},
|
||||||
'skip': 'All Nebula content requires authentication',
|
'skip': 'All Nebula content requires authentication',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
|
'url': 'https://watchnebula.com/videos/money-episode-1-the-draw',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
_NETRC_MACHINE = 'watchnebula'
|
_NETRC_MACHINE = 'watchnebula'
|
||||||
|
|
||||||
_nebula_token = None
|
_nebula_token = None
|
||||||
|
|
||||||
def _retrieve_nebula_auth(self):
|
def _retrieve_nebula_auth(self):
|
||||||
"""
|
"""
|
||||||
Log in to Nebula, and returns a Nebula API token
|
Log in to Nebula, and returns a Nebula API token
|
||||||
"""
|
"""
|
||||||
|
|
||||||
username, password = self._get_login_info()
|
username, password = self._get_login_info()
|
||||||
if not (username and password):
|
if not (username and password):
|
||||||
self.raise_login_required()
|
self.raise_login_required()
|
||||||
|
|
||||||
self.report_login()
|
self.report_login()
|
||||||
data = json.dumps({'email': username, 'password': password}).encode('utf8')
|
data = json.dumps({'email': username, 'password': password}).encode('utf8')
|
||||||
response = self._download_json(
|
response = self._download_json(
|
||||||
'https://api.watchnebula.com/api/v1/auth/login/',
|
'https://api.watchnebula.com/api/v1/auth/login/',
|
||||||
data=data, fatal=False, video_id=None,
|
data=data, fatal=False, video_id=None,
|
||||||
headers={
|
headers={
|
||||||
'content-type': 'application/json',
|
'content-type': 'application/json',
|
||||||
# Submitting the 'sessionid' cookie always causes a 403 on auth endpoint
|
# Submitting the 'sessionid' cookie always causes a 403 on auth endpoint
|
||||||
'cookie': ''
|
'cookie': ''
|
||||||
},
|
},
|
||||||
note='Authenticating to Nebula with supplied credentials',
|
note='Authenticating to Nebula with supplied credentials',
|
||||||
errnote='Authentication failed or rejected')
|
errnote='Authentication failed or rejected')
|
||||||
if not response or not response.get('key'):
|
if not response or not response.get('key'):
|
||||||
self.raise_login_required()
|
self.raise_login_required()
|
||||||
|
|
||||||
# save nebula token as cookie
|
# save nebula token as cookie
|
||||||
self._set_cookie(
|
self._set_cookie(
|
||||||
'nebula.app', 'nebula-auth',
|
'nebula.app', 'nebula-auth',
|
||||||
compat_urllib_parse_quote(
|
compat_urllib_parse_quote(
|
||||||
json.dumps({
|
json.dumps({
|
||||||
"apiToken": response["key"],
|
"apiToken": response["key"],
|
||||||
"isLoggingIn": False,
|
"isLoggingIn": False,
|
||||||
"isLoggingOut": False,
|
"isLoggingOut": False,
|
||||||
}, separators=(",", ":"))),
|
}, separators=(",", ":"))),
|
||||||
expire_time=int(time.time()) + 86400 * 365,
|
expire_time=int(time.time()) + 86400 * 365,
|
||||||
)
|
)
|
||||||
|
|
||||||
return response['key']
|
return response['key']
|
||||||
|
|
||||||
def _retrieve_zype_api_key(self, page_url, display_id):
|
def _retrieve_zype_api_key(self, page_url, display_id):
|
||||||
"""
|
"""
|
||||||
Retrieves the Zype API key
|
Retrieves the Zype API key
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Find the js that has the API key from the webpage and download it
|
# Find the js that has the API key from the webpage and download it
|
||||||
webpage = self._download_webpage(page_url, video_id=display_id)
|
webpage = self._download_webpage(page_url, video_id=display_id)
|
||||||
main_script_relpath = self._search_regex(
|
main_script_relpath = self._search_regex(
|
||||||
r'<script[^>]*src="(?P<script_relpath>[^"]*main.[0-9a-f]*.chunk.js)"[^>]*>', webpage,
|
r'<script[^>]*src="(?P<script_relpath>[^"]*main.[0-9a-f]*.chunk.js)"[^>]*>', webpage,
|
||||||
group='script_relpath', name='script relative path', fatal=True)
|
group='script_relpath', name='script relative path', fatal=True)
|
||||||
main_script_abspath = urljoin(page_url, main_script_relpath)
|
main_script_abspath = urljoin(page_url, main_script_relpath)
|
||||||
main_script = self._download_webpage(main_script_abspath, video_id=display_id,
|
main_script = self._download_webpage(main_script_abspath, video_id=display_id,
|
||||||
note='Retrieving Zype API key')
|
note='Retrieving Zype API key')
|
||||||
|
|
||||||
api_key = self._search_regex(
|
api_key = self._search_regex(
|
||||||
r'REACT_APP_ZYPE_API_KEY\s*:\s*"(?P<api_key>[\w-]*)"', main_script,
|
r'REACT_APP_ZYPE_API_KEY\s*:\s*"(?P<api_key>[\w-]*)"', main_script,
|
||||||
group='api_key', name='API key', fatal=True)
|
group='api_key', name='API key', fatal=True)
|
||||||
|
|
||||||
return api_key
|
return api_key
|
||||||
|
|
||||||
def _call_zype_api(self, path, params, video_id, api_key, note):
|
def _call_zype_api(self, path, params, video_id, api_key, note):
|
||||||
"""
|
"""
|
||||||
A helper for making calls to the Zype API.
|
A helper for making calls to the Zype API.
|
||||||
"""
|
"""
|
||||||
query = {'api_key': api_key, 'per_page': 1}
|
query = {'api_key': api_key, 'per_page': 1}
|
||||||
query.update(params)
|
query.update(params)
|
||||||
return self._download_json('https://api.zype.com' + path, video_id, query=query, note=note)
|
return self._download_json('https://api.zype.com' + path, video_id, query=query, note=note)
|
||||||
|
|
||||||
def _call_nebula_api(self, path, video_id, access_token, note):
|
def _call_nebula_api(self, path, video_id, access_token, note):
|
||||||
"""
|
"""
|
||||||
A helper for making calls to the Nebula API.
|
A helper for making calls to the Nebula API.
|
||||||
"""
|
"""
|
||||||
return self._download_json('https://api.watchnebula.com/api/v1' + path, video_id, headers={
|
return self._download_json('https://api.watchnebula.com/api/v1' + path, video_id, headers={
|
||||||
'Authorization': 'Token {access_token}'.format(access_token=access_token)
|
'Authorization': 'Token {access_token}'.format(access_token=access_token)
|
||||||
}, note=note)
|
}, note=note)
|
||||||
|
|
||||||
def _fetch_zype_access_token(self, video_id):
|
def _fetch_zype_access_token(self, video_id):
|
||||||
try:
|
try:
|
||||||
user_object = self._call_nebula_api('/auth/user/', video_id, self._nebula_token, note='Retrieving Zype access token')
|
user_object = self._call_nebula_api('/auth/user/', video_id, self._nebula_token, note='Retrieving Zype access token')
|
||||||
except ExtractorError as exc:
|
except ExtractorError as exc:
|
||||||
# if 401, attempt credential auth and retry
|
# if 401, attempt credential auth and retry
|
||||||
if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.code == 401:
|
if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.code == 401:
|
||||||
self._nebula_token = self._retrieve_nebula_auth()
|
self._nebula_token = self._retrieve_nebula_auth()
|
||||||
user_object = self._call_nebula_api('/auth/user/', video_id, self._nebula_token, note='Retrieving Zype access token')
|
user_object = self._call_nebula_api('/auth/user/', video_id, self._nebula_token, note='Retrieving Zype access token')
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], compat_str)
|
access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], compat_str)
|
||||||
if not access_token:
|
if not access_token:
|
||||||
if try_get(user_object, lambda x: x['is_subscribed'], bool):
|
if try_get(user_object, lambda x: x['is_subscribed'], bool):
|
||||||
# TODO: Reimplement the same Zype token polling the Nebula frontend implements
|
# TODO: Reimplement the same Zype token polling the Nebula frontend implements
|
||||||
# see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532
|
# see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532
|
||||||
raise ExtractorError(
|
raise ExtractorError(
|
||||||
'Unable to extract Zype access token from Nebula API authentication endpoint. '
|
'Unable to extract Zype access token from Nebula API authentication endpoint. '
|
||||||
'Open an arbitrary video in a browser with this account to generate a token',
|
'Open an arbitrary video in a browser with this account to generate a token',
|
||||||
expected=True)
|
expected=True)
|
||||||
raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')
|
raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint')
|
||||||
return access_token
|
return access_token
|
||||||
|
|
||||||
def _extract_channel_title(self, video_meta):
|
def _extract_channel_title(self, video_meta):
|
||||||
# TODO: Implement the API calls giving us the channel list,
|
# TODO: Implement the API calls giving us the channel list,
|
||||||
# so that we can do the title lookup and then figure out the channel URL
|
# so that we can do the title lookup and then figure out the channel URL
|
||||||
categories = video_meta.get('categories', []) if video_meta else []
|
categories = video_meta.get('categories', []) if video_meta else []
|
||||||
# the channel name is the value of the first category
|
# the channel name is the value of the first category
|
||||||
for category in categories:
|
for category in categories:
|
||||||
if category.get('value'):
|
if category.get('value'):
|
||||||
return category['value'][0]
|
return category['value'][0]
|
||||||
|
|
||||||
def _real_initialize(self):
|
def _real_initialize(self):
|
||||||
# check cookie jar for valid token
|
# check cookie jar for valid token
|
||||||
nebula_cookies = self._get_cookies('https://nebula.app')
|
nebula_cookies = self._get_cookies('https://nebula.app')
|
||||||
nebula_cookie = nebula_cookies.get('nebula-auth')
|
nebula_cookie = nebula_cookies.get('nebula-auth')
|
||||||
if nebula_cookie:
|
if nebula_cookie:
|
||||||
self.to_screen('Authenticating to Nebula with token from cookie jar')
|
self.to_screen('Authenticating to Nebula with token from cookie jar')
|
||||||
nebula_cookie_value = compat_urllib_parse_unquote(nebula_cookie.value)
|
nebula_cookie_value = compat_urllib_parse_unquote(nebula_cookie.value)
|
||||||
self._nebula_token = self._parse_json(nebula_cookie_value, None).get('apiToken')
|
self._nebula_token = self._parse_json(nebula_cookie_value, None).get('apiToken')
|
||||||
|
|
||||||
# try to authenticate using credentials if no valid token has been found
|
# try to authenticate using credentials if no valid token has been found
|
||||||
if not self._nebula_token:
|
if not self._nebula_token:
|
||||||
self._nebula_token = self._retrieve_nebula_auth()
|
self._nebula_token = self._retrieve_nebula_auth()
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
api_key = self._retrieve_zype_api_key(url, display_id)
|
api_key = self._retrieve_zype_api_key(url, display_id)
|
||||||
|
|
||||||
response = self._call_zype_api('/videos', {'friendly_title': display_id},
|
response = self._call_zype_api('/videos', {'friendly_title': display_id},
|
||||||
display_id, api_key, note='Retrieving metadata from Zype')
|
display_id, api_key, note='Retrieving metadata from Zype')
|
||||||
if len(response.get('response') or []) != 1:
|
if len(response.get('response') or []) != 1:
|
||||||
raise ExtractorError('Unable to find video on Zype API')
|
raise ExtractorError('Unable to find video on Zype API')
|
||||||
video_meta = response['response'][0]
|
video_meta = response['response'][0]
|
||||||
|
|
||||||
video_id = video_meta['_id']
|
video_id = video_meta['_id']
|
||||||
zype_access_token = self._fetch_zype_access_token(display_id)
|
zype_access_token = self._fetch_zype_access_token(display_id)
|
||||||
|
|
||||||
channel_title = self._extract_channel_title(video_meta)
|
channel_title = self._extract_channel_title(video_meta)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
'ie_key': 'Zype',
|
'ie_key': 'Zype',
|
||||||
'url': 'https://player.zype.com/embed/%s.html?access_token=%s' % (video_id, zype_access_token),
|
'url': 'https://player.zype.com/embed/%s.html?access_token=%s' % (video_id, zype_access_token),
|
||||||
'title': video_meta.get('title'),
|
'title': video_meta.get('title'),
|
||||||
'description': video_meta.get('description'),
|
'description': video_meta.get('description'),
|
||||||
'timestamp': parse_iso8601(video_meta.get('published_at')),
|
'timestamp': parse_iso8601(video_meta.get('published_at')),
|
||||||
'thumbnails': [{
|
'thumbnails': [{
|
||||||
'id': tn.get('name'), # this appears to be null
|
'id': tn.get('name'), # this appears to be null
|
||||||
'url': tn['url'],
|
'url': tn['url'],
|
||||||
'width': tn.get('width'),
|
'width': tn.get('width'),
|
||||||
'height': tn.get('height'),
|
'height': tn.get('height'),
|
||||||
} for tn in video_meta.get('thumbnails', [])],
|
} for tn in video_meta.get('thumbnails', [])],
|
||||||
'duration': video_meta.get('duration'),
|
'duration': video_meta.get('duration'),
|
||||||
'channel': channel_title,
|
'channel': channel_title,
|
||||||
'uploader': channel_title, # we chose uploader = channel name
|
'uploader': channel_title, # we chose uploader = channel name
|
||||||
# TODO: uploader_url, channel_id, channel_url
|
# TODO: uploader_url, channel_id, channel_url
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue