Automated Daily Builds

This commit is contained in:
Lesmiscore
2022-02-13 16:41:06 +09:00
committed by dirkf
parent 0861812d72
commit b3e42caaaf
5 changed files with 419 additions and 124 deletions

187
.github/workflows/build.yml vendored Normal file
View File

@@ -0,0 +1,187 @@
name: Build
on:
workflow_dispatch:
schedule:
- cron: "0 2 * * *"
jobs:
build_unix:
runs-on: ubuntu-latest
outputs:
version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
ytdl_version: ${{ steps.bump_version.outputs.ytdl_version }}
upload_url: ${{ steps.create_release.outputs.upload_url }}
sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }}
sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }}
sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install packages
run: sudo apt-get -y install zip pandoc man
- name: Set version suffix
id: version_suffix
run: echo ::set-output name=version_suffix::$( printf '114514\n40298\n43044\n810\n1919\n19419\n334' | sort -R | head -n1 )
- name: Bump version
id: bump_version
run: |
python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
make issuetemplates
- name: Push to release
id: push_release
run: echo ::set-output name=head_sha::$(git rev-parse HEAD)
- name: Get Changelog
id: get_changelog
run: |
changelog=$(cat Changelog.md | grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdl_version }}\n{2}).+?(?=\n{2,3}###)') || true
echo "changelog<<EOF" >> $GITHUB_ENV
echo "$changelog" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Run Make
run: make all tar
- name: Get SHA2-256SUMS for youtube-dl
id: sha256_bin
run: echo "::set-output name=sha256_bin::$(sha256sum youtube-dl | awk '{print $1}')"
- name: Get SHA2-256SUMS for youtube-dl.tar.gz
id: sha256_tar
run: echo "::set-output name=sha256_tar::$(sha256sum youtube-dl.tar.gz | awk '{print $1}')"
- name: Get SHA2-512SUMS for youtube-dl
id: sha512_bin
run: echo "::set-output name=sha512_bin::$(sha512sum youtube-dl | awk '{print $1}')"
- name: Get SHA2-512SUMS for youtube-dl.tar.gz
id: sha512_tar
run: echo "::set-output name=sha512_tar::$(sha512sum youtube-dl.tar.gz | awk '{print $1}')"
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.bump_version.outputs.ytdl_version }}
release_name: youtube-dl ${{ steps.bump_version.outputs.ytdl_version }}
commitish: ${{ steps.push_release.outputs.head_sha }}
body: This is a daily build of youtube-dl.
draft: false
prerelease: false
- name: Upload youtube-dl Unix binary
id: upload-release-asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./youtube-dl
asset_name: youtube-dl
asset_content_type: application/octet-stream
- name: Upload Source tar
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./youtube-dl.tar.gz
asset_name: youtube-dl-${{ steps.bump_version.outputs.ytdl_version }}.tar.gz
asset_content_type: application/gzip
build_windows:
runs-on: windows-2022
needs: build_unix
outputs:
sha256_win: ${{ steps.sha256_win.outputs.sha256_win }}
sha512_win: ${{ steps.sha512_win.outputs.sha512_win }}
steps:
- uses: actions/checkout@v2
# reason to choose 3.4: https://media.discordapp.net/attachments/807245652072857613/942409077701619742/unknown.png
- name: Set up Python 3.4
uses: actions/setup-python@v2
with:
python-version: '3.4'
architecture: 'x86'
- name: Install packages
# https://setuptools.pypa.io/en/latest/history.html#v44-0-0
# https://pypi.org/project/py2exe/0.9.2.2/
# https://pip.pypa.io/en/stable/news/#v19-2
# https://wheel.readthedocs.io/en/stable/news.html
run: python -m pip install --upgrade "pip<19.2" "setuptools<44" "wheel<0.34.0" py2exe==0.9.2.2
- name: Bump version
id: bump_version
env:
version_suffix: ${{ needs.build_unix.outputs.version_suffix }}
run: python devscripts/update-version.py ${{ env.version_suffix }}
# - name: Run PyInstaller Script
# run: python -m PyInstaller --onefile --console --distpath dist/ -n youtube-dl youtube_dl\__main__.py
- name: Build EXE file
run: python setup.py py2exe
- name: Upload youtube-dl.exe Windows binary
id: upload-release-windows
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ needs.build_unix.outputs.upload_url }}
asset_path: ./youtube-dl.exe
asset_name: youtube-dl.exe
asset_content_type: application/vnd.microsoft.portable-executable
- name: Get SHA2-256SUMS for youtube-dl.exe
id: sha256_win
run: echo "::set-output name=sha256_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA256).Hash.ToLower())"
- name: Get SHA2-512SUMS for youtube-dl.exe
id: sha512_win
run: echo "::set-output name=sha512_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA512).Hash.ToLower())"
finish:
runs-on: ubuntu-latest
needs: [build_unix, build_windows]
env:
YTDL_VERSION: ${{ needs.build_unix.outputs.ytdl_version }}
steps:
- name: Make SHA2-256SUMS file
env:
SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }}
SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }}
SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }}
run: |
echo "${{ env.SHA256_BIN }} youtube-dl" >> SHA2-256SUMS
echo "${{ env.SHA256_TAR }} youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-256SUMS
echo "${{ env.SHA256_WIN }} youtube-dl.exe" >> SHA2-256SUMS
- name: Upload 256SUMS file
id: upload-sums
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ needs.build_unix.outputs.upload_url }}
asset_path: ./SHA2-256SUMS
asset_name: SHA2-256SUMS
asset_content_type: text/plain
- name: Make SHA2-512SUMS file
env:
SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }}
SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }}
SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }}
run: |
echo "${{ env.SHA512_BIN }} youtube-dl" >> SHA2-512SUMS
echo "${{ env.SHA512_TAR }} youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-512SUMS
echo "${{ env.SHA512_WIN }} youtube-dl.exe" >> SHA2-512SUMS
- name: Upload 512SUMS file
id: upload-512sums
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ needs.build_unix.outputs.upload_url }}
asset_path: ./SHA2-512SUMS
asset_name: SHA2-512SUMS
asset_content_type: text/plain

View File

@@ -0,0 +1,17 @@
name: Rebase on Upstream
on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
jobs:
rebase:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
with:
fetch-depth: 0
- uses: ytdl-patched/rebase-upstream-action@master
with:
token: ${{ secrets.GH_PAT }}

View File

@@ -1,6 +1,9 @@
[![Build Status](https://github.com/ytdl-org/youtube-dl/workflows/CI/badge.svg)](https://github.com/ytdl-org/youtube-dl/actions?query=workflow%3ACI) [![Build Status](https://github.com/ytdl-org/youtube-dl/workflows/CI/badge.svg)](https://github.com/ytdl-org/youtube-dl/actions?query=workflow%3ACI)
**This repository is for unofficial daily builds.**
**Please do not open any issues nor pull requests.**
youtube-dl - download videos from youtube.com or other video platforms youtube-dl - download videos from youtube.com or other video platforms
- [INSTALLATION](#installation) - [INSTALLATION](#installation)

View File

@@ -0,0 +1,32 @@
#!/usr/bin/env python3
from datetime import datetime
import sys
with open('youtube_dl/version.py', 'rt') as f:
exec(compile(f.read(), 'youtube_dl/version.py', 'exec'))
old_version = locals()['__version__']
old_version_list = old_version.split('.')
old_ver = '.'.join(old_version_list[:3])
old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
ver = datetime.utcnow().strftime("%Y.%m.%d")
rev = (sys.argv[1:] or [''])[0] # Use first argument, if present as revision number
if not rev:
rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
VERSION = '.'.join((ver, rev)) if rev else ver
VERSION_FILE = '''# Autogenerated by devscripts/update-version.py
__version__ = {!r}
'''.format(VERSION)
with open('youtube_dl/version.py', 'wt') as f:
f.write(VERSION_FILE)
print('::set-output name=ytdl_version::' + VERSION)
print('\nVersion = %s' % VERSION)

View File

@@ -1,11 +1,12 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import json
import traceback
import hashlib import hashlib
import json
import os import os
import platform
import subprocess import subprocess
import sys import sys
import traceback
from zipimport import zipimporter from zipimport import zipimporter
from .compat import ( from .compat import (
@@ -17,176 +18,231 @@ from .utils import encode_compat_str
from .version import __version__ from .version import __version__
def rsa_verify(message, signature, key): def detect_variant():
from hashlib import sha256 if hasattr(sys, 'frozen'):
assert isinstance(message, bytes) if getattr(sys, '_MEIPASS', None):
byte_size = (len(bin(key[0])) - 2 + 8 - 1) // 8 return 'win_exe'
signature = ('%x' % pow(int(signature, 16), key[1], key[0])).encode() return 'py2exe'
signature = (byte_size * 2 - len(signature)) * b'0' + signature elif isinstance(globals().get('__loader__'), zipimporter):
asn1 = b'3031300d060960864801650304020105000420' return 'zip'
asn1 += sha256(message).hexdigest().encode() elif os.path.basename(sys.argv[0]) == '__main__.py':
if byte_size < len(asn1) // 2 + 11: return 'source'
return False return 'unknown'
expected = b'0001' + (byte_size - len(asn1) // 2 - 3) * b'ff' + b'00' + asn1
return expected == signature
def update_self(to_screen, verbose, opener): _NON_UPDATEABLE_REASONS = {
"""Update the program file with the latest version from the repository""" 'win_exe': None,
'zip': None,
'mac_exe': None,
'py2exe': None,
'win_dir': 'Auto-update is not supported for unpackaged windows executable; Re-download the latest release',
'mac_dir': 'Auto-update is not supported for unpackaged MacOS executable; Re-download the latest release',
'source': 'You cannot update when running from source code; Use git to pull the latest changes',
'unknown': 'It looks like you installed youtube-dl with a package manager, pip or setup.py; Use that to update',
}
UPDATE_URL = 'https://yt-dl.org/update/'
VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
JSON_URL = UPDATE_URL + 'versions.json'
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'): def is_non_updateable():
to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.') return _NON_UPDATEABLE_REASONS.get(detect_variant(), _NON_UPDATEABLE_REASONS['unknown'])
return
# Check if there is a new version
try: def run_update(ydl):
newversion = opener.open(VERSION_URL).read().decode('utf-8').strip() """
except Exception: Update the program file with the latest version from the repository
if verbose: Returns whether the program should terminate
to_screen(encode_compat_str(traceback.format_exc())) """
to_screen('ERROR: can\'t find the current version. Please try again later.')
return JSON_URL = 'https://api.github.com/repos/ytdl-patched/youtube-dl/releases/latest'
if newversion == __version__:
to_screen('youtube-dl is up-to-date (' + __version__ + ')') def report_error(msg, expected=False):
return ydl.report_error(msg, tb='' if expected else None)
def report_unable(action, expected=False):
report_error('Unable to %s' % action, expected)
def report_permission_error(file):
report_unable('write to %s; Try running as administrator' % file, True)
def report_network_error(action, delim=';'):
report_unable('%s%s Visit https://github.com/ytdl-patched/youtube-dl/releases/latest' % (action, delim), True)
def calc_sha256sum(path):
h = hashlib.sha256()
b = bytearray(128 * 1024)
mv = memoryview(b)
with open(os.path.realpath(path), 'rb', buffering=0) as f:
for n in iter(lambda: f.readinto(mv), 0):
h.update(mv[:n])
return h.hexdigest()
# Download and check versions info # Download and check versions info
try: try:
versions_info = opener.open(JSON_URL).read().decode('utf-8') version_info = ydl._opener.open(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info) version_info = json.loads(version_info)
except Exception: except Exception:
if verbose: return report_network_error('obtain version info', delim='; Please try again later or')
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: can\'t obtain versions info. Please try again later.')
return
if 'signature' not in versions_info:
to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
return
signature = versions_info['signature']
del versions_info['signature']
if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
to_screen('ERROR: the versions file signature is invalid. Aborting.')
return
version_id = versions_info['latest']
def version_tuple(version_str): def version_tuple(version_str):
return tuple(map(int, version_str.split('.'))) return tuple(map(int, version_str.split('.')))
version_id = version_info['tag_name']
ydl.to_screen('Latest version: %s, Current version: %s' % (version_id, __version__))
if version_tuple(__version__) >= version_tuple(version_id): if version_tuple(__version__) >= version_tuple(version_id):
to_screen('youtube-dl is up to date (%s)' % __version__) ydl.to_screen('youtube-dl is up to date (%s)' % __version__)
return return
to_screen('Updating to version ' + version_id + ' ...') err = is_non_updateable()
version = versions_info['versions'][version_id] if err:
return report_error(err, True)
print_notes(to_screen, versions_info['versions'])
# sys.executable is set to the full pathname of the exe-file for py2exe # sys.executable is set to the full pathname of the exe-file for py2exe
# though symlinks are not followed so that we need to do this manually # though symlinks are not followed so that we need to do this manually
# with help of realpath # with help of realpath
filename = compat_realpath(sys.executable if hasattr(sys, 'frozen') else sys.argv[0]) filename = compat_realpath(sys.executable if hasattr(sys, 'frozen') else sys.argv[0])
ydl.to_screen('Current Build Hash %s' % calc_sha256sum(filename))
ydl.to_screen('Updating to version %s ...' % version_id)
version_labels = {
'zip_3': '',
'py2exe_32': '.exe',
'py2exe_64': '.exe',
}
def get_bin_info(bin_or_exe, version):
label = version_labels['%s_%s' % (bin_or_exe, version)]
return next((i for i in version_info['assets'] if i['name'] == 'youtube-dl%s' % label), {})
def get_sha256sum(bin_or_exe, version):
filename = 'youtube-dl%s' % version_labels['%s_%s' % (bin_or_exe, version)]
urlh = next(
(i for i in version_info['assets'] if i['name'] in ('SHA2-256SUMS')),
{}).get('browser_download_url')
if not urlh:
return None
hash_data = ydl._opener.open(urlh).read().decode('utf-8')
return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename)
if not os.access(filename, os.W_OK): if not os.access(filename, os.W_OK):
to_screen('ERROR: no write permissions on %s' % filename) return report_permission_error(filename)
return
# Py2EXE # PyInstaller
if hasattr(sys, 'frozen'): variant = detect_variant()
exe = filename if variant in ('win_exe', 'py2exe'):
directory = os.path.dirname(exe) directory = os.path.dirname(filename)
if not os.access(directory, os.W_OK): if not os.access(directory, os.W_OK):
to_screen('ERROR: no write permissions on %s' % directory) return report_permission_error(directory)
return try:
if os.path.exists(filename + '.old'):
os.remove(filename + '.old')
except (IOError, OSError):
return report_unable('remove the old version')
try: try:
urlh = opener.open(version['exe'][0]) arch = platform.architecture()[0][:2]
url = get_bin_info(variant, arch).get('browser_download_url')
if not url:
return report_network_error('fetch updates')
urlh = ydl._opener.open(url)
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError): except (IOError, OSError):
if verbose: return report_network_error('download latest version')
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: unable to download latest version')
return
newcontent_hash = hashlib.sha256(newcontent).hexdigest()
if newcontent_hash != version['exe'][1]:
to_screen('ERROR: the downloaded file hash does not match. Aborting.')
return
try: try:
with open(exe + '.new', 'wb') as outf: with open(filename + '.new', 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError): except (IOError, OSError):
if verbose: return report_permission_error('%s.new' % filename)
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: unable to write the new version') expected_sum = get_sha256sum(variant, arch)
return if not expected_sum:
ydl.report_warning('no hash information found for the release')
elif calc_sha256sum(filename + '.new') != expected_sum:
report_network_error('verify the new executable')
try:
os.remove(filename + '.new')
except OSError:
return report_unable('remove corrupt download')
try: try:
bat = os.path.join(directory, 'youtube-dl-updater.bat') os.rename(filename, filename + '.old')
with open(bat, 'w') as batfile:
batfile.write('''
@echo off
echo Waiting for file handle to be closed ...
ping 127.0.0.1 -n 5 -w 1000 > NUL
move /Y "%s.new" "%s" > NUL
echo Updated youtube-dl to version %s.
start /b "" cmd /c del "%%~f0"&exit /b"
\n''' % (exe, exe, version_id))
subprocess.Popen([bat]) # Continues to run in the background
return # Do not show premature success messages
except (IOError, OSError): except (IOError, OSError):
if verbose: return report_unable('move current version')
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: unable to overwrite current version')
return
# Zip unix package
elif isinstance(globals().get('__loader__'), zipimporter):
try: try:
urlh = opener.open(version['bin'][0]) os.rename(filename + '.new', filename)
except (IOError, OSError):
report_unable('overwrite current version')
os.rename(filename + '.old', filename)
return
try:
# Continues to run in the background
subprocess.Popen(
'ping 127.0.0.1 -n 5 -w 1000 & del /F "%s.old"' % filename,
shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
ydl.to_screen('Updated youtube-dl to version %s' % version_id)
return True # Exit app
except OSError:
report_unable('delete the old version')
elif variant in ('zip', 'mac_exe'):
pack_type = '3' if variant == 'zip' else '64'
try:
url = get_bin_info(variant, pack_type).get('browser_download_url')
if not url:
return report_network_error('fetch updates')
urlh = ydl._opener.open(url)
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError): except (IOError, OSError):
if verbose: return report_network_error('download the latest version')
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: unable to download latest version')
return
newcontent_hash = hashlib.sha256(newcontent).hexdigest() expected_sum = get_sha256sum(variant, pack_type)
if newcontent_hash != version['bin'][1]: if not expected_sum:
to_screen('ERROR: the downloaded file hash does not match. Aborting.') ydl.report_warning('no hash information found for the release')
return elif hashlib.sha256(newcontent).hexdigest() != expected_sum:
return report_network_error('verify the new package')
try: try:
with open(filename, 'wb') as outf: with open(filename, 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError): except (IOError, OSError):
if verbose: return report_unable('overwrite current version')
to_screen(encode_compat_str(traceback.format_exc()))
to_screen('ERROR: unable to overwrite current version')
return
to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.') ydl.to_screen('Updated youtube-dl to version %s; Restart youtube-dl to use the new version' % version_id)
return
assert False, ('Unhandled variant: %s' % variant)
def get_notes(versions, fromVersion): # Deprecated
notes = [] def update_self(to_screen, verbose, opener):
for v, vdata in sorted(versions.items()):
if v > fromVersion:
notes.extend(vdata.get('notes', []))
return notes
printfn = to_screen
def print_notes(to_screen, versions, fromVersion=__version__): class FakeYDL():
notes = get_notes(versions, fromVersion) _opener = opener
if notes: to_screen = printfn
to_screen('PLEASE NOTE:')
for note in notes: @staticmethod
to_screen(note) def report_warning(msg, *args, **kwargs):
return printfn('WARNING: %s' % msg, *args, **kwargs)
@staticmethod
def report_error(msg, tb=None):
printfn('ERROR: %s' % msg)
if not verbose:
return
if tb is None:
# Copied from YoutubeDl.trouble
if sys.exc_info()[0]:
tb = ''
if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
tb += encode_compat_str(traceback.format_exc())
else:
tb_data = traceback.format_list(traceback.extract_stack())
tb = ''.join(tb_data)
if tb:
printfn(tb)
return run_update(FakeYDL())