From 1cf376f55a3d9335eb161c07c439ca143d86924e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 27 Feb 2021 18:11:23 +0530 Subject: [PATCH] Add option `--sleep-requests` to sleep b/w requests (Closes #106) * Also fix documentation of `sleep_interval_subtitles` Related issues: https://github.com/blackjack4494/yt-dlc/issues/158 https://github.com/blackjack4494/youtube-dlc/issues/195 https://github.com/ytdl-org/youtube-dl/pull/28270 https://github.com/ytdl-org/youtube-dl/pull/28144 https://github.com/ytdl-org/youtube-dl/issues/27767 https://github.com/ytdl-org/youtube-dl/issues/23638 https://github.com/ytdl-org/youtube-dl/issues/26287 https://github.com/ytdl-org/youtube-dl/issues/26319 --- README.md | 5 ++++- yt_dlp/YoutubeDL.py | 14 +++++--------- yt_dlp/__init__.py | 7 +++++++ yt_dlp/extractor/common.py | 8 ++++++++ yt_dlp/options.py | 6 +++++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 69813c2b7..8def97e4c 100644 --- a/README.md +++ b/README.md @@ -491,6 +491,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t --bidi-workaround Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH + --sleep-requests SECONDS Number of seconds to sleep between requests + during data extraction --sleep-interval SECONDS Number of seconds to sleep before each download when used alone or a lower bound of a range for randomized sleep before each @@ -501,7 +503,8 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t before each download (maximum possible number of seconds to sleep). Must only be used along with --min-sleep-interval - --sleep-subtitles SECONDS Enforce sleep interval on subtitles as well + --sleep-subtitles SECONDS Number of seconds to sleep before each + subtitle download ## Video Format Options: -f, --format FORMAT Video format code, see "FORMAT SELECTION" diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index a54755e15..3c53f4cd8 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -324,6 +324,8 @@ class YoutubeDL(object): source_address: Client-side IP address to bind to. call_home: Boolean, true iff we are allowed to contact the yt-dlp servers for debugging. (BROKEN) + sleep_interval_requests: Number of seconds to sleep between requests + during extraction sleep_interval: Number of seconds to sleep before each download when used alone or a lower bound of a range for randomized sleep before each download (minimum possible number @@ -334,6 +336,7 @@ class YoutubeDL(object): Must only be used along with sleep_interval. Actual sleep time will be a random float from range [sleep_interval; max_sleep_interval]. + sleep_interval_subtitles: Number of seconds to sleep before each subtitle download listformats: Print an overview of available video formats and exit. list_thumbnails: Print a table of all thumbnails and exit. match_filter: A function that gets called with the info_dict of @@ -406,6 +409,7 @@ class YoutubeDL(object): _ies = [] _pps = {'beforedl': [], 'aftermove': [], 'normal': []} __prepare_filename_warned = False + _first_webpage_request = True _download_retcode = None _num_downloads = None _playlist_level = 0 @@ -420,6 +424,7 @@ class YoutubeDL(object): self._ies_instances = {} self._pps = {'beforedl': [], 'aftermove': [], 'normal': []} self.__prepare_filename_warned = False + self._first_webpage_request = True self._post_hooks = [] self._progress_hooks = [] self._download_retcode = 0 @@ -2166,15 +2171,6 @@ class YoutubeDL(object): else: try: dl(sub_filename, sub_info, subtitle=True) - ''' - if self.params.get('sleep_interval_subtitles', False): - dl(sub_filename, sub_info) - else: - sub_data = ie._request_webpage( - sub_info['url'], info_dict['id'], note=False).read() - with io.open(encodeFilename(sub_filename), 'wb') as subfile: - subfile.write(sub_data) - ''' files_to_move[sub_filename] = sub_filename_final except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download subtitle for "%s": %s' % diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 91539daaf..15605ab31 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -169,6 +169,12 @@ def _real_main(argv=None): parser.error('max sleep interval must be greater than or equal to min sleep interval') else: opts.max_sleep_interval = opts.sleep_interval + if opts.sleep_interval_subtitles is not None: + if opts.sleep_interval_subtitles < 0: + parser.error('subtitles sleep interval must be positive or 0') + if opts.sleep_interval_requests is not None: + if opts.sleep_interval_requests < 0: + parser.error('requests sleep interval must be positive or 0') if opts.ap_mso and opts.ap_mso not in MSO_INFO: parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers') if opts.overwrites: @@ -524,6 +530,7 @@ def _real_main(argv=None): 'fixup': opts.fixup, 'source_address': opts.source_address, 'call_home': opts.call_home, + 'sleep_interval_requests': opts.sleep_interval_requests, 'sleep_interval': opts.sleep_interval, 'max_sleep_interval': opts.max_sleep_interval, 'sleep_interval_subtitles': opts.sleep_interval_subtitles, diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 677a61b86..47b91a00a 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -606,6 +606,14 @@ class InfoExtractor(object): See _download_webpage docstring for arguments specification. """ + if not self._downloader._first_webpage_request: + sleep_interval = float_or_none(self._downloader.params.get('sleep_interval_requests')) or 0 + if sleep_interval > 0: + self.to_screen('Sleeping %s seconds...' % sleep_interval) + time.sleep(sleep_interval) + else: + self._downloader._first_webpage_request = False + if note is None: self.report_download_webpage(video_id) elif note is not False: diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 5f200fb8f..ae11e6b8b 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -696,6 +696,10 @@ def parseOpts(overrideArguments=None): '--bidi-workaround', dest='bidi_workaround', action='store_true', help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') + workarounds.add_option( + '--sleep-requests', metavar='SECONDS', + dest='sleep_interval_requests', type=float, + help='Number of seconds to sleep between requests during data extraction') workarounds.add_option( '--sleep-interval', '--min-sleep-interval', metavar='SECONDS', dest='sleep_interval', type=float, @@ -714,7 +718,7 @@ def parseOpts(overrideArguments=None): workarounds.add_option( '--sleep-subtitles', metavar='SECONDS', dest='sleep_interval_subtitles', default=0, type=int, - help='Enforce sleep interval on subtitles as well') + help='Number of seconds to sleep before each subtitle download') verbosity = optparse.OptionGroup(parser, 'Verbosity and Simulation Options') verbosity.add_option(