From 35f4f764a786685ea45d84abe1cf1ad3847f4c97 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 21 Jan 2024 10:03:33 +1300 Subject: [PATCH] [rh:requests] Apply `remove_dot_segments` to absolute redirect locations Fixes https://github.com/yt-dlp/yt-dlp/issues/9020 Authored by: coletdjnz --- test/test_networking.py | 25 ++++++++++++++++--------- yt_dlp/networking/_requests.py | 5 +++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/test/test_networking.py b/test/test_networking.py index 62325aa8e..8cadd86f5 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -180,6 +180,12 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler): self.send_header('Location', '/a/b/./../../headers') self.send_header('Content-Length', '0') self.end_headers() + elif self.path == '/redirect_dotsegments_absolute': + self.send_response(301) + # redirect to /headers but with dot segments before - absolute url + self.send_header('Location', f'http://127.0.0.1:{http_server_port(self.server)}/a/b/./../../headers') + self.send_header('Content-Length', '0') + self.end_headers() elif self.path.startswith('/redirect_'): self._redirect() elif self.path.startswith('/method'): @@ -345,16 +351,17 @@ class TestHTTPRequestHandler(TestRequestHandlerBase): res.close() @pytest.mark.parametrize('handler', ['Urllib', 'Requests'], indirect=True) - def test_remove_dot_segments(self, handler): - with handler() as rh: + @pytest.mark.parametrize('path', [ + '/a/b/./../../headers', + '/redirect_dotsegments', + # https://github.com/yt-dlp/yt-dlp/issues/9020 + '/redirect_dotsegments_absolute', + ]) + def test_remove_dot_segments(self, handler, path): + with handler(verbose=True) as rh: # This isn't a comprehensive test, - # but it should be enough to check whether the handler is removing dot segments - res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/a/b/./../../headers')) - assert res.status == 200 - assert res.url == f'http://127.0.0.1:{self.http_port}/headers' - res.close() - - res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}/redirect_dotsegments')) + # but it should be enough to check whether the handler is removing dot segments in required scenarios + res = validate_and_send(rh, Request(f'http://127.0.0.1:{self.http_port}{path}')) assert res.status == 200 assert res.url == f'http://127.0.0.1:{self.http_port}/headers' res.close() diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py index e129110ca..00e4bdb49 100644 --- a/yt_dlp/networking/_requests.py +++ b/yt_dlp/networking/_requests.py @@ -8,6 +8,7 @@ import warnings from ..dependencies import brotli, requests, urllib3 from ..utils import bug_reports_message, int_or_none, variadic +from ..utils.networking import normalize_url if requests is None: raise ImportError('requests module is not installed') @@ -199,6 +200,10 @@ class RequestsSession(requests.sessions.Session): prepared_request.method = new_method + # Requests fails to resolve dot segments on absolute redirect locations + # See: https://github.com/yt-dlp/yt-dlp/issues/9020 + prepared_request.url = normalize_url(prepared_request.url) + def rebuild_auth(self, prepared_request, response): # HACK: undo status code change from rebuild_method, if applicable. # rebuild_auth runs after requests would remove headers/body based on status code