Back-port test_youtube_signature.py from yt-dlp and fix JSInterp accordingly

test/test_youtube_signature.py

@@ -14,9 +14,10 @@ import string
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.jsinterp import JSInterpreter
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@ _TESTS = [
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
@@ -95,35 +115,54 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
+def t_factory(name, sig_func, url_pattern):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
         assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+        test_id = m.group('id')
 
         def test_func(self):
-        basename = 'player-%s.js' % test_id
+            basename = 'player-{0}-{1}.js'.format(name, test_id)
             fn = os.path.join(self.TESTDATA_DIR, basename)
 
             if not os.path.exists(fn):
                 compat_urlretrieve(url, fn)
-
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
             with io.open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
+
+        test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
+
+
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
     src_sig = (
         compat_str(string.printable[:sig_input])
         if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
+    return func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
-
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
 
-for test_spec in _TESTS:
+def n_sig(jscode, sig_input):
-    make_tfunc(*test_spec)
+    # Pending implementation of _extract_n_function_name() or similar in
+    # youtube.py, hard-code here
+    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    import re
+    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
+    funcname = funcname and funcname.group(1)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
+
+
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
+
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':

youtube_dl/jsinterp.py

@@ -9,7 +9,8 @@ from .utils import (
     remove_quotes,
 )
 from .compat import (
-    compat_collections_abc
+    compat_collections_abc,
+    compat_str,
 )
 MutableMapping = compat_collections_abc.MutableMapping
 
@@ -372,7 +373,7 @@ class JSInterpreter(object):
                 # nonlocal member
                 member = nl.member
                 if variable == 'String':
-                    obj = str
+                    obj = compat_str
                 elif variable in local_vars:
                     obj = local_vars[variable]
                 else:
@@ -391,7 +392,7 @@ class JSInterpreter(object):
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == str:
+                if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
@@ -533,7 +534,7 @@ class JSInterpreter(object):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    [x.strip() for x in mobj.group('args').split(',')],
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)