Two more time

2025-06-05 21:59:24 +02:00 · 2023-04-27 21:13:26 -05:00
parent 446f38ee9d
commit ace4364339
1 changed files with 2 additions and 3 deletions
--- a/modeling/tokenizer.py
+++ b/modeling/tokenizer.py
@@ -28,10 +28,9 @@ class GenericTokenizer:
        return ret.ids

    def decode(self, tokens: Union[int, List[int], torch.Tensor]) -> str:
-        if isinstance(tokens, int):
-            tokens = [tokens]
-
        # TODO: Figure out why this breaks softprompts on some models
+        # if isinstance(tokens, int):
+        #     tokens = [tokens]
        # if isinstance(tokens, list):
        #     tokens = torch.tensor(tokens)
        # elif isinstance(tokens, torch.Tensor):