From ace436433900be313658ae34c178bbb8785d4b55 Mon Sep 17 00:00:00 2001 From: somebody Date: Thu, 27 Apr 2023 21:13:26 -0500 Subject: [PATCH] Two more time --- modeling/tokenizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modeling/tokenizer.py b/modeling/tokenizer.py index 2e58fedb..323c3885 100644 --- a/modeling/tokenizer.py +++ b/modeling/tokenizer.py @@ -28,10 +28,9 @@ class GenericTokenizer: return ret.ids def decode(self, tokens: Union[int, List[int], torch.Tensor]) -> str: - if isinstance(tokens, int): - tokens = [tokens] - # TODO: Figure out why this breaks softprompts on some models + # if isinstance(tokens, int): + # tokens = [tokens] # if isinstance(tokens, list): # tokens = torch.tensor(tokens) # elif isinstance(tokens, torch.Tensor):