rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Fix for tokenizer stuff on pythia

Browse Source
This commit is contained in:
somebody
2023-04-09 18:23:58 -05:00
parent 3e8e3a18b0
commit 334c09606b
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modeling/tokenizer.py
View File

@@ -22,9 +22,10 @@ class GenericTokenizer:
setattr(self.tokenizer, name, value)
def encode(self, text: str) -> list:
if isinstance(self.tokenizer, PreTrainedTokenizer):
return self.tokenizer.encode(text)
return self.tokenizer.encode(text).ids
ret = self.tokenizer.encode(text)
if isinstance(ret, list):
return ret
return ret.ids
def decode(self, tokens: Union[int, List[int], torch.Tensor]) -> str:
if isinstance(tokens, torch.Tensor):