rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Cut out things until it works

Browse Source
This commit is contained in:
one-some
2023-04-27 10:10:17 -05:00
parent 467f2f25eb
commit f9d162c001
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
aiserver.py
View File

@@ -2296,7 +2296,7 @@ def lua_encode(string):
from transformers import GPT2Tokenizer
global tokenizer
tokenizer = GPT2Tokenizer.from_pretrained("gpt2", revision=koboldai_vars.revision, cache_dir="cache")
return tokenizer.encode(utils.encodenewlines(string), max_length=int(4e9), truncation=True)
return tokenizer.encode(utils.encodenewlines(string))
#==================================================================#
# Computes context given a submission, Lua array of entry UIDs and a Lua array