rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Model: Fix eos/bos padding issue

Browse Source 
Weird config None assignments
This commit is contained in:
somebody
2023-03-10 18:36:22 -06:00
parent 3ac3e41dd8
commit 38c4edac40
2 changed files with 3 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
aiserver.py
View File

@@ -5493,26 +5493,7 @@ def final_startup():
# Precompile TPU backend if required # Precompile TPU backend if required
if isinstance(model, HFMTJInferenceModel): if isinstance(model, HFMTJInferenceModel):
import tpu_mtj_backend model.raw_generate([23403, 727, 20185], max_new=1)
soft_tokens = model.get_soft_tokens()
if(koboldai_vars.dynamicscan or (not koboldai_vars.nogenmod and koboldai_vars.has_genmod)):
tpool.execute(tpu_mtj_backend.infer_dynamic, np.tile(np.uint32((23403, 727, 20185)), (koboldai_vars.numseqs, 1)),
soft_embeddings= koboldai_vars.sp,
soft_tokens= soft_tokens,
gen_len= 1,
use_callback= False,
numseqs= koboldai_vars.numseqs,
excluded_world_info= list(set() for _ in range(koboldai_vars.numseqs))
)
else:
tpool.execute(
tpu_mtj_backend.infer_static,
np.uint32((23403, 727, 20185)),
soft_embeddings= koboldai_vars.sp,
soft_tokens= soft_tokens,
gen_len= 1,
numseqs= koboldai_vars.numseqs
)
# Set the initial RNG seed # Set the initial RNG seed
set_seed() set_seed()

4
tpu_mtj_backend.py
View File

@@ -779,9 +779,9 @@ def read_neox_checkpoint(state, path, config, checkpoint_shards=2):
def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None:
global thread_resources_env, seq, tokenizer, network, params, pad_token_id global thread_resources_env, seq, tokenizer, network, params, pad_token_id
if "pad_token_id" in kwargs: if kwargs.get("pad_token_id"):
pad_token_id = kwargs["pad_token_id"] pad_token_id = kwargs["pad_token_id"]
elif "eos_token_id" in kwargs: elif kwargs.get("eos_token_id"):
pad_token_id = kwargs["eos_token_id"] pad_token_id = kwargs["eos_token_id"]
if not hasattr(koboldai_vars, "sampler_order") or not koboldai_vars.sampler_order: if not hasattr(koboldai_vars, "sampler_order") or not koboldai_vars.sampler_order: