rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

MTJ Fix

Browse Source
This commit is contained in:
somebody
2022-09-22 21:34:54 -05:00
parent 92e240a6ba
commit ffbe50920e
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
aiserver.py
View File

@@ -4905,15 +4905,15 @@ def core_generate(text: list, min: int, max: int, found_entries: set):
__debug("generate result", result.__dict__) __debug("generate result", result.__dict__)
if result.is_whole_generation:
__debug("Outa here")
break
genout = result.encoded genout = result.encoded
already_generated += len(genout[0]) - len(gen_in[0]) already_generated += len(genout[0]) - len(gen_in[0])
assert already_generated <= koboldai_vars.genamt assert already_generated <= koboldai_vars.genamt
if result.is_whole_generation:
__debug("Outa here")
break
# Generation stopped; why? # Generation stopped; why?
# If we have been told to halt, we have reached our target token # If we have been told to halt, we have reached our target token
# amount (controlled by halt), or Dynamic WI has not told us to # amount (controlled by halt), or Dynamic WI has not told us to
@@ -5058,6 +5058,7 @@ def tpu_raw_generate(
# Mostly lifted from apiactionsubmit_tpumtjgenerate # Mostly lifted from apiactionsubmit_tpumtjgenerate
soft_tokens = tpumtjgetsofttokens() soft_tokens = tpumtjgetsofttokens()
__debug("we are generating with", prompt_tokens, "batch", batch_count, "soft tokens", soft_tokens) __debug("we are generating with", prompt_tokens, "batch", batch_count, "soft tokens", soft_tokens)
genout = tpool.execute( genout = tpool.execute(
tpu_mtj_backend.infer_static, tpu_mtj_backend.infer_static,
np.uint32(prompt_tokens), np.uint32(prompt_tokens),