mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-06-05 21:59:24 +02:00
TPU backend no longer needs to recompile after changing softprompt
This commit is contained in:
17
aiserver.py
17
aiserver.py
@ -1802,7 +1802,20 @@ def tpumtjgenerate(txt, minimum, maximum, found_entries=None):
|
|||||||
raise ValueError("Dynamic world info scanning is not supported by the TPU backend yet")
|
raise ValueError("Dynamic world info scanning is not supported by the TPU backend yet")
|
||||||
|
|
||||||
soft_tokens = None
|
soft_tokens = None
|
||||||
if(vars.sp is not None):
|
if(vars.sp is None):
|
||||||
|
global np
|
||||||
|
if 'np' not in globals():
|
||||||
|
import numpy as np
|
||||||
|
tensor = np.zeros((1, tpu_mtj_backend.params["d_model"]), dtype=np.float32)
|
||||||
|
rows = tensor.shape[0]
|
||||||
|
padding_amount = tpu_mtj_backend.params["seq"] - (tpu_mtj_backend.params["seq"] % -tpu_mtj_backend.params["cores_per_replica"]) - rows
|
||||||
|
tensor = np.pad(tensor, ((0, padding_amount), (0, 0)))
|
||||||
|
tensor = tensor.reshape(
|
||||||
|
tpu_mtj_backend.params["cores_per_replica"],
|
||||||
|
-1,
|
||||||
|
tpu_mtj_backend.params["d_model"],
|
||||||
|
)
|
||||||
|
vars.sp = tensor
|
||||||
soft_tokens = np.arange(
|
soft_tokens = np.arange(
|
||||||
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"],
|
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"],
|
||||||
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"] + vars.sp_length,
|
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"] + vars.sp_length,
|
||||||
@ -2676,7 +2689,7 @@ def spRequest(filename):
|
|||||||
|
|
||||||
if(vars.model in ("TPUMeshTransformerGPTJ",)):
|
if(vars.model in ("TPUMeshTransformerGPTJ",)):
|
||||||
rows = tensor.shape[0]
|
rows = tensor.shape[0]
|
||||||
padding_amount = -(rows % -tpu_mtj_backend.params["cores_per_replica"])
|
padding_amount = tpu_mtj_backend.params["seq"] - (tpu_mtj_backend.params["seq"] % -tpu_mtj_backend.params["cores_per_replica"]) - rows
|
||||||
tensor = np.pad(tensor, ((0, padding_amount), (0, 0)))
|
tensor = np.pad(tensor, ((0, padding_amount), (0, 0)))
|
||||||
tensor = tensor.reshape(
|
tensor = tensor.reshape(
|
||||||
tpu_mtj_backend.params["cores_per_replica"],
|
tpu_mtj_backend.params["cores_per_replica"],
|
||||||
|
Reference in New Issue
Block a user