rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Move overrides to better places

Browse Source
This commit is contained in:
somebody
2023-05-02 20:14:10 -05:00
parent 111028642e
commit efe268df60
3 changed files with 17 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modeling/inference_model.py
View File

@@ -197,12 +197,6 @@ class InferenceModel:
Returns:
AutoTokenizer: Tokenizer deemed fit for the location string. May be a fallback tokenizer.
"""
if utils.koboldai_vars.model_type == "xglm":
# Default to </s> newline mode if using XGLM
utils.koboldai_vars.newlinemode = "s"
elif utils.koboldai_vars.model_type in ["opt", "bloom"]:
# Handle </s> but don't convert newlines if using Fairseq models that have newlines trained in them
utils.koboldai_vars.newlinemode = "ns"
std_kwargs = {"revision": utils.koboldai_vars.revision, "cache_dir": "cache"}

17
modeling/inference_models/hf.py
View File

@@ -32,6 +32,23 @@ class HFInferenceModel(InferenceModel):
if utils.koboldai_vars.newlinemode == "n":
utils.koboldai_vars.badwordsids.append([self.tokenizer.eos_token_id])
# These are model specific tokenizer overrides if a model has bad defaults
if utils.koboldai_vars.model_type == "llama":
self.tokenizer.decode_with_prefix_space = True
self.tokenizer.add_bos_token = False
elif utils.koboldai_vars.model_type == "opt":
self.tokenizer._koboldai_header = self.tokenizer.encode("")
self.tokenizer.add_bos_token = False
self.tokenizer.add_prefix_space = False
# Change newline behavior to match model quirks
if utils.koboldai_vars.model_type == "xglm":
# Default to </s> newline mode if using XGLM
utils.koboldai_vars.newlinemode = "s"
elif utils.koboldai_vars.model_type in ["opt", "bloom"]:
# Handle </s> but don't convert newlines if using Fairseq models that have newlines trained in them
utils.koboldai_vars.newlinemode = "ns"
return super()._post_load()
def get_local_model_path(

9
modeling/inference_models/hf_torch.py
View File

@@ -132,15 +132,6 @@ class HFTorchInferenceModel(HFInferenceModel):
if not utils.koboldai_vars.model_type:
utils.koboldai_vars.model_type = m_self.get_model_type()
# These are model specific overrides if a model has bad defaults
if utils.koboldai_vars.model_type == "llama":
m_self.tokenizer.decode_with_prefix_space = True
m_self.tokenizer.add_bos_token = False
elif utils.koboldai_vars.model_type == "opt":
m_self.tokenizer._koboldai_header = m_self.tokenizer.encode("")
m_self.tokenizer.add_bos_token = False
m_self.tokenizer.add_prefix_space = False
# Patch stopping_criteria
class PTHStopper(StoppingCriteria):
def __call__(