rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Pull upstream changes, fix conflicts

Browse Source
This commit is contained in:
0cc4m
2023-07-15 23:01:52 +02:00
parent ed7ad00b59 0622810bc4
commit e78361fc8f
33 changed files with 3868 additions and 3147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modeling/inference_models/hf.py
View File

@@ -18,6 +18,7 @@ class HFInferenceModel(InferenceModel):
self.model_config = None
#self.model_name = model_name
self.hf_torch = False
self.model = None
self.tokenizer = None
self.badwordsids = koboldai_settings.badwordsids_default
@@ -158,7 +159,6 @@ class HFInferenceModel(InferenceModel):
def set_input_parameters(self, parameters):
if self.hf_torch and hasattr(self, "get_model_type") and self.get_model_type() != "gpt2":
import breakmodel
layer_count = self.model_config["n_layer"] if isinstance(self.model_config, dict) else self.model_config.num_layers if hasattr(self.model_config, "num_layers") else self.model_config.n_layer if hasattr(self.model_config, "n_layer") else self.model_config.num_hidden_layers if hasattr(self.model_config, 'num_hidden_layers') else None
if layer_count is not None and layer_count >= 0 and not self.nobreakmodel:
gpu_count = torch.cuda.device_count()
@@ -177,9 +177,8 @@ class HFInferenceModel(InferenceModel):
self.disk_layers = parameters['Disk_Layers'] if 'Disk_Layers' in parameters else 0
if isinstance(self.disk_layers, str):
self.disk_layers = int(self.disk_layers) if self.disk_layers.isnumeric() else 0
breakmodel.gpu_blocks = layers
breakmodel.disk_blocks = self.disk_layers
self.usegpu = self.cpu_layers == 0 and breakmodel.disk_blocks == 0 and sum(self.layers)-self.layers[0] == 0
print("TODO: Allow config")
# self.usegpu = self.cpu_layers == 0 and breakmodel.disk_blocks == 0 and sum(self.layers)-self.layers[0] == 0
self.model_type = self.get_model_type()
self.breakmodel = ((self.model_type != 'gpt2') or self.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not self.nobreakmodel
self.lazy_load = True