rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Add 4bit plugin

Browse Source
This commit is contained in:
0cc4m
2023-05-28 22:54:36 +02:00
parent d71a63fa49
commit e49d35afc9
5 changed files with 233 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -25,8 +25,12 @@ from modeling.inference_models.hf_torch import HFTorchInferenceModel
model_backend_name = "Huggingface"
class GenericHFTorchInferenceModel(HFTorchInferenceModel):
def load_config(self) -> None:
class model_backend(HFTorchInferenceModel):
def _initialize_model(self):
return
def _load(self, save_model: bool, initial_load: bool) -> None:
utils.koboldai_vars.allowsp = True
# Make model path the same as the model name to make this consistent
@@ -243,11 +247,6 @@ class GenericHFTorchInferenceModel(HFTorchInferenceModel):
)
shutil.rmtree("cache/")
if not self.lazy_load:
utils.layers_module_names = utils.get_layers_module_names(self.model)
utils.module_names = list(self.model.state_dict().keys())
utils.named_buffers = list(self.model.named_buffers(recurse=True))
self.patch_embedding()