rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Save the 4-bit flag to the model settings.

Browse Source
This commit is contained in:
ebolam
2023-07-17 09:48:01 -04:00
parent eafb699bbf
commit 66377fc09e
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -30,12 +30,16 @@ class model_backend(HFTorchInferenceModel):
dependency_exists = importlib.util.find_spec("bitsandbytes") dependency_exists = importlib.util.find_spec("bitsandbytes")
if dependency_exists: if dependency_exists:
if model_name != 'customhuggingface' or "custom_model_name" in parameters: if model_name != 'customhuggingface' or "custom_model_name" in parameters:
if os.path.exists("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_"))) and 'base_url' not in vars(self):
with open("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_")), "r") as f:
temp = json.load(f)
else temp = {}
requested_parameters.append({ requested_parameters.append({
"uitype": "toggle", "uitype": "toggle",
"unit": "bool", "unit": "bool",
"label": "Use 4-bit", "label": "Use 4-bit",
"id": "use_4_bit", "id": "use_4_bit",
"default": False, "default": temp['use_4_bit'] if 'use_4_bit' in temp else False,
"tooltip": "Whether or not to use BnB's 4-bit mode", "tooltip": "Whether or not to use BnB's 4-bit mode",
"menu_path": "Layers", "menu_path": "Layers",
"extra_classes": "", "extra_classes": "",
@@ -292,6 +296,7 @@ class model_backend(HFTorchInferenceModel):
"disk_layers": self.disk_layers "disk_layers": self.disk_layers
if "disk_layers" in vars(self) if "disk_layers" in vars(self)
else 0, else 0,
"use_4_bit": self.use_4_bit,
}, },
f, f,
indent="", indent="",