rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Make 4-bit the default part 2

Browse Source
This commit is contained in:
Henk
2023-07-24 00:06:20 +02:00
parent 3409853dfc
commit a963c97acb
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -58,7 +58,7 @@ class model_backend(HFTorchInferenceModel):
"unit": "text", "unit": "text",
"label": "Quantization", "label": "Quantization",
"id": "quantization", "id": "quantization",
"default": temp['quantization'] if 'quantization' in temp else 'none', "default": temp['quantization'] if 'quantization' in temp else '4bit' if dependency_exists else '16-bit',
"tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode", "tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode",
"menu_path": "Layers", "menu_path": "Layers",
"children": [{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}, {'text': '16-bit', 'value':'16-bit'}], "children": [{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}, {'text': '16-bit', 'value':'16-bit'}],
@@ -66,7 +66,7 @@ class model_backend(HFTorchInferenceModel):
"refresh_model_inputs": False "refresh_model_inputs": False
}) })
else: else:
logger.warning("Bitsandbytes is not installed, you can not use Huggingface models in 4-bit") logger.warning("Bitsandbytes is not installed, you can not use Quantization for Huggingface models")
return requested_parameters return requested_parameters
def set_input_parameters(self, parameters): def set_input_parameters(self, parameters):