From a963c97acba850bea0ebb13b6fd2510834fe7a8f Mon Sep 17 00:00:00 2001 From: Henk Date: Mon, 24 Jul 2023 00:06:20 +0200 Subject: [PATCH] Make 4-bit the default part 2 --- modeling/inference_models/generic_hf_torch/class.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modeling/inference_models/generic_hf_torch/class.py b/modeling/inference_models/generic_hf_torch/class.py index a0fdedc6..a0ac9b55 100644 --- a/modeling/inference_models/generic_hf_torch/class.py +++ b/modeling/inference_models/generic_hf_torch/class.py @@ -58,7 +58,7 @@ class model_backend(HFTorchInferenceModel): "unit": "text", "label": "Quantization", "id": "quantization", - "default": temp['quantization'] if 'quantization' in temp else 'none', + "default": temp['quantization'] if 'quantization' in temp else '4bit' if dependency_exists else '16-bit', "tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode", "menu_path": "Layers", "children": [{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}, {'text': '16-bit', 'value':'16-bit'}], @@ -66,7 +66,7 @@ class model_backend(HFTorchInferenceModel): "refresh_model_inputs": False }) else: - logger.warning("Bitsandbytes is not installed, you can not use Huggingface models in 4-bit") + logger.warning("Bitsandbytes is not installed, you can not use Quantization for Huggingface models") return requested_parameters def set_input_parameters(self, parameters):