From 8bef2e5fef52c88fc3318c7108e90d1441023731 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 16 Jul 2023 02:02:58 +0200 Subject: [PATCH] Fixes 16-bit if BnB is not installed --- modeling/inference_models/generic_hf_torch/class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modeling/inference_models/generic_hf_torch/class.py b/modeling/inference_models/generic_hf_torch/class.py index 10429897..40006dab 100644 --- a/modeling/inference_models/generic_hf_torch/class.py +++ b/modeling/inference_models/generic_hf_torch/class.py @@ -47,7 +47,7 @@ class model_backend(HFTorchInferenceModel): def set_input_parameters(self, parameters): super().set_input_parameters(parameters) - self.use_4_bit = parameters['use_4_bit'] + self.use_4_bit = parameters['use_4_bit'] if 'use_4_bit' in parameters else False def _load(self, save_model: bool, initial_load: bool) -> None: utils.koboldai_vars.allowsp = True