Removed extra load_in_4bit.

2025-06-05 21:59:24 +02:00 · 2023-07-17 04:53:47 -06:00
parent e9913d657a
commit 145a43a000
1 changed files with 0 additions and 1 deletions
--- a/modeling/inference_models/generic_hf_torch/class.py
+++ b/modeling/inference_models/generic_hf_torch/class.py
@@ -80,7 +80,6 @@ class model_backend(HFTorchInferenceModel):
        if self.use_4_bit:
            self.lazy_load = False
            tf_kwargs.update({
-                "load_in_4bit": True,
                "quantization_config":BitsAndBytesConfig(
                    load_in_4bit=True,
                    bnb_4bit_compute_dtype=torch.float16,