From 0f913275a93bb2e2ba4d557625845e79f7270018 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 23 Jul 2023 23:08:11 +0200 Subject: [PATCH] 4-bit as Default --- environments/huggingface.yml | 2 -- modeling/inference_models/generic_hf_torch/class.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/environments/huggingface.yml b/environments/huggingface.yml index 89cd4c89..64224dc3 100644 --- a/environments/huggingface.yml +++ b/environments/huggingface.yml @@ -52,5 +52,3 @@ dependencies: - einops - peft==0.3.0 - scipy - - --find-links=https://0cc4m.github.io/exllama/exllama-whl-links.html - - exllama==0.0.6 diff --git a/modeling/inference_models/generic_hf_torch/class.py b/modeling/inference_models/generic_hf_torch/class.py index 1bf38b99..a0fdedc6 100644 --- a/modeling/inference_models/generic_hf_torch/class.py +++ b/modeling/inference_models/generic_hf_torch/class.py @@ -61,7 +61,7 @@ class model_backend(HFTorchInferenceModel): "default": temp['quantization'] if 'quantization' in temp else 'none', "tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode", "menu_path": "Layers", - "children": [{'text': 'None', 'value':'none'},{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}], + "children": [{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}, {'text': '16-bit', 'value':'16-bit'}], "extra_classes": "", "refresh_model_inputs": False })