rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

4-bit as Default

Browse Source
This commit is contained in:
Henk
2023-07-23 23:08:11 +02:00
parent 89637ae9d7
commit 0f913275a9
2 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
environments/huggingface.yml
View File

@@ -52,5 +52,3 @@ dependencies:
- einops - einops
- peft==0.3.0 - peft==0.3.0
- scipy - scipy
- --find-links=https://0cc4m.github.io/exllama/exllama-whl-links.html
- exllama==0.0.6

2
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -61,7 +61,7 @@ class model_backend(HFTorchInferenceModel):
"default": temp['quantization'] if 'quantization' in temp else 'none', "default": temp['quantization'] if 'quantization' in temp else 'none',
"tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode", "tooltip": "Whether or not to use BnB's 4-bit or 8-bit mode",
"menu_path": "Layers", "menu_path": "Layers",
"children": [{'text': 'None', 'value':'none'},{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}], "children": [{'text': '4-bit', 'value': '4bit'}, {'text': '8-bit', 'value': '8bit'}, {'text': '16-bit', 'value':'16-bit'}],
"extra_classes": "", "extra_classes": "",
"refresh_model_inputs": False "refresh_model_inputs": False
}) })