rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Preliminary HF GPTQ changes

Browse Source
This commit is contained in:
Henk
2023-08-21 01:58:52 +02:00
parent 6f557befa9
commit 3dd0e91fbb
2 changed files with 32 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -57,6 +57,7 @@ class model_backend(HFTorchInferenceModel):
temp = json.load(f)
else:
temp = {}
if not hasattr(self.model_config, 'quantization_config'):
requested_parameters.append({
"uitype": "dropdown",
"unit": "text",
@@ -105,6 +106,7 @@ class model_backend(HFTorchInferenceModel):
"low_cpu_mem_usage": True,
}
if not hasattr(self.model_config, 'quantization_config'):
if self.quantization == "8bit":
tf_kwargs.update({
"quantization_config":BitsAndBytesConfig(

2
modeling/inference_models/gptq_hf_torch/class.py
View File

@@ -21,7 +21,7 @@ from pathlib import Path
model_backend_type = "GPTQ"
model_backend_name = "Huggingface GPTQ"
model_backend_name = "Legacy GPTQ"
def load_model_gptq_settings(path):