rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Don't disable exllama

Browse Source
This commit is contained in:
Henk
2023-08-21 13:17:30 +02:00
parent 8daa2f1adc
commit 5917737676
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modeling/inference_models/gptq_hf_torch/class.py
View File

@@ -389,7 +389,7 @@ class model_backend(HFTorchInferenceModel):
except:
autogptq_failed = True # Ugly hack to get it to free the VRAM of the last attempt like we do above, better suggestions welcome - Henk
if autogptq_failed:
model = AutoGPTQForCausalLM.from_quantized(location, model_basename=Path(gptq_file).stem, use_safetensors=gptq_file.endswith(".safetensors"), device_map=device_map, disable_exllama=True)
model = AutoGPTQForCausalLM.from_quantized(location, model_basename=Path(gptq_file).stem, use_safetensors=gptq_file.endswith(".safetensors"), device_map=device_map, inject_fused_attention=False)
# Patch in embeddings function
def get_input_embeddings(self):
return self.model.get_input_embeddings()