rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Use device_map="auto"

Browse Source
This commit is contained in:
somebody
2023-07-12 17:27:48 -05:00
parent 60473d4c23
commit d17ce8461d
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modeling/inference_models/basic_hf/class.py
View File

@@ -55,7 +55,7 @@ class model_backend(HFInferenceModel):
self.init_model_config()
self.model = AutoModelForCausalLM.from_pretrained(
self.get_local_model_path(), low_cpu_mem_usage=True
self.get_local_model_path(), low_cpu_mem_usage=True, device_map="auto"
)
if self.usegpu: