rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Not quite

Browse Source
This commit is contained in:
somebody
2023-05-28 14:57:45 -05:00
parent ed0728188a
commit ceaefa9f5e
4 changed files with 25 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modeling/inference_models/hf_torch.py
View File

@@ -289,7 +289,8 @@ class HFTorchInferenceModel(HFInferenceModel):
device_map = infer_auto_device_map(
model,
max_memory={0: "10GiB", 1: "7GiB", "cpu": "15GiB"},
no_split_module_classes=["GPTJBlock"],
no_split_module_classes=["GPTJBlock", "OPTDecoderLayer"],
dtype="float16",
)
return AutoModelForCausalLM.from_pretrained(