rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

CPU fixes

Browse Source
This commit is contained in:
Henk
2023-07-02 21:50:23 +02:00
parent 1da4580e8b
commit 81e72329af
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modeling/inference_models/hf_torch.py
View File

@@ -121,6 +121,8 @@ class HFTorchInferenceModel(HFInferenceModel):
return torch.float32 return torch.float32
elif utils.args.cpu: elif utils.args.cpu:
return torch.float32 return torch.float32
elif not self.usegpu and not self.breakmodel:
return torch.float32
return torch.float16 return torch.float16
def _apply_warpers( def _apply_warpers(
@@ -268,7 +270,9 @@ class HFTorchInferenceModel(HFInferenceModel):
gen_in = torch.tensor(prompt_tokens, dtype=torch.long)[None] gen_in = torch.tensor(prompt_tokens, dtype=torch.long)[None]
else: else:
gen_in = prompt_tokens gen_in = prompt_tokens
if not self.usegpu and not self.breakmodel:
gen_in = gen_in.to("cpu")
else:
device = utils.get_auxilary_device() device = utils.get_auxilary_device()
gen_in = gen_in.to(device) gen_in = gen_in.to(device)