rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #19 from henk7171/accelerate-offloading

Browse Source 
Remove wrong usegpu behavior
This commit is contained in:
one-some
2023-06-22 15:05:03 -05:00
committed by GitHub
parent 5ee20bd7d6 1da4580e8b
commit e62e3560bf
1 changed files with 0 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
modeling/inference_models/hf_torch.py
View File

@@ -114,9 +114,6 @@ class HFTorchInferenceModel(HFInferenceModel):
self.breakmodel_config.gpu_blocks = self.layers self.breakmodel_config.gpu_blocks = self.layers
self.breakmodel_config.disk_blocks = self.disk_layers self.breakmodel_config.disk_blocks = self.disk_layers
# HACK: Prevent get_auxiliary_device from returning cuda
utils.koboldai_vars.hascuda = self.usegpu
return ret return ret
def _get_target_dtype(self) -> Union[torch.float16, torch.float32]: def _get_target_dtype(self) -> Union[torch.float16, torch.float32]:
@@ -124,8 +121,6 @@ class HFTorchInferenceModel(HFInferenceModel):
return torch.float32 return torch.float32
elif utils.args.cpu: elif utils.args.cpu:
return torch.float32 return torch.float32
elif not self.usegpu:
return torch.float32
return torch.float16 return torch.float16
def _apply_warpers( def _apply_warpers(