rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Fix lazy-loading on 4-bit

Browse Source
This commit is contained in:
0cc4m
2023-04-17 07:21:18 +02:00
parent 4d34f9b7de
commit 1ef515f4c2
2 changed files with 11 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
modeling/inference_models/hf_torch.py
View File

@@ -412,14 +412,17 @@ class HFTorchInferenceModel(HFInferenceModel):
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
def get_original_key(key): def get_original_key(key):
return max( try:
( return max(
original_key (
for original_key in utils.module_names original_key
if original_key.endswith(key) for original_key in utils.module_names
), if original_key.endswith(key)
key=len, ),
) key=len,
)
except ValueError:
return key
for key, value in model_dict.items(): for key, value in model_dict.items():
original_key = get_original_key(key) original_key = get_original_key(key)

2
modeling/inference_models/hf_torch_4bit.py
View File

@@ -104,8 +104,6 @@ class HFTorch4BitInferenceModel(HFTorchInferenceModel):
) )
utils.koboldai_vars.model = self.model_name utils.koboldai_vars.model = self.model_name
self.lazy_load = False
self.init_model_config() self.init_model_config()
gpulayers = utils.args.breakmodel_gpulayers gpulayers = utils.args.breakmodel_gpulayers