rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Patches: Make lazyload work on quantized

Browse Source 
i wanna watch youtube while my model is loading without locking up my
system >:(
This commit is contained in:
somebody
2023-07-17 16:47:31 -05:00
parent e8d84bb787
commit 23b95343bd
2 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -78,7 +78,6 @@ class model_backend(HFTorchInferenceModel):
}
if self.use_4_bit:
self.lazy_load = False
tf_kwargs.update({
"quantization_config":BitsAndBytesConfig(
load_in_4bit=True,

4
modeling/patches.py
View File

@@ -181,7 +181,7 @@ class LazyloadPatches:
is_quantized = is_quantized or load_in_8bit
if is_quantized:
from .utils.bitsandbytes import set_module_8bit_tensor_to_device
from transformers.utils.bitsandbytes import set_module_quantized_tensor_to_device
error_msgs = []
@@ -299,7 +299,7 @@ class LazyloadPatches:
fp16_statistics = None
if "SCB" not in param_name:
set_module_8bit_tensor_to_device(
set_module_quantized_tensor_to_device(
model,
param_name,
param_device,