rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

4-bit on Colab

Browse Source
This commit is contained in:
Henk
2023-07-18 01:48:01 +02:00
parent da9226fba5
commit 5bbcdc47da
3 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -82,7 +82,7 @@ class model_backend(HFTorchInferenceModel):
"low_cpu_mem_usage": True,
}
if self.use_4_bit:
if self.use_4_bit or utils.koboldai_vars.colab_arg:
tf_kwargs.update({
"quantization_config":BitsAndBytesConfig(
load_in_4bit=True,