rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

BnB dependency check

Browse Source
This commit is contained in:
Henk
2023-07-15 18:56:13 +02:00
parent 160effb9ea
commit c43d60772b
1 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -11,6 +11,7 @@ from transformers import AutoModelForCausalLM, GPTNeoForCausalLM, GPT2LMHeadMode
import utils import utils
import modeling.lazy_loader as lazy_loader import modeling.lazy_loader as lazy_loader
import koboldai_settings import koboldai_settings
import importlib
from logger import logger from logger import logger
@@ -26,6 +27,8 @@ class model_backend(HFTorchInferenceModel):
def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}): def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}):
requested_parameters = super().get_requested_parameters(model_name, model_path, menu_path, parameters = {}) requested_parameters = super().get_requested_parameters(model_name, model_path, menu_path, parameters = {})
dependency_exists = importlib.util.find_spec("bitsandbytes")
if dependency_exists:
requested_parameters.append({ requested_parameters.append({
"uitype": "toggle", "uitype": "toggle",
"unit": "bool", "unit": "bool",
@@ -37,6 +40,8 @@ class model_backend(HFTorchInferenceModel):
"extra_classes": "", "extra_classes": "",
"refresh_model_inputs": False "refresh_model_inputs": False
}) })
else:
logger.warning("Bitsandbytes is not installed, you can not use Huggingface models in 4-bit")
return requested_parameters return requested_parameters
def set_input_parameters(self, parameters): def set_input_parameters(self, parameters):