rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Fix llama 4-bit loading error

Browse Source
This commit is contained in:
0cc4m
2023-05-04 18:31:37 +02:00
parent ef358fdf5a
commit d48fedcbfb
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
modeling/inference_models/hf_torch_4bit.py
View File

@@ -14,7 +14,7 @@ from transformers import AutoModelForCausalLM, GPTNeoForCausalLM, AutoTokenizer,
import utils import utils
import modeling.lazy_loader as lazy_loader import modeling.lazy_loader as lazy_loader
import koboldai_settings import koboldai_settings
from logger import logger, set_logger_verbosity, quiesce_logger from logger import logger, set_logger_verbosity
try: try:
import breakmodel import breakmodel
@@ -24,6 +24,7 @@ except ModuleNotFoundError as e:
raise e raise e
from modeling.inference_models.hf_torch import HFTorchInferenceModel from modeling.inference_models.hf_torch import HFTorchInferenceModel
from modeling.tokenizer import GenericTokenizer
# 4-bit dependencies # 4-bit dependencies
from pathlib import Path from pathlib import Path
@@ -362,4 +363,4 @@ class HFTorch4BitInferenceModel(HFTorchInferenceModel):
else: else:
tokenizer = AutoTokenizer.from_pretrained(utils.koboldai_vars.custmodpth) tokenizer = AutoTokenizer.from_pretrained(utils.koboldai_vars.custmodpth)
return tokenizer return GenericTokenizer(tokenizer)