From d48fedcbfb7ba8dca9623215822ab1cbb700612e Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Thu, 4 May 2023 18:31:37 +0200 Subject: [PATCH] Fix llama 4-bit loading error --- modeling/inference_models/hf_torch_4bit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modeling/inference_models/hf_torch_4bit.py b/modeling/inference_models/hf_torch_4bit.py index f0ff87b9..10ef0e56 100644 --- a/modeling/inference_models/hf_torch_4bit.py +++ b/modeling/inference_models/hf_torch_4bit.py @@ -14,7 +14,7 @@ from transformers import AutoModelForCausalLM, GPTNeoForCausalLM, AutoTokenizer, import utils import modeling.lazy_loader as lazy_loader import koboldai_settings -from logger import logger, set_logger_verbosity, quiesce_logger +from logger import logger, set_logger_verbosity try: import breakmodel @@ -24,6 +24,7 @@ except ModuleNotFoundError as e: raise e from modeling.inference_models.hf_torch import HFTorchInferenceModel +from modeling.tokenizer import GenericTokenizer # 4-bit dependencies from pathlib import Path @@ -362,4 +363,4 @@ class HFTorch4BitInferenceModel(HFTorchInferenceModel): else: tokenizer = AutoTokenizer.from_pretrained(utils.koboldai_vars.custmodpth) - return tokenizer + return GenericTokenizer(tokenizer)