From 8d008b87a608beb47e5f41473a40b437aa33d4b4 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Wed, 29 Mar 2023 13:25:06 +0000 Subject: [PATCH] Add OPT support --- aiserver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aiserver.py b/aiserver.py index 27cafd59..edce6bf1 100644 --- a/aiserver.py +++ b/aiserver.py @@ -94,6 +94,7 @@ sys.path.insert(0, os.path.abspath(Path("repos/gptq"))) from gptj import load_quant as gptj_load_quant from gptneox import load_quant as gptneox_load_quant from llama import load_quant as llama_load_quant +from opt import load_quant as opt_load_quant monkey_patched_4bit = False @@ -3169,6 +3170,9 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal elif koboldai_vars.model_type == "llama": model = llama_load_quant(koboldai_vars.custmodpth, path_4bit, 4, -1) tokenizer = LlamaTokenizer.from_pretrained(koboldai_vars.custmodpth) + elif koboldai_vars.model_type == "opt": + model = opt_load_quant(koboldai_vars.custmodpth, path_4bit, 4) + tokenizer = AutoTokenizer.from_pretrained(koboldai_vars.custmodpth) else: raise RuntimeError(f"4-bit load failed. Model type {koboldai_vars.model_type} not supported in 4-bit")