rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Hide exllama if not installed

Browse Source
This commit is contained in:
Henk
2023-09-11 01:21:29 +02:00
parent fff28daafd
commit 2455e5d60c
1 changed files with 33 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
modeling/inference_models/exllama/class.py
View File

@@ -1,5 +1,5 @@
from __future__ import annotations from __future__ import annotations
try:
import time, json import time, json
import torch import torch
import requests import requests
@@ -28,9 +28,13 @@ from modeling.inference_model import (
from modeling.tokenizer import GenericTokenizer from modeling.tokenizer import GenericTokenizer
from exllama.model import ExLlama, ExLlamaCache, ExLlamaConfig from exllama.model import ExLlama, ExLlamaCache, ExLlamaConfig
from transformers import LlamaTokenizer from transformers import LlamaTokenizer
from exllama.generator import ExLlamaGenerator from exllama.generator import ExLlamaGenerator
load_failed = False
except:
load_failed = True
model_backend_type = "GPTQ" model_backend_type = "GPTQ"
model_backend_name = "ExLlama" model_backend_name = "ExLlama"
@@ -101,10 +105,11 @@ class model_backend(InferenceModel):
stopper_hooks=True, stopper_hooks=True,
post_token_probs=False, post_token_probs=False,
) )
self.disable = load_failed
def is_valid(self, model_name, model_path, menu_path): def is_valid(self, model_name, model_path, menu_path):
gptq_model, _ = load_model_gptq_settings(model_path)
try: try:
gptq_model, _ = load_model_gptq_settings(model_path)
self.model_config = self._load_config(model_name, model_path) self.model_config = self._load_config(model_name, model_path)
return self.model_config and gptq_model return self.model_config and gptq_model
except: except: