rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Add sliders for exllama context size and related methods

Browse Source
This commit is contained in:
0cc4m
2023-07-23 07:11:28 +02:00
parent 58908ab846
commit 748e5ef318
2 changed files with 53 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modeling/inference_models/gptq_hf_torch/class.py
View File

@@ -56,7 +56,7 @@ def load_model_gptq_settings(path):
gptq_file = False
gptq_version = -1
gptq_legacy_files = glob.glob(os.path.join(path, "4bit*.pt")) + glob.glob(os.path.join(path, "4bit*.safetensors"))
gptq_legacy_files = glob.glob(os.path.join(path, "*4bit*.pt")) + glob.glob(os.path.join(path, "*4bit*.safetensors"))
if "gptq_bits" in js:
gptq_model = True
gptq_bits = js["gptq_bits"]
@@ -70,7 +70,7 @@ def load_model_gptq_settings(path):
gptq_bits = 4
gptq_file = gptq_legacy_files[0]
fname = Path(gptq_file).parts[-1]
g = re.findall("^(?:4bit)(?:-)(\\d+)(?:g-?)", fname)
g = re.findall("(?:4bit)(?:-)(\\d+)(?:g-?)", fname)
gptq_groupsize = int(g[0]) if g else -1
gptq_version = -1