From 49740aa5abf406f7b9f6a60e60e23815c3f7007f Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sun, 23 Jul 2023 21:56:48 +0200 Subject: [PATCH] Fix ntk alpha --- modeling/inference_models/exllama/class.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modeling/inference_models/exllama/class.py b/modeling/inference_models/exllama/class.py index aa37a7aa..e3c7a874 100644 --- a/modeling/inference_models/exllama/class.py +++ b/modeling/inference_models/exllama/class.py @@ -430,6 +430,7 @@ class model_backend(InferenceModel): self.model_config.max_seq_len = parameters["max_ctx"] self.model_config.compress_pos_emb = parameters["compress_emb"] self.model_config.alpha_value = parameters["ntk_alpha"] + self.model_config.calculate_rotary_embedding_base() # Disable half2 for HIP self.model_config.rmsnorm_no_half2 = bool(torch.version.hip)