From 0c7eaefb1acc522eeed0b2dc1af78ec894b84a8b Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Tue, 13 Jun 2023 10:11:29 +0200 Subject: [PATCH] Fix AMD ROCm exllama inference --- modeling/inference_models/exllama/class.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modeling/inference_models/exllama/class.py b/modeling/inference_models/exllama/class.py index b17d04bf..37681b4f 100644 --- a/modeling/inference_models/exllama/class.py +++ b/modeling/inference_models/exllama/class.py @@ -362,5 +362,10 @@ class model_backend(InferenceModel): self.model_config.device_map.lm_head = "cuda:0" self.model_config.device_map.norm = "cuda:0" + self.model_config.rmsnorm_no_half2 = bool(torch.version.hip) + self.model_config.rope_no_half2 = bool(torch.version.hip) + self.model_config.matmul_no_half2 = bool(torch.version.hip) + self.model_config.silu_no_half2 = bool(torch.version.hip) + self.model_name = parameters['custom_model_name'] if 'custom_model_name' in parameters else parameters['id'] self.path = parameters['path'] if 'path' in parameters else None