From fc7a07a62354b2fe59fda9d990a414d481541115 Mon Sep 17 00:00:00 2001 From: ebolam Date: Wed, 17 Jan 2024 18:00:12 -0500 Subject: [PATCH] Fix for koboldcpp model backend --- modeling/inference_models/koboldcpp/class.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modeling/inference_models/koboldcpp/class.py b/modeling/inference_models/koboldcpp/class.py index bdf511c0..802d7ca0 100644 --- a/modeling/inference_models/koboldcpp/class.py +++ b/modeling/inference_models/koboldcpp/class.py @@ -301,10 +301,10 @@ class model_backend(InferenceModel): # Store context in memory to use it for comparison with generated content utils.koboldai_vars.lastctx = decoded_prompt - self.input_queue.put({'command': 'generate', 'data': [(decoded_prompt,max_new,utils.koboldai_vars.max_length, - gen_settings.temp,int(gen_settings.top_k),gen_settings.top_a,gen_settings.top_p, - gen_settings.typical,gen_settings.tfs,gen_settings.rep_pen,gen_settings.rep_pen_range), - {"sampler_order": gen_settings.sampler_order, "use_default_badwordsids": utils.koboldai_vars.use_default_badwordsids} + self.input_queue.put({'command': 'generate', 'data': [(decoded_prompt,), {'max_length': max_new, 'max_context_length': utils.koboldai_vars.max_length, + 'temperature': gen_settings.temp, 'top_k': int(gen_settings.top_k), 'top_a': gen_settings.top_a, 'top_p': gen_settings.top_p, + 'typical_p': gen_settings.typical, 'tfs': gen_settings.tfs, 'rep_pen': gen_settings.rep_pen, 'rep_pen_range': gen_settings.rep_pen_range, + "sampler_order": gen_settings.sampler_order, "use_default_badwordsids": utils.koboldai_vars.use_default_badwordsids} ]}) #genresult = koboldcpp.generate(decoded_prompt,max_new,utils.koboldai_vars.max_length,