Track token generation progress

2025-06-05 21:59:24 +02:00 · 2023-06-19 19:14:04 +02:00
parent e874f0c1c2
commit a191855b37
1 changed files with 4 additions and 0 deletions
--- a/modeling/inference_models/exllama/class.py
+++ b/modeling/inference_models/exllama/class.py
@@ -300,8 +300,12 @@ class model_backend(InferenceModel):

            self._post_token_gen(self.generator.sequence)

+            utils.koboldai_vars.generated_tkns += 1
+
            if token.item() == self.tokenizer.eos_token_id: break

+        utils.koboldai_vars.generated_tkns = max_new
+
        return GenerationResult(
            model=self,
            out_batches=np.array(