From b9600e432940abfde4f2b28bfec7311b7c1322a8 Mon Sep 17 00:00:00 2001
From: ebolam <ebolam@gmail.com>
Date: Sat, 16 Sep 2023 21:17:26 -0400
Subject: [PATCH] better text size handling for tortoise

---
 koboldai_settings.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/koboldai_settings.py b/koboldai_settings.py
index ad819abe..5a9201d7 100644
--- a/koboldai_settings.py
+++ b/koboldai_settings.py
@@ -2134,8 +2134,17 @@ class KoboldStoryRegister(object):
                 if text.strip() == "":
                     shutil.copy("data/empty_audio.ogg", filename)
                 else:
-                    if len(text) > 400:
+                    if len(self.tortoise.tokenizer.encode(text)) > 400:
                         text = self.sentence_re.findall(text)
+                        i=0
+                        while i <= len(text)-2:
+                            if len(self.tortoise.tokenizer.encode(text[i] + text[i+1])) < 400:
+                                text[i] = text[i] + text[i+1]
+                                del text[i+1]
+                            else:
+                                i+=1
+                                    
+                        
                     else:
                         text = [text]
                 output = None