From d838c966ee289b4ded912420c7c73d8b74374e55 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sat, 16 Sep 2023 19:41:56 -0400 Subject: [PATCH 1/5] update for tortoise tts --- koboldai_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index b7408b93..80bfa5c2 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2121,7 +2121,7 @@ class KoboldStoryRegister(object): sample_rate = 24000 speaker = 'train_daws' if self.tortoise is None and importlib.util.find_spec("tortoise") is not None: - self.tortoise=api.TextToSpeech() + self.tortoise=api.TextToSpeech(use_deepspeed=True, kv_cache=True, half=True) if importlib.util.find_spec("tortoise") is not None: voice_samples, conditioning_latents = load_voices([speaker]) From 92a24a9f1e029e63280e152b80ef580a76763260 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sat, 16 Sep 2023 20:32:46 -0400 Subject: [PATCH 2/5] Changed tortoise tts to use environmental variables to overwrite defaults --- koboldai_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index 80bfa5c2..7297c336 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2121,7 +2121,7 @@ class KoboldStoryRegister(object): sample_rate = 24000 speaker = 'train_daws' if self.tortoise is None and importlib.util.find_spec("tortoise") is not None: - self.tortoise=api.TextToSpeech(use_deepspeed=True, kv_cache=True, half=True) + self.tortoise=api.TextToSpeech(use_deepspeed=os.environ.get('deepspeed', "true").lower()=="true", kv_cache=os.environ.get('kv_cache', "true").lower()=="true", half=True) if importlib.util.find_spec("tortoise") is not None: voice_samples, conditioning_latents = load_voices([speaker]) From 81705e4318067d6726d74e4f42f5697afcc02e87 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sat, 16 Sep 2023 20:37:15 -0400 Subject: [PATCH 3/5] Fix for blank actions in TTS export --- koboldai_settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index 7297c336..ff097a76 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2145,7 +2145,8 @@ class KoboldStoryRegister(object): output = pydub.AudioSegment(np.int16(audio * 2 ** 15).tobytes(), frame_rate=sample_rate, sample_width=2, channels=channels) else: output = output + pydub.AudioSegment(np.int16(audio * 2 ** 15).tobytes(), frame_rate=sample_rate, sample_width=2, channels=channels) - output.export(filename, format="ogg", bitrate="16k") + if output is not None: + output.export(filename, format="ogg", bitrate="16k") logger.info("Slow audio took {} for {} characters".format(time.time()-start_time, text_length)) def gen_all_audio(self, overwrite=False): From 92755a03dafa0a504cb3b3f6f681089601da00ce Mon Sep 17 00:00:00 2001 From: ebolam Date: Sat, 16 Sep 2023 20:51:29 -0400 Subject: [PATCH 4/5] Better text splitting for TTS --- koboldai_settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index ff097a76..ad819abe 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2114,7 +2114,8 @@ class KoboldStoryRegister(object): output = pydub.AudioSegment(np.int16(audio * 2 ** 15).tobytes(), frame_rate=sample_rate, sample_width=2, channels=channels) else: output = output + pydub.AudioSegment(np.int16(audio * 2 ** 15).tobytes(), frame_rate=sample_rate, sample_width=2, channels=channels) - output.export(filename, format="ogg", bitrate="16k") + if output is not None: + output.export(filename, format="ogg", bitrate="16k") def create_wave_slow(self, make_audio_queue_slow): import pydub @@ -2133,7 +2134,7 @@ class KoboldStoryRegister(object): if text.strip() == "": shutil.copy("data/empty_audio.ogg", filename) else: - if len(text) > 20000: + if len(text) > 400: text = self.sentence_re.findall(text) else: text = [text] From b9600e432940abfde4f2b28bfec7311b7c1322a8 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sat, 16 Sep 2023 21:17:26 -0400 Subject: [PATCH 5/5] better text size handling for tortoise --- koboldai_settings.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index ad819abe..5a9201d7 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2134,8 +2134,17 @@ class KoboldStoryRegister(object): if text.strip() == "": shutil.copy("data/empty_audio.ogg", filename) else: - if len(text) > 400: + if len(self.tortoise.tokenizer.encode(text)) > 400: text = self.sentence_re.findall(text) + i=0 + while i <= len(text)-2: + if len(self.tortoise.tokenizer.encode(text[i] + text[i+1])) < 400: + text[i] = text[i] + text[i+1] + del text[i+1] + else: + i+=1 + + else: text = [text] output = None