From d70ed69dae985ec53f3286dc61c78a4d8e7983b5 Mon Sep 17 00:00:00 2001 From: ebolam Date: Fri, 11 Nov 2022 18:36:48 -0500 Subject: [PATCH] Fixed new sampler showing in Original UI Initial skeleton for tts (experimental feature) --- aiserver.py | 15 +++++++- gensettings.py | 18 ++++++++++ koboldai_settings.py | 63 +++++++++++++++++++++++++++++++--- static/koboldai.js | 1 + templates/settings flyout.html | 3 ++ 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/aiserver.py b/aiserver.py index 61dc9d33..dee0f249 100644 --- a/aiserver.py +++ b/aiserver.py @@ -536,7 +536,7 @@ def UI_2_log_history(message): del web_log_history[0] web_log_history.append(data) -from flask import Flask, render_template, Response, request, copy_current_request_context, send_from_directory, session, jsonify, abort, redirect, has_request_context +from flask import Flask, render_template, Response, request, copy_current_request_context, send_from_directory, session, jsonify, abort, redirect, has_request_context, send_file from flask_socketio import SocketIO, emit, join_room, leave_room from flask_socketio import emit as _emit from flask_session import Session @@ -9564,6 +9564,19 @@ def UI_2_test_match(): koboldai_vars.assign_world_info_to_actions() return show_vars() + +@app.route("/audio") +@logger.catch +def UI_2_audio(): + action_id = int(request.args['id']) if 'id' in request.args else len(koboldai_vars.actions) + filename="stories/{}/{}.wav".format(koboldai_vars.story_id, action_id) + if not os.path.exists(filename): + koboldai_vars.actions.gen_audio(action_id) + return send_file( + filename, + mimetype="audio/wav") + + #==================================================================# # Test #==================================================================# diff --git a/gensettings.py b/gensettings.py index c2ed3f8a..9ef65062 100644 --- a/gensettings.py +++ b/gensettings.py @@ -106,6 +106,7 @@ gensettingstf = [ "name": "top_a" }, { + "UI_V2_Only": True, "uitype": "slider", "unit": "float", "label": "Penalty Alpha", @@ -604,6 +605,23 @@ gensettingstf = [ "name": "experimental_features" }, { + "UI_V2_Only": True, + "uitype": "toggle", + "unit": "bool", + "label": "Generate Audio", + "id": "gen_audio", + "min": 0, + "max": 1, + "step": 1, + "default": 0, + "tooltip": "If enabled, The system will generate audio files for each action.", + "menu_path": "Interface", + "sub_path": "UI", + "classname": "story", + "name": "gen_audio", + "extra_classes": "var_sync_alt_system_experimental_features" + }, + { "UI_V2_Only": True, "uitype": "text", "unit": "text", diff --git a/koboldai_settings.py b/koboldai_settings.py index 12e671e1..1c8f72ec 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -8,6 +8,7 @@ from collections import OrderedDict import multiprocessing from logger import logger import eventlet +import torch serverstarted = False queue = None @@ -794,6 +795,7 @@ class story_settings(settings): # {"target": "(r)", "substitution": "®", "enabled": False}, # {"target": "(tm)", "substitution": "™", "enabled": False}, ] + self.gen_audio = False # bias experiment self.memory_attn_bias = 1 @@ -907,8 +909,10 @@ class story_settings(settings): #Change game save state if name in ['story_name', 'prompt', 'memory', 'authornote', 'authornotetemplate', 'andepth', 'chatname', 'actionmode', 'dynamicscan', 'notes', 'biases']: self.gamesaved = False - - if name == 'useprompt': + + if name == "gen_audio" and value: + self.actions.gen_all_audio() + elif name == 'useprompt': ignore = self.koboldai_vars.calc_ai_text() elif name == 'actions': self.actions.story_settings = self @@ -1002,11 +1006,11 @@ class system_settings(settings): local_only_variables = ['socketio', 'lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer'] + 'summarizer', 'summary_tokenizer', 'tts_model'] no_save_variables = ['socketio', 'lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', - 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link'] + 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model'] settings_name = "system" def __init__(self, socketio, koboldai_var): self.socketio = socketio @@ -1160,6 +1164,9 @@ class KoboldStoryRegister(object): #Options being a list of dict with keys of "text", "Pinned", "Previous Selection", "Edited", "Probabilities" self.action_count = -1 self.story_settings = story_settings + self.tts_model = None + self.make_audio_thread = None + self.make_audio_queue = multiprocessing.Queue() for item in sequence: self.append(item) @@ -1266,6 +1273,7 @@ class KoboldStoryRegister(object): logger.debug("Calcing AI Text from Action __setitem__") ignore = self.koboldai_vars.calc_ai_text() self.set_game_saved() + self.gen_audio(i) def __len__(self): return self.action_count+1 if self.action_count >=0 else 0 @@ -1309,6 +1317,7 @@ class KoboldStoryRegister(object): logger.debug("Calcing AI Text from Action load from json") ignore = self.koboldai_vars.calc_ai_text() self.set_game_saved() + self.gen_all_audio() def append(self, text, action_id_offset=0, recalc=True): if self.koboldai_vars.remove_double_space: @@ -1341,6 +1350,7 @@ class KoboldStoryRegister(object): if recalc: logger.debug("Calcing AI Text from Action Append") ignore = self.koboldai_vars.calc_ai_text() + self.gen_audio(action_id) def append_options(self, option_list): if self.action_count+1 in self.actions: @@ -1448,6 +1458,7 @@ class KoboldStoryRegister(object): self.set_game_saved() logger.debug("Calcing AI Text from Action Use Option") ignore = self.koboldai_vars.calc_ai_text() + self.gen_audio(action_step) def delete_option(self, option_number, action_step=None): if action_step is None: @@ -1636,6 +1647,50 @@ class KoboldStoryRegister(object): #logger.debug("to_sentences: {}s".format(time.time()-start_time)) return action_text_split + def gen_audio(self, action_id=None, overwrite=True): + if self.story_settings.gen_audio and self.koboldai_vars.experimental_features: + if action_id is None: + action_id = self.action_count + + logger.info("Generating audio for action {}".format(action_id)) + + if self.tts_model is None: + language = 'en' + model_id = 'v3_en' + self.tts_model, example_text = torch.hub.load(repo_or_dir='snakers4/silero-models', + model='silero_tts', + language=language, + speaker=model_id) + #self.tts_model.to(torch.device(0)) # gpu or cpu + self.tts_model.to(torch.device("cpu")) # gpu or cpu + + filename="stories/{}/{}.wav".format(self.story_settings.story_id, action_id) + if not os.path.exists("stories/{}".format(self.story_settings.story_id)): + os.mkdir("stories/{}".format(self.story_settings.story_id)) + + if overwrite or not os.path.exists(filename): + self.make_audio_queue.put((self.actions[action_id]['Selected Text'], filename)) + if self.make_audio_thread is None or not self.make_audio_thread.is_alive(): + self.make_audio_thread = threading.Thread(target=self.create_wave, args=(self.tts_model, self.make_audio_queue)) + self.make_audio_thread.start() + + def create_wave(self, model, make_audio_queue): + sample_rate = 48000 + speaker = 'en_5' + while not make_audio_queue.empty(): + (text, filename) = make_audio_queue.get() + self.tts_model.save_wav(text=text, + speaker=speaker, + sample_rate=sample_rate, + audio_path=filename) + + def gen_all_audio(self, overwrite=False): + if self.story_settings.gen_audio and self.koboldai_vars.experimental_features: + for i in reversed(list(self.actions.keys())): + self.gen_audio(i, overwrite=False) + else: + print("{} and {}".format(self.story_settings.gen_audio, self.koboldai_vars.experimental_features)) + def __setattr__(self, name, value): new_variable = name not in self.__dict__ old_value = getattr(self, name, None) diff --git a/static/koboldai.js b/static/koboldai.js index fa91a8bd..95fc9ea4 100644 --- a/static/koboldai.js +++ b/static/koboldai.js @@ -373,6 +373,7 @@ function process_actions_data(data) { hide_show_prompt(); //console.log("Took "+((Date.now()-start_time)/1000)+"s to process"); + } function do_story_text_updates(action) { diff --git a/templates/settings flyout.html b/templates/settings flyout.html index 78d82939..01ba3880 100644 --- a/templates/settings flyout.html +++ b/templates/settings flyout.html @@ -83,6 +83,9 @@
+
+
{% with menu='Home' %} {% with sub_path='' %}