Merge branch 'staging' of https://github.com/SillyLossy/TavernAI into staging

public/scripts/extensions/expressions/index.js

@@ -9,6 +9,7 @@ const MODULE_NAME = 'expressions';
 const UPDATE_INTERVAL = 2000;
 const FALLBACK_EXPRESSION = 'joy';
 const DEFAULT_EXPRESSIONS = [
+    "live2d",
     "admiration",
     "amusement",
     "anger",
@@ -44,6 +45,9 @@ let lastCharacter = undefined;
 let lastMessage = null;
 let spriteCache = {};
 let inApiCall = false;
+let live2d_var = false;
+let previousSrc = null;
+
 
 function isVisualNovelMode() {
     return Boolean(!isMobile() && power_user.waifuMode && getContext().groupId);
@@ -392,6 +396,56 @@ function onExpressionsShowDefaultInput() {
     }
 }
 
+function loadLiveChar(value_name) {
+    let url = `${getApiUrl()}/api/live2d/load?loadchar=${location.origin}/characters/${value_name}`;
+    doExtrasFetch(url, {
+        method: 'GET',
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Bypass-Tunnel-Reminder': 'bypass',
+        },
+    })
+        .then(response => response.text())
+        .then(data => console.log(data))
+        .catch((error) => console.error('Error:', error));
+}
+
+
+function handleImageChange(isChecked) {
+    const imgElement = document.querySelector('img#expression-image.expression');
+
+    if (!imgElement) {
+        console.log("Cannot find addExpressionImage()");
+        return;
+    }
+
+    if (isChecked) {
+        // Method get IP of endpoint
+        if (imgElement.src !== getApiUrl() + '/api/live2d/result_feed') {
+            const expressionListItemElement = document.querySelector('#live2d');
+            const expressionImageElement = expressionListItemElement.querySelector('.expression_list_image');
+            const newSrc = expressionImageElement.src;
+
+            doExtrasFetch(newSrc, {
+                method: 'HEAD',
+            })
+                .then(response => {
+                    if (response.ok) {
+                        imgElement.src = getApiUrl() + '/api/live2d/result_feed';
+                    }
+                })
+                .catch(error => {
+                    console.error(error); // Log the error if necessary
+                });
+        } else if (previousSrc) {
+            imgElement.src = previousSrc; // Revert the src to its previous value
+        }
+    } else if (previousSrc !== null) {
+        imgElement.src = previousSrc; // Revert the src to its previous value
+    }
+    live2d_var = isChecked;
+}
+
 async function moduleWorker() {
     const context = getContext();
 
@@ -405,6 +459,21 @@ async function moduleWorker() {
     if (context.groupId !== lastCharacter && context.characterId !== lastCharacter) {
         removeExpression();
         spriteCache = {};
+
+        previousSrc = null;
+
+        //uncheck live image
+        let checkbox = document.getElementById('image_type_toggle');
+        if (checkbox.checked) {
+            checkbox.click();
+        }
+
+        //clear expression
+        let imgElement = document.getElementById('expression-image');
+        imgElement.src = "";
+
+        //Load new char
+        loadLiveChar(context.name2 + ".png");
     }
 
     const vnMode = isVisualNovelMode();
@@ -654,7 +723,6 @@ async function getSpritesList(name) {
 
     try {
         const result = await fetch(`/get_sprites?name=${encodeURIComponent(name)}`);
-
         let sprites = result.ok ? (await result.json()) : [];
         return sprites;
     }
@@ -697,6 +765,8 @@ async function getExpressionsList() {
 }
 
 async function setExpression(character, expression, force) {
+    if (live2d_var == false) {
+
         console.debug('entered setExpressions');
         await validateImages(character);
         const img = $('img.expression');
@@ -789,22 +859,32 @@ async function setExpression(character, expression, force) {
                         setDefault();
                     }
                 });
-        }
-
             } else {
                 if (extension_settings.expressions.showDefault) {
                     setDefault();
                 }
             }
+        }
 
         function setDefault() {
             console.debug('setting default');
             const defImgUrl = `/img/default-expressions/${expression}.png`;
-        console.log(defImgUrl);
+            //console.log(defImgUrl);
             img.attr('src', defImgUrl);
             img.addClass('default');
         }
         document.getElementById("expression-holder").style.display = '';
+
+        if (live2d_var == true) {
+            // Find the <img> element with id="expression-image" and class="expression"
+            const imgElement = document.querySelector('img#expression-image.expression');
+            //console.log("searching");
+            if (imgElement) {
+                console.log("setting value");
+                imgElement.src = getApiUrl() + '/api/live2d/result_feed';
+            }
+        }
+    }
 }
 
 function onClickExpressionImage() {
@@ -1052,7 +1132,6 @@ function setExpressionOverrideHtml(forceClear = false) {
         $('body').append(element);
     }
     function addSettings() {
-
         const html = `
         <div class="expression_settings">
             <div class="inline-drawer">
@@ -1060,7 +1139,15 @@ function setExpressionOverrideHtml(forceClear = false) {
                     <b>Character Expressions</b>
                     <div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
                 </div>
+
                 <div class="inline-drawer-content">
+					<!-- Toggle button for aituber/static images -->
+                    <div class="toggle_button">
+						<label class="switch">
+                            <input id="image_type_toggle" type="checkbox">
+                            <span class="slider round"></span>
+                        <label for="image_type_toggle">Image Type - Live2d (extras)</label>
+                  </div>
                                       <div class="offline_mode">
                         <small>You are in offline mode. Click on the image below to set the expression.</small>
                     </div>
@@ -1090,6 +1177,7 @@ function setExpressionOverrideHtml(forceClear = false) {
             </form>
         </div>
         `;
+
         $('#extensions_settings').append(html);
         $('#expression_override_button').on('click', onClickExpressionOverrideButton);
         $('#expressions_show_default').on('input', onExpressionsShowDefaultInput);
@@ -1105,6 +1193,17 @@ function setExpressionOverrideHtml(forceClear = false) {
         $(document).on('click', '.expression_list_delete', onClickExpressionDelete);
         $(window).on("resize", updateVisualNovelModeDebounced);
         $('.expression_settings').hide();
+
+
+        $('#image_type_toggle').on('change', function () {
+            const isChecked = this.checked;
+            const inputElement = document.querySelector('input[name="avatar_url"]');
+            const value_name = inputElement ? inputElement.value : '';
+            if (isChecked) {
+                loadLiveChar(value_name);
+            }
+            handleImageChange(isChecked);
+        });
     }
 
     addExpressionImage();
@@ -1116,6 +1215,7 @@ function setExpressionOverrideHtml(forceClear = false) {
     moduleWorker();
     dragElement($("#expression-holder"))
     eventSource.on(event_types.CHAT_CHANGED, () => {
+        //console.log("checked: " + live2d_var);
         setExpressionOverrideHtml();
 
         if (isVisualNovelMode()) {

public/scripts/extensions/objective/index.js

@@ -72,6 +72,9 @@ function getTaskByIdRecurse(taskId, task) {
 function substituteParamsPrompts(content) {
     content = content.replace(/{{objective}}/gi, currentObjective.description)
     content = content.replace(/{{task}}/gi, currentTask.description)
+    if (currentTask.parent){
+        content = content.replace(/{{parent}}/gi, currentTask.parent.description)
+    }
     content = substituteParams(content)
     return content
 }
@@ -97,7 +100,7 @@ async function generateTasks() {
     updateUiTaskList();
     setCurrentTask();
     console.info(`Response for Objective: '${taskTree.description}' was \n'${taskResponse}', \nwhich created tasks \n${JSON.stringify(globalTasks.map(v => {return v.toSaveState()}), null, 2)} `)
-    toastr.success(`Generated ${globalTasks.length} tasks`, 'Done!');
+    toastr.success(`Generated ${taskTree.length} tasks`, 'Done!');
 }
 
 // Call Quiet Generate to check if a task is completed
@@ -126,7 +129,7 @@ async function checkTaskCompleted() {
 function getNextIncompleteTaskRecurse(task){
     if (task.completed === false // Return task if incomplete
         && task.children.length === 0 // Ensure task has no children, it's subtasks will determine completeness
-        && task.parentId  // Must have parent id. Only root task will be missing this and we dont want that
+        && task.parentId !== ""  // Must have parent id. Only root task will be missing this and we dont want that
     ){
         return task
     }

public/scripts/extensions/speech-recognition/index.js

@@ -8,16 +8,21 @@ import { getContext, getApiUrl, modules, extension_settings, ModuleWorkerWrapper
 import { VoskSttProvider } from './vosk.js'
 import { WhisperSttProvider } from './whisper.js'
 import { BrowserSttProvider } from './browser.js'
+import { StreamingSttProvider } from './streaming.js'
 export { MODULE_NAME };
 
 const MODULE_NAME = 'Speech Recognition';
 const DEBUG_PREFIX = "<Speech Recognition module> "
+const UPDATE_INTERVAL = 100;
+
+let inApiCall = false;
 
 let sttProviders = {
     None: null,
     Browser: BrowserSttProvider,
     Whisper: WhisperSttProvider,
     Vosk: VoskSttProvider,
+    Streaming: StreamingSttProvider,
 }
 
 let sttProvider = null
@@ -27,6 +32,82 @@ let audioRecording = false
 const constraints = { audio: { sampleSize: 16, channelCount: 1, sampleRate: 16000 } };
 let audioChunks = [];
 
+async function moduleWorker() {
+    if (sttProviderName != "Streaming") {
+        return;
+    }
+
+    // API is busy
+    if (inApiCall) {
+        return;
+    }
+
+    try {
+        inApiCall = true;
+        const userMessageOriginal =  await sttProvider.getUserMessage();
+        let userMessageFormatted = userMessageOriginal.trim();
+
+        if (userMessageFormatted.length > 0)
+        {
+            console.debug(DEBUG_PREFIX+"recorded transcript: \""+userMessageFormatted+"\"");
+
+            let userMessageLower = userMessageFormatted.toLowerCase();
+            // remove punctuation
+            let userMessageRaw = userMessageLower.replace(/[^\w\s\']|_/g, "").replace(/\s+/g, " ");
+
+            console.debug(DEBUG_PREFIX+"raw transcript:",userMessageRaw);
+
+            // Detect trigger words
+            let messageStart = -1;
+
+            if (extension_settings.speech_recognition.Streaming.triggerWordsEnabled) {
+            
+                for (const triggerWord of extension_settings.speech_recognition.Streaming.triggerWords) {
+                    const triggerPos = userMessageRaw.indexOf(triggerWord.toLowerCase());
+                    
+                    // Trigger word not found or not starting message and just a substring
+                    if (triggerPos == -1){ // | (triggerPos > 0 & userMessageFormatted[triggerPos-1] != " ")) {
+                        console.debug(DEBUG_PREFIX+"trigger word not found: ", triggerWord);
+                    }
+                    else {
+                        console.debug(DEBUG_PREFIX+"Found trigger word: ", triggerWord, " at index ", triggerPos);
+                        if (triggerPos < messageStart | messageStart == -1) { // & (triggerPos + triggerWord.length) < userMessageFormatted.length)) {
+                            messageStart = triggerPos; // + triggerWord.length + 1;
+                        }
+                    }
+                }
+            } else {
+                messageStart = 0;
+            }
+
+            if (messageStart == -1) {
+                console.debug(DEBUG_PREFIX+"message ignored, no trigger word preceding a message. Voice transcript: \""+ userMessageOriginal +"\"");
+                if (extension_settings.speech_recognition.Streaming.debug) {
+                    toastr.info(
+                        "No trigger word preceding a message. Voice transcript: \""+ userMessageOriginal +"\"",
+                        DEBUG_PREFIX+"message ignored.",
+                        { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true },
+                    );
+                }
+            }
+            else{
+                userMessageFormatted = userMessageFormatted.substring(messageStart);
+                processTranscript(userMessageFormatted);
+            }
+        }
+        else
+        {
+            console.debug(DEBUG_PREFIX+"Received empty transcript, ignored");
+        }
+    }
+    catch (error) {
+        console.debug(error);
+    }
+    finally {
+        inApiCall = false;
+    }
+}
+
 async function processTranscript(transcript) {
     try {
         const transcriptOriginal =  transcript;
@@ -198,13 +279,21 @@ function loadSttProvider(provider) {
     if (sttProviderName == "Browser") {
         sttProvider.processTranscriptFunction = processTranscript;
         sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
-    }
-    else {
-        sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
-        loadNavigatorAudioRecording();
-
         $("#microphone_button").show();
     }
+
+    if (sttProviderName == "Vosk" | sttProviderName == "Whisper") {
+        sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
+        loadNavigatorAudioRecording();
+        $("#microphone_button").show();
+    }
+    
+    if (sttProviderName == "Streaming") {
+        sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
+        $("#microphone_button").off('click');
+        $("#microphone_button").hide();
+    }
+    
 }
 
 function onSttProviderChange() {
@@ -231,7 +320,7 @@ const defaultSettings = {
     messageMode: "append",
     messageMappingText: "",
     messageMapping: [],
-    messageMappingEnabled: false
+    messageMappingEnabled: false,
 }
 
 function loadSettings() {
@@ -344,8 +433,7 @@ $(document).ready(function () {
     addExtensionControls(); // No init dependencies
     loadSettings(); // Depends on Extension Controls and loadTtsProvider
     loadSttProvider(extension_settings.speech_recognition.currentProvider); // No dependencies
-
+    const wrapper = new ModuleWorkerWrapper(moduleWorker);
-    //const wrapper = new ModuleWorkerWrapper(moduleWorker);
+    setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL); // Init depends on all the things
-    //setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL); // Init depends on all the things
+    moduleWorker();
-    //moduleWorker();
 })

public/scripts/extensions/speech-recognition/streaming.js

@@ -0,0 +1,102 @@
+import { getApiUrl, doExtrasFetch, modules } from "../../extensions.js";
+export { StreamingSttProvider }
+
+const DEBUG_PREFIX = "<Speech Recognition module (streaming)> "
+
+class StreamingSttProvider {
+    //########//
+    // Config //
+    //########//
+
+    settings
+
+    defaultSettings = {
+        triggerWordsText: "",
+        triggerWords : [],
+        triggerWordsEnabled : false,
+        debug : false,
+    }
+
+    get settingsHtml() {
+        let html = '\
+        <div id="speech_recognition_streaming_trigger_words_div">\
+        <span>Trigger words</span>\
+        <textarea id="speech_recognition_streaming_trigger_words" class="text_pole textarea_compact" type="text" rows="4" placeholder="Enter comma separated words that triggers new message, example:\nhey, hey aqua, record, listen"></textarea>\
+        <label class="checkbox_label" for="speech_recognition_streaming_trigger_words_enabled">\
+            <input type="checkbox" id="speech_recognition_streaming_trigger_words_enabled" name="speech_recognition_trigger_words_enabled">\
+            <small>Enable trigger words</small>\
+        </label>\
+        <label class="checkbox_label" for="speech_recognition_streaming_debug">\
+            <input type="checkbox" id="speech_recognition_streaming_debug" name="speech_recognition_streaming_debug">\
+            <small>Enable debug pop ups</small>\
+        </label>\
+        </div>\
+        '
+        return html
+    }
+
+    onSettingsChange() {
+        this.settings.triggerWordsText = $('#speech_recognition_streaming_trigger_words').val();
+        let array = $('#speech_recognition_streaming_trigger_words').val().split(",");
+        array = array.map(element => {return element.trim().toLowerCase();});
+        array = array.filter((str) => str !== '');
+        this.settings.triggerWords = array;
+        this.settings.triggerWordsEnabled = $("#speech_recognition_streaming_trigger_words_enabled").is(':checked');
+        this.settings.debug = $("#speech_recognition_streaming_debug").is(':checked');
+        console.debug(DEBUG_PREFIX+" Updated settings: ", this.settings);
+        this.loadSettings(this.settings);
+    }
+
+    loadSettings(settings) {
+        // Populate Provider UI given input settings
+        if (Object.keys(settings).length == 0) {
+            console.debug(DEBUG_PREFIX+"Using default Whisper STT extension settings")
+        }
+
+        // Only accept keys defined in defaultSettings
+        this.settings = this.defaultSettings
+
+        for (const key in settings){
+            if (key in this.settings){
+                this.settings[key] = settings[key]
+            } else {
+                throw `Invalid setting passed to STT extension: ${key}`
+            }
+        }
+
+        $("#speech_recognition_streaming_trigger_words").val(this.settings.triggerWordsText);
+        $("#speech_recognition_streaming_trigger_words_enabled").prop('checked',this.settings.triggerWordsEnabled);
+        $("#speech_recognition_streaming_debug").prop('checked',this.settings.debug);
+
+        console.debug(DEBUG_PREFIX+"streaming STT settings loaded")
+    }
+
+    async getUserMessage() {
+        // Return if module is not loaded
+        if (!modules.includes('streaming-stt')) {
+            console.debug(DEBUG_PREFIX+"Module streaming-stt must be activated in Sillytavern Extras for streaming user voice.")
+            return "";
+        }
+
+        const url = new URL(getApiUrl());
+        url.pathname = '/api/speech-recognition/streaming/record-and-transcript';
+
+        const apiResult = await doExtrasFetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Bypass-Tunnel-Reminder': 'bypass',
+            },
+            body: JSON.stringify({ text: "" }),
+        });
+
+        if (!apiResult.ok) {
+            toastr.error(apiResult.statusText, DEBUG_PREFIX+'STT Generation Failed  (streaming)', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true });
+            throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`);
+        }
+
+        const data = await apiResult.json();
+        return data.transcript;
+    }
+
+}

public/scripts/extensions/tts/index.js

@@ -1,5 +1,5 @@
 import { callPopup, cancelTtsPlay, eventSource, event_types, isMultigenEnabled, is_send_press, saveSettingsDebounced } from '../../../script.js'
-import { ModuleWorkerWrapper, extension_settings, getContext } from '../../extensions.js'
+import { ModuleWorkerWrapper, doExtrasFetch, extension_settings, getApiUrl, getContext } from '../../extensions.js'
 import { escapeRegex, getStringHash } from '../../utils.js'
 import { EdgeTtsProvider } from './edge.js'
 import { ElevenLabsTtsProvider } from './elevenlabs.js'
@@ -7,14 +7,13 @@ import { SileroTtsProvider } from './silerotts.js'
 import { CoquiTtsProvider } from './coquitts.js'
 import { SystemTtsProvider } from './system.js'
 import { NovelTtsProvider } from './novel.js'
-import { isMobile } from '../../RossAscends-mods.js'
 import { power_user } from '../../power-user.js'
 
 const UPDATE_INTERVAL = 1000
 
 let voiceMap = {} // {charName:voiceid, charName2:voiceid2}
 let audioControl
-
+let storedvalue = false;
 let lastCharacterId = null
 let lastGroupId = null
 let lastChatId = null
@@ -164,6 +163,20 @@ async function moduleWorker() {
     ttsJobQueue.push(message)
 }
 
+function talkingAnimation(switchValue) {
+    const apiUrl = getApiUrl();
+    const animationType = switchValue ? "start" : "stop";
+
+    if (switchValue !== storedvalue) {
+        try {
+            console.log(animationType + " Talking Animation");
+            doExtrasFetch(`${apiUrl}/api/live2d/${animationType}_talking`);
+            storedvalue = switchValue; // Update the storedvalue to the current switchValue
+        } catch (error) {
+            // Handle the error here or simply ignore it to prevent logging
+        }
+    }
+}
 
 function resetTtsPlayback() {
     // Stop system TTS utterance
@@ -291,8 +304,10 @@ function updateUiAudioPlayState() {
         // Give user feedback that TTS is active by setting the stop icon if processing or playing
         if (!audioElement.paused || isTtsProcessing()) {
             img = 'fa-solid fa-stop-circle extensionsMenuExtensionButton'
+            talkingAnimation(true)
         } else {
             img = 'fa-solid fa-circle-play extensionsMenuExtensionButton'
+            talkingAnimation(false)
         }
         $('#tts_media_control').attr('class', img);
     } else {
@@ -354,6 +369,7 @@ async function processAudioJobQueue() {
         audioQueueProcessorReady = false
         currentAudioJob = audioJobQueue.pop()
         playAudioData(currentAudioJob)
+        talkingAnimation(true)
     } catch (error) {
         console.error(error)
         audioQueueProcessorReady = true