Merge pull request #1656 from Technologicat/talkinghead-talkinganim

Talkinghead: enable talking animation

public/scripts/extensions/expressions/index.js

@@ -10,6 +10,7 @@ export { MODULE_NAME };
 const MODULE_NAME = 'expressions';
 const UPDATE_INTERVAL = 2000;
 const STREAMING_UPDATE_INTERVAL = 6000;
+const TALKINGCHECK_UPDATE_INTERVAL = 500;
 const FALLBACK_EXPRESSION = 'joy';
 const DEFAULT_EXPRESSIONS = [
     'talkinghead',
@@ -46,6 +47,8 @@ const DEFAULT_EXPRESSIONS = [
 let expressionsList = null;
 let lastCharacter = undefined;
 let lastMessage = null;
+let lastTalkingState = false;
+let lastTalkingStateMessage = null;  // last message as seen by `updateTalkingState` (tracked separately, different timer)
 let spriteCache = {};
 let inApiCall = false;
 let lastServerResponseTime = 0;
@@ -385,6 +388,9 @@ function onExpressionsShowDefaultInput() {
     }
 }
 
+/**
+  * Stops animating a talkinghead.
+  */
 async function unloadTalkingHead() {
     if (!modules.includes('talkinghead')) {
         console.debug('talkinghead module is disabled');
@@ -404,6 +410,9 @@ async function unloadTalkingHead() {
     }
 }
 
+/**
+  * Posts `talkinghead.png` of the current character to the talkinghead module in SillyTavern-extras, to start animating it.
+  */
 async function loadTalkingHead() {
     if (!modules.includes('talkinghead')) {
         console.debug('talkinghead module is disabled');
@@ -563,9 +572,10 @@ async function moduleWorker() {
         return;
     }
 
+    const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastMessage === currentLastMessage.mes);
+
     // check if last message changed
-    if ((lastCharacter === context.characterId || lastCharacter === context.groupId)
-        && lastMessage === currentLastMessage.mes) {
+    if (!lastMessageChanged) {
         return;
     }
 
@@ -615,6 +625,64 @@ async function moduleWorker() {
     }
 }
 
+/**
+  * Starts/stops talkinghead talking animation.
+  *
+  * Talking starts only when all the following conditions are met:
+  *   - The LLM is currently streaming its output.
+  *   - The AI's current last message is non-empty, and also not just '...' (as produced by a swipe).
+  *   - The AI's current last message has changed from what we saw during the previous call.
+  *
+  * In all other cases, talking stops.
+  *
+  * A talkinghead API call is made only when the talking state changes.
+  */
+async function updateTalkingState() {
+    // Don't bother if talkinghead is disabled or not loaded.
+    if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) {
+        return;
+    }
+
+    const context = getContext();
+    const currentLastMessage = getLastCharacterMessage();
+
+    try {
+        // TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker`
+        //       (that similarly checks the streaming processor state) does that for some reason.
+        //       Talkinghead isn't currently designed to work with groups.
+        const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastTalkingStateMessage === currentLastMessage.mes);
+        const url = new URL(getApiUrl());
+        let newTalkingState;
+        if (context.streamingProcessor && !context.streamingProcessor.isFinished &&
+            currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) {
+            url.pathname = '/api/talkinghead/start_talking';
+            newTalkingState = true;
+        } else {
+            url.pathname = '/api/talkinghead/stop_talking';
+            newTalkingState = false;
+        }
+        try {
+            // Call the talkinghead API only if the talking state changed.
+            if (newTalkingState !== lastTalkingState) {
+                console.debug(`updateTalkingState: calling ${url.pathname}`);
+                await doExtrasFetch(url);
+            }
+        }
+        catch (error) {
+            // it's ok if not supported
+        }
+        finally {
+            lastTalkingState = newTalkingState;
+        }
+    }
+    catch (error) {
+        // console.log(error);
+    }
+    finally {
+        lastTalkingStateMessage = currentLastMessage.mes;
+    }
+}
+
 /**
  * Checks whether the current character has a talkinghead image available.
  * @returns {Promise<boolean>} True if the character has a talkinghead image available, false otherwise.
@@ -1011,7 +1079,7 @@ async function getExpressionsList() {
 }
 
 async function setExpression(character, expression, force) {
-    if (extension_settings.expressions.local || !extension_settings.expressions.talkinghead) {
+    if (!isTalkingHeadEnabled()) {
         console.debug('entered setExpressions');
         await validateImages(character);
         const img = $('img.expression');
@@ -1276,7 +1344,7 @@ async function onClickExpressionUpload(event) {
         e.target.form.reset();
 
         // In talkinghead mode, when a new talkinghead image is uploaded, refresh the live char.
-        if (extension_settings.expressions.talkinghead && !extension_settings.expressions.local && id === 'talkinghead') {
+        if (isTalkingHeadEnabled() && id === 'talkinghead') {
             await loadTalkingHead();
         }
     };
@@ -1505,6 +1573,11 @@ function setExpressionOverrideHtml(forceClear = false) {
     const updateFunction = wrapper.update.bind(wrapper);
     setInterval(updateFunction, UPDATE_INTERVAL);
     moduleWorker();
+    // For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule.
+    const wrapperTalkingState = new ModuleWorkerWrapper(updateTalkingState);
+    const updateTalkingStateFunction = wrapperTalkingState.update.bind(wrapperTalkingState);
+    setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL);
+    updateTalkingState();
     dragElement($('#expression-holder'));
     eventSource.on(event_types.CHAT_CHANGED, () => {
         // character changed