From 83e264db9ed9fc1dd4495883bf0eb57250cf1d76 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:40:03 +0200 Subject: [PATCH 01/10] add some debug messages --- public/scripts/extensions/expressions/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 39f296545..207da091b 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -396,6 +396,7 @@ async function unloadTalkingHead() { console.debug('talkinghead module is disabled'); return; } + console.debug('expressions: Stopping Talkinghead'); try { const url = new URL(getApiUrl()); @@ -418,6 +419,7 @@ async function loadTalkingHead() { console.debug('talkinghead module is disabled'); return; } + console.debug('expressions: Starting Talkinghead'); const spriteFolderName = getSpriteFolderName(); From 24b315a149fb285ebd2e9878827b72659e8c292f Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:44:26 +0200 Subject: [PATCH 02/10] comments The tech is "Talkinghead" (capital T), the Extras module is "talkinghead" (lowercase t). --- .../scripts/extensions/expressions/index.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 207da091b..d08b1cae1 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -389,7 +389,7 @@ function onExpressionsShowDefaultInput() { } /** - * Stops animating a talkinghead. + * Stops animating Talkinghead. */ async function unloadTalkingHead() { if (!modules.includes('talkinghead')) { @@ -560,7 +560,7 @@ function handleImageChange() { async function moduleWorker() { const context = getContext(); - // Hide and disable talkinghead while in local mode + // Hide and disable Talkinghead while in local mode $('#image_type_block').toggle(!extension_settings.expressions.local); if (extension_settings.expressions.local && extension_settings.expressions.talkinghead) { @@ -693,7 +693,7 @@ async function moduleWorker() { } /** - * Starts/stops talkinghead talking animation. + * Starts/stops Talkinghead talking animation. * * Talking starts only when all the following conditions are met: * - The LLM is currently streaming its output. @@ -702,10 +702,10 @@ async function moduleWorker() { * * In all other cases, talking stops. * - * A talkinghead API call is made only when the talking state changes. + * A Talkinghead API call is made only when the talking state changes. */ async function updateTalkingState() { - // Don't bother if talkinghead is disabled or not loaded. + // Don't bother if Talkinghead is disabled or not loaded. if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) { return; } @@ -729,7 +729,7 @@ async function updateTalkingState() { newTalkingState = false; } try { - // Call the talkinghead API only if the talking state changed. + // Call the Talkinghead API only if the talking state changed. if (newTalkingState !== lastTalkingState) { console.debug(`updateTalkingState: calling ${url.pathname}`); await doExtrasFetch(url); @@ -873,7 +873,7 @@ async function setSpriteSlashCommand(_, spriteId) { spriteId = spriteId.trim().toLowerCase(); - // In talkinghead mode, don't check for the existence of the sprite + // In Talkinghead mode, don't check for the existence of the sprite // (emotion names are the same as for sprites, but it only needs "talkinghead.png"). const currentLastMessage = getLastCharacterMessage(); const spriteFolderName = getSpriteFolderName(currentLastMessage, currentLastMessage.name); @@ -1257,8 +1257,8 @@ async function setExpression(character, expression, force) { document.getElementById('expression-holder').style.display = ''; } else { - // Set the talkinghead emotion to the specified expression - // TODO: For now, talkinghead emote only supported when VN mode is off; see also updateVisualNovelMode. + // Set the Talkinghead emotion to the specified expression + // TODO: For now, Talkinghead emote only supported when VN mode is off; see also updateVisualNovelMode. try { let result = await isTalkingHeadAvailable(); if (result) { @@ -1411,8 +1411,8 @@ async function onClickExpressionUpload(event) { // Reset the input e.target.form.reset(); - // In talkinghead mode, when a new talkinghead image is uploaded, refresh the live char. if (isTalkingHeadEnabled() && id === 'talkinghead') { + // In Talkinghead mode, when a new talkinghead image is uploaded, refresh the live char. await loadTalkingHead(); } }; @@ -1666,7 +1666,7 @@ async function fetchImagesNoCache() { const updateFunction = wrapper.update.bind(wrapper); setInterval(updateFunction, UPDATE_INTERVAL); moduleWorker(); - // For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule. + // For setting the Talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule. const wrapperTalkingState = new ModuleWorkerWrapper(updateTalkingState); const updateTalkingStateFunction = wrapperTalkingState.update.bind(wrapperTalkingState); setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL); From eb634d597fe0ac30e799b032b73b8f3b2748dead Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:44:39 +0200 Subject: [PATCH 03/10] add comment on TTS --- public/scripts/extensions/expressions/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index d08b1cae1..dbe58e2f8 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -703,6 +703,9 @@ async function moduleWorker() { * In all other cases, talking stops. * * A Talkinghead API call is made only when the talking state changes. + * + * Note that also the TTS system, if enabled, starts/stops the Talkinghead talking animation. + * See `talkingAnimation` in `SillyTavern/public/scripts/extensions/tts/index.js`. */ async function updateTalkingState() { // Don't bother if Talkinghead is disabled or not loaded. From 9e8f3e0def4f3c3064019e6e189ebd83e6e7310d Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:44:46 +0200 Subject: [PATCH 04/10] one more debug message --- public/scripts/extensions/expressions/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index dbe58e2f8..9293c4376 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -792,6 +792,7 @@ function getSpriteFolderName(characterMessage = null, characterName = null) { } function setTalkingHeadState(newState) { + console.debug(`expressions: New talkinghead state: ${newState}`); extension_settings.expressions.talkinghead = newState; // Store setting saveSettingsDebounced(); From 169b1c2c63fe993bbedbac199c7bab458b7622d6 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:45:20 +0200 Subject: [PATCH 05/10] talkinghead check: always check also whether the module is enabled --- public/scripts/extensions/expressions/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 9293c4376..03b586dfc 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -530,8 +530,8 @@ function handleImageChange() { return; } - if (isTalkingHeadEnabled()) { // Method get IP of endpoint + if (isTalkingHeadEnabled() && modules.includes('talkinghead')) { const talkingheadResultFeedSrc = `${getApiUrl()}/api/talkinghead/result_feed`; $('#expression-holder').css({ display: '' }); if (imgElement.src !== talkingheadResultFeedSrc) { @@ -882,7 +882,7 @@ async function setSpriteSlashCommand(_, spriteId) { const currentLastMessage = getLastCharacterMessage(); const spriteFolderName = getSpriteFolderName(currentLastMessage, currentLastMessage.name); let label = spriteId; - if (!isTalkingHeadEnabled()) { + if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) { await validateImages(spriteFolderName); // Fuzzy search for sprite @@ -1150,7 +1150,7 @@ async function getExpressionsList() { } async function setExpression(character, expression, force) { - if (!isTalkingHeadEnabled()) { + if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) { console.debug('entered setExpressions'); await validateImages(character); const img = $('img.expression'); @@ -1415,8 +1415,8 @@ async function onClickExpressionUpload(event) { // Reset the input e.target.form.reset(); - if (isTalkingHeadEnabled() && id === 'talkinghead') { // In Talkinghead mode, when a new talkinghead image is uploaded, refresh the live char. + if (id === 'talkinghead' && isTalkingHeadEnabled() && modules.includes('talkinghead')) { await loadTalkingHead(); } }; From 3b526ce207a5c2d2601c39c281be6618a01a70be Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:45:37 +0200 Subject: [PATCH 06/10] remove some useless comments --- public/scripts/extensions/expressions/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 03b586dfc..8ccfb382a 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -530,7 +530,6 @@ function handleImageChange() { return; } - // Method get IP of endpoint if (isTalkingHeadEnabled() && modules.includes('talkinghead')) { const talkingheadResultFeedSrc = `${getApiUrl()}/api/talkinghead/result_feed`; $('#expression-holder').css({ display: '' }); @@ -547,7 +546,7 @@ function handleImageChange() { } }) .catch(error => { - console.error(error); // Log the error if necessary + console.error(error); }); } } From 5ad2a0d06479f89cd9de2bfb2b17d8d07ca2d718 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:45:50 +0200 Subject: [PATCH 07/10] refresh talkinghead char on expression zip upload --- public/scripts/extensions/expressions/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 8ccfb382a..99405cec0 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -1525,6 +1525,11 @@ async function onClickExpressionUploadPackButton() { // Reset the input e.target.form.reset(); + + // In Talkinghead mode, refresh the live char. + if (isTalkingHeadEnabled() && modules.includes('talkinghead')) { + await loadTalkingHead(); + } }; $('#expression_upload_pack') From 91c4de66050198fa717e36d9bab7050a23372be1 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:46:23 +0200 Subject: [PATCH 08/10] add `/th` (alias `/talkinghead`) to toggle Talkinghead on/off --- public/scripts/extensions/expressions/index.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 99405cec0..2359ace5c 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -58,6 +58,17 @@ function isTalkingHeadEnabled() { return extension_settings.expressions.talkinghead && !extension_settings.expressions.local; } +/** + * Toggles Talkinghead mode on/off. + * + * Implements the `/th` slash command, which is meant to be bound to a Quick Reply button + * as a quick way to switch Talkinghead on or off (e.g. to conserve GPU resources when AFK + * for a long time). + */ +function toggleTalkingHeadCommand(_) { + setTalkingHeadState(!extension_settings.expressions.talkinghead); +} + function isVisualNovelMode() { return Boolean(!isMobile() && power_user.waifuMode && getContext().groupId); } @@ -1711,4 +1722,5 @@ async function fetchImagesNoCache() { registerSlashCommand('sprite', setSpriteSlashCommand, ['emote'], '(spriteId) – force sets the sprite for the current character', true, true); registerSlashCommand('spriteoverride', setSpriteSetCommand, ['costume'], '(optional folder) – sets an override sprite folder for the current character. If the name starts with a slash or a backslash, selects a sub-folder in the character-named folder. Empty value to reset to default.', true, true); registerSlashCommand('lastsprite', (_, value) => lastExpression[value.trim()] ?? '', [], '(charName) – Returns the last set sprite / expression for the named character.', true, true); + registerSlashCommand('th', toggleTalkingHeadCommand, ['talkinghead'], '– Character Expressions: toggles Image Type - talkinghead (extras) on/off.'); })(); From 2a39db799a4353d1a91a4d643ee9d7bf1d87a562 Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:46:44 +0200 Subject: [PATCH 09/10] auto-pause Talkinghead when ST tab is hidden to save GPU resources --- .../scripts/extensions/expressions/index.js | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 2359ace5c..cfbb87894 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -1678,6 +1678,34 @@ async function fetchImagesNoCache() { $('#expression_custom_remove').on('click', onClickExpressionRemoveCustom); } + // Pause Talkinghead to save resources when the ST tab is not visible or the window is minimized. + // We currently do this via loading/unloading. Could be improved by adding new pause/unpause endpoints to Extras. + document.addEventListener("visibilitychange", function (event) { + let pageIsVisible; + if (document.hidden) { + console.debug('expressions: SillyTavern is now hidden'); + pageIsVisible = false; + } else { + console.debug('expressions: SillyTavern is now visible'); + pageIsVisible = true; + } + + if (isTalkingHeadEnabled() && modules.includes('talkinghead')) { + isTalkingHeadAvailable().then(result => { + if (result) { + if (pageIsVisible) { + loadTalkingHead(); + } else { + unloadTalkingHead(); + } + handleImageChange(); // Change image as needed + } else { + //console.log("talkinghead does not exist."); + } + }); + } + }); + addExpressionImage(); addVisualNovelMode(); addSettings(); From ad48d6666abfca8124a03375de2ebd707cd0668a Mon Sep 17 00:00:00 2001 From: Juha Jeronen Date: Mon, 5 Feb 2024 00:47:14 +0200 Subject: [PATCH 10/10] fix bug: when switching talkinghead off, set character expression --- public/scripts/extensions/expressions/index.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index cfbb87894..2f7963e0a 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -562,8 +562,14 @@ function handleImageChange() { } } } else { - imgElement.src = ''; //remove incase char doesnt have expressions - setExpression(getContext().name2, FALLBACK_EXPRESSION, true); + imgElement.src = ''; // remove in case char doesn't have expressions + + // When switching Talkinghead off, force-set the character to the last known expression, if any. + // This preserves the same expression Talkinghead had at the moment it was switched off. + const charName = getContext().name2; + const last = lastExpression[charName]; + const targetExpression = last ? last : FALLBACK_EXPRESSION; + setExpression(charName, targetExpression, true); } }