rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enable talkinghead's talking animation while the LLM is streaming

This commit is contained in:
Juha Jeronen 2024-01-08 00:47:59 +02:00
parent 39c485ab25
commit 87b05e2e2d
1 changed files with 64 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
public/scripts/extensions/expressions/index.js
View File

@ -10,6 +10,7 @@ export { MODULE_NAME };
const MODULE_NAME = 'expressions';
const UPDATE_INTERVAL = 2000;
const STREAMING_UPDATE_INTERVAL = 6000;
const TALKINGCHECK_UPDATE_INTERVAL = 250;
const FALLBACK_EXPRESSION = 'joy';
const DEFAULT_EXPRESSIONS = [
'talkinghead',
@ -46,6 +47,8 @@ const DEFAULT_EXPRESSIONS = [
let expressionsList = null;
let lastCharacter = undefined;
let lastMessage = null;
let lastTalkingState = false;
let lastTalkingStateMessage = null; // last message as seen by `updateTalkingState` (tracked separately, different timer)
let spriteCache = {};
let inApiCall = false;
let lastServerResponseTime = 0;
@ -623,6 +626,62 @@ async function moduleWorker() {
}
}
/**
* Starts/stops talkinghead talking animation.
*
* Talking starts only when all the following conditions are met:
* - The LLM is currently streaming its output.
* - The AI's current last message is non-empty, and also not just '...' (as produced by a swipe).
* - The AI's current last message has changed from what we saw during the previous call.
*
* In all other cases, talking stops.
*
* A talkinghead API call is made only when the talking state changes.
*/
async function updateTalkingState() {
const context = getContext();
const currentLastMessage = getLastCharacterMessage();
try {
// TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker`
// (that similarly checks the streaming processor state) does that for some reason.
// Talkinghead isn't currently designed to work with groups.
if (isTalkingHeadEnabled()) {
const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId)
&& lastTalkingStateMessage === currentLastMessage.mes);
const url = new URL(getApiUrl());
let newTalkingState;
if (context.streamingProcessor && !context.streamingProcessor.isFinished &&
currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) {
url.pathname = '/api/talkinghead/start_talking';
newTalkingState = true;
} else {
url.pathname = '/api/talkinghead/stop_talking';
newTalkingState = false;
}
try {
// Call the talkinghead API only if the talking state changed.
if (newTalkingState !== lastTalkingState) {
console.debug(`updateTalkingState: calling ${url.pathname}`);
await doExtrasFetch(url);
}
}
catch (error) {
// it's ok if not supported
}
finally {
lastTalkingState = newTalkingState;
}
}
}
catch (error) {
// console.log(error);
}
finally {
lastTalkingStateMessage = currentLastMessage.mes;
}
}
/**
* Checks whether the current character has a talkinghead image available.
* @returns {Promise<boolean>} True if the character has a talkinghead image available, false otherwise.
@ -1513,6 +1572,11 @@ function setExpressionOverrideHtml(forceClear = false) {
const updateFunction = wrapper.update.bind(wrapper);
setInterval(updateFunction, UPDATE_INTERVAL);
moduleWorker();
// For setting the talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule.
const wrapper_talkingstate = new ModuleWorkerWrapper(updateTalkingState);
const updateTalkingStateFunction = wrapper_talkingstate.update.bind(wrapper_talkingstate);
setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL);
updateTalkingState();
dragElement($('#expression-holder'));
eventSource.on(event_types.CHAT_CHANGED, () => {
// character changed