mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
yoink talkinghead - goodbye extras dependency
This commit is contained in:
@ -1602,7 +1602,6 @@
|
|||||||
"Character Expressions": "Expressions de personnages",
|
"Character Expressions": "Expressions de personnages",
|
||||||
"Translate text to English before classification": "Traduire le texte en anglais avant de le classer",
|
"Translate text to English before classification": "Traduire le texte en anglais avant de le classer",
|
||||||
"Show default images (emojis) if sprite missing": "Afficher les images par défaut (emojis) si le sprite est manquant",
|
"Show default images (emojis) if sprite missing": "Afficher les images par défaut (emojis) si le sprite est manquant",
|
||||||
"Image Type - talkinghead (extras)": "Type d'image - talkinghead (extras)",
|
|
||||||
"Classifier API": "API de classification",
|
"Classifier API": "API de classification",
|
||||||
"Select the API for classifying expressions.": "Sélectionnez l'API pour classer les expressions.",
|
"Select the API for classifying expressions.": "Sélectionnez l'API pour classer les expressions.",
|
||||||
"Main API": "API principale",
|
"Main API": "API principale",
|
||||||
|
@ -1467,7 +1467,6 @@
|
|||||||
"menu within": "내의 메뉴",
|
"menu within": "내의 메뉴",
|
||||||
"Translate text to English before classification": "분류 전에 텍스트를 영어로 번역합니다.",
|
"Translate text to English before classification": "분류 전에 텍스트를 영어로 번역합니다.",
|
||||||
"Show default images (emojis) if sprite missing": "해당하는 스프라이트가 없으면 기본 이미지 (이모지들)을 표시합니다.",
|
"Show default images (emojis) if sprite missing": "해당하는 스프라이트가 없으면 기본 이미지 (이모지들)을 표시합니다.",
|
||||||
"Image Type - talkinghead (extras)": "이미지 유형 - 토킹 헤드 (부가 사항)",
|
|
||||||
"Classifier API": "분류를 위한 API",
|
"Classifier API": "분류를 위한 API",
|
||||||
"Select the API for classifying expressions.": "감정 이미지들을 분류할 API를 선택하세요.",
|
"Select the API for classifying expressions.": "감정 이미지들을 분류할 API를 선택하세요.",
|
||||||
"Local": "로컬",
|
"Local": "로컬",
|
||||||
|
@ -1349,7 +1349,6 @@
|
|||||||
"Character Expressions": "角色表情",
|
"Character Expressions": "角色表情",
|
||||||
"Translate text to English before classification": "分类之前将文本翻译成英文",
|
"Translate text to English before classification": "分类之前将文本翻译成英文",
|
||||||
"Show default images (emojis) if sprite missing": "如果表情包缺失,则显示默认图像(表情符号)",
|
"Show default images (emojis) if sprite missing": "如果表情包缺失,则显示默认图像(表情符号)",
|
||||||
"Image Type - talkinghead (extras)": "图像类型 - 说话头像(附加内容)",
|
|
||||||
"Classifier API": "分类器 API",
|
"Classifier API": "分类器 API",
|
||||||
"Select the API for classifying expressions.": "选择用于对表达式进行分类的API。",
|
"Select the API for classifying expressions.": "选择用于对表达式进行分类的API。",
|
||||||
"Main API": "主要 API",
|
"Main API": "主要 API",
|
||||||
|
@ -1653,7 +1653,6 @@
|
|||||||
"HuggingFace Token": "HuggingFace 符元",
|
"HuggingFace Token": "HuggingFace 符元",
|
||||||
"Image Captioning": "圖片註解",
|
"Image Captioning": "圖片註解",
|
||||||
"Generate Caption": "產生圖片註解",
|
"Generate Caption": "產生圖片註解",
|
||||||
"Image Type - talkinghead (extras)": "圖片類型 - talkinghead(額外選項)",
|
|
||||||
"Injection Position": "插入位置",
|
"Injection Position": "插入位置",
|
||||||
"Injection position. Relative (to other prompts in prompt manager) or In-chat @ Depth.": "插入位置(與提示詞管理器中的其他提示相比)或聊天中的深度位置。",
|
"Injection position. Relative (to other prompts in prompt manager) or In-chat @ Depth.": "插入位置(與提示詞管理器中的其他提示相比)或聊天中的深度位置。",
|
||||||
"Injection Template": "插入範本",
|
"Injection Template": "插入範本",
|
||||||
|
@ -38,11 +38,9 @@ export { MODULE_NAME };
|
|||||||
const MODULE_NAME = 'expressions';
|
const MODULE_NAME = 'expressions';
|
||||||
const UPDATE_INTERVAL = 2000;
|
const UPDATE_INTERVAL = 2000;
|
||||||
const STREAMING_UPDATE_INTERVAL = 10000;
|
const STREAMING_UPDATE_INTERVAL = 10000;
|
||||||
const TALKINGCHECK_UPDATE_INTERVAL = 500;
|
|
||||||
const DEFAULT_FALLBACK_EXPRESSION = 'joy';
|
const DEFAULT_FALLBACK_EXPRESSION = 'joy';
|
||||||
const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
|
const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
|
||||||
const DEFAULT_EXPRESSIONS = [
|
const DEFAULT_EXPRESSIONS = [
|
||||||
'talkinghead',
|
|
||||||
'admiration',
|
'admiration',
|
||||||
'amusement',
|
'amusement',
|
||||||
'anger',
|
'anger',
|
||||||
@ -86,8 +84,6 @@ const NO_IMAGE_PLACEHOLDER = { title: 'No Image', type: 'failure', fileName: 'No
|
|||||||
let expressionsList = null;
|
let expressionsList = null;
|
||||||
let lastCharacter = undefined;
|
let lastCharacter = undefined;
|
||||||
let lastMessage = null;
|
let lastMessage = null;
|
||||||
let lastTalkingState = false;
|
|
||||||
let lastTalkingStateMessage = null; // last message as seen by `updateTalkingState` (tracked separately, different timer)
|
|
||||||
/** @type {{[characterKey: string]: Expression[]}} */
|
/** @type {{[characterKey: string]: Expression[]}} */
|
||||||
let spriteCache = {};
|
let spriteCache = {};
|
||||||
let inApiCall = false;
|
let inApiCall = false;
|
||||||
@ -96,10 +92,6 @@ let lastServerResponseTime = 0;
|
|||||||
/** @type {{[characterName: string]: string}} */
|
/** @type {{[characterName: string]: string}} */
|
||||||
export let lastExpression = {};
|
export let lastExpression = {};
|
||||||
|
|
||||||
function isTalkingHeadEnabled() {
|
|
||||||
return extension_settings.expressions.talkinghead && extension_settings.expressions.api == EXPRESSION_API.extras;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the fallback expression if explicitly chosen, otherwise the default one
|
* Returns the fallback expression if explicitly chosen, otherwise the default one
|
||||||
* @returns {string} expression name
|
* @returns {string} expression name
|
||||||
@ -108,18 +100,6 @@ function getFallbackExpression() {
|
|||||||
return extension_settings.expressions.fallback_expression ?? DEFAULT_FALLBACK_EXPRESSION;
|
return extension_settings.expressions.fallback_expression ?? DEFAULT_FALLBACK_EXPRESSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Toggles Talkinghead mode on/off.
|
|
||||||
*
|
|
||||||
* Implements the `/th` slash command, which is meant to be bound to a Quick Reply button
|
|
||||||
* as a quick way to switch Talkinghead on or off (e.g. to conserve GPU resources when AFK
|
|
||||||
* for a long time).
|
|
||||||
*/
|
|
||||||
function toggleTalkingHeadCommand(_) {
|
|
||||||
setTalkingHeadState(!extension_settings.expressions.talkinghead);
|
|
||||||
return String(extension_settings.expressions.talkinghead);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isVisualNovelMode() {
|
function isVisualNovelMode() {
|
||||||
return Boolean(!isMobile() && power_user.waifuMode && getContext().groupId);
|
return Boolean(!isMobile() && power_user.waifuMode && getContext().groupId);
|
||||||
}
|
}
|
||||||
@ -451,199 +431,9 @@ function onExpressionsShowDefaultInput() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Stops animating Talkinghead.
|
|
||||||
*/
|
|
||||||
async function unloadTalkingHead() {
|
|
||||||
if (!modules.includes('talkinghead')) {
|
|
||||||
console.debug('talkinghead module is disabled');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
console.debug('expressions: Stopping Talkinghead');
|
|
||||||
|
|
||||||
try {
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
url.pathname = '/api/talkinghead/unload';
|
|
||||||
const loadResponse = await doExtrasFetch(url);
|
|
||||||
if (!loadResponse.ok) {
|
|
||||||
throw new Error(loadResponse.statusText);
|
|
||||||
}
|
|
||||||
//console.log(`Response: ${loadResponseText}`);
|
|
||||||
} catch (error) {
|
|
||||||
//console.error(`Error unloading - ${error}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Posts `talkinghead.png` of the current character to the talkinghead module in SillyTavern-extras, to start animating it.
|
|
||||||
*/
|
|
||||||
async function loadTalkingHead() {
|
|
||||||
if (!modules.includes('talkinghead')) {
|
|
||||||
console.debug('talkinghead module is disabled');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
console.debug('expressions: Starting Talkinghead');
|
|
||||||
|
|
||||||
const spriteFolderName = getSpriteFolderName();
|
|
||||||
|
|
||||||
const talkingheadPath = `/characters/${encodeURIComponent(spriteFolderName)}/talkinghead.png`;
|
|
||||||
const emotionsSettingsPath = `/characters/${encodeURIComponent(spriteFolderName)}/_emotions.json`;
|
|
||||||
const animatorSettingsPath = `/characters/${encodeURIComponent(spriteFolderName)}/_animator.json`;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const spriteResponse = await fetch(talkingheadPath);
|
|
||||||
|
|
||||||
if (!spriteResponse.ok) {
|
|
||||||
throw new Error(spriteResponse.statusText);
|
|
||||||
}
|
|
||||||
|
|
||||||
const spriteBlob = await spriteResponse.blob();
|
|
||||||
const spriteFile = new File([spriteBlob], 'talkinghead.png', { type: 'image/png' });
|
|
||||||
const formData = new FormData();
|
|
||||||
formData.append('file', spriteFile);
|
|
||||||
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
url.pathname = '/api/talkinghead/load';
|
|
||||||
|
|
||||||
const loadResponse = await doExtrasFetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
body: formData,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!loadResponse.ok) {
|
|
||||||
throw new Error(loadResponse.statusText);
|
|
||||||
}
|
|
||||||
|
|
||||||
const loadResponseText = await loadResponse.text();
|
|
||||||
console.log(`Load talkinghead response: ${loadResponseText}`);
|
|
||||||
|
|
||||||
// Optional: per-character emotion templates
|
|
||||||
let emotionsSettings;
|
|
||||||
try {
|
|
||||||
const emotionsResponse = await fetch(emotionsSettingsPath);
|
|
||||||
if (emotionsResponse.ok) {
|
|
||||||
emotionsSettings = await emotionsResponse.json();
|
|
||||||
console.log(`Loaded ${emotionsSettingsPath}`);
|
|
||||||
} else {
|
|
||||||
throw new Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
emotionsSettings = {}; // blank -> use server defaults (to unload the previous character's customizations)
|
|
||||||
console.log(`No valid config at ${emotionsSettingsPath}, using server defaults`);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
url.pathname = '/api/talkinghead/load_emotion_templates';
|
|
||||||
const apiResult = await doExtrasFetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Bypass-Tunnel-Reminder': 'bypass',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(emotionsSettings),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!apiResult.ok) {
|
|
||||||
throw new Error(apiResult.statusText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
// it's ok if not supported
|
|
||||||
console.log('Failed to send _emotions.json (backend too old?), ignoring');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Optional: per-character animator and postprocessor config
|
|
||||||
let animatorSettings;
|
|
||||||
try {
|
|
||||||
const animatorResponse = await fetch(animatorSettingsPath);
|
|
||||||
if (animatorResponse.ok) {
|
|
||||||
animatorSettings = await animatorResponse.json();
|
|
||||||
console.log(`Loaded ${animatorSettingsPath}`);
|
|
||||||
} else {
|
|
||||||
throw new Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
animatorSettings = {}; // blank -> use server defaults (to unload the previous character's customizations)
|
|
||||||
console.log(`No valid config at ${animatorSettingsPath}, using server defaults`);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
url.pathname = '/api/talkinghead/load_animator_settings';
|
|
||||||
const apiResult = await doExtrasFetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Bypass-Tunnel-Reminder': 'bypass',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(animatorSettings),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!apiResult.ok) {
|
|
||||||
throw new Error(apiResult.statusText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
// it's ok if not supported
|
|
||||||
console.log('Failed to send _animator.json (backend too old?), ignoring');
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error loading talkinghead image: ${talkingheadPath} - ${error}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function handleImageChange() {
|
|
||||||
const imgElement = document.querySelector('img#expression-image.expression');
|
|
||||||
|
|
||||||
if (!imgElement || !(imgElement instanceof HTMLImageElement)) {
|
|
||||||
console.log('Cannot find addExpressionImage()');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isTalkingHeadEnabled() && modules.includes('talkinghead')) {
|
|
||||||
const talkingheadResultFeedSrc = `${getApiUrl()}/api/talkinghead/result_feed`;
|
|
||||||
$('#expression-holder').css({ display: '' });
|
|
||||||
if (imgElement.src !== talkingheadResultFeedSrc) {
|
|
||||||
const expressionImageElement = document.querySelector('.expression_list_image');
|
|
||||||
|
|
||||||
if (expressionImageElement && expressionImageElement instanceof HTMLImageElement) {
|
|
||||||
doExtrasFetch(expressionImageElement.src, {
|
|
||||||
method: 'HEAD',
|
|
||||||
})
|
|
||||||
.then(response => {
|
|
||||||
if (response.ok) {
|
|
||||||
imgElement.src = talkingheadResultFeedSrc;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
console.error(error);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
imgElement.src = ''; // remove in case char doesn't have expressions
|
|
||||||
|
|
||||||
// When switching Talkinghead off, force-set the character to the last known expression, if any.
|
|
||||||
// This preserves the same expression Talkinghead had at the moment it was switched off.
|
|
||||||
const charName = getContext().name2;
|
|
||||||
const last = lastExpression[charName];
|
|
||||||
const targetExpression = last ? last : getFallbackExpression();
|
|
||||||
setExpression(charName, targetExpression, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function moduleWorker() {
|
async function moduleWorker() {
|
||||||
const context = getContext();
|
const context = getContext();
|
||||||
|
|
||||||
// Hide and disable Talkinghead while not in extras
|
|
||||||
$('#image_type_block').toggle(extension_settings.expressions.api == EXPRESSION_API.extras);
|
|
||||||
|
|
||||||
if (extension_settings.expressions.api != EXPRESSION_API.extras && extension_settings.expressions.talkinghead) {
|
|
||||||
$('#image_type_toggle').prop('checked', false);
|
|
||||||
setTalkingHeadState(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
// non-characters not supported
|
// non-characters not supported
|
||||||
if (!context.groupId && context.characterId === undefined) {
|
if (!context.groupId && context.characterId === undefined) {
|
||||||
removeExpression();
|
removeExpression();
|
||||||
@ -773,91 +563,6 @@ async function moduleWorker() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Starts/stops Talkinghead talking animation.
|
|
||||||
*
|
|
||||||
* Talking starts only when all the following conditions are met:
|
|
||||||
* - The LLM is currently streaming its output.
|
|
||||||
* - The AI's current last message is non-empty, and also not just '...' (as produced by a swipe).
|
|
||||||
* - The AI's current last message has changed from what we saw during the previous call.
|
|
||||||
*
|
|
||||||
* In all other cases, talking stops.
|
|
||||||
*
|
|
||||||
* A Talkinghead API call is made only when the talking state changes.
|
|
||||||
*
|
|
||||||
* Note that also the TTS system, if enabled, starts/stops the Talkinghead talking animation.
|
|
||||||
* See `talkingAnimation` in `SillyTavern/public/scripts/extensions/tts/index.js`.
|
|
||||||
*/
|
|
||||||
async function updateTalkingState() {
|
|
||||||
// Don't bother if Talkinghead is disabled or not loaded.
|
|
||||||
if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const context = getContext();
|
|
||||||
const currentLastMessage = getLastCharacterMessage();
|
|
||||||
|
|
||||||
try {
|
|
||||||
// TODO: Not sure if we need also "&& !context.groupId" here - the classify check in `moduleWorker`
|
|
||||||
// (that similarly checks the streaming processor state) does that for some reason.
|
|
||||||
// Talkinghead isn't currently designed to work with groups.
|
|
||||||
const lastMessageChanged = !((lastCharacter === context.characterId || lastCharacter === context.groupId) && lastTalkingStateMessage === currentLastMessage.mes);
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
let newTalkingState;
|
|
||||||
if (context.streamingProcessor && !context.streamingProcessor.isFinished &&
|
|
||||||
currentLastMessage.mes.length !== 0 && currentLastMessage.mes !== '...' && lastMessageChanged) {
|
|
||||||
url.pathname = '/api/talkinghead/start_talking';
|
|
||||||
newTalkingState = true;
|
|
||||||
} else {
|
|
||||||
url.pathname = '/api/talkinghead/stop_talking';
|
|
||||||
newTalkingState = false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
// Call the Talkinghead API only if the talking state changed.
|
|
||||||
if (newTalkingState !== lastTalkingState) {
|
|
||||||
console.debug(`updateTalkingState: calling ${url.pathname}`);
|
|
||||||
await doExtrasFetch(url);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
// it's ok if not supported
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
lastTalkingState = newTalkingState;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
// console.log(error);
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
lastTalkingStateMessage = currentLastMessage.mes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks whether the current character has a talkinghead image available.
|
|
||||||
* @returns {Promise<boolean>} True if the character has a talkinghead image available, false otherwise.
|
|
||||||
*/
|
|
||||||
async function isTalkingHeadAvailable() {
|
|
||||||
let spriteFolderName = getSpriteFolderName();
|
|
||||||
|
|
||||||
try {
|
|
||||||
await validateImages(spriteFolderName);
|
|
||||||
|
|
||||||
let talkingheadObj = spriteCache[spriteFolderName].find(obj => obj.label === 'talkinghead');
|
|
||||||
let talkingheadPath = talkingheadObj ? talkingheadObj.path : null;
|
|
||||||
|
|
||||||
if (talkingheadPath != null) {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
await unloadTalkingHead();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
return err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getSpriteFolderName(characterMessage = null, characterName = null) {
|
function getSpriteFolderName(characterMessage = null, characterName = null) {
|
||||||
const context = getContext();
|
const context = getContext();
|
||||||
let spriteFolderName = characterName ?? context.name2;
|
let spriteFolderName = characterName ?? context.name2;
|
||||||
@ -872,33 +577,6 @@ function getSpriteFolderName(characterMessage = null, characterName = null) {
|
|||||||
return spriteFolderName;
|
return spriteFolderName;
|
||||||
}
|
}
|
||||||
|
|
||||||
function setTalkingHeadState(newState) {
|
|
||||||
console.debug(`expressions: New talkinghead state: ${newState}`);
|
|
||||||
extension_settings.expressions.talkinghead = newState; // Store setting
|
|
||||||
saveSettingsDebounced();
|
|
||||||
|
|
||||||
if ([EXPRESSION_API.local, EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
isTalkingHeadAvailable().then(result => {
|
|
||||||
if (result) {
|
|
||||||
//console.log("talkinghead exists!");
|
|
||||||
|
|
||||||
if (extension_settings.expressions.talkinghead) {
|
|
||||||
loadTalkingHead();
|
|
||||||
} else {
|
|
||||||
unloadTalkingHead();
|
|
||||||
}
|
|
||||||
handleImageChange(); // Change image as needed
|
|
||||||
|
|
||||||
|
|
||||||
} else {
|
|
||||||
//console.log("talkinghead does not exist.");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getFolderNameByMessage(message) {
|
function getFolderNameByMessage(message) {
|
||||||
const context = getContext();
|
const context = getContext();
|
||||||
let avatarPath = '';
|
let avatarPath = '';
|
||||||
@ -976,19 +654,14 @@ async function classifyCallback(/** @type {{api: string?, prompt: string?}} */ {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function setSpriteSlashCommand(_, spriteId) {
|
async function setSpriteSlashCommand(_, spriteId) {
|
||||||
|
spriteId = spriteId.trim().toLowerCase();
|
||||||
if (!spriteId) {
|
if (!spriteId) {
|
||||||
console.log('No sprite id provided');
|
console.log('No sprite id provided');
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
spriteId = spriteId.trim().toLowerCase();
|
const spriteFolderName = getSpriteFolderName();
|
||||||
|
|
||||||
// In Talkinghead mode, don't check for the existence of the sprite
|
|
||||||
// (emotion names are the same as for sprites, but it only needs "talkinghead.png").
|
|
||||||
const currentLastMessage = getLastCharacterMessage();
|
|
||||||
const spriteFolderName = getSpriteFolderName(currentLastMessage, currentLastMessage.name);
|
|
||||||
let label = spriteId;
|
|
||||||
if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) {
|
|
||||||
await validateImages(spriteFolderName);
|
await validateImages(spriteFolderName);
|
||||||
|
|
||||||
// Fuzzy search for sprite
|
// Fuzzy search for sprite
|
||||||
@ -1001,8 +674,7 @@ async function setSpriteSlashCommand(_, spriteId) {
|
|||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
label = spriteItem.label;
|
const label = spriteItem.label;
|
||||||
}
|
|
||||||
|
|
||||||
const vnMode = isVisualNovelMode();
|
const vnMode = isVisualNovelMode();
|
||||||
await sendExpressionCall(spriteFolderName, label, true, vnMode);
|
await sendExpressionCall(spriteFolderName, label, true, vnMode);
|
||||||
@ -1190,7 +862,7 @@ function getJsonSchema(emotions) {
|
|||||||
function onTextGenSettingsReady(args) {
|
function onTextGenSettingsReady(args) {
|
||||||
// Only call if inside an API call
|
// Only call if inside an API call
|
||||||
if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {
|
if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {
|
||||||
const emotions = DEFAULT_EXPRESSIONS.filter((e) => e != 'talkinghead');
|
const emotions = DEFAULT_EXPRESSIONS;
|
||||||
Object.assign(args, {
|
Object.assign(args, {
|
||||||
top_k: 1,
|
top_k: 1,
|
||||||
stop: [],
|
stop: [],
|
||||||
@ -1581,7 +1253,7 @@ export async function getExpressionsList() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If there was no specific list, or an error, just return the default expressions
|
// If there was no specific list, or an error, just return the default expressions
|
||||||
expressionsList = DEFAULT_EXPRESSIONS.filter(e => e !== 'talkinghead').slice();
|
expressionsList = DEFAULT_EXPRESSIONS.slice();
|
||||||
return expressionsList;
|
return expressionsList;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1597,7 +1269,6 @@ export async function getExpressionsList() {
|
|||||||
* @returns {Promise<void>} A promise that resolves when the expression has been set.
|
* @returns {Promise<void>} A promise that resolves when the expression has been set.
|
||||||
*/
|
*/
|
||||||
async function setExpression(character, expression, force = false) {
|
async function setExpression(character, expression, force = false) {
|
||||||
if (!isTalkingHeadEnabled() || !modules.includes('talkinghead')) {
|
|
||||||
console.debug('entered setExpressions');
|
console.debug('entered setExpressions');
|
||||||
await validateImages(character);
|
await validateImages(character);
|
||||||
const img = $('img.expression');
|
const img = $('img.expression');
|
||||||
@ -1718,41 +1389,6 @@ async function setExpression(character, expression, force = false) {
|
|||||||
img.addClass('default');
|
img.addClass('default');
|
||||||
}
|
}
|
||||||
document.getElementById('expression-holder').style.display = '';
|
document.getElementById('expression-holder').style.display = '';
|
||||||
|
|
||||||
} else {
|
|
||||||
// Set the Talkinghead emotion to the specified expression
|
|
||||||
// TODO: For now, Talkinghead emote only supported when VN mode is off; see also updateVisualNovelMode.
|
|
||||||
try {
|
|
||||||
let result = await isTalkingHeadAvailable();
|
|
||||||
if (result) {
|
|
||||||
const url = new URL(getApiUrl());
|
|
||||||
url.pathname = '/api/talkinghead/set_emotion';
|
|
||||||
await doExtrasFetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ emotion_name: expression }),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
// `set_emotion` is not present in old versions, so let it 404.
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Find the <img> element with id="expression-image" and class="expression"
|
|
||||||
const imgElement = document.querySelector('img#expression-image.expression');
|
|
||||||
//console.log("searching");
|
|
||||||
if (imgElement && imgElement instanceof HTMLImageElement) {
|
|
||||||
//console.log("setting value");
|
|
||||||
imgElement.src = getApiUrl() + '/api/talkinghead/result_feed';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
//console.log("The fetch failed!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function onClickExpressionImage() {
|
function onClickExpressionImage() {
|
||||||
@ -1976,11 +1612,6 @@ async function onClickExpressionUpload(event) {
|
|||||||
|
|
||||||
// Reset the input
|
// Reset the input
|
||||||
e.target.form.reset();
|
e.target.form.reset();
|
||||||
|
|
||||||
// In Talkinghead mode, when a new talkinghead image is uploaded, refresh the live char.
|
|
||||||
if (expression === 'talkinghead' && isTalkingHeadEnabled() && modules.includes('talkinghead')) {
|
|
||||||
await loadTalkingHead();
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
$('#expression_upload')
|
$('#expression_upload')
|
||||||
@ -2091,11 +1722,6 @@ async function onClickExpressionUploadPackButton() {
|
|||||||
|
|
||||||
// Reset the input
|
// Reset the input
|
||||||
e.target.form.reset();
|
e.target.form.reset();
|
||||||
|
|
||||||
// In Talkinghead mode, refresh the live char.
|
|
||||||
if (isTalkingHeadEnabled() && modules.includes('talkinghead')) {
|
|
||||||
await loadTalkingHead();
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
$('#expression_upload_pack')
|
$('#expression_upload_pack')
|
||||||
@ -2254,18 +1880,12 @@ function migrateSettings() {
|
|||||||
$(window).on('resize', updateVisualNovelModeDebounced);
|
$(window).on('resize', updateVisualNovelModeDebounced);
|
||||||
$('#open_chat_expressions').hide();
|
$('#open_chat_expressions').hide();
|
||||||
|
|
||||||
$('#image_type_toggle').on('click', function () {
|
|
||||||
if (this instanceof HTMLInputElement) {
|
|
||||||
setTalkingHeadState(this.checked);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
await renderAdditionalExpressionSettings();
|
await renderAdditionalExpressionSettings();
|
||||||
$('#expression_api').val(extension_settings.expressions.api ?? EXPRESSION_API.extras);
|
$('#expression_api').val(extension_settings.expressions.api ?? EXPRESSION_API.extras);
|
||||||
$('.expression_llm_prompt_block').toggle([EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api));
|
$('.expression_llm_prompt_block').toggle([EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api));
|
||||||
$('#expression_llm_prompt').val(extension_settings.expressions.llmPrompt ?? '');
|
$('#expression_llm_prompt').val(extension_settings.expressions.llmPrompt ?? '');
|
||||||
$('#expression_llm_prompt').on('input', function () {
|
$('#expression_llm_prompt').on('input', function () {
|
||||||
extension_settings.expressions.llmPrompt = $(this).val();
|
extension_settings.expressions.llmPrompt = String($(this).val());
|
||||||
saveSettingsDebounced();
|
saveSettingsDebounced();
|
||||||
});
|
});
|
||||||
$('#expression_llm_prompt_restore').on('click', function () {
|
$('#expression_llm_prompt_restore').on('click', function () {
|
||||||
@ -2280,34 +1900,6 @@ function migrateSettings() {
|
|||||||
$('#expression_api').on('change', onExpressionApiChanged);
|
$('#expression_api').on('change', onExpressionApiChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pause Talkinghead to save resources when the ST tab is not visible or the window is minimized.
|
|
||||||
// We currently do this via loading/unloading. Could be improved by adding new pause/unpause endpoints to Extras.
|
|
||||||
document.addEventListener('visibilitychange', function (event) {
|
|
||||||
let pageIsVisible;
|
|
||||||
if (document.hidden) {
|
|
||||||
console.debug('expressions: SillyTavern is now hidden');
|
|
||||||
pageIsVisible = false;
|
|
||||||
} else {
|
|
||||||
console.debug('expressions: SillyTavern is now visible');
|
|
||||||
pageIsVisible = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isTalkingHeadEnabled() && modules.includes('talkinghead')) {
|
|
||||||
isTalkingHeadAvailable().then(result => {
|
|
||||||
if (result) {
|
|
||||||
if (pageIsVisible) {
|
|
||||||
loadTalkingHead();
|
|
||||||
} else {
|
|
||||||
unloadTalkingHead();
|
|
||||||
}
|
|
||||||
handleImageChange(); // Change image as needed
|
|
||||||
} else {
|
|
||||||
//console.log("talkinghead does not exist.");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
addExpressionImage();
|
addExpressionImage();
|
||||||
addVisualNovelMode();
|
addVisualNovelMode();
|
||||||
migrateSettings();
|
migrateSettings();
|
||||||
@ -2316,11 +1908,6 @@ function migrateSettings() {
|
|||||||
const updateFunction = wrapper.update.bind(wrapper);
|
const updateFunction = wrapper.update.bind(wrapper);
|
||||||
setInterval(updateFunction, UPDATE_INTERVAL);
|
setInterval(updateFunction, UPDATE_INTERVAL);
|
||||||
moduleWorker();
|
moduleWorker();
|
||||||
// For setting the Talkinghead talking animation on/off quickly enough for realtime use, we need another timer on a shorter schedule.
|
|
||||||
const wrapperTalkingState = new ModuleWorkerWrapper(updateTalkingState);
|
|
||||||
const updateTalkingStateFunction = wrapperTalkingState.update.bind(wrapperTalkingState);
|
|
||||||
setInterval(updateTalkingStateFunction, TALKINGCHECK_UPDATE_INTERVAL);
|
|
||||||
updateTalkingState();
|
|
||||||
dragElement($('#expression-holder'));
|
dragElement($('#expression-holder'));
|
||||||
eventSource.on(event_types.CHAT_CHANGED, () => {
|
eventSource.on(event_types.CHAT_CHANGED, () => {
|
||||||
// character changed
|
// character changed
|
||||||
@ -2334,12 +1921,6 @@ function migrateSettings() {
|
|||||||
imgElement.src = '';
|
imgElement.src = '';
|
||||||
}
|
}
|
||||||
|
|
||||||
//set checkbox to global var
|
|
||||||
$('#image_type_toggle').prop('checked', extension_settings.expressions.talkinghead);
|
|
||||||
if (extension_settings.expressions.talkinghead) {
|
|
||||||
setTalkingHeadState(extension_settings.expressions.talkinghead);
|
|
||||||
}
|
|
||||||
|
|
||||||
setExpressionOverrideHtml();
|
setExpressionOverrideHtml();
|
||||||
|
|
||||||
if (isVisualNovelMode()) {
|
if (isVisualNovelMode()) {
|
||||||
@ -2350,11 +1931,6 @@ function migrateSettings() {
|
|||||||
});
|
});
|
||||||
eventSource.on(event_types.MOVABLE_PANELS_RESET, updateVisualNovelModeDebounced);
|
eventSource.on(event_types.MOVABLE_PANELS_RESET, updateVisualNovelModeDebounced);
|
||||||
eventSource.on(event_types.GROUP_UPDATED, updateVisualNovelModeDebounced);
|
eventSource.on(event_types.GROUP_UPDATED, updateVisualNovelModeDebounced);
|
||||||
eventSource.on(event_types.EXTRAS_CONNECTED, () => {
|
|
||||||
if (extension_settings.expressions.talkinghead) {
|
|
||||||
setTalkingHeadState(extension_settings.expressions.talkinghead);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
const localEnumProviders = {
|
const localEnumProviders = {
|
||||||
expressions: () => getCachedExpressions().map(expression => {
|
expressions: () => getCachedExpressions().map(expression => {
|
||||||
@ -2420,13 +1996,6 @@ function migrateSettings() {
|
|||||||
],
|
],
|
||||||
helpString: 'Returns the last set expression for the named character.',
|
helpString: 'Returns the last set expression for the named character.',
|
||||||
}));
|
}));
|
||||||
SlashCommandParser.addCommandObject(SlashCommand.fromProps({
|
|
||||||
name: 'expression-talkinghead',
|
|
||||||
callback: toggleTalkingHeadCommand,
|
|
||||||
aliases: ['th', 'talkinghead'],
|
|
||||||
helpString: 'Character Expressions: toggles <i>Image Type - talkinghead (extras)</i> on/off.',
|
|
||||||
returns: 'the current state of the <i>Image Type - talkinghead (extras)</i> on/off.',
|
|
||||||
}));
|
|
||||||
SlashCommandParser.addCommandObject(SlashCommand.fromProps({
|
SlashCommandParser.addCommandObject(SlashCommand.fromProps({
|
||||||
name: 'expression-classify',
|
name: 'expression-classify',
|
||||||
aliases: ['classify-expressions', 'expressions'],
|
aliases: ['classify-expressions', 'expressions'],
|
||||||
|
@ -22,10 +22,6 @@
|
|||||||
<input id="expressions_reroll_if_same" type="checkbox">
|
<input id="expressions_reroll_if_same" type="checkbox">
|
||||||
<span data-i18n="Re-roll if same expression is used again">Re-roll if same sprite is used again</span>
|
<span data-i18n="Re-roll if same expression is used again">Re-roll if same sprite is used again</span>
|
||||||
</label>
|
</label>
|
||||||
<label id="image_type_block" class="checkbox_label" for="image_type_toggle">
|
|
||||||
<input id="image_type_toggle" type="checkbox">
|
|
||||||
<span data-i18n="Image Type - talkinghead (extras)">Image Type - talkinghead (extras)</span>
|
|
||||||
</label>
|
|
||||||
<div class="expression_api_block m-b-1 m-t-1">
|
<div class="expression_api_block m-b-1 m-t-1">
|
||||||
<label for="expression_api" data-i18n="Classifier API">Classifier API</label>
|
<label for="expression_api" data-i18n="Classifier API">Classifier API</label>
|
||||||
<small data-i18n="Select the API for classifying expressions.">Select the API for classifying expressions.</small>
|
<small data-i18n="Select the API for classifying expressions.">Select the API for classifying expressions.</small>
|
||||||
|
@ -27,13 +27,11 @@ import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashComm
|
|||||||
import { enumIcons } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
|
import { enumIcons } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
|
||||||
import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
|
import { POPUP_TYPE, callGenericPopup } from '../../popup.js';
|
||||||
import { GoogleTranslateTtsProvider } from './google-translate.js';
|
import { GoogleTranslateTtsProvider } from './google-translate.js';
|
||||||
export { talkingAnimation };
|
|
||||||
|
|
||||||
const UPDATE_INTERVAL = 1000;
|
const UPDATE_INTERVAL = 1000;
|
||||||
|
|
||||||
let voiceMapEntries = [];
|
let voiceMapEntries = [];
|
||||||
let voiceMap = {}; // {charName:voiceid, charName2:voiceid2}
|
let voiceMap = {}; // {charName:voiceid, charName2:voiceid2}
|
||||||
let talkingHeadState = false;
|
|
||||||
let lastChatId = null;
|
let lastChatId = null;
|
||||||
let lastMessage = null;
|
let lastMessage = null;
|
||||||
let lastMessageHash = null;
|
let lastMessageHash = null;
|
||||||
@ -165,27 +163,6 @@ async function moduleWorker() {
|
|||||||
updateUiAudioPlayState();
|
updateUiAudioPlayState();
|
||||||
}
|
}
|
||||||
|
|
||||||
function talkingAnimation(switchValue) {
|
|
||||||
if (!modules.includes('talkinghead')) {
|
|
||||||
console.debug('Talking Animation module not loaded');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const apiUrl = getApiUrl();
|
|
||||||
const animationType = switchValue ? 'start' : 'stop';
|
|
||||||
|
|
||||||
if (switchValue !== talkingHeadState) {
|
|
||||||
try {
|
|
||||||
console.log(animationType + ' Talking Animation');
|
|
||||||
doExtrasFetch(`${apiUrl}/api/talkinghead/${animationType}_talking`);
|
|
||||||
talkingHeadState = switchValue;
|
|
||||||
} catch (error) {
|
|
||||||
// Handle the error here or simply ignore it to prevent logging
|
|
||||||
}
|
|
||||||
}
|
|
||||||
updateUiAudioPlayState();
|
|
||||||
}
|
|
||||||
|
|
||||||
function resetTtsPlayback() {
|
function resetTtsPlayback() {
|
||||||
// Stop system TTS utterance
|
// Stop system TTS utterance
|
||||||
cancelTtsPlay();
|
cancelTtsPlay();
|
||||||
@ -347,7 +324,6 @@ function onAudioControlClicked() {
|
|||||||
// Not pausing, doing a full stop to anything TTS is doing. Better UX as pause is not as useful
|
// Not pausing, doing a full stop to anything TTS is doing. Better UX as pause is not as useful
|
||||||
if (!audioElement.paused || isTtsProcessing()) {
|
if (!audioElement.paused || isTtsProcessing()) {
|
||||||
resetTtsPlayback();
|
resetTtsPlayback();
|
||||||
talkingAnimation(false);
|
|
||||||
} else {
|
} else {
|
||||||
// Default play behavior if not processing or playing is to play the last message.
|
// Default play behavior if not processing or playing is to play the last message.
|
||||||
ttsJobQueue.push(context.chat[context.chat.length - 1]);
|
ttsJobQueue.push(context.chat[context.chat.length - 1]);
|
||||||
@ -374,7 +350,6 @@ function addAudioControl() {
|
|||||||
function completeCurrentAudioJob() {
|
function completeCurrentAudioJob() {
|
||||||
audioQueueProcessorReady = true;
|
audioQueueProcessorReady = true;
|
||||||
currentAudioJob = null;
|
currentAudioJob = null;
|
||||||
talkingAnimation(false); //stop lip animation
|
|
||||||
// updateUiPlayState();
|
// updateUiPlayState();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -404,7 +379,6 @@ async function processAudioJobQueue() {
|
|||||||
audioQueueProcessorReady = false;
|
audioQueueProcessorReady = false;
|
||||||
currentAudioJob = audioJobQueue.shift();
|
currentAudioJob = audioJobQueue.shift();
|
||||||
playAudioData(currentAudioJob);
|
playAudioData(currentAudioJob);
|
||||||
talkingAnimation(true);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
toastr.error(error.toString());
|
toastr.error(error.toString());
|
||||||
console.error(error);
|
console.error(error);
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
import { isMobile } from '../../RossAscends-mods.js';
|
import { isMobile } from '../../RossAscends-mods.js';
|
||||||
import { getPreviewString } from './index.js';
|
import { getPreviewString } from './index.js';
|
||||||
import { talkingAnimation } from './index.js';
|
|
||||||
import { saveTtsProviderSettings } from './index.js';
|
import { saveTtsProviderSettings } from './index.js';
|
||||||
export { SystemTtsProvider };
|
export { SystemTtsProvider };
|
||||||
|
|
||||||
@ -70,7 +69,6 @@ var speechUtteranceChunker = function (utt, settings, callback) {
|
|||||||
//placing the speak invocation inside a callback fixes ordering and onend issues.
|
//placing the speak invocation inside a callback fixes ordering and onend issues.
|
||||||
setTimeout(function () {
|
setTimeout(function () {
|
||||||
speechSynthesis.speak(newUtt);
|
speechSynthesis.speak(newUtt);
|
||||||
talkingAnimation(true);
|
|
||||||
}, 0);
|
}, 0);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -240,7 +238,6 @@ class SystemTtsProvider {
|
|||||||
//some code to execute when done
|
//some code to execute when done
|
||||||
resolve(silence);
|
resolve(silence);
|
||||||
console.log('System TTS done');
|
console.log('System TTS done');
|
||||||
talkingAnimation(false);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user