Merge branch 'staging' of https://github.com/SillyLossy/TavernAI into staging
This commit is contained in:
Cohee
commit
78d62d7be2
|
|
@ -9,6 +9,7 @@ const MODULE_NAME = 'expressions';
|
|||
const UPDATE_INTERVAL = 2000;
|
||||
const FALLBACK_EXPRESSION = 'joy';
|
||||
const DEFAULT_EXPRESSIONS = [
|
||||
"live2d",
|
||||
"admiration",
|
||||
"amusement",
|
||||
"anger",
|
||||
|
|
@ -44,6 +45,9 @@ let lastCharacter = undefined;
|
|||
let lastMessage = null;
|
||||
let spriteCache = {};
|
||||
let inApiCall = false;
|
||||
let live2d_var = false;
|
||||
let previousSrc = null;
|
||||
|
||||
|
||||
function isVisualNovelMode() {
|
||||
return Boolean(!isMobile() && power_user.waifuMode && getContext().groupId);
|
||||
|
|
@ -392,6 +396,56 @@ function onExpressionsShowDefaultInput() {
|
|||
}
|
||||
}
|
||||
|
||||
function loadLiveChar(value_name) {
|
||||
let url = `${getApiUrl()}/api/live2d/load?loadchar=${location.origin}/characters/${value_name}`;
|
||||
doExtrasFetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Bypass-Tunnel-Reminder': 'bypass',
|
||||
},
|
||||
})
|
||||
.then(response => response.text())
|
||||
.then(data => console.log(data))
|
||||
.catch((error) => console.error('Error:', error));
|
||||
}
|
||||
|
||||
|
||||
function handleImageChange(isChecked) {
|
||||
const imgElement = document.querySelector('img#expression-image.expression');
|
||||
|
||||
if (!imgElement) {
|
||||
console.log("Cannot find addExpressionImage()");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isChecked) {
|
||||
// Method get IP of endpoint
|
||||
if (imgElement.src !== getApiUrl() + '/api/live2d/result_feed') {
|
||||
const expressionListItemElement = document.querySelector('#live2d');
|
||||
const expressionImageElement = expressionListItemElement.querySelector('.expression_list_image');
|
||||
const newSrc = expressionImageElement.src;
|
||||
|
||||
doExtrasFetch(newSrc, {
|
||||
method: 'HEAD',
|
||||
})
|
||||
.then(response => {
|
||||
if (response.ok) {
|
||||
imgElement.src = getApiUrl() + '/api/live2d/result_feed';
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(error); // Log the error if necessary
|
||||
});
|
||||
} else if (previousSrc) {
|
||||
imgElement.src = previousSrc; // Revert the src to its previous value
|
||||
}
|
||||
} else if (previousSrc !== null) {
|
||||
imgElement.src = previousSrc; // Revert the src to its previous value
|
||||
}
|
||||
live2d_var = isChecked;
|
||||
}
|
||||
|
||||
async function moduleWorker() {
|
||||
const context = getContext();
|
||||
|
||||
|
|
@ -405,6 +459,21 @@ async function moduleWorker() {
|
|||
if (context.groupId !== lastCharacter && context.characterId !== lastCharacter) {
|
||||
removeExpression();
|
||||
spriteCache = {};
|
||||
|
||||
previousSrc = null;
|
||||
|
||||
//uncheck live image
|
||||
let checkbox = document.getElementById('image_type_toggle');
|
||||
if (checkbox.checked) {
|
||||
checkbox.click();
|
||||
}
|
||||
|
||||
//clear expression
|
||||
let imgElement = document.getElementById('expression-image');
|
||||
imgElement.src = "";
|
||||
|
||||
//Load new char
|
||||
loadLiveChar(context.name2 + ".png");
|
||||
}
|
||||
|
||||
const vnMode = isVisualNovelMode();
|
||||
|
|
@ -654,7 +723,6 @@ async function getSpritesList(name) {
|
|||
|
||||
try {
|
||||
const result = await fetch(`/get_sprites?name=${encodeURIComponent(name)}`);
|
||||
|
||||
let sprites = result.ok ? (await result.json()) : [];
|
||||
return sprites;
|
||||
}
|
||||
|
|
@ -697,114 +765,126 @@ async function getExpressionsList() {
|
|||
}
|
||||
|
||||
async function setExpression(character, expression, force) {
|
||||
console.debug('entered setExpressions');
|
||||
await validateImages(character);
|
||||
const img = $('img.expression');
|
||||
const prevExpressionSrc = img.attr('src');
|
||||
const expressionClone = img.clone()
|
||||
if (live2d_var == false) {
|
||||
|
||||
const sprite = (spriteCache[character] && spriteCache[character].find(x => x.label === expression));
|
||||
console.debug('checking for expression images to show..');
|
||||
if (sprite) {
|
||||
console.debug('setting expression from character images folder');
|
||||
console.debug('entered setExpressions');
|
||||
await validateImages(character);
|
||||
const img = $('img.expression');
|
||||
const prevExpressionSrc = img.attr('src');
|
||||
const expressionClone = img.clone()
|
||||
|
||||
if (force && isVisualNovelMode()) {
|
||||
const context = getContext();
|
||||
const group = context.groups.find(x => x.id === context.groupId);
|
||||
const sprite = (spriteCache[character] && spriteCache[character].find(x => x.label === expression));
|
||||
console.debug('checking for expression images to show..');
|
||||
if (sprite) {
|
||||
console.debug('setting expression from character images folder');
|
||||
|
||||
for (const member of group.members) {
|
||||
const groupMember = context.characters.find(x => x.avatar === member);
|
||||
if (force && isVisualNovelMode()) {
|
||||
const context = getContext();
|
||||
const group = context.groups.find(x => x.id === context.groupId);
|
||||
|
||||
if (!groupMember) {
|
||||
continue;
|
||||
for (const member of group.members) {
|
||||
const groupMember = context.characters.find(x => x.avatar === member);
|
||||
|
||||
if (!groupMember) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (groupMember.name == character) {
|
||||
await setImage($(`.expression-holder[data-avatar="${member}"] img`), sprite.path);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
//only swap expressions when necessary
|
||||
if (prevExpressionSrc !== sprite.path
|
||||
&& !img.hasClass('expression-animating')) {
|
||||
//clone expression
|
||||
expressionClone.addClass('expression-clone')
|
||||
//make invisible and remove id to prevent double ids
|
||||
//must be made invisible to start because they share the same Z-index
|
||||
expressionClone.attr('id', '').css({ opacity: 0 });
|
||||
//add new sprite path to clone src
|
||||
expressionClone.attr('src', sprite.path);
|
||||
//add invisible clone to html
|
||||
expressionClone.appendTo($("#expression-holder"))
|
||||
|
||||
if (groupMember.name == character) {
|
||||
await setImage($(`.expression-holder[data-avatar="${member}"] img`), sprite.path);
|
||||
return;
|
||||
const duration = 200;
|
||||
|
||||
//add animation flags to both images
|
||||
//to prevent multiple expression changes happening simultaneously
|
||||
img.addClass('expression-animating');
|
||||
|
||||
// Set the parent container's min width and height before running the transition
|
||||
const imgWidth = img.width();
|
||||
const imgHeight = img.height();
|
||||
const expressionHolder = img.parent();
|
||||
expressionHolder.css('min-width', imgWidth > 100 ? imgWidth : 100);
|
||||
expressionHolder.css('min-height', imgHeight > 100 ? imgHeight : 100);
|
||||
|
||||
//position absolute prevent the original from jumping around during transition
|
||||
img.css('position', 'absolute');
|
||||
expressionClone.addClass('expression-animating');
|
||||
//fade the clone in
|
||||
expressionClone.css({
|
||||
opacity: 0
|
||||
}).animate({
|
||||
opacity: 1
|
||||
}, duration)
|
||||
//when finshed fading in clone, fade out the original
|
||||
.promise().done(function () {
|
||||
img.animate({
|
||||
opacity: 0
|
||||
}, duration);
|
||||
//remove old expression
|
||||
img.remove();
|
||||
//replace ID so it becomes the new 'original' expression for next change
|
||||
expressionClone.attr('id', 'expression-image');
|
||||
expressionClone.removeClass('expression-animating');
|
||||
|
||||
// Reset the expression holder min height and width
|
||||
expressionHolder.css('min-width', 100);
|
||||
expressionHolder.css('min-height', 100);
|
||||
});
|
||||
|
||||
|
||||
expressionClone.removeClass('expression-clone');
|
||||
|
||||
expressionClone.removeClass('default');
|
||||
expressionClone.off('error');
|
||||
expressionClone.on('error', function () {
|
||||
console.debug('Expression image error', sprite.path);
|
||||
$(this).attr('src', '');
|
||||
$(this).off('error');
|
||||
if (force && extension_settings.expressions.showDefault) {
|
||||
setDefault();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
if (extension_settings.expressions.showDefault) {
|
||||
setDefault();
|
||||
}
|
||||
}
|
||||
}
|
||||
//only swap expressions when necessary
|
||||
if (prevExpressionSrc !== sprite.path
|
||||
&& !img.hasClass('expression-animating')) {
|
||||
//clone expression
|
||||
expressionClone.addClass('expression-clone')
|
||||
//make invisible and remove id to prevent double ids
|
||||
//must be made invisible to start because they share the same Z-index
|
||||
expressionClone.attr('id', '').css({ opacity: 0 });
|
||||
//add new sprite path to clone src
|
||||
expressionClone.attr('src', sprite.path);
|
||||
//add invisible clone to html
|
||||
expressionClone.appendTo($("#expression-holder"))
|
||||
|
||||
const duration = 200;
|
||||
|
||||
//add animation flags to both images
|
||||
//to prevent multiple expression changes happening simultaneously
|
||||
img.addClass('expression-animating');
|
||||
|
||||
// Set the parent container's min width and height before running the transition
|
||||
const imgWidth = img.width();
|
||||
const imgHeight = img.height();
|
||||
const expressionHolder = img.parent();
|
||||
expressionHolder.css('min-width', imgWidth > 100 ? imgWidth : 100);
|
||||
expressionHolder.css('min-height', imgHeight > 100 ? imgHeight : 100);
|
||||
|
||||
//position absolute prevent the original from jumping around during transition
|
||||
img.css('position', 'absolute');
|
||||
expressionClone.addClass('expression-animating');
|
||||
//fade the clone in
|
||||
expressionClone.css({
|
||||
opacity: 0
|
||||
}).animate({
|
||||
opacity: 1
|
||||
}, duration)
|
||||
//when finshed fading in clone, fade out the original
|
||||
.promise().done(function () {
|
||||
img.animate({
|
||||
opacity: 0
|
||||
}, duration);
|
||||
//remove old expression
|
||||
img.remove();
|
||||
//replace ID so it becomes the new 'original' expression for next change
|
||||
expressionClone.attr('id', 'expression-image');
|
||||
expressionClone.removeClass('expression-animating');
|
||||
|
||||
// Reset the expression holder min height and width
|
||||
expressionHolder.css('min-width', 100);
|
||||
expressionHolder.css('min-height', 100);
|
||||
});
|
||||
|
||||
|
||||
expressionClone.removeClass('expression-clone');
|
||||
|
||||
expressionClone.removeClass('default');
|
||||
expressionClone.off('error');
|
||||
expressionClone.on('error', function () {
|
||||
console.debug('Expression image error', sprite.path);
|
||||
$(this).attr('src', '');
|
||||
$(this).off('error');
|
||||
if (force && extension_settings.expressions.showDefault) {
|
||||
setDefault();
|
||||
}
|
||||
});
|
||||
function setDefault() {
|
||||
console.debug('setting default');
|
||||
const defImgUrl = `/img/default-expressions/${expression}.png`;
|
||||
//console.log(defImgUrl);
|
||||
img.attr('src', defImgUrl);
|
||||
img.addClass('default');
|
||||
}
|
||||
document.getElementById("expression-holder").style.display = '';
|
||||
|
||||
} else {
|
||||
if (extension_settings.expressions.showDefault) {
|
||||
setDefault();
|
||||
if (live2d_var == true) {
|
||||
// Find the <img> element with id="expression-image" and class="expression"
|
||||
const imgElement = document.querySelector('img#expression-image.expression');
|
||||
//console.log("searching");
|
||||
if (imgElement) {
|
||||
console.log("setting value");
|
||||
imgElement.src = getApiUrl() + '/api/live2d/result_feed';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setDefault() {
|
||||
console.debug('setting default');
|
||||
const defImgUrl = `/img/default-expressions/${expression}.png`;
|
||||
console.log(defImgUrl);
|
||||
img.attr('src', defImgUrl);
|
||||
img.addClass('default');
|
||||
}
|
||||
document.getElementById("expression-holder").style.display = '';
|
||||
}
|
||||
|
||||
function onClickExpressionImage() {
|
||||
|
|
@ -1052,7 +1132,6 @@ function setExpressionOverrideHtml(forceClear = false) {
|
|||
$('body').append(element);
|
||||
}
|
||||
function addSettings() {
|
||||
|
||||
const html = `
|
||||
<div class="expression_settings">
|
||||
<div class="inline-drawer">
|
||||
|
|
@ -1060,8 +1139,16 @@ function setExpressionOverrideHtml(forceClear = false) {
|
|||
<b>Character Expressions</b>
|
||||
<div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
|
||||
</div>
|
||||
|
||||
<div class="inline-drawer-content">
|
||||
<div class="offline_mode">
|
||||
<!-- Toggle button for aituber/static images -->
|
||||
<div class="toggle_button">
|
||||
<label class="switch">
|
||||
<input id="image_type_toggle" type="checkbox">
|
||||
<span class="slider round"></span>
|
||||
<label for="image_type_toggle">Image Type - Live2d (extras)</label>
|
||||
</div>
|
||||
<div class="offline_mode">
|
||||
<small>You are in offline mode. Click on the image below to set the expression.</small>
|
||||
</div>
|
||||
<div class="flex-container flexnowrap">
|
||||
|
|
@ -1090,6 +1177,7 @@ function setExpressionOverrideHtml(forceClear = false) {
|
|||
</form>
|
||||
</div>
|
||||
`;
|
||||
|
||||
$('#extensions_settings').append(html);
|
||||
$('#expression_override_button').on('click', onClickExpressionOverrideButton);
|
||||
$('#expressions_show_default').on('input', onExpressionsShowDefaultInput);
|
||||
|
|
@ -1105,6 +1193,17 @@ function setExpressionOverrideHtml(forceClear = false) {
|
|||
$(document).on('click', '.expression_list_delete', onClickExpressionDelete);
|
||||
$(window).on("resize", updateVisualNovelModeDebounced);
|
||||
$('.expression_settings').hide();
|
||||
|
||||
|
||||
$('#image_type_toggle').on('change', function () {
|
||||
const isChecked = this.checked;
|
||||
const inputElement = document.querySelector('input[name="avatar_url"]');
|
||||
const value_name = inputElement ? inputElement.value : '';
|
||||
if (isChecked) {
|
||||
loadLiveChar(value_name);
|
||||
}
|
||||
handleImageChange(isChecked);
|
||||
});
|
||||
}
|
||||
|
||||
addExpressionImage();
|
||||
|
|
@ -1116,6 +1215,7 @@ function setExpressionOverrideHtml(forceClear = false) {
|
|||
moduleWorker();
|
||||
dragElement($("#expression-holder"))
|
||||
eventSource.on(event_types.CHAT_CHANGED, () => {
|
||||
//console.log("checked: " + live2d_var);
|
||||
setExpressionOverrideHtml();
|
||||
|
||||
if (isVisualNovelMode()) {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,9 @@ function getTaskByIdRecurse(taskId, task) {
|
|||
function substituteParamsPrompts(content) {
|
||||
content = content.replace(/{{objective}}/gi, currentObjective.description)
|
||||
content = content.replace(/{{task}}/gi, currentTask.description)
|
||||
if (currentTask.parent){
|
||||
content = content.replace(/{{parent}}/gi, currentTask.parent.description)
|
||||
}
|
||||
content = substituteParams(content)
|
||||
return content
|
||||
}
|
||||
|
|
@ -97,7 +100,7 @@ async function generateTasks() {
|
|||
updateUiTaskList();
|
||||
setCurrentTask();
|
||||
console.info(`Response for Objective: '${taskTree.description}' was \n'${taskResponse}', \nwhich created tasks \n${JSON.stringify(globalTasks.map(v => {return v.toSaveState()}), null, 2)} `)
|
||||
toastr.success(`Generated ${globalTasks.length} tasks`, 'Done!');
|
||||
toastr.success(`Generated ${taskTree.length} tasks`, 'Done!');
|
||||
}
|
||||
|
||||
// Call Quiet Generate to check if a task is completed
|
||||
|
|
@ -126,7 +129,7 @@ async function checkTaskCompleted() {
|
|||
function getNextIncompleteTaskRecurse(task){
|
||||
if (task.completed === false // Return task if incomplete
|
||||
&& task.children.length === 0 // Ensure task has no children, it's subtasks will determine completeness
|
||||
&& task.parentId // Must have parent id. Only root task will be missing this and we dont want that
|
||||
&& task.parentId !== "" // Must have parent id. Only root task will be missing this and we dont want that
|
||||
){
|
||||
return task
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,16 +8,21 @@ import { getContext, getApiUrl, modules, extension_settings, ModuleWorkerWrapper
|
|||
import { VoskSttProvider } from './vosk.js'
|
||||
import { WhisperSttProvider } from './whisper.js'
|
||||
import { BrowserSttProvider } from './browser.js'
|
||||
import { StreamingSttProvider } from './streaming.js'
|
||||
export { MODULE_NAME };
|
||||
|
||||
const MODULE_NAME = 'Speech Recognition';
|
||||
const DEBUG_PREFIX = "<Speech Recognition module> "
|
||||
const UPDATE_INTERVAL = 100;
|
||||
|
||||
let inApiCall = false;
|
||||
|
||||
let sttProviders = {
|
||||
None: null,
|
||||
Browser: BrowserSttProvider,
|
||||
Whisper: WhisperSttProvider,
|
||||
Vosk: VoskSttProvider,
|
||||
Streaming: StreamingSttProvider,
|
||||
}
|
||||
|
||||
let sttProvider = null
|
||||
|
|
@ -27,6 +32,82 @@ let audioRecording = false
|
|||
const constraints = { audio: { sampleSize: 16, channelCount: 1, sampleRate: 16000 } };
|
||||
let audioChunks = [];
|
||||
|
||||
async function moduleWorker() {
|
||||
if (sttProviderName != "Streaming") {
|
||||
return;
|
||||
}
|
||||
|
||||
// API is busy
|
||||
if (inApiCall) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
inApiCall = true;
|
||||
const userMessageOriginal = await sttProvider.getUserMessage();
|
||||
let userMessageFormatted = userMessageOriginal.trim();
|
||||
|
||||
if (userMessageFormatted.length > 0)
|
||||
{
|
||||
console.debug(DEBUG_PREFIX+"recorded transcript: \""+userMessageFormatted+"\"");
|
||||
|
||||
let userMessageLower = userMessageFormatted.toLowerCase();
|
||||
// remove punctuation
|
||||
let userMessageRaw = userMessageLower.replace(/[^\w\s\']|_/g, "").replace(/\s+/g, " ");
|
||||
|
||||
console.debug(DEBUG_PREFIX+"raw transcript:",userMessageRaw);
|
||||
|
||||
// Detect trigger words
|
||||
let messageStart = -1;
|
||||
|
||||
if (extension_settings.speech_recognition.Streaming.triggerWordsEnabled) {
|
||||
|
||||
for (const triggerWord of extension_settings.speech_recognition.Streaming.triggerWords) {
|
||||
const triggerPos = userMessageRaw.indexOf(triggerWord.toLowerCase());
|
||||
|
||||
// Trigger word not found or not starting message and just a substring
|
||||
if (triggerPos == -1){ // | (triggerPos > 0 & userMessageFormatted[triggerPos-1] != " ")) {
|
||||
console.debug(DEBUG_PREFIX+"trigger word not found: ", triggerWord);
|
||||
}
|
||||
else {
|
||||
console.debug(DEBUG_PREFIX+"Found trigger word: ", triggerWord, " at index ", triggerPos);
|
||||
if (triggerPos < messageStart | messageStart == -1) { // & (triggerPos + triggerWord.length) < userMessageFormatted.length)) {
|
||||
messageStart = triggerPos; // + triggerWord.length + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
messageStart = 0;
|
||||
}
|
||||
|
||||
if (messageStart == -1) {
|
||||
console.debug(DEBUG_PREFIX+"message ignored, no trigger word preceding a message. Voice transcript: \""+ userMessageOriginal +"\"");
|
||||
if (extension_settings.speech_recognition.Streaming.debug) {
|
||||
toastr.info(
|
||||
"No trigger word preceding a message. Voice transcript: \""+ userMessageOriginal +"\"",
|
||||
DEBUG_PREFIX+"message ignored.",
|
||||
{ timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true },
|
||||
);
|
||||
}
|
||||
}
|
||||
else{
|
||||
userMessageFormatted = userMessageFormatted.substring(messageStart);
|
||||
processTranscript(userMessageFormatted);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
console.debug(DEBUG_PREFIX+"Received empty transcript, ignored");
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.debug(error);
|
||||
}
|
||||
finally {
|
||||
inApiCall = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function processTranscript(transcript) {
|
||||
try {
|
||||
const transcriptOriginal = transcript;
|
||||
|
|
@ -198,13 +279,21 @@ function loadSttProvider(provider) {
|
|||
if (sttProviderName == "Browser") {
|
||||
sttProvider.processTranscriptFunction = processTranscript;
|
||||
sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
|
||||
}
|
||||
else {
|
||||
sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
|
||||
loadNavigatorAudioRecording();
|
||||
|
||||
$("#microphone_button").show();
|
||||
}
|
||||
|
||||
if (sttProviderName == "Vosk" | sttProviderName == "Whisper") {
|
||||
sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
|
||||
loadNavigatorAudioRecording();
|
||||
$("#microphone_button").show();
|
||||
}
|
||||
|
||||
if (sttProviderName == "Streaming") {
|
||||
sttProvider.loadSettings(extension_settings.speech_recognition[sttProviderName]);
|
||||
$("#microphone_button").off('click');
|
||||
$("#microphone_button").hide();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function onSttProviderChange() {
|
||||
|
|
@ -231,7 +320,7 @@ const defaultSettings = {
|
|||
messageMode: "append",
|
||||
messageMappingText: "",
|
||||
messageMapping: [],
|
||||
messageMappingEnabled: false
|
||||
messageMappingEnabled: false,
|
||||
}
|
||||
|
||||
function loadSettings() {
|
||||
|
|
@ -344,8 +433,7 @@ $(document).ready(function () {
|
|||
addExtensionControls(); // No init dependencies
|
||||
loadSettings(); // Depends on Extension Controls and loadTtsProvider
|
||||
loadSttProvider(extension_settings.speech_recognition.currentProvider); // No dependencies
|
||||
|
||||
//const wrapper = new ModuleWorkerWrapper(moduleWorker);
|
||||
//setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL); // Init depends on all the things
|
||||
//moduleWorker();
|
||||
const wrapper = new ModuleWorkerWrapper(moduleWorker);
|
||||
setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL); // Init depends on all the things
|
||||
moduleWorker();
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
import { getApiUrl, doExtrasFetch, modules } from "../../extensions.js";
|
||||
export { StreamingSttProvider }
|
||||
|
||||
const DEBUG_PREFIX = "<Speech Recognition module (streaming)> "
|
||||
|
||||
class StreamingSttProvider {
|
||||
//########//
|
||||
// Config //
|
||||
//########//
|
||||
|
||||
settings
|
||||
|
||||
defaultSettings = {
|
||||
triggerWordsText: "",
|
||||
triggerWords : [],
|
||||
triggerWordsEnabled : false,
|
||||
debug : false,
|
||||
}
|
||||
|
||||
get settingsHtml() {
|
||||
let html = '\
|
||||
<div id="speech_recognition_streaming_trigger_words_div">\
|
||||
<span>Trigger words</span>\
|
||||
<textarea id="speech_recognition_streaming_trigger_words" class="text_pole textarea_compact" type="text" rows="4" placeholder="Enter comma separated words that triggers new message, example:\nhey, hey aqua, record, listen"></textarea>\
|
||||
<label class="checkbox_label" for="speech_recognition_streaming_trigger_words_enabled">\
|
||||
<input type="checkbox" id="speech_recognition_streaming_trigger_words_enabled" name="speech_recognition_trigger_words_enabled">\
|
||||
<small>Enable trigger words</small>\
|
||||
</label>\
|
||||
<label class="checkbox_label" for="speech_recognition_streaming_debug">\
|
||||
<input type="checkbox" id="speech_recognition_streaming_debug" name="speech_recognition_streaming_debug">\
|
||||
<small>Enable debug pop ups</small>\
|
||||
</label>\
|
||||
</div>\
|
||||
'
|
||||
return html
|
||||
}
|
||||
|
||||
onSettingsChange() {
|
||||
this.settings.triggerWordsText = $('#speech_recognition_streaming_trigger_words').val();
|
||||
let array = $('#speech_recognition_streaming_trigger_words').val().split(",");
|
||||
array = array.map(element => {return element.trim().toLowerCase();});
|
||||
array = array.filter((str) => str !== '');
|
||||
this.settings.triggerWords = array;
|
||||
this.settings.triggerWordsEnabled = $("#speech_recognition_streaming_trigger_words_enabled").is(':checked');
|
||||
this.settings.debug = $("#speech_recognition_streaming_debug").is(':checked');
|
||||
console.debug(DEBUG_PREFIX+" Updated settings: ", this.settings);
|
||||
this.loadSettings(this.settings);
|
||||
}
|
||||
|
||||
loadSettings(settings) {
|
||||
// Populate Provider UI given input settings
|
||||
if (Object.keys(settings).length == 0) {
|
||||
console.debug(DEBUG_PREFIX+"Using default Whisper STT extension settings")
|
||||
}
|
||||
|
||||
// Only accept keys defined in defaultSettings
|
||||
this.settings = this.defaultSettings
|
||||
|
||||
for (const key in settings){
|
||||
if (key in this.settings){
|
||||
this.settings[key] = settings[key]
|
||||
} else {
|
||||
throw `Invalid setting passed to STT extension: ${key}`
|
||||
}
|
||||
}
|
||||
|
||||
$("#speech_recognition_streaming_trigger_words").val(this.settings.triggerWordsText);
|
||||
$("#speech_recognition_streaming_trigger_words_enabled").prop('checked',this.settings.triggerWordsEnabled);
|
||||
$("#speech_recognition_streaming_debug").prop('checked',this.settings.debug);
|
||||
|
||||
console.debug(DEBUG_PREFIX+"streaming STT settings loaded")
|
||||
}
|
||||
|
||||
async getUserMessage() {
|
||||
// Return if module is not loaded
|
||||
if (!modules.includes('streaming-stt')) {
|
||||
console.debug(DEBUG_PREFIX+"Module streaming-stt must be activated in Sillytavern Extras for streaming user voice.")
|
||||
return "";
|
||||
}
|
||||
|
||||
const url = new URL(getApiUrl());
|
||||
url.pathname = '/api/speech-recognition/streaming/record-and-transcript';
|
||||
|
||||
const apiResult = await doExtrasFetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Bypass-Tunnel-Reminder': 'bypass',
|
||||
},
|
||||
body: JSON.stringify({ text: "" }),
|
||||
});
|
||||
|
||||
if (!apiResult.ok) {
|
||||
toastr.error(apiResult.statusText, DEBUG_PREFIX+'STT Generation Failed (streaming)', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true });
|
||||
throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`);
|
||||
}
|
||||
|
||||
const data = await apiResult.json();
|
||||
return data.transcript;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { callPopup, cancelTtsPlay, eventSource, event_types, isMultigenEnabled, is_send_press, saveSettingsDebounced } from '../../../script.js'
|
||||
import { ModuleWorkerWrapper, extension_settings, getContext } from '../../extensions.js'
|
||||
import { ModuleWorkerWrapper, doExtrasFetch, extension_settings, getApiUrl, getContext } from '../../extensions.js'
|
||||
import { escapeRegex, getStringHash } from '../../utils.js'
|
||||
import { EdgeTtsProvider } from './edge.js'
|
||||
import { ElevenLabsTtsProvider } from './elevenlabs.js'
|
||||
|
|
@ -7,14 +7,13 @@ import { SileroTtsProvider } from './silerotts.js'
|
|||
import { CoquiTtsProvider } from './coquitts.js'
|
||||
import { SystemTtsProvider } from './system.js'
|
||||
import { NovelTtsProvider } from './novel.js'
|
||||
import { isMobile } from '../../RossAscends-mods.js'
|
||||
import { power_user } from '../../power-user.js'
|
||||
|
||||
const UPDATE_INTERVAL = 1000
|
||||
|
||||
let voiceMap = {} // {charName:voiceid, charName2:voiceid2}
|
||||
let audioControl
|
||||
|
||||
let storedvalue = false;
|
||||
let lastCharacterId = null
|
||||
let lastGroupId = null
|
||||
let lastChatId = null
|
||||
|
|
@ -164,6 +163,20 @@ async function moduleWorker() {
|
|||
ttsJobQueue.push(message)
|
||||
}
|
||||
|
||||
function talkingAnimation(switchValue) {
|
||||
const apiUrl = getApiUrl();
|
||||
const animationType = switchValue ? "start" : "stop";
|
||||
|
||||
if (switchValue !== storedvalue) {
|
||||
try {
|
||||
console.log(animationType + " Talking Animation");
|
||||
doExtrasFetch(`${apiUrl}/api/live2d/${animationType}_talking`);
|
||||
storedvalue = switchValue; // Update the storedvalue to the current switchValue
|
||||
} catch (error) {
|
||||
// Handle the error here or simply ignore it to prevent logging
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resetTtsPlayback() {
|
||||
// Stop system TTS utterance
|
||||
|
|
@ -291,8 +304,10 @@ function updateUiAudioPlayState() {
|
|||
// Give user feedback that TTS is active by setting the stop icon if processing or playing
|
||||
if (!audioElement.paused || isTtsProcessing()) {
|
||||
img = 'fa-solid fa-stop-circle extensionsMenuExtensionButton'
|
||||
talkingAnimation(true)
|
||||
} else {
|
||||
img = 'fa-solid fa-circle-play extensionsMenuExtensionButton'
|
||||
talkingAnimation(false)
|
||||
}
|
||||
$('#tts_media_control').attr('class', img);
|
||||
} else {
|
||||
|
|
@ -354,6 +369,7 @@ async function processAudioJobQueue() {
|
|||
audioQueueProcessorReady = false
|
||||
currentAudioJob = audioJobQueue.pop()
|
||||
playAudioData(currentAudioJob)
|
||||
talkingAnimation(true)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
audioQueueProcessorReady = true
|
||||
|
|
|
|||
Loading…
Reference in New Issue