From 72d78fbe9695b1e0b2b27b43394bc5bb093a76fb Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 5 Jan 2024 07:00:23 +0100 Subject: [PATCH 1/9] Add VRM import map and assets endpoint. --- public/index.html | 13 +++++++++++++ src/endpoints/assets.js | 29 ++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index dfbdcf032..f2d903bcd 100644 --- a/public/index.html +++ b/public/index.html @@ -22,6 +22,19 @@ + + + + + diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 4ff1d91bb..0bd160cc2 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -8,7 +8,7 @@ const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const { jsonParser } = require('../express-common'); const { clientRelativePath } = require('../util'); -const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; +const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm']; /** * Validates the input filename for the asset. @@ -106,6 +106,33 @@ router.post('/get', jsonParser, async (_, response) => { continue; } + // VRM assets + if (folder == 'vrm') { + output[folder] = {'model':[], 'animation':[]}; + // Extract models + const vrm_model_folder = path.normalize(path.join(folderPath, 'vrm', 'model')); + let files = getFiles(vrm_model_folder); + //console.debug("FILE FOUND:",files) + for (let file of files) { + if (!file.endsWith('.placeholder')) { + //console.debug("Asset VRM model found:",file) + output['vrm']['model'].push(clientRelativePath(file)); + } + } + + // Extract models + const vrm_animation_folder = path.normalize(path.join(folderPath, 'vrm', 'animation')); + files = getFiles(vrm_animation_folder); + //console.debug("FILE FOUND:",files) + for (let file of files) { + if (!file.endsWith('.placeholder')) { + //console.debug("Asset VRM animation found:",file) + output['vrm']['animation'].push(clientRelativePath(file)); + } + } + continue; + } + // Other assets (bgm/ambient/blip) const files = fs.readdirSync(path.join(folderPath, folder)) .filter(filename => { From 4414de90879695e26d077f96f4b8a7cc9f44bce2 Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 5 Jan 2024 22:28:18 +0100 Subject: [PATCH 2/9] Add vrm assets folders. --- public/assets/vrm/animation/.placeholder | 1 + public/assets/vrm/model/.placeholder | 1 + 2 files changed, 2 insertions(+) create mode 100644 public/assets/vrm/animation/.placeholder create mode 100644 public/assets/vrm/model/.placeholder diff --git a/public/assets/vrm/animation/.placeholder b/public/assets/vrm/animation/.placeholder new file mode 100644 index 000000000..c7a29571f --- /dev/null +++ b/public/assets/vrm/animation/.placeholder @@ -0,0 +1 @@ +Put VRM animation files here diff --git a/public/assets/vrm/model/.placeholder b/public/assets/vrm/model/.placeholder new file mode 100644 index 000000000..14ce3cf88 --- /dev/null +++ b/public/assets/vrm/model/.placeholder @@ -0,0 +1 @@ +Put VRM model files here From 017ac8a51568c3b1b7a6e898bcdf13f4e1cf5f4d Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Sun, 14 Jan 2024 04:26:06 +0100 Subject: [PATCH 3/9] Added vrm tts lip sync call --- public/scripts/extensions/tts/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index 7bca9c258..c37e97ebc 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -474,6 +474,10 @@ async function tts(text, voiceId, char) { if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function') response = await window['rvcVoiceConversion'](response, char, text); + // VRM injection + if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') + await window['vrmLipSync'](response, char); + await addAudioJob(response); } From a8a993c5ba1a65d8f07dbe052150a4617c5e9364 Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Tue, 16 Jan 2024 04:54:14 +0100 Subject: [PATCH 4/9] Update tts processing to keep track of character for VRM lip sync. --- public/scripts/extensions/tts/index.js | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index c37e97ebc..1b4020a50 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -311,13 +311,18 @@ let currentAudioJob; let audioPaused = false; let audioQueueProcessorReady = true; -async function playAudioData(audioBlob) { +async function playAudioData(audioJob) { + const audioBlob = audioJob["audioBlob"]; // Since current audio job can be cancelled, don't playback if it is null if (currentAudioJob == null) { console.log('Cancelled TTS playback because currentAudioJob was null'); } if (audioBlob instanceof Blob) { const srcUrl = await getBase64Async(audioBlob); + // VRM inject + if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') { + await window['vrmLipSync'](audioBlob, audioJob["char"]); + } audioElement.src = srcUrl; } else if (typeof audioBlob === 'string') { audioElement.src = audioBlob; @@ -418,15 +423,15 @@ function completeCurrentAudioJob() { * Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback * @param {Response} response */ -async function addAudioJob(response) { +async function addAudioJob(response, char) { if (typeof response === 'string') { - audioJobQueue.push(response); + audioJobQueue.push({"audioBlob":response, "char":char}); } else { const audioData = await response.blob(); if (!audioData.type.startsWith('audio/')) { throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioData.type}`; } - audioJobQueue.push(audioData); + audioJobQueue.push({"audioBlob":audioData, "char":char}); } console.debug('Pushed audio job to queue.'); } @@ -474,11 +479,12 @@ async function tts(text, voiceId, char) { if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function') response = await window['rvcVoiceConversion'](response, char, text); - // VRM injection - if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') + /*/ VRM injection + if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') { await window['vrmLipSync'](response, char); + }*/ - await addAudioJob(response); + await addAudioJob(response, char); } let response = await ttsProvider.generateTts(text, voiceId); From 49a5031e58ea053101e14947f07f51fae3761c83 Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 19 Jan 2024 09:29:49 +0100 Subject: [PATCH 5/9] Clean debug comments --- public/index.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 81d9d5551..09416f5f3 100644 --- a/public/index.html +++ b/public/index.html @@ -22,8 +22,7 @@ - - + - - From b741f32ae941bab1ccdb5c2791ac3c698debef3e Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 19 Jan 2024 09:34:32 +0100 Subject: [PATCH 6/9] Clean comments --- public/scripts/extensions/tts/index.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index 36a80ecc5..d9143998a 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -321,10 +321,12 @@ async function playAudioData(audioJob) { } if (audioBlob instanceof Blob) { const srcUrl = await getBase64Async(audioBlob); - // VRM inject + + // VRM lip sync if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') { await window['vrmLipSync'](audioBlob, audioJob["char"]); } + audioElement.src = srcUrl; } else if (typeof audioBlob === 'string') { audioElement.src = audioBlob; @@ -481,11 +483,6 @@ async function tts(text, voiceId, char) { if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function') response = await window['rvcVoiceConversion'](response, char, text); - /*/ VRM injection - if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') { - await window['vrmLipSync'](response, char); - }*/ - await addAudioJob(response, char); } From bce5352c943014621408fbcaa0b0ef962ebfa6ea Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 19 Jan 2024 17:07:10 +0100 Subject: [PATCH 7/9] Removed VRM importmap. --- public/index.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/public/index.html b/public/index.html index 07af050ea..37e4971ed 100644 --- a/public/index.html +++ b/public/index.html @@ -22,15 +22,6 @@ - - From b7f46b1cdf545d7449ec8f2dbe67747f23d7eeb6 Mon Sep 17 00:00:00 2001 From: Tony Ribeiro Date: Fri, 19 Jan 2024 17:08:45 +0100 Subject: [PATCH 8/9] Remove typo --- public/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/public/index.html b/public/index.html index 37e4971ed..abdf53a5b 100644 --- a/public/index.html +++ b/public/index.html @@ -22,7 +22,6 @@ - From ffbf35e468627ef82764e785476282f6a5bb68e0 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 21 Jan 2024 15:11:47 +0200 Subject: [PATCH 9/9] Update index.js --- public/scripts/extensions/tts/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index d9143998a..c018cc030 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -323,7 +323,7 @@ async function playAudioData(audioJob) { const srcUrl = await getBase64Async(audioBlob); // VRM lip sync - if (extension_settings.vrm.enabled && typeof window['vrmLipSync'] === 'function') { + if (extension_settings.vrm?.enabled && typeof window['vrmLipSync'] === 'function') { await window['vrmLipSync'](audioBlob, audioJob["char"]); }