Merge pull request #1716 from Tony-sama/staging

VRM extension support

public/assets/vrm/animation/.placeholder

@@ -0,0 +1 @@
+Put VRM animation files here

public/assets/vrm/model/.placeholder

@@ -0,0 +1 @@
+Put VRM model files here

public/scripts/extensions/tts/index.js

@@ -313,13 +313,20 @@ let currentAudioJob;
 let audioPaused = false;
 let audioQueueProcessorReady = true;
 
-async function playAudioData(audioBlob) {
+async function playAudioData(audioJob) {
+    const audioBlob = audioJob["audioBlob"];
     // Since current audio job can be cancelled, don't playback if it is null
     if (currentAudioJob == null) {
         console.log('Cancelled TTS playback because currentAudioJob was null');
     }
     if (audioBlob instanceof Blob) {
         const srcUrl = await getBase64Async(audioBlob);
+
+        // VRM lip sync
+        if (extension_settings.vrm?.enabled && typeof window['vrmLipSync'] === 'function') {
+            await window['vrmLipSync'](audioBlob, audioJob["char"]);
+        }
+
         audioElement.src = srcUrl;
     } else if (typeof audioBlob === 'string') {
         audioElement.src = audioBlob;
@@ -420,15 +427,15 @@ function completeCurrentAudioJob() {
  * Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback
  * @param {Response} response
  */
-async function addAudioJob(response) {
+async function addAudioJob(response, char) {
     if (typeof response === 'string') {
-        audioJobQueue.push(response);
+        audioJobQueue.push({"audioBlob":response, "char":char});
     } else {
         const audioData = await response.blob();
         if (!audioData.type.startsWith('audio/')) {
             throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioData.type}`;
         }
-        audioJobQueue.push(audioData);
+        audioJobQueue.push({"audioBlob":audioData, "char":char});
     }
     console.debug('Pushed audio job to queue.');
 }
@@ -476,7 +483,7 @@ async function tts(text, voiceId, char) {
         if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function')
             response = await window['rvcVoiceConversion'](response, char, text);
 
-        await addAudioJob(response);
+        await addAudioJob(response, char);
     }
 
     let response = await ttsProvider.generateTts(text, voiceId);

src/endpoints/assets.js

@@ -8,7 +8,7 @@ const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
 const { jsonParser } = require('../express-common');
 const { clientRelativePath } = require('../util');
 
-const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
+const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm'];
 
 /**
  * Validates the input filename for the asset.
@@ -106,6 +106,33 @@ router.post('/get', jsonParser, async (_, response) => {
                     continue;
                 }
 
+                // VRM assets
+                if (folder == 'vrm') {
+                    output[folder] = {'model':[], 'animation':[]};
+                    // Extract models
+                    const vrm_model_folder = path.normalize(path.join(folderPath, 'vrm', 'model'));
+                    let files = getFiles(vrm_model_folder);
+                    //console.debug("FILE FOUND:",files)
+                    for (let file of files) {
+                        if (!file.endsWith('.placeholder')) {
+                            //console.debug("Asset VRM model found:",file)
+                            output['vrm']['model'].push(clientRelativePath(file));
+                        }
+                    }
+
+                    // Extract models
+                    const vrm_animation_folder = path.normalize(path.join(folderPath, 'vrm', 'animation'));
+                    files = getFiles(vrm_animation_folder);
+                    //console.debug("FILE FOUND:",files)
+                    for (let file of files) {
+                        if (!file.endsWith('.placeholder')) {
+                            //console.debug("Asset VRM animation found:",file)
+                            output['vrm']['animation'].push(clientRelativePath(file));
+                        }
+                    }
+                    continue;
+                }
+
                 // Other assets (bgm/ambient/blip)
                 const files = fs.readdirSync(path.join(folderPath, folder))
                     .filter(filename => {