rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-01-31 11:35:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1716 from Tony-sama/staging

Browse Source 
VRM extension support
This commit is contained in:
Cohee 2024-01-21 15:12:44 +02:00 committed by GitHub
commit f3fc3f12a7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 42 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
public/assets/vrm/animation/.placeholder Normal file
View File

@ -0,0 +1 @@
Put VRM animation files here

1
public/assets/vrm/model/.placeholder Normal file
View File

@ -0,0 +1 @@
Put VRM model files here

17
public/scripts/extensions/tts/index.js
View File

@ -313,13 +313,20 @@ let currentAudioJob;
let audioPaused = false;
let audioQueueProcessorReady = true;
async function playAudioData(audioBlob) {
async function playAudioData(audioJob) {
const audioBlob = audioJob["audioBlob"];
// Since current audio job can be cancelled, don't playback if it is null
if (currentAudioJob == null) {
console.log('Cancelled TTS playback because currentAudioJob was null');
}
if (audioBlob instanceof Blob) {
const srcUrl = await getBase64Async(audioBlob);
// VRM lip sync
if (extension_settings.vrm?.enabled && typeof window['vrmLipSync'] === 'function') {
await window['vrmLipSync'](audioBlob, audioJob["char"]);
}
audioElement.src = srcUrl;
} else if (typeof audioBlob === 'string') {
audioElement.src = audioBlob;
@ -420,15 +427,15 @@ function completeCurrentAudioJob() {
* Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback
* @param {Response} response
*/
async function addAudioJob(response) {
async function addAudioJob(response, char) {
if (typeof response === 'string') {
audioJobQueue.push(response);
audioJobQueue.push({"audioBlob":response, "char":char});
} else {
const audioData = await response.blob();
if (!audioData.type.startsWith('audio/')) {
throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioData.type}`;
}
audioJobQueue.push(audioData);
audioJobQueue.push({"audioBlob":audioData, "char":char});
}
console.debug('Pushed audio job to queue.');
}
@ -476,7 +483,7 @@ async function tts(text, voiceId, char) {
if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function')
response = await window['rvcVoiceConversion'](response, char, text);
await addAudioJob(response);
await addAudioJob(response, char);
}
let response = await ttsProvider.generateTts(text, voiceId);

29
src/endpoints/assets.js
View File

@ -8,7 +8,7 @@ const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const { clientRelativePath } = require('../util');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm'];
/**
* Validates the input filename for the asset.
@ -106,6 +106,33 @@ router.post('/get', jsonParser, async (_, response) => {
continue;
}
// VRM assets
if (folder == 'vrm') {
output[folder] = {'model':[], 'animation':[]};
// Extract models
const vrm_model_folder = path.normalize(path.join(folderPath, 'vrm', 'model'));
let files = getFiles(vrm_model_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
if (!file.endsWith('.placeholder')) {
//console.debug("Asset VRM model found:",file)
output['vrm']['model'].push(clientRelativePath(file));
}
}
// Extract models
const vrm_animation_folder = path.normalize(path.join(folderPath, 'vrm', 'animation'));
files = getFiles(vrm_animation_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
if (!file.endsWith('.placeholder')) {
//console.debug("Asset VRM animation found:",file)
output['vrm']['animation'].push(clientRelativePath(file));
}
}
continue;
}
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {