Add SpeechT5 TTS provider
This commit is contained in:
Cohee
parent
f1cadfde2b
commit
bdf7602fab
|
|
@ -12,6 +12,7 @@ import { registerSlashCommand } from '../../slash-commands.js';
|
|||
import { OpenAITtsProvider } from './openai.js';
|
||||
import { XTTSTtsProvider } from './xtts.js';
|
||||
import { AllTalkTtsProvider } from './alltalk.js';
|
||||
import { SpeechT5TtsProvider } from './speecht5.js';
|
||||
export { talkingAnimation };
|
||||
|
||||
const UPDATE_INTERVAL = 1000;
|
||||
|
|
@ -76,6 +77,7 @@ let ttsProviders = {
|
|||
Novel: NovelTtsProvider,
|
||||
OpenAI: OpenAITtsProvider,
|
||||
AllTalk: AllTalkTtsProvider,
|
||||
SpeechT5: SpeechT5TtsProvider,
|
||||
};
|
||||
let ttsProvider;
|
||||
let ttsProviderName;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,198 @@
|
|||
import { getPreviewString, saveTtsProviderSettings } from './index.js';
|
||||
import { getBase64Async } from '../../utils.js';
|
||||
import { getRequestHeaders } from '../../../script.js';
|
||||
|
||||
export { SpeechT5TtsProvider };
|
||||
|
||||
class SpeechT5TtsProvider {
|
||||
//########//
|
||||
// Config //
|
||||
//########//
|
||||
|
||||
settings;
|
||||
ready = false;
|
||||
voices = [];
|
||||
separator = ' .. ';
|
||||
audioElement = document.createElement('audio');
|
||||
|
||||
defaultSettings = {
|
||||
speakers: [],
|
||||
speaker: '',
|
||||
voiceMap: {},
|
||||
};
|
||||
|
||||
get settingsHtml() {
|
||||
let html = `
|
||||
<label for="speecht5_tts_speaker">Speaker:</label>
|
||||
<div class="flex-container">
|
||||
<select id="speecht5_tts_speaker" class="text_pole flex1">
|
||||
</select>
|
||||
<div id="speecht5_tts_speaker_upload_button" class="menu_button" title="Upload speaker">
|
||||
<i class="fa-solid fa-upload"></i>
|
||||
</div>
|
||||
<div id="speecht5_tts_delete_speaker_button" class="menu_button" title="Delete speaker">
|
||||
<i class="fa-solid fa-trash"></i>
|
||||
</div>
|
||||
</div>
|
||||
<input type="file" id="speecht5_tts_speaker_upload" class="displayNone">
|
||||
<div><i>Loading model for the first time may take a while!</i></div>
|
||||
`;
|
||||
return html;
|
||||
}
|
||||
|
||||
onSettingsChange() {
|
||||
// Used when provider settings are updated from UI
|
||||
this.settings.speaker = $('#speecht5_tts_speaker').val();
|
||||
saveTtsProviderSettings();
|
||||
}
|
||||
|
||||
async previewTtsVoice(voiceId) {
|
||||
this.audioElement.pause();
|
||||
this.audioElement.currentTime = 0;
|
||||
|
||||
const text = getPreviewString('en-US');
|
||||
const response = await this.fetchTtsGeneration(text, voiceId);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const audio = await response.blob();
|
||||
const url = URL.createObjectURL(audio);
|
||||
this.audioElement.src = url;
|
||||
this.audioElement.play();
|
||||
}
|
||||
|
||||
async loadSettings(settings) {
|
||||
// Pupulate Provider UI given input settings
|
||||
if (Object.keys(settings).length == 0) {
|
||||
console.info('Using default TTS Provider settings');
|
||||
}
|
||||
|
||||
// Only accept keys defined in defaultSettings
|
||||
this.settings = this.defaultSettings;
|
||||
|
||||
for (const key in settings) {
|
||||
if (key in this.settings) {
|
||||
this.settings[key] = settings[key];
|
||||
} else {
|
||||
throw `Invalid setting passed to TTS Provider: ${key}`;
|
||||
}
|
||||
}
|
||||
|
||||
for (const speaker of this.settings.speakers) {
|
||||
$('#speecht5_tts_speaker').append($('<option>', {
|
||||
value: speaker.voice_id,
|
||||
text: speaker.name,
|
||||
}));
|
||||
}
|
||||
|
||||
$('#speecht5_tts_speaker').val(this.settings.speaker);
|
||||
$('#speecht5_tts_speaker').on('change', this.onSettingsChange.bind(this));
|
||||
$('#speecht5_tts_speaker_upload_button').on('click', () => {
|
||||
$('#speecht5_tts_speaker_upload').trigger('click');
|
||||
});
|
||||
$('#speecht5_tts_speaker_upload').on('change', async (event) => {
|
||||
const file = event.target.files[0];
|
||||
if (file.size != 2048) {
|
||||
toastr.error('Invalid speaker file size, expected 2048 bytes');
|
||||
return;
|
||||
}
|
||||
|
||||
const data = await getBase64Async(file);
|
||||
const speaker = {
|
||||
voice_id: file.name,
|
||||
name: file.name,
|
||||
data: data,
|
||||
lang: 'en-US',
|
||||
preview_url: false,
|
||||
};
|
||||
this.settings.speakers.push(speaker);
|
||||
$('#speecht5_tts_speaker').append($('<option>', {
|
||||
value: speaker.voice_id,
|
||||
text: speaker.name,
|
||||
}));
|
||||
$('#speecht5_tts_speaker').val(speaker.name);
|
||||
this.onSettingsChange();
|
||||
});
|
||||
$('#speecht5_tts_delete_speaker_button').on('click', () => {
|
||||
const confirmDelete = confirm('Are you sure you want to delete this speaker?');
|
||||
|
||||
if (!confirmDelete) {
|
||||
return;
|
||||
}
|
||||
|
||||
const speaker = this.settings.speakers.find(s => s.voice_id === this.settings.speaker);
|
||||
if (!speaker) {
|
||||
toastr.error('Speaker not found');
|
||||
return;
|
||||
}
|
||||
|
||||
const index = this.settings.speakers.indexOf(speaker);
|
||||
this.settings.speakers.splice(index, 1);
|
||||
$(`#speecht5_tts_speaker option[value="${speaker.voice_id}"]`).remove();
|
||||
|
||||
if (this.settings.speakers.length == 0) {
|
||||
console.log('No speakers left');
|
||||
return;
|
||||
}
|
||||
|
||||
$('#speecht5_tts_speaker').val(this.settings.speakers[0].voice_id);
|
||||
this.onSettingsChange();
|
||||
});
|
||||
|
||||
await this.checkReady();
|
||||
|
||||
console.debug('SpeechT5: Settings loaded');
|
||||
}
|
||||
|
||||
async checkReady() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
async getVoice(voiceName) {
|
||||
return this.settings.speakers.find(s => s.voice_id === voiceName);
|
||||
}
|
||||
|
||||
async generateTts(text, voiceId) {
|
||||
const response = await this.fetchTtsGeneration(text, voiceId);
|
||||
return response;
|
||||
}
|
||||
|
||||
async fetchTtsVoiceObjects() {
|
||||
return this.settings.speakers;
|
||||
}
|
||||
|
||||
async fetchTtsGeneration(inputText, voiceId) {
|
||||
console.info(`Generating new TTS for voice_id ${voiceId}`);
|
||||
const speaker = await this.getVoice(voiceId);
|
||||
|
||||
if (!speaker) {
|
||||
toastr.error(`Speaker not found: ${voiceId}`, 'TTS Generation Failed');
|
||||
throw new Error(`Speaker not found: ${voiceId}`);
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
'/api/speech/synthesize',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify({
|
||||
'text': inputText,
|
||||
'speaker': speaker.data,
|
||||
'model': 'Xenova/speecht5_tts',
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
toastr.error(response.statusText, 'TTS Generation Failed');
|
||||
throw new Error(`HTTP ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
async fetchTtsFromHistory(history_item_id) {
|
||||
return Promise.resolve(history_item_id);
|
||||
}
|
||||
}
|
||||
|
|
@ -60,7 +60,7 @@ router.post('/synthesize', jsonParser, async (req, res) => {
|
|||
const module = await import('../transformers.mjs');
|
||||
const pipe = await module.default.getPipeline(TASK, model);
|
||||
const speaker_embeddings = speaker
|
||||
? new Float32Array(new Uint8Array(Buffer.from(speaker.split(',')[1], 'base64')).buffer)
|
||||
? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer)
|
||||
: null;
|
||||
const start = performance.now();
|
||||
const result = await pipe(text, { speaker_embeddings: speaker_embeddings });
|
||||
|
|
|
|||
Loading…
Reference in New Issue