TTS modularity better. Add Silero TTS

This commit is contained in:
ouoertheo
2023-04-28 16:23:36 -05:00
parent c354f8ee30
commit 99de695868
3 changed files with 328 additions and 147 deletions

View File

@@ -5,48 +5,24 @@ class ElevenLabsTtsProvider {
// Config //
//########//
API_KEY
settings = this.defaultSettings
settings
voices = []
set API_KEY(apiKey) {
this.API_KEY = apiKey
}
get API_KEY() {
return this.API_KEY
}
get settings() {
return this.settings
}
updateSettings(settings) {
console.info("Settings updated")
if("stability" in settings && "similarity_boost" in settings){
this.settings = settings
$('#elevenlabs_tts_stability').val(this.settings.stability)
$('#elevenlabs_tts_similarity_boost').val(this.settings.similarity_boost)
this.onSettingsChange()
} else {
throw `Invalid settings passed to ElevenLabs: ${JSON.stringify(settings)}`
}
}
defaultSettings = {
stability: 0.75,
similarity_boost: 0.75
}
onSettingsChange() {
this.settings = {
stability: $('#elevenlabs_tts_stability').val(),
similarity_boost: $('#elevenlabs_tts_similarity_boost').val()
}
$('#elevenlabs_tts_stability_output').text(this.settings.stability)
$('#elevenlabs_tts_similarity_boost_output').text(this.settings.similarity_boost)
similarity_boost: 0.75,
apiKey: "",
voiceMap: {}
}
get settingsHtml() {
let html = `
<label for="elevenlabs_tts_api_key">API Key</label>
<input id="elevenlabs_tts_api_key" type="text" class="text_pole" placeholder="<API Key>"/>
<label for="elevenlabs_tts_stability">Stability: <span id="elevenlabs_tts_stability_output"></span></label>
<input id="elevenlabs_tts_stability" type="range" value="${this.defaultSettings.stability}" min="0" max="1" step="0.05" />
<label for="elevenlabs_tts_similarity_boost">Similarity Boost: <span id="elevenlabs_tts_similarity_boost_output"></span></label>
@@ -55,9 +31,58 @@ class ElevenLabsTtsProvider {
return html
}
//#############//
// Management //
//#############//
onSettingsChange() {
// Update dynamically
this.settings.stability = $('#elevenlabs_tts_stability').val()
this.settings.similarity_boost = $('#elevenlabs_tts_similarity_boost').val()
}
loadSettings(settings) {
// Pupulate Provider UI given input settings
if (Object.keys(settings).length == 0) {
console.info("Using default TTS Provider settings")
}
// Only accept keys defined in defaultSettings
this.settings = this.defaultSettings
for (const key in settings){
if (key in this.settings){
this.settings[key] = settings[key]
} else {
throw `Invalid setting passed to TTS Provider: ${key}`
}
}
$('#elevenlabs_tts_stability').val(this.settings.stability)
$('#elevenlabs_tts_similarity_boost').val(this.settings.similarity_boost)
$('#elevenlabs_tts_api_key').val(this.settings.apiKey)
console.info("Settings loaded")
}
async onApplyClick() {
// Update on Apply click
return await this.updateApiKey().catch( (error) => {
throw error
})
}
async updateApiKey() {
// Using this call to validate API key
this.settings.apiKey = $('#elevenlabs_tts_api_key').val()
await this.fetchTtsVoiceIds().catch(error => {
throw `TTS API key validation failed`
})
this.settings.apiKey = this.settings.apiKey
console.debug(`Saved new API_KEY: ${this.settings.apiKey}`)
}
//#################//
// TTS Interfaces //
//#################//
async getVoice(voiceName) {
if (this.voices.length == 0) {
@@ -72,6 +97,25 @@ class ElevenLabsTtsProvider {
return match
}
async generateTts(text, voiceId){
const historyId = await this.findTtsGenerationInHistory(text, voiceId)
let response
if (historyId) {
console.debug(`Found existing TTS generation with id ${historyId}`)
response = await this.fetchTtsFromHistory(historyId)
} else {
console.debug(`No existing TTS generation found, requesting new generation`)
response = await this.fetchTtsGeneration(text, voiceId)
}
return response
}
//###################//
// Helper Functions //
//###################//
async findTtsGenerationInHistory(message, voiceId) {
const ttsHistory = await this.fetchTtsHistory()
for (const history of ttsHistory) {
@@ -85,12 +129,13 @@ class ElevenLabsTtsProvider {
return ''
}
//###########//
// API CALLS //
//###########//
async fetchTtsVoiceIds() {
const headers = {
'xi-api-key': this.API_KEY
'xi-api-key': this.settings.apiKey
}
const response = await fetch(`https://api.elevenlabs.io/v1/voices`, {
headers: headers
@@ -104,7 +149,7 @@ class ElevenLabsTtsProvider {
async fetchTtsVoiceSettings() {
const headers = {
'xi-api-key': this.API_KEY
'xi-api-key': this.settings.apiKey
}
const response = await fetch(
`https://api.elevenlabs.io/v1/voices/settings/default`,
@@ -125,7 +170,7 @@ class ElevenLabsTtsProvider {
{
method: 'POST',
headers: {
'xi-api-key': this.API_KEY,
'xi-api-key': this.settings.apiKey,
'Content-Type': 'application/json'
},
body: JSON.stringify({
@@ -146,7 +191,7 @@ class ElevenLabsTtsProvider {
`https://api.elevenlabs.io/v1/history/${history_item_id}/audio`,
{
headers: {
'xi-api-key': this.API_KEY
'xi-api-key': this.settings.apiKey
}
}
)
@@ -158,7 +203,7 @@ class ElevenLabsTtsProvider {
async fetchTtsHistory() {
const headers = {
'xi-api-key': this.API_KEY
'xi-api-key': this.settings.apiKey
}
const response = await fetch(`https://api.elevenlabs.io/v1/history`, {
headers: headers

View File

@@ -2,6 +2,7 @@ import { callPopup, saveSettingsDebounced } from '../../../script.js'
import { extension_settings, getContext } from '../../extensions.js'
import { getStringHash } from '../../utils.js'
import { ElevenLabsTtsProvider } from './elevenlabs.js'
import { SileroTtsProvider } from './silerotts.js'
const UPDATE_INTERVAL = 1000
@@ -15,7 +16,8 @@ let lastMessageHash = null
let ttsProviders = {
elevenLabs: ElevenLabsTtsProvider
ElevenLabs: ElevenLabsTtsProvider,
Silero: SileroTtsProvider
}
let ttsProvider
let ttsProviderName
@@ -130,6 +132,30 @@ async function onTtsVoicesClick() {
callPopup(popupText, 'text')
}
function updateUiAudioPlayState() {
if (extension_settings.tts.enabled == true) {
audioControl.style.display = 'flex'
const img = !audioElement.paused
? 'fa-solid fa-circle-pause'
: 'fa-solid fa-circle-play'
audioControl.className = img
} else {
audioControl.style.display = 'none'
}
}
function onAudioControlClicked() {
audioElement.paused ? audioElement.play() : audioElement.pause()
updateUiAudioPlayState()
}
function addAudioControl() {
$('#send_but_sheld').prepend('<div id="tts_media_control"/>')
$('#send_but_sheld').on('click', onAudioControlClicked)
audioControl = document.getElementById('tts_media_control')
updateUiAudioPlayState()
}
function completeCurrentAudioJob() {
queueProcessorReady = true
lastAudioPosition = 0
@@ -142,7 +168,7 @@ function completeCurrentAudioJob() {
*/
async function addAudioJob(response) {
const audioData = await response.blob()
if (audioData.type != 'audio/mpeg') {
if (!audioData.type in ['audio/mpeg', 'audio/wav']) {
throw `TTS received HTTP response with invalid data format. Expecting audio/mpeg, got ${audioData.type}`
}
audioJobQueue.push(audioData)
@@ -188,16 +214,7 @@ function saveLastValues() {
}
async function tts(text, voiceId) {
const historyId = await ttsProvider.findTtsGenerationInHistory(text, voiceId)
let response
if (historyId) {
console.debug(`Found existing TTS generation with id ${historyId}`)
response = await ttsProvider.fetchTtsFromHistory(historyId)
} else {
console.debug(`No existing TTS generation found, requesting new generation`)
response = await ttsProvider.fetchTtsGeneration(text, voiceId)
}
const response = await ttsProvider.generateTts(text, voiceId)
addAudioJob(response)
completeTtsJob()
}
@@ -242,32 +259,20 @@ window.playFullConversation = playFullConversation
//#############################//
function loadSettings() {
if (!(ttsProviderName in extension_settings.tts)){
extension_settings.tts[ttsProviderName] = {}
if (Object.keys(extension_settings.tts).length === 0) {
Object.assign(extension_settings.tts, defaultSettings)
}
if (Object.keys(extension_settings.tts[ttsProviderName]).length === 0) {
Object.assign(extension_settings.tts[ttsProviderName], defaultSettings)
extension_settings.tts[ttsProviderName].settings = Object.assign({}, ttsProvider.defaultSettings)
}
$('#tts_api_key').val(
extension_settings.tts[ttsProviderName].apiKey
)
$('#tts_voice_map').val(
extension_settings.tts[ttsProviderName].voiceMap
)
$('#tts_enabled').prop(
'checked',
extension_settings.tts.enabled
)
ttsProvider.updateSettings(extension_settings.tts[ttsProviderName].settings)
onApplyClick()
}
const defaultSettings = {
apiKey: '',
voiceMap: '',
ttsEnabled: false
ttsEnabled: false,
currentProvider: "ElevenLabs"
}
function setTtsStatus(status, success) {
@@ -279,21 +284,6 @@ function setTtsStatus(status, success) {
}
}
async function updateApiKey() {
const value = $('#tts_api_key').val()
// Using this call to validate API key
ttsProvider.API_KEY = String(value)
await ttsProvider.fetchTtsVoiceIds().catch(error => {
ttsProvider.API_KEY = null
throw `TTS API key invalid`
})
extension_settings.tts[ttsProviderName].apiKey = String(value)
console.debug(`Saved new API_KEY: ${value}`)
saveSettingsDebounced()
}
function parseVoiceMap(voiceMapString) {
let parsedVoiceMap = {}
for (const [charName, voiceId] of voiceMapString
@@ -323,13 +313,14 @@ async function voicemapIsValid(parsedVoiceMap) {
async function updateVoiceMap() {
let isValidResult = false
const context = getContext()
// console.debug("onvoiceMapSubmit");
const value = $('#tts_voice_map').val()
const parsedVoiceMap = parseVoiceMap(value)
isValidResult = await voicemapIsValid(parsedVoiceMap)
if (isValidResult) {
extension_settings.tts[ttsProviderName].voiceMap = String(value)
context.voiceMap = String(value)
ttsProvider.settings.voiceMap = String(value)
// console.debug(`ttsProvider.voiceMap: ${ttsProvider.settings.voiceMap}`)
voiceMap = parsedVoiceMap
console.debug(`Saved new voiceMap: ${value}`)
saveSettingsDebounced()
@@ -339,14 +330,18 @@ async function updateVoiceMap() {
}
function onApplyClick() {
Promise.all([updateApiKey(), updateVoiceMap()])
.then(([result1, result2]) => {
updateUiAudioPlayState()
setTtsStatus('Successfully applied settings', true)
})
.catch(error => {
setTtsStatus(error, false)
})
Promise.all([
ttsProvider.onApplyClick(),
updateVoiceMap()
]).catch(error => {
console.error(error)
setTtsStatus(error, false)
})
extension_settings.tts[ttsProviderName] = ttsProvider.settings
saveSettingsDebounced()
setTtsStatus('Successfully applied settings', true)
console.info(`Saved settings ${ttsProviderName} ${JSON.stringify(ttsProvider.settings)}`)
}
function onEnableClick() {
@@ -357,49 +352,62 @@ function onEnableClick() {
saveSettingsDebounced()
}
function updateUiAudioPlayState() {
if (extension_settings.tts.enabled == true) {
audioControl.style.display = 'flex'
const img = !audioElement.paused
? 'fa-solid fa-circle-pause'
: 'fa-solid fa-circle-play'
audioControl.className = img
} else {
audioControl.style.display = 'none'
//##############//
// TTS Provider //
//##############//
function loadTtsProvider(provider) {
//Clear the current config and add new config
$("#tts_provider_settings").html("")
if (!provider) {
provider
}
}
function onAudioControlClicked() {
audioElement.paused ? audioElement.play() : audioElement.pause()
updateUiAudioPlayState()
}
function addAudioControl() {
$('#send_but_sheld').prepend('<div id="tts_media_control"/>')
$('#send_but_sheld').on('click', onAudioControlClicked)
audioControl = document.getElementById('tts_media_control')
updateUiAudioPlayState()
}
function addUiTtsProviderConfig() {
$('#tts_provider_settings').append(ttsProvider.settingsHtml)
ttsProvider.onSettingsChange()
}
function loadTtsProvider(provider){
// Set up provider references. No init dependencies
// Init provider references
extension_settings.tts.currentProvider = provider
ttsProviderName = provider
ttsProvider = new ttsProviders[provider]
saveSettingsDebounced()
// Init provider settings
$('#tts_provider_settings').append(ttsProvider.settingsHtml)
if (!(ttsProviderName in extension_settings.tts)) {
console.warn(`Provider ${ttsProviderName} not in Extension Settings, initiatilizing provider in settings`)
extension_settings.tts[ttsProviderName] = {}
}
// Load voicemap settings
let voiceMapFromSettings
if ("voiceMap" in extension_settings.tts[ttsProviderName]) {
voiceMapFromSettings = extension_settings.tts[ttsProviderName].voiceMap
voiceMap = parseVoiceMap(voiceMapFromSettings)
} else {
voiceMapFromSettings = ""
voiceMap = {}
}
$('#tts_voice_map').val(voiceMapFromSettings)
$('#tts_provider').val(ttsProviderName)
ttsProvider.loadSettings(extension_settings.tts[ttsProviderName])
}
function onTtsProviderSettingsInput(){
ttsProvider.onSettingsChange()
extension_settings.tts[ttsProviderName].settings = ttsProvider.settings
saveSettingsDebounced()
function onTtsProviderChange() {
const ttsProviderSelection = $('#tts_provider').val()
loadTtsProvider(ttsProviderSelection)
}
function onTtsProviderSettingsInput() {
ttsProvider.onSettingsChange()
// Persist changes to SillyTavern tts extension settings
extension_settings.tts[ttsProviderName] = ttsProvider.setttings
saveSettingsDebounced()
console.info(`Saved settings ${ttsProviderName} ${JSON.stringify(ttsProvider.settings)}`)
}
$(document).ready(function () {
function addExtensionControls() {
const settingsHtml = `
@@ -410,14 +418,10 @@ $(document).ready(function () {
<div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
</div>
<div class="inline-drawer-content">
<label>API Key</label>
<input id="tts_api_key" type="text" class="text_pole" placeholder="<API Key>"/>
<label>Voice Map</label>
<textarea id="tts_voice_map" type="text" class="text_pole textarea_compact" rows="4"
placeholder="Enter comma separated map of charName:ttsName. Example: \nAqua:Bella,\nYou:Josh,"></textarea>
<div class="tts_buttons">
<input id="tts_apply" class="menu_button" type="submit" value="Apply" />
<input id="tts_voices" class="menu_button" type="submit" value="Available voices" />
<div>
<span>Select TTS Provider</span> </br>
<select id="tts_provider">
</select>
</div>
<div>
<label class="checkbox_label" for="tts_enabled">
@@ -425,16 +429,18 @@ $(document).ready(function () {
Enabled
</label>
</div>
<label>Voice Map</label>
<textarea id="tts_voice_map" type="text" class="text_pole textarea_compact" rows="4"
placeholder="Enter comma separated map of charName:ttsName. Example: \nAqua:Bella,\nYou:Josh,"></textarea>
<div id="tts_status">
</div>
<div class="inline-drawer">
<div class="inline-drawer-toggle inline-drawer-header">
<b>TTS Config</b>
<div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
<form id="tts_provider_settings" class="inline-drawer-content">
</form>
<div class="tts_buttons">
<input id="tts_apply" class="menu_button" type="submit" value="Apply" />
<input id="tts_voices" class="menu_button" type="submit" value="Available voices" />
</div>
<form id="tts_provider_settings" class="inline-drawer-content">
</form>
</div>
</div>
</div>
@@ -445,11 +451,14 @@ $(document).ready(function () {
$('#tts_enabled').on('click', onEnableClick)
$('#tts_voices').on('click', onTtsVoicesClick)
$('#tts_provider_settings').on('input', onTtsProviderSettingsInput)
for (const provider in ttsProviders) {
$('#tts_provider').append($("<option />").val(provider).text(provider))
}
$('#tts_provider').on('change', onTtsProviderChange)
}
loadTtsProvider("elevenLabs") // No init dependencies
addExtensionControls() // No init dependencies
addUiTtsProviderConfig() // Depends on ttsProvider being loaded
loadSettings() // Depends on Extension Controls and ttsProvider
loadSettings() // Depends on Extension Controls and loadTtsProvider
loadTtsProvider(extension_settings.tts.currentProvider) // No dependencies
addAudioControl() // Depends on Extension Controls
setInterval(moduleWorker, UPDATE_INTERVAL) // Init depends on all the things
})

View File

@@ -0,0 +1,127 @@
export { SileroTtsProvider }
class SileroTtsProvider {
//########//
// Config //
//########//
settings
voices = []
defaultSettings = {
provider_endpoint: "http://localhost:8001/tts",
voiceMap: {}
}
get settingsHtml() {
let html = `
<label for="silero_tts_endpoint">Provider Endpoint:</label>
<input id="silero_tts_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
<span> A simple Python Silero TTS Server can be found <a href="https://github.com/ouoertheo/silero-api-server">here</a>.</span>
`
return html
}
onSettingsChange() {
// Used when provider settings are updated from UI
this.settings.provider_endpoint = $('#silero_tts_endpoint').val()
}
loadSettings(settings) {
// Pupulate Provider UI given input settings
if (Object.keys(settings).length == 0) {
console.info("Using default TTS Provider settings")
}
// Only accept keys defined in defaultSettings
this.settings = this.defaultSettings
for (const key in settings){
if (key in this.settings){
this.settings[key] = settings[key]
} else {
throw `Invalid setting passed to TTS Provider: ${key}`
}
}
$('#silero_tts_endpoint').text(this.settings.provider_endpoint)
console.info("Settings loaded")
}
async onApplyClick() {
return
}
//#################//
// TTS Interfaces //
//#################//
async getVoice(voiceName) {
if (this.voices.length == 0) {
this.voices = await this.fetchTtsVoiceIds()
}
const match = this.voices.filter(
sileroVoice => sileroVoice.name == voiceName
)[0]
if (!match) {
throw `TTS Voice name ${voiceName} not found`
}
return match
}
async generateTts(text, voiceId){
const response = await this.fetchTtsGeneration(text, voiceId)
return response
}
//###########//
// API CALLS //
//###########//
async fetchTtsVoiceIds() {
const response = await fetch(`${this.settings.provider_endpoint}/speakers`)
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${await response.json()}`)
}
const responseJson = await response.json()
return responseJson
}
async fetchTtsGeneration(inputText, voiceId) {
console.info(`Generating new TTS for voice_id ${voiceId}`)
const response = await fetch(
`${this.settings.provider_endpoint}/generate`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
"text": inputText,
"speaker": voiceId
})
}
)
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${await response.json()}`)
}
return response
}
async fetchTtsFromHistory(history_item_id) {
console.info(`Fetched existing TTS with history_item_id ${history_item_id}`)
const response = await fetch(
`https://api.elevenlabs.io/v1/history/${history_item_id}/audio`,
{
headers: {
'xi-api-key': this.API_KEY
}
}
)
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${await response.json()}`)
}
return response
}
}