diff --git a/public/scripts/extensions/rvc/index.js b/public/scripts/extensions/rvc/index.js new file mode 100644 index 000000000..c2edda2de --- /dev/null +++ b/public/scripts/extensions/rvc/index.js @@ -0,0 +1,262 @@ +/* +TODO: + - try pseudo streaming audio by just sending chunk every X seconds and asking VOSK if it is full text. +*/ + +import { saveSettingsDebounced } from "../../../script.js"; +import { getContext, getApiUrl, modules, extension_settings, ModuleWorkerWrapper, doExtrasFetch } from "../../extensions.js"; +export { MODULE_NAME, rvcVoiceConversion}; + +const MODULE_NAME = 'RVC'; +const DEBUG_PREFIX = " " + +//let currentModel = null + +/*/ Load character model if needed +async function rvcLoadModel(model) { + const url = new URL(getApiUrl()); + url.pathname = '/api/voice-conversion/rvc/load-model'; + + const apiResult = await doExtrasFetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Bypass-Tunnel-Reminder': 'bypass', + }, + body: JSON.stringify({ "model_name": model }), + }); + + if (!apiResult.ok) { + toastr.error("May be a wrong model name in RVC voice map, please check console for details", 'RVC Voice model load Failed', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); + throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`); + } + + const result = await apiResult.json(); + console.log("Loaded RVC model:", result.model_loaded); + currentModel = model +} +*/ + +// Send an audio file to RVC to convert voice +async function rvcVoiceConversion(response, character) { + let apiResult + + // Check voice map + if (extension_settings.rvc.voiceMap[character] === undefined) { + toastr.error("No model is assigned to character '"+char+"', check RVC voice map in the extension menu.", 'RVC Voice map error', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); + throw new Error("No RVC model assign in voice map for current character "+char); + } + + // Load model if different from currently loaded + //if (currentModel === null | currentModel != extension_settings.rvc.voiceMap[character]) + // await rvcLoadModel(extension_settings.rvc.voiceMap[character]); + + const audioData = await response.blob() + if (!audioData.type in ['audio/mpeg', 'audio/wav', 'audio/x-wav', 'audio/wave', 'audio/webm']) { + throw `TTS received HTTP response with invalid data format. Expecting audio/mpeg, got ${audioData.type}` + } + console.log("Sending tts audio data to RVC on extras server") + + var requestData = new FormData(); + requestData.append('AudioFile', audioData, 'record.wav'); + requestData.append("json", JSON.stringify({ + "modelName": extension_settings.rvc.voiceMap[character], + "pitchOffset": extension_settings.rvc.pitchOffset, + "pitchExtraction": extension_settings.rvc.pitchExtraction, + "indexRate": extension_settings.rvc.indexRate, + "filterRadius": extension_settings.rvc.filterRadius, + //"rmsMixRate": extension_settings.rvc.rmsMixRate, + "protect": extension_settings.rvc.protect + })); + + const url = new URL(getApiUrl()); + url.pathname = '/api/voice-conversion/rvc/process-audio'; + + apiResult = await doExtrasFetch(url, { + method: 'POST', + body: requestData, + }); + + if (!apiResult.ok) { + toastr.error(apiResult.statusText, 'RVC Voice Conversion Failed', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); + throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`); + } + + return apiResult; +} + +//#############################// +// Extension UI and Settings // +//#############################// + +const defaultSettings = { + enabled: false, + model:"", + pitchOffset:0, + pitchExtraction:"dio", + indexRate:0.88, + filterRadius:3, + //rmsMixRate:1, + protect:0.33, + voicMapText: "", + voiceMap: {} +} + +function loadSettings() { + if (Object.keys(extension_settings.rvc).length === 0) { + Object.assign(extension_settings.rvc, defaultSettings) + } + $('#rvc_enabled').prop('checked',extension_settings.rvc.enabled); + $('#rvc_model').val(extension_settings.rvc.model); + + $('#rvc_pitch_offset').val(extension_settings.rvc.pitchOffset); + $('#rvc_pitch_offset_value').text(extension_settings.rvc.pitchOffset); + + $('#rvc_pitch_extraction').val(extension_settings.rvc.pitchExtraction); + $('#rvc_pitch_extractiont_value').text(extension_settings.rvc.pitchExtraction); + + $('#rvc_index_rate').val(extension_settings.rvc.indexRate); + $('#rvc_index_rate_value').text(extension_settings.rvc.indexRate); + + $('#rvc_filter_radius').val(extension_settings.rvc.filterRadius); + $("#rvc_filter_radius_value").text(extension_settings.rvc.filterRadius); + + //$('#rvc_mix_rate').val(extension_settings.rvc.rmsMixRate); + $('#rvc_protect').val(extension_settings.rvc.protect); + $("#rvc_protect_value").text(extension_settings.rvc.protect); + + $('#rvc_voice_map').val(extension_settings.rvc.voiceMapText); +} + +async function onApplyClick() { + let error = false; + let array = $('#rvc_voice_map').val().split(","); + array = array.map(element => {return element.trim();}); + array = array.filter((str) => str !== ''); + extension_settings.rvc.voiceMap = {}; + for (const text of array) { + if (text.includes("=")) { + const pair = text.split("=") + extension_settings.rvc.voiceMap[pair[0].trim()] = pair[1].trim() + console.debug(DEBUG_PREFIX+"Added mapping", pair[0],"=>", extension_settings.rvc.voiceMap[pair[0]]); + } + else { + $("#rvc_status").text("Voice map is invalid, check console for errors"); + $("#rvc_status").css("color", "red"); + console.error(DEBUG_PREFIX+"Wrong syntax for message mapping, no '=' found in:", text); + error = true; + } + } + + if (!error) { + $("#rvc_status").text("Successfully applied settings"); + $("#rvc_status").css("color", "green"); + console.debug(DEBUG_PREFIX+"Updated message mapping", extension_settings.rvc.voiceMap); + extension_settings.rvc.voiceMapText = $('#rvc_voice_map').val(); + saveSettingsDebounced(); + } +} + +async function onEnabledClick() { + extension_settings.rvc.enabled = $('#rvc_enabled').is(':checked'); + saveSettingsDebounced() +} + +async function onPitchExtractionChange() { + extension_settings.rvc.pitchExtraction = $('#rvc_pitch_extraction').val(); + saveSettingsDebounced() +} + +async function onIndexRateChange() { + extension_settings.rvc.indexRate = Number($('#rvc_index_rate').val()); + $("#rvc_index_rate_value").text(extension_settings.rvc.indexRate) + saveSettingsDebounced() +} + +async function onFilterRadiusChange() { + extension_settings.rvc.filterRadius = Number($('#rvc_filter_radius').val()); + $("#rvc_filter_radius_value").text(extension_settings.rvc.filterRadius) + saveSettingsDebounced() +} + +async function onPitchOffsetChange() { + extension_settings.rvc.pitchOffset = Number($('#rvc_pitch_offset').val()); + $("#rvc_pitch_offset_value").text(extension_settings.rvc.pitchOffset) + saveSettingsDebounced() +} + +async function onProtectChange() { + extension_settings.rvc.protect = Number($('#rvc_protect').val()); + $("#rvc_protect_value").text(extension_settings.rvc.protect) + saveSettingsDebounced() +} + +$(document).ready(function () { + function addExtensionControls() { + const settingsHtml = ` +
+
+
+ RVC +
+
+
+
+ +
+
+ Select Pitch Extraction
+ +
+
+ + + + + + + + + + + + + +
+
+
+ +
+
+
+
+
+ `; + $('#extensions_settings').append(settingsHtml); + $("#rvc_enabled").on("click", onEnabledClick); + $('#rvc_pitch_extraction').on('change', onPitchExtractionChange); + $('#rvc_index_rate').on('input', onIndexRateChange); + $('#rvc_filter_radius').on('input', onFilterRadiusChange); + $('#rvc_pitch_offset').on('input', onPitchOffsetChange); + $('#rvc_protect').on('input', onProtectChange); + $("#rvc_apply").on("click", onApplyClick); + + } + addExtensionControls(); // No init dependencies + loadSettings(); // Depends on Extension Controls + + console.log(getContext()); + console.log(getContext().name2); +}) diff --git a/public/scripts/extensions/rvc/manifest.json b/public/scripts/extensions/rvc/manifest.json new file mode 100644 index 000000000..c9b9d8b2f --- /dev/null +++ b/public/scripts/extensions/rvc/manifest.json @@ -0,0 +1,11 @@ +{ + "display_name": "RVC", + "loading_order": 13, + "requires": ["rvc"], + "optional": [], + "js": "index.js", + "css": "style.css", + "author": "Keij#6799", + "version": "0.1.0", + "homePage": "https://github.com/SillyTavern/SillyTavern" +} diff --git a/public/scripts/extensions/rvc/style.css b/public/scripts/extensions/rvc/style.css new file mode 100644 index 000000000..23ef2f78d --- /dev/null +++ b/public/scripts/extensions/rvc/style.css @@ -0,0 +1,3 @@ +.speech-toggle { + display: flex; +} diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index 00007430c..ad5514e2c 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -8,6 +8,7 @@ import { CoquiTtsProvider } from './coquitts.js' import { SystemTtsProvider } from './system.js' import { NovelTtsProvider } from './novel.js' import { power_user } from '../../power-user.js' +import { rvcVoiceConversion } from "../rvc/index.js" const UPDATE_INTERVAL = 1000 @@ -399,8 +400,13 @@ function saveLastValues() { ) } -async function tts(text, voiceId) { - const response = await ttsProvider.generateTts(text, voiceId) +async function tts(text, voiceId, char) { + let response = await ttsProvider.generateTts(text, voiceId) + + // RVC injection + if (extension_settings.rvc.enabled) + response = await rvcVoiceConversion(response, char) + addAudioJob(response) completeTtsJob() } @@ -450,7 +456,7 @@ async function processTtsQueue() { toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`) throw `Unable to attain voiceId for ${char}` } - tts(text, voiceId) + tts(text, voiceId, char) } catch (error) { console.error(error) currentTtsJob = null @@ -567,6 +573,7 @@ function onEnableClick() { saveSettingsDebounced() } + function onAutoGenerationClick() { extension_settings.tts.auto_generation = $('#tts_auto_generation').prop('checked'); saveSettingsDebounced()