/* TODO: - load RVC models list from extras - Settings per characters */ import { saveSettingsDebounced } from "../../../script.js"; import { getContext, getApiUrl, extension_settings, doExtrasFetch, ModuleWorkerWrapper, modules } from "../../extensions.js"; export { MODULE_NAME, rvcVoiceConversion }; const MODULE_NAME = 'RVC'; const DEBUG_PREFIX = " " const UPDATE_INTERVAL = 1000 let charactersList = [] // Updated with module worker let rvcModelsList = [] // Initialized only once let rvcModelsReceived = false; function updateVoiceMapText() { let voiceMapText = "" for (let i in extension_settings.rvc.voiceMap) { const voice_settings = extension_settings.rvc.voiceMap[i]; voiceMapText += i + ":" + voice_settings["modelName"] + "(" + voice_settings["pitchExtraction"] + "," + voice_settings["pitchOffset"] + "," + voice_settings["indexRate"] + "," + voice_settings["filterRadius"] + "," + voice_settings["rmsMixRate"] + "," + voice_settings["protect"] + "),\n" } extension_settings.rvc.voiceMapText = voiceMapText; $('#rvc_voice_map').val(voiceMapText); console.debug(DEBUG_PREFIX, "Updated voice map debug text to\n", voiceMapText) } //#############################// // Extension UI and Settings // //#############################// const defaultSettings = { enabled: false, model: "", pitchOffset: 0, pitchExtraction: "dio", indexRate: 0.88, filterRadius: 3, rmsMixRate: 1, protect: 0.33, voicMapText: "", voiceMap: {} } function loadSettings() { if (extension_settings.rvc === undefined) extension_settings.rvc = {}; if (Object.keys(extension_settings.rvc).length === 0) { Object.assign(extension_settings.rvc, defaultSettings) } $('#rvc_enabled').prop('checked', extension_settings.rvc.enabled); $('#rvc_model').val(extension_settings.rvc.model); $('#rvc_pitch_extraction').val(extension_settings.rvc.pitchExtraction); $('#rvc_pitch_extractiont_value').text(extension_settings.rvc.pitchExtraction); $('#rvc_index_rate').val(extension_settings.rvc.indexRate); $('#rvc_index_rate_value').text(extension_settings.rvc.indexRate); $('#rvc_filter_radius').val(extension_settings.rvc.filterRadius); $("#rvc_filter_radius_value").text(extension_settings.rvc.filterRadius); $('#rvc_pitch_offset').val(extension_settings.rvc.pitchOffset); $('#rvc_pitch_offset_value').text(extension_settings.rvc.pitchOffset); $('#rvc_rms_mix_rate').val(extension_settings.rvc.rmsMixRate); $("#rvc_rms_mix_rate_value").text(extension_settings.rvc.rmsMixRate); $('#rvc_protect').val(extension_settings.rvc.protect); $("#rvc_protect_value").text(extension_settings.rvc.protect); $('#rvc_voice_map').val(extension_settings.rvc.voiceMapText); } async function onEnabledClick() { extension_settings.rvc.enabled = $('#rvc_enabled').is(':checked'); saveSettingsDebounced() } async function onPitchExtractionChange() { extension_settings.rvc.pitchExtraction = $('#rvc_pitch_extraction').val(); saveSettingsDebounced() } async function onIndexRateChange() { extension_settings.rvc.indexRate = Number($('#rvc_index_rate').val()); $("#rvc_index_rate_value").text(extension_settings.rvc.indexRate) saveSettingsDebounced() } async function onFilterRadiusChange() { extension_settings.rvc.filterRadius = Number($('#rvc_filter_radius').val()); $("#rvc_filter_radius_value").text(extension_settings.rvc.filterRadius) saveSettingsDebounced() } async function onPitchOffsetChange() { extension_settings.rvc.pitchOffset = Number($('#rvc_pitch_offset').val()); $("#rvc_pitch_offset_value").text(extension_settings.rvc.pitchOffset) saveSettingsDebounced() } async function onRmsMixRateChange() { extension_settings.rvc.rmsMixRate = Number($('#rvc_rms_mix_rate').val()); $("#rvc_rms_mix_rate_value").text(extension_settings.rvc.rmsMixRate) saveSettingsDebounced() } async function onProtectChange() { extension_settings.rvc.protect = Number($('#rvc_protect').val()); $("#rvc_protect_value").text(extension_settings.rvc.protect) saveSettingsDebounced() } async function onApplyClick() { let error = false; const character = $("#rvc_character_select").val(); const model_name = $("#rvc_model_select").val(); const pitchExtraction = $("#rvc_pitch_extraction").val(); const indexRate = $("#rvc_index_rate").val(); const filterRadius = $("#rvc_filter_radius").val(); const pitchOffset = $("#rvc_pitch_offset").val(); const rmsMixRate = $("#rvc_rms_mix_rate").val(); const protect = $("#rvc_protect").val(); if (character === "none") { toastr.error("Character not selected.", DEBUG_PREFIX + " voice mapping apply", { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); return; } if (model_name == "none") { toastr.error("Model not selected.", DEBUG_PREFIX + " voice mapping apply", { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); return; } extension_settings.rvc.voiceMap[character] = { "modelName": model_name, "pitchExtraction": pitchExtraction, "indexRate": indexRate, "filterRadius": filterRadius, "pitchOffset": pitchOffset, "rmsMixRate": rmsMixRate, "protect": protect } updateVoiceMapText(); console.debug(DEBUG_PREFIX, "Updated settings of ", character, ":", extension_settings.rvc.voiceMap[character]) saveSettingsDebounced(); } async function onDeleteClick() { const character = $("#rvc_character_select").val(); if (character === "none") { toastr.error("Character not selected.", DEBUG_PREFIX + " voice mapping delete", { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); return; } delete extension_settings.rvc.voiceMap[character]; console.debug(DEBUG_PREFIX, "Deleted settings of ", character); updateVoiceMapText(); saveSettingsDebounced(); } async function onChangeUploadFiles() { const url = new URL(getApiUrl()); const inputFiles = $("#rvc_model_upload_files").get(0).files; let formData = new FormData(); for (const file of inputFiles) formData.append(file.name, file); console.debug(DEBUG_PREFIX, "Sending files:", formData); url.pathname = '/api/voice-conversion/rvc/upload-models'; const apiResult = await doExtrasFetch(url, { method: 'POST', body: formData }); if (!apiResult.ok) { toastr.error(apiResult.statusText, DEBUG_PREFIX + ' Check extras console for errors log'); throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`); } alert('The files have been uploaded successfully.'); } $(document).ready(function () { function addExtensionControls() { const settingsHtml = `
RVC

Characters Voice Mapping

Upload one archive per model. With .pth and .index (optional) inside.
Supported format: .zip .rar .7zip .7z

Model Settings

Tips: dio and pm faster, harvest slower but good.
Torchcrepe and rmvpe are good but uses GPU.
Controls accent strength, too high may produce artifact.
Higher can reduce breathiness but may increase run time.
Recommended +12 key for male to female conversion and -12 key for female to male conversion.
Closer to 0 is closer to TTS and 1 is closer to trained voice. Can help mask noise and sound more natural when set relatively low.
Avoid non voice sounds. Lower is more being ignored.
`; $('#extensions_settings').append(settingsHtml); $("#rvc_enabled").on("click", onEnabledClick); $("#rvc_voice_map").attr("disabled", "disabled");; $('#rvc_pitch_extraction').on('change', onPitchExtractionChange); $('#rvc_index_rate').on('input', onIndexRateChange); $('#rvc_filter_radius').on('input', onFilterRadiusChange); $('#rvc_pitch_offset').on('input', onPitchOffsetChange); $('#rvc_rms_mix_rate').on('input', onRmsMixRateChange); $('#rvc_protect').on('input', onProtectChange); $("#rvc_apply").on("click", onApplyClick); $("#rvc_delete").on("click", onDeleteClick); $("#rvc_model_upload_files").hide(); $("#rvc_model_upload_select_button").on("click", function() {$("#rvc_model_upload_files").click()}); $("#rvc_model_upload_files").on("change", onChangeUploadFiles); //$("#rvc_model_upload_button").on("click", onClickUpload); $("#rvc_model_refresh_button").on("click", refreshVoiceList); } addExtensionControls(); // No init dependencies loadSettings(); // Depends on Extension Controls const wrapper = new ModuleWorkerWrapper(moduleWorker); setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL); moduleWorker(); }) //#############################// // API Calls // //#############################// /* Check model installation state, return one of ["installed", "corrupted", "absent"] */ async function get_models_list(model_id) { const url = new URL(getApiUrl()); url.pathname = '/api/voice-conversion/rvc/get-models-list'; const apiResult = await doExtrasFetch(url, { method: 'POST' }); if (!apiResult.ok) { toastr.error(apiResult.statusText, DEBUG_PREFIX + ' Check model state request failed'); throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`); } return apiResult } /* Send an audio file to RVC to convert voice */ async function rvcVoiceConversion(response, character, text) { let apiResult // Check voice map if (extension_settings.rvc.voiceMap[character] === undefined) { //toastr.error("No model is assigned to character '"+character+"', check RVC voice map in the extension menu.", DEBUG_PREFIX+'RVC Voice map error', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); console.info(DEBUG_PREFIX, "No RVC model assign in voice map for current character " + character); return response; } const audioData = await response.blob() if (!audioData.type in ['audio/mpeg', 'audio/wav', 'audio/x-wav', 'audio/wave', 'audio/webm']) { throw `TTS received HTTP response with invalid data format. Expecting audio/mpeg, got ${audioData.type}` } console.log("Audio type received:", audioData.type) const voice_settings = extension_settings.rvc.voiceMap[character]; var requestData = new FormData(); requestData.append('AudioFile', audioData, 'record'); requestData.append("json", JSON.stringify({ "modelName": voice_settings["modelName"], "pitchExtraction": voice_settings["pitchExtraction"], "pitchOffset": voice_settings["pitchOffset"], "indexRate": voice_settings["indexRate"], "filterRadius": voice_settings["filterRadius"], "rmsMixRate": voice_settings["rmsMixRate"], "protect": voice_settings["protect"], "text": text })); console.log("Sending tts audio data to RVC on extras server",requestData) const url = new URL(getApiUrl()); url.pathname = '/api/voice-conversion/rvc/process-audio'; apiResult = await doExtrasFetch(url, { method: 'POST', body: requestData, }); if (!apiResult.ok) { toastr.error(apiResult.statusText, DEBUG_PREFIX + ' RVC Voice Conversion Failed', { timeOut: 10000, extendedTimeOut: 20000, preventDuplicates: true }); throw new Error(`HTTP ${apiResult.status}: ${await apiResult.text()}`); } return apiResult; } //#############################// // Module Worker // //#############################// async function refreshVoiceList() { let result = await get_models_list(); result = await result.json(); rvcModelsList = result["models_list"] $('#rvc_model_select') .find('option') .remove() .end() .append('') .val('none') for (const modelName of rvcModelsList) { $("#rvc_model_select").append(new Option(modelName, modelName)); } rvcModelsReceived = true console.debug(DEBUG_PREFIX, "Updated model list to:", rvcModelsList); } async function moduleWorker() { updateCharactersList(); if (modules.includes('rvc') && !rvcModelsReceived) { refreshVoiceList(); } } function updateCharactersList() { let currentcharacters = new Set(); const context = getContext(); for (const i of context.characters) { currentcharacters.add(i.name); } currentcharacters = Array.from(currentcharacters); currentcharacters.unshift(context.name1); if (JSON.stringify(charactersList) !== JSON.stringify(currentcharacters)) { charactersList = currentcharacters $('#rvc_character_select') .find('option') .remove() .end() .append('') .val('none') for (const charName of charactersList) { $("#rvc_character_select").append(new Option(charName, charName)); } console.debug(DEBUG_PREFIX, "Updated character list to:", charactersList); } }