Audio module proper extension code. Dedicated Audio extension scripts. Simple UI to adjust bgm/ambient. Load bgm per character from character folder and switch audio dynamically w.r.t. expression. Ambient sound updated when background change.

2023-08-20 02:31:17 +02:00 · 2023-08-20 02:31:17 +02:00 · 25daa61e83
parent 8590de7229
commit 25daa61e83
12 changed files with 417 additions and 27 deletions
--- a/public/backgrounds/audio/custom/.placeholder
+++ b/public/backgrounds/audio/custom/.placeholder
@ -0,0 +1 @@
+Place your custom background ambient sound file here
--- a/public/backgrounds/audio/default/bedroom
+++ b/public/backgrounds/audio/default/bedroom
--- a/public/backgrounds/audio/default/cityscape
+++ b/public/backgrounds/audio/default/cityscape
--- a/public/backgrounds/audio/default/cityscape
+++ b/public/backgrounds/audio/default/cityscape
--- a/public/backgrounds/audio/default/great-tree.mp3
+++ b/public/backgrounds/audio/default/great-tree.mp3
--- a/public/backgrounds/audio/default/landscape
+++ b/public/backgrounds/audio/default/landscape
--- a/public/backgrounds/audio/default/landscape
+++ b/public/backgrounds/audio/default/landscape
--- a/public/scripts/extensions/audio/index.js
+++ b/public/scripts/extensions/audio/index.js
@ -0,0 +1,352 @@
+/*
+TODO:
+ - Emotion-based BGM
+    - per character bgms OK
+    - simple fade out/in when switching OK
+    - cross fading ?
+ - Background based ambient sounds
+    - global sounds OK
+    - global overides ?
+    - import option on background UI ?
+ - One UI with different mixing options OK
+ - Allow background music edition using background menu
+    - https://fontawesome.com/icons/music?f=classic&s=solid
+    - https://codepen.io/noirsociety/pen/rNQxQwm
+    - https://codepen.io/xrocker/pen/abdKVGy
+*/
+
+import { saveSettingsDebounced } from "../../../script.js";
+import { getContext, getApiUrl, extension_settings, doExtrasFetch, ModuleWorkerWrapper, modules } from "../../extensions.js";
+export { MODULE_NAME };
+
+const MODULE_NAME = 'Audio';
+const DEBUG_PREFIX = "<Audio module> ";
+const UPDATE_INTERVAL = 1000;
+
+const MUSIC_FALLBACK = "default"
+const EXPRESSIONS_LIST = ["default","anger","fear","joy","love","sadness","surprise"];
+const SPRITE_DOM_ID = "#expression-image"
+const AMBIENT_FOLDER = "backgrounds/audio/default/"
+
+let characterMusics = {}; // Updated with module worker
+let ambientMusics = []; // Initialized only once
+
+let currentCharacter = null;
+let currentExpression = "default";
+let currentBackground = "default"
+
+//#############################//
+//  Extension UI and Settings  //
+//#############################//
+
+const defaultSettings = {
+    enabled: false,
+    bgm_muted: false,
+    ambient_muted: false,
+    bgm_volume: 50,
+    ambient_volume: 50,
+}
+
+function loadSettings() {
+    if (Object.keys(extension_settings.audio).length === 0) {
+        Object.assign(extension_settings.audio, defaultSettings)
+    }
+    $("#audio_enabled").prop('checked', extension_settings.audio.enabled);
+
+    $("#audio_character_bgm_volume").text(extension_settings.audio.bgm_volume);
+    $("#audio_ambient_volume").text(extension_settings.audio.ambient_volume);
+
+    $("#audio_character_bgm_volume_slider").val(extension_settings.audio.bgm_volume);
+    $("#audio_ambient_volume_slider").val(extension_settings.audio.ambient_volume);
+    
+    if (extension_settings.audio.bgm_muted) {
+        $("#audio_character_bgm_mute_icon").toggleClass("fa-volume-high");
+        $("#audio_character_bgm_mute_icon").toggleClass("fa-volume-mute");
+        $("#audio_character_bgm").prop("muted", true);
+    }
+
+    if (extension_settings.audio.ambient_muted) {
+        $("#audio_ambient_mute_icon").toggleClass("fa-volume-high");
+        $("#audio_ambient_mute_icon").toggleClass("fa-volume-mute");
+        $("#audio_ambient").prop("muted", true);
+    }
+}
+
+async function onEnabledClick() {
+    extension_settings.audio.enabled = $('#audio_enabled').is(':checked');
+    if (extension_settings.audio.enabled) {
+        $("#audio_character_bgm").play();
+        $("#audio_ambient").play();
+    } else {
+        $("#audio_character_bgm").pause();
+        $("#audio_ambient").pause();
+    }
+    saveSettingsDebounced();
+}
+
+async function onBGMMuteClick() {
+    extension_settings.audio.bgm_muted = !extension_settings.audio.bgm_muted;
+    $("#audio_character_bgm_mute_icon").toggleClass("fa-volume-high");
+    $("#audio_character_bgm_mute_icon").toggleClass("fa-volume-mute");
+    $("#audio_character_bgm").prop("muted", !$("#audio_character_bgm").prop("muted"));
+    saveSettingsDebounced();
+}
+
+async function onAmbientMuteClick() {
+    extension_settings.audio.ambient_muted = !extension_settings.audio.ambient_muted;
+    $("#audio_ambient_mute_icon").toggleClass("fa-volume-high");
+    $("#audio_ambient_mute_icon").toggleClass("fa-volume-mute");
+    $("#audio_ambient").prop("muted", !$("#audio_ambient").prop("muted"));
+    saveSettingsDebounced();
+}
+
+async function onBGMVolumeChange() {
+    extension_settings.audio.bgm_volume = ~~($("#audio_character_bgm_volume_slider").val());
+    $("#audio_character_bgm").prop("volume",extension_settings.audio.bgm_volume * 0.01);
+    $("#audio_character_bgm_volume").text(extension_settings.audio.bgm_volume);
+    saveSettingsDebounced();
+    console.debug(DEBUG_PREFIX,"UPDATED BGM MAX TO",extension_settings.audio.bgm_volume)
+}
+
+async function onAmbientVolumeChange() {
+    extension_settings.audio.ambient_volume = ~~($("#audio_ambient_volume_slider").val());
+    $("#audio_ambient").prop("volume",extension_settings.audio.ambient_volume * 0.01);
+    $("#audio_ambient_volume").text(extension_settings.audio.ambient_volume);
+    saveSettingsDebounced();
+    console.debug(DEBUG_PREFIX,"UPDATED Ambient MAX TO",extension_settings.audio.ambient_volume)
+}
+
+
+$(document).ready(function () {
+    function addExtensionControls() {
+        const settingsHtml = `
+        <div id="audio_settings">
+            <div class="inline-drawer">
+                <div class="inline-drawer-toggle inline-drawer-header">
+                    <b>Audio</b>
+                    <div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
+                </div>
+                <div class="inline-drawer-content">
+                    <div>
+                        <label class="checkbox_label" for="audio_enabled">
+                            <input type="checkbox" id="audio_enabled" name="audio_enabled">
+                            <small>Enabled</small>
+                        </label>
+                        <label class="checkbox_label" for="audio_debug">
+                            <input type="checkbox" id="audio_debug" name="audio_debug">
+                            <small>Debug</small>
+                        </label>
+                    </div>
+                    <div>
+                        <div>
+                            <label for="audio_character_bgm_volume_slider">Music <span id="audio_character_bgm_volume"></span></label>
+                            <div class="mixer-div">
+                                <div id="audio_character_bgm_mute" class="menu_button audio-mute-button">
+                                    <i class="fa-solid fa-volume-high fa-lg" id="audio_character_bgm_mute_icon"></i>
+                                </div>
+                                <input type="range" class ="slider" id ="audio_character_bgm_volume_slider" value = "0" maxlength ="100">
+                            </div>
+                            <audio id="audio_character_bgm" controls src="">
+                        </div>
+                        <div>
+                            <label for="audio_ambient_volume_slider">Ambient <span id="audio_ambient_volume"></span></label>
+                            <div class="mixer-div">
+                                <div id="audio_character_ambient_mute" class="menu_button audio-mute-button">
+                                    <i class="fa-solid fa-volume-high fa-lg" id="audio_ambient_mute_icon"></i>
+                                </div>
+                                <input type="range" class ="slider" id ="audio_ambient_volume_slider" value = "0" maxlength ="100">
+                            </div>
+                            <audio id="audio_ambient" controls src="">
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        `;
+        $('#extensions_settings').append(settingsHtml);
+    }
+    
+    addExtensionControls(); // No init dependencies
+    loadSettings(); // Depends on Extension Controls
+    
+    $("#audio_character_bgm").attr("loop",true);
+    $("#audio_ambient").attr("loop",true);
+
+    $("#audio_character_bgm").hide();
+    $("#audio_ambient").hide();
+    $("#audio_character_bgm_mute").on("click",onBGMMuteClick);
+    $("#audio_character_ambient_mute").on("click",onAmbientMuteClick);
+
+    $("#audio_enabled").on("click", onEnabledClick);
+    $("#audio_character_bgm_volume_slider").on("input", onBGMVolumeChange);
+    $("#audio_ambient_volume_slider").on("input", onAmbientVolumeChange);
+
+    // DBG
+    $("#audio_debug").on("click",function() {
+        if($("#audio_debug").is(':checked')) {
+            $("#audio_character_bgm").show();
+            $("#audio_ambient").show();
+        }
+        else {
+            $("#audio_character_bgm").hide();
+            $("#audio_ambient").hide();
+        }
+    });
+    //
+
+    const wrapper = new ModuleWorkerWrapper(moduleWorker);
+    setInterval(wrapper.update.bind(wrapper), UPDATE_INTERVAL);
+    moduleWorker();
+})
+
+//#############################//
+//  API Calls                  //
+//#############################//
+
+async function getMusicsList(name) {
+    console.debug(DEBUG_PREFIX, "getting bgm list for", name);
+
+    try {
+        const result = await fetch(`/get_character_background_musics?name=${encodeURIComponent(name)}`);
+        let musics = result.ok ? (await result.json()) : [];
+        return musics;
+    }
+    catch (err) {
+        console.log(err);
+        return [];
+    }
+}
+
+//#############################//
+//  Module Worker              //
+//#############################//
+
+/*
+    - Update character BGM
+    - Update ambient sound
+*/
+async function moduleWorker() {
+    const moduleEnabled = extension_settings.audio.enabled;
+
+    //console.debug(DEBUG_PREFIX, getContext());
+
+    if (moduleEnabled) {
+
+        // Update ambient audio
+        let newBackground = $("#bg1").css("background-image");
+        newBackground = newBackground.substring(newBackground.lastIndexOf("/")+1).replace(/\.[^/.]+$/, "");
+
+        console.debug(DEBUG_PREFIX,"Current backgroung:",newBackground);
+
+        if (currentBackground !== newBackground & newBackground != "none") {
+            currentBackground = newBackground;
+
+            console.debug(DEBUG_PREFIX,"Changing ambient audio");
+            updateAmbient();
+        }
+
+        const newCharacter = getContext().name2;
+
+        // Case 1: character changed (new chat or group chat)
+        if (currentCharacter !== newCharacter) {
+            currentCharacter = newCharacter;
+
+            console.debug(DEBUG_PREFIX,"Updated current character to",currentCharacter);
+            
+            // 1.1) First time character appear, load its music folder
+            if (characterMusics[currentCharacter] === undefined) {
+                const audio_file_paths = await getMusicsList(currentCharacter);
+                console.debug(DEBUG_PREFIX, "Recieved", audio_file_paths);
+                
+                // Initialise expression/files mapping
+                characterMusics[currentCharacter] = {};
+                for(const e of EXPRESSIONS_LIST)
+                    characterMusics[currentCharacter][e] = [];
+
+                for(const i of audio_file_paths) {
+                    //console.debug(DEBUG_PREFIX,"File found:",i);
+                    for(const e of EXPRESSIONS_LIST)
+                        if (i["label"].includes(e))
+                            characterMusics[currentCharacter][e].push(i["path"]);
+                }
+                console.debug(DEBUG_PREFIX,"Updated BGM map of",currentCharacter,"to",characterMusics[currentCharacter]);
+            }
+            
+            // Character changed require BGM/ambient update whatever the expression
+            currentExpression = "default";
+            updateBGM();
+            return;
+        }
+
+        let newExpression = MUSIC_FALLBACK; 
+
+        if (!$(SPRITE_DOM_ID).length) {
+            console.error(DEBUG_PREFIX,"ERROR: expression sprite does not exist, cannot extract expression from ",SPRITE_DOM_ID)
+        }
+
+        // HACK: use sprite file name as expression detection
+        const spriteFile = $("#expression-image").attr("src");
+        newExpression = spriteFile.substring(spriteFile.lastIndexOf("/")+1).replace(/\.[^/.]+$/, "");
+        console.debug(DEBUG_PREFIX,"Current expression",newExpression);
+
+        if (!EXPRESSIONS_LIST.includes(newExpression)) {
+            console.debug(DEBUG_PREFIX,"Not a valid expression, ignored");
+            return;
+        }
+
+        // Case 2: Same character but different expression
+        if (currentExpression !== newExpression) {
+            currentExpression = newExpression;
+            console.debug(DEBUG_PREFIX,"Updated current character expression to",currentExpression);
+            updateBGM();
+            return;
+        }
+
+        // Case 3: Same character/expression keep playing same BGM
+    }
+}
+
+async function updateBGM() {
+    const audio_files = characterMusics[currentCharacter][currentExpression];
+    const audio_file_path = audio_files[Math.floor(Math.random() * audio_files.length)]; // random pick
+    console.log("<MUSIC module> Checking audio file",audio_file_path)
+    fetch(audio_file_path)
+    .then(response => {
+        if (!response.ok) {
+            console.log("<MUSIC module> File not found!")
+        }
+        else {
+            console.log("<MUSIC module> Playing emotion",currentExpression)
+            const audio = $("#audio_character_bgm");
+
+            audio.animate({volume: 0.0}, 2000, function() {
+                audio.attr("src",audio_file_path);
+                audio[0].play();
+                audio.volume = extension_settings.audio.bgm_volume * 0.01;
+                audio.animate({volume: extension_settings.audio.bgm_volume * 0.01}, 2000);
+            })
+        }
+    });
+}
+
+async function updateAmbient() {
+    const audio_file_path = AMBIENT_FOLDER+currentBackground+".mp3";
+    console.log("<MUSIC module> Changing ambient audio for",audio_file_path)
+    fetch(audio_file_path)
+    .then(response => {
+        if (!response.ok) {
+            console.log("<MUSIC module> File not found!")
+        }
+        else {
+            console.log("<MUSIC module> Changing ambient audio for",currentBackground)
+            const audio = $("#audio_ambient");
+
+            audio.animate({volume: 0.0}, 2000, function() {
+                audio.attr("src",audio_file_path);
+                audio[0].play();
+                audio.volume = extension_settings.audio.ambient_volume * 0.01;
+                audio.animate({volume: extension_settings.audio.ambient_volume * 0.01}, 2000);
+            })
+        }
+    });
+}
--- a/public/scripts/extensions/audio/manifest.json
+++ b/public/scripts/extensions/audio/manifest.json
@ -0,0 +1,11 @@
+{
+    "display_name": "Sounds",
+    "loading_order": 14,
+    "requires": [],
+    "optional": ["classify"],
+    "js": "index.js",
+    "css": "style.css",
+    "author": "Keij#6799 and Deffcolony",
+    "version": "0.1.0",
+    "homePage": "https://github.com/SillyTavern/SillyTavern"
+}
--- a/public/scripts/extensions/audio/style.css
+++ b/public/scripts/extensions/audio/style.css
@ -0,0 +1,13 @@
+.mixer-div {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: center;
+    padding: 5px;
+}
+
+.audio-mute-button {
+    padding: 5px;
+    width: 50px;
+    height: 30px;
+}
--- a/public/scripts/extensions/expressions/index.js
+++ b/public/scripts/extensions/expressions/index.js
@ -991,32 +991,8 @@ async function setExpression(character, expression, force) {
            }
        });

-
-
+        
    }
-
-    // HACK: music update
-    console.error("<MUSIC module> CODE INJECTION START")
-    const music_file_path = "/characters/"+character+"/music/"+expression+"_0.mp3"
-    console.log("<MUSIC module> Checking audio file",music_file_path)
-    fetch(music_file_path)
-    .then(response => {
-        if (!response.ok) {
-            console.log("<MUSIC module> File not found!")
-        }
-        else {
-            console.log("<MUSIC module> Playing emotion",expression)
-            const audio = $("#music_background");
-
-            audio.animate({volume: 0.0}, 2000, function() {
-                audio.attr("src",music_file_path)
-                audio[0].play();
-                audio.volume = 1.0;
-                audio.animate({volume: 1.0}, 2000);
-            })
-        }
-    });
-    console.error("<MUSIC module> CODE INJECTION END")
 }

 function onClickExpressionImage() {
@ -1308,8 +1284,6 @@ function setExpressionOverrideHtml(forceClear = false) {
                <input type="file" id="expression_upload_pack" name="expression_upload_pack" accept="application/zip" hidden>
                <input type="file" id="expression_upload" name="expression_upload" accept="image/*" hidden>
            </form>
-            <!-- HACK: Musique module -->
-            <audio id="music_background" controls src="">
        </div>
        `;

--- a/server.js
+++ b/server.js
@ -318,6 +318,7 @@ const directories = {
    context: 'public/context',
    backups: 'backups/',
    quickreplies: 'public/QuickReplies'
+    // TODO: add ambient music here
 };

 // CSRF Protection //
@ -4932,3 +4933,41 @@ app.post('/delete_extension', jsonParser, async (request, response) => {
        return response.status(500).send(`Server Error: ${error.message}`);
    }
 });
+/**
+ * HTTP POST handler function to retrieve a character background music list.
+ *
+ * @param {Object} request - HTTP Request object, expects a character name in the query.
+ * @param {Object} response - HTTP Response object will contain a list of audio file path.
+ *
+ * @returns {void}
+ */
+app.get('/get_character_background_musics', jsonParser, function (request, response) {
+    const AUDIO_FOLDER = "audio"
+    const BGM_FOLDER = "bgm"
+    const name = request.query.name;
+    const musicsPath = path.join(directories.characters, name, AUDIO_FOLDER,BGM_FOLDER);
+    let musics = [];
+
+    try {
+        if (fs.existsSync(musicsPath) && fs.statSync(musicsPath).isDirectory()) {
+            musics = fs.readdirSync(musicsPath)
+                .filter(file => {
+                    const mimeType = mime.lookup(file);
+                    return mimeType && mimeType.startsWith('audio/');
+                })
+                .map((file) => {
+                    const pathToMusic = path.join(musicsPath, file);
+                    return {
+                        label: path.parse(pathToMusic).name.toLowerCase(),
+                        path: `/characters/${name}/${AUDIO_FOLDER}/${BGM_FOLDER}/${file}`,
+                    };
+                });
+        }
+    }
+    catch (err) {
+        console.log(err);
+    }
+    finally {
+        return response.send(musics);
+    }
+});
				`@ -0,0 +1 @@`
				`Place your custom background ambient sound file here`