Merge branch 'staging' of https://github.com/Cohee1207/SillyTavern into staging

public/index.html

@@ -403,7 +403,7 @@
                                     </div>
                                     <div class="range-block-range-and-counter">
                                         <div class="range-block-range">
-                                            <input type="range" id="rep_pen_slope_novel" name="volume" min="0.01" max="10" step="0.01">
+                                            <input type="range" id="rep_pen_slope_novel" name="volume" min="0" max="10" step="0.01">
                                         </div>
                                         <div class="range-block-counter">
                                             <div contenteditable="true" data-for="rep_pen_slope_novel" id="rep_pen_slope_counter_novel">

public/script.js

@@ -143,7 +143,7 @@ import {
     onlyUnique,
 } from "./scripts/utils.js";
 
-import { extension_settings, getContext, loadExtensionSettings, processExtensionHelpers, registerExtensionHelper, runGenerationInterceptors, saveMetadataDebounced } from "./scripts/extensions.js";
+import { extension_settings, getContext, installExtension, loadExtensionSettings, processExtensionHelpers, registerExtensionHelper, runGenerationInterceptors, saveMetadataDebounced } from "./scripts/extensions.js";
 import { COMMENT_NAME_DEFAULT, executeSlashCommands, getSlashCommandsHelp, registerSlashCommand } from "./scripts/slash-commands.js";
 import {
     tag_map,
@@ -8898,25 +8898,7 @@ jQuery(async function () {
         }
 
         const url = input.trim();
-        console.debug('Extension import started', url);
+        await installExtension(url);
-
-        const request = await fetch('/api/extensions/install', {
-            method: 'POST',
-            headers: getRequestHeaders(),
-            body: JSON.stringify({ url }),
-        });
-
-        if (!request.ok) {
-            toastr.info(request.statusText, 'Extension import failed');
-            console.error('Extension import failed', request.status, request.statusText);
-            return;
-        }
-
-        const response = await request.json();
-        toastr.success(`Extension "${response.display_name}" by ${response.author} (version ${response.version}) has been imported successfully!`, 'Extension import successful');
-        console.debug(`Extension "${response.display_name}" has been imported successfully at ${response.extensionPath}`);
-        await loadExtensionSettings(settings);
-        eventSource.emit(event_types.EXTENSION_SETTINGS_LOADED);
     });
 
 

public/scripts/extensions.js

@@ -11,7 +11,7 @@ export {
     ModuleWorkerWrapper,
 };
 
-let extensionNames = [];
+export let extensionNames = [];
 let manifests = {};
 const defaultUrl = "http://localhost:5100";
 
@@ -639,23 +639,26 @@ async function onDeleteClick() {
     // use callPopup to create a popup for the user to confirm before delete
     const confirmation = await callPopup(`Are you sure you want to delete ${extensionName}?`, 'delete_extension');
     if (confirmation) {
-        try {
+        await deleteExtension(extensionName);
-            const response = await fetch('/api/extensions/delete', {
-                method: 'POST',
-                headers: getRequestHeaders(),
-                body: JSON.stringify({ extensionName })
-            });
-        } catch (error) {
-            console.error('Error:', error);
-        }
-        toastr.success(`Extension ${extensionName} deleted`);
-        showExtensionsDetails();
-        // reload the page to remove the extension from the list
-        location.reload();
     }
 };
 
+export async function deleteExtension(extensionName) {
+    try {
+        const response = await fetch('/api/extensions/delete', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({ extensionName })
+        });
+    } catch (error) {
+        console.error('Error:', error);
+    }
 
+    toastr.success(`Extension ${extensionName} deleted`);
+    showExtensionsDetails();
+    // reload the page to remove the extension from the list
+    location.reload();
+}
 
 /**
  * Fetches the version details of a specific extension.
@@ -680,7 +683,32 @@ async function getExtensionVersion(extensionName) {
     }
 }
 
+/**
+ * Installs a third-party extension via the API.
+ * @param {string} url Extension repository URL
+ * @returns {Promise<void>}
+ */
+export async function installExtension(url) {
+    console.debug('Extension import started', url);
 
+    const request = await fetch('/api/extensions/install', {
+        method: 'POST',
+        headers: getRequestHeaders(),
+        body: JSON.stringify({ url }),
+    });
+
+    if (!request.ok) {
+        toastr.info(request.statusText, 'Extension import failed');
+        console.error('Extension import failed', request.status, request.statusText);
+        return;
+    }
+
+    const response = await request.json();
+    toastr.success(`Extension "${response.display_name}" by ${response.author} (version ${response.version}) has been imported successfully!`, 'Extension import successful');
+    console.debug(`Extension "${response.display_name}" has been imported successfully at ${response.extensionPath}`);
+    await loadExtensionSettings({});
+    eventSource.emit(event_types.EXTENSION_SETTINGS_LOADED);
+}
 
 async function loadExtensionSettings(settings) {
     if (settings.extension_settings) {

public/scripts/extensions/assets/index.js

@@ -5,6 +5,7 @@ TODO:
 //const DEBUG_TONY_SAMA_FORK_MODE = false
 
 import { getRequestHeaders, callPopup } from "../../../script.js";
+import { deleteExtension, extensionNames, installExtension } from "../../extensions.js";
 export { MODULE_NAME };
 
 const MODULE_NAME = 'Assets';
@@ -116,9 +117,13 @@ function downloadAssetsList(url) {
 
                         console.debug(DEBUG_PREFIX, "Created element for BGM", asset["id"])
 
+                        const displayName = DOMPurify.sanitize(asset["name"] || asset["id"]);
+                        const description = DOMPurify.sanitize(asset["description"] || "");
+
                         $(`<i></i>`)
                             .append(element)
-                            .append(`<span>${asset["id"]}</span>`)
+                            .append(`<span>${displayName}</span>`)
+                            .append(`<span>${description}</span>`)
                             .appendTo(assetTypeMenu);
                     }
                     assetTypeMenu.appendTo("#assets_menu");
@@ -136,7 +141,14 @@ function downloadAssetsList(url) {
 }
 
 function isAssetInstalled(assetType, filename) {
-    for (const i of currentAssets[assetType]) {
+    let assetList = currentAssets[assetType];
+
+    if (assetType == 'extension') {
+        const thirdPartyMarker = "third-party/";
+        assetList = extensionNames.filter(x => x.startsWith(thirdPartyMarker)).map(x => x.replace(thirdPartyMarker, ''));
+    }
+
+    for (const i of assetList) {
         //console.debug(DEBUG_PREFIX,i,filename)
         if (i.includes(filename))
             return true;
@@ -149,6 +161,13 @@ async function installAsset(url, assetType, filename) {
     console.debug(DEBUG_PREFIX, "Downloading ", url);
     const category = assetType;
     try {
+        if (category === 'extension') {
+            console.debug(DEBUG_PREFIX, "Installing extension ", url)
+            await installExtension(url);
+            console.debug(DEBUG_PREFIX, "Extension installed.")
+            return;
+        }
+
         const body = { url, category, filename };
         const result = await fetch('/api/assets/download', {
             method: 'POST',
@@ -170,6 +189,12 @@ async function deleteAsset(assetType, filename) {
     console.debug(DEBUG_PREFIX, "Deleting ", assetType, filename);
     const category = assetType;
     try {
+        if (category === 'extension') {
+            console.debug(DEBUG_PREFIX, "Deleting extension ", filename)
+            await deleteExtension(filename);
+            console.debug(DEBUG_PREFIX, "Extension deleted.")
+        }
+
         const body = { category, filename };
         const result = await fetch('/api/assets/delete', {
             method: 'POST',

public/scripts/extensions/assets/style.css

@@ -19,6 +19,7 @@
     align-items: center;
     justify-content: left;
     padding: 5px;
+    font-style: normal;
 }
 
 .assets-list-div i span{
@@ -34,7 +35,7 @@
     border-radius: 2px;
     cursor: pointer;
   }
-  
+
 .asset-download-button:active {
     background: #007a63;
 }
@@ -75,5 +76,3 @@ to {
     transform: rotate(1turn);
 }
 }
-  
-  

public/scripts/extensions/stable-diffusion/index.js

@@ -156,6 +156,7 @@ const defaultSettings = {
     horde: false,
     horde_nsfw: false,
     horde_karras: true,
+    horde_sanitize: true,
 
     // Refine mode
     refine_mode: false,
@@ -252,6 +253,7 @@ async function loadSettings() {
     $('#sd_horde').prop('checked', extension_settings.sd.horde);
     $('#sd_horde_nsfw').prop('checked', extension_settings.sd.horde_nsfw);
     $('#sd_horde_karras').prop('checked', extension_settings.sd.horde_karras);
+    $('#sd_horde_sanitize').prop('checked', extension_settings.sd.horde_sanitize);
     $('#sd_restore_faces').prop('checked', extension_settings.sd.restore_faces);
     $('#sd_enable_hr').prop('checked', extension_settings.sd.enable_hr);
     $('#sd_refine_mode').prop('checked', extension_settings.sd.refine_mode);
@@ -439,16 +441,21 @@ function onNovelAnlasGuardInput() {
     saveSettingsDebounced();
 }
 
-async function onHordeNsfwInput() {
+function onHordeNsfwInput() {
     extension_settings.sd.horde_nsfw = !!$(this).prop('checked');
     saveSettingsDebounced();
 }
 
-async function onHordeKarrasInput() {
+function onHordeKarrasInput() {
     extension_settings.sd.horde_karras = !!$(this).prop('checked');
     saveSettingsDebounced();
 }
 
+function onHordeSanitizeInput() {
+    extension_settings.sd.horde_sanitize = !!$(this).prop('checked');
+    saveSettingsDebounced();
+}
+
 function onRestoreFacesInput() {
     extension_settings.sd.restore_faces = !!$(this).prop('checked');
     saveSettingsDebounced();
@@ -1243,6 +1250,7 @@ async function generateHordeImage(prompt) {
             nsfw: extension_settings.sd.horde_nsfw,
             restore_faces: !!extension_settings.sd.restore_faces,
             enable_hr: !!extension_settings.sd.enable_hr,
+            sanitize: !!extension_settings.sd.horde_sanitize,
         }),
     });
 
@@ -1584,6 +1592,7 @@ jQuery(async () => {
     $('#sd_height').on('input', onHeightInput);
     $('#sd_horde_nsfw').on('input', onHordeNsfwInput);
     $('#sd_horde_karras').on('input', onHordeKarrasInput);
+    $('#sd_horde_sanitize').on('input', onHordeSanitizeInput);
     $('#sd_restore_faces').on('input', onRestoreFacesInput);
     $('#sd_enable_hr').on('input', onHighResFixInput);
     $('#sd_refine_mode').on('input', onRefineModeInput);

public/scripts/extensions/stable-diffusion/settings.html

@@ -58,6 +58,12 @@
                         Allow NSFW images from Horde
                     </span>
                 </label>
+                <label for="sd_horde_sanitize" class="checkbox_label">
+                    <input id="sd_horde_sanitize" type="checkbox" />
+                    <span data-i18n="Sanitize prompts (recommended)">
+                        Sanitize prompts (recommended)
+                    </span>
+                </label>
                 <label for="sd_horde_karras" class="checkbox_label">
                     <input id="sd_horde_karras" type="checkbox" />
                     <span data-i18n="Karras (not all samplers supported)">

public/scripts/textgen-settings.js

@@ -227,6 +227,10 @@ export function isAphrodite() {
     return textgenerationwebui_settings.type === textgen_types.APHRODITE;
 }
 
+export function isOoba() {
+    return textgenerationwebui_settings.type === textgen_types.OOBA;
+}
+
 export function getTextGenUrlSourceId() {
     switch (textgenerationwebui_settings.type) {
         case textgen_types.MANCER:
@@ -327,6 +331,18 @@ async function generateTextGenWithStreaming(generate_data, signal) {
         streamingUrl = api_server_textgenerationwebui;
     }
 
+    if (isMancer() || isOoba()) {
+        try {
+            const parsedUrl = new URL(streamingUrl);
+            if (parsedUrl.protocol !== 'ws:' && parsedUrl.protocol !== 'wss:') {
+                throw new Error('Invalid protocol');
+            }
+        } catch {
+            toastr.error('Invalid URL for streaming. Make sure it starts with ws:// or wss://');
+            return async function* () { throw new Error('Invalid URL for streaming.'); }
+        }
+    }
+
     const response = await fetch('/generate_textgenerationwebui', {
         headers: {
             ...getRequestHeaders(),

server.js

@@ -552,8 +552,18 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
         });
 
         async function* readWebsocket() {
-            const streamingUrl = new URL(streamingUrlString);
+            /** @type {WebSocket} */
-            const websocket = new WebSocket(streamingUrl);
+            let websocket;
+            /** @type {URL} */
+            let streamingUrl;
+
+            try {
+                const streamingUrl = new URL(streamingUrlString);
+                websocket = new WebSocket(streamingUrl);
+            } catch (error) {
+                console.log("[SillyTavern] Socket error", error);
+                return;
+            }
 
             websocket.on('open', async function () {
                 console.log('WebSocket opened');

src/horde.js

@@ -17,6 +17,41 @@ async function getHordeClient() {
     return ai_horde;
 }
 
+/**
+ * Removes dirty no-no words from the prompt.
+ * Taken verbatim from KAI Lite's implementation (AGPLv3).
+ * https://github.com/LostRuins/lite.koboldai.net/blob/main/index.html#L7786C2-L7811C1
+ * @param {string} prompt Prompt to sanitize
+ * @returns {string} Sanitized prompt
+ */
+function sanitizeHordeImagePrompt(prompt) {
+    if (!prompt) {
+        return "";
+    }
+
+    //to avoid flagging from some image models, always swap these words
+    prompt = prompt.replace(/\b(girl)\b/gmi, "woman");
+    prompt = prompt.replace(/\b(boy)\b/gmi, "man");
+    prompt = prompt.replace(/\b(girls)\b/gmi, "women");
+    prompt = prompt.replace(/\b(boys)\b/gmi, "men");
+
+    //always remove these high risk words from prompt, as they add little value to image gen while increasing the risk the prompt gets flagged
+    prompt = prompt.replace(/\b(under.age|under.aged|underage|underaged|loli|pedo|pedophile|(\w+).year.old|(\w+).years.old|minor|prepubescent|minors|shota)\b/gmi, "");
+
+    //if nsfw is detected, do not remove it but apply additional precautions
+    let isNsfw = prompt.match(/\b(cock|ahegao|hentai|uncensored|lewd|cocks|deepthroat|deepthroating|dick|dicks|cumshot|lesbian|fuck|fucked|fucking|sperm|naked|nipples|tits|boobs|breasts|boob|breast|topless|ass|butt|fingering|masturbate|masturbating|bitch|blowjob|pussy|piss|asshole|dildo|dildos|vibrator|erection|foreskin|handjob|nude|penis|porn|vibrator|virgin|vagina|vulva|threesome|orgy|bdsm|hickey|condom|testicles|anal|bareback|bukkake|creampie|stripper|strap-on|missionary|clitoris|clit|clitty|cowgirl|fleshlight|sex|buttplug|milf|oral|sucking|bondage|orgasm|scissoring|railed|slut|sluts|slutty|cumming|cunt|faggot|sissy|anal|anus|cum|semen|scat|nsfw|xxx|explicit|erotic|horny|aroused|jizz|moan|rape|raped|raping|throbbing|humping)\b/gmi);
+
+    if (isNsfw) {
+        //replace risky subject nouns with person
+        prompt = prompt.replace(/\b(youngster|infant|baby|toddler|child|teen|kid|kiddie|kiddo|teenager|student|preteen|pre.teen)\b/gmi, "person");
+
+        //remove risky adjectives and related words
+        prompt = prompt.replace(/\b(young|younger|youthful|youth|small|smaller|smallest|girly|boyish|lil|tiny|teenaged|lit[tl]le|school.aged|school|highschool|kindergarten|teens|children|kids)\b/gmi, "");
+    }
+
+    return prompt;
+}
+
 /**
  *
  * @param {import("express").Express} app
@@ -108,6 +143,17 @@ function registerEndpoints(app, jsonParser) {
                 request.body.prompt = String(request.body.prompt).substring(0, maxLength);
             }
 
+            // Sanitize prompt if requested
+            if (request.body.sanitize) {
+                const sanitized = sanitizeHordeImagePrompt(request.body.prompt);
+
+                if (request.body.prompt !== sanitized) {
+                    console.log('Stable Horde prompt was sanitized.');
+                }
+
+                request.body.prompt = sanitized;
+            }
+
             const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
             console.log('Stable Horde request:', request.body);