Merge branch 'staging' into support-multiple-expressions

2025-06-05 21:59:27 +02:00 · 2025-01-26 23:11:37 +01:00
parent 239c3f1db7 6d43eea1bd
commit 5c34c93a84
85 changed files with 2970 additions and 917 deletions
--- a/src/endpoints/avatars.js
+++ b/src/endpoints/avatars.js
@@ -9,6 +9,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import { jsonParser, urlencodedParser } from '../express-common.js';
 import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
 import { getImages, tryParse } from '../util.js';
+import { getFileNameValidationFunction } from '../middleware/validateFileName.js';

 export const router = express.Router();

@@ -17,7 +18,7 @@ router.post('/get', jsonParser, function (request, response) {
    response.send(JSON.stringify(images));
 });

-router.post('/delete', jsonParser, function (request, response) {
+router.post('/delete', jsonParser, getFileNameValidationFunction('avatar'), function (request, response) {
    if (!request.body) return response.sendStatus(400);

    if (request.body.avatar !== sanitize(request.body.avatar)) {
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -37,6 +37,8 @@ import {
    getTiktokenTokenizer,
    sentencepieceTokenizers,
    TEXT_COMPLETION_MODELS,
+    webTokenizers,
+    getWebTokenizer,
 } from '../tokenizers.js';

 const API_OPENAI = 'https://api.openai.com/v1';
@@ -61,6 +63,7 @@ const API_DEEPSEEK = 'https://api.deepseek.com/beta';
 * @returns
 */
 function postProcessPrompt(messages, type, names) {
+    const addAssistantPrefix = x => x.length && (x[x.length - 1].role !== 'assistant' || (x[x.length - 1].prefix = true)) ? x : x;
    switch (type) {
        case 'merge':
        case 'claude':
@@ -70,7 +73,9 @@ function postProcessPrompt(messages, type, names) {
        case 'strict':
            return mergeMessages(messages, names, true, true);
        case 'deepseek':
-            return (x => x.length && (x[x.length - 1].role !== 'assistant' || (x[x.length - 1].prefix = true)) ? x : x)(mergeMessages(messages, names, true, false));
+            return addAssistantPrefix(mergeMessages(messages, names, true, false));
+        case 'deepseek-reasoner':
+            return addAssistantPrefix(mergeMessages(messages, names, true, true));
        default:
            return messages;
    }
@@ -636,6 +641,89 @@ async function sendCohereRequest(request, response) {
    }
 }

+/**
+ * Sends a request to DeepSeek API.
+ * @param {express.Request} request Express request
+ * @param {express.Response} response Express response
+ */
+async function sendDeepSeekRequest(request, response) {
+    const apiUrl = new URL(request.body.reverse_proxy || API_DEEPSEEK).toString();
+    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
+
+    if (!apiKey && !request.body.reverse_proxy) {
+        console.log('DeepSeek API key is missing.');
+        return response.status(400).send({ error: true });
+    }
+
+    const controller = new AbortController();
+    request.socket.removeAllListeners('close');
+    request.socket.on('close', function () {
+        controller.abort();
+    });
+
+    try {
+        let bodyParams = {};
+
+        if (request.body.logprobs > 0) {
+            bodyParams['top_logprobs'] = request.body.logprobs;
+            bodyParams['logprobs'] = true;
+        }
+
+        const postProcessType = String(request.body.model).endsWith('-reasoner') ? 'deepseek-reasoner' : 'deepseek';
+        const processedMessages = postProcessPrompt(request.body.messages, postProcessType, getPromptNames(request));
+
+        const requestBody = {
+            'messages': processedMessages,
+            'model': request.body.model,
+            'temperature': request.body.temperature,
+            'max_tokens': request.body.max_tokens,
+            'stream': request.body.stream,
+            'presence_penalty': request.body.presence_penalty,
+            'frequency_penalty': request.body.frequency_penalty,
+            'top_p': request.body.top_p,
+            'stop': request.body.stop,
+            'seed': request.body.seed,
+            ...bodyParams,
+        };
+
+        const config = {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': 'Bearer ' + apiKey,
+            },
+            body: JSON.stringify(requestBody),
+            signal: controller.signal,
+        };
+
+        console.log('DeepSeek request:', requestBody);
+
+        const generateResponse = await fetch(apiUrl + '/chat/completions', config);
+
+        if (request.body.stream) {
+            forwardFetchResponse(generateResponse, response);
+        } else {
+            if (!generateResponse.ok) {
+                const errorText = await generateResponse.text();
+                console.log(`DeepSeek API returned error: ${generateResponse.status} ${generateResponse.statusText} ${errorText}`);
+                const errorJson = tryParse(errorText) ?? { error: true };
+                return response.status(500).send(errorJson);
+            }
+            const generateResponseJson = await generateResponse.json();
+            console.log('DeepSeek response:', generateResponseJson);
+            return response.send(generateResponseJson);
+        }
+    } catch (error) {
+        console.log('Error communicating with DeepSeek API: ', error);
+        if (!response.headersSent) {
+            response.send({ error: true });
+        } else {
+            response.end();
+        }
+    }
+}
+
+
 export const router = express.Router();

 router.post('/status', jsonParser, async function (request, response_getstatus_openai) {
@@ -680,8 +768,8 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
        api_key_openai = readSecret(request.user.directories, SECRET_KEYS.NANOGPT);
        headers = {};
    } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
-        api_url = API_DEEPSEEK.replace('/beta', '');
-        api_key_openai = readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
+        api_url = new URL(request.body.reverse_proxy || API_DEEPSEEK.replace('/beta', ''));
+        api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
        headers = {};
    } else {
        console.log('This chat completion source is not supported yet.');
@@ -777,6 +865,14 @@ router.post('/bias', jsonParser, async function (request, response) {
                return response.send({});
            }
            encodeFunction = (text) => new Uint32Array(instance.encodeIds(text));
+        } else if (webTokenizers.includes(model)) {
+            const tokenizer = getWebTokenizer(model);
+            const instance = await tokenizer?.get();
+            if (!instance) {
+                console.warn('Tokenizer not initialized:', model);
+                return response.send({});
+            }
+            encodeFunction = (text) => new Uint32Array(instance.encode(text));
        } else {
            const tokenizer = getTiktokenTokenizer(model);
            encodeFunction = (tokenizer.encode.bind(tokenizer));
@@ -841,6 +937,7 @@ router.post('/generate', jsonParser, function (request, response) {
        case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response);
        case CHAT_COMPLETION_SOURCES.MISTRALAI: return sendMistralAIRequest(request, response);
        case CHAT_COMPLETION_SOURCES.COHERE: return sendCohereRequest(request, response);
+        case CHAT_COMPLETION_SOURCES.DEEPSEEK: return sendDeepSeekRequest(request, response);
    }

    let apiUrl;
@@ -954,18 +1051,6 @@ router.post('/generate', jsonParser, function (request, response) {
        apiKey = readSecret(request.user.directories, SECRET_KEYS.BLOCKENTROPY);
        headers = {};
        bodyParams = {};
-    } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.DEEPSEEK) {
-        apiUrl = API_DEEPSEEK;
-        apiKey = readSecret(request.user.directories, SECRET_KEYS.DEEPSEEK);
-        headers = {};
-        bodyParams = {};
-
-        if (request.body.logprobs > 0) {
-            bodyParams['top_logprobs'] = request.body.logprobs;
-            bodyParams['logprobs'] = true;
-        }
-
-        request.body.messages = postProcessPrompt(request.body.messages, 'deepseek', getPromptNames(request));
    } else {
        console.log('This chat completion source is not supported yet.');
        return response.status(400).send({ error: true });
@@ -1103,4 +1188,3 @@ router.post('/generate', jsonParser, function (request, response) {
        }
    }
 });
-
--- a/src/endpoints/backgrounds.js
+++ b/src/endpoints/backgrounds.js
@@ -7,6 +7,7 @@ import sanitize from 'sanitize-filename';
 import { jsonParser, urlencodedParser } from '../express-common.js';
 import { invalidateThumbnail } from './thumbnails.js';
 import { getImages } from '../util.js';
+import { getFileNameValidationFunction } from '../middleware/validateFileName.js';

 export const router = express.Router();

@@ -15,7 +16,7 @@ router.post('/all', jsonParser, function (request, response) {
    response.send(JSON.stringify(images));
 });

-router.post('/delete', jsonParser, function (request, response) {
+router.post('/delete', jsonParser, getFileNameValidationFunction('bg'), function (request, response) {
    if (!request.body) return response.sendStatus(400);

    if (request.body.bg !== sanitize(request.body.bg)) {
--- a/src/endpoints/characters.js
+++ b/src/endpoints/characters.js
@@ -14,6 +14,7 @@ import jimp from 'jimp';

 import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
 import { jsonParser, urlencodedParser } from '../express-common.js';
+import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction } from '../middleware/validateFileName.js';
 import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer, MemoryLimitedMap, getConfigValue } from '../util.js';
 import { TavernCardValidator } from '../validator/TavernCardValidator.js';
 import { parse, write } from '../character-card-parser.js';
@@ -73,12 +74,18 @@ async function writeCharacterData(inputFile, data, outputFile, request, crop = u
         * Read the image, resize, and save it as a PNG into the buffer.
         * @returns {Promise<Buffer>} Image buffer
         */
-        function getInputImage() {
-            if (Buffer.isBuffer(inputFile)) {
-                return parseImageBuffer(inputFile, crop);
-            }
+        async function getInputImage() {
+            try {
+                if (Buffer.isBuffer(inputFile)) {
+                    return await parseImageBuffer(inputFile, crop);
+                }

-            return tryReadImage(inputFile, crop);
+                return await tryReadImage(inputFile, crop);
+            } catch (error) {
+                const message = Buffer.isBuffer(inputFile) ? 'Failed to read image buffer.' : `Failed to read image: ${inputFile}.`;
+                console.warn(message, 'Using a fallback image.', error);
+                return await fs.promises.readFile(defaultAvatarPath);
+            }
        }

        const inputImage = await getInputImage();
@@ -756,7 +763,7 @@ router.post('/create', urlencodedParser, async function (request, response) {
    }
 });

-router.post('/rename', jsonParser, async function (request, response) {
+router.post('/rename', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body.avatar_url || !request.body.new_name) {
        return response.sendStatus(400);
    }
@@ -803,7 +810,7 @@ router.post('/rename', jsonParser, async function (request, response) {
    }
 });

-router.post('/edit', urlencodedParser, async function (request, response) {
+router.post('/edit', urlencodedParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body) {
        console.error('Error: no response body detected');
        response.status(400).send('Error: no response body detected');
@@ -852,7 +859,7 @@ router.post('/edit', urlencodedParser, async function (request, response) {
 * @param {Object} response - The HTTP response object.
 * @returns {void}
 */
-router.post('/edit-attribute', jsonParser, async function (request, response) {
+router.post('/edit-attribute', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    console.log(request.body);
    if (!request.body) {
        console.error('Error: no response body detected');
@@ -898,7 +905,7 @@ router.post('/edit-attribute', jsonParser, async function (request, response) {
 *
 * @returns {void}
 * */
-router.post('/merge-attributes', jsonParser, async function (request, response) {
+router.post('/merge-attributes', jsonParser, getFileNameValidationFunction('avatar'), async function (request, response) {
    try {
        const update = request.body;
        const avatarPath = path.join(request.user.directories.characters, update.avatar);
@@ -929,7 +936,7 @@ router.post('/merge-attributes', jsonParser, async function (request, response)
    }
 });

-router.post('/delete', jsonParser, async function (request, response) {
+router.post('/delete', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body || !request.body.avatar_url) {
        return response.sendStatus(400);
    }
@@ -992,7 +999,7 @@ router.post('/all', jsonParser, async function (request, response) {
    }
 });

-router.post('/get', jsonParser, async function (request, response) {
+router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    try {
        if (!request.body) return response.sendStatus(400);
        const item = request.body.avatar_url;
@@ -1011,7 +1018,7 @@ router.post('/get', jsonParser, async function (request, response) {
    }
 });

-router.post('/chats', jsonParser, async function (request, response) {
+router.post('/chats', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body) return response.sendStatus(400);

    const characterDirectory = (request.body.avatar_url).replace('.png', '');
@@ -1160,7 +1167,7 @@ router.post('/import', urlencodedParser, async function (request, response) {
    }
 });

-router.post('/duplicate', jsonParser, async function (request, response) {
+router.post('/duplicate', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    try {
        if (!request.body.avatar_url) {
            console.log('avatar URL not found in request body');
@@ -1207,7 +1214,7 @@ router.post('/duplicate', jsonParser, async function (request, response) {
    }
 });

-router.post('/export', jsonParser, async function (request, response) {
+router.post('/export', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    try {
        if (!request.body.format || !request.body.avatar_url) {
            return response.sendStatus(400);
--- a/src/endpoints/chats.js
+++ b/src/endpoints/chats.js
@@ -9,6 +9,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import _ from 'lodash';

 import { jsonParser, urlencodedParser } from '../express-common.js';
+import validateAvatarUrlMiddleware from '../middleware/validateFileName.js';
 import {
    getConfigValue,
    humanizedISO8601DateTime,
@@ -264,9 +265,37 @@ function flattenChubChat(userName, characterName, lines) {
    return (lines ?? []).map(convert).join('\n');
 }

+/**
+ * Imports a chat from RisuAI format.
+ * @param {string} userName User name
+ * @param {string} characterName Character name
+ * @param {object} jsonData Imported chat data
+ * @returns {string} Chat data
+ */
+function importRisuChat(userName, characterName, jsonData) {
+    /** @type {object[]} */
+    const chat = [{
+        user_name: userName,
+        character_name: characterName,
+        create_date: humanizedISO8601DateTime(),
+    }];
+
+    for (const message of jsonData.data.message) {
+        const isUser = message.role === 'user';
+        chat.push({
+            name: message.name ?? (isUser ? userName : characterName),
+            is_user: isUser,
+            send_date: Number(message.time ?? Date.now()),
+            mes: message.data ?? '',
+        });
+    }
+
+    return chat.map(obj => JSON.stringify(obj)).join('\n');
+}
+
 export const router = express.Router();

-router.post('/save', jsonParser, function (request, response) {
+router.post('/save', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
    try {
        const directoryName = String(request.body.avatar_url).replace('.png', '');
        const chatData = request.body.chat;
@@ -282,7 +311,7 @@ router.post('/save', jsonParser, function (request, response) {
    }
 });

-router.post('/get', jsonParser, function (request, response) {
+router.post('/get', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
    try {
        const dirName = String(request.body.avatar_url).replace('.png', '');
        const directoryPath = path.join(request.user.directories.chats, dirName);
@@ -319,7 +348,7 @@ router.post('/get', jsonParser, function (request, response) {
 });


-router.post('/rename', jsonParser, async function (request, response) {
+router.post('/rename', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body || !request.body.original_file || !request.body.renamed_file) {
        return response.sendStatus(400);
    }
@@ -344,7 +373,7 @@ router.post('/rename', jsonParser, async function (request, response) {
    return response.send({ ok: true, sanitizedFileName });
 });

-router.post('/delete', jsonParser, function (request, response) {
+router.post('/delete', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
    const dirName = String(request.body.avatar_url).replace('.png', '');
    const fileName = String(request.body.chatfile);
    const filePath = path.join(request.user.directories.chats, dirName, sanitize(fileName));
@@ -360,7 +389,7 @@ router.post('/delete', jsonParser, function (request, response) {
    return response.send('ok');
 });

-router.post('/export', jsonParser, async function (request, response) {
+router.post('/export', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
    if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
        return response.sendStatus(400);
    }
@@ -450,7 +479,7 @@ router.post('/group/import', urlencodedParser, function (request, response) {
    }
 });

-router.post('/import', urlencodedParser, function (request, response) {
+router.post('/import', urlencodedParser, validateAvatarUrlMiddleware, function (request, response) {
    if (!request.body) return response.sendStatus(400);

    const format = request.body.file_type;
@@ -481,6 +510,8 @@ router.post('/import', urlencodedParser, function (request, response) {
                importFunc = importOobaChat;
            } else if (Array.isArray(jsonData.messages)) { // Agnai's format
                importFunc = importAgnaiChat;
+            } else if (jsonData.type === 'risuChat') { // RisuAI format
+                importFunc = importRisuChat;
            } else { // Unknown format
                console.log('Incorrect chat format .json');
                return response.send({ error: true });
@@ -596,7 +627,7 @@ router.post('/group/save', jsonParser, (request, response) => {
    return response.send({ ok: true });
 });

-router.post('/search', jsonParser, function (request, response) {
+router.post('/search', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
    try {
        const { query, avatar_url, group_id } = request.body;
        let chatFiles = [];
--- a/src/endpoints/search.js
+++ b/src/endpoints/search.js
@@ -4,6 +4,8 @@ import express from 'express';
 import { decode } from 'html-entities';
 import { readSecret, SECRET_KEYS } from './secrets.js';
 import { jsonParser } from '../express-common.js';
+import { trimV1 } from '../util.js';
+import { setAdditionalHeaders } from '../additional-headers.js';

 export const router = express.Router();

@@ -257,6 +259,41 @@ router.post('/tavily', jsonParser, async (request, response) => {
    }
 });

+router.post('/koboldcpp', jsonParser, async (request, response) => {
+    try {
+        const { query, url } = request.body;
+
+        if (!url) {
+            console.error('No URL provided for KoboldCpp search');
+            return response.sendStatus(400);
+        }
+
+        console.debug('KoboldCpp search query', query);
+
+        const baseUrl = trimV1(url);
+        const args = {
+            method: 'POST',
+            headers: {},
+            body: JSON.stringify({ q: query }),
+        };
+
+        setAdditionalHeaders(request, args, baseUrl);
+        const result = await fetch(`${baseUrl}/api/extra/websearch`, args);
+
+        if (!result.ok) {
+            const text = await result.text();
+            console.error('KoboldCpp request failed', result.statusText, text);
+            return response.status(500).send(text);
+        }
+
+        const data = await result.json();
+        return response.json(data);
+    } catch (error) {
+        console.error(error);
+        return response.sendStatus(500);
+    }
+});
+
 router.post('/visit', jsonParser, async (request, response) => {
    try {
        const url = request.body.url;
--- a/src/endpoints/settings.js
+++ b/src/endpoints/settings.js
@@ -9,9 +9,10 @@ import { SETTINGS_FILE } from '../constants.js';
 import { getConfigValue, generateTimestamp, removeOldBackups } from '../util.js';
 import { jsonParser } from '../express-common.js';
 import { getAllUserHandles, getUserDirectories } from '../users.js';
+import { getFileNameValidationFunction } from '../middleware/validateFileName.js';

-const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true);
-const ENABLE_EXTENSIONS_AUTO_UPDATE = getConfigValue('enableExtensionsAutoUpdate', true);
+const ENABLE_EXTENSIONS = !!getConfigValue('extensions.enabled', true);
+const ENABLE_EXTENSIONS_AUTO_UPDATE = !!getConfigValue('extensions.autoUpdate', true);
 const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);

 // 10 minutes
@@ -296,7 +297,7 @@ router.post('/get-snapshots', jsonParser, async (request, response) => {
    }
 });

-router.post('/load-snapshot', jsonParser, async (request, response) => {
+router.post('/load-snapshot', jsonParser, getFileNameValidationFunction('name'), async (request, response) => {
    try {
        const userFilesPattern = getFilePrefix(request.user.profile.handle);

@@ -330,7 +331,7 @@ router.post('/make-snapshot', jsonParser, async (request, response) => {
    }
 });

-router.post('/restore-snapshot', jsonParser, async (request, response) => {
+router.post('/restore-snapshot', jsonParser, getFileNameValidationFunction('name'), async (request, response) => {
    try {
        const userFilesPattern = getFilePrefix(request.user.profile.handle);

--- a/src/endpoints/tokenizers.js
+++ b/src/endpoints/tokenizers.js
@@ -238,6 +238,15 @@ export const sentencepieceTokenizers = [
    'jamba',
 ];

+export const webTokenizers = [
+    'claude',
+    'llama3',
+    'command-r',
+    'qwen2',
+    'nemo',
+    'deepseek',
+];
+
 /**
 * Gets the Sentencepiece tokenizer by the model name.
 * @param {string} model Sentencepiece model name
@@ -275,6 +284,39 @@ export function getSentencepiceTokenizer(model) {
    return null;
 }

+/**
+ * Gets the Web tokenizer by the model name.
+ * @param {string} model Web tokenizer model name
+ * @returns {WebTokenizer|null} Web tokenizer
+ */
+export function getWebTokenizer(model) {
+    if (model.includes('llama3')) {
+        return llama3_tokenizer;
+    }
+
+    if (model.includes('claude')) {
+        return claude_tokenizer;
+    }
+
+    if (model.includes('command-r')) {
+        return commandTokenizer;
+    }
+
+    if (model.includes('qwen2')) {
+        return qwen2Tokenizer;
+    }
+
+    if (model.includes('nemo')) {
+        return nemoTokenizer;
+    }
+
+    if (model.includes('deepseek')) {
+        return deepseekTokenizer;
+    }
+
+    return null;
+}
+
 /**
 * Counts the token ids for the given text using the Sentencepiece tokenizer.
 * @param {SentencePieceTokenizer} tokenizer Sentencepiece tokenizer
--- a/src/endpoints/users-private.js
+++ b/src/endpoints/users-private.js
@@ -23,6 +23,7 @@ router.post('/logout', async (request, response) => {
        }

        request.session.handle = null;
+        request.session.csrfToken = null;
        request.session = null;
        return response.sendStatus(204);
    } catch (error) {
--- a/src/endpoints/vectors.js
+++ b/src/endpoints/vectors.js
@@ -164,7 +164,7 @@ function getSourceSettings(source, request) {
            };
        case 'transformers':
            return {
-                model: getConfigValue('extras.embeddingModel', ''),
+                model: getConfigValue('extensions.models.embedding', ''),
            };
        case 'palm':
            return {
--- a/src/middleware/basicAuth.js
+++ b/src/middleware/basicAuth.js
@@ -5,17 +5,18 @@
 import { Buffer } from 'node:buffer';
 import storage from 'node-persist';
 import { getAllUserHandles, toKey, getPasswordHash } from '../users.js';
-import { getConfig, getConfigValue } from '../util.js';
+import { getConfig, getConfigValue, safeReadFileSync } from '../util.js';

 const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false);
 const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);

-const unauthorizedResponse = (res) => {
-    res.set('WWW-Authenticate', 'Basic realm="SillyTavern", charset="UTF-8"');
-    return res.status(401).send('Authentication required');
-};
-
 const basicAuthMiddleware = async function (request, response, callback) {
+    const unauthorizedWebpage = safeReadFileSync('./public/error/unauthorized.html') ?? '';
+    const unauthorizedResponse = (res) => {
+        res.set('WWW-Authenticate', 'Basic realm="SillyTavern", charset="UTF-8"');
+        return res.status(401).send(unauthorizedWebpage);
+    };
+
    const config = getConfig();
    const authHeader = request.headers.authorization;

--- a/src/middleware/validateFileName.js
+++ b/src/middleware/validateFileName.js
@@ -0,0 +1,34 @@
+import path from 'node:path';
+
+/**
+ * Gets a middleware function that validates the field in the request body.
+ * @param {string} fieldName Field name
+ * @returns {import('express').RequestHandler} Middleware function
+ */
+export function getFileNameValidationFunction(fieldName) {
+    /**
+    * Validates the field in the request body.
+    * @param {import('express').Request} req Request object
+    * @param {import('express').Response} res Response object
+    * @param {import('express').NextFunction} next Next middleware
+    */
+    return function validateAvatarUrlMiddleware(req, res, next) {
+        if (req.body && fieldName in req.body && typeof req.body[fieldName] === 'string') {
+            const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
+            if (forbiddenRegExp.test(req.body[fieldName])) {
+                console.error('An error occurred while validating the request body', {
+                    handle: req.user.profile.handle,
+                    path: req.originalUrl,
+                    field: fieldName,
+                    value: req.body[fieldName],
+                });
+                return res.sendStatus(400);
+            }
+        }
+
+        next();
+    };
+}
+
+const avatarUrlValidationFunction = getFileNameValidationFunction('avatar_url');
+export default avatarUrlValidationFunction;
--- a/src/middleware/webpack-serve.js
+++ b/src/middleware/webpack-serve.js
@@ -5,8 +5,6 @@ import { publicLibConfig } from '../../webpack.config.js';
 export default function getWebpackServeMiddleware() {
    const outputPath = publicLibConfig.output?.path;
    const outputFile = publicLibConfig.output?.filename;
-    /** @type {import('webpack').Compiler|null} */
-    let compiler = webpack(publicLibConfig);

    /**
     * A very spartan recreation of webpack-dev-middleware.
@@ -28,12 +26,9 @@ export default function getWebpackServeMiddleware() {
     * @returns {Promise<void>}
     */
    devMiddleware.runWebpackCompiler = () => {
-        return new Promise((resolve) => {
-            if (compiler === null) {
-                console.warn('Webpack compiler is already closed.');
-                return resolve();
-            }
+        const compiler = webpack(publicLibConfig);

+        return new Promise((resolve) => {
            console.log();
            console.log('Compiling frontend libraries...');
            compiler.run((_error, stats) => {
@@ -42,11 +37,7 @@ export default function getWebpackServeMiddleware() {
                    console.log(output);
                    console.log();
                }
-                if (compiler === null) {
-                    return resolve();
-                }
                compiler.close(() => {
-                    compiler = null;
                    resolve();
                });
            });
--- a/src/middleware/whitelist.js
+++ b/src/middleware/whitelist.js
@@ -1,10 +1,11 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import process from 'node:process';
+import Handlebars from 'handlebars';
 import ipMatching from 'ip-matching';

 import { getIpFromRequest } from '../express-common.js';
-import { color, getConfigValue } from '../util.js';
+import { color, getConfigValue, safeReadFileSync } from '../util.js';

 const whitelistPath = path.join(process.cwd(), './whitelist.txt');
 const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false);
@@ -52,12 +53,16 @@ function getForwardedIp(req) {
 * @returns {import('express').RequestHandler} The middleware function
 */
 export default function whitelistMiddleware(whitelistMode, listen) {
+    const forbiddenWebpage = Handlebars.compile(
+        safeReadFileSync('./public/error/forbidden-by-whitelist.html') ?? '',
+    );
+
    return function (req, res, next) {
        const clientIp = getIpFromRequest(req);
        const forwardedIp = getForwardedIp(req);
+        const userAgent = req.headers['user-agent'];

        if (listen && !knownIPs.has(clientIp)) {
-            const userAgent = req.headers['user-agent'];
            console.log(color.yellow(`New connection from ${clientIp}; User Agent: ${userAgent}\n`));
            knownIPs.add(clientIp);

@@ -76,9 +81,15 @@ export default function whitelistMiddleware(whitelistMode, listen) {
            || forwardedIp && whitelistMode === true && !whitelist.some(x => ipMatching.matches(forwardedIp, ipMatching.getMatch(x)))
        ) {
            // Log the connection attempt with real IP address
-            const ipDetails = forwardedIp ? `${clientIp} (forwarded from ${forwardedIp})` : clientIp;
-            console.log(color.red('Forbidden: Connection attempt from ' + ipDetails + '. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.\n'));
-            return res.status(403).send('<b>Forbidden</b>: Connection attempt from <b>' + ipDetails + '</b>. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.');
+            const ipDetails = forwardedIp
+                ? `${clientIp} (forwarded from ${forwardedIp})`
+                : clientIp;
+            console.log(
+                color.red(
+                    `Blocked connection from ${clientIp}; User Agent: ${userAgent}\n\tTo allow this connection, add its IP address to the whitelist or disable whitelist mode by editing config.yaml in the root directory of your SillyTavern installation.\n`,
+                ),
+            );
+            return res.status(403).send(forbiddenWebpage({ ipDetails }));
        }
        next();
    };
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -360,6 +360,8 @@ export function convertCohereMessages(messages, names) {
 */
 export function convertGooglePrompt(messages, model, useSysPrompt, names) {
    const visionSupportedModels = [
+        'gemini-2.0-flash-thinking-exp',
+        'gemini-2.0-flash-thinking-exp-01-21',
        'gemini-2.0-flash-thinking-exp-1219',
        'gemini-2.0-flash-exp',
        'gemini-1.5-flash',
--- a/src/transformers.js
+++ b/src/transformers.js
@@ -19,31 +19,31 @@ const tasks = {
    'text-classification': {
        defaultModel: 'Cohee/distilbert-base-uncased-go-emotions-onnx',
        pipeline: null,
-        configField: 'extras.classificationModel',
+        configField: 'extensions.models.classification',
        quantized: true,
    },
    'image-to-text': {
        defaultModel: 'Xenova/vit-gpt2-image-captioning',
        pipeline: null,
-        configField: 'extras.captioningModel',
+        configField: 'extensions.models.captioning',
        quantized: true,
    },
    'feature-extraction': {
        defaultModel: 'Xenova/all-mpnet-base-v2',
        pipeline: null,
-        configField: 'extras.embeddingModel',
+        configField: 'extensions.models.embedding',
        quantized: true,
    },
    'automatic-speech-recognition': {
        defaultModel: 'Xenova/whisper-small',
        pipeline: null,
-        configField: 'extras.speechToTextModel',
+        configField: 'extensions.models.speechToText',
        quantized: true,
    },
    'text-to-speech': {
        defaultModel: 'Xenova/speecht5_tts',
        pipeline: null,
-        configField: 'extras.textToSpeechModel',
+        configField: 'extensions.models.textToSpeech',
        quantized: false,
    },
 };
@@ -132,7 +132,7 @@ export async function getPipeline(task, forceModel = '') {

    const cacheDir = path.join(globalThis.DATA_ROOT, '_cache');
    const model = forceModel || getModelForTask(task);
-    const localOnly = getConfigValue('extras.disableAutoDownload', false);
+    const localOnly = !getConfigValue('extensions.models.autoDownload', true);
    console.log('Initializing transformers.js pipeline for task', task, 'with model', model);
    const instance = await pipeline(task, model, { cache_dir: cacheDir, quantized: tasks[task].quantized ?? true, local_files_only: localOnly });
    tasks[task].pipeline = instance;
--- a/src/users.js
+++ b/src/users.js
@@ -458,7 +458,8 @@ export function getPasswordSalt() {
 */
 export function getCookieSessionName() {
    // Get server hostname and hash it to generate a session suffix
-    const suffix = crypto.createHash('sha256').update(os.hostname()).digest('hex').slice(0, 8);
+    const hostname = os.hostname() || 'localhost';
+    const suffix = crypto.createHash('sha256').update(hostname).digest('hex').slice(0, 8);
    return `session-${suffix}`;
 }

--- a/src/util.js
+++ b/src/util.js
@@ -871,3 +871,14 @@ export class MemoryLimitedMap {
        return this.map[Symbol.iterator]();
    }
 }
+
+/**
+ * A 'safe' version of `fs.readFileSync()`. Returns the contents of a file if it exists, falling back to a default value if not.
+ * @param {string} filePath Path of the file to be read.
+ * @param {Parameters<typeof fs.readFileSync>[1]} options Options object to pass through to `fs.readFileSync()` (default: `{ encoding: 'utf-8' }`).
+ * @returns The contents at `filePath` if it exists, or `null` if not.
+ */
+export function safeReadFileSync(filePath, options = { encoding: 'utf-8' }) {
+    if (fs.existsSync(filePath)) return fs.readFileSync(filePath, options);
+    return null;
+}