Merge branch 'staging' into 202411-backend-maxctx

src/endpoints/backends/chat-completions.js

@@ -102,7 +102,7 @@ async function sendClaudeRequest(request, response) {
         const additionalHeaders = {};
         const useTools = request.body.model.startsWith('claude-3') && Array.isArray(request.body.tools) && request.body.tools.length > 0;
         const useSystemPrompt = (request.body.model.startsWith('claude-2') || request.body.model.startsWith('claude-3')) && request.body.claude_use_sysprompt;
-        const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, request.body.human_sysprompt_message, request.body.char_name, request.body.user_name);
+        const convertedPrompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, useSystemPrompt, useTools, request.body.char_name, request.body.user_name);
         // Add custom stop sequences
         const stopSequences = [];
         if (Array.isArray(request.body.stop)) {

src/endpoints/characters.js

@@ -14,7 +14,7 @@ import jimp from 'jimp';
 
 import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
 import { jsonParser, urlencodedParser } from '../express-common.js';
-import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } from '../util.js';
+import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer, MemoryLimitedMap } from '../util.js';
 import { TavernCardValidator } from '../validator/TavernCardValidator.js';
 import { parse, write } from '../character-card-parser.js';
 import { readWorldInfoFile } from './worldinfo.js';
@@ -23,7 +23,8 @@ import { importRisuSprites } from './sprites.js';
 const defaultAvatarPath = './public/img/ai4.png';
 
 // KV-store for parsed character data
-const characterDataCache = new Map();
+// 100 MB limit. Would take roughly 3000 characters to reach this limit
+const characterDataCache = new MemoryLimitedMap(1024 * 1024 * 100);
 // Some Android devices require tighter memory management
 const isAndroid = process.platform === 'android';
 
@@ -58,6 +59,9 @@ async function writeCharacterData(inputFile, data, outputFile, request, crop = u
     try {
         // Reset the cache
         for (const key of characterDataCache.keys()) {
+            if (Buffer.isBuffer(inputFile)) {
+                break;
+            }
             if (key.startsWith(inputFile)) {
                 characterDataCache.delete(key);
                 break;

src/middleware/webpack-serve.js

@@ -1,20 +1,45 @@
-import process from 'node:process';
+import path from 'node:path';
 import webpack from 'webpack';
-import middleware from 'webpack-dev-middleware';
 import { publicLibConfig } from '../../webpack.config.js';
 
 export default function getWebpackServeMiddleware() {
+    const outputPath = publicLibConfig.output?.path;
+    const outputFile = publicLibConfig.output?.filename;
     const compiler = webpack(publicLibConfig);
 
-    if (process.env.NODE_ENV === 'production' || process.platform === 'android') {
-        compiler.hooks.done.tap('serve', () => {
-            if (compiler.watching) {
-                compiler.watching.close(() => { });
-            }
-            compiler.watchFileSystem = null;
-            compiler.watchMode = false;
-        });
+    /**
+     * A very spartan recreation of webpack-dev-middleware.
+     * @param {import('express').Request} req Request object.
+     * @param {import('express').Response} res Response object.
+     * @param {import('express').NextFunction} next Next function.
+     * @type {import('express').RequestHandler}
+     */
+    function devMiddleware(req, res, next) {
+        if (req.method === 'GET' && path.parse(req.path).base === outputFile) {
+            return res.sendFile(outputFile, { root: outputPath });
+        }
+
+        next();
     }
 
-    return middleware(compiler, {});
+    /**
+     * Wait until Webpack is done compiling.
+     * @returns {Promise<void>}
+     */
+    devMiddleware.runWebpackCompiler = () => {
+        return new Promise((resolve) => {
+            console.log();
+            console.log('Compiling frontend libraries...');
+            compiler.run((_error, stats) => {
+                const output = stats?.toString(publicLibConfig.stats);
+                if (output) {
+                    console.log(output);
+                    console.log();
+                }
+                resolve();
+            });
+        });
+    };
+
+    return devMiddleware;
 }

src/prompt-converters.js

@@ -91,11 +91,10 @@ export function convertClaudePrompt(messages, addAssistantPostfix, addAssistantP
  * @param {string}   prefillString User determined prefill string
  * @param {boolean}  useSysPrompt See if we want to use a system prompt
  * @param {boolean}  useTools See if we want to use tools
- * @param {string}   humanMsgFix Add Human message between system prompt and assistant.
  * @param {string}   charName Character name
  * @param {string}   userName User name
  */
-export function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') {
+export function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, charName, userName) {
     let systemPrompt = [];
     if (useSysPrompt) {
         // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
@@ -122,10 +121,10 @@ export function convertClaudeMessages(messages, prefillString, useSysPrompt, use
 
         // Check if the first message in the array is of type user, if not, interject with humanMsgFix or a blank message.
         // Also prevents erroring out if the messages array is empty.
-        if (messages.length === 0 || (messages.length > 0 && messages[0].role !== 'user')) {
+        if (messages.length === 0) {
             messages.unshift({
                 role: 'user',
-                content: humanMsgFix || PROMPT_PLACEHOLDER,
+                content: PROMPT_PLACEHOLDER,
             });
         }
     }

src/util.js

@@ -670,3 +670,182 @@ export function isValidUrl(url) {
         return false;
     }
 }
+
+/**
+ * MemoryLimitedMap class that limits the memory usage of string values.
+ */
+export class MemoryLimitedMap {
+    /**
+     * Creates an instance of MemoryLimitedMap.
+     * @param {number} maxMemoryInBytes - The maximum allowed memory in bytes for string values.
+     */
+    constructor(maxMemoryInBytes) {
+        if (typeof maxMemoryInBytes !== 'number' || maxMemoryInBytes <= 0) {
+            throw new Error('maxMemoryInBytes must be a positive number');
+        }
+        this.maxMemory = maxMemoryInBytes;
+        this.currentMemory = 0;
+        this.map = new Map();
+        this.queue = [];
+    }
+
+    /**
+     * Estimates the memory usage of a string in bytes.
+     * Assumes each character occupies 2 bytes (UTF-16).
+     * @param {string} str
+     * @returns {number}
+     */
+    static estimateStringSize(str) {
+        return str ? str.length * 2 : 0;
+    }
+
+    /**
+     * Adds or updates a key-value pair in the map.
+     * If adding the new value exceeds the memory limit, evicts oldest entries.
+     * @param {string} key
+     * @param {string} value
+     */
+    set(key, value) {
+        if (typeof key !== 'string' || typeof value !== 'string') {
+            return;
+        }
+
+        const newValueSize = MemoryLimitedMap.estimateStringSize(value);
+
+        // If the new value itself exceeds the max memory, reject it
+        if (newValueSize > this.maxMemory) {
+            return;
+        }
+
+        // Check if the key already exists to adjust memory accordingly
+        if (this.map.has(key)) {
+            const oldValue = this.map.get(key);
+            const oldValueSize = MemoryLimitedMap.estimateStringSize(oldValue);
+            this.currentMemory -= oldValueSize;
+            // Remove the key from its current position in the queue
+            const index = this.queue.indexOf(key);
+            if (index > -1) {
+                this.queue.splice(index, 1);
+            }
+        }
+
+        // Evict oldest entries until there's enough space
+        while (this.currentMemory + newValueSize > this.maxMemory && this.queue.length > 0) {
+            const oldestKey = this.queue.shift();
+            const oldestValue = this.map.get(oldestKey);
+            const oldestValueSize = MemoryLimitedMap.estimateStringSize(oldestValue);
+            this.map.delete(oldestKey);
+            this.currentMemory -= oldestValueSize;
+        }
+
+        // After eviction, check again if there's enough space
+        if (this.currentMemory + newValueSize > this.maxMemory) {
+            return;
+        }
+
+        // Add the new key-value pair
+        this.map.set(key, value);
+        this.queue.push(key);
+        this.currentMemory += newValueSize;
+    }
+
+    /**
+     * Retrieves the value associated with the given key.
+     * @param {string} key
+     * @returns {string | undefined}
+     */
+    get(key) {
+        return this.map.get(key);
+    }
+
+    /**
+     * Checks if the map contains the given key.
+     * @param {string} key
+     * @returns {boolean}
+     */
+    has(key) {
+        return this.map.has(key);
+    }
+
+    /**
+     * Deletes the key-value pair associated with the given key.
+     * @param {string} key
+     * @returns {boolean} - Returns true if the key was found and deleted, else false.
+     */
+    delete(key) {
+        if (!this.map.has(key)) {
+            return false;
+        }
+        const value = this.map.get(key);
+        const valueSize = MemoryLimitedMap.estimateStringSize(value);
+        this.map.delete(key);
+        this.currentMemory -= valueSize;
+
+        // Remove the key from the queue
+        const index = this.queue.indexOf(key);
+        if (index > -1) {
+            this.queue.splice(index, 1);
+        }
+
+        return true;
+    }
+
+    /**
+     * Clears all entries from the map.
+     */
+    clear() {
+        this.map.clear();
+        this.queue = [];
+        this.currentMemory = 0;
+    }
+
+    /**
+     * Returns the number of key-value pairs in the map.
+     * @returns {number}
+     */
+    size() {
+        return this.map.size;
+    }
+
+    /**
+     * Returns the current memory usage in bytes.
+     * @returns {number}
+     */
+    totalMemory() {
+        return this.currentMemory;
+    }
+
+    /**
+     * Returns an iterator over the keys in the map.
+     * @returns {IterableIterator<string>}
+     */
+    keys() {
+        return this.map.keys();
+    }
+
+    /**
+     * Returns an iterator over the values in the map.
+     * @returns {IterableIterator<string>}
+     */
+    values() {
+        return this.map.values();
+    }
+
+    /**
+     * Iterates over the map in insertion order.
+     * @param {Function} callback - Function to execute for each element.
+     */
+    forEach(callback) {
+        this.map.forEach((value, key) => {
+            callback(value, key, this);
+        });
+    }
+
+    /**
+     * Makes the MemoryLimitedMap iterable.
+     * @returns {Iterator} - Iterator over [key, value] pairs.
+     */
+    [Symbol.iterator]() {
+        return this.map[Symbol.iterator]();
+    }
+}