Merge branch 'staging' into sysprompt-divorce

Dockerfile

@@ -19,7 +19,7 @@ ENV NODE_ENV=production
 COPY package*.json post-install.js ./
 RUN \
   echo "*** Install npm packages ***" && \
-  npm i --no-audit --no-fund --quiet --omit=dev && npm cache clean --force
+  npm i --no-audit --no-fund --loglevel=error --no-progress --omit=dev && npm cache clean --force
 
 # Bundle app source
 COPY . ./

Start.bat

@@ -1,7 +1,7 @@
 @echo off
 pushd %~dp0
 set NODE_ENV=production
-call npm install --no-audit --no-fund --quiet --omit=dev
+call npm install --no-audit --no-fund --loglevel=error --no-progress --omit=dev
 node server.js %*
 pause
 popd

UpdateAndStart.bat

@@ -12,7 +12,7 @@ if %errorlevel% neq 0 (
     )
 )
 set NODE_ENV=production
-call npm install --no-audit --no-fund --quiet --omit=dev
+call npm install --no-audit --no-fund --loglevel=error --no-progress --omit=dev
 node server.js %*
 pause
 popd

UpdateForkAndStart.bat

@@ -42,7 +42,7 @@ if NOT "!AUTO_SWITCH!"=="" (
         SET TARGET_BRANCH=release
         goto update
     )
-    
+
     echo Auto-switching defined to stay on current branch
     goto update
 )
@@ -95,7 +95,7 @@ if %errorlevel% neq 0 (
 
 echo Installing npm packages and starting server
 set NODE_ENV=production
-call npm install --no-audit --no-fund --quiet --omit=dev
+call npm install --no-audit --no-fund --loglevel=error --no-progress --omit=dev
 node server.js %*
 
 :end

public/css/promptmanager.css

@@ -316,3 +316,15 @@
         margin-left: 0.5em;
     }
 }
+
+.completion_prompt_manager_popup_entry_form_control:has(#completion_prompt_manager_popup_entry_form_prompt:disabled) > div:first-child::after {
+    content: 'The content of this prompt is pulled from elsewhere and cannot be edited here.';
+    display: block;
+    width: 100%;
+    font-weight: 600;
+    text-align: center;
+}
+
+.completion_prompt_manager_popup_entry_form_control #completion_prompt_manager_popup_entry_form_prompt:disabled {
+    visibility: hidden;
+}

public/index.html

@@ -3297,7 +3297,8 @@
                                     <option value="16">Command-R</option>
                                     <option value="4">NerdStash (NovelAI Clio)</option>
                                     <option value="5">NerdStash v2 (NovelAI Kayra)</option>
-                                    <option value="7">Mistral</option>
+                                    <option value="7">Mistral V1</option>
+                                    <option value="17">Mistral Nemo</option>
                                     <option value="8">Yi</option>
                                     <option value="11">Claude 1/2</option>
                                     <option value="6">API (WebUI / koboldcpp)</option>

public/script.js

@@ -6923,15 +6923,13 @@ export async function displayPastChats() {
             }
             // Check whether `text` {string} includes all of the `fragments` {string[]}.
             function matchFragments(fragments, text) {
-                if (!text) {
+                if (!text || !text.toLowerCase) return false;
-                    return false;
+                return fragments.every(item => text.toLowerCase().includes(item));
-                }
-                return fragments.every(item => text.includes(item));
             }
             const fragments = makeQueryFragments(searchQuery);
             // At least one chat message must match *all* the fragments.
             // Currently, this doesn't match if the fragment matches are distributed across several chat messages.
-            return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes?.toLowerCase()));
+            return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes));
         });
 
         console.debug(filteredData);

public/scripts/PromptManager.js

@@ -427,12 +427,13 @@ class PromptManager {
 
             document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_name').value = prompt.name;
             document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_role').value = 'system';
-            document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_prompt').value = prompt.content;
+            document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_prompt').value = prompt.content ?? '';
             document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_injection_position').value = prompt.injection_position ?? 0;
             document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_injection_depth').value = prompt.injection_depth ?? DEFAULT_DEPTH;
             document.getElementById(this.configuration.prefix + 'prompt_manager_depth_block').style.visibility = prompt.injection_position === INJECTION_POSITION.ABSOLUTE ? 'visible' : 'hidden';
             document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_forbid_overrides').checked = prompt.forbid_overrides ?? false;
             document.getElementById(this.configuration.prefix + 'prompt_manager_forbid_overrides_block').style.visibility = this.overridablePrompts.includes(prompt.identifier) ? 'visible' : 'hidden';
+            document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_prompt').disabled = prompt.marker ?? false;
 
             if (!this.systemPrompts.includes(promptId)) {
                 document.getElementById(this.configuration.prefix + 'prompt_manager_popup_entry_form_injection_position').removeAttribute('disabled');
@@ -920,7 +921,15 @@ class PromptManager {
      * @returns {boolean} True if the prompt can be edited, false otherwise.
      */
     isPromptEditAllowed(prompt) {
-        return !prompt.marker;
+        const forceEditPrompts = [
+            'charDescription',
+            'charPersonality',
+            'scenario',
+            'personaDescription',
+            'worldInfoBefore',
+            'worldInfoAfter',
+        ];
+        return forceEditPrompts.includes(prompt.identifier) || !prompt.marker;
     }
 
     /**
@@ -929,7 +938,17 @@ class PromptManager {
      * @returns {boolean} True if the prompt can be deleted, false otherwise.
      */
     isPromptToggleAllowed(prompt) {
-        const forceTogglePrompts = ['charDescription', 'charPersonality', 'scenario', 'personaDescription', 'worldInfoBefore', 'worldInfoAfter', 'main', 'chatHistory', 'dialogueExamples'];
+        const forceTogglePrompts = [
+            'charDescription',
+            'charPersonality',
+            'scenario',
+            'personaDescription',
+            'worldInfoBefore',
+            'worldInfoAfter',
+            'main',
+            'chatHistory',
+            'dialogueExamples',
+        ];
         return prompt.marker && !forceTogglePrompts.includes(prompt.identifier) ? false : !this.configuration.toggleDisabled.includes(prompt.identifier);
     }
 
@@ -1182,8 +1201,9 @@ class PromptManager {
         const forbidOverridesBlock = document.getElementById(this.configuration.prefix + 'prompt_manager_forbid_overrides_block');
 
         nameField.value = prompt.name ?? '';
-        roleField.value = prompt.role ?? '';
+        roleField.value = prompt.role ?? 'system';
         promptField.value = prompt.content ?? '';
+        promptField.disabled = prompt.marker ?? false;
         injectionPositionField.value = prompt.injection_position ?? INJECTION_POSITION.RELATIVE;
         injectionDepthField.value = prompt.injection_depth ?? DEFAULT_DEPTH;
         injectionDepthBlock.style.visibility = prompt.injection_position === INJECTION_POSITION.ABSOLUTE ? 'visible' : 'hidden';
@@ -1279,6 +1299,7 @@ class PromptManager {
         nameField.value = '';
         roleField.selectedIndex = 0;
         promptField.value = '';
+        promptField.disabled = false;
         injectionPositionField.selectedIndex = 0;
         injectionPositionField.removeAttribute('disabled');
         injectionDepthField.value = DEFAULT_DEPTH;

public/scripts/openai.js

@@ -970,6 +970,12 @@ async function populateChatCompletion(prompts, chatCompletion, { bias, quietProm
         }
 
         const prompt = prompts.get(source);
+
+        if (prompt.injection_position === INJECTION_POSITION.ABSOLUTE) {
+            promptManager.log(`Skipping prompt ${source} because it is an absolute prompt`);
+            return;
+        }
+
         const index = target ? prompts.index(target) : prompts.index(source);
         const collection = new MessageCollection(source);
         collection.add(Message.fromPrompt(prompt));
@@ -1014,8 +1020,8 @@ async function populateChatCompletion(prompts, chatCompletion, { bias, quietProm
             acc.push(prompt.identifier);
             return acc;
         }, []);
-    const userAbsolutePrompts = prompts.collection
+    const absolutePrompts = prompts.collection
-        .filter((prompt) => false === prompt.system_prompt && prompt.injection_position === INJECTION_POSITION.ABSOLUTE)
+        .filter((prompt) => prompt.injection_position === INJECTION_POSITION.ABSOLUTE)
         .reduce((acc, prompt) => {
             acc.push(prompt);
             return acc;
@@ -1080,7 +1086,7 @@ async function populateChatCompletion(prompts, chatCompletion, { bias, quietProm
     }
 
     // Add in-chat injections
-    messages = populationInjectionPrompts(userAbsolutePrompts, messages);
+    messages = populationInjectionPrompts(absolutePrompts, messages);
 
     // Decide whether dialogue examples should always be added
     if (power_user.pin_examples) {
@@ -1217,6 +1223,18 @@ function preparePromptsForChatCompletion({ Scenario, charPersonality, name2, wor
 
     // Merge system prompts with prompt manager prompts
     systemPrompts.forEach(prompt => {
+        const collectionPrompt = prompts.get(prompt.identifier);
+
+        // Apply system prompt role/depth overrides if they set in the prompt manager
+        if (collectionPrompt) {
+            // In-Chat / Relative
+            prompt.injection_position = collectionPrompt.injection_position ?? prompt.injection_position;
+            // Depth for In-Chat
+            prompt.injection_depth = collectionPrompt.injection_depth ?? prompt.injection_depth;
+            // Role (system, user, assistant)
+            prompt.role = collectionPrompt.role ?? prompt.role;
+        }
+
         const newPrompt = promptManager.preparePrompt(prompt);
         const markerIndex = prompts.index(prompt.identifier);
 

public/scripts/tokenizers.js

@@ -30,6 +30,7 @@ export const tokenizers = {
     JAMBA: 14,
     QWEN2: 15,
     COMMAND_R: 16,
+    NEMO: 17,
     BEST_MATCH: 99,
 };
 
@@ -43,6 +44,7 @@ export const ENCODE_TOKENIZERS = [
     tokenizers.JAMBA,
     tokenizers.QWEN2,
     tokenizers.COMMAND_R,
+    tokenizers.NEMO,
     // uncomment when NovelAI releases Kayra and Clio weights, lol
     //tokenizers.NERD,
     //tokenizers.NERD2,
@@ -121,6 +123,11 @@ const TOKENIZER_URLS = {
         decode: '/api/tokenizers/command-r/decode',
         count: '/api/tokenizers/command-r/encode',
     },
+    [tokenizers.NEMO]: {
+        encode: '/api/tokenizers/nemo/encode',
+        decode: '/api/tokenizers/nemo/decode',
+        count: '/api/tokenizers/nemo/encode',
+    },
     [tokenizers.API_TEXTGENERATIONWEBUI]: {
         encode: '/api/tokenizers/remote/textgenerationwebui/encode',
         count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -535,6 +542,7 @@ export function getTokenizerModel() {
     const jambaTokenizer = 'jamba';
     const qwen2Tokenizer = 'qwen2';
     const commandRTokenizer = 'command-r';
+    const nemoTokenizer = 'nemo';
 
     // Assuming no one would use it for different models.. right?
     if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -628,6 +636,9 @@ export function getTokenizerModel() {
     }
 
     if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
+        if (oai_settings.mistralai_model.includes('nemo') || oai_settings.mistralai_model.includes('pixtral')) {
+            return nemoTokenizer;
+        }
         return mistralTokenizer;
     }
 

public/scripts/world-info.js

@@ -4134,10 +4134,10 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) {
 
         switch (entry.position) {
             case world_info_position.before:
-                WIBeforeEntries.unshift(substituteParams(content));
+                WIBeforeEntries.unshift(content);
                 break;
             case world_info_position.after:
-                WIAfterEntries.unshift(substituteParams(content));
+                WIAfterEntries.unshift(content);
                 break;
             case world_info_position.EMTop:
                 EMEntries.unshift(

src/endpoints/chats.js

@@ -92,8 +92,7 @@ function importOobaChat(userName, characterName, jsonData) {
         }
     }
 
-    const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
+    return chat.map(obj => JSON.stringify(obj)).join('\n');
-    return chatContent;
 }
 
 /**
@@ -121,8 +120,7 @@ function importAgnaiChat(userName, characterName, jsonData) {
         });
     }
 
-    const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
+    return chat.map(obj => JSON.stringify(obj)).join('\n');
-    return chatContent;
 }
 
 /**
@@ -159,6 +157,37 @@ function importCAIChat(userName, characterName, jsonData) {
     return newChats;
 }
 
+/**
+ * Flattens `msg` and `swipes` data from Chub Chat format.
+ * Only changes enough to make it compatible with the standard chat serialization format.
+ * @param {string} userName User name
+ * @param {string} characterName Character name
+ * @param {string[]} lines serialised JSONL data
+ * @returns {string} Converted data
+ */
+function flattenChubChat(userName, characterName, lines) {
+    function flattenSwipe(swipe) {
+        return swipe.message ? swipe.message : swipe;
+    }
+
+    function convert(line) {
+        const lineData = tryParse(line);
+        if (!lineData) return line;
+
+        if (lineData.mes && lineData.mes.message) {
+            lineData.mes = lineData?.mes.message;
+        }
+
+        if (lineData?.swipes && Array.isArray(lineData.swipes)) {
+            lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));
+        }
+
+        return JSON.stringify(lineData);
+    }
+
+    return (lines ?? []).map(convert).join('\n');
+}
+
 const router = express.Router();
 
 router.post('/save', jsonParser, function (request, response) {
@@ -273,7 +302,7 @@ router.post('/export', jsonParser, async function (request, response) {
     }
     try {
         // Short path for JSONL files
-        if (request.body.format == 'jsonl') {
+        if (request.body.format === 'jsonl') {
             try {
                 const rawFile = fs.readFileSync(filename, 'utf8');
                 const successMessage = {
@@ -283,8 +312,7 @@ router.post('/export', jsonParser, async function (request, response) {
 
                 console.log(`Chat exported as ${exportfilename}`);
                 return response.status(200).json(successMessage);
-            }
+            } catch (err) {
-            catch (err) {
                 console.error(err);
                 const errorMessage = {
                     message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
@@ -319,8 +347,7 @@ router.post('/export', jsonParser, async function (request, response) {
             console.log(`Chat exported as ${exportfilename}`);
             return response.status(200).json(successMessage);
         });
-    }
+    } catch (err) {
-    catch (err) {
         console.log('chat export failed.');
         console.log(err);
         return response.sendStatus(400);
@@ -396,20 +423,36 @@ router.post('/import', urlencodedParser, function (request, response) {
         }
 
         if (format === 'jsonl') {
-            const line = data.split('\n')[0];
+            let lines = data.split('\n');
+            const header = lines[0];
 
-            const jsonData = JSON.parse(line);
+            const jsonData = JSON.parse(header);
 
-            if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
+            if (!(jsonData.user_name !== undefined || jsonData.name !== undefined)) {
-                const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
-                const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
-                fs.copyFileSync(pathToUpload, filePath);
-                fs.unlinkSync(pathToUpload);
-                response.send({ res: true });
-            } else {
                 console.log('Incorrect chat format .jsonl');
                 return response.send({ error: true });
             }
+
+            // Do a tiny bit of work to import Chub Chat data
+            // Processing the entire file is so fast that it's not worth checking if it's a Chub chat first
+            let flattenedChat;
+            try {
+                // flattening is unlikely to break, but it's not worth failing to
+                // import normal chats in an attempt to import a Chub chat
+                flattenedChat = flattenChubChat(userName, characterName, lines);
+            } catch (error) {
+                console.warn('Failed to flatten Chub Chat data: ', error);
+            }
+
+            const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
+            const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
+            if (flattenedChat !== data) {
+                writeFileAtomicSync(filePath, flattenedChat, 'utf8');
+            } else {
+                fs.copyFileSync(pathToUpload, filePath);
+            }
+            fs.unlinkSync(pathToUpload);
+            response.send({ res: true });
         }
     } catch (error) {
         console.error(error);

src/endpoints/tokenizers.js

@@ -221,6 +221,7 @@ const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
 const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
 const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
 const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
+const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
 
 const sentencepieceTokenizers = [
     'llama',
@@ -418,6 +419,10 @@ function getTokenizerModel(requestModel) {
         return 'command-r';
     }
 
+    if (requestModel.includes('nemo')) {
+        return 'nemo';
+    }
+
     // default
     return 'gpt-3.5-turbo';
 }
@@ -645,6 +650,7 @@ router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(clau
 router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
 router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
 router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
+router.post('/nemo/encode', jsonParser, createWebTokenizerEncodingHandler(nemoTokenizer));
 router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
 router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
 router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@@ -657,6 +663,7 @@ router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(clau
 router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
 router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
 router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
+router.post('/nemo/decode', jsonParser, createWebTokenizerDecodingHandler(nemoTokenizer));
 
 router.post('/openai/encode', jsonParser, async function (req, res) {
     try {
@@ -707,6 +714,11 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
             return handler(req, res);
         }
 
+        if (queryModel.includes('nemo')) {
+            const handler = createWebTokenizerEncodingHandler(nemoTokenizer);
+            return handler(req, res);
+        }
+
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenEncodingHandler(model);
         return handler(req, res);
@@ -765,6 +777,11 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
             return handler(req, res);
         }
 
+        if (queryModel.includes('nemo')) {
+            const handler = createWebTokenizerDecodingHandler(nemoTokenizer);
+            return handler(req, res);
+        }
+
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenDecodingHandler(model);
         return handler(req, res);
@@ -835,6 +852,13 @@ router.post('/openai/count', jsonParser, async function (req, res) {
             return res.send({ 'token_count': num_tokens });
         }
 
+        if (model === 'nemo') {
+            const instance = await nemoTokenizer.get();
+            if (!instance) throw new Error('Failed to load the Nemo tokenizer');
+            num_tokens = countWebTokenizerTokens(instance, req.body);
+            return res.send({ 'token_count': num_tokens });
+        }
+
         const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
         const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
         const tokensPadding = 3;

start.sh

@@ -26,7 +26,7 @@ fi
 
 echo "Installing Node Modules..."
 export NODE_ENV=production
-npm i --no-audit --no-fund --quiet --omit=dev
+npm i --no-audit --no-fund --loglevel=error --no-progress --omit=dev
 
 echo "Entering SillyTavern..."
 node "server.js" "$@"