From 2f5e40691c9d39be97a9fbbde71b78f2d84e7998 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 5 Sep 2024 20:55:06 +0300
Subject: [PATCH 1/4] Implement more smart textarea autosizes

---
 public/index.html                             |  6 ++--
 .../extensions/stable-diffusion/index.js      | 28 +++++++++++--------
 .../extensions/stable-diffusion/settings.html |  8 +++---
 .../scripts/extensions/token-counter/index.js |  6 ++--
 .../extensions/token-counter/style.css        |  5 ++++
 public/scripts/group-chats.js                 |  7 +++--
 6 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/public/index.html b/public/index.html
index 9a495629b..ea16adabc 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1820,12 +1820,12 @@
                                                 <span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
                                                 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_prefill" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
                                             </div>
-                                            <textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
+                                            <textarea id="claude_assistant_prefill" class="text_pole textarea_compact autoSetHeight" name="assistant_prefill" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
                                             <div class="flex-container alignItemsCenter">
                                                 <span id="claude_assistant_impersonation_text" data-i18n="Assistant Impersonation Prefill">Assistant Impersonation Prefill</span>
                                                 <i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_impersonation" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
                                             </div>
-                                            <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact" name="assistant_impersonation" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
+                                            <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
                                         </div>
                                         <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
                                             <input id="claude_use_sysprompt" type="checkbox" />
@@ -1845,7 +1845,7 @@
                                                     <div class="fa-solid fa-clock-rotate-left"></div>
                                                 </div>
                                             </div>
-                                            <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
+                                            <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact autoSetHeight" rows="2" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
                                         </div>
                                     </div>
                                 </div>
diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js
index 2f13329a2..6f33b5bf2 100644
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -724,7 +724,7 @@ function onChatChanged() {
 }
 
 async function adjustElementScrollHeight() {
-    if (!$('.sd_settings').is(':visible')) {
+    if (CSS.supports('field-sizing', 'content') || !$('.sd_settings').is(':visible')) {
         return;
     }
 
@@ -737,17 +737,19 @@ async function adjustElementScrollHeight() {
 async function onCharacterPromptInput() {
     const key = getCharaFilename(this_chid);
     extension_settings.sd.character_prompts[key] = $('#sd_character_prompt').val();
-    await resetScrollHeight($(this));
     saveSettingsDebounced();
     writePromptFieldsDebounced(this_chid);
+    if (CSS.supports('field-sizing', 'content')) return;
+    await resetScrollHeight($(this));
 }
 
 async function onCharacterNegativePromptInput() {
     const key = getCharaFilename(this_chid);
     extension_settings.sd.character_negative_prompts[key] = $('#sd_character_negative_prompt').val();
-    await resetScrollHeight($(this));
     saveSettingsDebounced();
     writePromptFieldsDebounced(this_chid);
+    if (CSS.supports('field-sizing', 'content')) return;
+    await resetScrollHeight($(this));
 }
 
 function getCharacterPrefix() {
@@ -856,14 +858,16 @@ function onStepsInput() {
 
 async function onPromptPrefixInput() {
     extension_settings.sd.prompt_prefix = $('#sd_prompt_prefix').val();
-    await resetScrollHeight($(this));
     saveSettingsDebounced();
+    if (CSS.supports('field-sizing', 'content')) return;
+    await resetScrollHeight($(this));
 }
 
 async function onNegativePromptInput() {
     extension_settings.sd.negative_prompt = $('#sd_negative_prompt').val();
-    await resetScrollHeight($(this));
     saveSettingsDebounced();
+    if (CSS.supports('field-sizing', 'content')) return;
+    await resetScrollHeight($(this));
 }
 
 function onSamplerChange() {
@@ -3911,12 +3915,14 @@ jQuery(async () => {
     $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
     $('#sd_huggingface_model_id').on('input', onHFModelInput);
 
-    $('.sd_settings .inline-drawer-toggle').on('click', function () {
-        initScrollHeight($('#sd_prompt_prefix'));
-        initScrollHeight($('#sd_negative_prompt'));
-        initScrollHeight($('#sd_character_prompt'));
-        initScrollHeight($('#sd_character_negative_prompt'));
-    });
+    if (!CSS.supports('field-sizing', 'content')) {
+        $('.sd_settings .inline-drawer-toggle').on('click', function () {
+            initScrollHeight($('#sd_prompt_prefix'));
+            initScrollHeight($('#sd_negative_prompt'));
+            initScrollHeight($('#sd_character_prompt'));
+            initScrollHeight($('#sd_character_negative_prompt'));
+        });
+    }
 
     for (const [key, value] of Object.entries(resolutionOptions)) {
         const option = document.createElement('option');
diff --git a/public/scripts/extensions/stable-diffusion/settings.html b/public/scripts/extensions/stable-diffusion/settings.html
index 821b5f791..46c1fd2d7 100644
--- a/public/scripts/extensions/stable-diffusion/settings.html
+++ b/public/scripts/extensions/stable-diffusion/settings.html
@@ -408,16 +408,16 @@
                 </div>
             </div>
             <label for="sd_prompt_prefix" data-i18n="Common prompt prefix">Common prompt prefix</label>
-            <textarea id="sd_prompt_prefix" class="text_pole textarea_compact" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
+            <textarea id="sd_prompt_prefix" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
             <label for="sd_negative_prompt" data-i18n="Negative common prompt prefix">Negative common prompt prefix</label>
-            <textarea id="sd_negative_prompt" class="text_pole textarea_compact"></textarea>
+            <textarea id="sd_negative_prompt" class="text_pole textarea_compact autoSetHeight"></textarea>
             <div id="sd_character_prompt_block">
                 <label for="sd_character_prompt" data-i18n="Character-specific prompt prefix">Character-specific prompt prefix</label>
                 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
-                <textarea id="sd_character_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
+                <textarea id="sd_character_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
                 <label for="sd_character_negative_prompt" data-i18n="Character-specific negative prompt prefix">Character-specific negative prompt prefix</label>
                 <small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
-                <textarea id="sd_character_negative_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
+                <textarea id="sd_character_negative_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
                 <label for="sd_character_prompt_share" class="checkbox_label flexWrap marginTop5">
                     <input id="sd_character_prompt_share" type="checkbox" />
                     <span data-i18n="Shareable">
diff --git a/public/scripts/extensions/token-counter/index.js b/public/scripts/extensions/token-counter/index.js
index cca4c5a51..2db5d1e53 100644
--- a/public/scripts/extensions/token-counter/index.js
+++ b/public/scripts/extensions/token-counter/index.js
@@ -59,8 +59,10 @@ async function doTokenCounter() {
             $('#tokenized_chunks_display').text('—');
         }
 
-        await resetScrollHeight($('#token_counter_textarea'));
-        await resetScrollHeight($('#token_counter_ids'));
+        if (!CSS.supports('field-sizing', 'content')) {
+            await resetScrollHeight($('#token_counter_textarea'));
+            await resetScrollHeight($('#token_counter_ids'));
+        }
     }, debounce_timeout.relaxed);
     dialog.find('#token_counter_textarea').on('input', () => countDebounced());
 
diff --git a/public/scripts/extensions/token-counter/style.css b/public/scripts/extensions/token-counter/style.css
index bb3294412..e9aec1c06 100644
--- a/public/scripts/extensions/token-counter/style.css
+++ b/public/scripts/extensions/token-counter/style.css
@@ -4,3 +4,8 @@
     padding: 2px;
     display: inline-block;
 }
+
+#token_counter_textarea,
+#token_counter_ids {
+    field-sizing: content;
+}
diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js
index 8b1c60f80..3e0b6955a 100644
--- a/public/scripts/group-chats.js
+++ b/public/scripts/group-chats.js
@@ -1414,8 +1414,11 @@ function toggleHiddenControls(group, generationMode = null) {
     const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);
     $('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);
     $('#rm_group_generation_mode_join_suffix').parent().toggle(isJoin);
-    initScrollHeight($('#rm_group_generation_mode_join_prefix'));
-    initScrollHeight($('#rm_group_generation_mode_join_suffix'));
+
+    if (!CSS.supports('field-sizing', 'content')) {
+        initScrollHeight($('#rm_group_generation_mode_join_prefix'));
+        initScrollHeight($('#rm_group_generation_mode_join_suffix'));
+    }
 }
 
 function select_group_chats(groupId, skipAnimation) {

From 81251b073aeb2f97f8e888f712b34a72f31915b5 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 6 Sep 2024 16:28:34 +0000
Subject: [PATCH 2/4] Implement downloadable tokenizers Closes #2574, #2754

---
 default/config.yaml              |   3 +
 public/index.html                |   2 +
 public/scripts/textgen-models.js |   7 ++
 public/scripts/tokenizers.js     |  30 +++++++
 src/endpoints/tokenizers.js      | 140 ++++++++++++++++++++++++++++---
 src/util.js                      |  15 ++++
 6 files changed, 187 insertions(+), 10 deletions(-)

diff --git a/default/config.yaml b/default/config.yaml
index 2d760b39f..58db04a49 100644
--- a/default/config.yaml
+++ b/default/config.yaml
@@ -95,6 +95,9 @@ requestOverrides: []
 enableExtensions: true
 # Automatically update extensions when a release version changes
 enableExtensionsAutoUpdate: true
+# Additional model tokenizers can be downloaded on demand.
+# Disabling will fallback to another locally available tokenizer.
+enableDownloadableTokenizers: true
 # Extension settings
 extras:
   # Disables automatic model download from HuggingFace
diff --git a/public/index.html b/public/index.html
index f7861290b..010cec225 100644
--- a/public/index.html
+++ b/public/index.html
@@ -3283,6 +3283,8 @@
                                     <option value="12">Llama 3</option>
                                     <option value="13">Gemma / Gemini</option>
                                     <option value="14">Jamba</option>
+                                    <option value="15">Qwen2</option>
+                                    <option value="16">Command-R</option>
                                     <option value="4">NerdStash (NovelAI Clio)</option>
                                     <option value="5">NerdStash v2 (NovelAI Kayra)</option>
                                     <option value="7">Mistral</option>
diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js
index 35090b810..d35f2b724 100644
--- a/public/scripts/textgen-models.js
+++ b/public/scripts/textgen-models.js
@@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
 export function getCurrentOpenRouterModelTokenizer() {
     const modelId = textgen_settings.openrouter_model;
     const model = openRouterModels.find(x => x.id === modelId);
+    if (modelId?.includes('jamba')) {
+        return tokenizers.JAMBA;
+    }
     switch (model?.architecture?.tokenizer) {
         case 'Llama2':
             return tokenizers.LLAMA;
@@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
             return tokenizers.GEMMA;
         case 'Claude':
             return tokenizers.CLAUDE;
+        case 'Cohere':
+            return tokenizers.COMMAND_R;
+        case 'Qwen':
+            return tokenizers.QWEN2;
         default:
             return tokenizers.OPENAI;
     }
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index e06fab97b..61884fe68 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -28,6 +28,8 @@ export const tokenizers = {
     LLAMA3: 12,
     GEMMA: 13,
     JAMBA: 14,
+    QWEN2: 15,
+    COMMAND_R: 16,
     BEST_MATCH: 99,
 };
 
@@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
         decode: '/api/tokenizers/jamba/decode',
         count: '/api/tokenizers/jamba/encode',
     },
+    [tokenizers.QWEN2]: {
+        encode: '/api/tokenizers/qwen2/encode',
+        decode: '/api/tokenizers/qwen2/decode',
+        count: '/api/tokenizers/qwen2/encode',
+    },
+    [tokenizers.COMMAND_R]: {
+        encode: '/api/tokenizers/command-r/encode',
+        decode: '/api/tokenizers/command-r/decode',
+        count: '/api/tokenizers/command-r/encode',
+    },
     [tokenizers.API_TEXTGENERATIONWEBUI]: {
         encode: '/api/tokenizers/remote/textgenerationwebui/encode',
         count: '/api/tokenizers/remote/textgenerationwebui/encode',
@@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
             if (model.includes('jamba')) {
                 return tokenizers.JAMBA;
             }
+            if (model.includes('command-r')) {
+                return tokenizers.COMMAND_R;
+            }
+            if (model.includes('qwen2')) {
+                return tokenizers.QWEN2;
+            }
         }
 
         return tokenizers.LLAMA;
@@ -511,6 +529,8 @@ export function getTokenizerModel() {
     const yiTokenizer = 'yi';
     const gemmaTokenizer = 'gemma';
     const jambaTokenizer = 'jamba';
+    const qwen2Tokenizer = 'qwen2';
+    const commandRTokenizer = 'command-r';
 
     // Assuming no one would use it for different models.. right?
     if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@@ -558,6 +578,12 @@ export function getTokenizerModel() {
         else if (model?.architecture?.tokenizer === 'Gemini') {
             return gemmaTokenizer;
         }
+        else if (model?.architecture?.tokenizer === 'Qwen') {
+            return qwen2Tokenizer;
+        }
+        else if (model?.architecture?.tokenizer === 'Cohere') {
+            return commandRTokenizer;
+        }
         else if (oai_settings.openrouter_model.includes('gpt-4o')) {
             return gpt4oTokenizer;
         }
@@ -581,6 +607,10 @@ export function getTokenizerModel() {
         }
     }
 
+    if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
+        return commandRTokenizer;
+    }
+
     if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
         return gemmaTokenizer;
     }
diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js
index 99ae527a6..cd42348c9 100644
--- a/src/endpoints/tokenizers.js
+++ b/src/endpoints/tokenizers.js
@@ -4,13 +4,12 @@ const express = require('express');
 const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
 const tiktoken = require('tiktoken');
 const { Tokenizer } = require('@agnai/web-tokenizers');
-const { convertClaudePrompt, convertGooglePrompt } = require('../prompt-converters');
-const { readSecret, SECRET_KEYS } = require('./secrets');
+const { convertClaudePrompt } = require('../prompt-converters');
 const { TEXTGEN_TYPES } = require('../constants');
 const { jsonParser } = require('../express-common');
 const { setAdditionalHeaders } = require('../additional-headers');
-
-const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
+const { getConfigValue, isValidUrl } = require('../util');
+const writeFileAtomicSync = require('write-file-atomic').sync;
 
 /**
  * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
@@ -53,6 +52,65 @@ const TEXT_COMPLETION_MODELS = [
 ];
 
 const CHARS_PER_TOKEN = 3.35;
+const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true);
+
+/**
+ * Gets a path to the tokenizer model. Downloads the model if it's a URL.
+ * @param {string} model Model URL or path
+ * @param {string|undefined} fallbackModel Fallback model path\
+ * @returns {Promise<string>} Path to the tokenizer model
+ */
+async function getPathToTokenizer(model, fallbackModel) {
+    if (!isValidUrl(model)) {
+        return model;
+    }
+
+    try {
+        const url = new URL(model);
+
+        if (!['https:', 'http:'].includes(url.protocol)) {
+            throw new Error('Invalid URL protocol');
+        }
+
+        const fileName = url.pathname.split('/').pop();
+
+        if (!fileName) {
+            throw new Error('Failed to extract the file name from the URL');
+        }
+
+        const CACHE_PATH = path.join(global.DATA_ROOT, '_cache');
+        if (!fs.existsSync(CACHE_PATH)) {
+            fs.mkdirSync(CACHE_PATH, { recursive: true });
+        }
+
+        const cachedFile = path.join(CACHE_PATH, fileName);
+        if (fs.existsSync(cachedFile)) {
+            return cachedFile;
+        }
+
+        if (!IS_DOWNLOAD_ALLOWED) {
+            throw new Error('Downloading tokenizers is disabled, the model is not cached');
+        }
+
+        console.log('Downloading tokenizer model:', model);
+        const response = await fetch(model);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch the model: ${response.status} ${response.statusText}`);
+        }
+
+        const arrayBuffer = await response.arrayBuffer();
+        writeFileAtomicSync(cachedFile, Buffer.from(arrayBuffer));
+        return cachedFile;
+    } catch (error) {
+        const getLastSegment = str => str?.split('/')?.pop() || '';
+        if (fallbackModel) {
+            console.log(`Could not get a tokenizer from ${getLastSegment(model)}. Reason: ${error.message}. Using a fallback model: ${getLastSegment(fallbackModel)}.`);
+            return fallbackModel;
+        }
+
+        throw new Error(`Failed to instantiate a tokenizer and fallback is not provided. Reason: ${error.message}`);
+    }
+}
 
 /**
  * Sentencepiece tokenizer for tokenizing text.
@@ -66,13 +124,19 @@ class SentencePieceTokenizer {
      * @type {string} Path to the tokenizer model
      */
     #model;
+    /**
+     * @type {string|undefined} Path to the fallback model
+     */
+    #fallbackModel;
 
     /**
      * Creates a new Sentencepiece tokenizer.
      * @param {string} model Path to the tokenizer model
+     * @param {string} [fallbackModel] Path to the fallback model
      */
-    constructor(model) {
+    constructor(model, fallbackModel) {
         this.#model = model;
+        this.#fallbackModel = fallbackModel;
     }
 
     /**
@@ -85,9 +149,10 @@ class SentencePieceTokenizer {
         }
 
         try {
+            const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
             this.#instance = new SentencePieceProcessor();
-            await this.#instance.load(this.#model);
-            console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
+            await this.#instance.load(pathToModel);
+            console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
             return this.#instance;
         } catch (error) {
             console.error('Sentencepiece tokenizer failed to load: ' + this.#model, error);
@@ -108,13 +173,19 @@ class WebTokenizer {
      * @type {string} Path to the tokenizer model
      */
     #model;
+    /**
+     * @type {string|undefined} Path to the fallback model
+     */
+    #fallbackModel;
 
     /**
      * Creates a new Web tokenizer.
      * @param {string} model Path to the tokenizer model
+     * @param {string} [fallbackModel] Path to the fallback model
      */
-    constructor(model) {
+    constructor(model, fallbackModel) {
         this.#model = model;
+        this.#fallbackModel = fallbackModel;
     }
 
     /**
@@ -127,9 +198,10 @@ class WebTokenizer {
         }
 
         try {
-            const arrayBuffer = fs.readFileSync(this.#model).buffer;
+            const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
+            const arrayBuffer = fs.readFileSync(pathToModel).buffer;
             this.#instance = await Tokenizer.fromJSON(arrayBuffer);
-            console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
+            console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
             return this.#instance;
         } catch (error) {
             console.error('Web tokenizer failed to load: ' + this.#model, error);
@@ -147,6 +219,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
 const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
 const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
 const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
+const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
+const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
 
 const sentencepieceTokenizers = [
     'llama',
@@ -332,6 +406,14 @@ function getTokenizerModel(requestModel) {
         return 'jamba';
     }
 
+    if (requestModel.includes('qwen2')) {
+        return 'qwen2';
+    }
+
+    if (requestModel.includes('command-r')) {
+        return 'command-r';
+    }
+
     // default
     return 'gpt-3.5-turbo';
 }
@@ -557,6 +639,8 @@ router.post('/jamba/encode', jsonParser, createSentencepieceEncodingHandler(spp_
 router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));
 router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));
 router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
+router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
+router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
 router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
 router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
 router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@@ -567,6 +651,8 @@ router.post('/jamba/decode', jsonParser, createSentencepieceDecodingHandler(spp_
 router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));
 router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));
 router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
+router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
+router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
 
 router.post('/openai/encode', jsonParser, async function (req, res) {
     try {
@@ -607,6 +693,16 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
             return handler(req, res);
         }
 
+        if (queryModel.includes('qwen2')) {
+            const handler = createWebTokenizerEncodingHandler(qwen2Tokenizer);
+            return handler(req, res);
+        }
+
+        if (queryModel.includes('command-r')) {
+            const handler = createWebTokenizerEncodingHandler(commandTokenizer);
+            return handler(req, res);
+        }
+
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenEncodingHandler(model);
         return handler(req, res);
@@ -655,6 +751,16 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
             return handler(req, res);
         }
 
+        if (queryModel.includes('qwen2')) {
+            const handler = createWebTokenizerDecodingHandler(qwen2Tokenizer);
+            return handler(req, res);
+        }
+
+        if (queryModel.includes('command-r')) {
+            const handler = createWebTokenizerDecodingHandler(commandTokenizer);
+            return handler(req, res);
+        }
+
         const model = getTokenizerModel(queryModel);
         const handler = createTiktokenDecodingHandler(model);
         return handler(req, res);
@@ -711,6 +817,20 @@ router.post('/openai/count', jsonParser, async function (req, res) {
             return res.send({ 'token_count': num_tokens });
         }
 
+        if (model === 'qwen2') {
+            const instance = await qwen2Tokenizer.get();
+            if (!instance) throw new Error('Failed to load the Qwen2 tokenizer');
+            num_tokens = countWebTokenizerTokens(instance, req.body);
+            return res.send({ 'token_count': num_tokens });
+        }
+
+        if (model === 'command-r') {
+            const instance = await commandTokenizer.get();
+            if (!instance) throw new Error('Failed to load the Command-R tokenizer');
+            num_tokens = countWebTokenizerTokens(instance, req.body);
+            return res.send({ 'token_count': num_tokens });
+        }
+
         const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
         const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
         const tokensPadding = 3;
diff --git a/src/util.js b/src/util.js
index 609f8cecb..b513cba3f 100644
--- a/src/util.js
+++ b/src/util.js
@@ -647,6 +647,20 @@ function getSeparator(n) {
     return '='.repeat(n);
 }
 
+/**
+ * Checks if the string is a valid URL.
+ * @param {string} url String to check
+ * @returns {boolean} If the URL is valid
+ */
+function isValidUrl(url) {
+    try {
+        new URL(url);
+        return true;
+    } catch (error) {
+        return false;
+    }
+}
+
 module.exports = {
     getConfig,
     getConfigValue,
@@ -676,4 +690,5 @@ module.exports = {
     makeHttp2Request,
     removeColorFormatting,
     getSeparator,
+    isValidUrl,
 };

From 97a2c4b2c739bd7e8c900e6f968b1a49ad55525d Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 7 Sep 2024 21:40:21 +0300
Subject: [PATCH 3/4] Add quiet arg to /context

---
 public/script.js                | 20 +++++++++++++++-----
 public/scripts/instruct-mode.js |  7 ++++---
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/public/script.js b/public/script.js
index 099651236..22aa140f3 100644
--- a/public/script.js
+++ b/public/script.js
@@ -8503,22 +8503,23 @@ for (const chatCompletionSource of Object.values(chat_completion_sources)) {
     };
 }
 
-async function selectContextCallback(_, name) {
+async function selectContextCallback(args, name) {
     if (!name) {
         return power_user.context.preset;
     }
 
+    const quiet = isTrueBoolean(args?.quiet);
     const contextNames = context_presets.map(preset => preset.name);
     const fuse = new Fuse(contextNames);
     const result = fuse.search(name);
 
     if (result.length === 0) {
-        toastr.warning(`Context template "${name}" not found`);
+        !quiet && toastr.warning(`Context template "${name}" not found`);
         return '';
     }
 
     const foundName = result[0].item;
-    selectContextPreset(foundName);
+    selectContextPreset(foundName, quiet);
     return foundName;
 }
 
@@ -8527,16 +8528,16 @@ async function selectInstructCallback(args, name) {
         return power_user.instruct.preset;
     }
 
+    const quiet = isTrueBoolean(args?.quiet);
     const instructNames = instruct_presets.map(preset => preset.name);
     const fuse = new Fuse(instructNames);
     const result = fuse.search(name);
 
     if (result.length === 0) {
-        toastr.warning(`Instruct template "${name}" not found`);
+        !quiet && toastr.warning(`Instruct template "${name}" not found`);
         return '';
     }
 
-    const quiet = isTrueBoolean(args?.quiet);
     const foundName = result[0].item;
     selectInstructPreset(foundName, quiet);
     return foundName;
@@ -9282,6 +9283,15 @@ jQuery(async function () {
         name: 'context',
         callback: selectContextCallback,
         returns: 'template name',
+        namedArgumentList: [
+            SlashCommandNamedArgument.fromProps({
+                name: 'quiet',
+                description: 'Suppress the toast message on template change',
+                typeList: [ARGUMENT_TYPE.BOOLEAN],
+                defaultValue: 'false',
+                enumList: commonEnumProviders.boolean('trueFalse')(),
+            }),
+        ],
         unnamedArgumentList: [
             SlashCommandArgument.fromProps({
                 description: 'context template name',
diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js
index 520f1c4f1..3319ebd23 100644
--- a/public/scripts/instruct-mode.js
+++ b/public/scripts/instruct-mode.js
@@ -130,19 +130,20 @@ function highlightDefaultPreset() {
 /**
  * Select context template if not already selected.
  * @param {string} preset Preset name.
+ * @param {boolean} quiet Suppress info message.
  */
-export function selectContextPreset(preset) {
+export function selectContextPreset(preset, quiet) {
     // If context template is not already selected, select it
     if (preset !== power_user.context.preset) {
         $('#context_presets').val(preset).trigger('change');
-        toastr.info(`Context Template: preset "${preset}" auto-selected`);
+        !quiet && toastr.info(`Context Template: preset "${preset}" auto-selected`);
     }
 
     // If instruct mode is disabled, enable it, except for default context template
     if (!power_user.instruct.enabled && preset !== power_user.default_context) {
         power_user.instruct.enabled = true;
         $('#instruct_enabled').prop('checked', true).trigger('change');
-        toastr.info('Instruct Mode enabled');
+        !quiet && toastr.info('Instruct Mode enabled');
     }
 
     saveSettingsDebounced();

From 5e0a61bab2ada59c22276bea6f16db6c5a2509e9 Mon Sep 17 00:00:00 2001
From: Wolfsblvt <wolfsblvt@gmail.com>
Date: Sat, 7 Sep 2024 21:33:36 +0200
Subject: [PATCH 4/4] Half-fix for scrollbar white corner on Chrome

---
 public/style.css | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/public/style.css b/public/style.css
index 68a62dfbb..91726dd63 100644
--- a/public/style.css
+++ b/public/style.css
@@ -182,6 +182,10 @@ body.movingUI ::-webkit-scrollbar-thumb:vertical {
     min-width: 40px;
 }
 
+::-webkit-scrollbar-corner {
+    background-color: transparent;
+}
+
 table.responsiveTable {
     width: 100%;
     margin: 10px 0;
@@ -4675,10 +4679,6 @@ body:not(.sd) .mes_img_swipes {
     width: 100%;
 }
 
-.img_enlarged_holder::-webkit-scrollbar-corner {
-    background-color: transparent;
-}
-
 .img_enlarged_container pre code {
     position: relative;
     display: block;