Merge e77b16662c into be7eb8b2b5

style: `.mes_block { overflow-y: clip; }`

public/index.html

@@ -2237,6 +2237,7 @@
                                     </a>
                                 </div>
                                 <h4 data-i18n="Tabby API key">Tabby API key</h4>
+                                <small data-i18n="Access TabbyAPI Admin">To access TabbyAPI's model switching, enter an admin key.</small>
                                 <div class="flex-container">
                                     <input id="api_key_tabby" name="api_key_tabby" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
                                     <div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_tabby">
@@ -2250,6 +2251,14 @@
                                     <small data-i18n="Example: 127.0.0.1:5000">Example: http://127.0.0.1:5000</small>
                                     <input id="tabby_api_url_text" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="tabby">
                                 </div>
+                                <div id="tabby_api_model_selection">
+                                    <h4 data-i18n="Tabby API Model">Tabby API Model</h4>
+                                    <select id="tabby_api_model">
+                                        <option data-i18n="-- Connect to the API --">
+                                            -- Connect to the API --
+                                        </option>
+                                    </select>
+                                </div>
                             </div>
                             <div data-tg-type="koboldcpp">
                                 <div class="flex-container flexFlowColumn">

public/script.js

@@ -22,7 +22,7 @@ import {
     parseTabbyLogprobs,
 } from './scripts/textgen-settings.js';
 
-const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER } = textgen_types;
+const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER , TABBY } = textgen_types;
 
 import {
     world_info,
@@ -1112,6 +1112,9 @@ async function getStatusTextgen() {
         } else if (textgen_settings.type === APHRODITE) {
             loadAphroditeModels(data?.data);
             online_status = textgen_settings.aphrodite_model;
+        } else if (textgen_settings.type === TABBY) {
+            loadTabbyApiModels(data?.result, data?.data);
+            online_status = textgen_settings.tabby_api_model;
         } else {
             online_status = data?.result;
         }
@@ -9171,11 +9174,35 @@ jQuery(async function () {
             }
         }
 
+        if (SECRET_KEYS.TABBY.length) {
+            const endpoint = getTextGenServer();
+
+            if (!endpoint) {
+                console.warn('No endpoint for key check');
+                online_status = 'no_connection';
+            }
+
+            const response = await fetch('/api/backends/text-completions/tabbyapi/verify-key', {
+                method: 'POST',
+                headers: getRequestHeaders(),
+                body: JSON.stringify({
+                    api_server: endpoint,
+                }),
+            });
+
+            const data = await response.json();
+            if (data?.isAdminKey) {
+                $('#tabby_api_model_selection').show();
+            } else {
+                $('#tabby_api_model_selection').hide();
+            }
+        }
+
         validateTextGenUrl();
         startStatusLoading();
         main_api = 'textgenerationwebui';
         saveSettingsDebounced();
-        getStatusTextgen();
+        await getStatusTextgen();
     });
 
     $('#api_button_novel').on('click', async function (e) {

public/scripts/openai.js

@@ -3250,7 +3250,8 @@ async function onExportPresetClick() {
     delete preset.proxy_password;
 
     const presetJsonString = JSON.stringify(preset, null, 4);
-    download(presetJsonString, oai_settings.preset_settings_openai, 'application/json');
+    const presetFileName = `${oai_settings.preset_settings_openai}.json`;
+    download(presetJsonString, presetFileName, 'application/json');
 }
 
 async function onLogitBiasPresetImportFileChange(e) {
@@ -3298,7 +3299,8 @@ function onLogitBiasPresetExportClick() {
     }
 
     const presetJsonString = JSON.stringify(oai_settings.bias_presets[oai_settings.bias_preset_selected], null, 4);
-    download(presetJsonString, oai_settings.bias_preset_selected, 'application/json');
+    const presetFileName = `${oai_settings.bias_preset_selected}.json`;
+    download(presetJsonString, presetFileName, 'application/json');
 }
 
 async function onDeletePresetClick() {

public/scripts/preset-manager.js

@@ -310,6 +310,7 @@ class PresetManager {
             'togetherai_model',
             'ollama_model',
             'aphrodite_model',
+            'tabby_api_model',
             'server_urls',
             'type',
             'custom_model',

public/scripts/secrets.js

@@ -5,6 +5,7 @@ export const SECRET_KEYS = {
     MANCER: 'api_key_mancer',
     APHRODITE: 'api_key_aphrodite',
     TABBY: 'api_key_tabby',
+    TABBY_ADMIN: 'api_key_admin_tabby',
     OPENAI: 'api_key_openai',
     NOVEL: 'api_key_novel',
     CLAUDE: 'api_key_claude',

public/scripts/textgen-models.js

@@ -1,13 +1,24 @@
 import { isMobile } from './RossAscends-mods.js';
-import { amount_gen, callPopup, eventSource, event_types, getRequestHeaders, max_context, setGenerationParamsFromPreset } from '../script.js';
+import {
+    amount_gen,
+    callPopup,
+    eventSource,
+    event_types,
+    getRequestHeaders,
+    max_context,
+    setGenerationParamsFromPreset,
+    token, settings,
+} from '../script.js';
 import { textgenerationwebui_settings as textgen_settings, textgen_types } from './textgen-settings.js';
 import { tokenizers } from './tokenizers.js';
+import { findSecret, readSecretState, SECRET_KEYS } from './secrets.js';
 
 let mancerModels = [];
 let togetherModels = [];
 let infermaticAIModels = [];
 let dreamGenModels = [];
 let aphroditeModels = [];
+let tabbyApiModels = [];
 export let openRouterModels = [];
 
 export async function loadOllamaModels(data) {
@@ -178,6 +189,32 @@ export async function loadAphroditeModels(data) {
     }
 }
 
+export async function loadTabbyApiModels(active, data) {
+    if (!Array.isArray(data)) {
+        console.error('Invalid Tabby API models data', data);
+        return;
+    }
+
+    tabbyApiModels = data;
+
+    if (textgen_settings.tabby_api_model !== active) {
+        textgen_settings.tabby_api_model = active;
+    }
+
+    if (!data.find(x => x.id === textgen_settings.tabby_api_model)) {
+        textgen_settings.tabby_api_model = data[0]?.id || '';
+    }
+
+    $('#tabby_api_model').empty();
+    for (const model of data) {
+        const option = document.createElement('option');
+        option.value = model.id;
+        option.text = model.id;
+        option.selected = model.id === textgen_settings.tabby_api_model;
+        $('#tabby_api_model').append(option);
+    }
+}
+
 function onMancerModelSelect() {
     const modelId = String($('#mancer_model').val());
     textgen_settings.mancer_model = modelId;
@@ -230,6 +267,37 @@ function onAphroditeModelSelect() {
     $('#api_button_textgenerationwebui').trigger('click');
 }
 
+async function onTabbyApiModelSelect() {
+    const modelId = String($('#tabby_api_model').val());
+    textgen_settings.tabby_api_model = modelId;
+
+    try {
+        const url = String($('#tabby_api_url_text').val()) + 'v1/model/load';
+        fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-CSRF-Token': token,
+                'x-admin-key': await findSecret(SECRET_KEYS.TABBY),
+            },
+            body: JSON.stringify({
+                name:  textgen_settings.tabby_api_model,
+            }),
+        }).then(
+            (value) => {
+                console.log(value); // Success!
+            },
+            (reason) => {
+                console.error('Could not load model: ' + reason); // Error!
+            },
+        );
+    } catch (err) {
+        console.error('Error setting model ', err);
+    }
+
+    $('#api_button_textgenerationwebui').trigger('click');
+}
+
 function getMancerModelTemplate(option) {
     const model = mancerModels.find(x => x.id === option?.element?.value);
 
@@ -324,6 +392,20 @@ function getAphroditeModelTemplate(option) {
     `));
 }
 
+function getTabbyApiModelTemplate(option) {
+    const model = tabbyApiModels.find(x => x.id === option?.element?.value);
+
+    if (!option.id || !model) {
+        return option.text;
+    }
+
+    return $((`
+        <div class="flex-container flexFlowColumn">
+            <div><strong>${DOMPurify.sanitize(model.id)}</strong></div>
+        </div>
+    `));
+}
+
 async function downloadOllamaModel() {
     try {
         const serverUrl = textgen_settings.server_urls[textgen_types.OLLAMA];
@@ -427,6 +509,7 @@ jQuery(function () {
     $('#openrouter_model').on('change', onOpenRouterModelSelect);
     $('#ollama_download_model').on('click', downloadOllamaModel);
     $('#aphrodite_model').on('change', onAphroditeModelSelect);
+    $('#tabby_api_model').on('change', onTabbyApiModelSelect);
 
     if (!isMobile()) {
         $('#mancer_model').select2({
@@ -477,5 +560,12 @@ jQuery(function () {
             width: '100%',
             templateResult: getAphroditeModelTemplate,
         });
+        $('#tabby_api_model').select2({
+            placeholder: 'Select a model',
+            searchInputPlaceholder: 'Search models...',
+            searchInputCssClass: 'text_pole',
+            width: '100%',
+            templateResult: getTabbyApiModelTemplate,
+        });
     }
 });

public/scripts/textgen-settings.js

@@ -147,6 +147,7 @@ const settings = {
     ollama_model: '',
     openrouter_model: 'openrouter/auto',
     aphrodite_model: '',
+    tabby_api_model: '',
     dreamgen_model: 'opus-v1-xl/text',
     legacy_api: false,
     sampler_order: KOBOLDCPP_ORDER,
@@ -1119,4 +1120,3 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
 
     return params;
 }
-

public/style.css

@@ -1000,6 +1000,7 @@ body .panelControlBar {
     padding-left: 10px;
     width: 100%;
     overflow-x: hidden;
+    overflow-y: clip;
 }
 
 .mes_text {

server.js

@@ -45,7 +45,6 @@ const {
     forwardFetchResponse,
 } = require('./src/util');
 const { ensureThumbnailCache } = require('./src/endpoints/thumbnails');
-const { loadTokenizers } = require('./src/endpoints/tokenizers');
 
 // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
 // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
@@ -543,22 +542,12 @@ const setupTasks = async function () {
     }
     console.log();
 
-    // TODO: do endpoint init functions depend on certain directories existing or not existing? They should be callable
-    // in any order for encapsulation reasons, but right now it's unknown if that would break anything.
-    await userModule.initUserStorage(dataRoot);
-
-    if (listen && !basicAuthMode && enableAccounts) {
-        await userModule.checkAccountsProtection();
-    }
-
-    await settingsEndpoint.init();
-    const directories = await userModule.ensurePublicDirectoriesExist();
-    await userModule.migrateUserData();
+    const directories = await userModule.getUserDirectoriesList();
     await contentManager.checkForNewContent(directories);
     await ensureThumbnailCache();
     cleanUploads();
 
-    await loadTokenizers();
+    await settingsEndpoint.init();
     await statsEndpoint.init();
 
     const cleanupPlugins = await loadPlugins();
@@ -581,7 +570,6 @@ const setupTasks = async function () {
         exitProcess();
     });
 
-
     console.log('Launching...');
 
     if (autorun) open(autorunUrl.toString());
@@ -601,6 +589,9 @@ const setupTasks = async function () {
         }
     }
 
+    if (listen && !basicAuthMode && enableAccounts) {
+        await userModule.checkAccountsProtection();
+    }
 };
 
 /**
@@ -642,21 +633,27 @@ function setWindowTitle(title) {
     }
 }
 
-if (cliArguments.ssl) {
-    https.createServer(
-        {
-            cert: fs.readFileSync(cliArguments.certPath),
-            key: fs.readFileSync(cliArguments.keyPath),
-        }, app)
-        .listen(
-            Number(tavernUrl.port) || 443,
-            tavernUrl.hostname,
-            setupTasks,
-        );
-} else {
-    http.createServer(app).listen(
-        Number(tavernUrl.port) || 80,
-        tavernUrl.hostname,
-        setupTasks,
-    );
-}
+// User storage module needs to be initialized before starting the server
+userModule.initUserStorage(dataRoot)
+    .then(userModule.ensurePublicDirectoriesExist)
+    .then(userModule.migrateUserData)
+    .finally(() => {
+        if (cliArguments.ssl) {
+            https.createServer(
+                {
+                    cert: fs.readFileSync(cliArguments.certPath),
+                    key: fs.readFileSync(cliArguments.keyPath),
+                }, app)
+                .listen(
+                    Number(tavernUrl.port) || 443,
+                    tavernUrl.hostname,
+                    setupTasks,
+                );
+        } else {
+            http.createServer(app).listen(
+                Number(tavernUrl.port) || 80,
+                tavernUrl.hostname,
+                setupTasks,
+            );
+        }
+    });

src/endpoints/backends/text-completions.js

@@ -7,6 +7,7 @@ const { jsonParser } = require('../../express-common');
 const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS, DREAMGEN_KEYS } = require('../../constants');
 const { forwardFetchResponse, trimV1 } = require('../../util');
 const { setAdditionalHeaders } = require('../../additional-headers');
+const { readSecret, SECRET_KEYS } = require('../secrets');
 
 const router = express.Router();
 
@@ -473,6 +474,42 @@ llamacpp.post('/caption-image', jsonParser, async function (request, response) {
     }
 });
 
+const tabbyapi = express.Router();
+
+tabbyapi.post('/verify-key', jsonParser, async function (request, response) {
+    try {
+        if (request.body.api_server.indexOf('localhost') !== -1) {
+            request.body.api_server = request.body.api_server.replace('localhost', '127.0.0.1');
+        }
+
+        console.log('Trying to connect to API:', request.body);
+        const baseUrl = trimV1(request.body.api_server);
+        const apiKey = readSecret(SECRET_KEYS.TABBY);
+
+        const fetchResponse = await fetch(`${baseUrl}/v1/auth/permission`, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': apiKey,
+            },
+            timeout: 0,
+        });
+
+        if (!fetchResponse.ok) {
+            console.log('TabbyAPI error:', fetchResponse.status, fetchResponse.statusText);
+            return response.status(500).send({ error: true });
+        }
+
+        const data = await fetchResponse.json();
+
+        return response.send({ 'isAdminKey': data?.permission === 'admin' });
+
+    } catch (error) {
+        console.error(error);
+        return response.status(500);
+    }
+});
+
 llamacpp.post('/props', jsonParser, async function (request, response) {
     try {
         if (!request.body.server_url) {
@@ -557,5 +594,6 @@ llamacpp.post('/slots', jsonParser, async function (request, response) {
 
 router.use('/ollama', ollama);
 router.use('/llamacpp', llamacpp);
+router.use('/tabbyapi', tabbyapi);
 
 module.exports = { router };

src/endpoints/tokenizers.js

@@ -10,6 +10,10 @@ const { TEXTGEN_TYPES } = require('../constants');
 const { jsonParser } = require('../express-common');
 const { setAdditionalHeaders } = require('../additional-headers');
 
+/**
+ * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
+ */
+
 /**
  * @type {{[key: string]: import("@dqbd/tiktoken").Tiktoken}} Tokenizers cache
  */
@@ -48,16 +52,30 @@ const TEXT_COMPLETION_MODELS = [
 
 const CHARS_PER_TOKEN = 3.35;
 
+/**
+ * Sentencepiece tokenizer for tokenizing text.
+ */
 class SentencePieceTokenizer {
+    /**
+     * @type {import('@agnai/sentencepiece-js').SentencePieceProcessor} Sentencepiece tokenizer instance
+     */
     #instance;
+    /**
+     * @type {string} Path to the tokenizer model
+     */
     #model;
 
+    /**
+     * Creates a new Sentencepiece tokenizer.
+     * @param {string} model Path to the tokenizer model
+     */
     constructor(model) {
         this.#model = model;
     }
 
     /**
      * Gets the Sentencepiece tokenizer instance.
+     * @returns {Promise<import('@agnai/sentencepiece-js').SentencePieceProcessor|null>} Sentencepiece tokenizer instance
      */
     async get() {
         if (this.#instance) {
@@ -76,18 +94,61 @@ class SentencePieceTokenizer {
     }
 }
 
-const spp_llama = new SentencePieceTokenizer('src/sentencepiece/llama.model');
-const spp_nerd = new SentencePieceTokenizer('src/sentencepiece/nerdstash.model');
-const spp_nerd_v2 = new SentencePieceTokenizer('src/sentencepiece/nerdstash_v2.model');
-const spp_mistral = new SentencePieceTokenizer('src/sentencepiece/mistral.model');
-const spp_yi = new SentencePieceTokenizer('src/sentencepiece/yi.model');
-let claude_tokenizer;
+/**
+ * Web tokenizer for tokenizing text.
+ */
+class WebTokenizer {
+    /**
+     * @type {Tokenizer} Web tokenizer instance
+     */
+    #instance;
+    /**
+     * @type {string} Path to the tokenizer model
+     */
+    #model;
+
+    /**
+     * Creates a new Web tokenizer.
+     * @param {string} model Path to the tokenizer model
+     */
+    constructor(model) {
+        this.#model = model;
+    }
+
+    /**
+     * Gets the Web tokenizer instance.
+     * @returns {Promise<Tokenizer|null>} Web tokenizer instance
+     */
+    async get() {
+        if (this.#instance) {
+            return this.#instance;
+        }
+
+        try {
+            const arrayBuffer = fs.readFileSync(this.#model).buffer;
+            this.#instance = await Tokenizer.fromJSON(arrayBuffer);
+            console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
+            return this.#instance;
+        } catch (error) {
+            console.error('Web tokenizer failed to load: ' + this.#model, error);
+            return null;
+        }
+    }
+}
+
+const spp_llama = new SentencePieceTokenizer('src/tokenizers/llama.model');
+const spp_nerd = new SentencePieceTokenizer('src/tokenizers/nerdstash.model');
+const spp_nerd_v2 = new SentencePieceTokenizer('src/tokenizers/nerdstash_v2.model');
+const spp_mistral = new SentencePieceTokenizer('src/tokenizers/mistral.model');
+const spp_yi = new SentencePieceTokenizer('src/tokenizers/yi.model');
+const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
 
 const sentencepieceTokenizers = [
     'llama',
     'nerdstash',
     'nerdstash_v2',
     'mistral',
+    'yi',
 ];
 
 /**
@@ -112,6 +173,10 @@ function getSentencepiceTokenizer(model) {
         return spp_nerd_v2;
     }
 
+    if (model.includes('yi')) {
+        return spp_yi;
+    }
+
     return null;
 }
 
@@ -168,13 +233,23 @@ async function getTiktokenChunks(tokenizer, ids) {
     return chunks;
 }
 
-async function getWebTokenizersChunks(tokenizer, ids) {
+/**
+ * Gets the token chunks for the given token IDs using the Web tokenizer.
+ * @param {Tokenizer} tokenizer Web tokenizer instance
+ * @param {number[]} ids Token IDs
+ * @returns {string[]} Token chunks
+ */
+function getWebTokenizersChunks(tokenizer, ids) {
     const chunks = [];
 
-    for (let i = 0; i < ids.length; i++) {
-        const id = ids[i];
-        const chunkText = await tokenizer.decode(new Uint32Array([id]));
+    for (let i = 0, lastProcessed = 0; i < ids.length; i++) {
+        const chunkIds = ids.slice(lastProcessed, i + 1);
+        const chunkText = tokenizer.decode(new Int32Array(chunkIds));
+        if (chunkText === '<27>') {
+            continue;
+        }
         chunks.push(chunkText);
+        lastProcessed = i + 1;
     }
 
     return chunks;
@@ -237,17 +312,12 @@ function getTiktokenTokenizer(model) {
     return tokenizer;
 }
 
-async function loadClaudeTokenizer(modelPath) {
-    try {
-        const arrayBuffer = fs.readFileSync(modelPath).buffer;
-        const instance = await Tokenizer.fromJSON(arrayBuffer);
-        return instance;
-    } catch (error) {
-        console.error('Claude tokenizer failed to load: ' + modelPath, error);
-        return null;
-    }
-}
-
+/**
+ * Counts the tokens for the given messages using the Claude tokenizer.
+ * @param {Tokenizer} tokenizer Web tokenizer
+ * @param {object[]} messages Array of messages
+ * @returns {number} Number of tokens
+ */
 function countClaudeTokens(tokenizer, messages) {
     // Should be fine if we use the old conversion method instead of the messages API one i think?
     const convertedPrompt = convertClaudePrompt(messages, false, '', false, false, '', false);
@@ -264,9 +334,14 @@ function countClaudeTokens(tokenizer, messages) {
 /**
  * Creates an API handler for encoding Sentencepiece tokens.
  * @param {SentencePieceTokenizer} tokenizer Sentencepiece tokenizer
- * @returns {any} Handler function
+ * @returns {TokenizationHandler} Handler function
  */
 function createSentencepieceEncodingHandler(tokenizer) {
+    /**
+     * Request handler for encoding Sentencepiece tokens.
+     * @param {import('express').Request} request
+     * @param {import('express').Response} response
+     */
     return async function (request, response) {
         try {
             if (!request.body) {
@@ -276,7 +351,7 @@ function createSentencepieceEncodingHandler(tokenizer) {
             const text = request.body.text || '';
             const instance = await tokenizer?.get();
             const { ids, count } = await countSentencepieceTokens(tokenizer, text);
-            const chunks = await instance?.encodePieces(text);
+            const chunks = instance?.encodePieces(text);
             return response.send({ ids, count, chunks });
         } catch (error) {
             console.log(error);
@@ -288,9 +363,14 @@ function createSentencepieceEncodingHandler(tokenizer) {
 /**
  * Creates an API handler for decoding Sentencepiece tokens.
  * @param {SentencePieceTokenizer} tokenizer Sentencepiece tokenizer
- * @returns {any} Handler function
+ * @returns {TokenizationHandler} Handler function
  */
 function createSentencepieceDecodingHandler(tokenizer) {
+    /**
+     * Request handler for decoding Sentencepiece tokens.
+     * @param {import('express').Request} request
+     * @param {import('express').Response} response
+     */
     return async function (request, response) {
         try {
             if (!request.body) {
@@ -299,6 +379,7 @@ function createSentencepieceDecodingHandler(tokenizer) {
 
             const ids = request.body.ids || [];
             const instance = await tokenizer?.get();
+            if (!instance) throw new Error('Failed to load the Sentencepiece tokenizer');
             const ops = ids.map(id => instance.decodeIds([id]));
             const chunks = await Promise.all(ops);
             const text = chunks.join('');
@@ -313,9 +394,14 @@ function createSentencepieceDecodingHandler(tokenizer) {
 /**
  * Creates an API handler for encoding Tiktoken tokens.
  * @param {string} modelId Tiktoken model ID
- * @returns {any} Handler function
+ * @returns {TokenizationHandler} Handler function
  */
 function createTiktokenEncodingHandler(modelId) {
+    /**
+     * Request handler for encoding Tiktoken tokens.
+     * @param {import('express').Request} request
+     * @param {import('express').Response} response
+     */
     return async function (request, response) {
         try {
             if (!request.body) {
@@ -337,9 +423,14 @@ function createTiktokenEncodingHandler(modelId) {
 /**
  * Creates an API handler for decoding Tiktoken tokens.
  * @param {string} modelId Tiktoken model ID
- * @returns {any} Handler function
+ * @returns {TokenizationHandler} Handler function
  */
 function createTiktokenDecodingHandler(modelId) {
+    /**
+     * Request handler for decoding Tiktoken tokens.
+     * @param {import('express').Request} request
+     * @param {import('express').Response} response
+     */
     return async function (request, response) {
         try {
             if (!request.body) {
@@ -358,14 +449,6 @@ function createTiktokenDecodingHandler(modelId) {
     };
 }
 
-/**
- * Loads the model tokenizers.
- * @returns {Promise<void>} Promise that resolves when the tokenizers are loaded
- */
-async function loadTokenizers() {
-    claude_tokenizer = await loadClaudeTokenizer('src/claude.json');
-}
-
 const router = express.Router();
 
 router.post('/ai21/count', jsonParser, async function (req, res) {
@@ -446,8 +529,10 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
 
         if (queryModel.includes('claude')) {
             const text = req.body.text || '';
-            const tokens = Object.values(claude_tokenizer.encode(text));
-            const chunks = await getWebTokenizersChunks(claude_tokenizer, tokens);
+            const instance = await claude_tokenizer.get();
+            if (!instance) throw new Error('Failed to load the Claude tokenizer');
+            const tokens = Object.values(instance.encode(text));
+            const chunks = getWebTokenizersChunks(instance, tokens);
             return res.send({ ids: tokens, count: tokens.length, chunks });
         }
 
@@ -481,7 +566,9 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
 
         if (queryModel.includes('claude')) {
             const ids = req.body.ids || [];
-            const chunkText = await claude_tokenizer.decode(new Uint32Array(ids));
+            const instance = await claude_tokenizer.get();
+            if (!instance) throw new Error('Failed to load the Claude tokenizer');
+            const chunkText = instance.decode(new Int32Array(ids));
             return res.send({ text: chunkText });
         }
 
@@ -503,7 +590,9 @@ router.post('/openai/count', jsonParser, async function (req, res) {
         const model = getTokenizerModel(queryModel);
 
         if (model === 'claude') {
-            num_tokens = countClaudeTokens(claude_tokenizer, req.body);
+            const instance = await claude_tokenizer.get();
+            if (!instance) throw new Error('Failed to load the Claude tokenizer');
+            num_tokens = countClaudeTokens(instance, req.body);
             return res.send({ 'token_count': num_tokens });
         }
 
@@ -665,7 +754,6 @@ module.exports = {
     getTokenizerModel,
     getTiktokenTokenizer,
     countClaudeTokens,
-    loadTokenizers,
     getSentencepiceTokenizer,
     sentencepieceTokenizers,
     router,

src/users.js

@@ -112,6 +112,16 @@ async function ensurePublicDirectoriesExist() {
     return directoriesList;
 }
 
+/**
+ * Gets a list of all user directories.
+ * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
+ */
+async function getUserDirectoriesList() {
+    const userHandles = await getAllUserHandles();
+    const directoriesList = userHandles.map(handle => getUserDirectories(handle));
+    return directoriesList;
+}
+
 /**
  * Perform migration from the old user data format to the new one.
  */
@@ -707,6 +717,7 @@ module.exports = {
     toAvatarKey,
     initUserStorage,
     ensurePublicDirectoriesExist,
+    getUserDirectoriesList,
     getAllUserHandles,
     getUserDirectories,
     setUserDataMiddleware,