From 5f92b8a09e3638475756b575e4859f7f4e9f907a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eug=C3=A9n=20Cowie?= Date: Mon, 19 Aug 2024 00:13:26 +0100 Subject: [PATCH 1/5] Add slash command to set tokenizer --- public/script.js | 43 +++++++++++++++++++++++++++++++++++- public/scripts/tokenizers.js | 18 +++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index daea9cc50..4599e1cdc 100644 --- a/public/script.js +++ b/public/script.js @@ -212,7 +212,7 @@ import { selectContextPreset, } from './scripts/instruct-mode.js'; import { initLocales, t, translate } from './scripts/i18n.js'; -import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache } from './scripts/tokenizers.js'; +import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js'; import { user_avatar, getUserAvatars, @@ -8451,6 +8451,25 @@ async function selectInstructCallback(_, name) { return foundName; } +async function selectTokenizerCallback(_, name) { + if (!name) { + return TOKENIZER_NAME_MAP[power_user.tokenizer]; + } + + const tokenizerNames = Object.values(TOKENIZER_NAME_MAP); + const fuse = new Fuse(tokenizerNames); + const result = fuse.search(name); + + if (result.length === 0) { + toastr.warning(`Tokenizer "${name}" not found`); + return ''; + } + + const foundName = result[0].item; + selectTokenizer(tokenizers[foundName.toUpperCase()]); + return foundName; +} + async function enableInstructCallback() { $('#instruct_enabled').prop('checked', true).trigger('change'); return ''; @@ -9095,6 +9114,28 @@ jQuery(async function () { `, })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tokenizer', + callback: selectTokenizerCallback, + returns: 'current tokenizer', + unnamedArgumentList: [ + SlashCommandArgument.fromProps({ + description: 'tokenizer name', + typeList: [ARGUMENT_TYPE.STRING], + enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer => + new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)), + }), + ], + helpString: ` +
+ Selects tokenizer by name. Gets the current tokenizer if no name is provided. +
+
+ Available tokenizers: + 
${Object.values(TOKENIZER_NAME_MAP).join(', ')}
+
+ ` + })); SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'instruct-on', callback: enableInstructCallback, diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index c27775acb..5732706d3 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -147,6 +147,24 @@ async function resetTokenCache() { } } +/** + * Maps tokenizer IDs to their names. + * @example { 0: 'none', 1: 'gpt2', ... } + */ +export const TOKENIZER_NAME_MAP = Object.fromEntries( + Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()])); + +/** + * Selects tokenizer if not already selected. + * @param {number} tokenizerId Tokenizer ID. + */ +export function selectTokenizer(tokenizerId) { + if (tokenizerId !== power_user.tokenizer) { + $('#tokenizer').val(tokenizerId).trigger('change'); + toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`); + } +} + /** * Gets the friendly name of the current tokenizer. * @param {string} forApi API to get the tokenizer for. Defaults to the main API. From 62818055af881170e0ebce3c90c4af8c12ff79ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eug=C3=A9n=20Cowie?= Date: Mon, 19 Aug 2024 15:36:56 +0100 Subject: [PATCH 2/5] Get available tokenizers from UI --- public/script.js | 20 ++++++++++---------- public/scripts/tokenizers.js | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/public/script.js b/public/script.js index 4599e1cdc..357976ee5 100644 --- a/public/script.js +++ b/public/script.js @@ -212,7 +212,7 @@ import { selectContextPreset, } from './scripts/instruct-mode.js'; import { initLocales, t, translate } from './scripts/i18n.js'; -import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js'; +import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js'; import { user_avatar, getUserAvatars, @@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) { async function selectTokenizerCallback(_, name) { if (!name) { - return TOKENIZER_NAME_MAP[power_user.tokenizer]; + return getFriendlyTokenizerName(main_api).tokenizerName; } - const tokenizerNames = Object.values(TOKENIZER_NAME_MAP); - const fuse = new Fuse(tokenizerNames); + const tokenizers = getAvailableTokenizers(); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] }); const result = fuse.search(name); if (result.length === 0) { @@ -8465,9 +8465,9 @@ async function selectTokenizerCallback(_, name) { return ''; } - const foundName = result[0].item; - selectTokenizer(tokenizers[foundName.toUpperCase()]); - return foundName; + const foundTokenizer = result[0].item; + selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); + return foundTokenizer; } async function enableInstructCallback() { @@ -9122,8 +9122,8 @@ jQuery(async function () { SlashCommandArgument.fromProps({ description: 'tokenizer name', typeList: [ARGUMENT_TYPE.STRING], - enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer => - new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)), + enumList: getAvailableTokenizers().map(tokenizer => + new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)), }), ], helpString: ` @@ -9132,7 +9132,7 @@ jQuery(async function () {
Available tokenizers: - 
${Object.values(TOKENIZER_NAME_MAP).join(', ')}
+ 
${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}
` })); diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 5732706d3..c4411bd3f 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -148,20 +148,26 @@ async function resetTokenCache() { } /** - * Maps tokenizer IDs to their names. - * @example { 0: 'none', 1: 'gpt2', ... } + * Gets all tokenizers available to the user. + * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info. */ -export const TOKENIZER_NAME_MAP = Object.fromEntries( - Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()])); +export function getAvailableTokenizers() { + const tokenizerOptions = $('#tokenizer').find('option').toArray(); + return tokenizerOptions.map(tokenizerOption => ({ + tokenizerName: tokenizerOption.text, + tokenizerId: Number(tokenizerOption.value), + })) +} /** * Selects tokenizer if not already selected. + * @param {string} tokenizerName Tokenizer name. * @param {number} tokenizerId Tokenizer ID. */ -export function selectTokenizer(tokenizerId) { +export function selectTokenizer(tokenizerName, tokenizerId) { if (tokenizerId !== power_user.tokenizer) { $('#tokenizer').val(tokenizerId).trigger('change'); - toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`); + toastr.info(`Tokenizer: "${tokenizerName}" selected`); } } From 53c71d80598bca435df9c97de54432bf7b8dab7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eug=C3=A9n=20Cowie?= Date: Mon, 19 Aug 2024 17:21:51 +0100 Subject: [PATCH 3/5] Self review fix --- public/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 357976ee5..5b89ac5f4 100644 --- a/public/script.js +++ b/public/script.js @@ -8467,7 +8467,7 @@ async function selectTokenizerCallback(_, name) { const foundTokenizer = result[0].item; selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); - return foundTokenizer; + return foundTokenizer.tokenizerName; } async function enableInstructCallback() { From b68969ee3b9ae3fdac13b39f5492c811ee28f95d Mon Sep 17 00:00:00 2001 From: Wolfsblvt Date: Mon, 19 Aug 2024 20:18:44 +0200 Subject: [PATCH 4/5] Small update to enum display + consistency --- public/script.js | 14 ++++++++------ public/scripts/tokenizers.js | 30 ++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/public/script.js b/public/script.js index 5b89ac5f4..273852398 100644 --- a/public/script.js +++ b/public/script.js @@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) { async function selectTokenizerCallback(_, name) { if (!name) { - return getFriendlyTokenizerName(main_api).tokenizerName; + return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; } const tokenizers = getAvailableTokenizers(); - const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] }); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); const result = fuse.search(name); if (result.length === 0) { @@ -8465,9 +8465,11 @@ async function selectTokenizerCallback(_, name) { return ''; } + /** @type {import('./scripts/tokenizers.js').Tokenizer} */ const foundTokenizer = result[0].item; - selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); - return foundTokenizer.tokenizerName; + selectTokenizer(foundTokenizer.tokenizerId); + + return foundTokenizer.tokenizerKey; } async function enableInstructCallback() { @@ -9123,7 +9125,7 @@ jQuery(async function () { description: 'tokenizer name', typeList: [ARGUMENT_TYPE.STRING], enumList: getAvailableTokenizers().map(tokenizer => - new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)), + new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), }), ], helpString: ` @@ -9132,7 +9134,7 @@ jQuery(async function () {
Available tokenizers: - 
${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}
+ 
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
` })); diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index c4411bd3f..e72a44ad6 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -147,34 +147,46 @@ async function resetTokenCache() { } } +/** + * @typedef {object} Tokenizer + * @property {number} tokenizerId - The id of the tokenizer option + * @property {string} tokenizerKey - Internal name/key of the tokenizer + * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI) + */ + /** * Gets all tokenizers available to the user. - * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info. + * @returns {Tokenizer[]} Tokenizer info. */ export function getAvailableTokenizers() { const tokenizerOptions = $('#tokenizer').find('option').toArray(); return tokenizerOptions.map(tokenizerOption => ({ - tokenizerName: tokenizerOption.text, tokenizerId: Number(tokenizerOption.value), + tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(), + tokenizerName: tokenizerOption.text, })) } /** * Selects tokenizer if not already selected. - * @param {string} tokenizerName Tokenizer name. * @param {number} tokenizerId Tokenizer ID. */ -export function selectTokenizer(tokenizerName, tokenizerId) { +export function selectTokenizer(tokenizerId) { if (tokenizerId !== power_user.tokenizer) { - $('#tokenizer').val(tokenizerId).trigger('change'); - toastr.info(`Tokenizer: "${tokenizerName}" selected`); + const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId); + if (!tokenizer) { + console.warn('Failed to find tokenizer with id', tokenizerId); + return; + } + $('#tokenizer').val(tokenizer.tokenizerId).trigger('change'); + toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`); } } /** * Gets the friendly name of the current tokenizer. * @param {string} forApi API to get the tokenizer for. Defaults to the main API. - * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info + * @returns {Tokenizer} Tokenizer info */ export function getFriendlyTokenizerName(forApi) { if (!forApi) { @@ -209,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) { ? tokenizers.OPENAI : tokenizerId; - return { tokenizerName, tokenizerId }; + const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase(); + + return { tokenizerName, tokenizerKey, tokenizerId }; } /** From 976be98dd755b1e2ff5bb8f3f9cceef5520f6c4e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 19 Aug 2024 21:31:58 +0300 Subject: [PATCH 5/5] Move command registration --- public/script.js | 45 +------------------------------- public/scripts/slash-commands.js | 45 +++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/public/script.js b/public/script.js index 273852398..daea9cc50 100644 --- a/public/script.js +++ b/public/script.js @@ -212,7 +212,7 @@ import { selectContextPreset, } from './scripts/instruct-mode.js'; import { initLocales, t, translate } from './scripts/i18n.js'; -import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js'; +import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache } from './scripts/tokenizers.js'; import { user_avatar, getUserAvatars, @@ -8451,27 +8451,6 @@ async function selectInstructCallback(_, name) { return foundName; } -async function selectTokenizerCallback(_, name) { - if (!name) { - return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; - } - - const tokenizers = getAvailableTokenizers(); - const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); - const result = fuse.search(name); - - if (result.length === 0) { - toastr.warning(`Tokenizer "${name}" not found`); - return ''; - } - - /** @type {import('./scripts/tokenizers.js').Tokenizer} */ - const foundTokenizer = result[0].item; - selectTokenizer(foundTokenizer.tokenizerId); - - return foundTokenizer.tokenizerKey; -} - async function enableInstructCallback() { $('#instruct_enabled').prop('checked', true).trigger('change'); return ''; @@ -9116,28 +9095,6 @@ jQuery(async function () { `, })); - SlashCommandParser.addCommandObject(SlashCommand.fromProps({ - name: 'tokenizer', - callback: selectTokenizerCallback, - returns: 'current tokenizer', - unnamedArgumentList: [ - SlashCommandArgument.fromProps({ - description: 'tokenizer name', - typeList: [ARGUMENT_TYPE.STRING], - enumList: getAvailableTokenizers().map(tokenizer => - new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), - }), - ], - helpString: ` -
- Selects tokenizer by name. Gets the current tokenizer if no name is provided. -
-
- Available tokenizers: - 
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
-
- ` - })); SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'instruct-on', callback: enableInstructCallback, diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index d9d27f0db..6dcac6a80 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -52,7 +52,7 @@ import { chat_completion_sources, oai_settings, setupChatCompletionPromptManager import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js'; import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js'; import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js'; -import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js'; +import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js'; import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js'; import { registerVariableCommands, resolveVariable } from './variables.js'; import { background_settings } from './backgrounds.js'; @@ -1558,6 +1558,28 @@ export function initDefaultSlashCommands() { `, })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tokenizer', + callback: selectTokenizerCallback, + returns: 'current tokenizer', + unnamedArgumentList: [ + SlashCommandArgument.fromProps({ + description: 'tokenizer name', + typeList: [ARGUMENT_TYPE.STRING], + enumList: getAvailableTokenizers().map(tokenizer => + new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), + }), + ], + helpString: ` +
+ Selects tokenizer by name. Gets the current tokenizer if no name is provided. +
+
+ Available tokenizers: + 
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
+
+ `, + })); registerVariableCommands(); } @@ -3562,6 +3584,27 @@ async function setApiUrlCallback({ api = null, connect = 'true' }, url) { return textgenerationwebui_settings.server_urls[type] ?? ''; } +async function selectTokenizerCallback(_, name) { + if (!name) { + return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; + } + + const tokenizers = getAvailableTokenizers(); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); + const result = fuse.search(name); + + if (result.length === 0) { + toastr.warning(`Tokenizer "${name}" not found`); + return ''; + } + + /** @type {import('./tokenizers.js').Tokenizer} */ + const foundTokenizer = result[0].item; + selectTokenizer(foundTokenizer.tokenizerId); + + return foundTokenizer.tokenizerKey; +} + export let isExecutingCommandsFromChatInput = false; export let commandsFromChatInputAbortController;