diff --git a/public/script.js b/public/script.js
index 5b89ac5f4..273852398 100644
--- a/public/script.js
+++ b/public/script.js
@@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) {
async function selectTokenizerCallback(_, name) {
if (!name) {
- return getFriendlyTokenizerName(main_api).tokenizerName;
+ return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? '';
}
const tokenizers = getAvailableTokenizers();
- const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] });
+ const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] });
const result = fuse.search(name);
if (result.length === 0) {
@@ -8465,9 +8465,11 @@ async function selectTokenizerCallback(_, name) {
return '';
}
+ /** @type {import('./scripts/tokenizers.js').Tokenizer} */
const foundTokenizer = result[0].item;
- selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId);
- return foundTokenizer.tokenizerName;
+ selectTokenizer(foundTokenizer.tokenizerId);
+
+ return foundTokenizer.tokenizerKey;
}
async function enableInstructCallback() {
@@ -9123,7 +9125,7 @@ jQuery(async function () {
description: 'tokenizer name',
typeList: [ARGUMENT_TYPE.STRING],
enumList: getAvailableTokenizers().map(tokenizer =>
- new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)),
+ new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)),
}),
],
helpString: `
@@ -9132,7 +9134,7 @@ jQuery(async function () {
Available tokenizers:
-
${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}
+
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
`
}));
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index c4411bd3f..e72a44ad6 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -147,34 +147,46 @@ async function resetTokenCache() {
}
}
+/**
+ * @typedef {object} Tokenizer
+ * @property {number} tokenizerId - The id of the tokenizer option
+ * @property {string} tokenizerKey - Internal name/key of the tokenizer
+ * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI)
+ */
+
/**
* Gets all tokenizers available to the user.
- * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info.
+ * @returns {Tokenizer[]} Tokenizer info.
*/
export function getAvailableTokenizers() {
const tokenizerOptions = $('#tokenizer').find('option').toArray();
return tokenizerOptions.map(tokenizerOption => ({
- tokenizerName: tokenizerOption.text,
tokenizerId: Number(tokenizerOption.value),
+ tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(),
+ tokenizerName: tokenizerOption.text,
}))
}
/**
* Selects tokenizer if not already selected.
- * @param {string} tokenizerName Tokenizer name.
* @param {number} tokenizerId Tokenizer ID.
*/
-export function selectTokenizer(tokenizerName, tokenizerId) {
+export function selectTokenizer(tokenizerId) {
if (tokenizerId !== power_user.tokenizer) {
- $('#tokenizer').val(tokenizerId).trigger('change');
- toastr.info(`Tokenizer: "${tokenizerName}" selected`);
+ const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId);
+ if (!tokenizer) {
+ console.warn('Failed to find tokenizer with id', tokenizerId);
+ return;
+ }
+ $('#tokenizer').val(tokenizer.tokenizerId).trigger('change');
+ toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`);
}
}
/**
* Gets the friendly name of the current tokenizer.
* @param {string} forApi API to get the tokenizer for. Defaults to the main API.
- * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info
+ * @returns {Tokenizer} Tokenizer info
*/
export function getFriendlyTokenizerName(forApi) {
if (!forApi) {
@@ -209,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) {
? tokenizers.OPENAI
: tokenizerId;
- return { tokenizerName, tokenizerId };
+ const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase();
+
+ return { tokenizerName, tokenizerKey, tokenizerId };
}
/**