rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Get available tokenizers from UI

Browse Source
This commit is contained in:
Eugén Cowie
2024-08-19 15:36:56 +01:00
parent 5f92b8a09e
commit 62818055af
2 changed files with 22 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
public/script.js
View File

@@ -212,7 +212,7 @@ import {
selectContextPreset, selectContextPreset,
} from './scripts/instruct-mode.js'; } from './scripts/instruct-mode.js';
import { initLocales, t, translate } from './scripts/i18n.js'; import { initLocales, t, translate } from './scripts/i18n.js';
import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js'; import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js';
import { import {
user_avatar, user_avatar,
getUserAvatars, getUserAvatars,
@@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) {
async function selectTokenizerCallback(_, name) { async function selectTokenizerCallback(_, name) {
if (!name) { if (!name) {
return TOKENIZER_NAME_MAP[power_user.tokenizer]; return getFriendlyTokenizerName(main_api).tokenizerName;
} }
const tokenizerNames = Object.values(TOKENIZER_NAME_MAP); const tokenizers = getAvailableTokenizers();
const fuse = new Fuse(tokenizerNames); const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] });
const result = fuse.search(name); const result = fuse.search(name);
if (result.length === 0) { if (result.length === 0) {
@@ -8465,9 +8465,9 @@ async function selectTokenizerCallback(_, name) {
return ''; return '';
} }
const foundName = result[0].item; const foundTokenizer = result[0].item;
selectTokenizer(tokenizers[foundName.toUpperCase()]); selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId);
return foundName; return foundTokenizer;
} }
async function enableInstructCallback() { async function enableInstructCallback() {
@@ -9122,8 +9122,8 @@ jQuery(async function () {
SlashCommandArgument.fromProps({ SlashCommandArgument.fromProps({
description: 'tokenizer name', description: 'tokenizer name',
typeList: [ARGUMENT_TYPE.STRING], typeList: [ARGUMENT_TYPE.STRING],
enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer => enumList: getAvailableTokenizers().map(tokenizer =>
new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)), new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)),
}), }),
], ],
helpString: ` helpString: `
@@ -9132,7 +9132,7 @@ jQuery(async function () {
</div> </div>
<div> <div>
<strong>Available tokenizers:</strong> <strong>Available tokenizers:</strong>
<pre><code>${Object.values(TOKENIZER_NAME_MAP).join(', ')}</code></pre> <pre><code>${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}</code></pre>
</div> </div>
` `
})); }));

18
public/scripts/tokenizers.js
View File

@@ -148,20 +148,26 @@ async function resetTokenCache() {
} }
/** /**
* Maps tokenizer IDs to their names. * Gets all tokenizers available to the user.
* @example { 0: 'none', 1: 'gpt2', ... } * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info.
*/ */
export const TOKENIZER_NAME_MAP = Object.fromEntries( export function getAvailableTokenizers() {
Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()])); const tokenizerOptions = $('#tokenizer').find('option').toArray();
return tokenizerOptions.map(tokenizerOption => ({
tokenizerName: tokenizerOption.text,
tokenizerId: Number(tokenizerOption.value),
}))
}
/** /**
* Selects tokenizer if not already selected. * Selects tokenizer if not already selected.
* @param {string} tokenizerName Tokenizer name.
* @param {number} tokenizerId Tokenizer ID. * @param {number} tokenizerId Tokenizer ID.
*/ */
export function selectTokenizer(tokenizerId) { export function selectTokenizer(tokenizerName, tokenizerId) {
if (tokenizerId !== power_user.tokenizer) { if (tokenizerId !== power_user.tokenizer) {
$('#tokenizer').val(tokenizerId).trigger('change'); $('#tokenizer').val(tokenizerId).trigger('change');
toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`); toastr.info(`Tokenizer: "${tokenizerName}" selected`);
} }
} }