rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add slash command to set tokenizer

Browse Source
This commit is contained in:
Eugén Cowie
2024-08-19 00:13:26 +01:00
parent 44e62156b7
commit 5f92b8a09e
2 changed files with 60 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
public/scripts/tokenizers.js
View File

@@ -147,6 +147,24 @@ async function resetTokenCache() {
}
}
/**
* Maps tokenizer IDs to their names.
* @example { 0: 'none', 1: 'gpt2', ... }
*/
export const TOKENIZER_NAME_MAP = Object.fromEntries(
Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()]));
/**
* Selects tokenizer if not already selected.
* @param {number} tokenizerId Tokenizer ID.
*/
export function selectTokenizer(tokenizerId) {
if (tokenizerId !== power_user.tokenizer) {
$('#tokenizer').val(tokenizerId).trigger('change');
toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`);
}
}
/**
* Gets the friendly name of the current tokenizer.
* @param {string} forApi API to get the tokenizer for. Defaults to the main API.