rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add a token counter for persona descriptions

Browse Source
This commit is contained in:
Cohee
2023-08-22 18:32:18 +03:00
parent ea4d4a8fd6
commit 042c0b84a1
4 changed files with 24 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
public/script.js
View File

@ -543,6 +543,12 @@ function getTokenizerBestMatch() {
return power_user.NONE;
}
/**
* Gets the token count for a string using the current model tokenizer.
* @param {string} str String to tokenize
* @param {number | undefined} padding Optional padding tokens. Defaults to 0.
* @returns {number} Token count.
*/
function getTokenCount(str, padding = undefined) {
if (typeof str !== 'string' || !str?.length) {
return 0;
@ -4520,6 +4526,7 @@ function setPersonaDescription() {
.val(power_user.persona_description_position)
.find(`option[value='${power_user.persona_description_position}']`)
.attr("selected", true);
countPersonaDescriptionTokens();
}
function onPersonaDescriptionPositionInput() {
@ -4544,8 +4551,18 @@ function onPersonaDescriptionPositionInput() {
saveSettingsDebounced();
}
/**
* Counts the number of tokens in a persona description.
*/
const countPersonaDescriptionTokens = debounce(() => {
const description = String($("#persona_description").val());
const count = getTokenCount(description);
$("#persona_description_token_count").text(String(count));
}, durationSaveEdit);
function onPersonaDescriptionInput() {
power_user.persona_description = String($("#persona_description").val());
countPersonaDescriptionTokens();
if (power_user.personas[user_avatar]) {
let object = power_user.persona_descriptions[user_avatar];