rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Better character tokens counting

Browse Source
This commit is contained in:
SillyLossy
2023-04-26 01:19:36 +03:00
parent 8f29e98cba
commit 1c4f1ea96c
2 changed files with 51 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
public/scripts/RossAscends-mods.js
View File

@ -140,66 +140,75 @@ $("#rm_button_create").on("click", function () { //when "+New Ch
create_save_mes_example = ""; create_save_mes_example = "";
$("#result_info").html('Type to start counting tokens!'); $("#result_info").html('Type to start counting tokens!');
}); });
$("#rm_ch_create_block").on("input", function () { RA_CountCharTokens(); }); //when any input is made to the create/edit character form textareas //when any input is made to the create/edit character form textareas
$("#character_popup").on("input", function () { RA_CountCharTokens(); }); //when any input is made to the advanced editing popup textareas $("#rm_ch_create_block").on("input", function () { RA_CountCharTokens(); });
//when any input is made to the advanced editing popup textareas
$("#character_popup").on("input", function () { RA_CountCharTokens(); });
//function: //function:
function RA_CountCharTokens() { function RA_CountCharTokens() {
$("#result_info").html(""); $("#result_info").html("");
//console.log('RA_TC -- starting with this_chid = ' + this_chid); //console.log('RA_TC -- starting with this_chid = ' + this_chid);
if (document.getElementById('name_div').style.display == "block") { //if new char if (document.getElementById('name_div').style.display == "block") { //if new char
function saveFormVariables() {
$("#form_create").on("input", function () { //fill temp vars with form_create values
create_save_name = $("#character_name_pole").val(); create_save_name = $("#character_name_pole").val();
create_save_description = $("#description_textarea").val(); create_save_description = $("#description_textarea").val();
create_save_first_message = $("#firstmessage_textarea").val(); create_save_first_message = $("#firstmessage_textarea").val();
}); }
$("#character_popup").on("input", function () { //fill temp vars with advanced popup values
function savePopupVariables() {
create_save_personality = $("#personality_textarea").val(); create_save_personality = $("#personality_textarea").val();
create_save_scenario = $("#scenario_pole").val(); create_save_scenario = $("#scenario_pole").val();
create_save_mes_example = $("#mes_example_textarea").val(); create_save_mes_example = $("#mes_example_textarea").val();
}
}); saveFormVariables();
savePopupVariables();
//count total tokens, including those that will be removed from context once chat history is long //count total tokens, including those that will be removed from context once chat history is long
count_tokens = getTokenCount(JSON.stringify( let count_string = [
create_save_name + create_save_name,
create_save_description + create_save_description,
create_save_personality + create_save_personality,
create_save_scenario + create_save_scenario,
create_save_first_message + create_save_first_message,
create_save_mes_example create_save_mes_example,
).replace(/\r/gm, '')); ].join('\n').replace(/\r/gm, '').trim();
count_tokens = getTokenCount(count_string);
//count permanent tokens that will never get flushed out of context //count permanent tokens that will never get flushed out of context
perm_tokens = getTokenCount(JSON.stringify( let perm_string = [
create_save_name + create_save_name,
create_save_description + create_save_description,
create_save_personality + create_save_personality,
create_save_scenario create_save_scenario,
).replace(/\r/gm, '')); // add examples to permanent if they are pinned
(power_user.pin_examples ? create_save_mes_example : ''),
].join('\n').replace(/\r/gm, '').trim();
perm_tokens = getTokenCount(perm_string);
} else { } else {
if (this_chid !== undefined && this_chid !== "invalid-safety-id") { // if we are counting a valid pre-saved char if (this_chid !== undefined && this_chid !== "invalid-safety-id") { // if we are counting a valid pre-saved char
//same as above, all tokens including temporary ones //same as above, all tokens including temporary ones
count_tokens = getTokenCount( let count_string = [
JSON.stringify( characters[this_chid].description,
characters[this_chid].description + characters[this_chid].personality,
characters[this_chid].personality + characters[this_chid].scenario,
characters[this_chid].scenario + characters[this_chid].first_mes,
characters[this_chid].first_mes + characters[this_chid].mes_example,
characters[this_chid].mes_example ].join('\n').replace(/\r/gm, '').trim();
)); count_tokens = getTokenCount(count_string);
//permanent tokens count //permanent tokens count
perm_tokens = getTokenCount( let perm_string = [
JSON.stringify( characters[this_chid].name,
characters[this_chid].name + characters[this_chid].description,
characters[this_chid].description + characters[this_chid].personality,
characters[this_chid].personality + characters[this_chid].scenario,
characters[this_chid].scenario + // add examples to permanent if they are pinned
(power_user.pin_examples ? characters[this_chid].mes_example : '') // add examples to permanent if they are pinned (power_user.pin_examples ? characters[this_chid].mes_example : ''),
)); ].join('\n').replace(/\r/gm, '').trim();
perm_tokens = getTokenCount(perm_string);
} else { console.log("RA_TC -- no valid char found, closing."); } // if neither, probably safety char or some error in loading } else { console.log("RA_TC -- no valid char found, closing."); } // if neither, probably safety char or some error in loading
} }
// display the counted tokens // display the counted tokens
@ -209,7 +218,7 @@ function RA_CountCharTokens() {
$("#result_info").html(` $("#result_info").html(`
<span class="neutral_warning">${count_tokens}</span>&nbsp;Tokens (<span class="neutral_warning">${perm_tokens}</span><span>&nbsp;Permanent Tokens) <span class="neutral_warning">${count_tokens}</span>&nbsp;Tokens (<span class="neutral_warning">${perm_tokens}</span><span>&nbsp;Permanent Tokens)
<br> <br>
<div id="chartokenwarning" class="menu_button whitespacenowrap"><a href="/notes/token-limits.html" target="_blank">Learn More About Token 'Limits'</a></div>`); <div id="chartokenwarning" class="menu_button whitespacenowrap"><a href="/notes#charactertokens" target="_blank">Learn More About Token 'Limits'</a></div>`);
} //warn if either are over 1024 } //warn if either are over 1024
} }
//Auto Load Last Charcter -- (fires when active_character is defined and auto_load_chat is true) //Auto Load Last Charcter -- (fires when active_character is defined and auto_load_chat is true)

4
public/scripts/power-user.js
View File

@ -595,6 +595,10 @@ $(document).ready(() => {
const value = $(this).find(':selected').val(); const value = $(this).find(':selected').val();
power_user.tokenizer = Number(value); power_user.tokenizer = Number(value);
saveSettingsDebounced(); saveSettingsDebounced();
// Trigger character editor re-tokenize
$("#rm_ch_create_block").trigger('input');
$("#character_popup").trigger('input');
}); });
$(window).on('focus', function () { $(window).on('focus', function () {