rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Silently redirect tokenization to the next matching tokenizer

This commit is contained in:
Cohee 2024-09-22 17:25:56 +03:00
parent ba52ceb979
commit 8c5b02d2ea
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
public/scripts/tokenizers.js
View File

@ -923,15 +923,20 @@ function countTokensFromTextgenAPI(str, resolve) {
function apiFailureTokenCount(str) {
console.error('Error counting tokens');
let shouldTryAgain = false;
if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
toastr.warning(
'Your selected API doesn\'t support the tokenization endpoint. Using estimated counts.',
'Error counting tokens',
{ timeOut: 10000, preventDuplicates: true },
);
const bestMatchBefore = getTokenizerBestMatch(main_api);
sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
const bestMatchAfter = getTokenizerBestMatch(main_api);
if ([tokenizers.API_TEXTGENERATIONWEBUI, tokenizers.API_KOBOLD].includes(bestMatchBefore) && bestMatchBefore !== bestMatchAfter) {
shouldTryAgain = true;
}
}
// Only try again if we guarantee not to be looped by the same error
if (shouldTryAgain && power_user.tokenizer === tokenizers.BEST_MATCH) {
return getTokenCount(str);
}
return guesstimate(str);