rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-03-03 03:17:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

Silently redirect tokenization to the next matching tokenizer

Browse Source
This commit is contained in:
Cohee 2024-09-22 17:25:56 +03:00
parent ba52ceb979
commit 8c5b02d2ea
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
public/scripts/tokenizers.js
View File

@ -923,15 +923,20 @@ function countTokensFromTextgenAPI(str, resolve) {
function apiFailureTokenCount(str) { function apiFailureTokenCount(str) {
console.error('Error counting tokens'); console.error('Error counting tokens');
let shouldTryAgain = false;
if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) { if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
toastr.warning( const bestMatchBefore = getTokenizerBestMatch(main_api);
'Your selected API doesn\'t support the tokenization endpoint. Using estimated counts.',
'Error counting tokens',
{ timeOut: 10000, preventDuplicates: true },
);
sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true)); sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
const bestMatchAfter = getTokenizerBestMatch(main_api);
if ([tokenizers.API_TEXTGENERATIONWEBUI, tokenizers.API_KOBOLD].includes(bestMatchBefore) && bestMatchBefore !== bestMatchAfter) {
shouldTryAgain = true;
}
}
// Only try again if we guarantee not to be looped by the same error
if (shouldTryAgain && power_user.tokenizer === tokenizers.BEST_MATCH) {
return getTokenCount(str);
} }
return guesstimate(str); return guesstimate(str);