rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #701 from 50h100a/api-tokenize

Browse Source 
Added tokenizer option for Ooba-like APIs
This commit is contained in:
Cohee
2023-07-12 11:28:50 +03:00
committed by GitHub
parent 9bbf23e56f 6b0e13e064
commit cb294983e7
4 changed files with 25 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
public/script.js
View File

@@ -581,6 +581,8 @@ function getTokenCount(str, padding = undefined) {
return countTokensRemote('/tokenize_nerdstash', str, padding);
case tokenizers.NERD2:
return countTokensRemote('/tokenize_nerdstash_v2', str, padding);
case tokenizers.API:
return countTokensRemote('/tokenize_via_api', str, padding);
default:
console.warn("Unknown tokenizer type", tokenizerType);
return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO) + padding;