rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer changes and fixes. + a toggle

Browse Source
This commit is contained in:
based
2023-12-14 16:31:08 +10:00
parent e26159c00d
commit 3e82a7d439
5 changed files with 24 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
public/scripts/tokenizers.js
View File

@ -393,7 +393,7 @@ export function getTokenizerModel() {
*/
export function countTokensOpenAI(messages, full = false) {
const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE;
const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE && oai_settings.use_google_tokenizer;
let tokenizerEndpoint = '';
if(shouldTokenizeAI21) {
tokenizerEndpoint = '/api/tokenizers/ai21/count';