rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

use appropriate tokenizer with mistral completion source

Browse Source
This commit is contained in:
based
2023-12-16 08:47:51 +10:00
parent 5dd2e8cd88
commit 7acb61ab68
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
public/scripts/tokenizers.js
View File

@@ -384,6 +384,10 @@ export function getTokenizerModel() {
return claudeTokenizer; return claudeTokenizer;
} }
if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
return mistralTokenizer;
}
// Default to Turbo 3.5 // Default to Turbo 3.5
return turboTokenizer; return turboTokenizer;
} }