diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 196f3ec9c..32cd44b5e 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -384,6 +384,10 @@ export function getTokenizerModel() { return claudeTokenizer; } + if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) { + return mistralTokenizer; + } + // Default to Turbo 3.5 return turboTokenizer; }