From 7acb61ab681dcc796daa2a4053eff5a7f40484fe Mon Sep 17 00:00:00 2001
From: based <phrog@riseup.net>
Date: Sat, 16 Dec 2023 08:47:51 +1000
Subject: [PATCH] use appropriate tokenizer with mistral completion source

---
 public/scripts/tokenizers.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index 196f3ec9c..32cd44b5e 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -384,6 +384,10 @@ export function getTokenizerModel() {
         return claudeTokenizer;
     }
 
+    if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
+        return mistralTokenizer;
+    }
+
     // Default to Turbo 3.5
     return turboTokenizer;
 }