From 40b05bb6b15d91743838acbfc6ac4fd8bdac0229 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 2 Jan 2025 23:22:53 +0200
Subject: [PATCH] Add best match tokenizer logic for DeepSeek and Nemo

---
 public/scripts/tokenizers.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index b0162192d..a39be642d 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -325,6 +325,12 @@ export function getTokenizerBestMatch(forApi) {
             if (model.includes('gemma')) {
                 return tokenizers.GEMMA;
             }
+            if (model.includes('nemo') || model.includes('pixtral')) {
+                return tokenizers.NEMO;
+            }
+            if (model.includes('deepseek')) {
+                return tokenizers.DEEPSEEK;
+            }
             if (model.includes('yi')) {
                 return tokenizers.YI;
             }