From 84283bc2b49b9cc3aad7bbbb0c6a1efaf4644c59 Mon Sep 17 00:00:00 2001
From: Cohee <titan0599@gmail.com>
Date: Fri, 4 Aug 2023 14:17:05 +0300
Subject: [PATCH] Add "Best match" tokenizer option

---
 default/settings.json        |  2 +-
 public/index.html            |  7 ++++---
 public/script.js             | 25 ++++++++++++++++++++++++-
 public/scripts/power-user.js |  3 ++-
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/default/settings.json b/default/settings.json
index 09e42ad90..ddf021549 100644
--- a/default/settings.json
+++ b/default/settings.json
@@ -59,7 +59,7 @@
         "trusted_workers_only": false
     },
     "power_user": {
-        "tokenizer": 3,
+        "tokenizer": 99,
         "token_padding": 64,
         "collapse_newlines": false,
         "pygmalion_formatting": 0,
diff --git a/public/index.html b/public/index.html
index 9b66b8a56..eb4d99024 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1556,11 +1556,11 @@
                     </div>
                     <div id="textgenerationwebui_api" style="display: none;position: relative;">
                         <form action="javascript:void(null);" method="post" enctype="multipart/form-data">
-                            If you are using: 
+                            If you are using:
                             <div class="flex-container indent20p">
                                 <a href="https://github.com/oobabooga/text-generation-webui" target="_blank">
                                     oobabooga/text-generation-webui
-                                </a>, 
+                                </a>
                                 <span data-i18n="Make sure you run it with">
                                     Make sure you run it with <tt>--api</tt> flag
                                 </span>
@@ -1568,7 +1568,7 @@
                             <div class="flex-container indent20p">
                                 <a href="https://mancer.tech/" target="_blank">
                                     Mancer AI
-                                </a>, 
+                                </a>
                                 <label class="checkbox_label" for="use-mancer-api-checkbox">
                                     <span data-i18n="Use API key (Only required for Mancer)">
                                         Click this box (and add your API key!):
@@ -1961,6 +1961,7 @@
                                 </a>
                             </h4>
                             <select id="tokenizer">
+                                <option value="99">Best match (recommended)</option>
                                 <option value="0">None / Estimated</option>
                                 <option value="1">GPT-3 (OpenAI)</option>
                                 <option value="2">GPT-3 (Alternative / Classic)</option>
diff --git a/public/script.js b/public/script.js
index 70205f9d1..ff8b45380 100644
--- a/public/script.js
+++ b/public/script.js
@@ -546,6 +546,25 @@ async function getClientVersion() {
     }
 }
 
+function getTokenizerBestMatch() {
+    if (main_api === 'novel') {
+        if (nai_settings.model_novel.includes('krake') || nai_settings.model_novel.includes('euterpe')) {
+            return tokenizers.CLASSIC;
+        }
+        if (nai_settings.model_novel.includes('clio')) {
+            return tokenizers.NERD;
+        }
+        if (nai_settings.model_novel.includes('kayra')) {
+            return tokenizers.NERD2;
+        }
+    }
+    if (main_api === 'kobold' || main_api === 'textgenerationwebui' || main_api === 'koboldhorde') {
+        return tokenizers.LLAMA;
+    }
+
+    return power_user.NONE;
+}
+
 function getTokenCount(str, padding = undefined) {
     if (typeof str !== 'string') {
         return 0;
@@ -563,6 +582,10 @@ function getTokenCount(str, padding = undefined) {
         }
     }
 
+    if (tokenizerType === tokenizers.BEST_MATCH) {
+        tokenizerType = getTokenizerBestMatch();
+    }
+
     if (padding === undefined) {
         padding = 0;
     }
@@ -7729,7 +7752,7 @@ $(document).ready(function () {
         const enabled = $("#use-mancer-api-checkbox").prop("checked");
         $("#mancer-api-ui").toggle(enabled);
         api_use_mancer_webui = enabled;
-        saveSettingsDebounced(); 
+        saveSettingsDebounced();
         getStatus();
     });
 
diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js
index d28119a61..0ff013843 100644
--- a/public/scripts/power-user.js
+++ b/public/scripts/power-user.js
@@ -71,6 +71,7 @@ const tokenizers = {
     NERD: 4,
     NERD2: 5,
     API: 6,
+    BEST_MATCH: 99,
 }
 
 const send_on_enter_options = {
@@ -87,7 +88,7 @@ export const persona_description_positions = {
 }
 
 let power_user = {
-    tokenizer: tokenizers.CLASSIC,
+    tokenizer: tokenizers.BEST_MATCH,
     token_padding: 64,
     collapse_newlines: false,
     pygmalion_formatting: pygmalion_options.AUTO,