Split up Kobold and textgenerationwebui endpoints

The endpoint was one big if/else statement that did two entirely
different things depending on the value of main_api. It makes more sense
for those to be two separate endpoints.

public/scripts/tokenizers.js

@@ -1,4 +1,4 @@
-import { characters, getAPIServerUrl, main_api, nai_settings, online_status, this_chid } from '../script.js';
+import { characters, main_api, api_server, api_server_textgenerationwebui, nai_settings, online_status, this_chid } from '../script.js';
 import { power_user, registerDebugFunction } from './power-user.js';
 import { chat_completion_sources, model_list, oai_settings } from './openai.js';
 import { groups, selected_group } from './group-chats.js';
@@ -174,9 +174,9 @@ function callTokenizer(type, str, padding) {
         case tokenizers.YI:
             return countTokensFromServer('/api/tokenizers/yi/encode', str, padding);
         case tokenizers.API_KOBOLD:
-            return countTokensFromKoboldAPI('/api/tokenizers/remote/encode', str, padding);
+            return countTokensFromKoboldAPI('/api/tokenizers/remote/kobold/count', str, padding);
         case tokenizers.API_TEXTGENERATIONWEBUI:
-            return countTokensFromTextgenAPI('/api/tokenizers/remote/encode', str, padding);
+            return countTokensFromTextgenAPI('/api/tokenizers/remote/textgenerationwebui/encode', str, padding);
         default:
             console.warn('Unknown tokenizer type', type);
             return callTokenizer(tokenizers.NONE, str, padding);
@@ -403,17 +403,15 @@ function getServerTokenizationParams(str) {
 function getKoboldAPITokenizationParams(str) {
     return {
         text: str,
-        main_api: 'kobold',
+        url: api_server,
-        url: getAPIServerUrl(),
     };
 }
 
 function getTextgenAPITokenizationParams(str) {
     return {
         text: str,
-        main_api: 'textgenerationwebui',
         api_type: textgen_settings.type,
-        url: getAPIServerUrl(),
+        url: api_server_textgenerationwebui,
         legacy_api:
             textgen_settings.legacy_api &&
             textgen_settings.type !== MANCER,
@@ -627,7 +625,7 @@ export function getTextTokens(tokenizerType, str) {
             return getTextTokensFromServer('/api/tokenizers/openai/encode', str, model);
         }
         case tokenizers.API_TEXTGENERATIONWEBUI:
-            return getTextTokensFromTextgenAPI('/api/tokenizers/remote/encode', str);
+            return getTextTokensFromTextgenAPI('/api/tokenizers/textgenerationwebui/encode', str);
         default:
             console.warn('Calling getTextTokens with unsupported tokenizer type', tokenizerType);
             return [];

src/endpoints/tokenizers.js

@@ -536,17 +536,48 @@ router.post('/openai/count', jsonParser, async function (req, res) {
     }
 });
 
-router.post('/remote/encode', jsonParser, async function (request, response) {
+router.post('/remote/kobold/count', jsonParser, async function (request, response) {
+    if (!request.body) {
+        return response.sendStatus(400);
+    }
+    const text = String(request.body.text) || '';
+    const baseUrl = String(request.body.url);
+
+    try {
+        const args = {
+            method: 'POST',
+            body: JSON.stringify({ 'prompt': text }),
+            headers: { 'Content-Type': 'application/json' },
+        };
+
+        let url = String(baseUrl).replace(/\/$/, '');
+        url += '/extra/tokencount';
+
+        const result = await fetch(url, args);
+
+        if (!result.ok) {
+            console.log(`API returned error: ${result.status} ${result.statusText}`);
+            return response.send({ error: true });
+        }
+
+        const data = await result.json();
+        const count = data['value'];
+        return response.send({ count, ids: [] });
+    } catch (error) {
+        console.log(error);
+        return response.send({ error: true });
+    }
+});
+
+router.post('/remote/textgenerationwebui/encode', jsonParser, async function (request, response) {
     if (!request.body) {
         return response.sendStatus(400);
     }
     const text = String(request.body.text) || '';
-    const api = String(request.body.main_api);
     const baseUrl = String(request.body.url);
     const legacyApi = Boolean(request.body.legacy_api);
 
     try {
-        if (api == 'textgenerationwebui') {
         const args = {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
@@ -589,34 +620,6 @@ router.post('/remote/encode', jsonParser, async function (request, response) {
         const ids = legacyApi ? [] : (data?.tokens ?? []);
 
         return response.send({ count, ids });
-        }
-
-        else if (api == 'kobold') {
-            const args = {
-                method: 'POST',
-                body: JSON.stringify({ 'prompt': text }),
-                headers: { 'Content-Type': 'application/json' },
-            };
-
-            let url = String(baseUrl).replace(/\/$/, '');
-            url += '/extra/tokencount';
-
-            const result = await fetch(url, args);
-
-            if (!result.ok) {
-                console.log(`API returned error: ${result.status} ${result.statusText}`);
-                return response.send({ error: true });
-            }
-
-            const data = await result.json();
-            const count = data['value'];
-            return response.send({ count: count, ids: [] });
-        }
-
-        else {
-            console.log('Unknown API', api);
-            return response.send({ error: true });
-        }
     } catch (error) {
         console.log(error);
         return response.send({ error: true });