Add API token ids from KoboldCpp

public/scripts/tokenizers.js

@@ -63,6 +63,7 @@ const TOKENIZER_URLS = {
     },
     [tokenizers.API_KOBOLD]: {
         count: '/api/tokenizers/remote/kobold/count',
+        encode: '/api/tokenizers/remote/kobold/count',
     },
     [tokenizers.MISTRAL]: {
         encode: '/api/tokenizers/mistral/encode',
@@ -617,6 +618,32 @@ function getTextTokensFromTextgenAPI(str) {
     return ids;
 }
 
+/**
+ * Calls the AI provider's tokenize API to encode a string to tokens.
+ * @param {string} str String to tokenize.
+ * @returns {number[]} Array of token ids.
+ */
+function getTextTokensFromKoboldAPI(str) {
+    let ids = [];
+
+    jQuery.ajax({
+        async: false,
+        type: 'POST',
+        url: TOKENIZER_URLS[tokenizers.API_KOBOLD].encode,
+        data: JSON.stringify({
+            text: str,
+            url: api_server,
+        }),
+        dataType: 'json',
+        contentType: 'application/json',
+        success: function (data) {
+            ids = data.ids;
+        },
+    });
+
+    return ids;
+}
+
 /**
  * Calls the underlying tokenizer model to decode token ids to text.
  * @param {string} endpoint API endpoint.
@@ -650,6 +677,8 @@ export function getTextTokens(tokenizerType, str) {
             return getTextTokens(currentRemoteTokenizerAPI(), str);
         case tokenizers.API_TEXTGENERATIONWEBUI:
             return getTextTokensFromTextgenAPI(str);
+        case tokenizers.API_KOBOLD:
+            return getTextTokensFromKoboldAPI(str);
         default: {
             const tokenizerEndpoints = TOKENIZER_URLS[tokenizerType];
             if (!tokenizerEndpoints) {

src/endpoints/tokenizers.js

@@ -562,7 +562,8 @@ router.post('/remote/kobold/count', jsonParser, async function (request, respons
 
         const data = await result.json();
         const count = data['value'];
-        return response.send({ count, ids: [] });
+        const ids = data['ids'] ?? [];
+        return response.send({ count, ids });
     } catch (error) {
         console.log(error);
         return response.send({ error: true });
@@ -617,7 +618,7 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re
 
         const data = await result.json();
         const count = legacyApi ? data?.results[0]?.tokens : (data?.length ?? data?.value);
-        const ids = legacyApi ? [] : (data?.tokens ?? []);
+        const ids = legacyApi ? [] : (data?.tokens ?? data?.ids ?? []);
 
         return response.send({ count, ids });
     } catch (error) {