From ea7268febd28603452a96c61a6049b1bbac97ece Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Tue, 15 Aug 2023 18:08:34 +0300
Subject: [PATCH] Replace tokenizer packages with agnai versions (not crash
 dumping to console)

---
 package-lock.json | 30 +++++++++++++++---------------
 package.json      |  4 ++--
 server.js         |  4 ++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 28d9620b8..7573bc7cf 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,8 +9,9 @@
             "version": "1.9.7",
             "license": "AGPL-3.0",
             "dependencies": {
+                "@agnai/sentencepiece-js": "^1.1.1",
+                "@agnai/web-tokenizers": "^0.1.3",
                 "@dqbd/tiktoken": "^1.0.2",
-                "@mlc-ai/web-tokenizers": "^0.1.0",
                 "axios": "^1.4.0",
                 "command-exists": "^1.2.9",
                 "compression": "^1",
@@ -39,7 +40,6 @@
                 "png-chunks-extract": "^1.0.0",
                 "response-time": "^2.3.2",
                 "sanitize-filename": "^1.6.3",
-                "sentencepiece-js": "^1.1.0",
                 "simple-git": "^3.19.1",
                 "uniqolor": "^1.1.0",
                 "webp-converter": "2.3.2",
@@ -56,6 +56,19 @@
                 "toastr": "^2.1.4"
             }
         },
+        "node_modules/@agnai/sentencepiece-js": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/@agnai/sentencepiece-js/-/sentencepiece-js-1.1.1.tgz",
+            "integrity": "sha512-h2+XPrJVLuVLl+2+3iZPWcTw6Fs2NNulnxyh7LoI1hzHHib1wDC6KTmTrDJlLq7/lr5QFYpeMz2rlTFQrS0C0g==",
+            "dependencies": {
+                "app-root-path": "^3.1.0"
+            }
+        },
+        "node_modules/@agnai/web-tokenizers": {
+            "version": "0.1.3",
+            "resolved": "https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.3.tgz",
+            "integrity": "sha512-KlmTftToTtmb6aLVdne4NluS+POWputPF5J8v25UN/EQS+K9vahWEIe1NPRSFqBQclObkqHaj7JOnFrmnSm5MA=="
+        },
         "node_modules/@babel/generator": {
             "version": "7.18.2",
             "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.2.tgz",
@@ -596,11 +609,6 @@
             "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
             "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
         },
-        "node_modules/@mlc-ai/web-tokenizers": {
-            "version": "0.1.0",
-            "resolved": "https://registry.npmjs.org/@mlc-ai/web-tokenizers/-/web-tokenizers-0.1.0.tgz",
-            "integrity": "sha512-whiQ+40ohtAFoFOGcje1Io7BMr434Wh3hM3nBCWlJMpXxL5Rlig/AH9wjyUPsytKwWTEe7RoYPyXSbFw5Vs6Tw=="
-        },
         "node_modules/@nodelib/fs.scandir": {
             "version": "2.1.5",
             "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -3033,14 +3041,6 @@
             "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
             "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
         },
-        "node_modules/sentencepiece-js": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/sentencepiece-js/-/sentencepiece-js-1.1.0.tgz",
-            "integrity": "sha512-HN6teKCRO9tz37zbaNI3i+vMZ/JRWDt6kmZ7OVpzQv1jZHyYNmf5tE7CFpIYN86+y9TLB0cuscMdA3OHhT/MhQ==",
-            "dependencies": {
-                "app-root-path": "^3.1.0"
-            }
-        },
         "node_modules/serve-static": {
             "version": "1.15.0",
             "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
diff --git a/package.json b/package.json
index a148295e2..12c2a4738 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "dependencies": {
         "@dqbd/tiktoken": "^1.0.2",
-        "@mlc-ai/web-tokenizers": "^0.1.0",
+        "@agnai/web-tokenizers": "^0.1.3",
         "axios": "^1.4.0",
         "command-exists": "^1.2.9",
         "compression": "^1",
@@ -30,7 +30,7 @@
         "png-chunks-extract": "^1.0.0",
         "response-time": "^2.3.2",
         "sanitize-filename": "^1.6.3",
-        "sentencepiece-js": "^1.1.0",
+        "@agnai/sentencepiece-js": "^1.1.1",
         "simple-git": "^3.19.1",
         "uniqolor": "^1.1.0",
         "webp-converter": "2.3.2",
diff --git a/server.js b/server.js
index f73778eb4..5e28f6831 100644
--- a/server.js
+++ b/server.js
@@ -172,8 +172,8 @@ function get_mancer_headers() {
 
 const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-const { SentencePieceProcessor, cleanText } = require("sentencepiece-js");
-const { Tokenizer } = require('@mlc-ai/web-tokenizers');
+const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
+const { Tokenizer } = require('@agnai/web-tokenizers');
 const CHARS_PER_TOKEN = 3.35;
 
 let spp_llama;