Replace tokenizer packages with agnai versions (not crash dumping to console)

package-lock.json

@@ -9,8 +9,9 @@
             "version": "1.9.7",
             "license": "AGPL-3.0",
             "dependencies": {
+                "@agnai/sentencepiece-js": "^1.1.1",
+                "@agnai/web-tokenizers": "^0.1.3",
                 "@dqbd/tiktoken": "^1.0.2",
-                "@mlc-ai/web-tokenizers": "^0.1.0",
                 "axios": "^1.4.0",
                 "command-exists": "^1.2.9",
                 "compression": "^1",
@@ -39,7 +40,6 @@
                 "png-chunks-extract": "^1.0.0",
                 "response-time": "^2.3.2",
                 "sanitize-filename": "^1.6.3",
-                "sentencepiece-js": "^1.1.0",
                 "simple-git": "^3.19.1",
                 "uniqolor": "^1.1.0",
                 "webp-converter": "2.3.2",
@@ -56,6 +56,19 @@
                 "toastr": "^2.1.4"
             }
         },
+        "node_modules/@agnai/sentencepiece-js": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/@agnai/sentencepiece-js/-/sentencepiece-js-1.1.1.tgz",
+            "integrity": "sha512-h2+XPrJVLuVLl+2+3iZPWcTw6Fs2NNulnxyh7LoI1hzHHib1wDC6KTmTrDJlLq7/lr5QFYpeMz2rlTFQrS0C0g==",
+            "dependencies": {
+                "app-root-path": "^3.1.0"
+            }
+        },
+        "node_modules/@agnai/web-tokenizers": {
+            "version": "0.1.3",
+            "resolved": "https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.3.tgz",
+            "integrity": "sha512-KlmTftToTtmb6aLVdne4NluS+POWputPF5J8v25UN/EQS+K9vahWEIe1NPRSFqBQclObkqHaj7JOnFrmnSm5MA=="
+        },
         "node_modules/@babel/generator": {
             "version": "7.18.2",
             "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.2.tgz",
@@ -596,11 +609,6 @@
             "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
             "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
         },
-        "node_modules/@mlc-ai/web-tokenizers": {
-            "version": "0.1.0",
-            "resolved": "https://registry.npmjs.org/@mlc-ai/web-tokenizers/-/web-tokenizers-0.1.0.tgz",
-            "integrity": "sha512-whiQ+40ohtAFoFOGcje1Io7BMr434Wh3hM3nBCWlJMpXxL5Rlig/AH9wjyUPsytKwWTEe7RoYPyXSbFw5Vs6Tw=="
-        },
         "node_modules/@nodelib/fs.scandir": {
             "version": "2.1.5",
             "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -3033,14 +3041,6 @@
             "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
             "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
         },
-        "node_modules/sentencepiece-js": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/sentencepiece-js/-/sentencepiece-js-1.1.0.tgz",
-            "integrity": "sha512-HN6teKCRO9tz37zbaNI3i+vMZ/JRWDt6kmZ7OVpzQv1jZHyYNmf5tE7CFpIYN86+y9TLB0cuscMdA3OHhT/MhQ==",
-            "dependencies": {
-                "app-root-path": "^3.1.0"
-            }
-        },
         "node_modules/serve-static": {
             "version": "1.15.0",
             "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",

package.json

@@ -1,7 +1,7 @@
 {
     "dependencies": {
         "@dqbd/tiktoken": "^1.0.2",
-        "@mlc-ai/web-tokenizers": "^0.1.0",
+        "@agnai/web-tokenizers": "^0.1.3",
         "axios": "^1.4.0",
         "command-exists": "^1.2.9",
         "compression": "^1",
@@ -30,7 +30,7 @@
         "png-chunks-extract": "^1.0.0",
         "response-time": "^2.3.2",
         "sanitize-filename": "^1.6.3",
-        "sentencepiece-js": "^1.1.0",
+        "@agnai/sentencepiece-js": "^1.1.1",
         "simple-git": "^3.19.1",
         "uniqolor": "^1.1.0",
         "webp-converter": "2.3.2",

server.js

@@ -172,8 +172,8 @@ function get_mancer_headers() {
 
 const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-const { SentencePieceProcessor, cleanText } = require("sentencepiece-js");
+const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
-const { Tokenizer } = require('@mlc-ai/web-tokenizers');
+const { Tokenizer } = require('@agnai/web-tokenizers');
 const CHARS_PER_TOKEN = 3.35;
 
 let spp_llama;