rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Replace tokenizer packages with agnai versions (not crash dumping to console)

Browse Source
This commit is contained in:
Cohee
2023-08-15 18:08:34 +03:00
parent e9e9421d8a
commit ea7268febd
3 changed files with 19 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
package-lock.json generated
View File

@@ -9,8 +9,9 @@
"version": "1.9.7",
"license": "AGPL-3.0",
"dependencies": {
"@agnai/sentencepiece-js": "^1.1.1",
"@agnai/web-tokenizers": "^0.1.3",
"@dqbd/tiktoken": "^1.0.2",
"@mlc-ai/web-tokenizers": "^0.1.0",
"axios": "^1.4.0",
"command-exists": "^1.2.9",
"compression": "^1",
@@ -39,7 +40,6 @@
"png-chunks-extract": "^1.0.0",
"response-time": "^2.3.2",
"sanitize-filename": "^1.6.3",
"sentencepiece-js": "^1.1.0",
"simple-git": "^3.19.1",
"uniqolor": "^1.1.0",
"webp-converter": "2.3.2",
@@ -56,6 +56,19 @@
"toastr": "^2.1.4"
}
},
"node_modules/@agnai/sentencepiece-js": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@agnai/sentencepiece-js/-/sentencepiece-js-1.1.1.tgz",
"integrity": "sha512-h2+XPrJVLuVLl+2+3iZPWcTw6Fs2NNulnxyh7LoI1hzHHib1wDC6KTmTrDJlLq7/lr5QFYpeMz2rlTFQrS0C0g==",
"dependencies": {
"app-root-path": "^3.1.0"
}
},
"node_modules/@agnai/web-tokenizers": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@agnai/web-tokenizers/-/web-tokenizers-0.1.3.tgz",
"integrity": "sha512-KlmTftToTtmb6aLVdne4NluS+POWputPF5J8v25UN/EQS+K9vahWEIe1NPRSFqBQclObkqHaj7JOnFrmnSm5MA=="
},
"node_modules/@babel/generator": {
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.2.tgz",
@@ -596,11 +609,6 @@
"resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
"integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
},
"node_modules/@mlc-ai/web-tokenizers": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/@mlc-ai/web-tokenizers/-/web-tokenizers-0.1.0.tgz",
"integrity": "sha512-whiQ+40ohtAFoFOGcje1Io7BMr434Wh3hM3nBCWlJMpXxL5Rlig/AH9wjyUPsytKwWTEe7RoYPyXSbFw5Vs6Tw=="
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -3033,14 +3041,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/sentencepiece-js": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/sentencepiece-js/-/sentencepiece-js-1.1.0.tgz",
"integrity": "sha512-HN6teKCRO9tz37zbaNI3i+vMZ/JRWDt6kmZ7OVpzQv1jZHyYNmf5tE7CFpIYN86+y9TLB0cuscMdA3OHhT/MhQ==",
"dependencies": {
"app-root-path": "^3.1.0"
}
},
"node_modules/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",

4
package.json
View File

@@ -1,7 +1,7 @@
{
"dependencies": {
"@dqbd/tiktoken": "^1.0.2",
"@mlc-ai/web-tokenizers": "^0.1.0",
"@agnai/web-tokenizers": "^0.1.3",
"axios": "^1.4.0",
"command-exists": "^1.2.9",
"compression": "^1",
@@ -30,7 +30,7 @@
"png-chunks-extract": "^1.0.0",
"response-time": "^2.3.2",
"sanitize-filename": "^1.6.3",
"sentencepiece-js": "^1.1.0",
"@agnai/sentencepiece-js": "^1.1.1",
"simple-git": "^3.19.1",
"uniqolor": "^1.1.0",
"webp-converter": "2.3.2",

4
server.js
View File

@@ -172,8 +172,8 @@ function get_mancer_headers() {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
const { SentencePieceProcessor, cleanText } = require("sentencepiece-js");
const { Tokenizer } = require('@mlc-ai/web-tokenizers');
const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
const { Tokenizer } = require('@agnai/web-tokenizers');
const CHARS_PER_TOKEN = 3.35;
let spp_llama;