rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Replace tokenizer packages with agnai versions (not crash dumping to console)

Browse Source
This commit is contained in:
Cohee
2023-08-15 18:08:34 +03:00
parent e9e9421d8a
commit ea7268febd
3 changed files with 19 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server.js
View File

@@ -172,8 +172,8 @@ function get_mancer_headers() {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
const { SentencePieceProcessor, cleanText } = require("sentencepiece-js");
const { Tokenizer } = require('@mlc-ai/web-tokenizers');
const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
const { Tokenizer } = require('@agnai/web-tokenizers');
const CHARS_PER_TOKEN = 3.35;
let spp_llama;