rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use proper tokenizer for Poe

Browse Source
This commit is contained in:
SillyLossy
2023-04-13 16:26:08 +03:00
parent 661b41341e
commit 14cc5ba937
10 changed files with 539 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
public/scripts/world-info.js
View File

@@ -1,6 +1,5 @@
import { saveSettings, callPopup, token, substituteParams } from "../script.js";
import { saveSettings, callPopup, token, substituteParams, getTokenCount } from "../script.js";
import { download, debounce } from "./utils.js";
import { encode } from "./gpt-2-3-tokenizer/mod.js";
export {
world_info,
@@ -218,7 +217,7 @@ function appendWorldEntry(entry) {
saveWorldInfo();
// count tokens
const numberOfTokens = encode(value).length;
const numberOfTokens = getTokenCount(value);
$(this)
.closest(".world_entry")
.find(".world_entry_form_token_counter")
@@ -526,7 +525,7 @@ function checkWorldInfo(chat) {
}
if (
encode(worldInfoBefore + worldInfoAfter).length >= world_info_budget
getTokenCount(worldInfoBefore + worldInfoAfter) >= world_info_budget
) {
needsToScan = false;
break;