rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Put tokenizer functions to a separate file. Cache local models token counts

Browse Source
This commit is contained in:
Cohee
2023-08-23 02:38:43 +03:00
parent 7250770c5d
commit bc5fc67906
11 changed files with 360 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
public/scripts/extensions/infinity-context/index.js
View File

@ -1,6 +1,7 @@
import { saveSettingsDebounced, getCurrentChatId, system_message_types, extension_prompt_types, eventSource, event_types, getRequestHeaders, CHARACTERS_PER_TOKEN_RATIO, substituteParams, max_context, } from "../../../script.js";
import { saveSettingsDebounced, getCurrentChatId, system_message_types, extension_prompt_types, eventSource, event_types, getRequestHeaders, substituteParams, } from "../../../script.js";
import { humanizedDateTime } from "../../RossAscends-mods.js";
import { getApiUrl, extension_settings, getContext, doExtrasFetch } from "../../extensions.js";
import { CHARACTERS_PER_TOKEN_RATIO } from "../../tokenizers.js";
import { getFileText, onlyUnique, splitRecursive } from "../../utils.js";
export { MODULE_NAME };