Put tokenizer functions to a separate file. Cache local models token counts

This commit is contained in:
Cohee
2023-08-23 02:38:43 +03:00
parent 7250770c5d
commit bc5fc67906
11 changed files with 360 additions and 308 deletions

View File

@@ -1,14 +1,14 @@
import {
getRequestHeaders,
getStoppingStrings,
getTextTokens,
max_context,
novelai_setting_names,
saveSettingsDebounced,
setGenerationParamsFromPreset
} from "../script.js";
import { getCfgPrompt } from "./extensions/cfg/util.js";
import { MAX_CONTEXT_DEFAULT, tokenizers } from "./power-user.js";
import { MAX_CONTEXT_DEFAULT } from "./power-user.js";
import { getTextTokens, tokenizers } from "./tokenizers.js";
import {
getSortableDelay,
getStringHash,