rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2024-12-12 01:17:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

Actually use getCurrentDreamGenModelTokenizer

Browse Source
This commit is contained in:
Cohee 2024-03-08 08:40:03 +02:00
parent 7639efb0fb
commit 2cdfda9d69
2 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
public/scripts/textgen-settings.js
View File

@ -14,7 +14,7 @@ import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasList
import { power_user, registerDebugFunction } from './power-user.js'; import { power_user, registerDebugFunction } from './power-user.js';
import EventSourceStream from './sse-stream.js'; import EventSourceStream from './sse-stream.js';
import { getCurrentOpenRouterModelTokenizer } from './textgen-models.js'; import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js'; import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
import { getSortableDelay, onlyUnique } from './utils.js'; import { getSortableDelay, onlyUnique } from './utils.js';
@ -319,6 +319,10 @@ function getTokenizerForTokenIds() {
return getCurrentOpenRouterModelTokenizer(); return getCurrentOpenRouterModelTokenizer();
} }
if (settings.type === DREAMGEN) {
return getCurrentDreamGenModelTokenizer();
}
return tokenizers.LLAMA; return tokenizers.LLAMA;
} }

7
public/scripts/tokenizers.js
View File

@ -5,9 +5,9 @@ import { groups, selected_group } from './group-chats.js';
import { getStringHash } from './utils.js'; import { getStringHash } from './utils.js';
import { kai_flags } from './kai-settings.js'; import { kai_flags } from './kai-settings.js';
import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer } from './textgen-settings.js'; import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer } from './textgen-settings.js';
import { getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js'; import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js';
const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP, OPENROUTER } = textgen_types; const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP, OPENROUTER, DREAMGEN } = textgen_types;
export const CHARACTERS_PER_TOKEN_RATIO = 3.35; export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown'; const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@ -206,6 +206,9 @@ export function getTokenizerBestMatch(forApi) {
if (forApi === 'textgenerationwebui' && textgen_settings.type === OPENROUTER) { if (forApi === 'textgenerationwebui' && textgen_settings.type === OPENROUTER) {
return getCurrentOpenRouterModelTokenizer(); return getCurrentOpenRouterModelTokenizer();
} }
if (forApi === 'textgenerationwebui' && textgen_settings.type === DREAMGEN) {
return getCurrentDreamGenModelTokenizer();
}
} }
return tokenizers.LLAMA; return tokenizers.LLAMA;