rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Actually use getCurrentDreamGenModelTokenizer

Browse Source
This commit is contained in:
Cohee
2024-03-08 08:40:03 +02:00
parent 7639efb0fb
commit 2cdfda9d69
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
public/scripts/textgen-settings.js
View File

@ -14,7 +14,7 @@ import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasList
import { power_user, registerDebugFunction } from './power-user.js';
import EventSourceStream from './sse-stream.js';
import { getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
import { getSortableDelay, onlyUnique } from './utils.js';
@ -319,6 +319,10 @@ function getTokenizerForTokenIds() {
return getCurrentOpenRouterModelTokenizer();
}
if (settings.type === DREAMGEN) {
return getCurrentDreamGenModelTokenizer();
}
return tokenizers.LLAMA;
}