rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use mistral and yi tokenizers for custom token bans

Browse Source
This commit is contained in:
Cohee
2023-11-21 01:04:27 +02:00
parent 9b75e49b54
commit 1ebfddf07e
3 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
public/scripts/textgen-settings.js
View File

@@ -14,7 +14,7 @@ import {
power_user,
registerDebugFunction,
} from "./power-user.js";
import { getTextTokens, tokenizers } from "./tokenizers.js";
import { SENTENCEPIECE_TOKENIZERS, getTextTokens, tokenizers } from "./tokenizers.js";
import { onlyUnique } from "./utils.js";
export {
@@ -187,6 +187,7 @@ function getCustomTokenBans() {
return '';
}
const tokenizer = SENTENCEPIECE_TOKENIZERS.includes(power_user.tokenizer) ? power_user.tokenizer : tokenizers.LLAMA;
const result = [];
const sequences = textgenerationwebui_settings.banned_tokens
.split('\n')
@@ -218,7 +219,7 @@ function getCustomTokenBans() {
}
} else {
try {
const tokens = getTextTokens(tokenizers.LLAMA, line);
const tokens = getTextTokens(tokenizer, line);
result.push(...tokens);
} catch {
console.log(`Could not tokenize raw text: ${line}`);

9
public/scripts/tokenizers.js
View File

@@ -22,6 +22,15 @@ export const tokenizers = {
BEST_MATCH: 99,
};
export const SENTENCEPIECE_TOKENIZERS = [
tokenizers.LLAMA,
tokenizers.MISTRAL,
tokenizers.YI,
// uncomment when NovelAI releases Kayra and Clio weights, lol
//tokenizers.NERD,
//tokenizers.NERD2,
];
const objectStore = new localforage.createInstance({ name: "SillyTavern_ChatCompletions" });
let tokenCache = {};