rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement downloadable tokenizers

Browse Source 
Closes #2574, #2754
This commit is contained in:
Cohee
2024-09-06 16:28:34 +00:00
parent 4a9401bfe2
commit 81251b073a
6 changed files with 187 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
public/scripts/tokenizers.js
View File

@ -28,6 +28,8 @@ export const tokenizers = {
LLAMA3: 12,
GEMMA: 13,
JAMBA: 14,
QWEN2: 15,
COMMAND_R: 16,
BEST_MATCH: 99,
};
@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/jamba/decode',
count: '/api/tokenizers/jamba/encode',
},
[tokenizers.QWEN2]: {
encode: '/api/tokenizers/qwen2/encode',
decode: '/api/tokenizers/qwen2/decode',
count: '/api/tokenizers/qwen2/encode',
},
[tokenizers.COMMAND_R]: {
encode: '/api/tokenizers/command-r/encode',
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.API_TEXTGENERATIONWEBUI]: {
encode: '/api/tokenizers/remote/textgenerationwebui/encode',
count: '/api/tokenizers/remote/textgenerationwebui/encode',
@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
if (model.includes('jamba')) {
return tokenizers.JAMBA;
}
if (model.includes('command-r')) {
return tokenizers.COMMAND_R;
}
if (model.includes('qwen2')) {
return tokenizers.QWEN2;
}
}
return tokenizers.LLAMA;
@ -511,6 +529,8 @@ export function getTokenizerModel() {
const yiTokenizer = 'yi';
const gemmaTokenizer = 'gemma';
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
// Assuming no one would use it for different models.. right?
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@ -558,6 +578,12 @@ export function getTokenizerModel() {
else if (model?.architecture?.tokenizer === 'Gemini') {
return gemmaTokenizer;
}
else if (model?.architecture?.tokenizer === 'Qwen') {
return qwen2Tokenizer;
}
else if (model?.architecture?.tokenizer === 'Cohere') {
return commandRTokenizer;
}
else if (oai_settings.openrouter_model.includes('gpt-4o')) {
return gpt4oTokenizer;
}
@ -581,6 +607,10 @@ export function getTokenizerModel() {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
return commandRTokenizer;
}
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
return gemmaTokenizer;
}