rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2794 from SillyTavern/more-tokenizers

Browse Source 
Implement downloadable tokenizers
This commit is contained in:
Cohee
2024-09-07 23:17:52 +03:00
committed by GitHub
parent a6e5024122 81251b073a
commit 37b3184cda
6 changed files with 187 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
public/scripts/textgen-models.js
View File

@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
export function getCurrentOpenRouterModelTokenizer() {
const modelId = textgen_settings.openrouter_model;
const model = openRouterModels.find(x => x.id === modelId);
if (modelId?.includes('jamba')) {
return tokenizers.JAMBA;
}
switch (model?.architecture?.tokenizer) {
case 'Llama2':
return tokenizers.LLAMA;
@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
return tokenizers.GEMMA;
case 'Claude':
return tokenizers.CLAUDE;
case 'Cohere':
return tokenizers.COMMAND_R;
case 'Qwen':
return tokenizers.QWEN2;
default:
return tokenizers.OPENAI;
}

30
public/scripts/tokenizers.js
View File

@ -28,6 +28,8 @@ export const tokenizers = {
LLAMA3: 12,
GEMMA: 13,
JAMBA: 14,
QWEN2: 15,
COMMAND_R: 16,
BEST_MATCH: 99,
};
@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/jamba/decode',
count: '/api/tokenizers/jamba/encode',
},
[tokenizers.QWEN2]: {
encode: '/api/tokenizers/qwen2/encode',
decode: '/api/tokenizers/qwen2/decode',
count: '/api/tokenizers/qwen2/encode',
},
[tokenizers.COMMAND_R]: {
encode: '/api/tokenizers/command-r/encode',
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.API_TEXTGENERATIONWEBUI]: {
encode: '/api/tokenizers/remote/textgenerationwebui/encode',
count: '/api/tokenizers/remote/textgenerationwebui/encode',
@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
if (model.includes('jamba')) {
return tokenizers.JAMBA;
}
if (model.includes('command-r')) {
return tokenizers.COMMAND_R;
}
if (model.includes('qwen2')) {
return tokenizers.QWEN2;
}
}
return tokenizers.LLAMA;
@ -511,6 +529,8 @@ export function getTokenizerModel() {
const yiTokenizer = 'yi';
const gemmaTokenizer = 'gemma';
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
// Assuming no one would use it for different models.. right?
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@ -558,6 +578,12 @@ export function getTokenizerModel() {
else if (model?.architecture?.tokenizer === 'Gemini') {
return gemmaTokenizer;
}
else if (model?.architecture?.tokenizer === 'Qwen') {
return qwen2Tokenizer;
}
else if (model?.architecture?.tokenizer === 'Cohere') {
return commandRTokenizer;
}
else if (oai_settings.openrouter_model.includes('gpt-4o')) {
return gpt4oTokenizer;
}
@ -581,6 +607,10 @@ export function getTokenizerModel() {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
return commandRTokenizer;
}
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
return gemmaTokenizer;
}