rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added command-a-03-2025 and command-a tokenizer

Browse Source
This commit is contained in:
bmen25124
2025-03-13 21:16:08 +03:00
parent 79a8080b7d
commit a77f4045f8
4 changed files with 57 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
public/scripts/tokenizers.js
View File

@@ -32,6 +32,7 @@ export const tokenizers = {
COMMAND_R: 16,
NEMO: 17,
DEEPSEEK: 18,
COMMAND_A: 19,
BEST_MATCH: 99,
};
@@ -45,6 +46,7 @@ export const ENCODE_TOKENIZERS = [
tokenizers.JAMBA,
tokenizers.QWEN2,
tokenizers.COMMAND_R,
tokenizers.COMMAND_A,
tokenizers.NEMO,
tokenizers.DEEPSEEK,
// uncomment when NovelAI releases Kayra and Clio weights, lol
@@ -129,6 +131,11 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.COMMAND_A]: {
encode: '/api/tokenizers/command-a/encode',
decode: '/api/tokenizers/command-a/decode',
count: '/api/tokenizers/command-a/encode',
},
[tokenizers.NEMO]: {
encode: '/api/tokenizers/nemo/encode',
decode: '/api/tokenizers/nemo/decode',
@@ -340,6 +347,9 @@ export function getTokenizerBestMatch(forApi) {
if (model.includes('command-r')) {
return tokenizers.COMMAND_R;
}
if (model.includes('command-a')) {
return tokenizers.COMMAND_A;
}
if (model.includes('qwen2')) {
return tokenizers.QWEN2;
}
@@ -572,6 +582,7 @@ export function getTokenizerModel() {
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
const commandATokenizer = 'command-a';
const nemoTokenizer = 'nemo';
const deepseekTokenizer = 'deepseek';
@@ -626,6 +637,9 @@ export function getTokenizerModel() {
return qwen2Tokenizer;
}
else if (model?.architecture?.tokenizer === 'Cohere') {
if (oai_settings.openrouter_model.includes('command-a')) {
return commandATokenizer;
}
return commandRTokenizer;
}
else if (oai_settings.openrouter_model.includes('gpt-4o')) {
@@ -652,6 +666,9 @@ export function getTokenizerModel() {
}
if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
if (oai_settings.cohere_model.includes('command-a')) {
return commandATokenizer;
}
return commandRTokenizer;
}