rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added command-a-03-2025 and command-a tokenizer

Browse Source
This commit is contained in:
bmen25124
2025-03-13 21:16:08 +03:00
parent 79a8080b7d
commit a77f4045f8
4 changed files with 57 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
public/index.html
View File

@@ -3333,6 +3333,7 @@
<option value="command-r-08-2024">command-r-08-2024</option>
<option value="command-r-plus-08-2024">command-r-plus-08-2024</option>
<option value="command-r7b-12-2024">command-r7b-12-2024</option>
<option value="command-a-03-2025">command-a-03-2025</option>
</optgroup>
<optgroup label="Nightly">
<option value="command-light-nightly">command-light-nightly</option>

3
public/scripts/openai.js
View File

@@ -4448,6 +4448,9 @@ async function onModelChange() {
else if (oai_settings.cohere_model.includes('command-r') || ['c4ai-aya-23', 'c4ai-aya-expanse-32b', 'command-nightly'].includes(oai_settings.cohere_model)) {
$('#openai_max_context').attr('max', max_128k);
}
else if (['command-a-03-2025'].includes(oai_settings.cohere_model)) {
$('#openai_max_context').attr('max', max_256k);
}
else if (['c4ai-aya-23-8b', 'c4ai-aya-expanse-8b'].includes(oai_settings.cohere_model)) {
$('#openai_max_context').attr('max', max_8k);
}

17
public/scripts/tokenizers.js
View File

@@ -32,6 +32,7 @@ export const tokenizers = {
COMMAND_R: 16,
NEMO: 17,
DEEPSEEK: 18,
COMMAND_A: 19,
BEST_MATCH: 99,
};
@@ -45,6 +46,7 @@ export const ENCODE_TOKENIZERS = [
tokenizers.JAMBA,
tokenizers.QWEN2,
tokenizers.COMMAND_R,
tokenizers.COMMAND_A,
tokenizers.NEMO,
tokenizers.DEEPSEEK,
// uncomment when NovelAI releases Kayra and Clio weights, lol
@@ -129,6 +131,11 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.COMMAND_A]: {
encode: '/api/tokenizers/command-a/encode',
decode: '/api/tokenizers/command-a/decode',
count: '/api/tokenizers/command-a/encode',
},
[tokenizers.NEMO]: {
encode: '/api/tokenizers/nemo/encode',
decode: '/api/tokenizers/nemo/decode',
@@ -340,6 +347,9 @@ export function getTokenizerBestMatch(forApi) {
if (model.includes('command-r')) {
return tokenizers.COMMAND_R;
}
if (model.includes('command-a')) {
return tokenizers.COMMAND_A;
}
if (model.includes('qwen2')) {
return tokenizers.QWEN2;
}
@@ -572,6 +582,7 @@ export function getTokenizerModel() {
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
const commandATokenizer = 'command-a';
const nemoTokenizer = 'nemo';
const deepseekTokenizer = 'deepseek';
@@ -626,6 +637,9 @@ export function getTokenizerModel() {
return qwen2Tokenizer;
}
else if (model?.architecture?.tokenizer === 'Cohere') {
if (oai_settings.openrouter_model.includes('command-a')) {
return commandATokenizer;
}
return commandRTokenizer;
}
else if (oai_settings.openrouter_model.includes('gpt-4o')) {
@@ -652,6 +666,9 @@ export function getTokenizerModel() {
}
if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
if (oai_settings.cohere_model.includes('command-a')) {
return commandATokenizer;
}
return commandRTokenizer;
}

43
src/endpoints/tokenizers.js
View File

@@ -222,7 +222,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
const commandRTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
const commandATokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-a.json', 'src/tokenizers/llama3.json');
const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
const deepseekTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/deepseek.json', 'src/tokenizers/llama3.json');
@@ -241,6 +242,7 @@ export const webTokenizers = [
'claude',
'llama3',
'command-r',
'command-a',
'qwen2',
'nemo',
'deepseek',
@@ -298,7 +300,11 @@ export function getWebTokenizer(model) {
}
if (model.includes('command-r')) {
return commandTokenizer;
return commandRTokenizer;
}
if (model.includes('command-a')) {
return commandATokenizer;
}
if (model.includes('qwen2')) {
@@ -469,6 +475,10 @@ export function getTokenizerModel(requestModel) {
return 'command-r';
}
if (requestModel.includes('command-a')) {
return 'command-a';
}
if (requestModel.includes('nemo')) {
return 'nemo';
}
@@ -699,7 +709,8 @@ router.post('/gpt2/encode', createTiktokenEncodingHandler('gpt2'));
router.post('/claude/encode', createWebTokenizerEncodingHandler(claude_tokenizer));
router.post('/llama3/encode', createWebTokenizerEncodingHandler(llama3_tokenizer));
router.post('/qwen2/encode', createWebTokenizerEncodingHandler(qwen2Tokenizer));
router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandTokenizer));
router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandRTokenizer));
router.post('/command-a/encode', createWebTokenizerEncodingHandler(commandATokenizer));
router.post('/nemo/encode', createWebTokenizerEncodingHandler(nemoTokenizer));
router.post('/deepseek/encode', createWebTokenizerEncodingHandler(deepseekTokenizer));
router.post('/llama/decode', createSentencepieceDecodingHandler(spp_llama));
@@ -713,7 +724,8 @@ router.post('/gpt2/decode', createTiktokenDecodingHandler('gpt2'));
router.post('/claude/decode', createWebTokenizerDecodingHandler(claude_tokenizer));
router.post('/llama3/decode', createWebTokenizerDecodingHandler(llama3_tokenizer));
router.post('/qwen2/decode', createWebTokenizerDecodingHandler(qwen2Tokenizer));
router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandTokenizer));
router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandRTokenizer));
router.post('/command-a/decode', createWebTokenizerDecodingHandler(commandATokenizer));
router.post('/nemo/decode', createWebTokenizerDecodingHandler(nemoTokenizer));
router.post('/deepseek/decode', createWebTokenizerDecodingHandler(deepseekTokenizer));
@@ -762,7 +774,12 @@ router.post('/openai/encode', async function (req, res) {
}
if (queryModel.includes('command-r')) {
const handler = createWebTokenizerEncodingHandler(commandTokenizer);
const handler = createWebTokenizerEncodingHandler(commandRTokenizer);
return handler(req, res);
}
if (queryModel.includes('command-a')) {
const handler = createWebTokenizerEncodingHandler(commandATokenizer);
return handler(req, res);
}
@@ -830,7 +847,12 @@ router.post('/openai/decode', async function (req, res) {
}
if (queryModel.includes('command-r')) {
const handler = createWebTokenizerDecodingHandler(commandTokenizer);
const handler = createWebTokenizerDecodingHandler(commandRTokenizer);
return handler(req, res);
}
if (queryModel.includes('command-a')) {
const handler = createWebTokenizerDecodingHandler(commandATokenizer);
return handler(req, res);
}
@@ -908,12 +930,19 @@ router.post('/openai/count', async function (req, res) {
}
if (model === 'command-r') {
const instance = await commandTokenizer.get();
const instance = await commandRTokenizer.get();
if (!instance) throw new Error('Failed to load the Command-R tokenizer');
num_tokens = countWebTokenizerTokens(instance, req.body);
return res.send({ 'token_count': num_tokens });
}
if (model === 'command-a') {
const instance = await commandATokenizer.get();
if (!instance) throw new Error('Failed to load the Command-A tokenizer');
num_tokens = countWebTokenizerTokens(instance, req.body);
return res.send({ 'token_count': num_tokens });
}
if (model === 'nemo') {
const instance = await nemoTokenizer.get();
if (!instance) throw new Error('Failed to load the Nemo tokenizer');