diff --git a/public/index.html b/public/index.html index 9403982b0..63caa13df 100644 --- a/public/index.html +++ b/public/index.html @@ -3333,6 +3333,7 @@ + diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 1b9b6db0c..a8f1725b3 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -4448,6 +4448,9 @@ async function onModelChange() { else if (oai_settings.cohere_model.includes('command-r') || ['c4ai-aya-23', 'c4ai-aya-expanse-32b', 'command-nightly'].includes(oai_settings.cohere_model)) { $('#openai_max_context').attr('max', max_128k); } + else if (['command-a-03-2025'].includes(oai_settings.cohere_model)) { + $('#openai_max_context').attr('max', max_256k); + } else if (['c4ai-aya-23-8b', 'c4ai-aya-expanse-8b'].includes(oai_settings.cohere_model)) { $('#openai_max_context').attr('max', max_8k); } diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 7cb88ab78..2abfbae21 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -32,6 +32,7 @@ export const tokenizers = { COMMAND_R: 16, NEMO: 17, DEEPSEEK: 18, + COMMAND_A: 19, BEST_MATCH: 99, }; @@ -45,6 +46,7 @@ export const ENCODE_TOKENIZERS = [ tokenizers.JAMBA, tokenizers.QWEN2, tokenizers.COMMAND_R, + tokenizers.COMMAND_A, tokenizers.NEMO, tokenizers.DEEPSEEK, // uncomment when NovelAI releases Kayra and Clio weights, lol @@ -129,6 +131,11 @@ const TOKENIZER_URLS = { decode: '/api/tokenizers/command-r/decode', count: '/api/tokenizers/command-r/encode', }, + [tokenizers.COMMAND_A]: { + encode: '/api/tokenizers/command-a/encode', + decode: '/api/tokenizers/command-a/decode', + count: '/api/tokenizers/command-a/encode', + }, [tokenizers.NEMO]: { encode: '/api/tokenizers/nemo/encode', decode: '/api/tokenizers/nemo/decode', @@ -340,6 +347,9 @@ export function getTokenizerBestMatch(forApi) { if (model.includes('command-r')) { return tokenizers.COMMAND_R; } + if (model.includes('command-a')) { + return tokenizers.COMMAND_A; + } if (model.includes('qwen2')) { return tokenizers.QWEN2; } @@ -572,6 +582,7 @@ export function getTokenizerModel() { const jambaTokenizer = 'jamba'; const qwen2Tokenizer = 'qwen2'; const commandRTokenizer = 'command-r'; + const commandATokenizer = 'command-a'; const nemoTokenizer = 'nemo'; const deepseekTokenizer = 'deepseek'; @@ -626,6 +637,9 @@ export function getTokenizerModel() { return qwen2Tokenizer; } else if (model?.architecture?.tokenizer === 'Cohere') { + if (oai_settings.openrouter_model.includes('command-a')) { + return commandATokenizer; + } return commandRTokenizer; } else if (oai_settings.openrouter_model.includes('gpt-4o')) { @@ -652,6 +666,9 @@ export function getTokenizerModel() { } if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) { + if (oai_settings.cohere_model.includes('command-a')) { + return commandATokenizer; + } return commandRTokenizer; } diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index c1468ad36..7b2783b4d 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -222,7 +222,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model'); const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model'); const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json'); const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json'); -const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json'); +const commandRTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json'); +const commandATokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-a.json', 'src/tokenizers/llama3.json'); const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json'); const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json'); const deepseekTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/deepseek.json', 'src/tokenizers/llama3.json'); @@ -241,6 +242,7 @@ export const webTokenizers = [ 'claude', 'llama3', 'command-r', + 'command-a', 'qwen2', 'nemo', 'deepseek', @@ -298,7 +300,11 @@ export function getWebTokenizer(model) { } if (model.includes('command-r')) { - return commandTokenizer; + return commandRTokenizer; + } + + if (model.includes('command-a')) { + return commandATokenizer; } if (model.includes('qwen2')) { @@ -469,6 +475,10 @@ export function getTokenizerModel(requestModel) { return 'command-r'; } + if (requestModel.includes('command-a')) { + return 'command-a'; + } + if (requestModel.includes('nemo')) { return 'nemo'; } @@ -699,7 +709,8 @@ router.post('/gpt2/encode', createTiktokenEncodingHandler('gpt2')); router.post('/claude/encode', createWebTokenizerEncodingHandler(claude_tokenizer)); router.post('/llama3/encode', createWebTokenizerEncodingHandler(llama3_tokenizer)); router.post('/qwen2/encode', createWebTokenizerEncodingHandler(qwen2Tokenizer)); -router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandTokenizer)); +router.post('/command-r/encode', createWebTokenizerEncodingHandler(commandRTokenizer)); +router.post('/command-a/encode', createWebTokenizerEncodingHandler(commandATokenizer)); router.post('/nemo/encode', createWebTokenizerEncodingHandler(nemoTokenizer)); router.post('/deepseek/encode', createWebTokenizerEncodingHandler(deepseekTokenizer)); router.post('/llama/decode', createSentencepieceDecodingHandler(spp_llama)); @@ -713,7 +724,8 @@ router.post('/gpt2/decode', createTiktokenDecodingHandler('gpt2')); router.post('/claude/decode', createWebTokenizerDecodingHandler(claude_tokenizer)); router.post('/llama3/decode', createWebTokenizerDecodingHandler(llama3_tokenizer)); router.post('/qwen2/decode', createWebTokenizerDecodingHandler(qwen2Tokenizer)); -router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandTokenizer)); +router.post('/command-r/decode', createWebTokenizerDecodingHandler(commandRTokenizer)); +router.post('/command-a/decode', createWebTokenizerDecodingHandler(commandATokenizer)); router.post('/nemo/decode', createWebTokenizerDecodingHandler(nemoTokenizer)); router.post('/deepseek/decode', createWebTokenizerDecodingHandler(deepseekTokenizer)); @@ -762,7 +774,12 @@ router.post('/openai/encode', async function (req, res) { } if (queryModel.includes('command-r')) { - const handler = createWebTokenizerEncodingHandler(commandTokenizer); + const handler = createWebTokenizerEncodingHandler(commandRTokenizer); + return handler(req, res); + } + + if (queryModel.includes('command-a')) { + const handler = createWebTokenizerEncodingHandler(commandATokenizer); return handler(req, res); } @@ -830,7 +847,12 @@ router.post('/openai/decode', async function (req, res) { } if (queryModel.includes('command-r')) { - const handler = createWebTokenizerDecodingHandler(commandTokenizer); + const handler = createWebTokenizerDecodingHandler(commandRTokenizer); + return handler(req, res); + } + + if (queryModel.includes('command-a')) { + const handler = createWebTokenizerDecodingHandler(commandATokenizer); return handler(req, res); } @@ -908,12 +930,19 @@ router.post('/openai/count', async function (req, res) { } if (model === 'command-r') { - const instance = await commandTokenizer.get(); + const instance = await commandRTokenizer.get(); if (!instance) throw new Error('Failed to load the Command-R tokenizer'); num_tokens = countWebTokenizerTokens(instance, req.body); return res.send({ 'token_count': num_tokens }); } + if (model === 'command-a') { + const instance = await commandATokenizer.get(); + if (!instance) throw new Error('Failed to load the Command-A tokenizer'); + num_tokens = countWebTokenizerTokens(instance, req.body); + return res.send({ 'token_count': num_tokens }); + } + if (model === 'nemo') { const instance = await nemoTokenizer.get(); if (!instance) throw new Error('Failed to load the Nemo tokenizer');