rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Mistral Nemo downloadable tokenizer

Browse Source
This commit is contained in:
Cohee
2024-09-19 00:10:22 +03:00
parent 6d731bbefe
commit a782dcec33
3 changed files with 37 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
public/scripts/tokenizers.js
View File

@ -30,6 +30,7 @@ export const tokenizers = {
JAMBA: 14,
QWEN2: 15,
COMMAND_R: 16,
NEMO: 17,
BEST_MATCH: 99,
};
@ -43,6 +44,7 @@ export const ENCODE_TOKENIZERS = [
tokenizers.JAMBA,
tokenizers.QWEN2,
tokenizers.COMMAND_R,
tokenizers.NEMO,
// uncomment when NovelAI releases Kayra and Clio weights, lol
//tokenizers.NERD,
//tokenizers.NERD2,
@ -121,6 +123,11 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.NEMO]: {
encode: '/api/tokenizers/nemo/encode',
decode: '/api/tokenizers/nemo/decode',
count: '/api/tokenizers/nemo/encode',
},
[tokenizers.API_TEXTGENERATIONWEBUI]: {
encode: '/api/tokenizers/remote/textgenerationwebui/encode',
count: '/api/tokenizers/remote/textgenerationwebui/encode',
@ -535,6 +542,7 @@ export function getTokenizerModel() {
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
const nemoTokenizer = 'nemo';
// Assuming no one would use it for different models.. right?
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@ -628,6 +636,9 @@ export function getTokenizerModel() {
}
if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
if (oai_settings.mistralai_model.includes('nemo') || oai_settings.mistralai_model.includes('pixtral')) {
return nemoTokenizer;
}
return mistralTokenizer;
}