rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement downloadable tokenizers

Browse Source 
Closes #2574, #2754
This commit is contained in:
Cohee
2024-09-06 16:28:34 +00:00
parent 4a9401bfe2
commit 81251b073a
6 changed files with 187 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
public/scripts/textgen-models.js
View File

@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
export function getCurrentOpenRouterModelTokenizer() {
const modelId = textgen_settings.openrouter_model;
const model = openRouterModels.find(x => x.id === modelId);
if (modelId?.includes('jamba')) {
return tokenizers.JAMBA;
}
switch (model?.architecture?.tokenizer) {
case 'Llama2':
return tokenizers.LLAMA;
@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
return tokenizers.GEMMA;
case 'Claude':
return tokenizers.CLAUDE;
case 'Cohere':
return tokenizers.COMMAND_R;
case 'Qwen':
return tokenizers.QWEN2;
default:
return tokenizers.OPENAI;
}