rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add tokenizer for Gemma/Gemini

Browse Source
This commit is contained in:
Cohee
2024-08-16 00:00:43 +03:00
parent ce8b0aae96
commit e707def7dd
5 changed files with 52 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
public/scripts/textgen-models.js
View File

@ -599,6 +599,10 @@ export function getCurrentOpenRouterModelTokenizer() {
return tokenizers.YI;
case 'Mistral':
return tokenizers.MISTRAL;
case 'Gemini':
return tokenizers.GEMMA;
case 'Claude':
return tokenizers.CLAUDE;
default:
return tokenizers.OPENAI;
}