Add llama 3 tokenizer

2025-06-05 21:59:27 +02:00 · 2024-05-03 23:59:39 +03:00
parent 7bc87b6e28
commit 7bfd666321
8 changed files with 143 additions and 21 deletions
--- a/public/scripts/textgen-models.js
+++ b/public/scripts/textgen-models.js
@@ -440,6 +440,10 @@ export function getCurrentOpenRouterModelTokenizer() {
    switch (model?.architecture?.tokenizer) {
        case 'Llama2':
            return tokenizers.LLAMA;
+        case 'Llama3':
+            return tokenizers.LLAMA3;
+        case 'Yi':
+            return tokenizers.YI;
        case 'Mistral':
            return tokenizers.MISTRAL;
        default: