From f248367ca304b8cdef9d451c12eec95b71b22f34 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Mon, 6 Nov 2023 01:26:13 +0200
Subject: [PATCH] Add Mistral tokenizer
---
public/index.html | 5 ++--
public/scripts/tokenizers.js | 11 ++++++++
.../{tokenizer.model => llama.model} | Bin
src/sentencepiece/mistral.model | Bin 0 -> 493443 bytes
src/tokenizers.js | 25 +++++++++++++++---
5 files changed, 36 insertions(+), 5 deletions(-)
rename src/sentencepiece/{tokenizer.model => llama.model} (100%)
create mode 100644 src/sentencepiece/mistral.model
diff --git a/public/index.html b/public/index.html
index 8fa60544f..28d9d71c1 100644
--- a/public/index.html
+++ b/public/index.html
@@ -2384,9 +2384,10 @@
-
+
+
@@ -5044,4 +5045,4 @@
-