rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/update-tokenizer' into dev

Browse Source
This commit is contained in:
SillyLossy
2023-03-08 16:53:08 +02:00
parent d15ad3ef8f 5bea179f3b
commit f91bcb6c73
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
public/scripts/gpt-2-3-tokenizer/README.md
View File

@@ -5,20 +5,20 @@ GPT-2/3 byte pair encoder/decoder/tokenizer based on [@latitudegames/GPT-3-Encod
See also: [JS byte pair encoder for OpenAI's CLIP model](https://github.com/josephrocca/clip-bpe-js).
```js
import {encode, decode} from "https://deno.land/x/gpt_2_3_tokenizer@v0.0.1/mod.js";
import {encode, decode} from "https://deno.land/x/gpt_2_3_tokenizer@v0.0.2/mod.js";
let text = "hello world";
console.log(encode(text)); // [258, 18798, 995]
console.log(decode(encode(text))); // "hello world"
```
or:
```js
let mod = await import("https://deno.land/x/gpt_2_3_tokenizer@v0.0.1/mod.js");
let mod = await import("https://deno.land/x/gpt_2_3_tokenizer@v0.0.2/mod.js");
mod.encode("hello world"); // [258, 18798, 995]
```
or to include it as a global variable in the browser:
```html
<script type=module>
import tokenizer from "https://deno.land/x/gpt_2_3_tokenizer@v0.0.1/mod.js";
import tokenizer from "https://deno.land/x/gpt_2_3_tokenizer@v0.0.2/mod.js";
window.tokenizer = tokenizer;
</script>
```

4
public/scripts/gpt-2-3-tokenizer/mod.js
View File

@@ -81,7 +81,7 @@ const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length))
const cache = {}
function bpe(token) {
if (token in cache) {
if (Object.hasOwn(cache, token)) {
return cache[token]
}
@@ -107,7 +107,7 @@ function bpe(token) {
}
))]
if (!(bigram in bpe_ranks)) {
if (!(Object.hasOwn(bpe_ranks, bigram))) {
break
}