Update GPT tokenizer

Fixes being unable to tokenize words like constructor/toString/etc
This commit is contained in:
SillyLossy
2023-02-23 15:07:05 +02:00
parent e13c1a3877
commit 5bea179f3b
2 changed files with 5 additions and 5 deletions

View File

@ -81,7 +81,7 @@ const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length))
const cache = {}
function bpe(token) {
if (token in cache) {
if (Object.hasOwn(cache, token)) {
return cache[token]
}
@ -107,7 +107,7 @@ function bpe(token) {
}
))]
if (!(bigram in bpe_ranks)) {
if (!(Object.hasOwn(bpe_ranks, bigram))) {
break
}