mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Update GPT tokenizer
Fixes being unable to tokenize words like constructor/toString/etc
This commit is contained in:
@ -81,7 +81,7 @@ const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length))
|
||||
const cache = {}
|
||||
|
||||
function bpe(token) {
|
||||
if (token in cache) {
|
||||
if (Object.hasOwn(cache, token)) {
|
||||
return cache[token]
|
||||
}
|
||||
|
||||
@ -107,7 +107,7 @@ function bpe(token) {
|
||||
}
|
||||
))]
|
||||
|
||||
if (!(bigram in bpe_ranks)) {
|
||||
if (!(Object.hasOwn(bpe_ranks, bigram))) {
|
||||
break
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user