rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

#709 Add extra type safety for token counting

Browse Source
This commit is contained in:
Cohee
2023-07-14 15:33:55 +03:00
parent 5812e34dcb
commit 8987534403
2 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
public/script.js
View File

@@ -552,6 +552,10 @@ async function getClientVersion() {
} }
function getTokenCount(str, padding = undefined) { function getTokenCount(str, padding = undefined) {
if (typeof str !== 'string') {
return 0;
}
let tokenizerType = power_user.tokenizer; let tokenizerType = power_user.tokenizer;
if (main_api === 'openai') { if (main_api === 'openai') {

4
public/scripts/utils.js
View File

@@ -86,6 +86,10 @@ export async function parseJsonFile(file) {
} }
export function getStringHash(str, seed = 0) { export function getStringHash(str, seed = 0) {
if (typeof str !== 'string') {
return 0;
}
let h1 = 0xdeadbeef ^ seed, let h1 = 0xdeadbeef ^ seed,
h2 = 0x41c6ce57 ^ seed; h2 = 0x41c6ce57 ^ seed;
for (let i = 0, ch; i < str.length; i++) { for (let i = 0, ch; i < str.length; i++) {