rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use tokenizer-specific token cache

Browse Source
This commit is contained in:
Cohee
2023-08-05 19:29:57 +03:00
committed by maver
parent 55cc559fd0
commit dd89009ecd
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
public/scripts/openai.js
View File

@ -1283,14 +1283,15 @@ function countTokens(messages, full = false) {
let token_count = -1; let token_count = -1;
for (const message of messages) { for (const message of messages) {
const model = getTokenizerModel();
const hash = getStringHash(message.content); const hash = getStringHash(message.content);
const cachedCount = tokenCache[chatId][hash]; const cacheKey = `${model}-${hash}`;
const cachedCount = tokenCache[chatId][cacheKey];
if (cachedCount) { if (typeof cachedCount === 'number') {
token_count += cachedCount; token_count += cachedCount;
} }
else { else {
let model = getTokenizerModel();
jQuery.ajax({ jQuery.ajax({
async: false, async: false,
@ -1300,8 +1301,8 @@ function countTokens(messages, full = false) {
dataType: "json", dataType: "json",
contentType: "application/json", contentType: "application/json",
success: function (data) { success: function (data) {
token_count += data.token_count; token_count += Number(data.token_count);
tokenCache[chatId][hash] = data.token_count; tokenCache[chatId][cacheKey] = Number(data.token_count);
} }
}); });
} }