rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix updated tokenization via ooba API

Browse Source
This commit is contained in:
Cohee
2023-11-09 19:39:08 +02:00
parent 3780321ae6
commit 81fe9aa699
2 changed files with 22 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
public/scripts/tokenizers.js
View File

@ -363,6 +363,15 @@ function getTokenCacheObject() {
return tokenCache[String(chatId)];
}
function getRemoteTokenizationParams(str) {
return {
text: str,
api: main_api,
url: getAPIServerUrl(),
legacy_api: main_api === 'textgenerationwebui' && textgenerationwebui_settings.legacy_api && !isMancer(),
};
}
/**
* Counts token using the remote server API.
* @param {string} endpoint API endpoint.
@ -377,12 +386,7 @@ function countTokensRemote(endpoint, str, padding) {
async: false,
type: 'POST',
url: endpoint,
data: JSON.stringify({
text: str,
api: main_api,
url: getAPIServerUrl(),
legacy_api: main_api === 'textgenerationwebui' && textgenerationwebui_settings.legacy_api && !isMancer() ,
}),
data: JSON.stringify(getRemoteTokenizationParams(str)),
dataType: "json",
contentType: "application/json",
success: function (data) {
@ -425,7 +429,7 @@ function getTextTokensRemote(endpoint, str, model = '') {
async: false,
type: 'POST',
url: endpoint,
data: JSON.stringify({ text: str }),
data: JSON.stringify(getRemoteTokenizationParams(str)),
dataType: "json",
contentType: "application/json",
success: function (data) {
@ -482,6 +486,8 @@ export function getTextTokens(tokenizerType, str) {
case tokenizers.OPENAI:
const model = getTokenizerModel();
return getTextTokensRemote('/api/tokenize/openai-encode', str, model);
case tokenizers.API:
return getTextTokensRemote('/tokenize_via_api', str);
default:
console.warn("Calling getTextTokens with unsupported tokenizer type", tokenizerType);
return [];