rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Inline most get(...)TokenizerParams calls 

For everything except textgenerationwebui, these params are now simple
enough that it doesn't make sense for them to be in a separate function.
This commit is contained in:
valadaptive 2023-12-09 20:35:11 -05:00
parent 30502ac949
commit 09465fbb97
1 changed files with 17 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
public/scripts/tokenizers.js
View File

@ -394,30 +394,6 @@ function getTokenCacheObject() {
return tokenCache[String(chatId)]; return tokenCache[String(chatId)];
} }
function getServerTokenizationParams(str) {
return {
text: str,
};
}
function getKoboldAPITokenizationParams(str) {
return {
text: str,
url: api_server,
};
}
function getTextgenAPITokenizationParams(str) {
return {
text: str,
api_type: textgen_settings.type,
url: api_server_textgenerationwebui,
legacy_api:
textgen_settings.legacy_api &&
textgen_settings.type !== MANCER,
};
}
/** /**
* Count tokens using the server API. * Count tokens using the server API.
* @param {string} endpoint API endpoint. * @param {string} endpoint API endpoint.
@ -432,7 +408,7 @@ function countTokensFromServer(endpoint, str, padding) {
async: false, async: false,
type: 'POST', type: 'POST',
url: endpoint, url: endpoint,
data: JSON.stringify(getServerTokenizationParams(str)), data: JSON.stringify({ text: str }),
dataType: 'json', dataType: 'json',
contentType: 'application/json', contentType: 'application/json',
success: function (data) { success: function (data) {
@ -461,7 +437,10 @@ function countTokensFromKoboldAPI(endpoint, str, padding) {
async: false, async: false,
type: 'POST', type: 'POST',
url: endpoint, url: endpoint,
data: JSON.stringify(getKoboldAPITokenizationParams(str)), data: JSON.stringify({
text: str,
url: api_server,
}),
dataType: 'json', dataType: 'json',
contentType: 'application/json', contentType: 'application/json',
success: function (data) { success: function (data) {
@ -476,6 +455,17 @@ function countTokensFromKoboldAPI(endpoint, str, padding) {
return tokenCount + padding; return tokenCount + padding;
} }
function getTextgenAPITokenizationParams(str) {
return {
text: str,
api_type: textgen_settings.type,
url: api_server_textgenerationwebui,
legacy_api:
textgen_settings.legacy_api &&
textgen_settings.type !== MANCER,
};
}
/** /**
* Count tokens using the AI provider's API. * Count tokens using the AI provider's API.
* @param {string} endpoint API endpoint. * @param {string} endpoint API endpoint.
@ -538,7 +528,7 @@ function getTextTokensFromServer(endpoint, str, model = '') {
async: false, async: false,
type: 'POST', type: 'POST',
url: endpoint, url: endpoint,
data: JSON.stringify(getServerTokenizationParams(str)), data: JSON.stringify({ text: str }),
dataType: 'json', dataType: 'json',
contentType: 'application/json', contentType: 'application/json',
success: function (data) { success: function (data) {