rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Horde doesn't support API tokenizers

Browse Source
This commit is contained in:
Cohee
2023-12-10 16:21:06 +02:00
parent f54bf99006
commit 9acef0fae6
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
public/scripts/tokenizers.js
View File

@@ -211,7 +211,6 @@ export function getTokenizerBestMatch(forApi) {
function currentRemoteTokenizerAPI() { function currentRemoteTokenizerAPI() {
switch (main_api) { switch (main_api) {
case 'kobold': case 'kobold':
case 'koboldhorde':
return tokenizers.API_KOBOLD; return tokenizers.API_KOBOLD;
case 'textgenerationwebui': case 'textgenerationwebui':
return tokenizers.API_TEXTGENERATIONWEBUI; return tokenizers.API_TEXTGENERATIONWEBUI;
@@ -240,7 +239,7 @@ function callTokenizer(type, str) {
const endpointUrl = TOKENIZER_URLS[type]?.count; const endpointUrl = TOKENIZER_URLS[type]?.count;
if (!endpointUrl) { if (!endpointUrl) {
console.warn('Unknown tokenizer type', type); console.warn('Unknown tokenizer type', type);
return callTokenizer(tokenizers.NONE, str); return apiFailureTokenCount(str);
} }
return countTokensFromServer(endpointUrl, str); return countTokensFromServer(endpointUrl, str);
} }
@@ -654,11 +653,13 @@ export function getTextTokens(tokenizerType, str) {
default: { default: {
const tokenizerEndpoints = TOKENIZER_URLS[tokenizerType]; const tokenizerEndpoints = TOKENIZER_URLS[tokenizerType];
if (!tokenizerEndpoints) { if (!tokenizerEndpoints) {
apiFailureTokenCount(str);
console.warn('Unknown tokenizer type', tokenizerType); console.warn('Unknown tokenizer type', tokenizerType);
return []; return [];
} }
let endpointUrl = tokenizerEndpoints.encode; let endpointUrl = tokenizerEndpoints.encode;
if (!endpointUrl) { if (!endpointUrl) {
apiFailureTokenCount(str);
console.warn('This tokenizer type does not support encoding', tokenizerType); console.warn('This tokenizer type does not support encoding', tokenizerType);
return []; return [];
} }