rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-03-02 10:57:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add custom endpoint type

Browse Source
This commit is contained in:
Cohee 2023-12-20 18:29:03 +02:00
parent 041b9d4b01
commit 5734dbd17c
10 changed files with 144 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
public/index.html
View File

@ -437,14 +437,15 @@
Streaming</span>
</label>
<div class="toggle-description justifyLeft">
<span data-i18n="Display the response bit by bit as it is generated.">Display
the response bit by bit as it is generated.</span><br>
<span data-i18n="When this is off, responses will be displayed all at once when they are complete.">When
this is off, responses will be displayed all at once when they are
complete.</span>
<span data-i18n="Display the response bit by bit as it is generated.">
Display the response bit by bit as it is generated.
</span><br>
<span data-i18n="When this is off, responses will be displayed all at once when they are complete.">
When this is off, responses will be displayed all at once when they are complete.
</span>
</div>
</div>
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,mistralai">
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,mistralai,custom">
<div class="range-block-title" data-i18n="Temperature">
Temperature
</div>
@ -457,7 +458,7 @@
</div>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21">
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom">
<div class="range-block-title" data-i18n="Frequency Penalty">
Frequency Penalty
</div>
@ -470,7 +471,7 @@
</div>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21">
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,ai21,custom">
<div class="range-block-title" data-i18n="Presence Penalty">
Presence Penalty
</div>
@ -509,7 +510,7 @@
</div>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="openai,claude,openrouter,ai21,scale,makersuite,mistralai">
<div data-newbie-hidden class="range-block" data-source="openai,claude,openrouter,ai21,scale,makersuite,mistralai,custom">
<div class="range-block-title" data-i18n="Top-p">
Top P
</div>
@ -733,6 +734,9 @@
</div>
<div class="wide100p">
<input id="openai_reverse_proxy" type="text" class="text_pole" placeholder="https://api.openai.com/v1" maxlength="500" />
<small class="reverse_proxy_warning">
Doesn't work? Try adding <tt>/v1</tt> at the end!
</small>
</div>
</div>
<div class="range-block" data-source="openai,claude">
@ -749,7 +753,7 @@
<div id="openai_proxy_password_show" title="Peek a password" class="menu_button fa-solid fa-eye-slash fa-fw"></div>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,mistralai">
<div data-newbie-hidden class="range-block" data-source="openai,openrouter,mistralai,custom">
<div class="range-block-title justifyLeft" data-i18n="Seed">
Seed
</div>
@ -1921,6 +1925,7 @@
<option value="ai21">AI21</option>
<option value="makersuite">Google MakerSuite</option>
<option value="mistralai">MistralAI</option>
<option value="custom">Custom (OpenAI-compatible)</option>
</select>
<form id="openai_form" data-source="openai" action="javascript:void(null);" method="post" enctype="multipart/form-data">
<h4><span data-i18n="OpenAI API key">OpenAI API key</span></h4>
@ -2225,6 +2230,35 @@
</select>
</div>
</form>
<form id="custom_form" data-source="custom">
<h4 data-i18n="Endpoint">Endpoint URL</h4>
<div class="flex-container">
<input id="custom_api_url_text" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" placeholder="Example: http://localhost:1234/v1">
</div>
<div>
<small>
Doesn't work? Try adding <code>/v1</code> at the end of the URL!
</small>
</div>
<h4 data-i18n="Custom API Key">Custom API Key</h4>
<div class="flex-container">
<input id="api_key_custom" name="api_key_custom" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_custom"></div>
</div>
<div data-for="api_key_custom" class="neutral_warning">
For privacy reasons, your API key will be hidden after you reload the page.
</div>
<h4>Enter a Model ID</h4>
<div class="flex-container">
<input id="custom_model_id" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" placeholder="Example: gpt-3.5-turbo">
</div>
<h4 data-i18n="Available Models">Available Models</h4>
<div class="flex-container">
<select id="model_custom_select">
<option>-- Connect to the API --</option>
</select>
</div>
</form>
<div class="flex-container flex">
<div id="api_button_openai" class="api_button menu_button menu_button_icon" type="submit" data-i18n="Connect">Connect</div>
<div class="api_loading menu_button" data-i18n="Cancel">Cancel</div>

22
public/script.js
View File

@ -5453,6 +5453,7 @@ function changeMainAPI() {
case chat_completion_sources.AI21:
case chat_completion_sources.MAKERSUITE:
case chat_completion_sources.MISTRALAI:
case chat_completion_sources.CUSTOM:
default:
setupChatCompletionPromptManager(oai_settings);
break;
@ -7572,43 +7573,48 @@ const CONNECT_API_MAP = {
},
'oai': {
selected: 'openai',
source: 'openai',
button: '#api_button_openai',
source: chat_completion_sources.OPENAI,
},
'claude': {
selected: 'openai',
source: 'claude',
button: '#api_button_openai',
source: chat_completion_sources.CLAUDE,
},
'windowai': {
selected: 'openai',
source: 'windowai',
button: '#api_button_openai',
source: chat_completion_sources.WINDOWAI,
},
'openrouter': {
selected: 'openai',
source: 'openrouter',
button: '#api_button_openai',
source: chat_completion_sources.OPENROUTER,
},
'scale': {
selected: 'openai',
source: 'scale',
button: '#api_button_openai',
source: chat_completion_sources.SCALE,
},
'ai21': {
selected: 'openai',
source: 'ai21',
button: '#api_button_openai',
source: chat_completion_sources.AI21,
},
'makersuite': {
selected: 'openai',
source: 'makersuite',
button: '#api_button_openai',
source: chat_completion_sources.MAKERSUITE,
},
'mistralai': {
selected: 'openai',
source: 'mistralai',
button: '#api_button_openai',
source: chat_completion_sources.MISTRALAI,
},
'custom': {
selected: 'openai',
button: '#api_button_openai',
source: chat_completion_sources.CUSTOM,
},
};

1
public/scripts/RossAscends-mods.js
View File

@ -399,6 +399,7 @@ function RA_autoconnect(PrevApi) {
|| (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21)
|| (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE)
|| (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI)
|| (secret_state[SECRET_KEYS.CUSTOM] && oai_settings.chat_completion_source == chat_completion_sources.CUSTOM)
) {
$('#api_button_openai').trigger('click');
}

61
public/scripts/openai.js
View File

@ -165,6 +165,7 @@ export const chat_completion_sources = {
AI21: 'ai21',
MAKERSUITE: 'makersuite',
MISTRALAI: 'mistralai',
CUSTOM: 'custom',
};
const prefixMap = selected_group ? {
@ -210,6 +211,8 @@ const default_settings = {
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium',
custom_model: '',
custom_url: '',
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
@ -266,6 +269,8 @@ const oai_settings = {
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium',
custom_model: '',
custom_url: '',
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
@ -1266,6 +1271,8 @@ function getChatCompletionModel() {
return oai_settings.ai21_model;
case chat_completion_sources.MISTRALAI:
return oai_settings.mistralai_model;
case chat_completion_sources.CUSTOM:
return oai_settings.custom_model;
default:
throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`);
}
@ -1480,7 +1487,7 @@ async function sendOpenAIRequest(type, messages, signal) {
return sendWindowAIRequest(messages, signal, stream);
}
const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER, chat_completion_sources.SCALE];
const logitBiasSources = [chat_completion_sources.OPENAI, chat_completion_sources.OPENROUTER, chat_completion_sources.SCALE, chat_completion_sources.CUSTOM];
if (oai_settings.bias_preset_selected
&& logitBiasSources.includes(oai_settings.chat_completion_source)
&& Array.isArray(oai_settings.bias_presets[oai_settings.bias_preset_selected])
@ -2311,6 +2318,8 @@ function loadOpenAISettings(data, settings) {
oai_settings.openrouter_force_instruct = settings.openrouter_force_instruct ?? default_settings.openrouter_force_instruct;
oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model;
oai_settings.custom_model = settings.custom_model ?? default_settings.custom_model;
oai_settings.custom_url = settings.custom_url ?? default_settings.custom_url;
oai_settings.google_model = settings.google_model ?? default_settings.google_model;
oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
@ -2355,6 +2364,8 @@ function loadOpenAISettings(data, settings) {
$(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).attr('selected', true);
$('#model_mistralai_select').val(oai_settings.mistralai_model);
$(`#model_mistralai_select option[value="${oai_settings.mistralai_model}"`).attr('selected', true);
$('#custom_model_id').val(oai_settings.custom_model);
$('#custom_api_url_text').val(oai_settings.custom_url);
$('#openai_max_context').val(oai_settings.openai_max_context);
$('#openai_max_context_counter').val(`${oai_settings.openai_max_context}`);
$('#model_openrouter_select').val(oai_settings.openrouter_model);
@ -2466,7 +2477,11 @@ async function getStatusOpen() {
validateReverseProxy();
}
const canBypass = oai_settings.chat_completion_source === chat_completion_sources.OPENAI && oai_settings.bypass_status_check;
if (oai_settings.chat_completion_source === chat_completion_sources.CUSTOM) {
custom_url: oai_settings.custom_url;
}
const canBypass = (oai_settings.chat_completion_source === chat_completion_sources.OPENAI && oai_settings.bypass_status_check) || oai_settings.chat_completion_source === chat_completion_sources.CUSTOM;
if (canBypass) {
setOnlineStatus('Status check bypassed');
}
@ -2533,6 +2548,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
openrouter_sort_models: settings.openrouter_sort_models,
ai21_model: settings.ai21_model,
mistralai_model: settings.mistralai_model,
custom_model: settings.custom_model,
google_model: settings.google_model,
temperature: settings.temp_openai,
frequency_penalty: settings.freq_pen_openai,
@ -2905,6 +2921,8 @@ function onSettingsPresetChange() {
openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false],
ai21_model: ['#model_ai21_select', 'ai21_model', false],
mistralai_model: ['#model_mistralai_select', 'mistralai_model', false],
custom_model: ['#custom_model_id', 'custom_model', false],
custom_url: ['#custom_api_url_text', 'custom_url', false],
google_model: ['#model_google_select', 'google_model', false],
openai_max_context: ['#openai_max_context', 'openai_max_context', false],
openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false],
@ -3094,6 +3112,12 @@ async function onModelChange() {
oai_settings.mistralai_model = value;
}
if (value && $(this).is('#model_custom_select')) {
console.log('Custom model changed to', value);
oai_settings.custom_model = value;
$('#custom_model_id').val(value).trigger('input');
}
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
@ -3241,6 +3265,12 @@ async function onModelChange() {
$('#top_k_openai').attr('max', 200).val(oai_settings.top_k_openai).trigger('input');
}
if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
$('#openai_max_context').attr('max', unlocked_max);
oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context);
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
}
$('#openai_max_context_counter').attr('max', Number($('#openai_max_context').attr('max')));
saveSettingsDebounced();
@ -3383,6 +3413,19 @@ async function onConnectButtonClick(e) {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
const api_key_custom = String($('#api_key_custom').val()).trim();
if (api_key_custom.length) {
await writeSecret(SECRET_KEYS.CUSTOM, api_key_custom);
}
if (!oai_settings.custom_url) {
console.log('No API URL saved for Custom');
return;
}
}
startStatusLoading();
saveSettingsDebounced();
await getStatusOpen();
@ -3418,6 +3461,9 @@ function toggleChatCompletionForms() {
else if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) {
$('#model_mistralai_select').trigger('change');
}
else if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
$('#model_custom_select').trigger('change');
}
$('[data-source]').each(function () {
const validSources = $(this).data('source').split(',');
$(this).toggle(validSources.includes(oai_settings.chat_completion_source));
@ -3780,6 +3826,16 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#custom_api_url_text').on('input', function () {
oai_settings.custom_url = String($(this).val());
saveSettingsDebounced();
});
$('#custom_model_id').on('input', function () {
oai_settings.custom_model = String($(this).val());
saveSettingsDebounced();
});
$(document).on('input', '#openai_settings .autoSetHeight', function () {
resetScrollHeight($(this));
});
@ -3796,6 +3852,7 @@ $(document).ready(async function () {
$('#openrouter_sort_models').on('change', onOpenrouterModelSortChange);
$('#model_ai21_select').on('change', onModelChange);
$('#model_mistralai_select').on('change', onModelChange);
$('#model_custom_select').on('change', onModelChange);
$('#settings_preset_openai').on('change', onSettingsPresetChange);
$('#new_oai_preset').on('click', onNewPresetClick);
$('#delete_oai_preset').on('click', onDeletePresetClick);

2
public/scripts/secrets.js
View File

@ -16,6 +16,7 @@ export const SECRET_KEYS = {
SERPAPI: 'api_key_serpapi',
MISTRALAI: 'api_key_mistralai',
TOGETHERAI: 'api_key_togetherai',
CUSTOM: 'api_key_custom',
};
const INPUT_MAP = {
@ -32,6 +33,7 @@ const INPUT_MAP = {
[SECRET_KEYS.APHRODITE]: '#api_key_aphrodite',
[SECRET_KEYS.TABBY]: '#api_key_tabby',
[SECRET_KEYS.MISTRALAI]: '#api_key_mistralai',
[SECRET_KEYS.CUSTOM]: '#api_key_custom',
[SECRET_KEYS.TOGETHERAI]: '#api_key_togetherai',
};

4
public/scripts/tokenizers.js
View File

@ -388,6 +388,10 @@ export function getTokenizerModel() {
return mistralTokenizer;
}
if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
return oai_settings.custom_model;
}
// Default to Turbo 3.5
return turboTokenizer;
}

4
public/style.css
View File

@ -3531,11 +3531,11 @@ a {
display: none;
}
.reverse_proxy_warning {
.reverse_proxy_warning:not(small) {
color: var(--warning);
background-color: var(--black70a);
text-shadow: none !important;
margin-top: 12px !important;
margin-top: 5px !important;
border-radius: 5px;
padding: 3px;
border: 1px solid var(--SmartThemeBorderColor);

1
src/constants.js
View File

@ -160,6 +160,7 @@ const CHAT_COMPLETION_SOURCES = {
AI21: 'ai21',
MAKERSUITE: 'makersuite',
MISTRALAI: 'mistralai',
CUSTOM: 'custom',
};
const UPLOADS_PATH = './uploads';

20
src/endpoints/backends/chat-completions.js
View File

@ -502,12 +502,16 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
api_url = 'https://api.mistral.ai/v1';
api_key_openai = readSecret(SECRET_KEYS.MISTRALAI);
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
api_url = request.body.custom_url;
api_key_openai = readSecret(SECRET_KEYS.CUSTOM);
headers = {};
} else {
console.log('This chat completion source is not supported yet.');
return response_getstatus_openai.status(400).send({ error: true });
}
if (!api_key_openai && !request.body.reverse_proxy) {
if (!api_key_openai && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
console.log('OpenAI API key is missing.');
return response_getstatus_openai.status(400).send({ error: true });
}
@ -657,7 +661,7 @@ router.post('/generate', jsonParser, function (request, response) {
let headers;
let bodyParams;
if (request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER) {
if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.OPENAI);
headers = {};
@ -666,7 +670,7 @@ router.post('/generate', jsonParser, function (request, response) {
if (getConfigValue('openai.randomizeUserId', false)) {
bodyParams['user'] = uuidv4();
}
} else {
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
apiUrl = 'https://openrouter.ai/api/v1';
apiKey = readSecret(SECRET_KEYS.OPENROUTER);
// OpenRouter needs to pass the referer: https://openrouter.ai/docs
@ -676,9 +680,17 @@ router.post('/generate', jsonParser, function (request, response) {
if (request.body.use_fallback) {
bodyParams['route'] = 'fallback';
}
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
apiUrl = request.body.custom_url;
apiKey = readSecret(SECRET_KEYS.CUSTOM);
headers = {};
bodyParams = {};
} else {
console.log('This chat completion source is not supported yet.');
return response.status(400).send({ error: true });
}
if (!apiKey && !request.body.reverse_proxy) {
if (!apiKey && !request.body.reverse_proxy && request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.CUSTOM) {
console.log('OpenAI API key is missing.');
return response.status(400).send({ error: true });
}

1
src/endpoints/secrets.js
View File

@ -27,6 +27,7 @@ const SECRET_KEYS = {
SERPAPI: 'api_key_serpapi',
TOGETHERAI: 'api_key_togetherai',
MISTRALAI: 'api_key_mistralai',
CUSTOM: 'api_key_custom',
};
/**