rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add additional parameters for custom endpoints

This commit is contained in:
Cohee 2023-12-20 23:39:10 +02:00
parent e42daa4098
commit 3001db3a47
6 changed files with 156 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
public/index.html
View File

@ -2263,6 +2263,7 @@
<div class="flex-container flex">
<div id="api_button_openai" class="api_button menu_button menu_button_icon" type="submit" data-i18n="Connect">Connect</div>
<div class="api_loading menu_button" data-i18n="Cancel">Cancel</div>
<div data-source="custom" id="customize_additional_parameters" class="menu_button menu_button_icon">Additional Parameters</div>
<div data-source="openrouter" id="openrouter_authorize" class="menu_button menu_button_icon" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">Authorize</div>
<div id="test_api_button" class="menu_button menu_button_icon" title="Verifies your API connection by sending a short test message. Be aware that you'll be credited for it!" data-i18n="[title]Verifies your API connection by sending a short test message. Be aware that you'll be credited for it!">Test Message</div>
</div>

3
public/scripts/extensions/shared.js
View File

@ -72,6 +72,9 @@ export async function getMultimodalCaption(base64Img, prompt) {
if (isCustom) {
requestBody.server_url = oai_settings.custom_url;
requestBody.model = oai_settings.custom_model || 'gpt-4-vision-preview';
requestBody.custom_include_headers = oai_settings.custom_include_headers;
requestBody.custom_include_body = oai_settings.custom_include_body;
requestBody.custom_exclude_body = oai_settings.custom_exclude_body;
}
function getEndpointUrl() {

55
public/scripts/openai.js
View File

@ -213,6 +213,9 @@ const default_settings = {
mistralai_model: 'mistral-medium',
custom_model: '',
custom_url: '',
custom_include_body: '',
custom_exclude_body: '',
custom_include_headers: '',
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
@ -271,6 +274,9 @@ const oai_settings = {
mistralai_model: 'mistral-medium',
custom_model: '',
custom_url: '',
custom_include_body: '',
custom_exclude_body: '',
custom_include_headers: '',
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
@ -1592,6 +1598,9 @@ async function sendOpenAIRequest(type, messages, signal) {
if (isCustom) {
generate_data['custom_url'] = oai_settings.custom_url;
generate_data['custom_include_body'] = oai_settings.custom_include_body;
generate_data['custom_exclude_body'] = oai_settings.custom_exclude_body;
generate_data['custom_include_headers'] = oai_settings.custom_include_headers;
}
if ((isOAI || isOpenRouter || isMistral || isCustom) && oai_settings.seed >= 0) {
@ -2342,6 +2351,9 @@ function loadOpenAISettings(data, settings) {
oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model;
oai_settings.custom_model = settings.custom_model ?? default_settings.custom_model;
oai_settings.custom_url = settings.custom_url ?? default_settings.custom_url;
oai_settings.custom_include_body = settings.custom_include_body ?? default_settings.custom_include_body;
oai_settings.custom_exclude_body = settings.custom_exclude_body ?? default_settings.custom_exclude_body;
oai_settings.custom_include_headers = settings.custom_include_headers ?? default_settings.custom_include_headers;
oai_settings.google_model = settings.google_model ?? default_settings.google_model;
oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
@ -2502,6 +2514,7 @@ async function getStatusOpen() {
if (oai_settings.chat_completion_source === chat_completion_sources.CUSTOM) {
$('#model_custom_select').empty();
data.custom_url = oai_settings.custom_url;
data.custom_include_headers = oai_settings.custom_include_headers;
}
const canBypass = (oai_settings.chat_completion_source === chat_completion_sources.OPENAI && oai_settings.bypass_status_check) || oai_settings.chat_completion_source === chat_completion_sources.CUSTOM;
@ -2946,6 +2959,9 @@ function onSettingsPresetChange() {
mistralai_model: ['#model_mistralai_select', 'mistralai_model', false],
custom_model: ['#custom_model_id', 'custom_model', false],
custom_url: ['#custom_api_url_text', 'custom_url', false],
custom_include_body: ['#custom_include_body', 'custom_include_body', false],
custom_exclude_body: ['#custom_exclude_body', 'custom_exclude_body', false],
custom_include_headers: ['#custom_include_headers', 'custom_include_headers', false],
google_model: ['#model_google_select', 'google_model', false],
openai_max_context: ['#openai_max_context', 'openai_max_context', false],
openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false],
@ -3308,7 +3324,7 @@ async function onNewPresetClick() {
const popupText = `
<h3>Preset name:</h3>
<h4>Hint: Use a character/group name to bind preset to a specific chat.</h4>`;
const name = await callPopup(popupText, 'input');
const name = await callPopup(popupText, 'input', oai_settings.preset_settings_openai);
if (!name) {
return;
@ -3537,6 +3553,42 @@ function updateScaleForm() {
}
}
function onCustomizeParametersClick() {
const template = $(`
<div class="flex-container flexFlowColumn height100p">
<h3>Additional Parameters</h3>
<div class="flex1 flex-container flexFlowColumn">
<h4>Include Body Parameters</h4>
<textarea id="custom_include_body" class="flex1" placeholder="Parameters to be included in the Chat Completion request body (YAML object)&#10;&#10;Example:&#10;- top_k: 20&#10;- repetition_penalty: 1.1"></textarea>
</div>
<div class="flex1 flex-container flexFlowColumn">
<h4>Exclude Body Parameters</h4>
<textarea id="custom_exclude_body" class="flex1" placeholder="Parameters to be excluded from the Chat Completion request body (YAML array)&#10;&#10;Example:&#10;- frequency_penalty&#10;- presence_penalty"></textarea>
</div>
<div class="flex1 flex-container flexFlowColumn">
<h4>Include Request Headers</h4>
<textarea id="custom_include_headers" class="flex1" placeholder="Additional headers for Chat Completion requests (YAML object)&#10;&#10;Example:&#10;- CustomHeader: custom-value&#10;- AnotherHeader: custom-value"></textarea>
</div>
</div>`);
template.find('#custom_include_body').val(oai_settings.custom_include_body).on('input', function() {
oai_settings.custom_include_body = String($(this).val());
saveSettingsDebounced();
});
template.find('#custom_exclude_body').val(oai_settings.custom_exclude_body).on('input', function() {
oai_settings.custom_exclude_body = String($(this).val());
saveSettingsDebounced();
});
template.find('#custom_include_headers').val(oai_settings.custom_include_headers).on('input', function() {
oai_settings.custom_include_headers = String($(this).val());
saveSettingsDebounced();
});
callPopup(template, 'text', '', { wide: true, large: true });
}
/**
* Check if the model supports image inlining
* @returns {boolean} True if the model supports image inlining
@ -3892,4 +3944,5 @@ $(document).ready(async function () {
$('#openai_logit_bias_delete_preset').on('click', onLogitBiasPresetDeleteClick);
$('#import_oai_preset').on('click', onImportPresetClick);
$('#openai_proxy_password_show').on('click', onProxyPasswordShowClick);
$('#customize_additional_parameters').on('click', onCustomizeParametersClick);
});

45
src/endpoints/backends/chat-completions.js
View File

@ -4,7 +4,7 @@ const { Readable } = require('stream');
const { jsonParser } = require('../../express-common');
const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY } = require('../../constants');
const { forwardFetchResponse, getConfigValue, tryParse, uuidv4 } = require('../../util');
const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml } = require('../../util');
const { convertClaudePrompt, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters');
const { readSecret, SECRET_KEYS } = require('../secrets');
@ -506,6 +506,7 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
api_url = request.body.custom_url;
api_key_openai = readSecret(SECRET_KEYS.CUSTOM);
headers = {};
mergeObjectWithYaml(headers, request.body.custom_include_headers);
} else {
console.log('This chat completion source is not supported yet.');
return response_getstatus_openai.status(400).send({ error: true });
@ -685,6 +686,8 @@ router.post('/generate', jsonParser, function (request, response) {
apiKey = readSecret(SECRET_KEYS.CUSTOM);
headers = {};
bodyParams = {};
mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
mergeObjectWithYaml(headers, request.body.custom_include_headers);
} else {
console.log('This chat completion source is not supported yet.');
return response.status(400).send({ error: true });
@ -712,6 +715,27 @@ router.post('/generate', jsonParser, function (request, response) {
controller.abort();
});
const requestBody = {
'messages': isTextCompletion === false ? request.body.messages : undefined,
'prompt': isTextCompletion === true ? textPrompt : undefined,
'model': request.body.model,
'temperature': request.body.temperature,
'max_tokens': request.body.max_tokens,
'stream': request.body.stream,
'presence_penalty': request.body.presence_penalty,
'frequency_penalty': request.body.frequency_penalty,
'top_p': request.body.top_p,
'top_k': request.body.top_k,
'stop': isTextCompletion === false ? request.body.stop : undefined,
'logit_bias': request.body.logit_bias,
'seed': request.body.seed,
...bodyParams,
};
if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
excludeKeysByYaml(requestBody, request.body.custom_exclude_body);
}
/** @type {import('node-fetch').RequestInit} */
const config = {
method: 'post',
@ -720,27 +744,12 @@ router.post('/generate', jsonParser, function (request, response) {
'Authorization': 'Bearer ' + apiKey,
...headers,
},
body: JSON.stringify({
'messages': isTextCompletion === false ? request.body.messages : undefined,
'prompt': isTextCompletion === true ? textPrompt : undefined,
'model': request.body.model,
'temperature': request.body.temperature,
'max_tokens': request.body.max_tokens,
'stream': request.body.stream,
'presence_penalty': request.body.presence_penalty,
'frequency_penalty': request.body.frequency_penalty,
'top_p': request.body.top_p,
'top_k': request.body.top_k,
'stop': isTextCompletion === false ? request.body.stop : undefined,
'logit_bias': request.body.logit_bias,
'seed': request.body.seed,
...bodyParams,
}),
body: JSON.stringify(requestBody),
signal: controller.signal,
timeout: 0,
};
console.log(JSON.parse(String(config.body)));
console.log(requestBody);
makeRequest(config, response, request);

12
src/endpoints/openai.js
View File

@ -4,13 +4,15 @@ const express = require('express');
const FormData = require('form-data');
const fs = require('fs');
const { jsonParser, urlencodedParser } = require('../express-common');
const { getConfigValue } = require('../util');
const { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml } = require('../util');
const router = express.Router();
router.post('/caption-image', jsonParser, async (request, response) => {
try {
let key = '';
let headers = {};
let bodyParams = {};
if (request.body.api === 'openai' && !request.body.reverse_proxy) {
key = readSecret(SECRET_KEYS.OPENAI);
@ -26,6 +28,8 @@ router.post('/caption-image', jsonParser, async (request, response) => {
if (request.body.api === 'custom') {
key = readSecret(SECRET_KEYS.CUSTOM);
mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
mergeObjectWithYaml(headers, request.body.custom_include_headers);
}
if (!key && !request.body.reverse_proxy && request.body.api !== 'custom') {
@ -45,6 +49,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
},
],
max_tokens: 500,
...bodyParams,
};
const captionSystemPrompt = getConfigValue('openai.captionSystemPrompt');
@ -55,10 +60,13 @@ router.post('/caption-image', jsonParser, async (request, response) => {
});
}
if (request.body.api === 'custom') {
excludeKeysByYaml(body, request.body.custom_exclude_body);
}
console.log('Multimodal captioning request', body);
let apiUrl = '';
let headers = {};
if (request.body.api === 'openrouter') {
apiUrl = 'https://openrouter.ai/api/v1/chat/completions';

61
src/util.js
View File

@ -399,6 +399,65 @@ function forwardFetchResponse(from, to) {
});
}
/**
* Adds YAML-serialized object to the object.
* @param {object} obj Object
* @param {string} yamlString YAML-serialized object
* @returns
*/
function mergeObjectWithYaml(obj, yamlString) {
if (!yamlString) {
return;
}
try {
const parsedObject = yaml.parse(yamlString);
if (Array.isArray(parsedObject)) {
for (const item of parsedObject) {
if (typeof item === 'object' && item && !Array.isArray(item)) {
Object.assign(obj, item);
}
}
}
else if (parsedObject && typeof parsedObject === 'object') {
Object.assign(obj, parsedObject);
}
} catch {
// Do nothing
}
}
/**
* Removes keys from the object by YAML-serialized array.
* @param {object} obj Object
* @param {string} yamlString YAML-serialized array
* @returns {void} Nothing
*/
function excludeKeysByYaml(obj, yamlString) {
if (!yamlString) {
return;
}
try {
const parsedObject = yaml.parse(yamlString);
if (Array.isArray(parsedObject)) {
parsedObject.forEach(key => {
delete obj[key];
});
} else if (typeof parsedObject === 'object') {
Object.keys(parsedObject).forEach(key => {
delete obj[key];
});
} else if (typeof parsedObject === 'string') {
delete obj[parsedObject];
}
} catch {
// Do nothing
}
}
module.exports = {
getConfig,
getConfigValue,
@ -420,4 +479,6 @@ module.exports = {
getImages,
forwardFetchResponse,
getHexString,
mergeObjectWithYaml,
excludeKeysByYaml,
};