From 7adc6d38e29672ac27acebf1d588f61be97d68d1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 24 Dec 2024 21:51:47 +0200 Subject: [PATCH] OpenRouter: Add control for middle-out transform Closes #3033 --- public/index.html | 13 +++++++++++++ public/scripts/openai.js | 18 ++++++++++++++++++ src/endpoints/backends/chat-completions.js | 20 +++++++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index c08b04a6d..7ea940004 100644 --- a/public/index.html +++ b/public/index.html @@ -651,6 +651,19 @@ +
+
+ Middle-out Transform + +
+
+ +
+
Max prompt cost: Unknown
diff --git a/public/scripts/openai.js b/public/scripts/openai.js index c94a88541..41b929ef3 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -207,6 +207,12 @@ const custom_prompt_post_processing_types = { STRICT: 'strict', }; +const openrouter_middleout_types = { + AUTO: 'auto', + ON: 'on', + OFF: 'off', +}; + const sensitiveFields = [ 'reverse_proxy', 'proxy_password', @@ -267,6 +273,7 @@ const default_settings = { openrouter_sort_models: 'alphabetically', openrouter_providers: [], openrouter_allow_fallbacks: true, + openrouter_middleout: openrouter_middleout_types.ON, jailbreak_system: false, reverse_proxy: '', chat_completion_source: chat_completion_sources.OPENAI, @@ -343,6 +350,7 @@ const oai_settings = { openrouter_sort_models: 'alphabetically', openrouter_providers: [], openrouter_allow_fallbacks: true, + openrouter_middleout: openrouter_middleout_types.ON, jailbreak_system: false, reverse_proxy: '', chat_completion_source: chat_completion_sources.OPENAI, @@ -1920,6 +1928,7 @@ async function sendOpenAIRequest(type, messages, signal) { generate_data['use_fallback'] = oai_settings.openrouter_use_fallback; generate_data['provider'] = oai_settings.openrouter_providers; generate_data['allow_fallbacks'] = oai_settings.openrouter_allow_fallbacks; + generate_data['middleout'] = oai_settings.openrouter_middleout; if (isTextCompletion) { generate_data['stop'] = getStoppingStrings(isImpersonate, isContinue); @@ -3022,6 +3031,7 @@ function loadOpenAISettings(data, settings) { oai_settings.openrouter_sort_models = settings.openrouter_sort_models ?? default_settings.openrouter_sort_models; oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback; oai_settings.openrouter_allow_fallbacks = settings.openrouter_allow_fallbacks ?? default_settings.openrouter_allow_fallbacks; + oai_settings.openrouter_middleout = settings.openrouter_middleout ?? default_settings.openrouter_middleout; oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model; oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model; oai_settings.cohere_model = settings.cohere_model ?? default_settings.cohere_model; @@ -3130,6 +3140,7 @@ function loadOpenAISettings(data, settings) { $('#openrouter_group_models').prop('checked', oai_settings.openrouter_group_models); $('#openrouter_allow_fallbacks').prop('checked', oai_settings.openrouter_allow_fallbacks); $('#openrouter_providers_chat').val(oai_settings.openrouter_providers).trigger('change'); + $('#openrouter_middleout').val(oai_settings.openrouter_middleout); $('#squash_system_messages').prop('checked', oai_settings.squash_system_messages); $('#continue_prefill').prop('checked', oai_settings.continue_prefill); $('#openai_function_calling').prop('checked', oai_settings.function_calling); @@ -3370,6 +3381,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { openrouter_sort_models: settings.openrouter_sort_models, openrouter_providers: settings.openrouter_providers, openrouter_allow_fallbacks: settings.openrouter_allow_fallbacks, + openrouter_middleout: settings.openrouter_middleout, ai21_model: settings.ai21_model, mistralai_model: settings.mistralai_model, cohere_model: settings.cohere_model, @@ -3836,6 +3848,7 @@ function onSettingsPresetChange() { openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false], openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false], openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true], + openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false], ai21_model: ['#model_ai21_select', 'ai21_model', false], mistralai_model: ['#model_mistralai_select', 'mistralai_model', false], cohere_model: ['#model_cohere_select', 'cohere_model', false], @@ -5261,6 +5274,11 @@ export function initOpenAI() { saveSettingsDebounced(); }); + $('#openrouter_middleout').on('input', function () { + oai_settings.openrouter_middleout = String($(this).val()); + saveSettingsDebounced(); + }); + $('#squash_system_messages').on('input', function () { oai_settings.squash_system_messages = !!$(this).prop('checked'); saveSettingsDebounced(); diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 64e6ab345..4e327bd5b 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -71,6 +71,22 @@ function postProcessPrompt(messages, type, names) { } } +/** + * Gets OpenRouter transforms based on the request. + * @param {import('express').Request} request Express request + * @returns {string[] | undefined} OpenRouter transforms + */ +function getOpenRouterTransforms(request) { + switch (request.body.middleout) { + case 'on': + return ['middle-out']; + case 'off': + return []; + case 'auto': + return undefined; + } +} + /** * Sends a request to Claude API. * @param {express.Request} request Express request @@ -834,7 +850,9 @@ router.post('/generate', jsonParser, function (request, response) { apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER); // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests headers = { ...OPENROUTER_HEADERS }; - bodyParams = { 'transforms': ['middle-out'] }; + bodyParams = { + 'transforms': getOpenRouterTransforms(request), + }; if (request.body.min_p !== undefined) { bodyParams['min_p'] = request.body.min_p;