OpenRouter: Add control for middle-out transform

Closes #3033

public/index.html

@@ -651,6 +651,19 @@
                                         <input type="number" id="n_openai" name="n_openai" class="text_pole" min="1" value="1">
                                     </div>
                                 </div>
+                                <div data-source="openrouter" class="range-block">
+                                    <div class="range-block-title" title="Allow compressing requests by removing messages from the middle of the prompt.">
+                                        <span data-i18n="Middle-out Transform">Middle-out Transform</span>
+                                        <a href="https://openrouter.ai/docs/transforms" target="_blank" rel="noreferrer noopener"  class="note-link-span fa-solid fa-circle-question"></a>
+                                    </div>
+                                    <div class="wide100p">
+                                        <select id="openrouter_middleout" class="text_pole">
+                                            <option value="auto" data-i18n="Auto">Auto</option>
+                                            <option value="on" data-i18n="Allow">Allow</option>
+                                            <option value="off" data-i18n="Forbid">Forbid</option>
+                                        </select>
+                                    </div>
+                                </div>
                                 <div data-source="openrouter">
                                     Max prompt cost: <span id="openrouter_max_prompt_cost">Unknown</span>
                                 </div>

public/scripts/openai.js

@@ -207,6 +207,12 @@ const custom_prompt_post_processing_types = {
     STRICT: 'strict',
 };
 
+const openrouter_middleout_types = {
+    AUTO: 'auto',
+    ON: 'on',
+    OFF: 'off',
+};
+
 const sensitiveFields = [
     'reverse_proxy',
     'proxy_password',
@@ -267,6 +273,7 @@ const default_settings = {
     openrouter_sort_models: 'alphabetically',
     openrouter_providers: [],
     openrouter_allow_fallbacks: true,
+    openrouter_middleout: openrouter_middleout_types.ON,
     jailbreak_system: false,
     reverse_proxy: '',
     chat_completion_source: chat_completion_sources.OPENAI,
@@ -343,6 +350,7 @@ const oai_settings = {
     openrouter_sort_models: 'alphabetically',
     openrouter_providers: [],
     openrouter_allow_fallbacks: true,
+    openrouter_middleout: openrouter_middleout_types.ON,
     jailbreak_system: false,
     reverse_proxy: '',
     chat_completion_source: chat_completion_sources.OPENAI,
@@ -1920,6 +1928,7 @@ async function sendOpenAIRequest(type, messages, signal) {
         generate_data['use_fallback'] = oai_settings.openrouter_use_fallback;
         generate_data['provider'] = oai_settings.openrouter_providers;
         generate_data['allow_fallbacks'] = oai_settings.openrouter_allow_fallbacks;
+        generate_data['middleout'] = oai_settings.openrouter_middleout;
 
         if (isTextCompletion) {
             generate_data['stop'] = getStoppingStrings(isImpersonate, isContinue);
@@ -3022,6 +3031,7 @@ function loadOpenAISettings(data, settings) {
     oai_settings.openrouter_sort_models = settings.openrouter_sort_models ?? default_settings.openrouter_sort_models;
     oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback;
     oai_settings.openrouter_allow_fallbacks = settings.openrouter_allow_fallbacks ?? default_settings.openrouter_allow_fallbacks;
+    oai_settings.openrouter_middleout = settings.openrouter_middleout ?? default_settings.openrouter_middleout;
     oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
     oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model;
     oai_settings.cohere_model = settings.cohere_model ?? default_settings.cohere_model;
@@ -3130,6 +3140,7 @@ function loadOpenAISettings(data, settings) {
     $('#openrouter_group_models').prop('checked', oai_settings.openrouter_group_models);
     $('#openrouter_allow_fallbacks').prop('checked', oai_settings.openrouter_allow_fallbacks);
     $('#openrouter_providers_chat').val(oai_settings.openrouter_providers).trigger('change');
+    $('#openrouter_middleout').val(oai_settings.openrouter_middleout);
     $('#squash_system_messages').prop('checked', oai_settings.squash_system_messages);
     $('#continue_prefill').prop('checked', oai_settings.continue_prefill);
     $('#openai_function_calling').prop('checked', oai_settings.function_calling);
@@ -3370,6 +3381,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
         openrouter_sort_models: settings.openrouter_sort_models,
         openrouter_providers: settings.openrouter_providers,
         openrouter_allow_fallbacks: settings.openrouter_allow_fallbacks,
+        openrouter_middleout: settings.openrouter_middleout,
         ai21_model: settings.ai21_model,
         mistralai_model: settings.mistralai_model,
         cohere_model: settings.cohere_model,
@@ -3836,6 +3848,7 @@ function onSettingsPresetChange() {
         openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false],
         openrouter_providers: ['#openrouter_providers_chat', 'openrouter_providers', false],
         openrouter_allow_fallbacks: ['#openrouter_allow_fallbacks', 'openrouter_allow_fallbacks', true],
+        openrouter_middleout: ['#openrouter_middleout', 'openrouter_middleout', false],
         ai21_model: ['#model_ai21_select', 'ai21_model', false],
         mistralai_model: ['#model_mistralai_select', 'mistralai_model', false],
         cohere_model: ['#model_cohere_select', 'cohere_model', false],
@@ -5261,6 +5274,11 @@ export function initOpenAI() {
         saveSettingsDebounced();
     });
 
+    $('#openrouter_middleout').on('input', function () {
+        oai_settings.openrouter_middleout = String($(this).val());
+        saveSettingsDebounced();
+    });
+
     $('#squash_system_messages').on('input', function () {
         oai_settings.squash_system_messages = !!$(this).prop('checked');
         saveSettingsDebounced();

src/endpoints/backends/chat-completions.js

@@ -71,6 +71,22 @@ function postProcessPrompt(messages, type, names) {
     }
 }
 
+/**
+ * Gets OpenRouter transforms based on the request.
+ * @param {import('express').Request} request Express request
+ * @returns {string[] | undefined} OpenRouter transforms
+ */
+function getOpenRouterTransforms(request) {
+    switch (request.body.middleout) {
+        case 'on':
+            return ['middle-out'];
+        case 'off':
+            return [];
+        case 'auto':
+            return undefined;
+    }
+}
+
 /**
  * Sends a request to Claude API.
  * @param {express.Request} request Express request
@@ -834,7 +850,9 @@ router.post('/generate', jsonParser, function (request, response) {
         apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
         // OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
         headers = { ...OPENROUTER_HEADERS };
-        bodyParams = { 'transforms': ['middle-out'] };
+        bodyParams = {
+            'transforms': getOpenRouterTransforms(request),
+        };
 
         if (request.body.min_p !== undefined) {
             bodyParams['min_p'] = request.body.min_p;