Auto-add OpenRouter captioning models

public/scripts/extensions/caption/index.js

@@ -1,6 +1,6 @@
 import { ensureImageFormatSupported, getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js';
 import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js';
-import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
+import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, main_api, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
 import { getMessageTimeStamp } from '../../RossAscends-mods.js';
 import { SECRET_KEYS, secret_state } from '../../secrets.js';
 import { getMultimodalCaption } from '../shared.js';
@@ -431,14 +431,9 @@ jQuery(async function () {
         $('#form_sheld').append(imgForm);
         $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
     }
-    function switchMultimodalBlocks() {
+    async function switchMultimodalBlocks() {
+        await addOpenRouterModels();
         const isMultimodal = extension_settings.caption.source === 'multimodal';
-        $('#caption_ollama_pull').on('click', (e) => {
-            const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
-            e.preventDefault();
-            $('#ollama_download_model').trigger('click');
-            $('#dialogue_popup_input').val(presetModel);
-        });
         $('#caption_multimodal_block').toggle(isMultimodal);
         $('#caption_prompt_block').toggle(isMultimodal);
         $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);
@@ -448,30 +443,48 @@ jQuery(async function () {
             const types = type.split(',');
             $(this).toggle(types.includes(extension_settings.caption.multimodal_api));
         });
-        $('#caption_multimodal_api').on('change', () => {
-            const api = String($('#caption_multimodal_api').val());
-            const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
-            extension_settings.caption.multimodal_api = api;
-            extension_settings.caption.multimodal_model = model;
-            saveSettingsDebounced();
-            switchMultimodalBlocks();
-        });
-        $('#caption_multimodal_model').on('change', () => {
-            extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
-            saveSettingsDebounced();
-        });
     }
     async function addSettings() {
         const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });
         $('#caption_container').append(html);
     }
+    async function addOpenRouterModels() {
+        const dropdown = document.getElementById('caption_multimodal_model');
+        if (!(dropdown instanceof HTMLSelectElement)) {
+            return;
+        }
+        if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter') {
+            return;
+        }
+        const options = Array.from(dropdown.options);
+        const response = await fetch('/api/openrouter/models/multimodal', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+        });
+        if (!response.ok) {
+            return;
+        }
+        const modelIds = await response.json();
+        if (Array.isArray(modelIds) && modelIds.length > 0) {
+            modelIds.forEach((modelId) => {
+                if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {
+                    return;
+                }
+                const option = document.createElement('option');
+                option.value = modelId;
+                option.textContent = modelId;
+                option.dataset.type = 'openrouter';
+                dropdown.add(option);
+            });
+        }
+    }
 
     await addSettings();
     addPictureSendForm();
     addSendPictureButton();
     setImageIcon();
     migrateSettings();
-    switchMultimodalBlocks();
+    await switchMultimodalBlocks();
 
     $('#caption_refine_mode').prop('checked', !!(extension_settings.caption.refine_mode));
     $('#caption_allow_reverse_proxy').prop('checked', !!(extension_settings.caption.allow_reverse_proxy));
@@ -506,6 +519,24 @@ jQuery(async function () {
         extension_settings.caption.auto_mode = !!$('#caption_auto_mode').prop('checked');
         saveSettingsDebounced();
     });
+    $('#caption_ollama_pull').on('click', (e) => {
+        const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
+        e.preventDefault();
+        $('#ollama_download_model').trigger('click');
+        $('#dialogue_popup_input').val(presetModel);
+    });
+    $('#caption_multimodal_api').on('change', () => {
+        const api = String($('#caption_multimodal_api').val());
+        const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
+        extension_settings.caption.multimodal_api = api;
+        extension_settings.caption.multimodal_model = model;
+        saveSettingsDebounced();
+        switchMultimodalBlocks();
+    });
+    $('#caption_multimodal_model').on('change', () => {
+        extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
+        saveSettingsDebounced();
+    });
 
     const onMessageEvent = async (index) => {
         if (!extension_settings.caption.auto_mode) {
@@ -531,14 +562,15 @@ jQuery(async function () {
             await captionExistingMessage(data);
             appendMediaToMessage(data, messageBlock, false);
             await saveChatConditional();
-        } catch(e) {
+        } catch (e) {
             console.error('Message image recaption failed', e);
         } finally {
             messageImg.removeClass(animationClass);
         }
     });
 
-    SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'caption',
+    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+        name: 'caption',
         callback: captionCommandCallback,
         returns: 'caption',
         namedArgumentList: [

server.js

@@ -610,6 +610,9 @@ app.use('/api/search', require('./src/endpoints/search').router);
 // Ooba/OpenAI text completions
 app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router);
 
+// OpenRouter
+app.use('/api/openrouter', require('./src/endpoints/openrouter').router);
+
 // KoboldAI
 app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router);
 

src/endpoints/openrouter.js

@@ -0,0 +1,32 @@
+const express = require('express');
+const { jsonParser } = require('../express-common');
+
+const router = express.Router();
+const API_OPENROUTER = 'https://openrouter.ai/api/v1';
+
+router.post('/models/multimodal', jsonParser, async (_req, res) => {
+    try {
+        // The endpoint is available without authentication
+        const response = await fetch(`${API_OPENROUTER}/models`, {
+            method: 'GET',
+            headers: {
+                'Accept': 'application/json',
+            },
+        });
+
+        if (!response.ok) {
+            return res.json([]);
+        }
+
+        const data = await response.json();
+        const models = data?.data || [];
+        const multimodalModels = models.filter(m => m?.architecture?.modality === 'text+image->text').map(m => m.id);
+
+        return res.json(multimodalModels);
+    } catch (error) {
+        console.error(error);
+        return res.sendStatus(500);
+    }
+});
+
+module.exports = { router };