diff --git a/public/scripts/extensions/caption/index.js b/public/scripts/extensions/caption/index.js index 3ec6b8470..efd2c1a55 100644 --- a/public/scripts/extensions/caption/index.js +++ b/public/scripts/extensions/caption/index.js @@ -1,6 +1,6 @@ import { ensureImageFormatSupported, getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js'; import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js'; -import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js'; +import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, main_api, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js'; import { getMessageTimeStamp } from '../../RossAscends-mods.js'; import { SECRET_KEYS, secret_state } from '../../secrets.js'; import { getMultimodalCaption } from '../shared.js'; @@ -431,14 +431,9 @@ jQuery(async function () { $('#form_sheld').append(imgForm); $('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false)); } - function switchMultimodalBlocks() { + async function switchMultimodalBlocks() { + await addOpenRouterModels(); const isMultimodal = extension_settings.caption.source === 'multimodal'; - $('#caption_ollama_pull').on('click', (e) => { - const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : ''; - e.preventDefault(); - $('#ollama_download_model').trigger('click'); - $('#dialogue_popup_input').val(presetModel); - }); $('#caption_multimodal_block').toggle(isMultimodal); $('#caption_prompt_block').toggle(isMultimodal); $('#caption_multimodal_api').val(extension_settings.caption.multimodal_api); @@ -448,30 +443,48 @@ jQuery(async function () { const types = type.split(','); $(this).toggle(types.includes(extension_settings.caption.multimodal_api)); }); - $('#caption_multimodal_api').on('change', () => { - const api = String($('#caption_multimodal_api').val()); - const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val()); - extension_settings.caption.multimodal_api = api; - extension_settings.caption.multimodal_model = model; - saveSettingsDebounced(); - switchMultimodalBlocks(); - }); - $('#caption_multimodal_model').on('change', () => { - extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val()); - saveSettingsDebounced(); - }); } async function addSettings() { const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT }); $('#caption_container').append(html); } + async function addOpenRouterModels() { + const dropdown = document.getElementById('caption_multimodal_model'); + if (!(dropdown instanceof HTMLSelectElement)) { + return; + } + if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter') { + return; + } + const options = Array.from(dropdown.options); + const response = await fetch('/api/openrouter/models/multimodal', { + method: 'POST', + headers: getRequestHeaders(), + }); + if (!response.ok) { + return; + } + const modelIds = await response.json(); + if (Array.isArray(modelIds) && modelIds.length > 0) { + modelIds.forEach((modelId) => { + if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) { + return; + } + const option = document.createElement('option'); + option.value = modelId; + option.textContent = modelId; + option.dataset.type = 'openrouter'; + dropdown.add(option); + }); + } + } await addSettings(); addPictureSendForm(); addSendPictureButton(); setImageIcon(); migrateSettings(); - switchMultimodalBlocks(); + await switchMultimodalBlocks(); $('#caption_refine_mode').prop('checked', !!(extension_settings.caption.refine_mode)); $('#caption_allow_reverse_proxy').prop('checked', !!(extension_settings.caption.allow_reverse_proxy)); @@ -506,6 +519,24 @@ jQuery(async function () { extension_settings.caption.auto_mode = !!$('#caption_auto_mode').prop('checked'); saveSettingsDebounced(); }); + $('#caption_ollama_pull').on('click', (e) => { + const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : ''; + e.preventDefault(); + $('#ollama_download_model').trigger('click'); + $('#dialogue_popup_input').val(presetModel); + }); + $('#caption_multimodal_api').on('change', () => { + const api = String($('#caption_multimodal_api').val()); + const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val()); + extension_settings.caption.multimodal_api = api; + extension_settings.caption.multimodal_model = model; + saveSettingsDebounced(); + switchMultimodalBlocks(); + }); + $('#caption_multimodal_model').on('change', () => { + extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val()); + saveSettingsDebounced(); + }); const onMessageEvent = async (index) => { if (!extension_settings.caption.auto_mode) { @@ -531,14 +562,15 @@ jQuery(async function () { await captionExistingMessage(data); appendMediaToMessage(data, messageBlock, false); await saveChatConditional(); - } catch(e) { + } catch (e) { console.error('Message image recaption failed', e); } finally { messageImg.removeClass(animationClass); } }); - SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'caption', + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'caption', callback: captionCommandCallback, returns: 'caption', namedArgumentList: [ diff --git a/server.js b/server.js index b9fff9bc7..84a5f9fe9 100644 --- a/server.js +++ b/server.js @@ -610,6 +610,9 @@ app.use('/api/search', require('./src/endpoints/search').router); // Ooba/OpenAI text completions app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router); +// OpenRouter +app.use('/api/openrouter', require('./src/endpoints/openrouter').router); + // KoboldAI app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router); diff --git a/src/endpoints/openrouter.js b/src/endpoints/openrouter.js new file mode 100644 index 000000000..6ac69a720 --- /dev/null +++ b/src/endpoints/openrouter.js @@ -0,0 +1,32 @@ +const express = require('express'); +const { jsonParser } = require('../express-common'); + +const router = express.Router(); +const API_OPENROUTER = 'https://openrouter.ai/api/v1'; + +router.post('/models/multimodal', jsonParser, async (_req, res) => { + try { + // The endpoint is available without authentication + const response = await fetch(`${API_OPENROUTER}/models`, { + method: 'GET', + headers: { + 'Accept': 'application/json', + }, + }); + + if (!response.ok) { + return res.json([]); + } + + const data = await response.json(); + const models = data?.data || []; + const multimodalModels = models.filter(m => m?.architecture?.modality === 'text+image->text').map(m => m.id); + + return res.json(multimodalModels); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } +}); + +module.exports = { router };