rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Auto-add OpenRouter captioning models

Browse Source
This commit is contained in:
Cohee
2024-09-15 18:38:44 +03:00
parent 44cc81fa73
commit 6393eda81e
3 changed files with 90 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
public/scripts/extensions/caption/index.js
View File

@@ -1,6 +1,6 @@
import { ensureImageFormatSupported, getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js';
import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js';
import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
import { appendMediaToMessage, callPopup, eventSource, event_types, getRequestHeaders, main_api, saveChatConditional, saveSettingsDebounced, substituteParamsExtended } from '../../../script.js';
import { getMessageTimeStamp } from '../../RossAscends-mods.js';
import { SECRET_KEYS, secret_state } from '../../secrets.js';
import { getMultimodalCaption } from '../shared.js';
@@ -431,14 +431,9 @@ jQuery(async function () {
$('#form_sheld').append(imgForm);
$('#img_file').on('change', (e) => onSelectImage(e.originalEvent, '', false));
}
function switchMultimodalBlocks() {
async function switchMultimodalBlocks() {
await addOpenRouterModels();
const isMultimodal = extension_settings.caption.source === 'multimodal';
$('#caption_ollama_pull').on('click', (e) => {
const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
e.preventDefault();
$('#ollama_download_model').trigger('click');
$('#dialogue_popup_input').val(presetModel);
});
$('#caption_multimodal_block').toggle(isMultimodal);
$('#caption_prompt_block').toggle(isMultimodal);
$('#caption_multimodal_api').val(extension_settings.caption.multimodal_api);
@@ -448,30 +443,48 @@ jQuery(async function () {
const types = type.split(',');
$(this).toggle(types.includes(extension_settings.caption.multimodal_api));
});
$('#caption_multimodal_api').on('change', () => {
const api = String($('#caption_multimodal_api').val());
const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
extension_settings.caption.multimodal_api = api;
extension_settings.caption.multimodal_model = model;
saveSettingsDebounced();
switchMultimodalBlocks();
});
$('#caption_multimodal_model').on('change', () => {
extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
saveSettingsDebounced();
});
}
async function addSettings() {
const html = await renderExtensionTemplateAsync('caption', 'settings', { TEMPLATE_DEFAULT, PROMPT_DEFAULT });
$('#caption_container').append(html);
}
async function addOpenRouterModels() {
const dropdown = document.getElementById('caption_multimodal_model');
if (!(dropdown instanceof HTMLSelectElement)) {
return;
}
if (extension_settings.caption.source !== 'multimodal' || extension_settings.caption.multimodal_api !== 'openrouter') {
return;
}
const options = Array.from(dropdown.options);
const response = await fetch('/api/openrouter/models/multimodal', {
method: 'POST',
headers: getRequestHeaders(),
});
if (!response.ok) {
return;
}
const modelIds = await response.json();
if (Array.isArray(modelIds) && modelIds.length > 0) {
modelIds.forEach((modelId) => {
if (!modelId || typeof modelId !== 'string' || options.some(o => o.value === modelId)) {
return;
}
const option = document.createElement('option');
option.value = modelId;
option.textContent = modelId;
option.dataset.type = 'openrouter';
dropdown.add(option);
});
}
}
await addSettings();
addPictureSendForm();
addSendPictureButton();
setImageIcon();
migrateSettings();
switchMultimodalBlocks();
await switchMultimodalBlocks();
$('#caption_refine_mode').prop('checked', !!(extension_settings.caption.refine_mode));
$('#caption_allow_reverse_proxy').prop('checked', !!(extension_settings.caption.allow_reverse_proxy));
@@ -506,6 +519,24 @@ jQuery(async function () {
extension_settings.caption.auto_mode = !!$('#caption_auto_mode').prop('checked');
saveSettingsDebounced();
});
$('#caption_ollama_pull').on('click', (e) => {
const presetModel = extension_settings.caption.multimodal_model !== 'ollama_current' ? extension_settings.caption.multimodal_model : '';
e.preventDefault();
$('#ollama_download_model').trigger('click');
$('#dialogue_popup_input').val(presetModel);
});
$('#caption_multimodal_api').on('change', () => {
const api = String($('#caption_multimodal_api').val());
const model = String($(`#caption_multimodal_model option[data-type="${api}"]`).first().val());
extension_settings.caption.multimodal_api = api;
extension_settings.caption.multimodal_model = model;
saveSettingsDebounced();
switchMultimodalBlocks();
});
$('#caption_multimodal_model').on('change', () => {
extension_settings.caption.multimodal_model = String($('#caption_multimodal_model').val());
saveSettingsDebounced();
});
const onMessageEvent = async (index) => {
if (!extension_settings.caption.auto_mode) {
@@ -531,14 +562,15 @@ jQuery(async function () {
await captionExistingMessage(data);
appendMediaToMessage(data, messageBlock, false);
await saveChatConditional();
} catch(e) {
} catch (e) {
console.error('Message image recaption failed', e);
} finally {
messageImg.removeClass(animationClass);
}
});
SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'caption',
SlashCommandParser.addCommandObject(SlashCommand.fromProps({
name: 'caption',
callback: captionCommandCallback,
returns: 'caption',
namedArgumentList: [

3
server.js
View File

@@ -610,6 +610,9 @@ app.use('/api/search', require('./src/endpoints/search').router);
// Ooba/OpenAI text completions
app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router);
// OpenRouter
app.use('/api/openrouter', require('./src/endpoints/openrouter').router);
// KoboldAI
app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router);

32
src/endpoints/openrouter.js Normal file
View File

@@ -0,0 +1,32 @@
const express = require('express');
const { jsonParser } = require('../express-common');
const router = express.Router();
const API_OPENROUTER = 'https://openrouter.ai/api/v1';
router.post('/models/multimodal', jsonParser, async (_req, res) => {
try {
// The endpoint is available without authentication
const response = await fetch(`${API_OPENROUTER}/models`, {
method: 'GET',
headers: {
'Accept': 'application/json',
},
});
if (!response.ok) {
return res.json([]);
}
const data = await response.json();
const models = data?.data || [];
const multimodalModels = models.filter(m => m?.architecture?.modality === 'text+image->text').map(m => m.id);
return res.json(multimodalModels);
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
module.exports = { router };