rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add koboldcpp as a multimodal captioning source

Browse Source
This commit is contained in:
Cohee
2024-03-14 01:03:51 +02:00
parent bd223486de
commit 6ac8ef1b48
3 changed files with 26 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
public/scripts/extensions/caption/index.js
View File

@ -288,6 +288,7 @@ jQuery(function () {
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'koboldcpp' && textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'custom') ||
extension_settings.caption.source === 'local' ||
extension_settings.caption.source === 'horde';
@ -355,6 +356,7 @@ jQuery(function () {
<select id="caption_multimodal_api" class="flex1 text_pole">
<option value="llamacpp">llama.cpp</option>
<option value="ooba">Text Generation WebUI (oobabooga)</option>
<option value="koboldcpp">KoboldCpp</option>
<option value="ollama">Ollama</option>
<option value="openai">OpenAI</option>
<option value="anthropic">Anthropic</option>
@ -378,6 +380,7 @@ jQuery(function () {
<option data-type="ollama" value="llava:latest">llava:latest</option>
<option data-type="llamacpp" value="llamacpp_current">[Currently loaded]</option>
<option data-type="ooba" value="ooba_current">[Currently loaded]</option>
<option data-type="koboldcpp" value="koboldcpp_current">[Currently loaded]</option>
<option data-type="custom" value="custom_current">[Currently selected]</option>
</select>
</div>

13
public/scripts/extensions/shared.js
View File

@ -21,16 +21,17 @@ export async function getMultimodalCaption(base64Img, prompt) {
}
// OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
// Ooba requires all images to be JPEGs.
// Ooba requires all images to be JPEGs. Koboldcpp just asked nicely.
const isGoogle = extension_settings.caption.multimodal_api === 'google';
const isClaude = extension_settings.caption.multimodal_api === 'anthropic';
const isOllama = extension_settings.caption.multimodal_api === 'ollama';
const isLlamaCpp = extension_settings.caption.multimodal_api === 'llamacpp';
const isCustom = extension_settings.caption.multimodal_api === 'custom';
const isOoba = extension_settings.caption.multimodal_api === 'ooba';
const isKoboldCpp = extension_settings.caption.multimodal_api === 'koboldcpp';
const base64Bytes = base64Img.length * 0.75;
const compressionLimit = 2 * 1024 * 1024;
if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba) {
if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
const maxSide = 1024;
base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');
@ -76,6 +77,10 @@ export async function getMultimodalCaption(base64Img, prompt) {
requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OOBA];
}
if (isKoboldCpp) {
requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP];
}
if (isCustom) {
requestBody.server_url = oai_settings.custom_url;
requestBody.model = oai_settings.custom_model || 'gpt-4-vision-preview';
@ -142,6 +147,10 @@ function throwIfInvalidModel() {
throw new Error('Text Generation WebUI server URL is not set.');
}
if (extension_settings.caption.multimodal_api === 'koboldcpp' && !textgenerationwebui_settings.server_urls[textgen_types.KOBOLDCPP]) {
throw new Error('KoboldCpp server URL is not set.');
}
if (extension_settings.caption.multimodal_api === 'custom' && !oai_settings.custom_url) {
throw new Error('Custom API URL is not set.');
}

13
src/endpoints/openai.js
View File

@ -5,6 +5,7 @@ const FormData = require('form-data');
const fs = require('fs');
const { jsonParser, urlencodedParser } = require('../express-common');
const { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } = require('../util');
const { setAdditionalHeaders } = require('../additional-headers');
const router = express.Router();
@ -37,7 +38,11 @@ router.post('/caption-image', jsonParser, async (request, response) => {
bodyParams.temperature = 0.1;
}
if (!key && !request.body.reverse_proxy && request.body.api !== 'custom' && request.body.api !== 'ooba') {
if (request.body.api === 'koboldcpp') {
key = readSecret(SECRET_KEYS.KOBOLDCPP);
}
if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp'].includes(request.body.api) === false) {
console.log('No key found for API', request.body.api);
return response.sendStatus(400);
}
@ -104,6 +109,12 @@ router.post('/caption-image', jsonParser, async (request, response) => {
});
}
if (request.body.api === 'koboldcpp') {
apiUrl = `${trimV1(request.body.server_url)}/v1/chat/completions`;
}
setAdditionalHeaders(request, { headers }, apiUrl);
const result = await fetch(apiUrl, {
method: 'POST',
headers: {