rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add vLLM as multimodal captioning source

Browse Source
This commit is contained in:
Cohee
2024-06-29 00:33:12 +03:00
parent a287ccfca2
commit 003066a036
4 changed files with 26 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
public/scripts/extensions/shared.js
View File

@ -34,6 +34,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
const isCustom = extension_settings.caption.multimodal_api === 'custom';
const isOoba = extension_settings.caption.multimodal_api === 'ooba';
const isKoboldCpp = extension_settings.caption.multimodal_api === 'koboldcpp';
const isVllm = extension_settings.caption.multimodal_api === 'vllm';
const base64Bytes = base64Img.length * 0.75;
const compressionLimit = 2 * 1024 * 1024;
if ((['google', 'openrouter'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) {
@ -65,6 +66,14 @@ export async function getMultimodalCaption(base64Img, prompt) {
requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.OLLAMA];
}
if (isVllm) {
if (extension_settings.caption.multimodal_model === 'vllm_current') {
requestBody.model = textgenerationwebui_settings.vllm_model;
}
requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.VLLM];
}
if (isLlamaCpp) {
requestBody.server_url = textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP];
}
@ -151,6 +160,14 @@ function throwIfInvalidModel(useReverseProxy) {
throw new Error('KoboldCpp server URL is not set.');
}
if (extension_settings.caption.multimodal_api === 'vllm' && !textgenerationwebui_settings.server_urls[textgen_types.VLLM]) {
throw new Error('vLLM server URL is not set.');
}
if (extension_settings.caption.multimodal_api === 'vllm' && extension_settings.caption.multimodal_model === 'vllm_current' && !textgenerationwebui_settings.vllm_model) {
throw new Error('vLLM model is not set.');
}
if (extension_settings.caption.multimodal_api === 'custom' && !oai_settings.custom_url) {
throw new Error('Custom API URL is not set.');
}