rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

message inlining vision support

Browse Source
This commit is contained in:
based
2023-12-15 01:28:54 +10:00
parent 178b07f414
commit d5bcd96eef
3 changed files with 57 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
public/scripts/openai.js
View File

@ -37,8 +37,8 @@ import {
chatCompletionDefaultPrompts,
INJECTION_POSITION,
Prompt,
promptManagerDefaultPromptOrders,
PromptManager,
promptManagerDefaultPromptOrders,
} from './PromptManager.js';
import {getCustomStoppingStrings, persona_description_positions, power_user} from './power-user.js';
@ -1795,13 +1795,15 @@ class Message {
async addImage(image) {
const textContent = this.content;
const isDataUrl = isDataURL(image);
if (!isDataUrl) {
try {
const response = await fetch(image, { method: 'GET', cache: 'force-cache' });
if (!response.ok) throw new Error('Failed to fetch image');
const blob = await response.blob();
image = await getBase64Async(blob);
if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) {
image = image.split(',')[1];
}
} catch (error) {
console.error('Image adding skipped', error);
return;
@ -3087,7 +3089,8 @@ async function onModelChange() {
} else {
$('#openai_max_context').attr('max', max_8k);
}
oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai);
$('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input');
oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context);
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
}
@ -3435,7 +3438,7 @@ export function isImageInliningSupported() {
case chat_completion_sources.OPENAI:
return oai_settings.openai_model.includes(gpt4v);
case chat_completion_sources.MAKERSUITE:
return oai_settings.openai_model.includes(geminiProV);
return oai_settings.google_model.includes(geminiProV);
case chat_completion_sources.OPENROUTER:
return oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava13b);
default:

7
server.js
View File

@ -1002,6 +1002,9 @@ async function sendMakerSuiteRequest(request, response) {
return response.status(400).send({ error: true });
}
const google_model = request.body.model;
const should_stream = request.body.stream;
const generationConfig = {
stopSequences: request.body.stop,
candidateCount: 1,
@ -1012,13 +1015,11 @@ async function sendMakerSuiteRequest(request, response) {
};
const body = {
contents: convertGooglePrompt(request.body.messages),
contents: convertGooglePrompt(request.body.messages, google_model),
safetySettings: MAKERSUITE_SAFETY,
generationConfig: generationConfig,
};
const google_model = request.body.model;
const should_stream = request.body.stream;
try {
const controller = new AbortController();
request.socket.removeAllListeners('close');

27
src/chat-completion.js
View File

@ -72,10 +72,33 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with
return requestPrompt;
}
function convertGooglePrompt(messages) {
function convertGooglePrompt(messages, type) {
const contents = [];
let lastRole = '';
let currentText = '';
const isMultimodal = type === 'gemini-pro-vision';
if (isMultimodal) {
const combinedText = messages.map((message) => {
const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: ';
return role + message.content;
}).join('\n\n').trim();
const imageEntry = messages.find((message) => message.content[1]?.image_url);
contents.push({
parts: [
{ text: combinedText },
{
inlineData: {
mimeType: 'image/png',
data: imageEntry.content[1].image_url.url ?? '',
},
},
],
role: 'user',
});
} else {
messages.forEach((message, index) => {
const role = message.role === 'assistant' ? 'model' : 'user';
if (lastRole === role) {
@ -97,6 +120,8 @@ function convertGooglePrompt(messages) {
});
}
});
}
return contents;
}