diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 878254b21..1d22348ef 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -31,18 +31,18 @@ import { system_message_types, this_chid, } from '../script.js'; -import { groups, selected_group } from './group-chats.js'; +import {groups, selected_group} from './group-chats.js'; import { chatCompletionDefaultPrompts, INJECTION_POSITION, Prompt, - promptManagerDefaultPromptOrders, PromptManager, + promptManagerDefaultPromptOrders, } from './PromptManager.js'; -import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js'; -import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js'; +import {getCustomStoppingStrings, persona_description_positions, power_user} from './power-user.js'; +import {SECRET_KEYS, secret_state, writeSecret} from './secrets.js'; import EventSourceStream from './sse-stream.js'; import { @@ -56,7 +56,7 @@ import { resetScrollHeight, stringFormat, } from './utils.js'; -import { countTokensOpenAI, getTokenizerModel } from './tokenizers.js'; +import {countTokensOpenAI, getTokenizerModel} from './tokenizers.js'; import { formatInstructModeChat, formatInstructModeExamples, @@ -1795,13 +1795,15 @@ class Message { async addImage(image) { const textContent = this.content; const isDataUrl = isDataURL(image); - if (!isDataUrl) { try { const response = await fetch(image, { method: 'GET', cache: 'force-cache' }); if (!response.ok) throw new Error('Failed to fetch image'); const blob = await response.blob(); image = await getBase64Async(blob); + if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) { + image = image.split(',')[1]; + } } catch (error) { console.error('Image adding skipped', error); return; @@ -3087,7 +3089,8 @@ async function onModelChange() { } else { $('#openai_max_context').attr('max', max_8k); } - + oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai); + $('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input'); oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context); $('#openai_max_context').val(oai_settings.openai_max_context).trigger('input'); } @@ -3435,7 +3438,7 @@ export function isImageInliningSupported() { case chat_completion_sources.OPENAI: return oai_settings.openai_model.includes(gpt4v); case chat_completion_sources.MAKERSUITE: - return oai_settings.openai_model.includes(geminiProV); + return oai_settings.google_model.includes(geminiProV); case chat_completion_sources.OPENROUTER: return oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava13b); default: diff --git a/server.js b/server.js index 4a19f3cd6..57264347d 100644 --- a/server.js +++ b/server.js @@ -1002,6 +1002,9 @@ async function sendMakerSuiteRequest(request, response) { return response.status(400).send({ error: true }); } + const google_model = request.body.model; + const should_stream = request.body.stream; + const generationConfig = { stopSequences: request.body.stop, candidateCount: 1, @@ -1012,13 +1015,11 @@ async function sendMakerSuiteRequest(request, response) { }; const body = { - contents: convertGooglePrompt(request.body.messages), + contents: convertGooglePrompt(request.body.messages, google_model), safetySettings: MAKERSUITE_SAFETY, generationConfig: generationConfig, }; - const google_model = request.body.model; - const should_stream = request.body.stream; try { const controller = new AbortController(); request.socket.removeAllListeners('close'); diff --git a/src/chat-completion.js b/src/chat-completion.js index cbed76abf..268a044e5 100644 --- a/src/chat-completion.js +++ b/src/chat-completion.js @@ -72,31 +72,56 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with return requestPrompt; } -function convertGooglePrompt(messages) { +function convertGooglePrompt(messages, type) { const contents = []; let lastRole = ''; let currentText = ''; - messages.forEach((message, index) => { - const role = message.role === 'assistant' ? 'model' : 'user'; - if (lastRole === role) { - currentText += '\n\n' + message.content; - } else { - if (currentText !== '') { + + const isMultimodal = type === 'gemini-pro-vision'; + + if (isMultimodal) { + const combinedText = messages.map((message) => { + const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: '; + return role + message.content; + }).join('\n\n').trim(); + + const imageEntry = messages.find((message) => message.content[1]?.image_url); + contents.push({ + parts: [ + { text: combinedText }, + { + inlineData: { + mimeType: 'image/png', + data: imageEntry.content[1].image_url.url ?? '', + }, + }, + ], + role: 'user', + }); + } else { + messages.forEach((message, index) => { + const role = message.role === 'assistant' ? 'model' : 'user'; + if (lastRole === role) { + currentText += '\n\n' + message.content; + } else { + if (currentText !== '') { + contents.push({ + parts: [{ text: currentText.trim() }], + role: lastRole, + }); + } + currentText = message.content; + lastRole = role; + } + if (index === messages.length - 1) { contents.push({ parts: [{ text: currentText.trim() }], role: lastRole, }); } - currentText = message.content; - lastRole = role; - } - if (index === messages.length - 1) { - contents.push({ - parts: [{ text: currentText.trim() }], - role: lastRole, - }); - } - }); + }); + } + return contents; }