From 6a51855f193955925e295cf05574d46532f364b5 Mon Sep 17 00:00:00 2001 From: based Date: Wed, 27 Mar 2024 13:52:51 +1000 Subject: [PATCH 1/2] Update Makersuite models --- public/index.html | 19 ++++++++++++++----- public/scripts/openai.js | 25 +++++++++++++++---------- src/prompt-converters.js | 8 +++++++- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/public/index.html b/public/index.html index 751cce6b3..baf5c2d3d 100644 --- a/public/index.html +++ b/public/index.html @@ -2604,11 +2604,20 @@

Google Model

diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 5acecfe7d..3a722dc97 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -3507,11 +3507,11 @@ async function onModelChange() { if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); - } else if (value === 'gemini-1.5-pro') { + } else if (value === 'gemini-1.5-pro-latest') { $('#openai_max_context').attr('max', max_1mil); - } else if (value === 'gemini-pro') { + } else if (value === 'gemini-ultra' || value === 'gemini-1.0-pro-latest' || value === 'gemini-pro' || value === 'gemini-1.0-ultra-latest') { $('#openai_max_context').attr('max', max_32k); - } else if (value === 'gemini-pro-vision') { + } else if (value === 'gemini-1.0-pro-vision-latest' || value === 'gemini-pro-vision') { $('#openai_max_context').attr('max', max_16k); } else { $('#openai_max_context').attr('max', max_8k); @@ -3939,21 +3939,26 @@ export function isImageInliningSupported() { return false; } - const gpt4v = 'gpt-4-vision'; - const geminiProV = 'gemini-pro-vision'; - const claude = 'claude-3'; - if (!oai_settings.image_inlining) { return false; } + // gultra just isn't being offered as multimodal, thanks google. + const visionSupportedModels = [ + 'gpt-4-vision', + 'gemini-1.0-pro-vision-latest', + 'gemini-1.5-pro-latest', + 'gemini-pro-vision', + 'claude-3' + ]; + switch (oai_settings.chat_completion_source) { case chat_completion_sources.OPENAI: - return oai_settings.openai_model.includes(gpt4v); + return visionSupportedModels.some(model => oai_settings.openai_model.includes(model)); case chat_completion_sources.MAKERSUITE: - return oai_settings.google_model.includes(geminiProV); + return visionSupportedModels.some(model => oai_settings.google_model.includes(model)); case chat_completion_sources.CLAUDE: - return oai_settings.claude_model.includes(claude); + return visionSupportedModels.some(model => oai_settings.claude_model.includes(model)); case chat_completion_sources.OPENROUTER: return !oai_settings.openrouter_force_instruct; case chat_completion_sources.CUSTOM: diff --git a/src/prompt-converters.js b/src/prompt-converters.js index 52161b661..3648fa1f8 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -196,7 +196,13 @@ function convertGooglePrompt(messages, model) { let lastRole = ''; let currentText = ''; - const isMultimodal = model === 'gemini-pro-vision'; + const visionSupportedModels = [ + 'gemini-1.0-pro-vision-latest', + 'gemini-1.5-pro-latest', + 'gemini-pro-vision', + ]; + + const isMultimodal = visionSupportedModels.includes(model); if (isMultimodal) { const combinedText = messages.map((message) => { From f3b9920f227780f091d2cd57526e78c4544b885d Mon Sep 17 00:00:00 2001 From: based Date: Wed, 27 Mar 2024 15:48:26 +1000 Subject: [PATCH 2/2] actually convert the prompts properly --- src/prompt-converters.js | 94 +++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/src/prompt-converters.js b/src/prompt-converters.js index 3648fa1f8..0b5af8f80 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -192,9 +192,6 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi function convertGooglePrompt(messages, model) { // This is a 1x1 transparent PNG const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='; - const contents = []; - let lastRole = ''; - let currentText = ''; const visionSupportedModels = [ 'gemini-1.0-pro-vision-latest', @@ -203,48 +200,65 @@ function convertGooglePrompt(messages, model) { ]; const isMultimodal = visionSupportedModels.includes(model); + let hasImage = false; - if (isMultimodal) { - const combinedText = messages.map((message) => { - const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: '; - return role + message.content; - }).join('\n\n').trim(); + const contents = []; + messages.forEach((message, index) => { + // fix the roles + if (message.role === 'system') { + message.role = 'user'; + } else if (message.role === 'assistant') { + message.role = 'model'; + } - const imageEntry = messages.find((message) => message.content?.[1]?.image_url); - const imageData = imageEntry?.content?.[1]?.image_url?.data ?? PNG_PIXEL; - contents.push({ - parts: [ - { text: combinedText }, - { - inlineData: { - mimeType: 'image/png', - data: imageData, - }, - }, - ], - role: 'user', - }); - } else { - messages.forEach((message, index) => { - const role = message.role === 'assistant' ? 'model' : 'user'; - if (lastRole === role) { - currentText += '\n\n' + message.content; + // similar story as claude + if (message.name) { + if (Array.isArray(message.content)) { + message.content[0].text = `${message.name}: ${message.content[0].text}`; } else { - if (currentText !== '') { - contents.push({ - parts: [{ text: currentText.trim() }], - role: lastRole, + message.content = `${message.name}: ${message.content}`; + } + delete message.name; + } + + //create the prompt parts + const parts = []; + if (typeof message.content === 'string') { + parts.push({ text: message.content }); + } else if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part.type === 'text') { + parts.push({ text: part.text }); + } else if (part.type === 'image_url' && isMultimodal) { + parts.push({ + inlineData: { + mimeType: 'image/png', + data: part.image_url.url, + }, }); + hasImage = true; } - currentText = message.content; - lastRole = role; - } - if (index === messages.length - 1) { - contents.push({ - parts: [{ text: currentText.trim() }], - role: lastRole, - }); - } + }); + } + + // merge consecutive messages with the same role + if (index > 0 && message.role === contents[contents.length - 1].role) { + contents[contents.length - 1].parts[0].text += '\n\n' + parts[0].text; + } else { + contents.push({ + role: message.role, + parts: parts, + }); + } + }); + + // pro 1.5 doesn't require a dummy image to be attached, other vision models do + if (isMultimodal && model !== 'gemini-1.5-pro-latest' && !hasImage) { + contents[0].parts.push({ + inlineData: { + mimeType: 'image/png', + data: PNG_PIXEL, + }, }); }