From 65aec223a310ac6f13de898500a8a1538089b0fd Mon Sep 17 00:00:00 2001 From: equal-l2 Date: Wed, 23 Apr 2025 23:31:07 +0900 Subject: [PATCH 1/5] Vision models clean-up --- public/scripts/openai.js | 76 ++++++++++++---------------------------- src/prompt-converters.js | 35 +++--------------- 2 files changed, 27 insertions(+), 84 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 4f83f2b17..4b6a02368 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -5098,68 +5098,36 @@ export function isImageInliningSupported() { // gultra just isn't being offered as multimodal, thanks google. const visionSupportedModels = [ + // OpenAI + 'chatgpt-4o-latest', + 'gpt-4-turbo', 'gpt-4-vision', - 'gemini-2.5-pro-exp-03-25', - 'gemini-2.5-pro-preview-03-25', - 'gemini-2.0-pro-exp', - 'gemini-2.0-pro-exp-02-05', - 'gemini-2.5-flash-preview-04-17', - 'gemini-2.0-flash-lite-preview', - 'gemini-2.0-flash-lite-preview-02-05', - 'gemini-2.0-flash', - 'gemini-2.0-flash-001', - 'gemini-2.0-flash-thinking-exp-1219', - 'gemini-2.0-flash-thinking-exp-01-21', - 'gemini-2.0-flash-thinking-exp', - 'gemini-2.0-flash-exp', - 'gemini-2.0-flash-exp-image-generation', - 'gemini-1.5-flash', - 'gemini-1.5-flash-latest', - 'gemini-1.5-flash-001', - 'gemini-1.5-flash-002', - 'gemini-1.5-flash-exp-0827', - 'gemini-1.5-flash-8b', - 'gemini-1.5-flash-8b-exp-0827', - 'gemini-1.5-flash-8b-exp-0924', + 'gpt-4.1', + 'gpt-4.5-preview', + 'gpt-4o', + 'o1', + 'o3', + 'o4-mini', + // 01.AI (Yi) + 'yi-vision', + // Claude + 'claude-3', + // Cohere + 'c4ai-aya-vision', + // Google AI Studio + 'gemini-1.5', + 'gemini-2.0', + 'gemini-2.5', 'gemini-exp-1114', 'gemini-exp-1121', 'gemini-exp-1206', - 'gemini-1.0-pro-vision-latest', - 'gemini-1.5-pro', - 'gemini-1.5-pro-latest', - 'gemini-1.5-pro-001', - 'gemini-1.5-pro-002', - 'gemini-1.5-pro-exp-0801', - 'gemini-1.5-pro-exp-0827', - 'claude-3', - 'claude-3-5', - 'claude-3-7', - 'gpt-4-turbo', - 'gpt-4o', - 'gpt-4o-mini', - 'gpt-4.5-preview', - 'gpt-4.5-preview-2025-02-27', - 'o1', - 'o1-2024-12-17', - 'chatgpt-4o-latest', - 'yi-vision', - 'mistral-large-pixtral-2411', + // MistralAI 'mistral-small-2503', 'mistral-small-latest', - 'pixtral-12b-latest', - 'pixtral-12b', - 'pixtral-12b-2409', - 'pixtral-large-latest', - 'pixtral-large-2411', - 'c4ai-aya-vision-8b', - 'c4ai-aya-vision-32b', + 'pixtral', + // xAI (Grok) 'grok-2-vision', 'grok-vision', - 'gpt-4.1', - 'o3', - 'o3-2025-04-16', - 'o4-mini', - 'o4-mini-2025-04-16', ]; switch (oai_settings.chat_completion_source) { diff --git a/src/prompt-converters.js b/src/prompt-converters.js index 8ea3ab896..54b0f4df1 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -359,41 +359,16 @@ export function convertCohereMessages(messages, names) { * @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models */ export function convertGooglePrompt(messages, model, useSysPrompt, names) { - const visionSupportedModels = [ - 'gemini-2.5-pro-preview-03-25', - 'gemini-2.5-pro-exp-03-25', - 'gemini-2.0-pro-exp', - 'gemini-2.0-pro-exp-02-05', - 'gemini-2.5-flash-preview-04-17', - 'gemini-2.0-flash-lite-preview', - 'gemini-2.0-flash-lite-preview-02-05', - 'gemini-2.0-flash', - 'gemini-2.0-flash-001', - 'gemini-2.0-flash-thinking-exp', - 'gemini-2.0-flash-thinking-exp-01-21', - 'gemini-2.0-flash-thinking-exp-1219', - 'gemini-2.0-flash-exp', - 'gemini-2.0-flash-exp-image-generation', - 'gemini-1.5-flash', - 'gemini-1.5-flash-latest', - 'gemini-1.5-flash-001', - 'gemini-1.5-flash-002', - 'gemini-1.5-flash-exp-0827', - 'gemini-1.5-flash-8b', - 'gemini-1.5-flash-8b-exp-0827', - 'gemini-1.5-flash-8b-exp-0924', + const visionSupportedModelPrefix = [ + 'gemini-1.5', + 'gemini-2.0', + 'gemini-2.5', 'gemini-exp-1114', 'gemini-exp-1121', 'gemini-exp-1206', - 'gemini-1.5-pro', - 'gemini-1.5-pro-latest', - 'gemini-1.5-pro-001', - 'gemini-1.5-pro-002', - 'gemini-1.5-pro-exp-0801', - 'gemini-1.5-pro-exp-0827', ]; - const isMultimodal = visionSupportedModels.includes(model); + const isMultimodal = visionSupportedModelPrefix.some(prefix => model.startsWith(prefix)); let sys_prompt = ''; if (useSysPrompt) { From 44c5ce9a3047b3c1479bf3ad2df55ef4362b6b47 Mon Sep 17 00:00:00 2001 From: equal-l2 Date: Wed, 23 Apr 2025 23:32:43 +0900 Subject: [PATCH 2/5] Exclude o1-mini from vision supported models --- public/scripts/openai.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 4b6a02368..9f1dd771c 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -5132,7 +5132,12 @@ export function isImageInliningSupported() { switch (oai_settings.chat_completion_source) { case chat_completion_sources.OPENAI: - return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) && !oai_settings.openai_model.includes('gpt-4-turbo-preview') && !oai_settings.openai_model.includes('o3-mini')); + return visionSupportedModels.some(model => + oai_settings.openai_model.includes(model) + && !oai_settings.openai_model.includes('gpt-4-turbo-preview') + && !oai_settings.openai_model.includes('o1-mini') + && !oai_settings.openai_model.includes('o3-mini') + ); case chat_completion_sources.MAKERSUITE: return visionSupportedModels.some(model => oai_settings.google_model.includes(model)); case chat_completion_sources.CLAUDE: From 3e8f9e268027a87b912783cd01cf9531121cf71b Mon Sep 17 00:00:00 2001 From: equal-l2 Date: Thu, 24 Apr 2025 00:02:43 +0900 Subject: [PATCH 3/5] Fix for eslint --- public/scripts/openai.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 9f1dd771c..81312affd 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -5132,11 +5132,11 @@ export function isImageInliningSupported() { switch (oai_settings.chat_completion_source) { case chat_completion_sources.OPENAI: - return visionSupportedModels.some(model => + return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) && !oai_settings.openai_model.includes('gpt-4-turbo-preview') && !oai_settings.openai_model.includes('o1-mini') - && !oai_settings.openai_model.includes('o3-mini') + && !oai_settings.openai_model.includes('o3-mini'), ); case chat_completion_sources.MAKERSUITE: return visionSupportedModels.some(model => oai_settings.google_model.includes(model)); From 903839c9c50055db58fd2c6eb6010ded385df7f7 Mon Sep 17 00:00:00 2001 From: equal-l2 Date: Fri, 25 Apr 2025 01:40:13 +0900 Subject: [PATCH 4/5] Use array syntax for excluding non-vision OpenAI models Co-authored-by: Wolfsblvt --- public/scripts/openai.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 81312affd..02ab85b2b 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -5134,9 +5134,7 @@ export function isImageInliningSupported() { case chat_completion_sources.OPENAI: return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) - && !oai_settings.openai_model.includes('gpt-4-turbo-preview') - && !oai_settings.openai_model.includes('o1-mini') - && !oai_settings.openai_model.includes('o3-mini'), + && ['gpt-4-turbo-preview', 'o1-mini', 'o3-mini'].some(x => !oai_settings.openai_model.includes(x)) ); case chat_completion_sources.MAKERSUITE: return visionSupportedModels.some(model => oai_settings.google_model.includes(model)); From e621f0d967d94d8c2c984dd7261759fa1c96783d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 26 Apr 2025 00:34:21 +0300 Subject: [PATCH 5/5] Remove model name check in convertGooglePrompt --- src/prompt-converters.js | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/prompt-converters.js b/src/prompt-converters.js index 75ee89cc8..421455b7a 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -362,22 +362,12 @@ export function convertCohereMessages(messages, names) { /** * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models. * @param {object[]} messages Array of messages - * @param {string} model Model name + * @param {string} _model Model name * @param {boolean} useSysPrompt Use system prompt * @param {PromptNames} names Prompt names * @returns {{contents: *[], system_instruction: {parts: {text: string}[]}}} Prompt for Google MakerSuite models */ -export function convertGooglePrompt(messages, model, useSysPrompt, names) { - const visionSupportedModelPrefix = [ - 'gemini-1.5', - 'gemini-2.0', - 'gemini-2.5', - 'gemini-exp-1114', - 'gemini-exp-1121', - 'gemini-exp-1206', - ]; - - const isMultimodal = visionSupportedModelPrefix.some(prefix => model.startsWith(prefix)); +export function convertGooglePrompt(messages, _model, useSysPrompt, names) { const sysPrompt = []; if (useSysPrompt) { @@ -477,7 +467,7 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) { toolNameMap[toolCall.id] = toolCall.function.name; }); - } else if (part.type === 'image_url' && isMultimodal) { + } else if (part.type === 'image_url') { const mimeType = part.image_url.url.split(';')[0].split(':')[1]; const base64Data = part.image_url.url.split(',')[1]; parts.push({