rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix Gemini multimodal with JPG images

Browse Source 
Fixes #2763
This commit is contained in:
Cohee
2024-09-08 10:48:28 +03:00
parent 85ebf3116e
commit 8bb964515a
4 changed files with 33 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
public/scripts/openai.js
View File

@ -47,6 +47,7 @@ import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js';
import { getEventSourceStream } from './sse-stream.js';
import {
createThumbnail,
delay,
download,
getBase64Async,
@ -2440,15 +2441,14 @@ class Message {
if (!response.ok) throw new Error('Failed to fetch image');
const blob = await response.blob();
image = await getBase64Async(blob);
if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) {
image = image.split(',')[1];
}
} catch (error) {
console.error('Image adding skipped', error);
return;
}
}
image = await this.compressImage(image);
const quality = oai_settings.inline_image_quality || default_settings.inline_image_quality;
this.content = [
{ type: 'text', text: textContent },
@ -2464,6 +2464,29 @@ class Message {
}
}
/**
* Compress an image if it exceeds the size threshold for the current chat completion source.
* @param {string} image Data URL of the image.
* @returns {Promise<string>} Compressed image as a Data URL.
*/
async compressImage(image) {
if ([chat_completion_sources.OPENROUTER, chat_completion_sources.MAKERSUITE].includes(oai_settings.chat_completion_source)) {
const sizeThreshold = 2 * 1024 * 1024;
const dataSize = image.length * 0.75;
const maxSide = 1024;
if (dataSize > sizeThreshold) {
image = await createThumbnail(image, maxSide);
}
}
return image;
}
/**
* Get the token cost of an image.
* @param {string} dataUrl Data URL of the image.
* @param {string} quality String representing the quality of the image. Can be 'low', 'auto', or 'high'.
* @returns
*/
async getImageTokenCost(dataUrl, quality) {
if (quality === 'low') {
return Message.tokensPerImage;