rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add inline image control

Browse Source
This commit is contained in:
Cohee
2024-05-14 01:08:31 +03:00
parent 16660e995e
commit 49cb8daf7d
3 changed files with 80 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
public/scripts/openai.js
View File

@ -50,6 +50,7 @@ import {
download,
getBase64Async,
getFileText,
getImageSizeFromDataURL,
getSortableDelay,
isDataURL,
parseJsonFile,
@ -273,6 +274,7 @@ const default_settings = {
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
inline_image_quality: 'low',
bypass_status_check: false,
continue_prefill: false,
names_behavior: character_names_behavior.NONE,
@ -348,6 +350,7 @@ const oai_settings = {
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
inline_image_quality: 'low',
bypass_status_check: false,
continue_prefill: false,
names_behavior: character_names_behavior.NONE,
@ -2188,12 +2191,47 @@ class Message {
}
}
const quality = oai_settings.inline_image_quality || default_settings.inline_image_quality;
this.content = [
{ type: 'text', text: textContent },
{ type: 'image_url', image_url: { 'url': image, 'detail': 'low' } },
{ type: 'image_url', image_url: { 'url': image, 'detail': quality } },
];
this.tokens += Message.tokensPerImage;
const tokens = await this.getImageTokenCost(image, quality);
this.tokens += tokens;
}
async getImageTokenCost(dataUrl, quality) {
if (quality === 'low') {
return Message.tokensPerImage;
}
const size = await getImageSizeFromDataURL(dataUrl);
// If the image is small enough, we can use the low quality token cost
if (quality === 'auto' && size.width <= 512 && size.height <= 512) {
return Message.tokensPerImage;
}
/*
* Images are first scaled to fit within a 2048 x 2048 square, maintaining their aspect ratio.
* Then, they are scaled such that the shortest side of the image is 768px long.
* Finally, we count how many 512px squares the image consists of.
* Each of those squares costs 170 tokens. Another 85 tokens are always added to the final total.
* https://platform.openai.com/docs/guides/vision/calculating-costs
*/
const scale = 2048 / Math.min(size.width, size.height);
const scaledWidth = Math.round(size.width * scale);
const scaledHeight = Math.round(size.height * scale);
const finalScale = 768 / Math.min(scaledWidth, scaledHeight);
const finalWidth = Math.round(scaledWidth * finalScale);
const finalHeight = Math.round(scaledHeight * finalScale);
const squares = Math.ceil(finalWidth / 512) * Math.ceil(finalHeight / 512);
const tokens = squares * 170 + 85;
return tokens;
}
/**
@ -2722,6 +2760,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.assistant_prefill = settings.assistant_prefill ?? default_settings.assistant_prefill;
oai_settings.human_sysprompt_message = settings.human_sysprompt_message ?? default_settings.human_sysprompt_message;
oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
oai_settings.inline_image_quality = settings.inline_image_quality ?? default_settings.inline_image_quality;
oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
oai_settings.seed = settings.seed ?? default_settings.seed;
oai_settings.n = settings.n ?? default_settings.n;
@ -2759,6 +2798,9 @@ function loadOpenAISettings(data, settings) {
$('#openai_image_inlining').prop('checked', oai_settings.image_inlining);
$('#openai_bypass_status_check').prop('checked', oai_settings.bypass_status_check);
$('#openai_inline_image_quality').val(oai_settings.inline_image_quality);
$(`#openai_inline_image_quality option[value="${oai_settings.inline_image_quality}"]`).prop('selected', true);
$('#model_openai_select').val(oai_settings.openai_model);
$(`#model_openai_select option[value="${oai_settings.openai_model}"`).attr('selected', true);
$('#model_claude_select').val(oai_settings.claude_model);
@ -3079,6 +3121,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
inline_image_quality: settings.inline_image_quality,
bypass_status_check: settings.bypass_status_check,
continue_prefill: settings.continue_prefill,
continue_postfix: settings.continue_postfix,
@ -3464,6 +3507,7 @@ function onSettingsPresetChange() {
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
inline_image_quality: ['#openai_inline_image_quality', 'inline_image_quality', false],
continue_prefill: ['#continue_prefill', 'continue_prefill', true],
continue_postfix: ['#continue_postfix', 'continue_postfix', false],
seed: ['#seed_openai', 'seed', false],
@ -4708,6 +4752,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#openai_inline_image_quality').on('input', function () {
oai_settings.inline_image_quality = String($(this).val());
saveSettingsDebounced();
});
$('#continue_prefill').on('input', function () {
oai_settings.continue_prefill = !!$(this).prop('checked');
saveSettingsDebounced();