rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-01-31 11:35:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add inline image control

Browse Source
This commit is contained in:
Cohee 2024-05-14 01:08:31 +03:00
parent 16660e995e
commit 49cb8daf7d
3 changed files with 80 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
public/index.html
View File

@ -1680,7 +1680,7 @@
</div>
</div>
<div class="range-block" data-source="openai,openrouter,makersuite,claude,custom">
<label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand">
<label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand marginBot10">
<input id="openai_image_inlining" type="checkbox" />
<span data-i18n="Send inline images">Send inline images</span>
<div id="image_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
@ -1689,6 +1689,16 @@
<code><i class="fa-solid fa-wand-magic-sparkles"></i></code> menu to attach an image file to the chat.
</div>
</label>
<div class="flex-container flexFlowColumn wide100p textAlignCenter">
<label for="openai_inline_image_quality">
Inline Image Quality
</label>
<select id="openai_inline_image_quality">
<option value="auto">Auto</option>
<option value="low">Low</option>
<option value="high">High</option>
</select>
</div>
</div>
<div class="range-block" data-source="ai21">
<label for="use_ai21_tokenizer" title="Use AI21 Tokenizer" class="checkbox_label widthFreeExpand">

53
public/scripts/openai.js
View File

@ -50,6 +50,7 @@ import {
download,
getBase64Async,
getFileText,
getImageSizeFromDataURL,
getSortableDelay,
isDataURL,
parseJsonFile,
@ -273,6 +274,7 @@ const default_settings = {
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
inline_image_quality: 'low',
bypass_status_check: false,
continue_prefill: false,
names_behavior: character_names_behavior.NONE,
@ -348,6 +350,7 @@ const oai_settings = {
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
inline_image_quality: 'low',
bypass_status_check: false,
continue_prefill: false,
names_behavior: character_names_behavior.NONE,
@ -2188,12 +2191,47 @@ class Message {
}
}
const quality = oai_settings.inline_image_quality || default_settings.inline_image_quality;
this.content = [
{ type: 'text', text: textContent },
{ type: 'image_url', image_url: { 'url': image, 'detail': 'low' } },
{ type: 'image_url', image_url: { 'url': image, 'detail': quality } },
];
this.tokens += Message.tokensPerImage;
const tokens = await this.getImageTokenCost(image, quality);
this.tokens += tokens;
}
async getImageTokenCost(dataUrl, quality) {
if (quality === 'low') {
return Message.tokensPerImage;
}
const size = await getImageSizeFromDataURL(dataUrl);
// If the image is small enough, we can use the low quality token cost
if (quality === 'auto' && size.width <= 512 && size.height <= 512) {
return Message.tokensPerImage;
}
/*
* Images are first scaled to fit within a 2048 x 2048 square, maintaining their aspect ratio.
* Then, they are scaled such that the shortest side of the image is 768px long.
* Finally, we count how many 512px squares the image consists of.
* Each of those squares costs 170 tokens. Another 85 tokens are always added to the final total.
* https://platform.openai.com/docs/guides/vision/calculating-costs
*/
const scale = 2048 / Math.min(size.width, size.height);
const scaledWidth = Math.round(size.width * scale);
const scaledHeight = Math.round(size.height * scale);
const finalScale = 768 / Math.min(scaledWidth, scaledHeight);
const finalWidth = Math.round(scaledWidth * finalScale);
const finalHeight = Math.round(scaledHeight * finalScale);
const squares = Math.ceil(finalWidth / 512) * Math.ceil(finalHeight / 512);
const tokens = squares * 170 + 85;
return tokens;
}
/**
@ -2722,6 +2760,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.assistant_prefill = settings.assistant_prefill ?? default_settings.assistant_prefill;
oai_settings.human_sysprompt_message = settings.human_sysprompt_message ?? default_settings.human_sysprompt_message;
oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
oai_settings.inline_image_quality = settings.inline_image_quality ?? default_settings.inline_image_quality;
oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
oai_settings.seed = settings.seed ?? default_settings.seed;
oai_settings.n = settings.n ?? default_settings.n;
@ -2759,6 +2798,9 @@ function loadOpenAISettings(data, settings) {
$('#openai_image_inlining').prop('checked', oai_settings.image_inlining);
$('#openai_bypass_status_check').prop('checked', oai_settings.bypass_status_check);
$('#openai_inline_image_quality').val(oai_settings.inline_image_quality);
$(`#openai_inline_image_quality option[value="${oai_settings.inline_image_quality}"]`).prop('selected', true);
$('#model_openai_select').val(oai_settings.openai_model);
$(`#model_openai_select option[value="${oai_settings.openai_model}"`).attr('selected', true);
$('#model_claude_select').val(oai_settings.claude_model);
@ -3079,6 +3121,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
inline_image_quality: settings.inline_image_quality,
bypass_status_check: settings.bypass_status_check,
continue_prefill: settings.continue_prefill,
continue_postfix: settings.continue_postfix,
@ -3464,6 +3507,7 @@ function onSettingsPresetChange() {
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
inline_image_quality: ['#openai_inline_image_quality', 'inline_image_quality', false],
continue_prefill: ['#continue_prefill', 'continue_prefill', true],
continue_postfix: ['#continue_postfix', 'continue_postfix', false],
seed: ['#seed_openai', 'seed', false],
@ -4708,6 +4752,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#openai_inline_image_quality').on('input', function () {
oai_settings.inline_image_quality = String($(this).val());
saveSettingsDebounced();
});
$('#continue_prefill').on('input', function () {
oai_settings.continue_prefill = !!$(this).prop('checked');
saveSettingsDebounced();

18
public/scripts/utils.js
View File

@ -732,6 +732,24 @@ export function isDataURL(str) {
return regex.test(str);
}
/**
* Gets the size of an image from a data URL.
* @param {string} dataUrl Image data URL
* @returns {Promise<{ width: number, height: number }>} Image size
*/
export function getImageSizeFromDataURL(dataUrl) {
const image = new Image();
image.src = dataUrl;
return new Promise((resolve, reject) => {
image.onload = function () {
resolve({ width: image.width, height: image.height });
};
image.onerror = function () {
reject(new Error('Failed to load image'));
};
});
}
export function getCharaFilename(chid) {
const context = getContext();
const fileName = context.characters[chid ?? context.characterId].avatar;