From e1a29b36f5355ef4dd081d3a589d429287314f32 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 13 Aug 2024 23:21:00 +0300 Subject: [PATCH 1/4] Image Generation: Add swipes for generated images Supersedes #2648 --- public/index.html | 5 + public/script.js | 22 ++++- .../extensions/stable-diffusion/index.js | 97 +++++++++++++++++++ public/style.css | 23 +++++ 4 files changed, 146 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index 45dfcd88c..9629415cb 100644 --- a/public/index.html +++ b/public/index.html @@ -5801,6 +5801,11 @@
+
+
+
1/1
+
+
diff --git a/public/script.js b/public/script.js index 798667b9d..8808320c8 100644 --- a/public/script.js +++ b/public/script.js @@ -459,6 +459,7 @@ export const event_types = { LLM_FUNCTION_TOOL_REGISTER: 'llm_function_tool_register', LLM_FUNCTION_TOOL_CALL: 'llm_function_tool_call', ONLINE_STATUS_CHANGED: 'online_status_changed', + IMAGE_SWIPED: 'image_swiped', }; export const eventSource = new EventEmitter(); @@ -2112,6 +2113,7 @@ export function updateMessageBlock(messageId, message) { export function appendMediaToMessage(mes, messageElement, adjustScroll = true) { // Add image to message if (mes.extra?.image) { + const container = messageElement.find('.mes_img_container'); const chatHeight = $('#chat').prop('scrollHeight'); const image = messageElement.find('.mes_img'); const text = messageElement.find('.mes_text'); @@ -2127,9 +2129,27 @@ export function appendMediaToMessage(mes, messageElement, adjustScroll = true) { }); image.attr('src', mes.extra?.image); image.attr('title', mes.extra?.title || mes.title || ''); - messageElement.find('.mes_img_container').addClass('img_extra'); + container.addClass('img_extra'); image.toggleClass('img_inline', isInline); text.toggleClass('displayNone', !isInline); + + const imageSwipes = mes.extra.image_swipes; + if (Array.isArray(imageSwipes) && imageSwipes.length > 0) { + container.addClass('img_swipes'); + const counter = container.find('.mes_img_swipe_counter'); + const currentImage = imageSwipes.indexOf(mes.extra.image) + 1; + counter.text(`${currentImage}/${imageSwipes.length}`); + + const swipeLeft = container.find('.mes_img_swipe_left'); + swipeLeft.off('click').on('click', function () { + eventSource.emit(event_types.IMAGE_SWIPED, { message: mes, element: messageElement, direction: 'left' }); + }); + + const swipeRight = container.find('.mes_img_swipe_right'); + swipeRight.off('click').on('click', function () { + eventSource.emit(event_types.IMAGE_SWIPED, { message: mes, element: messageElement, direction: 'right' }); + }); + } } // Add file to message diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 4e942e4be..4edb1bacf 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -59,6 +59,7 @@ const initiators = { action: 'action', interactive: 'interactive', wand: 'wand', + swipe: 'swipe', }; const generationMode = { @@ -3429,6 +3430,7 @@ async function sendMessage(prompt, image, generationType, additionalNegativePref generationType: generationType, negative: additionalNegativePrefix, inline_image: false, + image_swipes: [image], }, }; context.chat.push(message); @@ -3657,6 +3659,99 @@ async function writePromptFields(characterId) { await writeExtensionField(characterId, 'sd_character_prompt', promptObject); } +/** + * Switches an image to the next or previous one in the swipe list. + * @param {object} args Event arguments + * @param {any} args.message Message object + * @param {JQuery} args.element Message element + * @param {string} args.direction Swipe direction + * @returns {Promise} + */ +async function onImageSwiped({ message, element, direction }) { + const context = getContext(); + const animationClass = 'fa-fade'; + const messageImg = element.find('.mes_img'); + + // Current image is already animating + if (messageImg.hasClass(animationClass)) { + return; + } + + const swipes = message?.extra?.image_swipes; + + if (!Array.isArray(swipes)) { + console.warn('No image swipes found in the message'); + return; + } + + const currentIndex = swipes.indexOf(message.extra.image); + + if (currentIndex === -1) { + console.warn('Current image not found in the swipes'); + return; + } + + // Switch to previous image or wrap around if at the beginning + if (direction === 'left') { + const newIndex = currentIndex === 0 ? swipes.length - 1 : currentIndex - 1; + message.extra.image = swipes[newIndex]; + + // Update the image in the message + appendMediaToMessage(message, element, false); + } + + // Switch to next image or generate a new one if at the end + if (direction === 'right') { + const newIndex = currentIndex === swipes.length - 1 ? swipes.length : currentIndex + 1; + + if (newIndex === swipes.length) { + const abortController = new AbortController(); + const swipeControls = element.find('.mes_img_swipes'); + const stopButton = document.getElementById('sd_stop_gen'); + const stopListener = () => abortController.abort('Aborted by user'); + const generationType = message?.extra?.generationType ?? generationMode.FREE; + const dimensions = setTypeSpecificDimensions(generationType); + const originalSeed = extension_settings.sd.seed; + extension_settings.sd.seed = Math.round(Math.random() * Number.MAX_SAFE_INTEGER); + let imagePath = ''; + + try { + $(stopButton).show(); + eventSource.once(CUSTOM_STOP_EVENT, stopListener); + const callback = () => { }; + const hasNegative = message.extra.negative; + const prompt = await refinePrompt(message.extra.title, false, false); + const negativePromptPrefix = hasNegative ? await refinePrompt(message.extra.negative, false, true) : ''; + const characterName = context.characterId + ? context.characters[context.characterId].name + : context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString(); + + messageImg.addClass(animationClass); + swipeControls.hide(); + imagePath = await sendGenerationRequest(generationType, prompt, negativePromptPrefix, characterName, callback, initiators.swipe, abortController.signal); + } finally { + $(stopButton).hide(); + messageImg.removeClass(animationClass); + swipeControls.show(); + eventSource.removeListener(CUSTOM_STOP_EVENT, stopListener); + restoreOriginalDimensions(dimensions); + extension_settings.sd.seed = originalSeed; + } + + if (!imagePath) { + return; + } + + swipes.push(imagePath); + } + + message.extra.image = swipes[newIndex]; + appendMediaToMessage(message, element, false); + } + + await context.saveChat(); +} + jQuery(async () => { await addSDGenButtons(); @@ -3795,6 +3890,8 @@ jQuery(async () => { } }); + eventSource.on(event_types.IMAGE_SWIPED, onImageSwiped); + eventSource.on(event_types.CHAT_CHANGED, onChatChanged); await loadSettings(); diff --git a/public/style.css b/public/style.css index 84d97fb9e..742a370ea 100644 --- a/public/style.css +++ b/public/style.css @@ -4569,6 +4569,7 @@ a { image-rendering: -webkit-optimize-contrast; } +.mes_img_swipes, .mes_img_controls { position: absolute; top: 0.1em; @@ -4578,9 +4579,16 @@ a { opacity: 0; flex-direction: row; justify-content: space-between; + align-items: center; padding: 1em; } +.mes_img_swipes { + top: unset; + bottom: 0.1rem; +} + +.mes_img_swipes .right_menu_button, .mes_img_controls .right_menu_button { filter: brightness(90%); text-shadow: 1px 1px var(--SmartThemeShadowColor) !important; @@ -4589,16 +4597,20 @@ a { width: 1.25em; } +.mes_img_swipes .right_menu_button::before, .mes_img_controls .right_menu_button::before { /* Fix weird alignment with this font-awesome icons on focus */ position: relative; top: 0.6125em; } +.mes_img_swipes .right_menu_button:hover, .mes_img_controls .right_menu_button:hover { filter: brightness(150%); } +.mes_img_container:hover .mes_img_swipes, +.mes_img_container:focus-within .mes_img_swipes, .mes_img_container:hover .mes_img_controls, .mes_img_container:focus-within .mes_img_controls { opacity: 1; @@ -4612,6 +4624,17 @@ body:not(.caption) .mes_img_caption { display: none; } +.mes_img_container:not(.img_swipes) .mes_img_swipes, +body:not(.sd) .mes_img_swipes { + display: none; +} + +.mes_img_swipe_counter { + font-weight: 600; + filter: drop-shadow(2px 4px 6px black); + cursor: default; +} + .img_enlarged_holder { /* Scaling via flex-grow and object-fit only works if we have some kind of base-height set */ min-height: 120px; From 4fd8d8e0eedabc955653884ee87e49e05bcce404 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 13 Aug 2024 23:35:49 +0300 Subject: [PATCH 2/4] Add swipes when using paintbrush --- .../scripts/extensions/stable-diffusion/index.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 4edb1bacf..fe81d722e 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -3617,10 +3617,23 @@ async function sdMessageButton(e) { function saveGeneratedImage(prompt, image, generationType, negative) { // Some message sources may not create the extra object - if (typeof message.extra !== 'object') { + if (typeof message.extra !== 'object' || message.extra === null) { message.extra = {}; } + // Add image to the swipe list if it's not already there + if (!Array.isArray(message.extra.image_swipes)) { + message.extra.image_swipes = []; + } + + const swipes = message.extra.image_swipes; + + if (message.extra.image && !swipes.includes(message.extra.image)) { + swipes.push(message.extra.image); + } + + swipes.push(image); + // If already contains an image and it's not inline - leave it as is message.extra.inline_image = message.extra.image && !message.extra.inline_image ? false : true; message.extra.image = image; From a08b3ec7fc63c5d4301a91d07717c53dce016c89 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 15 Aug 2024 00:40:08 +0300 Subject: [PATCH 3/4] Sort gallery images by date --- src/endpoints/images.js | 2 +- src/util.js | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/endpoints/images.js b/src/endpoints/images.js index a01e34073..7290307ca 100644 --- a/src/endpoints/images.js +++ b/src/endpoints/images.js @@ -82,7 +82,7 @@ router.post('/list/:folder', (request, response) => { } try { - const images = getImages(directoryPath); + const images = getImages(directoryPath, 'date'); return response.send(images); } catch (error) { console.error(error); diff --git a/src/util.js b/src/util.js index b82007156..9f42afb13 100644 --- a/src/util.js +++ b/src/util.js @@ -382,14 +382,31 @@ function removeOldBackups(directory, prefix) { } } -function getImages(path) { +/** + * Get a list of images in a directory. + * @param {string} directoryPath Path to the directory containing the images + * @param {'name' | 'date'} sortBy Sort images by name or date + * @returns {string[]} List of image file names + */ +function getImages(directoryPath, sortBy = 'name') { + function getSortFunction() { + switch (sortBy) { + case 'name': + return Intl.Collator().compare; + case 'date': + return (a, b) => fs.statSync(path.join(directoryPath, a)).mtimeMs - fs.statSync(path.join(directoryPath, b)).mtimeMs; + default: + return (_a, _b) => 0; + } + } + return fs - .readdirSync(path) + .readdirSync(directoryPath) .filter(file => { const type = mime.lookup(file); return type && type.startsWith('image/'); }) - .sort(Intl.Collator().compare); + .sort(getSortFunction()); } /** From 58db8440abcb8671a9fcf674bdc037357d5bdefc Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 15 Aug 2024 00:49:50 +0300 Subject: [PATCH 4/4] Invert groupId check --- .../scripts/extensions/stable-diffusion/index.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index fe81d722e..98eb28ae4 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -2312,9 +2312,9 @@ async function generatePicture(initiator, args, trigger, message, callback) { const quietPrompt = getQuietPrompt(generationType, trigger); const context = getContext(); - // if context.characterId is not null, then we get context.characters[context.characterId].avatar, else we get groupId and context.groups[groupId].id - // sadly, groups is not an array, but is a dict with keys being index numbers, so we have to filter it - const characterName = context.characterId ? context.characters[context.characterId].name : context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString(); + const characterName = context.groupId + ? context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString() + : context.characters[context.characterId]?.name; if (generationType == generationMode.BACKGROUND) { const callbackOriginal = callback; @@ -3573,7 +3573,9 @@ async function sdMessageButton(e) { const $mes = $icon.closest('.mes'); const message_id = $mes.attr('mesid'); const message = context.chat[message_id]; - const characterFileName = context.characterId ? context.characters[context.characterId].name : context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString(); + const characterFileName = context.groupId + ? context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString() + : context.characters[context.characterId]?.name; const messageText = message?.mes; const hasSavedImage = message?.extra?.image && message?.extra?.title; const hasSavedNegative = message?.extra?.negative; @@ -3735,9 +3737,9 @@ async function onImageSwiped({ message, element, direction }) { const hasNegative = message.extra.negative; const prompt = await refinePrompt(message.extra.title, false, false); const negativePromptPrefix = hasNegative ? await refinePrompt(message.extra.negative, false, true) : ''; - const characterName = context.characterId - ? context.characters[context.characterId].name - : context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString(); + const characterName = context.groupId + ? context.groups[Object.keys(context.groups).filter(x => context.groups[x].id === context.groupId)[0]]?.id?.toString() + : context.characters[context.characterId]?.name; messageImg.addClass(animationClass); swipeControls.hide();