From b164084c0c9889c13958e221649171010137860f Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 00:19:50 -0700 Subject: [PATCH 01/26] Enhancement: Make buttons scrollable --- public/scripts/slash-commands.js | 37 ++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index a34b4e544..6a4c5f63f 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2074,7 +2074,32 @@ async function buttonsCallback(args, text) { let popup; const buttonContainer = document.createElement('div'); - buttonContainer.classList.add('flex-container', 'flexFlowColumn', 'wide100p', 'm-t-1'); + buttonContainer.classList.add('flex-container', 'flexFlowColumn', 'wide100p'); + + const scrollableContainer = document.createElement('div'); + scrollableContainer.style.maxHeight = '50vh'; // Use viewport height instead of fixed pixels + scrollableContainer.style.overflowY = 'auto'; + scrollableContainer.style.WebkitOverflowScrolling = 'touch'; // Enable momentum scrolling on iOS + scrollableContainer.classList.add('m-t-1', 'scrollable-buttons'); + + // Add custom CSS for better mobile scrolling + const style = document.createElement('style'); + style.textContent = ` + .scrollable-buttons { + -webkit-overflow-scrolling: touch; + overflow-y: auto; + flex-shrink: 1; + min-height: 0; + } + .scrollable-buttons::-webkit-scrollbar { + width: 6px; + } + .scrollable-buttons::-webkit-scrollbar-thumb { + background-color: rgba(0, 0, 0, 0.2); + border-radius: 3px; + } + `; + document.head.appendChild(style); for (const [result, button] of resultToButtonMap) { const buttonElement = document.createElement('div'); @@ -2087,9 +2112,16 @@ async function buttonsCallback(args, text) { buttonContainer.appendChild(buttonElement); } + scrollableContainer.appendChild(buttonContainer); + const popupContainer = document.createElement('div'); popupContainer.innerHTML = safeValue; - popupContainer.appendChild(buttonContainer); + popupContainer.appendChild(scrollableContainer); + + // Ensure the popup uses flex layout + popupContainer.style.display = 'flex'; + popupContainer.style.flexDirection = 'column'; + popupContainer.style.maxHeight = '80vh'; // Limit the overall height of the popup popup = new Popup(popupContainer, POPUP_TYPE.TEXT, '', { okButton: 'Cancel' }); popup.show() @@ -4272,3 +4304,4 @@ sendTextarea.addEventListener('input', () => { sendTextarea.style.fontFamily = null; } }); + From 8f09aced83ab44088bb28ec81941136562fbfce2 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 00:57:15 -0700 Subject: [PATCH 02/26] Add scroll bar to make it obvious --- public/scripts/slash-commands.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 6a4c5f63f..08ea48536 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2090,12 +2090,14 @@ async function buttonsCallback(args, text) { overflow-y: auto; flex-shrink: 1; min-height: 0; + scrollbar-width: thin; + scrollbar-color: rgba(255, 255, 255, 0.3) transparent; } .scrollable-buttons::-webkit-scrollbar { width: 6px; } .scrollable-buttons::-webkit-scrollbar-thumb { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgba(255, 255, 255, 0.3); border-radius: 3px; } `; @@ -4304,4 +4306,3 @@ sendTextarea.addEventListener('input', () => { sendTextarea.style.fontFamily = null; } }); - From 9d77140ea589e08d86c2d135ae4556f6eb9830fb Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 03:51:36 -0700 Subject: [PATCH 03/26] Fix: Inconsistent Textarea resizing in small windows --- public/scripts/RossAscends-mods.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index ecc31df8b..bfcb8e809 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -697,20 +697,26 @@ const chatBlock = document.getElementById('chat'); const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; /** - * this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height) + * Auto-resizes the send message textarea to fit its content, up to a maximum height defined by CSS. + * This function preserves chat scroll position, resets the textarea height for accurate measurement, + * calculates the required height, sets the new height, and then restores the chat scroll position. + * Firefox-specific scrolling adjustments are included for smoother behavior. */ function autoFitSendTextArea() { - const originalScrollBottom = chatBlock.scrollHeight - (chatBlock.scrollTop + chatBlock.offsetHeight); - if (Math.ceil(sendTextArea.scrollHeight + 3) >= Math.floor(sendTextArea.offsetHeight)) { - const sendTextAreaMinHeight = '0px'; - sendTextArea.style.height = sendTextAreaMinHeight; - } - const newHeight = sendTextArea.scrollHeight + 3; + const originalScrollTop = chatBlock.scrollTop; + const originalScrollHeight = chatBlock.scrollHeight; + + sendTextArea.style.height = '1px'; + + const newHeight = sendTextArea.scrollHeight; + sendTextArea.style.height = `${newHeight}px`; + // Restore chat scroll position (Firefox-specific adjustment for smoothness). if (!isFirefox) { - const newScrollTop = Math.round(chatBlock.scrollHeight - (chatBlock.offsetHeight + originalScrollBottom)); - chatBlock.scrollTop = newScrollTop; + chatBlock.scrollTop = originalScrollTop + (chatBlock.scrollHeight - originalScrollHeight); + } else { + chatBlock.scrollTo({ top: chatBlock.scrollHeight, behavior: 'auto' }); } } export const autoFitSendTextAreaDebounced = debounce(autoFitSendTextArea, debounce_timeout.short); From 5e883e446a20a5a8b074a28eaee111ba6deb3ee5 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Sun, 17 Nov 2024 00:36:06 +1100 Subject: [PATCH 04/26] Make dynamic reroll available without use of modifier key Linting --- public/css/logprobs.css | 14 +++ public/scripts/logprobs.js | 232 ++++++++++++++++++++----------------- 2 files changed, 137 insertions(+), 109 deletions(-) diff --git a/public/css/logprobs.css b/public/css/logprobs.css index 91a716c7d..f9b5e6b42 100644 --- a/public/css/logprobs.css +++ b/public/css/logprobs.css @@ -72,6 +72,20 @@ opacity: 0.5; } +.logprobs_output_prefix:hover { + background-color: rgba(255, 0, 50, 0.4); +} + +.logprobs_output_prefix:hover ~ .logprobs_output_prefix { + background-color: rgba(255, 0, 50, 0.4); +} + +#logprobsReroll { + float: right; /* Position the button to the right */ + margin: 5px 0 5px 10px; /* Add spacing (top, right, bottom, left) */ + clear: right; /* Ensure it starts on a new line */ +} + .logprobs_candidate_list { grid-row-start: 3; grid-row-end: 4; diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index e82294782..49007a9bf 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -1,6 +1,5 @@ import { animation_duration, - callPopup, chat, cleanUpMessage, event_types, @@ -13,6 +12,8 @@ import { import { debounce, delay, getStringHash } from './utils.js'; import { decodeTextTokens, getTokenizerBestMatch } from './tokenizers.js'; import { power_user } from './power-user.js'; +import { callGenericPopup, POPUP_TYPE } from './popup.js'; +import { t } from './i18n.js'; const TINTS = 4; const MAX_MESSAGE_LOGPROBS = 100; @@ -43,17 +44,26 @@ const MAX_MESSAGE_LOGPROBS = 100; * @property {Candidate[]} topLogprobs - Array of top candidate tokens */ -let state = { - /** @type {TokenLogprobs | null} */ +/** + * State object for Token Probabilities + * @typedef {Object} LogprobsState + * @property {?TokenLogprobs} selectedTokenLogprobs Log probabilities for + * currently-selected token. + * @property {Map} messageLogprobs Log probabilities for + * each message, keyed by message hash. + */ + +/** + * @type {LogprobsState} state + */ +const state = { selectedTokenLogprobs: null, - /** @type {Map} */ messageLogprobs: new Map(), }; /** - * renderAlternativeTokensView renders the Token Probabilities UI and all - * subviews with the active message's logprobs data. If the message has no token - * logprobs, a zero-state is rendered. + * Renders the Token Probabilities UI and all subviews with the active message's + * logprobs data. If the message has no token logprobs, a message is displayed. */ function renderAlternativeTokensView() { const view = $('#logprobs_generation_output'); @@ -68,13 +78,14 @@ function renderAlternativeTokensView() { const usingSmoothStreaming = isStreamingEnabled() && power_user.smooth_streaming; if (!messageLogprobs?.length || usingSmoothStreaming) { const emptyState = $('
'); - const noTokensMsg = usingSmoothStreaming - ? 'Token probabilities are not available when using Smooth Streaming.' - : 'No token probabilities available for the current message.'; - const msg = power_user.request_token_probabilities - ? noTokensMsg - : 'Enable Request token probabilities in the User Settings menu to use this feature.'; - emptyState.html(msg); + const noTokensMsg = !power_user.request_token_probabilities + ? 'Enable Request token probabilities in the User Settings menu to use this feature.' + : usingSmoothStreaming + ? t`Token probabilities are not available when using Smooth Streaming.` + : is_send_press + ? t`Generation in progress...` + : t`No token probabilities available for the current message.`; + emptyState.html(noTokensMsg); emptyState.addClass('logprobs_empty_state'); view.append(emptyState); return; @@ -84,16 +95,39 @@ function renderAlternativeTokensView() { const tokenSpans = []; if (prefix) { - const prefixSpan = $(''); - prefixSpan.text(prefix); - prefixSpan.html(prefixSpan.html().replace(/\n/g, '
')); - prefixSpan.addClass('logprobs_output_prefix'); - prefixSpan.attr('title', 'Select to reroll the last \'Continue\' generation.\nHold the CTRL key when clicking to reroll from before that word.'); - prefixSpan.click(onPrefixClicked); - addKeyboardProps(prefixSpan); - tokenSpans.push(...withVirtualWhitespace(prefix, prefixSpan)); + const rerollButton = $(''); + rerollButton.attr('title', t`Reroll with the entire prefix`); + rerollButton.on('click', () => onPrefixClicked(prefix.length)); + tokenSpans.push(rerollButton); + + let cumulativeOffset = 0; + const words = prefix.split(/\s+/); + const delimiters = prefix.match(/\s+/g) || []; // Capture the actual delimiters + + words.forEach((word, i) => { + const span = $(''); + span.text(`${word} `); + + span.addClass('logprobs_output_prefix'); + span.attr('title', t`Reroll from this point`); + + let offset = cumulativeOffset; + span.on('click', () => onPrefixClicked(offset)); + addKeyboardProps(span); + + tokenSpans.push(span); + tokenSpans.push(delimiters[i]?.includes('\n') + ? document.createElement('br') + : document.createTextNode(delimiters[i] || ' '), + ); + + cumulativeOffset += word.length + (delimiters[i]?.length || 0); + }); } + messageLogprobs.forEach((tokenData, i) => { const { token } = tokenData; const span = $(''); @@ -101,7 +135,7 @@ function renderAlternativeTokensView() { span.text(text); span.addClass('logprobs_output_token'); span.addClass('logprobs_tint_' + (i % TINTS)); - span.click(() => onSelectedTokenChanged(tokenData, span)); + span.on('click', () => onSelectedTokenChanged(tokenData, span)); addKeyboardProps(span); tokenSpans.push(...withVirtualWhitespace(token, span)); }); @@ -129,6 +163,10 @@ function addKeyboardProps(element) { /** * renderTopLogprobs renders the top logprobs subview with the currently * selected token highlighted. If no token is selected, the subview is hidden. + * + * Callers: + * - renderAlternativeTokensView, to render the entire view + * - onSelectedTokenChanged, to update the view when a token is selected */ function renderTopLogprobs() { $('#logprobs_top_logprobs_hint').hide(); @@ -150,8 +188,7 @@ function renderTopLogprobs() { const probability = Math.exp(log); sum += probability; return [text, probability, log]; - } - else { + } else { return [text, log, null]; } }); @@ -175,7 +212,7 @@ function renderTopLogprobs() { } addKeyboardProps(container); if (token !== '') { - container.click(() => onAlternativeClicked(state.selectedTokenLogprobs, token)); + container.on('click', () => onAlternativeClicked(state.selectedTokenLogprobs, token)); } else { container.prop('disabled', true); } @@ -192,9 +229,8 @@ function renderTopLogprobs() { } /** - * onSelectedTokenChanged is called when the user clicks on a token in the - * token output view. It updates the selected token state and re-renders the - * top logprobs view, or deselects the token if it was already selected. + * User clicks on a token in the token output view. It updates the selected token state + * and re-renders the top logprobs view, or deselects the token if it was already selected. * @param {TokenLogprobs} logprobs - logprob data for the selected token * @param {Element} span - target span node that was clicked */ @@ -223,7 +259,10 @@ function onAlternativeClicked(tokenLogprobs, alternative) { } if (getGeneratingApi() === 'openai') { - return callPopup('

Feature unavailable

Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.

', 'text'); + const title = t`Feature unavailable`; + const message = t`Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.`; + const content = `

${title}

${message}

`; + return callGenericPopup(content, POPUP_TYPE.TEXT); } const { messageLogprobs, continueFrom } = getActiveMessageLogprobData(); @@ -234,79 +273,29 @@ function onAlternativeClicked(tokenLogprobs, alternative) { const prefix = continueFrom || ''; const prompt = prefix + tokens.join(''); - const messageId = chat.length - 1; - createSwipe(messageId, prompt); - - $('.swipe_right:last').click(); // :see_no_evil: - - Generate('continue').then(_ => void _); + addGeneration(prompt); } /** - * getTextBeforeClickedWord retrieves the portion of text within a span - * that appears before the word clicked by the user. Using the x and y - * coordinates from a PointerEvent, this function identifies the exact - * word clicked and returns the text preceding it within the span. + * User clicks on the reroll button in the token output view, or on a word in the + * prefix. Retrieve the prefix for the current message and truncate it at the + * offset for the selected word. Then request a `continue` completion from the + * model with the new prompt. * - * If the clicked position does not resolve to a valid word or text node, - * the entire span text is returned as a fallback. + * If no offset is provided, the entire prefix will be rerolled. * - * @param {PointerEvent} event - The click event containing the x and y coordinates. - * @param {string} spanText - The full text content of the span element. - * @returns {string} The text before the clicked word, or the entire span text as fallback. + * @param {number} offset - index of the token in the prefix to reroll from + * @returns {void} + * @param offset */ -function getTextBeforeClickedWord(event, spanText) { - const x = event.clientX; - const y = event.clientY; - const range = document.caretRangeFromPoint(x, y); - - if (range && range.startContainer.nodeType === Node.TEXT_NODE) { - const textNode = range.startContainer; - const offset = range.startOffset; - - // Get the full text content of the text node - const text = textNode.nodeValue; - - // Find the boundaries of the clicked word - const start = text.lastIndexOf(' ', offset - 1) + 1; - - // Return the text before the clicked word - return text.slice(0, start); - } - - // If we can't determine the exact word, return the full span text as a fallback - return spanText; -} - - -/** - * onPrefixClicked is called when the user clicks on the carried-over prefix - * in the token output view. It allows them to reroll the last 'continue' - * completion with none of the output generated from it, in case they don't - * like the results. - * - * If the user holds the Ctrl key while clicking, only the portion of text - * before the clicked word is retained as the prefix for rerolling - */ -function onPrefixClicked() { +function onPrefixClicked(offset = undefined) { if (!checkGenerateReady()) { return; } - const { continueFrom } = getActiveMessageLogprobData(); - const messageId = chat.length - 1; - - // Check if Ctrl key is pressed during the click - let prefix = continueFrom || ''; - if (event.ctrlKey) { - // Ctrl is pressed - use the text before the clicked word - prefix = getTextBeforeClickedWord(event, continueFrom); - } - - // Use the determined `prefix` - createSwipe(messageId, prefix); - $('.swipe_right:last').click(); - Generate('continue').then(_ => void _); + const { continueFrom } = getActiveMessageLogprobData() || {}; + const prefix = continueFrom ? continueFrom.substring(0, offset) : ''; + addGeneration(prefix); } function checkGenerateReady() { @@ -317,6 +306,22 @@ function checkGenerateReady() { return true; } +/** + * Generates a new swipe as a continuation of the given prompt, when user selects + * an alternative token or rerolls from a prefix. + * + * @param prompt + */ +function addGeneration(prompt) { + const messageId = chat.length - 1; + if (prompt && prompt.length > 0) { + createSwipe(messageId, prompt); + $('.swipe_right:last').trigger('click'); + void Generate('continue'); + } else { + $('.swipe_right:last').trigger('click'); + } +} /** * onToggleLogprobsPanel is called when the user performs an action that toggles @@ -356,8 +361,7 @@ function onToggleLogprobsPanel() { } /** - * createSwipe appends a new swipe to the target chat message with the given - * text. + * Appends a new swipe to the target chat message with the given text. * @param {number} messageId - target chat message ID * @param {string} prompt - initial prompt text which will be continued */ @@ -400,9 +404,10 @@ function toVisibleWhitespace(input) { * allow text to wrap despite whitespace characters being replaced with a dot. * @param {string} text - token text being evaluated for whitespace * @param {Element} span - target span node to be wrapped - * @returns {Element[]} array of nodes to be appended to the DOM + * @returns {Node[]} - array of nodes to be appended to the parent element */ function withVirtualWhitespace(text, span) { + /** @type {Node[]} */ const result = [span]; if (text.match(/^\s/)) { result.unshift(document.createTextNode('\u200b')); @@ -430,12 +435,16 @@ function withVirtualWhitespace(text, span) { } /** - * saveLogprobsForActiveMessage receives an array of TokenLogprobs objects - * representing the top logprobs for each token in a message and associates it - * with the active message. + * Receives the top logprobs for each token in a message and associates it with the active message. + * + * Ensure the active message has been updated and rendered before calling this function + * or the logprobs data will be saved to the wrong message. + * + * Callers: + * - Generate:onSuccess via saveLogprobsForActiveMessage, for non-streaming text completion + * - StreamingProcessor:onFinishStreaming, for streaming text completion + * - sendOpenAIRequest, for non-streaming chat completion * - * **Ensure the active message has been updated and rendered before calling - * this function or the logprobs data will be saved to the wrong message.** * @param {TokenLogprobs[]} logprobs - array of logprobs data for each token * @param {string | null} continueFrom - for 'continue' generations, the prompt */ @@ -445,7 +454,10 @@ export function saveLogprobsForActiveMessage(logprobs, continueFrom) { return; } - convertTokenIdLogprobsToText(logprobs); + // NovelAI only returns token IDs in logprobs data; convert to text tokens in-place + if (getGeneratingApi() === 'novel') { + convertTokenIdLogprobsToText(logprobs); + } const msgId = chat.length - 1; /** @type {MessageLogprobData} */ @@ -491,17 +503,18 @@ function getActiveMessageLogprobData() { return state.messageLogprobs.get(hash) || null; } + /** - * convertLogprobTokenIdsToText mutates the given logprobs data's topLogprobs - * field keyed by token text instead of token ID. This is only necessary for - * APIs which only return token IDs in their logprobs data; for others this - * function is a no-op. + * convertLogprobTokenIdsToText replaces token IDs in logprobs data with text tokens, + * for APIs that return token IDs instead of text tokens, to wit: NovelAI. + * * @param {TokenLogprobs[]} input - logprobs data with numeric token IDs */ function convertTokenIdLogprobsToText(input) { const api = getGeneratingApi(); if (api !== 'novel') { - return input; + // should have been checked by the caller + throw new Error('convertTokenIdLogprobsToText should only be called for NovelAI'); } const tokenizerId = getTokenizerBestMatch(api); @@ -512,6 +525,7 @@ function convertTokenIdLogprobsToText(input) { ))); // Submit token IDs to tokenizer to get token text, then build ID->text map + // noinspection JSCheckFunctionSignatures - mutates input in-place const { chunks } = decodeTextTokens(tokenizerId, tokenIds); const tokenIdText = new Map(tokenIds.map((id, i) => [id, chunks[i]])); @@ -526,8 +540,8 @@ function convertTokenIdLogprobsToText(input) { export function initLogprobs() { const debouncedRender = debounce(renderAlternativeTokensView); - $('#logprobsViewerClose').click(onToggleLogprobsPanel); - $('#option_toggle_logprobs').click(onToggleLogprobsPanel); + $('#logprobsViewerClose').on('click', onToggleLogprobsPanel); + $('#option_toggle_logprobs').on('click', onToggleLogprobsPanel); eventSource.on(event_types.CHAT_CHANGED, debouncedRender); eventSource.on(event_types.CHARACTER_MESSAGE_RENDERED, debouncedRender); eventSource.on(event_types.IMPERSONATE_READY, debouncedRender); From 65469a477411f45e299077c99de7ae7b682cd3a2 Mon Sep 17 00:00:00 2001 From: poipoi300 Date: Sat, 16 Nov 2024 15:56:11 -0500 Subject: [PATCH 05/26] Fixed typo in HTML title --- public/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index 42f8bc686..4b444d56f 100644 --- a/public/index.html +++ b/public/index.html @@ -4503,7 +4503,7 @@
-
+
AutoComplete Settings
From de4246f7b79ee74665f85ca9e626553b18b88b58 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 17 Nov 2024 18:45:53 +0200 Subject: [PATCH 06/26] Move reroll button --- public/css/logprobs.css | 6 ------ public/index.html | 5 ++++- public/scripts/logprobs.js | 11 ++++------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/public/css/logprobs.css b/public/css/logprobs.css index f9b5e6b42..71b319539 100644 --- a/public/css/logprobs.css +++ b/public/css/logprobs.css @@ -80,12 +80,6 @@ background-color: rgba(255, 0, 50, 0.4); } -#logprobsReroll { - float: right; /* Position the button to the right */ - margin: 5px 0 5px 10px; /* Add spacing (top, right, bottom, left) */ - clear: right; /* Ensure it starts on a new line */ -} - .logprobs_candidate_list { grid-row-start: 3; grid-row-end: 4; diff --git a/public/index.html b/public/index.html index 7c130227b..d7483b3e0 100644 --- a/public/index.html +++ b/public/index.html @@ -6631,8 +6631,11 @@
- + Select a token to see alternatives considered by the AI. +
diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 49007a9bf..e8bafa56c 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -17,6 +17,7 @@ import { t } from './i18n.js'; const TINTS = 4; const MAX_MESSAGE_LOGPROBS = 100; +const REROLL_BUTTON = $('#logprobsReroll'); /** * Tuple of a candidate token and its logarithm of probability of being chosen @@ -93,14 +94,10 @@ function renderAlternativeTokensView() { const prefix = continueFrom || ''; const tokenSpans = []; + REROLL_BUTTON.toggle(!!prefix); if (prefix) { - const rerollButton = $(''); - rerollButton.attr('title', t`Reroll with the entire prefix`); - rerollButton.on('click', () => onPrefixClicked(prefix.length)); - tokenSpans.push(rerollButton); + REROLL_BUTTON.off('click').on('click', () => onPrefixClicked(prefix.length)); let cumulativeOffset = 0; const words = prefix.split(/\s+/); @@ -127,7 +124,6 @@ function renderAlternativeTokensView() { }); } - messageLogprobs.forEach((tokenData, i) => { const { token } = tokenData; const span = $(''); @@ -539,6 +535,7 @@ function convertTokenIdLogprobsToText(input) { } export function initLogprobs() { + REROLL_BUTTON.hide(); const debouncedRender = debounce(renderAlternativeTokensView); $('#logprobsViewerClose').on('click', onToggleLogprobsPanel); $('#option_toggle_logprobs').on('click', onToggleLogprobsPanel); From 14d5c669b1252cd0833b4c914d204454db01cc49 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 17 Nov 2024 19:44:08 +0200 Subject: [PATCH 07/26] Fix trimming incomplete sentences in logprobs reroll --- public/scripts/logprobs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index e82294782..db9aa7536 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -364,7 +364,7 @@ function onToggleLogprobsPanel() { function createSwipe(messageId, prompt) { // need to call `cleanUpMessage` on our new prompt, because we were working // with raw model output and our new prompt is missing trimming/macro replacements - const cleanedPrompt = cleanUpMessage(prompt, false, false); + const cleanedPrompt = cleanUpMessage(prompt, false, false, true); const msg = chat[messageId]; const newSwipeInfo = { From b143b3c8208c83264e70fc2337fd0cce6378ef76 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 17 Nov 2024 20:10:51 +0200 Subject: [PATCH 08/26] Fix type errors --- public/scripts/logprobs.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 9bde4f660..fa7ff0d15 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -24,6 +24,10 @@ const REROLL_BUTTON = $('#logprobsReroll'); * @typedef {[string, number]} Candidate - (token, logprob) */ +/** + * @typedef {(Node|JQuery|JQuery)[]} NodeArray - Array of DOM nodes + */ + /** * Logprob data for a single message * @typedef {Object} MessageLogprobData @@ -200,15 +204,15 @@ function renderTopLogprobs() { container.addClass('selected'); } - const tokenText = $('').text(`${toVisibleWhitespace(token)}`); - const percentText = $('').text(`${(probability * 100).toFixed(2)}%`); + const tokenText = $('').text(`${toVisibleWhitespace(token.toString())}`); + const percentText = $('').text(`${(+probability * 100).toFixed(2)}%`); container.append(tokenText, percentText); if (log) { container.attr('title', `logarithm: ${log}`); } addKeyboardProps(container); if (token !== '') { - container.on('click', () => onAlternativeClicked(state.selectedTokenLogprobs, token)); + container.on('click', () => onAlternativeClicked(state.selectedTokenLogprobs, token.toString())); } else { container.prop('disabled', true); } @@ -228,7 +232,7 @@ function renderTopLogprobs() { * User clicks on a token in the token output view. It updates the selected token state * and re-renders the top logprobs view, or deselects the token if it was already selected. * @param {TokenLogprobs} logprobs - logprob data for the selected token - * @param {Element} span - target span node that was clicked + * @param {Node|JQuery} span - target span node that was clicked */ function onSelectedTokenChanged(logprobs, span) { $('.logprobs_output_token.selected').removeClass('selected'); @@ -399,11 +403,11 @@ function toVisibleWhitespace(input) { * after the span node if its token begins or ends with whitespace in order to * allow text to wrap despite whitespace characters being replaced with a dot. * @param {string} text - token text being evaluated for whitespace - * @param {Element} span - target span node to be wrapped - * @returns {Node[]} - array of nodes to be appended to the parent element + * @param {Node|JQuery} span - target span node to be wrapped + * @returns {NodeArray} - array of nodes to be appended to the parent element */ function withVirtualWhitespace(text, span) { - /** @type {Node[]} */ + /** @type {NodeArray} */ const result = [span]; if (text.match(/^\s/)) { result.unshift(document.createTextNode('\u200b')); @@ -522,7 +526,7 @@ function convertTokenIdLogprobsToText(input) { // Submit token IDs to tokenizer to get token text, then build ID->text map // noinspection JSCheckFunctionSignatures - mutates input in-place - const { chunks } = decodeTextTokens(tokenizerId, tokenIds); + const { chunks } = decodeTextTokens(tokenizerId, tokenIds.map(parseInt)); const tokenIdText = new Map(tokenIds.map((id, i) => [id, chunks[i]])); // Fixup logprobs data with token text From b1b6f0473a2c20eea0fdab39620875c517485a18 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 00:19:50 -0700 Subject: [PATCH 09/26] Enhancement: Make buttons scrollable --- public/scripts/slash-commands.js | 37 ++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index a34b4e544..6a4c5f63f 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2074,7 +2074,32 @@ async function buttonsCallback(args, text) { let popup; const buttonContainer = document.createElement('div'); - buttonContainer.classList.add('flex-container', 'flexFlowColumn', 'wide100p', 'm-t-1'); + buttonContainer.classList.add('flex-container', 'flexFlowColumn', 'wide100p'); + + const scrollableContainer = document.createElement('div'); + scrollableContainer.style.maxHeight = '50vh'; // Use viewport height instead of fixed pixels + scrollableContainer.style.overflowY = 'auto'; + scrollableContainer.style.WebkitOverflowScrolling = 'touch'; // Enable momentum scrolling on iOS + scrollableContainer.classList.add('m-t-1', 'scrollable-buttons'); + + // Add custom CSS for better mobile scrolling + const style = document.createElement('style'); + style.textContent = ` + .scrollable-buttons { + -webkit-overflow-scrolling: touch; + overflow-y: auto; + flex-shrink: 1; + min-height: 0; + } + .scrollable-buttons::-webkit-scrollbar { + width: 6px; + } + .scrollable-buttons::-webkit-scrollbar-thumb { + background-color: rgba(0, 0, 0, 0.2); + border-radius: 3px; + } + `; + document.head.appendChild(style); for (const [result, button] of resultToButtonMap) { const buttonElement = document.createElement('div'); @@ -2087,9 +2112,16 @@ async function buttonsCallback(args, text) { buttonContainer.appendChild(buttonElement); } + scrollableContainer.appendChild(buttonContainer); + const popupContainer = document.createElement('div'); popupContainer.innerHTML = safeValue; - popupContainer.appendChild(buttonContainer); + popupContainer.appendChild(scrollableContainer); + + // Ensure the popup uses flex layout + popupContainer.style.display = 'flex'; + popupContainer.style.flexDirection = 'column'; + popupContainer.style.maxHeight = '80vh'; // Limit the overall height of the popup popup = new Popup(popupContainer, POPUP_TYPE.TEXT, '', { okButton: 'Cancel' }); popup.show() @@ -4272,3 +4304,4 @@ sendTextarea.addEventListener('input', () => { sendTextarea.style.fontFamily = null; } }); + From 8722c6a62b295ba9f4257825e2b83e5caad234d3 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 00:57:15 -0700 Subject: [PATCH 10/26] Add scroll bar to make it obvious --- public/scripts/slash-commands.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 6a4c5f63f..08ea48536 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2090,12 +2090,14 @@ async function buttonsCallback(args, text) { overflow-y: auto; flex-shrink: 1; min-height: 0; + scrollbar-width: thin; + scrollbar-color: rgba(255, 255, 255, 0.3) transparent; } .scrollable-buttons::-webkit-scrollbar { width: 6px; } .scrollable-buttons::-webkit-scrollbar-thumb { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgba(255, 255, 255, 0.3); border-radius: 3px; } `; @@ -4304,4 +4306,3 @@ sendTextarea.addEventListener('input', () => { sendTextarea.style.fontFamily = null; } }); - From 9533b7e59a9f626f1ca9ad48238ffc634ae99ea3 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 17 Nov 2024 15:54:09 -0800 Subject: [PATCH 11/26] review: Move css out of js code --- public/css/scrollable-button.css | 19 +++++++++++++++++++ public/scripts/slash-commands.js | 26 +------------------------- public/style.css | 1 + 3 files changed, 21 insertions(+), 25 deletions(-) create mode 100644 public/css/scrollable-button.css diff --git a/public/css/scrollable-button.css b/public/css/scrollable-button.css new file mode 100644 index 000000000..c77032a1e --- /dev/null +++ b/public/css/scrollable-button.css @@ -0,0 +1,19 @@ +.scrollable-buttons-container { + max-height: 50vh; /* Use viewport height instead of fixed pixels */ + overflow-y: auto; + -webkit-overflow-scrolling: touch; /* Momentum scrolling on iOS */ + margin-top: 1rem; /* m-t-1 is equivalent to margin-top: 1rem; */ + flex-shrink: 1; + min-height: 0; + scrollbar-width: thin; + scrollbar-color: rgba(255, 255, 255, 0.3) transparent; +} + +.scrollable-buttons-container::-webkit-scrollbar { + width: 6px; +} + +.scrollable-buttons-container::-webkit-scrollbar-thumb { + background-color: rgba(255, 255, 255, 0.3); + border-radius: 3px; +} diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 91bd0d8ed..e10c89ae6 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2084,31 +2084,7 @@ async function buttonsCallback(args, text) { buttonContainer.classList.add('flex-container', 'flexFlowColumn', 'wide100p'); const scrollableContainer = document.createElement('div'); - scrollableContainer.style.maxHeight = '50vh'; // Use viewport height instead of fixed pixels - scrollableContainer.style.overflowY = 'auto'; - scrollableContainer.style.WebkitOverflowScrolling = 'touch'; // Enable momentum scrolling on iOS - scrollableContainer.classList.add('m-t-1', 'scrollable-buttons'); - - // Add custom CSS for better mobile scrolling - const style = document.createElement('style'); - style.textContent = ` - .scrollable-buttons { - -webkit-overflow-scrolling: touch; - overflow-y: auto; - flex-shrink: 1; - min-height: 0; - scrollbar-width: thin; - scrollbar-color: rgba(255, 255, 255, 0.3) transparent; - } - .scrollable-buttons::-webkit-scrollbar { - width: 6px; - } - .scrollable-buttons::-webkit-scrollbar-thumb { - background-color: rgba(255, 255, 255, 0.3); - border-radius: 3px; - } - `; - document.head.appendChild(style); + scrollableContainer.classList.add('scrollable-buttons-container'); for (const [result, button] of resultToButtonMap) { const buttonElement = document.createElement('div'); diff --git a/public/style.css b/public/style.css index 217c22bd9..3d4a1bd26 100644 --- a/public/style.css +++ b/public/style.css @@ -9,6 +9,7 @@ @import url(css/logprobs.css); @import url(css/accounts.css); @import url(css/tags.css); +@import url(css/scrollable-button.css); :root { --doc-height: 100%; From 6e29ad4b50fb1b99e0f3dbeab1fe68f3fa58f23a Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Oct 2024 03:51:36 -0700 Subject: [PATCH 12/26] Fix: Inconsistent Textarea resizing in small windows --- public/scripts/RossAscends-mods.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 32e4fdcb4..d1fe5f3ec 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -699,20 +699,26 @@ const chatBlock = document.getElementById('chat'); const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; /** - * this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height) + * Auto-resizes the send message textarea to fit its content, up to a maximum height defined by CSS. + * This function preserves chat scroll position, resets the textarea height for accurate measurement, + * calculates the required height, sets the new height, and then restores the chat scroll position. + * Firefox-specific scrolling adjustments are included for smoother behavior. */ function autoFitSendTextArea() { - const originalScrollBottom = chatBlock.scrollHeight - (chatBlock.scrollTop + chatBlock.offsetHeight); - if (Math.ceil(sendTextArea.scrollHeight + 3) >= Math.floor(sendTextArea.offsetHeight)) { - const sendTextAreaMinHeight = '0px'; - sendTextArea.style.height = sendTextAreaMinHeight; - } - const newHeight = sendTextArea.scrollHeight + 3; + const originalScrollTop = chatBlock.scrollTop; + const originalScrollHeight = chatBlock.scrollHeight; + + sendTextArea.style.height = '1px'; + + const newHeight = sendTextArea.scrollHeight; + sendTextArea.style.height = `${newHeight}px`; + // Restore chat scroll position (Firefox-specific adjustment for smoothness). if (!isFirefox) { - const newScrollTop = Math.round(chatBlock.scrollHeight - (chatBlock.offsetHeight + originalScrollBottom)); - chatBlock.scrollTop = newScrollTop; + chatBlock.scrollTop = originalScrollTop + (chatBlock.scrollHeight - originalScrollHeight); + } else { + chatBlock.scrollTo({ top: chatBlock.scrollHeight, behavior: 'auto' }); } } export const autoFitSendTextAreaDebounced = debounce(autoFitSendTextArea, debounce_timeout.short); From 328e3622f2340f9516723b0df9d7b23e545eb975 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 17 Nov 2024 18:38:38 -0800 Subject: [PATCH 13/26] review: Fix chrome inconsistent scroll and firefox scroll to bottom --- public/scripts/RossAscends-mods.js | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index d1fe5f3ec..391f34ce9 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -699,26 +699,18 @@ const chatBlock = document.getElementById('chat'); const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; /** - * Auto-resizes the send message textarea to fit its content, up to a maximum height defined by CSS. - * This function preserves chat scroll position, resets the textarea height for accurate measurement, - * calculates the required height, sets the new height, and then restores the chat scroll position. - * Firefox-specific scrolling adjustments are included for smoother behavior. + * Resizes the chat input textarea vertically to match its text content, up to a maximum height defined in CSS. + * Preserves scroll position in Chrome. In Firefox, the textarea grows to cover the chat history. */ function autoFitSendTextArea() { - const originalScrollTop = chatBlock.scrollTop; - const originalScrollHeight = chatBlock.scrollHeight; - - sendTextArea.style.height = '1px'; - - const newHeight = sendTextArea.scrollHeight; + const originalScrollBottom = chatBlock.scrollHeight - (chatBlock.scrollTop + chatBlock.offsetHeight); + sendTextArea.style.height = '1px'; // Reset height to 1px to force recalculation of scrollHeight + const newHeight = sendTextArea.scrollHeight; sendTextArea.style.height = `${newHeight}px`; - // Restore chat scroll position (Firefox-specific adjustment for smoothness). if (!isFirefox) { - chatBlock.scrollTop = originalScrollTop + (chatBlock.scrollHeight - originalScrollHeight); - } else { - chatBlock.scrollTo({ top: chatBlock.scrollHeight, behavior: 'auto' }); + chatBlock.scrollTop = chatBlock.scrollHeight - (chatBlock.offsetHeight + originalScrollBottom); } } export const autoFitSendTextAreaDebounced = debounce(autoFitSendTextArea, debounce_timeout.short); From 17cce528f8f24e2de321f34bc0bca8ec5c977e1c Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 17 Nov 2024 18:44:23 -0800 Subject: [PATCH 14/26] Undo docstring changes --- public/scripts/RossAscends-mods.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 391f34ce9..fb55e42dd 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -699,8 +699,7 @@ const chatBlock = document.getElementById('chat'); const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; /** - * Resizes the chat input textarea vertically to match its text content, up to a maximum height defined in CSS. - * Preserves scroll position in Chrome. In Firefox, the textarea grows to cover the chat history. + * this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height) */ function autoFitSendTextArea() { const originalScrollBottom = chatBlock.scrollHeight - (chatBlock.scrollTop + chatBlock.offsetHeight); From 5992117904ed9be5af20c2e8582a979a37235cdd Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 17:55:30 +1100 Subject: [PATCH 15/26] Add GGUF models and denoise parameter for ComfyUI --- .../stable-diffusion/comfyWorkflowEditor.html | 1 + .../extensions/stable-diffusion/index.js | 4 ++ .../extensions/stable-diffusion/settings.html | 2 +- src/endpoints/stable-diffusion.js | 42 ++++++++++++------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html index 2427fa6fb..5ac02209b 100644 --- a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html +++ b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html @@ -17,6 +17,7 @@
  • "%scheduler%"
  • "%steps%"
  • "%scale%"
    • +
  • "%denoise%"
  • "%clip_skip%"
  • "%width%"
  • "%height%"
    • diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 8bffdaafe..a176f2e6e 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -3269,6 +3269,10 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const seed = extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : Math.round(Math.random() * Number.MAX_SAFE_INTEGER); workflow = workflow.replaceAll('"%seed%"', JSON.stringify(seed)); + + const denoising_strength = extension_settings.sd.denoising_strength === undefined ? 1.0 : extension_settings.sd.denoising_strength; + workflow = workflow.replaceAll('"%denoise%"', JSON.stringify(denoising_strength)); + placeholders.forEach(ph => { workflow = workflow.replaceAll(`"%${ph}%"`, JSON.stringify(extension_settings.sd[ph])); }); diff --git a/public/scripts/extensions/stable-diffusion/settings.html b/public/scripts/extensions/stable-diffusion/settings.html index d07504dca..e3ee1eea9 100644 --- a/public/scripts/extensions/stable-diffusion/settings.html +++ b/public/scripts/extensions/stable-diffusion/settings.html @@ -319,7 +319,7 @@
    -
    +
    Denoising strength diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 115b540f2..3679d9ea1 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -7,7 +7,7 @@ import sanitize from 'sanitize-filename'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; import FormData from 'form-data'; -import { getBasicAuthHeader, delay } from '../util.js'; +import { delay, getBasicAuthHeader } from '../util.js'; import { jsonParser } from '../express-common.js'; import { readSecret, SECRET_KEYS } from './secrets.js'; @@ -19,7 +19,7 @@ import { readSecret, SECRET_KEYS } from './secrets.js'; function getComfyWorkflows(directories) { return fs .readdirSync(directories.comfyWorkflows) - .filter(file => file[0] != '.' && file.toLowerCase().endsWith('.json')) + .filter(file => file[0] !== '.' && file.toLowerCase().endsWith('.json')) .sort(Intl.Collator().compare); } @@ -67,8 +67,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { /** @type {any} */ const data = await result.json(); - const names = data.map(x => x.name); - return names; + return data.map(x => x.name); } async function getLatentUpscalers() { @@ -88,8 +87,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { /** @type {any} */ const data = await result.json(); - const names = data.map(x => x.name); - return names; + return data.map(x => x.name); } const [upscalers, latentUpscalers] = await Promise.all([getUpscalerModels(), getLatentUpscalers()]); @@ -241,8 +239,7 @@ router.post('/set-model', jsonParser, async (request, response) => { 'Authorization': getBasicAuthHeader(request.body.auth), }, }); - const data = await result.json(); - return data; + return await result.json(); } const url = new URL(request.body.url); @@ -274,7 +271,7 @@ router.post('/set-model', jsonParser, async (request, response) => { const progress = progressState['progress']; const jobCount = progressState['state']['job_count']; - if (progress == 0.0 && jobCount === 0) { + if (progress === 0.0 && jobCount === 0) { break; } @@ -412,8 +409,18 @@ comfy.post('/models', jsonParser, async (request, response) => { } /** @type {any} */ const data = await result.json(); - return response.send(data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it }))); - } catch (error) { + + const ckpts = data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })) || []; + + // load list of GGUF unets from diffusion_models if the loader node is available + const ggufs = data.UnetLoaderGGUF?.input.required.unet_name[0].map(it => ({ value: it, text: `GGUF: ${it}` })) || []; + const models = ckpts.concat(ggufs); + + // make the display names of the models somewhat presentable + models.forEach(it => it.text = it.text.replace(/\.[^.]*$/, '').replace(/_/g, ' ')); + + return response.send(models); + } catch (error) { console.log(error); return response.sendStatus(500); } @@ -550,7 +557,13 @@ comfy.post('/generate', jsonParser, async (request, response) => { await delay(100); } if (item.status.status_str === 'error') { - throw new Error('ComfyUI generation did not succeed.'); + // Report node tracebacks if available + const errorMessages = item.status?.messages + ?.filter(it => it[0] === 'execution_error') + .map(it => it[1]) + .map(it => `${it.node_type} [${it.node_id}] ${it.exception_type}: ${it.exception_message}`) + .join('\n') || ''; + throw new Error(`ComfyUI generation did not succeed.\n\n${errorMessages}`.trim()); } const imgInfo = Object.keys(item.outputs).map(it => item.outputs[it].images).flat()[0]; const imgUrl = new URL(request.body.url); @@ -563,8 +576,9 @@ comfy.post('/generate', jsonParser, async (request, response) => { const imgBuffer = await imgResponse.buffer(); return response.send(imgBuffer.toString('base64')); } catch (error) { - console.log(error); - return response.sendStatus(500); + console.log('ComfyUI error:', error); + response.status(500).send(`${error.message}`); + return response; } }); From 8927de45dd0bd437c9ff4824d6fff78e4bc31334 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 18:30:31 +1100 Subject: [PATCH 16/26] Convert data URL to plain base64-encoded image data --- public/scripts/extensions/stable-diffusion/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index a176f2e6e..2283d219f 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -3283,7 +3283,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const response = await fetch(getUserAvatarUrl()); if (response.ok) { const avatarBlob = await response.blob(); - const avatarBase64 = await getBase64Async(avatarBlob); + const avatarBase64DataUrl = await getBase64Async(avatarBlob); + const avatarBase64 = avatarBase64DataUrl.split(',')[1]; workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(avatarBase64)); } else { workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(PNG_PIXEL)); @@ -3293,7 +3294,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const response = await fetch(getCharacterAvatarUrl()); if (response.ok) { const avatarBlob = await response.blob(); - const avatarBase64 = await getBase64Async(avatarBlob); + const avatarBase64DataUrl = await getBase64Async(avatarBlob); + const avatarBase64 = avatarBase64DataUrl.split(',')[1]; workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(avatarBase64)); } else { workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(PNG_PIXEL)); From b46dae0d6e54b44b4bcea55db51de2f4e0836187 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 18:49:38 +1100 Subject: [PATCH 17/26] Add sample workflow for ComfyUI img2img --- .../content/Char_Avatar_Comfy_Workflow.json | 137 ++++++++++++++++++ default/content/index.json | 4 + 2 files changed, 141 insertions(+) create mode 100644 default/content/Char_Avatar_Comfy_Workflow.json diff --git a/default/content/Char_Avatar_Comfy_Workflow.json b/default/content/Char_Avatar_Comfy_Workflow.json new file mode 100644 index 000000000..c40bd8fb8 --- /dev/null +++ b/default/content/Char_Avatar_Comfy_Workflow.json @@ -0,0 +1,137 @@ +{ + "3": { + "inputs": { + "seed": "%seed%", + "steps": "%steps%", + "cfg": "%scale%", + "sampler_name": "%sampler%", + "scheduler": "%scheduler%", + "denoise": "%denoise%", + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "12", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "%model%" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "6": { + "inputs": { + "text": "%prompt%", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "%negative_prompt%", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Negative Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "SillyTavern", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "10": { + "inputs": { + "image": "%char_avatar%" + }, + "class_type": "ETN_LoadImageBase64", + "_meta": { + "title": "Load Image (Base64) [https://github.com/Acly/comfyui-tooling-nodes]" + } + }, + "12": { + "inputs": { + "pixels": [ + "13", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "13": { + "inputs": { + "upscale_method": "bicubic", + "width": "%width%", + "height": "%height%", + "crop": "center", + "image": [ + "10", + 0 + ] + }, + "class_type": "ImageScale", + "_meta": { + "title": "Upscale Image" + } + } +} diff --git a/default/content/index.json b/default/content/index.json index 8c2ca66be..24e73e5c9 100644 --- a/default/content/index.json +++ b/default/content/index.json @@ -135,6 +135,10 @@ "filename": "Default_Comfy_Workflow.json", "type": "workflow" }, + { + "filename": "Char_Avatar_Comfy_Workflow.json", + "type": "workflow" + }, { "filename": "presets/kobold/Ace of Spades.json", "type": "kobold_preset" From 44a8283e7ea356469a5bd68f4cc13e73139cc5bf Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:42:37 +0000 Subject: [PATCH 18/26] Fix npm audit --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2e684c8ec..e3d0eade1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3019,9 +3019,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz", + "integrity": "sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==", "license": "MIT", "dependencies": { "path-key": "^3.1.0", From 26aad5fa80966e3f8846573127758ff19ed627ec Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:55:25 +0000 Subject: [PATCH 19/26] Fix reroll button at home --- public/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/index.html b/public/index.html index 9bf1dc617..9c88da12b 100644 --- a/public/index.html +++ b/public/index.html @@ -6631,9 +6631,9 @@
    - + Select a token to see alternatives considered by the AI. - From bcf127e38717f93569677034bc69f0377c3fa9ee Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 22:17:19 +1100 Subject: [PATCH 20/26] Add diffusion unets to ComfyUI models list rm string interpolation --- src/endpoints/stable-diffusion.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 3679d9ea1..567dadf13 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -411,10 +411,11 @@ comfy.post('/models', jsonParser, async (request, response) => { const data = await result.json(); const ckpts = data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })) || []; + const unets = data.UNETLoader.input.required.unet_name[0].map(it => ({ value: it, text: `UNet: ${it}` })) || []; // load list of GGUF unets from diffusion_models if the loader node is available const ggufs = data.UnetLoaderGGUF?.input.required.unet_name[0].map(it => ({ value: it, text: `GGUF: ${it}` })) || []; - const models = ckpts.concat(ggufs); + const models = [...ckpts, ...unets, ...ggufs]; // make the display names of the models somewhat presentable models.forEach(it => it.text = it.text.replace(/\.[^.]*$/, '').replace(/_/g, ' ')); @@ -577,7 +578,7 @@ comfy.post('/generate', jsonParser, async (request, response) => { return response.send(imgBuffer.toString('base64')); } catch (error) { console.log('ComfyUI error:', error); - response.status(500).send(`${error.message}`); + response.status(500).send(error.message); return response; } }); From aaed09f6063e47543da807fcc20ea8d256f1d4b5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:18:45 +0000 Subject: [PATCH 21/26] Data Bank: display file ingestion progress --- public/scripts/extensions/vectors/index.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 89a7f8c6d..f26184edb 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -44,6 +44,9 @@ const MODULE_NAME = 'vectors'; export const EXTENSION_PROMPT_TAG = '3_vectors'; export const EXTENSION_PROMPT_TAG_DB = '4_vectors_data_bank'; +// Force solo chunks for sources that don't support batching. +const getBatchSize = () => ['transformers', 'palm', 'ollama'].includes(settings.source) ? 1 : 5; + const settings = { // For both source: 'transformers', @@ -125,7 +128,7 @@ async function onVectorizeAllClick() { // upon request of a full vectorise cachedSummaries.clear(); - const batchSize = 5; + const batchSize = getBatchSize(); const elapsedLog = []; let finished = false; $('#vectorize_progress').show(); @@ -560,7 +563,9 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overla fileText = translatedText; } - const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`); + const batchSize = getBatchSize(); + const toastBody = $('').text('This may take a while. Please wait...'); + const toast = toastr.info(toastBody, `Ingesting file ${fileName}`, { closeButton: false, escapeHtml: false, timeOut: 0, extendedTimeOut: 0 }); const overlapSize = Math.round(chunkSize * overlapPercent / 100); const delimiters = getChunkDelimiters(); // Overlap should not be included in chunk size. It will be later compensated by overlapChunks @@ -569,7 +574,12 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overla console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks); const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index })); - await insertVectorItems(collectionId, items); + + for (let i = 0; i < items.length; i += batchSize) { + toastBody.text(`${i}/${items.length} (${Math.round((i / items.length) * 100)}%) chunks processed`); + const chunkedBatch = items.slice(i, i + batchSize); + await insertVectorItems(collectionId, chunkedBatch); + } toastr.clear(toast); console.log(`Vectors: Inserted ${chunks.length} vector items for file ${fileName} into ${collectionId}`); From 378dfd3626330dbb96a38f70e275ab3c41501e30 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:24:26 +0000 Subject: [PATCH 22/26] Vectors: escape file names --- public/scripts/extensions/vectors/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index f26184edb..28689dccb 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -23,7 +23,7 @@ import { import { collapseNewlines, registerDebugFunction } from '../../power-user.js'; import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js'; import { getDataBankAttachments, getDataBankAttachmentsForSource, getFileAttachment } from '../../chats.js'; -import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive, trimToStartSentence, trimToEndSentence } from '../../utils.js'; +import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive, trimToStartSentence, trimToEndSentence, escapeHtml } from '../../utils.js'; import { debounce_timeout } from '../../constants.js'; import { getSortedEntries } from '../../world-info.js'; import { textgen_types, textgenerationwebui_settings } from '../../textgen-settings.js'; @@ -565,7 +565,7 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overla const batchSize = getBatchSize(); const toastBody = $('').text('This may take a while. Please wait...'); - const toast = toastr.info(toastBody, `Ingesting file ${fileName}`, { closeButton: false, escapeHtml: false, timeOut: 0, extendedTimeOut: 0 }); + const toast = toastr.info(toastBody, `Ingesting file ${escapeHtml(fileName)}`, { closeButton: false, escapeHtml: false, timeOut: 0, extendedTimeOut: 0 }); const overlapSize = Math.round(chunkSize * overlapPercent / 100); const delimiters = getChunkDelimiters(); // Overlap should not be included in chunk size. It will be later compensated by overlapChunks @@ -1060,7 +1060,7 @@ async function onViewStatsClick() { toastr.info(`Total hashes: ${totalHashes}
    Unique hashes: ${uniqueHashes}

    I'll mark collected messages with a green circle.`, - `Stats for chat ${chatId}`, + `Stats for chat ${escapeHtml(chatId)}`, { timeOut: 10000, escapeHtml: false }, ); From b651d31780d72afae0ca8e29a4180366df2b6e9c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:10:20 +0000 Subject: [PATCH 23/26] MistralAI: new pixtral large model --- public/index.html | 3 +++ public/scripts/extensions/caption/settings.html | 2 ++ public/scripts/openai.js | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index 9c88da12b..fd19d2878 100644 --- a/public/index.html +++ b/public/index.html @@ -3026,6 +3026,7 @@ + @@ -3040,10 +3041,12 @@ + + diff --git a/public/scripts/extensions/caption/settings.html b/public/scripts/extensions/caption/settings.html index e9363eebe..64ef9aee9 100644 --- a/public/scripts/extensions/caption/settings.html +++ b/public/scripts/extensions/caption/settings.html @@ -37,6 +37,8 @@