From f1bc217e79f7854c094090cbb378cd633d9cf218 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 01:14:50 +0200 Subject: [PATCH 1/9] Expressions: Add WebLLM extension classification --- .../scripts/extensions/expressions/index.js | 70 ++++++++++++++----- .../extensions/expressions/settings.html | 3 +- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index fdc146c1a..e31807893 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -15,6 +15,7 @@ import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashComm import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js'; import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js'; import { SlashCommandClosure } from '../../slash-commands/SlashCommandClosure.js'; +import { generateWebLlmChatPrompt, isWebLlmSupported } from '../shared.js'; export { MODULE_NAME }; const MODULE_NAME = 'expressions'; @@ -59,6 +60,7 @@ const EXPRESSION_API = { local: 0, extras: 1, llm: 2, + webllm: 3, }; let expressionsList = null; @@ -698,8 +700,8 @@ async function moduleWorker() { } // If using LLM api then check if streamingProcessor is finished to avoid sending multiple requests to the API - if (extension_settings.expressions.api === EXPRESSION_API.llm && context.streamingProcessor && !context.streamingProcessor.isFinished) { - return; + if (extension_settings.expressions.api === EXPRESSION_API.llm && context.streamingProcessor && !context.streamingProcessor.isFinished) { + return; } // API is busy @@ -852,7 +854,7 @@ function setTalkingHeadState(newState) { extension_settings.expressions.talkinghead = newState; // Store setting saveSettingsDebounced(); - if (extension_settings.expressions.api == EXPRESSION_API.local || extension_settings.expressions.api == EXPRESSION_API.llm) { + if ([EXPRESSION_API.local, EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api)) { return; } @@ -1057,11 +1059,39 @@ function parseLlmResponse(emotionResponse, labels) { console.debug(`fuzzy search found: ${result[0].item} as closest for the LLM response:`, emotionResponse); return result[0].item; } + const lowerCaseResponse = String(emotionResponse || '').toLowerCase(); + for (const label of labels) { + if (lowerCaseResponse.includes(label.toLowerCase())) { + console.debug(`Found label ${label} in the LLM response:`, emotionResponse); + return label; + } + } } throw new Error('Could not parse emotion response ' + emotionResponse); } +/** + * Gets the JSON schema for the LLM API. + * @param {string[]} emotions A list of emotions to search for. + * @returns {object} The JSON schema for the LLM API. + */ +function getJsonSchema(emotions) { + return { + $schema: 'http://json-schema.org/draft-04/schema#', + type: 'object', + properties: { + emotion: { + type: 'string', + enum: emotions, + }, + }, + required: [ + 'emotion', + ], + }; +} + function onTextGenSettingsReady(args) { // Only call if inside an API call if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) { @@ -1071,19 +1101,7 @@ function onTextGenSettingsReady(args) { stop: [], stopping_strings: [], custom_token_bans: [], - json_schema: { - $schema: 'http://json-schema.org/draft-04/schema#', - type: 'object', - properties: { - emotion: { - type: 'string', - enum: emotions, - }, - }, - required: [ - 'emotion', - ], - }, + json_schema: getJsonSchema(emotions), }); } } @@ -1139,6 +1157,22 @@ export async function getExpressionLabel(text, expressionsApi = extension_settin const emotionResponse = await generateRaw(text, main_api, false, false, prompt); return parseLlmResponse(emotionResponse, expressionsList); } + // Using WebLLM + case EXPRESSION_API.webllm: { + if (!isWebLlmSupported()) { + console.warn('WebLLM is not supported. Using fallback expression'); + return getFallbackExpression(); + } + + const expressionsList = await getExpressionsList(); + const prompt = substituteParamsExtended(customPrompt, { labels: expressionsList }) || await getLlmPrompt(expressionsList); + const messages = [ + { role: 'user', content: text + '\n\n' + prompt }, + ]; + + const emotionResponse = await generateWebLlmChatPrompt(messages); + return parseLlmResponse(emotionResponse, expressionsList); + } // Extras default: { const url = new URL(getApiUrl()); @@ -1603,7 +1637,7 @@ function onExpressionApiChanged() { const tempApi = this.value; if (tempApi) { extension_settings.expressions.api = Number(tempApi); - $('.expression_llm_prompt_block').toggle(extension_settings.expressions.api === EXPRESSION_API.llm); + $('.expression_llm_prompt_block').toggle([EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api)); expressionsList = null; spriteCache = {}; moduleWorker(); @@ -1940,7 +1974,7 @@ function migrateSettings() { await renderAdditionalExpressionSettings(); $('#expression_api').val(extension_settings.expressions.api ?? EXPRESSION_API.extras); - $('.expression_llm_prompt_block').toggle(extension_settings.expressions.api === EXPRESSION_API.llm); + $('.expression_llm_prompt_block').toggle([EXPRESSION_API.llm, EXPRESSION_API.webllm].includes(extension_settings.expressions.api)); $('#expression_llm_prompt').val(extension_settings.expressions.llmPrompt ?? ''); $('#expression_llm_prompt').on('input', function () { extension_settings.expressions.llmPrompt = $(this).val(); diff --git a/public/scripts/extensions/expressions/settings.html b/public/scripts/extensions/expressions/settings.html index f2b7b79ac..dc22debbd 100644 --- a/public/scripts/extensions/expressions/settings.html +++ b/public/scripts/extensions/expressions/settings.html @@ -24,7 +24,8 @@
From d27d750cb2888c6c8a7980f00cb08e4c55901041 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 02:50:42 +0200 Subject: [PATCH 2/9] Use CSS resizing for send textarea --- public/scripts/RossAscends-mods.js | 44 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 750e9cdd2..213f9f07e 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -887,14 +887,44 @@ export function initRossMods() { saveSettingsDebounced(); }); + const cssAutofit = CSS.supports('field-sizing', 'content'); + + if (cssAutofit) { + sendTextArea.style['fieldSizing'] = 'content'; + sendTextArea.style['height'] = 'auto'; + + let lastHeight = chatBlock.offsetHeight; + const chatBlockResizeObserver = new ResizeObserver((entries) => { + for (const entry of entries) { + if (entry.target !== chatBlock) { + continue; + } + + const threshold = 1; + const newHeight = chatBlock.offsetHeight; + const deltaHeight = newHeight - lastHeight; + const isScrollAtBottom = Math.abs(chatBlock.scrollHeight - chatBlock.scrollTop - newHeight) <= threshold; + + if (!isScrollAtBottom && Math.abs(deltaHeight) > threshold) { + chatBlock.scrollTop -= deltaHeight; + } + lastHeight = newHeight; + } + }); + + chatBlockResizeObserver.observe(chatBlock); + } + sendTextArea.addEventListener('input', () => { - const hasContent = sendTextArea.value !== ''; - const fitsCurrentSize = sendTextArea.scrollHeight <= sendTextArea.offsetHeight; - const isScrollbarShown = sendTextArea.clientWidth < sendTextArea.offsetWidth; - const isHalfScreenHeight = sendTextArea.offsetHeight >= window.innerHeight / 2; - const needsDebounce = hasContent && (fitsCurrentSize || (isScrollbarShown && isHalfScreenHeight)); - if (needsDebounce) autoFitSendTextAreaDebounced(); - else autoFitSendTextArea(); + if (!cssAutofit) { + const hasContent = sendTextArea.value !== ''; + const fitsCurrentSize = sendTextArea.scrollHeight <= sendTextArea.offsetHeight; + const isScrollbarShown = sendTextArea.clientWidth < sendTextArea.offsetWidth; + const isHalfScreenHeight = sendTextArea.offsetHeight >= window.innerHeight / 2; + const needsDebounce = hasContent && (fitsCurrentSize || (isScrollbarShown && isHalfScreenHeight)); + if (needsDebounce) autoFitSendTextAreaDebounced(); + else autoFitSendTextArea(); + } saveUserInputDebounced(); }); From f649e4698b749c62c6f3c413605fb77d874f2569 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 02:52:05 +0200 Subject: [PATCH 3/9] Invert if cssAutofit --- public/scripts/RossAscends-mods.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 213f9f07e..5334019d4 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -916,16 +916,19 @@ export function initRossMods() { } sendTextArea.addEventListener('input', () => { - if (!cssAutofit) { - const hasContent = sendTextArea.value !== ''; - const fitsCurrentSize = sendTextArea.scrollHeight <= sendTextArea.offsetHeight; - const isScrollbarShown = sendTextArea.clientWidth < sendTextArea.offsetWidth; - const isHalfScreenHeight = sendTextArea.offsetHeight >= window.innerHeight / 2; - const needsDebounce = hasContent && (fitsCurrentSize || (isScrollbarShown && isHalfScreenHeight)); - if (needsDebounce) autoFitSendTextAreaDebounced(); - else autoFitSendTextArea(); - } saveUserInputDebounced(); + + if (cssAutofit) { + return; + } + + const hasContent = sendTextArea.value !== ''; + const fitsCurrentSize = sendTextArea.scrollHeight <= sendTextArea.offsetHeight; + const isScrollbarShown = sendTextArea.clientWidth < sendTextArea.offsetWidth; + const isHalfScreenHeight = sendTextArea.offsetHeight >= window.innerHeight / 2; + const needsDebounce = hasContent && (fitsCurrentSize || (isScrollbarShown && isHalfScreenHeight)); + if (needsDebounce) autoFitSendTextAreaDebounced(); + else autoFitSendTextArea(); }); restoreUserInput(); From 222edd5c3696cc36500b6d68ce998990b46ad27f Mon Sep 17 00:00:00 2001 From: cloak1505 Date: Sat, 21 Dec 2024 01:30:35 -0600 Subject: [PATCH 4/9] OpenRouter: Update providers list --- public/scripts/textgen-models.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index 6323505f5..851c3940d 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -25,6 +25,7 @@ const OPENROUTER_PROVIDERS = [ 'Anthropic', 'Google', 'Google AI Studio', + 'Amazon Bedrock', 'Groq', 'SambaNova', 'Cohere', @@ -50,6 +51,8 @@ const OPENROUTER_PROVIDERS = [ 'Featherless', 'Inflection', 'xAI', + 'Cloudflare', + 'SF Compute', '01.AI', 'HuggingFace', 'Mancer', From 2fbe689605beddaa84cfc3b867c4e324901f365e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 16:47:25 +0200 Subject: [PATCH 5/9] Implement autofit for edit textarea --- public/script.js | 54 ++++++++++++++---------------- public/scripts/RossAscends-mods.js | 5 ++- public/style.css | 2 ++ 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/public/script.js b/public/script.js index 04eb8bc09..3fcfbc55c 100644 --- a/public/script.js +++ b/public/script.js @@ -849,11 +849,9 @@ export let is_send_press = false; //Send generation let this_del_mes = -1; -//message editing and chat scroll position persistence +//message editing var this_edit_mes_chname = ''; var this_edit_mes_id; -var scroll_holder = 0; -var is_use_scroll_holder = false; //settings export let settings; @@ -9727,25 +9725,7 @@ jQuery(async function () { chooseBogusFolder($(this), tagId); }); - /** - * Sets the scroll height of the edit textarea to fit the content. - * @param {HTMLTextAreaElement} e Textarea element to auto-fit - */ - function autoFitEditTextArea(e) { - scroll_holder = chatElement[0].scrollTop; - e.style.height = '0px'; - const newHeight = e.scrollHeight + 4; - e.style.height = `${newHeight}px`; - is_use_scroll_holder = true; - } - const autoFitEditTextAreaDebounced = debounce(autoFitEditTextArea, debounce_timeout.short); - document.addEventListener('input', e => { - if (e.target instanceof HTMLTextAreaElement && e.target.classList.contains('edit_textarea')) { - const scrollbarShown = e.target.clientWidth < e.target.offsetWidth && e.target.offsetHeight >= window.innerHeight * 0.75; - const immediately = (e.target.scrollHeight > e.target.offsetHeight && !scrollbarShown) || e.target.value === ''; - immediately ? autoFitEditTextArea(e.target) : autoFitEditTextAreaDebounced(e.target); - } - }); + const cssAutofit = CSS.supports('field-sizing', 'content'); const chatElementScroll = document.getElementById('chat'); const chatScrollHandler = function () { if (power_user.waifuMode) { @@ -9767,12 +9747,26 @@ jQuery(async function () { }; chatElementScroll.addEventListener('wheel', chatScrollHandler, { passive: true }); chatElementScroll.addEventListener('touchmove', chatScrollHandler, { passive: true }); - chatElementScroll.addEventListener('scroll', function () { - if (is_use_scroll_holder) { - this.scrollTop = scroll_holder; - is_use_scroll_holder = false; + + if (!cssAutofit) { + /** + * Sets the scroll height of the edit textarea to fit the content. + * @param {HTMLTextAreaElement} e Textarea element to auto-fit + */ + function autoFitEditTextArea(e) { + e.style.height = '0px'; + const newHeight = e.scrollHeight + 4; + e.style.height = `${newHeight}px`; } - }, { passive: true }); + const autoFitEditTextAreaDebounced = debounce(autoFitEditTextArea, debounce_timeout.short); + document.addEventListener('input', e => { + if (e.target instanceof HTMLTextAreaElement && e.target.classList.contains('edit_textarea')) { + const scrollbarShown = e.target.clientWidth < e.target.offsetWidth && e.target.offsetHeight >= window.innerHeight * 0.75; + const immediately = (e.target.scrollHeight > e.target.offsetHeight && !scrollbarShown) || e.target.value === ''; + immediately ? autoFitEditTextArea(e.target) : autoFitEditTextAreaDebounced(e.target); + } + }); + } $(document).on('click', '.mes', function () { //when a 'delete message' parent div is clicked @@ -10517,8 +10511,10 @@ jQuery(async function () { let edit_textarea = $(this) .closest('.mes_block') .find('.edit_textarea'); - edit_textarea.height(0); - edit_textarea.height(edit_textarea[0].scrollHeight); + if (!cssAutofit) { + edit_textarea.height(0); + edit_textarea.height(edit_textarea[0].scrollHeight); + } edit_textarea.focus(); edit_textarea[0].setSelectionRange( //this sets the cursor at the end of the text String(edit_textarea.val()).length, diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 5334019d4..acef7b5f6 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -890,9 +890,6 @@ export function initRossMods() { const cssAutofit = CSS.supports('field-sizing', 'content'); if (cssAutofit) { - sendTextArea.style['fieldSizing'] = 'content'; - sendTextArea.style['height'] = 'auto'; - let lastHeight = chatBlock.offsetHeight; const chatBlockResizeObserver = new ResizeObserver((entries) => { for (const entry of entries) { @@ -919,6 +916,8 @@ export function initRossMods() { saveUserInputDebounced(); if (cssAutofit) { + // Unset modifications made with a manual resize + sendTextArea.style.height = 'auto'; return; } diff --git a/public/style.css b/public/style.css index c9033bcad..d47c8aaf6 100644 --- a/public/style.css +++ b/public/style.css @@ -1264,6 +1264,7 @@ button { text-shadow: 0px 0px calc(var(--shadowWidth) * 1px) var(--SmartThemeShadowColor); flex: 1; order: 3; + field-sizing: content; --progColor: rgb(146, 190, 252); --progFlashColor: rgb(215, 136, 114); @@ -4111,6 +4112,7 @@ input[type="range"]::-webkit-slider-thumb { line-height: calc(var(--mainFontSize) + .25rem); max-height: 75vh; max-height: 75dvh; + field-sizing: content; } #anchor_order { From 85ce522270bd67b3d72de1c92e768bd30120e388 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:24:54 +0200 Subject: [PATCH 6/9] Remove invalid style --- public/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 3fcfbc55c..44e946693 100644 --- a/public/script.js +++ b/public/script.js @@ -10505,7 +10505,7 @@ jQuery(async function () { .closest('.mes_block') .find('.mes_text') .append( - '', + '', ); $('#curEditTextarea').val(text); let edit_textarea = $(this) From 252043ae11f51c637ebe1c5d3a58aae4352e3b2a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:33:18 +0200 Subject: [PATCH 7/9] Move code for cleaner diff --- public/script.js | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/public/script.js b/public/script.js index 44e946693..0d7d47e24 100644 --- a/public/script.js +++ b/public/script.js @@ -9726,6 +9726,26 @@ jQuery(async function () { }); const cssAutofit = CSS.supports('field-sizing', 'content'); + if (!cssAutofit) { + /** + * Sets the scroll height of the edit textarea to fit the content. + * @param {HTMLTextAreaElement} e Textarea element to auto-fit + */ + function autoFitEditTextArea(e) { + e.style.height = '0px'; + const newHeight = e.scrollHeight + 4; + e.style.height = `${newHeight}px`; + } + const autoFitEditTextAreaDebounced = debounce(autoFitEditTextArea, debounce_timeout.short); + document.addEventListener('input', e => { + if (e.target instanceof HTMLTextAreaElement && e.target.classList.contains('edit_textarea')) { + const scrollbarShown = e.target.clientWidth < e.target.offsetWidth && e.target.offsetHeight >= window.innerHeight * 0.75; + const immediately = (e.target.scrollHeight > e.target.offsetHeight && !scrollbarShown) || e.target.value === ''; + immediately ? autoFitEditTextArea(e.target) : autoFitEditTextAreaDebounced(e.target); + } + }); + } + const chatElementScroll = document.getElementById('chat'); const chatScrollHandler = function () { if (power_user.waifuMode) { @@ -9748,26 +9768,6 @@ jQuery(async function () { chatElementScroll.addEventListener('wheel', chatScrollHandler, { passive: true }); chatElementScroll.addEventListener('touchmove', chatScrollHandler, { passive: true }); - if (!cssAutofit) { - /** - * Sets the scroll height of the edit textarea to fit the content. - * @param {HTMLTextAreaElement} e Textarea element to auto-fit - */ - function autoFitEditTextArea(e) { - e.style.height = '0px'; - const newHeight = e.scrollHeight + 4; - e.style.height = `${newHeight}px`; - } - const autoFitEditTextAreaDebounced = debounce(autoFitEditTextArea, debounce_timeout.short); - document.addEventListener('input', e => { - if (e.target instanceof HTMLTextAreaElement && e.target.classList.contains('edit_textarea')) { - const scrollbarShown = e.target.clientWidth < e.target.offsetWidth && e.target.offsetHeight >= window.innerHeight * 0.75; - const immediately = (e.target.scrollHeight > e.target.offsetHeight && !scrollbarShown) || e.target.value === ''; - immediately ? autoFitEditTextArea(e.target) : autoFitEditTextAreaDebounced(e.target); - } - }); - } - $(document).on('click', '.mes', function () { //when a 'delete message' parent div is clicked // and we are in delete mode and del_checkbox is visible From d07ee76784b74f28ec7bbe9b16ab8cc6ffbb8e7c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:48:12 +0200 Subject: [PATCH 8/9] Add overscroll contain for edit textarea --- public/style.css | 1 + 1 file changed, 1 insertion(+) diff --git a/public/style.css b/public/style.css index d47c8aaf6..725b33514 100644 --- a/public/style.css +++ b/public/style.css @@ -4113,6 +4113,7 @@ input[type="range"]::-webkit-slider-thumb { max-height: 75vh; max-height: 75dvh; field-sizing: content; + overscroll-behavior-y: contain; } #anchor_order { From 21ee072677ae89ef55a28472ffeacd3af8a8b9c0 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:50:37 +0200 Subject: [PATCH 9/9] Revert "Add overscroll contain for edit textarea" This reverts commit d07ee76784b74f28ec7bbe9b16ab8cc6ffbb8e7c. --- public/style.css | 1 - 1 file changed, 1 deletion(-) diff --git a/public/style.css b/public/style.css index 725b33514..d47c8aaf6 100644 --- a/public/style.css +++ b/public/style.css @@ -4113,7 +4113,6 @@ input[type="range"]::-webkit-slider-thumb { max-height: 75vh; max-height: 75dvh; field-sizing: content; - overscroll-behavior-y: contain; } #anchor_order {