From e9ce2853cc60fec7e73ae8282ba7f043922a87ee Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:48:53 -0400 Subject: [PATCH 01/16] Allow split on newlines (wip) --- public/script.js | 2 +- .../extensions/infinity-context/index.js | 25 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/public/script.js b/public/script.js index 33469565b..2a8ce192e 100644 --- a/public/script.js +++ b/public/script.js @@ -688,7 +688,7 @@ $.get("/csrf-token").then(async (data) => { function checkOnlineStatus() { ///////// REMOVED LINES THAT DUPLICATE RA_CHeckOnlineStatus FEATURES - + console.log(online_status); if (online_status == "no_connection") { $("#online_status_indicator2").css("background-color", "red"); //Kobold $("#online_status_text2").html("No connection..."); diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index ccc82031a..56282fb96 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -433,8 +433,13 @@ async function onSelectInjectFile(e) { try { toastr.info('This may take some time, depending on the file size', 'Processing...'); const text = await getFileText(file); - - const split = splitRecursive(text, extension_settings.chromadb.file_split_length).filter(onlyUnique); + extension_settings.chromadb.file_split_type = "newline"; + //allow splitting on newlines or splitrecursively + if(extension_settings.chromadb.file_split_type == "newline"){ + var split = text.split(/\r?\n/).filter(onlyUnique); + } else { + const split = splitRecursive(text, extension_settings.chromadb.file_split_length).filter(onlyUnique); + } const baseDate = Date.now(); const messages = split.map((m, i) => ({ @@ -533,10 +538,19 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { let recallMsg = extension_settings.chromadb.recall_msg || defaultSettings.chroma_default_msg; const chromaDepth = extension_settings.chromadb.chroma_depth; const chromaSortStrategy = extension_settings.chromadb.sort_strategy; + + //log the current settings + console.debug("CHROMADB: Current settings: %o", extension_settings.chromadb); + + if (currentChatId) { const messagesToStore = chat.slice(0, -extension_settings.chromadb.keep_context); + //log the messages to store + console.debug("CHROMADB: Messages to store: %o", messagesToStore); if (messagesToStore.length > 0 || extension_settings.chromadb.freeze) { + //log the messages to store length vs keep context + console.debug("CHROMADB: Messages to store length vs keep context: %o vs %o", messagesToStore.length, extension_settings.chromadb.keep_context); await addMessages(currentChatId, messagesToStore); const lastMessage = chat[chat.length - 1]; @@ -549,13 +563,16 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { queriedMessages = await queryMultiMessages(currentChatId, lastMessage.mes); } else{ + console.log("Utilizing single chat"); queriedMessages = await queryMessages(currentChatId, lastMessage.mes); } if(chromaSortStrategy === "date"){ + console.log("Sorting by date"); queriedMessages.sort((a, b) => a.date - b.date); } else{ + console.log("Sorting by distance"); queriedMessages.sort((a, b) => b.distance - a.distance); } @@ -630,7 +647,11 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { if (selectedStrategy === 'original') { //removes .length # messages from the start of 'kept messages' //replaces them with chromaDB results (with no separator) + console.log('ChromaDB chat before injection', chat); + console.log('ChromaDB newChat', newChat); + console.log('ChromaDB queriedMessages', queriedMessages); newChat.push(...queriedMessages.map(m => m.meta).filter(onlyUnique).map(JSON.parse)); + console.log('ChromaDB newChat after push', newChat); chat.splice(0, messagesToStore.length, ...newChat); } From 1d640a2cbf2a4d72deccd2d7f82821b2e51712cc Mon Sep 17 00:00:00 2001 From: phiharri Date: Thu, 6 Jul 2023 01:43:57 +0100 Subject: [PATCH 02/16] Optional delete message confirmation --- public/index.html | 5 ++++- public/script.js | 8 +++++--- public/scripts/power-user.js | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 06878afbd..51601ad5c 100644 --- a/public/index.html +++ b/public/index.html @@ -2344,6 +2344,9 @@ +
@@ -3561,4 +3564,4 @@ - \ No newline at end of file + diff --git a/public/script.js b/public/script.js index f22757020..b30c2c997 100644 --- a/public/script.js +++ b/public/script.js @@ -7672,9 +7672,11 @@ $(document).ready(function () { $(document).on("click", ".mes_edit_delete", async function () { - const confirmation = await callPopup("Are you sure you want to delete this message?", 'confirm'); - if (!confirmation) { - return; + if (power_user.confirm_message_delete) { + const confirmation = await callPopup("Are you sure you want to delete this message?", 'confirm'); + if (!confirmation) { + return; + } } const mes = $(this).closest(".mes"); diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index c1a548a22..4700a4817 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -632,6 +632,7 @@ function loadPowerUserSettings(settings, data) { $(`#pygmalion_formatting option[value=${power_user.pygmalion_formatting}]`).attr("selected", true); $(`#send_on_enter option[value=${power_user.send_on_enter}]`).attr("selected", true); $("#import_card_tags").prop("checked", power_user.import_card_tags); + $("#confirm_message_delete").prop("checked", power_user.confirm_message_delete !== undefined ? !!power_user.confirm_message_delete : true); $("#collapse-newlines-checkbox").prop("checked", power_user.collapse_newlines); $("#pin-examples-checkbox").prop("checked", power_user.pin_examples); $("#disable-description-formatting-checkbox").prop("checked", power_user.disable_description_formatting); @@ -1697,6 +1698,11 @@ $(document).ready(() => { saveSettingsDebounced(); }); + $("#confirm_message_delete").on('input', function () { + power_user.confirm_message_delete = !!$(this).prop('checked'); + saveSettingsDebounced(); + }); + $("#render_formulas").on("input", function () { power_user.render_formulas = !!$(this).prop('checked'); reloadMarkdownProcessor(power_user.render_formulas); From b6ac73631fd2c56b1c680def593d5508d1c768e1 Mon Sep 17 00:00:00 2001 From: Cohee Date: Thu, 6 Jul 2023 12:35:03 +0300 Subject: [PATCH 03/16] #640 Fix group candidates filtering --- public/scripts/group-chats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index f6a9e08fd..3da831a75 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -1234,7 +1234,7 @@ function filterGroupMembers() { $("#rm_group_add_members .group_member").removeClass('hiddenBySearch'); } else { $("#rm_group_add_members .group_member").each(function () { - const isValidSearch = $(this).children(".ch_name").text().toLowerCase().includes(searchValue); + const isValidSearch = $(this).find(".ch_name").text().toLowerCase().includes(searchValue); $(this).toggleClass('hiddenBySearch', !isValidSearch); }); } From a2fc3ec1153a6285784e4e4a3fe3d74c8a62275b Mon Sep 17 00:00:00 2001 From: phiharri Date: Thu, 6 Jul 2023 20:56:08 +0100 Subject: [PATCH 04/16] set default value for confirm_message_delete --- public/scripts/power-user.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 4700a4817..0c18458b5 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -121,6 +121,7 @@ let power_user = { play_message_sound: false, play_sound_unfocused: true, auto_save_msg_edits: false, + confirm_message_delete: true, sort_field: 'name', sort_order: 'asc', From 08fd83ca5d6d032dd9150c395c4899483c608d8b Mon Sep 17 00:00:00 2001 From: vbd537 Date: Thu, 6 Jul 2023 21:00:55 -0300 Subject: [PATCH 05/16] Temporary Poe Fix By adding "/Sage" or any other page name that exists to the end of the "home_url" variable, Poe's problem is solved and it can be used normally again --- src/poe-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/poe-client.js b/src/poe-client.js index 4e0365ba9..23080a70a 100644 --- a/src/poe-client.js +++ b/src/poe-client.js @@ -312,7 +312,7 @@ function logObjectStructure(obj, indent = 0, depth = Infinity) { class Client { gql_url = "https://poe.com/api/gql_POST"; gql_recv_url = "https://poe.com/api/receive_POST"; - home_url = "https://poe.com"; + home_url = "https://poe.com/Sage"; settings_url = "https://poe.com/api/settings"; formkey = ""; From 82624ff55bdf199556ec0ccd2f5dd84eb5feaa34 Mon Sep 17 00:00:00 2001 From: kingbri Date: Thu, 6 Jul 2023 22:21:04 -0400 Subject: [PATCH 06/16] Regex: fix multiple script bug Multiple scripts were not running due to improper variable assingment. For efficiency's sake, do not do a string comparison before returning and instead do another variable assignment in the parent function. Doing this reduces the length of regex hooks in the parent calls, but also removes the need for unnecessary O(n) complexity of comparing two string variables. If there are errors, it would be advisable to add string comparison and revert back to the old logic in parent function calls. Signed-off-by: kingbri --- public/script.js | 25 +++++------------------ public/scripts/extensions/regex/engine.js | 12 +++++------ public/scripts/slash-commands.js | 10 ++------- 3 files changed, 13 insertions(+), 34 deletions(-) diff --git a/public/script.js b/public/script.js index 970963cc8..91421ca0a 100644 --- a/public/script.js +++ b/public/script.js @@ -1137,10 +1137,7 @@ function messageFormatting(mes, ch_name, isSystem, isUser) { mes = mes.replaceAll(substituteParams(power_user.user_prompt_bias), ""); } - const regexResult = getRegexedString(mes, regex_placement.MD_DISPLAY); - if (regexResult) { - mes = regexResult; - } + mes = getRegexedString(mes, regex_placement.MD_DISPLAY); if (power_user.auto_fix_generated_markdown) { mes = fixMarkdown(mes); @@ -2956,10 +2953,7 @@ export function replaceBiasMarkup(str) { } export async function sendMessageAsUser(textareaText, messageBias) { - const regexResult = getRegexedString(textareaText, regex_placement.USER_INPUT); - if (regexResult) { - textareaText = regexResult; - } + textareaText = getRegexedString(textareaText, regex_placement.USER_INPUT); chat[chat.length] = {}; chat[chat.length - 1]['name'] = name1; @@ -3551,10 +3545,7 @@ function cleanUpMessage(getMessage, isImpersonate, displayIncompleteSentences = } // Regex uses vars, so add before formatting - const regexResult = getRegexedString(getMessage, isImpersonate ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT); - if (regexResult) { - getMessage = regexResult; - } + getMessage = getRegexedString(getMessage, isImpersonate ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT); if (!displayIncompleteSentences && power_user.trim_sentences) { getMessage = end_trim_to_sentence(getMessage, power_user.include_newline); @@ -4985,16 +4976,13 @@ function updateMessage(div) { regexPlacement = regex_placement.SYSTEM; } - const regexResult = getRegexedString( + text = getRegexedString( text, regexPlacement, { characterOverride: regexPlacement === regex_placement.SENDAS ? mes.name : undefined } ); - if (regexResult) { - text = regexResult; - } if (power_user.trim_spaces) { text = text.trim(); @@ -6113,10 +6101,7 @@ async function createOrEditCharacter(e) { ) { // MARK - kingbri: Regex on character greeting message // May need to be placed somewhere else - const regexResult = getRegexedString(this_ch_mes, regex_placement.AI_OUTPUT); - if (regexResult) { - this_ch_mes = regexResult; - } + this_ch_mes = getRegexedString(this_ch_mes, regex_placement.AI_OUTPUT); clearChat(); chat.length = 0; diff --git a/public/scripts/extensions/regex/engine.js b/public/scripts/extensions/regex/engine.js index 0a8554a1f..22bef65e0 100644 --- a/public/scripts/extensions/regex/engine.js +++ b/public/scripts/extensions/regex/engine.js @@ -39,14 +39,14 @@ function regexFromString(input) { // Parent function to fetch a regexed version of a raw string function getRegexedString(rawString, placement, { characterOverride } = {}) { + let finalString = rawString; if (extension_settings.disabledExtensions.includes("regex") || !rawString || placement === undefined) { - return; + return finalString; } - let finalString; extension_settings.regex.forEach((script) => { if (script.placement.includes(placement)) { - finalString = runRegexScript(script, rawString, { characterOverride }); + finalString = runRegexScript(script, finalString, { characterOverride }); } }); @@ -55,17 +55,17 @@ function getRegexedString(rawString, placement, { characterOverride } = {}) { // Runs the provided regex script on the given string function runRegexScript(regexScript, rawString, { characterOverride } = {}) { + let newString = rawString; if (!regexScript || !!(regexScript.disabled) || !regexScript?.findRegex || !rawString) { - return; + return newString; } let match; - let newString; const findRegex = regexFromString(regexScript.substituteRegex ? substituteParams(regexScript.findRegex) : regexScript.findRegex); // The user skill issued. Return with nothing. if (!findRegex) { - return; + return newString; } while ((match = findRegex.exec(rawString)) !== null) { diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 2020f9df9..21fb911cd 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -226,10 +226,7 @@ async function sendMessageAs(_, text) { const name = parts.shift().trim(); let mesText = parts.join('\n').trim(); - const regexResult = getRegexedString(mesText, regex_placement.SENDAS, { characterOverride: name }); - if (regexResult) { - mesText = regexResult; - } + mesText = getRegexedString(mesText, regex_placement.SENDAS, { characterOverride: name }); // Messages that do nothing but set bias will be hidden from the context const bias = extractMessageBias(mesText); @@ -273,10 +270,7 @@ async function sendNarratorMessage(_, text) { return; } - const regexResult = getRegexedString(text, regex_placement.SYSTEM); - if (regexResult) { - text = regexResult; - } + text = getRegexedString(text, regex_placement.SYSTEM); const name = chat_metadata[NARRATOR_NAME_KEY] || NARRATOR_NAME_DEFAULT; // Messages that do nothing but set bias will be hidden from the context From 02fb1bc26ff7854ea724e6f256ee11b0bae95c72 Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Thu, 6 Jul 2023 23:06:44 -0400 Subject: [PATCH 07/16] Revert to the not evals. --- public/script.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 970963cc8..a8903427d 100644 --- a/public/script.js +++ b/public/script.js @@ -6494,7 +6494,12 @@ const swipe_right = () => { } export function updateCharacterCount(characterSelector) { - const visibleCharacters = $(characterSelector).filter(":visible"); + const visibleCharacters = $(characterSelector) + .not(".hiddenBySearch") + .not(".hiddenByTag") + .not(".hiddenByGroup") + .not(".hiddenByGroupMember") + .not(".hiddenByFav"); const visibleCharacterCount = visibleCharacters.length; const totalCharacterCount = $(characterSelector).length; From f854609512d11b10c92120bcfe7b350c0e5aedcb Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Fri, 7 Jul 2023 00:23:28 -0400 Subject: [PATCH 08/16] Option to split on newlines --- .../extensions/infinity-context/index.js | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index a2f3bd56d..6230a8fad 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -186,6 +186,16 @@ function onFileSplitLengthInput() { saveSettingsDebounced(); } +function onChunkNLInput() { + let shouldSplit = $('#onChunkNLInput').is(':checked'); + if(shouldSplit){ + extension_settings.chromadb.file_split_type = "newline"; + } else { + extension_settings.chromadb.file_split_type = "length"; + } + saveSettingsDebounced(); +} + function checkChatId(chat_id) { if (!chat_id || chat_id.trim() === '') { toastr.error('Please select a character and try again.'); @@ -564,20 +574,16 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { console.debug(recallStrategy) if (lastMessage) { if (recallStrategy === 'multichat'){ - console.log("Utilizing multichat") queriedMessages = await queryMultiMessages(currentChatId, lastMessage.mes); } else{ - console.log("Utilizing single chat"); queriedMessages = await queryMessages(currentChatId, lastMessage.mes); } if(chromaSortStrategy === "date"){ - console.log("Sorting by date"); queriedMessages.sort((a, b) => a.date - b.date); } else{ - console.log("Sorting by distance"); queriedMessages.sort((a, b) => b.distance - a.distance); } @@ -652,15 +658,10 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { if (selectedStrategy === 'original') { //removes .length # messages from the start of 'kept messages' //replaces them with chromaDB results (with no separator) - console.log('ChromaDB chat before injection', chat); - console.log('ChromaDB newChat', newChat); - console.log('ChromaDB queriedMessages', queriedMessages); newChat.push(...queriedMessages.map(m => m.meta).filter(onlyUnique).map(JSON.parse)); - console.log('ChromaDB newChat after push', newChat); chat.splice(0, messagesToStore.length, ...newChat); } - console.log('ChromaDB chat after injection', chat); } } } @@ -746,6 +747,10 @@ jQuery(async () => { Use % strategy +