From e7b6738fe859bffc64019f18bdad702ab622fd82 Mon Sep 17 00:00:00 2001 From: based Date: Sun, 31 Dec 2023 05:04:37 +1000 Subject: [PATCH 01/40] merge --- src/endpoints/prompt-converters.js | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/endpoints/prompt-converters.js b/src/endpoints/prompt-converters.js index 12efd1cdc..8b2d3a0fb 100644 --- a/src/endpoints/prompt-converters.js +++ b/src/endpoints/prompt-converters.js @@ -65,6 +65,53 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, return requestPrompt; } +/** + * Convert ChatML objects into working with Anthropic's new Messaging API. + * @param {object[]} messages Array of messages + * @param {boolean} addAssistantPostfix Add Assistant postfix. + * @param {string} addAssistantPrefill Add Assistant prefill after the assistant postfix. + * @param {boolean} withSysPromptSupport Indicates if the Claude model supports the system prompt format. + * @param {string} addSysHumanMsg Add Human message between system prompt and assistant. + */ +function convertClaudeMessages(messages, addAssistantPostfix, addAssistantPrefill, addSysHumanMsg) { + // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array. + let systemPrompt = ''; + let i; + for (i = 0; i < messages.length; i++) { + if (messages[i].role !== 'system') { + break; + } + systemPrompt += `${messages[i].content}\n\n`; + } + + messages.splice(0, i); + + // Check if the first message in the array is of type user, if not, interject with addSysHumanMsg or a blank message. + if (messages.length > 0 && messages[0].role !== 'user') { + messages.unshift({ + role: 'user', + content: addSysHumanMsg || '', + }); + } + + // Now replace all further messages that have the role 'system' with the role 'user'. + messages.forEach((message) => { + if (message.role === 'system') { + message.role = 'user'; + } + }); + + // Postfix and prefill + if (addAssistantPostfix) { + messages.push({ + role: 'assistant', + content: addAssistantPrefill || '', + }); + } + + return { messages: messages, systemPrompt: systemPrompt.trim() }; +} + /** * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models. * @param {object[]} messages Array of messages @@ -154,6 +201,7 @@ function convertTextCompletionPrompt(messages) { module.exports = { convertClaudePrompt, + convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, }; From a38bfe39b548e2fd4bd2e0f0732630c09920c968 Mon Sep 17 00:00:00 2001 From: berbant <33601955+berbant@users.noreply.github.com> Date: Sat, 2 Mar 2024 23:21:53 +0400 Subject: [PATCH 02/40] Add Link to Source to Dropdown menu --- public/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/index.html b/public/index.html index e91ea5cae..4add787d5 100644 --- a/public/index.html +++ b/public/index.html @@ -3990,6 +3990,9 @@ + +
+
+
+ Description + + + + +
+ +
+ Tokens: counting... +
+
+
+
+
+ First message + + + + +
+
+ + Alt. Greetings + +
+
+ +
+ Tokens: counting... +
+
@@ -4417,7 +4421,7 @@ Chat History
- - +
diff --git a/public/script.js b/public/script.js index dcab37d2a..29b848769 100644 --- a/public/script.js +++ b/public/script.js @@ -6442,7 +6442,7 @@ export function select_selected_character(chid) { $('#description_textarea').val(characters[chid].description); $('#character_world').val(characters[chid].data?.extensions?.world || ''); $('#creator_notes_textarea').val(characters[chid].data?.creator_notes || characters[chid].creatorcomment); - $('#creator_notes_spoiler').text(characters[chid].data?.creator_notes || characters[chid].creatorcomment); + $('#creator_notes_spoiler').html(converter.makeHtml(characters[chid].data?.creator_notes || characters[chid].creatorcomment)); $('#character_version_textarea').val(characters[chid].data?.character_version || ''); $('#system_prompt_textarea').val(characters[chid].data?.system_prompt || ''); $('#post_history_instructions_textarea').val(characters[chid].data?.post_history_instructions || ''); @@ -6512,7 +6512,7 @@ function select_rm_create() { $('#description_textarea').val(create_save.description); $('#character_world').val(create_save.world); $('#creator_notes_textarea').val(create_save.creator_notes); - $('#creator_notes_spoiler').text(create_save.creator_notes); + $('#creator_notes_spoiler').html(converter.makeHtml(create_save.creator_notes)); $('#post_history_instructions_textarea').val(create_save.post_history_instructions); $('#system_prompt_textarea').val(create_save.system_prompt); $('#tags_textarea').val(create_save.tags); From be38359d661462078c8b76b339bac3ffd5267eb2 Mon Sep 17 00:00:00 2001 From: RossAscends <124905043+RossAscends@users.noreply.github.com> Date: Sun, 3 Mar 2024 22:25:20 +0900 Subject: [PATCH 08/40] add maximize button to creator note --- public/index.html | 2 +- public/style.css | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index d255d3fca..2d2fff432 100644 --- a/public/index.html +++ b/public/index.html @@ -4298,7 +4298,7 @@
-

Creator's Notes

+

Creator's Notes

diff --git a/public/style.css b/public/style.css index 969efc27e..ff27b1250 100644 --- a/public/style.css +++ b/public/style.css @@ -1018,6 +1018,19 @@ select { order: 3; } +#character_popup .editor_maximize { + cursor: pointer; + margin: 5px; + opacity: 0.75; + filter: grayscale(1); + -webkit-transition: all 250ms ease-in-out; + transition: all 250ms ease-in-out; +} + +#character_popup .editor_maximize:hover { + opacity: 1; +} + .text_pole::placeholder { color: rgb(139, 139, 139); } From 39c588f30e4d642738ce0d0aeffd116cd859e09a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 15:26:25 +0200 Subject: [PATCH 09/40] Showdown: parse single underscores as italics --- public/script.js | 2 ++ public/scripts/showdown-underscore.js | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 public/scripts/showdown-underscore.js diff --git a/public/script.js b/public/script.js index 29b848769..a664c12d2 100644 --- a/public/script.js +++ b/public/script.js @@ -172,6 +172,7 @@ import { } from './scripts/secrets.js'; import { EventEmitter } from './lib/eventemitter.js'; import { markdownExclusionExt } from './scripts/showdown-exclusion.js'; +import { markdownUnderscoreExt } from './scripts/showdown-underscore.js'; import { NOTE_MODULE_NAME, initAuthorsNote, metadata_keys, setFloatingPrompt, shouldWIAddPrompt } from './scripts/authors-note.js'; import { registerPromptManagerMigration } from './scripts/PromptManager.js'; import { getRegexedString, regex_placement } from './scripts/extensions/regex/engine.js'; @@ -690,6 +691,7 @@ function reloadMarkdownProcessor(render_formulas = false) { parseImgDimensions: true, tables: true, underline: true, + extensions: [markdownUnderscoreExt()], }); } diff --git a/public/scripts/showdown-underscore.js b/public/scripts/showdown-underscore.js new file mode 100644 index 000000000..1ff3bdabd --- /dev/null +++ b/public/scripts/showdown-underscore.js @@ -0,0 +1,22 @@ +// Showdown extension that replaces words surrounded by singular underscores with tags +export const markdownUnderscoreExt = () => { + if (!canUseNegativeLookbehind()) { + console.log('Showdown-underscore extension: Negative lookbehind not supported. Skipping.'); + return []; + } + + return [{ + type: 'lang', + regex: /\b(?$1', + }]; +}; + +function canUseNegativeLookbehind() { + try { + new RegExp('(? Date: Sun, 3 Mar 2024 16:04:48 +0200 Subject: [PATCH 10/40] Clean-up /fuzzy command doc comments --- public/scripts/slash-commands.js | 50 ++++++++++++++------------------ 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 5376d14b1..b1f84b208 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -173,7 +173,7 @@ parser.addCommand('gen', generateCallback, [], '(lock=on parser.addCommand('genraw', generateRawCallback, [], '(lock=on/off [prompt]) – generates text using the provided prompt and passes it to the next command through the pipe, optionally locking user input while generating. Does not include chat history or character card. Use instruct=off to skip instruct formatting, e.g. /genraw instruct=off Why is the sky blue?. Use stop=... with a JSON-serialized array to add one-time custom stop strings, e.g. /genraw stop=["\\n"] Say hi', true, true); parser.addCommand('addswipe', addSwipeCallback, ['swipeadd'], '(text) – adds a swipe to the last chat message.', true, true); parser.addCommand('abort', abortCallback, [], ' – aborts the slash command batch execution', true, true); -parser.addCommand('fuzzy', fuzzyCallback, [], 'list=["a","b","c"] threshold=0.4 (text to search) – performs a fuzzy match of each items of list within the text to search. If any item matches then its name is returned. If no item list matches the text to search then undefined is returned. The optional threshold (default is 0.4) allows some control over the matching. A low value (min 0.0) means the match is very strict. At 1.0 (max) the match is very loose and probably matches anything. The returned value passes to the next command through the pipe.', true, true);parser.addCommand('pass', (_, arg) => arg, ['return'], '(text) – passes the text to the next command through the pipe.', true, true); +parser.addCommand('fuzzy', fuzzyCallback, [], 'list=["a","b","c"] threshold=0.4 (text to search) – performs a fuzzy match of each items of list within the text to search. If any item matches then its name is returned. If no item list matches the text to search then no value is returned. The optional threshold (default is 0.4) allows some control over the matching. A low value (min 0.0) means the match is very strict. At 1.0 (max) the match is very loose and probably matches anything. The returned value passes to the next command through the pipe.', true, true); parser.addCommand('pass', (_, arg) => arg, ['return'], '(text) – passes the text to the next command through the pipe.', true, true); parser.addCommand('delay', delayCallback, ['wait', 'sleep'], '(milliseconds) – delays the next command in the pipe by the specified number of milliseconds.', true, true); parser.addCommand('input', inputCallback, ['prompt'], '(default="string" large=on/off wide=on/off okButton="string" rows=number [text]) – Shows a popup with the provided text and an input field. The default argument is the default value of the input field, and the text argument is the text to display.', true, true); parser.addCommand('run', runCallback, ['call', 'exec'], '[key1=value key2=value ...] ([qrSet.]qrLabel) – runs a Quick Reply with the specified name from a currently active preset or from another preset, named arguments can be referenced in a QR with {{arg::key}}.', true, true); @@ -503,23 +503,15 @@ async function inputCallback(args, prompt) { /** * Each item in "args.list" is searched within "search_item" using fuzzy search. If any matches it returns the matched "item". - * @param {any} args - arguments containing "list" (JSON array) and optionaly "threshold" (float between 0.0 and 1.0) - * @param {list} args.list - list of words you search into search_in_value - * @param {number} args.threshold - sensitivity of search, the lower the strictier, default value is 0.4 - * @param {string} search_in_value - the string where items of list are searched - * @returns {string} + * @param {FuzzyCommandArgs} args - arguments containing "list" (JSON array) and optionaly "threshold" (float between 0.0 and 1.0) + * @param {string} searchInValue - the string where items of list are searched + * @returns {string} - the matched item from the list + * @typedef {{list: string, threshold: string}} FuzzyCommandArgs - arguments for /fuzzy command + * @example /fuzzy list=["down","left","up","right"] "he looks up" | /echo // should return "up" + * @link https://www.fusejs.io/ */ -function fuzzyCallback(args, search_in_value) { - // - // args.list : list of words (no space) you search into search_in_value - // args.threshold : sensitivity of search, lower the more strict (added to give more flexibility) - // search_in_value: the text where you want to search - // - // /fuzzy list=["down","left","up","right"] "he looks up" | /echo - // should return "up" - // https://www.fusejs.io/ - - if (!value) { +function fuzzyCallback(args, searchInValue) { + if (!searchInValue) { console.warn('WARN: No argument provided for /fuzzy command'); return ''; } @@ -543,27 +535,27 @@ function fuzzyCallback(args, search_in_value) { threshold: 0.4, }; // threshold determines how strict is the match, low threshold value is very strict, at 1 (nearly?) everything matches - if ( 'threshold' in args ) { + if ('threshold' in args) { params.threshold = parseFloat(resolveVariable(args.threshold)); - if ( isNaN(params.threshold) ) { + if (isNaN(params.threshold)) { console.warn('WARN: \'threshold\' argument must be a float between 0.0 and 1.0 for /fuzzy command'); return ''; } - if ( params.threshold < 0 ) { + if (params.threshold < 0) { params.threshold = 0; } - if ( params.threshold > 1 ) { + if (params.threshold > 1) { params.threshold = 1; } } - const fuse = new Fuse([search_in_value], params); + const fuse = new Fuse([searchInValue], params); // each item in the "list" is searched within "search_item", if any matches it returns the matched "item" - for (let search_item of list) { - let result = fuse.search(search_item); - if ( result.length > 0 ) { - console.info('fuzzyCallback Matched: ' + search_item); - return search_item; + for (const searchItem of list) { + const result = fuse.search(searchItem); + if (result.length > 0) { + console.info('fuzzyCallback Matched: ' + searchItem); + return searchItem; } } return ''; @@ -1623,7 +1615,7 @@ async function executeSlashCommands(text, unescape = false) { ?.replace(/\\\|/g, '|') ?.replace(/\\\{/g, '{') ?.replace(/\\\}/g, '}') - ; + ; } for (const [key, value] of Object.entries(result.args)) { @@ -1632,7 +1624,7 @@ async function executeSlashCommands(text, unescape = false) { .replace(/\\\|/g, '|') .replace(/\\\{/g, '{') .replace(/\\\}/g, '}') - ; + ; } } From 112e8f224cfa6c70a41e9ecf57872342ea366b38 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 16:45:26 +0200 Subject: [PATCH 11/40] Use alternate method of determining URLs, consolidate logs format --- public/script.js | 3 ++- src/endpoints/content-manager.js | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/public/script.js b/public/script.js index 8999f3f98..cee684a60 100644 --- a/public/script.js +++ b/public/script.js @@ -148,6 +148,7 @@ import { getBase64Async, humanFileSize, Stopwatch, + isValidUrl, } from './scripts/utils.js'; import { ModuleWorkerWrapper, doDailyExtensionUpdatesCheck, extension_settings, getContext, loadExtensionSettings, renderExtensionTemplate, runGenerationInterceptors, saveMetadataDebounced } from './scripts/extensions.js'; @@ -9988,7 +9989,7 @@ jQuery(async function () { const url = input.trim(); var request; - if (url.includes("https")) { + if (isValidUrl(url)) { console.debug('Custom content import started for URL: ', url); request = await fetch('/api/content/importURL', { method: 'POST', diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index ef18395ac..e40c8ff85 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -422,18 +422,17 @@ router.post('/importUUID', jsonParser, async (request, response) => { const uuid = request.body.url; let result; - const isJannny = uuid.includes("_character") - const isPygmalion = (!isJannny && uuid.length == 36) - const uuidType = uuid.includes("lorebook") ? "lorebook" : "character"; + const isJannny = uuid.includes('_character'); + const isPygmalion = (!isJannny && uuid.length == 36); + const uuidType = uuid.includes('lorebook') ? 'lorebook' : 'character'; if (isPygmalion) { - console.debug("We have a Pyg character") + console.log('Downloading Pygmalion character:', uuid); result = await downloadPygmalionCharacter(uuid); } else if (isJannny) { - console.debug("We have a Janny character") - result = await downloadJannyCharacter(uuid.split("_")[0]); + console.log('Downloading Janitor character:', uuid.split('_')[0]); + result = await downloadJannyCharacter(uuid.split('_')[0]); } else { - console.debug("We have something from Chub?") if (uuidType === 'character') { console.log('Downloading chub character:', uuid); result = await downloadChubCharacter(uuid); From 8cf1671d56b67e1ea9da169c3b4c4383033990e8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 16:49:44 +0200 Subject: [PATCH 12/40] Sanitize creator's notes --- public/script.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/script.js b/public/script.js index 2e69fc678..bafb96628 100644 --- a/public/script.js +++ b/public/script.js @@ -6445,7 +6445,7 @@ export function select_selected_character(chid) { $('#description_textarea').val(characters[chid].description); $('#character_world').val(characters[chid].data?.extensions?.world || ''); $('#creator_notes_textarea').val(characters[chid].data?.creator_notes || characters[chid].creatorcomment); - $('#creator_notes_spoiler').html(converter.makeHtml(characters[chid].data?.creator_notes || characters[chid].creatorcomment)); + $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(characters[chid].data?.creator_notes || characters[chid].creatorcomment))); $('#character_version_textarea').val(characters[chid].data?.character_version || ''); $('#system_prompt_textarea').val(characters[chid].data?.system_prompt || ''); $('#post_history_instructions_textarea').val(characters[chid].data?.post_history_instructions || ''); @@ -6515,7 +6515,7 @@ function select_rm_create() { $('#description_textarea').val(create_save.description); $('#character_world').val(create_save.world); $('#creator_notes_textarea').val(create_save.creator_notes); - $('#creator_notes_spoiler').html(converter.makeHtml(create_save.creator_notes)); + $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(create_save.creator_notes))); $('#post_history_instructions_textarea').val(create_save.post_history_instructions); $('#system_prompt_textarea').val(create_save.system_prompt); $('#tags_textarea').val(create_save.tags); From 8767c2a90b311270c62f709b3b90bf32351c3f80 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 16:57:38 +0200 Subject: [PATCH 13/40] Respect external media preferences in creator notes --- public/script.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/script.js b/public/script.js index bafb96628..cfb453486 100644 --- a/public/script.js +++ b/public/script.js @@ -6445,7 +6445,7 @@ export function select_selected_character(chid) { $('#description_textarea').val(characters[chid].description); $('#character_world').val(characters[chid].data?.extensions?.world || ''); $('#creator_notes_textarea').val(characters[chid].data?.creator_notes || characters[chid].creatorcomment); - $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(characters[chid].data?.creator_notes || characters[chid].creatorcomment))); + $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(characters[chid].data?.creator_notes || characters[chid].creatorcomment), { MESSAGE_SANITIZE: true })); $('#character_version_textarea').val(characters[chid].data?.character_version || ''); $('#system_prompt_textarea').val(characters[chid].data?.system_prompt || ''); $('#post_history_instructions_textarea').val(characters[chid].data?.post_history_instructions || ''); @@ -6515,7 +6515,7 @@ function select_rm_create() { $('#description_textarea').val(create_save.description); $('#character_world').val(create_save.world); $('#creator_notes_textarea').val(create_save.creator_notes); - $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(create_save.creator_notes))); + $('#creator_notes_spoiler').html(DOMPurify.sanitize(converter.makeHtml(create_save.creator_notes), { MESSAGE_SANITIZE: true })); $('#post_history_instructions_textarea').val(create_save.post_history_instructions); $('#system_prompt_textarea').val(create_save.system_prompt); $('#tags_textarea').val(create_save.tags); From 23c2a0d8f5512d7081254365ec22763d865c25e2 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 18:56:05 +0200 Subject: [PATCH 14/40] Fix card fields replace if missing --- public/script.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/script.js b/public/script.js index cfb453486..961ce65c1 100644 --- a/public/script.js +++ b/public/script.js @@ -2491,12 +2491,12 @@ export function getCharacterCardFields() { return result; } - const scenarioText = chat_metadata['scenario'] || characters[this_chid].scenario; - result.description = baseChatReplace(characters[this_chid].description.trim(), name1, name2); - result.personality = baseChatReplace(characters[this_chid].personality.trim(), name1, name2); + const scenarioText = chat_metadata['scenario'] || characters[this_chid]?.scenario; + result.description = baseChatReplace(characters[this_chid].description?.trim(), name1, name2); + result.personality = baseChatReplace(characters[this_chid].personality?.trim(), name1, name2); result.scenario = baseChatReplace(scenarioText.trim(), name1, name2); - result.mesExamples = baseChatReplace(characters[this_chid].mes_example.trim(), name1, name2); - result.persona = baseChatReplace(power_user.persona_description.trim(), name1, name2); + result.mesExamples = baseChatReplace(characters[this_chid].mes_example?.trim(), name1, name2); + result.persona = baseChatReplace(power_user.persona_description?.trim(), name1, name2); result.system = power_user.prefer_character_prompt ? baseChatReplace(characters[this_chid].data?.system_prompt?.trim(), name1, name2) : ''; result.jailbreak = power_user.prefer_character_jailbreak ? baseChatReplace(characters[this_chid].data?.post_history_instructions?.trim(), name1, name2) : ''; From 6ea2cf2abedfd1070618934e7ca99c277f69e527 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 19:06:37 +0200 Subject: [PATCH 15/40] Indicate OR instruct override as legacy --- public/index.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index 2d2fff432..d587c3064 100644 --- a/public/index.html +++ b/public/index.html @@ -2440,6 +2440,10 @@
@@ -5559,4 +5563,4 @@ - \ No newline at end of file + From b9392893dc9cafa32d03448ee667859a837cbd88 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 3 Mar 2024 19:12:20 +0200 Subject: [PATCH 16/40] [FEATURE_REQUEST] Option to toggle disable instruct formatting for example dialogue insertion #1881 --- public/index.html | 4 ++++ public/scripts/instruct-mode.js | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/public/index.html b/public/index.html index d587c3064..d04f46373 100644 --- a/public/index.html +++ b/public/index.html @@ -2756,6 +2756,10 @@ Replace Macro in Sequences +