diff --git a/.gitignore b/.gitignore index d6a5061bb..fbef3d33f 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ access.log public/css/user.css /plugins/ /data +/default/scaffold diff --git a/default/content/presets/openai/Default.json b/default/content/presets/openai/Default.json index dbf3b9619..132abeea0 100644 --- a/default/content/presets/openai/Default.json +++ b/default/content/presets/openai/Default.json @@ -231,6 +231,7 @@ "api_url_scale": "", "show_external_models": false, "assistant_prefill": "", + "assistant_impersonation": "", "human_sysprompt_message": "Let's get started. Please generate your response based on the information and instructions provided above.", "use_ai21_tokenizer": false, "use_google_tokenizer": false, diff --git a/default/content/settings.json b/default/content/settings.json index 3f4c25865..c04b88bfb 100644 --- a/default/content/settings.json +++ b/default/content/settings.json @@ -624,6 +624,7 @@ "show_external_models": false, "proxy_password": "", "assistant_prefill": "", + "assistant_impersonation": "", "use_ai21_tokenizer": false } } diff --git a/default/scaffold/README.md b/default/scaffold/README.md new file mode 100644 index 000000000..b1272cf8f --- /dev/null +++ b/default/scaffold/README.md @@ -0,0 +1,26 @@ +# Content Scaffolding + +Content files in this folder will be copied for all users (old and new) on the server startup. + +1. You **must** create an `index.json` file in `/default/scaffold` for it to work. The syntax is the same as for default content. +2. All file paths should be relative to `/default/scaffold`, the use of subdirectories is allowed. +3. Scaffolded files are copied first, so they override any of the default files (presets/settings/etc.) that have the same file name. + +## Example + +```json +[ + { + "filename": "themes/Midnight.json", + "type": "theme" + }, + { + "filename": "backgrounds/city.png", + "type": "background" + }, + { + "filename": "characters/Charlie.png", + "type": "character" + } +] +``` diff --git a/package-lock.json b/package-lock.json index 93c8ed8ee..2a99a2099 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,17 @@ { "name": "sillytavern", - "version": "1.12.0-preview", + "version": "1.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sillytavern", - "version": "1.12.0-preview", + "version": "1.12.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", - "@dqbd/tiktoken": "^1.0.13", "@zeldafan0225/ai_horde": "^4.0.1", "archiver": "^7.0.1", "bing-translate-api": "^2.9.1", @@ -46,6 +45,7 @@ "sanitize-filename": "^1.6.3", "sillytavern-transformers": "^2.14.6", "simple-git": "^3.19.1", + "tiktoken": "^1.0.15", "vectra": "^0.2.2", "wavefile": "^11.0.0", "write-file-atomic": "^5.0.1", @@ -82,10 +82,6 @@ "version": "0.1.3", "license": "Apache-2.0" }, - "node_modules/@dqbd/tiktoken": { - "version": "1.0.13", - "license": "MIT" - }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "dev": true, @@ -4403,6 +4399,11 @@ "dev": true, "license": "MIT" }, + "node_modules/tiktoken": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.15.tgz", + "integrity": "sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw==" + }, "node_modules/timm": { "version": "1.7.1", "license": "MIT" diff --git a/package.json b/package.json index 278b63283..e8cbda243 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "dependencies": { "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", - "@dqbd/tiktoken": "^1.0.13", "@zeldafan0225/ai_horde": "^4.0.1", "archiver": "^7.0.1", "bing-translate-api": "^2.9.1", @@ -36,6 +35,7 @@ "sanitize-filename": "^1.6.3", "sillytavern-transformers": "^2.14.6", "simple-git": "^3.19.1", + "tiktoken": "^1.0.15", "vectra": "^0.2.2", "wavefile": "^11.0.0", "write-file-atomic": "^5.0.1", @@ -68,14 +68,15 @@ "type": "git", "url": "https://github.com/SillyTavern/SillyTavern.git" }, - "version": "1.12.0-preview", + "version": "1.12.0", "scripts": { "start": "node server.js", "start:no-csrf": "node server.js --disableCsrf", "postinstall": "node post-install.js", "lint": "eslint \"src/**/*.js\" \"public/**/*.js\" ./*.js", "lint:fix": "eslint \"src/**/*.js\" \"public/**/*.js\" ./*.js --fix", - "plugins:update": "node plugins update" + "plugins:update": "node plugins update", + "plugins:install": "node plugins install" }, "bin": { "sillytavern": "./server.js" diff --git a/plugins.js b/plugins.js index 4615951c6..63d21778a 100644 --- a/plugins.js +++ b/plugins.js @@ -15,6 +15,12 @@ if (command === 'update') { updatePlugins(); } +if (command === 'install') { + const pluginName = process.argv[3]; + console.log('Installing a new plugin', color.green(pluginName)); + installPlugin(pluginName); +} + async function updatePlugins() { const directories = fs.readdirSync(pluginsPath) .filter(file => !file.startsWith('.')) @@ -51,3 +57,19 @@ async function updatePlugins() { console.log(color.magenta('All plugins updated!')); } + +async function installPlugin(pluginName) { + try { + const pluginPath = path.join(pluginsPath, path.basename(pluginName, '.git')); + + if (fs.existsSync(pluginPath)) { + return console.log(color.yellow(`Directory already exists at ${pluginPath}`)); + } + + await git().clone(pluginName, pluginPath, { '--depth': 1 }); + console.log(`Plugin ${color.green(pluginName)} installed to ${color.cyan(pluginPath)}`); + } + catch (error) { + console.error(color.red(`Failed to install plugin ${pluginName}`), error); + } +} diff --git a/public/css/select2-overrides.css b/public/css/select2-overrides.css index cdab229d8..776bf48ed 100644 --- a/public/css/select2-overrides.css +++ b/public/css/select2-overrides.css @@ -171,3 +171,78 @@ .select2-results__option.select2-results__message::before { display: none; } + +.select2-selection__choice__display { + /* Fix weird alignment on the left side */ + margin-left: 1px; +} + +/* Styling for choice remove icon */ +span.select2.select2-container .select2-selection__choice__remove { + cursor: pointer; + transition: background-color 0.3s; + color: var(--SmartThemeBodyColor); + background-color: var(--black50a); +} + +span.select2.select2-container .select2-selection__choice__remove:hover { + color: var(--SmartThemeBodyColor); + background-color: var(--white30a); +} + +/* Custom class to support styling to show clickable choice options */ +.select2_choice_clickable+span.select2-container .select2-selection__choice__display { + cursor: pointer; +} +.select2_choice_clickable_buttonstyle+span.select2-container .select2-selection__choice__display { + cursor: pointer; + transition: background-color 0.3s; + color: var(--SmartThemeBodyColor); + background-color: var(--black50a); +} + +.select2_choice_clickable_buttonstyle+span.select2-container .select2-selection__choice__display:hover { + background-color: var(--white30a); +} + +/* Custom class to support same line multi inputs of select2 controls */ +.select2_multi_sameline+span.select2-container .select2-selection--multiple { + display: flex; + flex-wrap: wrap; +}.select2_multi_sameline+span.select2-container .select2-selection--multiple .select2-search--inline { + /* Allow search placeholder to take up all space if needed */ + flex-grow: 1; +} + +.select2_multi_sameline+span.select2-container .select2-selection--multiple .select2-selection__rendered { + /* Fix weird styling choice or huge margin around selected options */ + margin-block-start: 2px; + margin-block-end: 2px; +} + +.select2_multi_sameline+span.select2-container .select2-selection--multiple .select2-search__field { + /* Min height to reserve spacing */ + min-height: calc(var(--mainFontSize) + 13px); + /* Min width to be clickable */ + min-width: 4em; + align-content: center; + /* Fix search textarea alignment issue with UL elements */ + margin-top: 0px; + height: unset; + /* Prevent height from jumping around when input is focused */ + line-height: 1; +} + +.select2_multi_sameline+span.select2-container .select2-selection--multiple .select2-selection__rendered { + /* Min height to reserve spacing */ + min-height: calc(var(--mainFontSize) + 13px); +} + +/* Make search bar invisible unless the select2 is active, to save space */ +.select2_multi_sameline+span.select2-container .select2-selection--multiple .select2-search--inline { + height: 1px; +} + +.select2_multi_sameline+span.select2-container.select2-container--focus .select2-selection--multiple .select2-search--inline { + height: unset; +} diff --git a/public/css/tags.css b/public/css/tags.css index 11806c69a..f9896d992 100644 --- a/public/css/tags.css +++ b/public/css/tags.css @@ -103,7 +103,8 @@ } #bulkTagsList, -#tagList .tag { +#tagList .tag, +#groupTagList .tag { opacity: 0.6; } @@ -193,7 +194,8 @@ filter: brightness(75%) saturate(0.6); } -.tag_as_folder:hover { +.tag_as_folder:hover, +.tag_as_folder.flash { filter: brightness(150%) saturate(0.6) !important; } diff --git a/public/css/world-info.css b/public/css/world-info.css index ef814a104..78af36950 100644 --- a/public/css/world-info.css +++ b/public/css/world-info.css @@ -76,6 +76,12 @@ .world_entry_form_control { display: flex; flex-direction: column; + position: relative; +} + +.world_entry_form_control .keyprimarytextpole, +.world_entry_form_control .keysecondarytextpole { + padding-right: 25px; } .world_entry_thin_controls { @@ -101,7 +107,7 @@ height: auto; margin-top: 0; margin-bottom: 0; - min-height: calc(var(--mainFontSize) + 13px); + min-height: calc(var(--mainFontSize) + 14px); } .delete_entry_button { @@ -197,20 +203,57 @@ display: none; } -#world_info+span.select2-container .select2-selection__choice__remove, -#world_info+span.select2-container .select2-selection__choice__display { - cursor: pointer; - transition: background-color 0.3s; +span.select2-container .select2-selection__choice__display:has(> .regex_item), +span.select2-container .select2-results__option:has(> .result_block .regex_item) { + background-color: #D27D2D30; +} + +.regex_item .regex_icon { + background-color: var(--black30a); color: var(--SmartThemeBodyColor); - background-color: var(--black50a); + border: 1px solid var(--SmartThemeBorderColor); + border-radius: 7px; + font-weight: bold; + font-size: calc(var(--mainFontSize) * 0.75); + padding: 0px 3px; + position: relative; + top: -1px; + margin-right: 3px; } -#world_info+span.select2-container .select2-selection__choice__display { - /* Fix weird alignment on the left side */ - margin-left: 1px; +.select2-results__option .regex_item .regex_icon { + margin-right: 6px; } -#world_info+span.select2-container .select2-selection__choice__remove:hover, -#world_info+span.select2-container .select2-selection__choice__display:hover { - background-color: var(--white30a); +.select2-results__option .item_count { + margin-left: 10px; + float: right; +} + +select.keyselect+span.select2-container .select2-selection--multiple { + padding-right: 30px; +} + +.switch_input_type_icon { + cursor: pointer; + font-weight: bold; + height: 20px; + width: fit-content; + margin-right: 5px; + margin-top: calc(5px + var(--mainFontSize)); + position: absolute; + right: 0; + padding: 1px; + + background-color: transparent; + border: none; + font-size: 1em; + + opacity: 0.5; + color: var(--SmartThemeBodyColor); + transition: opacity 0.3s; +} + +.switch_input_type_icon:hover { + opacity: 1; } diff --git a/public/index.html b/public/index.html index ae0ba93de..5c2f065d7 100644 --- a/public/index.html +++ b/public/index.html @@ -116,7 +116,7 @@
@@ -134,7 +134,7 @@

Chat Completion Presets

@@ -246,7 +246,7 @@
-
+
Temperature
@@ -942,7 +942,7 @@
-
+
Logit Bias
@@ -1131,6 +1131,11 @@
+ + + +
+
Multiple swipes per generation @@ -1267,14 +1272,13 @@
-
-

-
+
+

+
-
- Dynamic Temperature + Dynamic Temperature

@@ -1296,7 +1300,7 @@

-
+

@@ -1325,7 +1329,7 @@

-
+

-
+
Seed
-
+

Banned Tokens @@ -1423,7 +1427,7 @@

-
+
Logit Bias
@@ -1437,7 +1441,7 @@
-
+

CFG
@@ -1459,7 +1463,7 @@

-
+ -
+
Samplers Order @@ -1680,7 +1684,7 @@
-
-
+
Logit Bias
@@ -2405,6 +2422,10 @@ + + + + @@ -2634,6 +2655,8 @@

Google Model

+
@@ -3426,8 +3449,8 @@
Chat Style:
@@ -3981,8 +4004,8 @@
- - + + (0 = All)
@@ -4065,7 +4088,7 @@ -

AutoComplete Settings

+
@@ -4171,14 +4200,14 @@ @@ -4473,6 +4502,9 @@ + +
+ Here you can toggle the display of individual samplers. (WIP) +
+
`); + + const listContainer = $('
'); + const APISamplers = await listSamplers(main_api); + listContainer.append(APISamplers); + html.append(listContainer); + + callPopup(html, 'text', null, { allowVerticalScrolling: true }); + + setSamplerListListeners(); + + $('#resetSelectedSamplers').off('click').on('click', async function () { + console.log('saw sampler select reset click'); + userDisabledSamplers = []; + userShownSamplers = []; + power_user.selectSamplers.forceShown = []; + power_user.selectSamplers.forceHidden = []; + await validateDisabledSamplers(true); + }); +} + +function setSamplerListListeners() { + // Goal 2: hide unchecked samplers from DOM + let listContainer = $('#apiSamplersList'); + listContainer.find('input').off('change').on('change', async function () { + + const samplerName = this.name.replace('_checkbox', ''); + let relatedDOMElement = $(`#${samplerName}_${main_api}`).parent(); + let targetDisplayType = 'flex'; + + if (samplerName === 'json_schema') { + relatedDOMElement = $('#json_schema_block'); + targetDisplayType = 'block'; + } + + if (samplerName === 'grammar_string') { + relatedDOMElement = $('#grammar_block_ooba'); + targetDisplayType = 'block'; + } + + if (samplerName === 'guidance_scale') { + relatedDOMElement = $('#cfg_block_ooba'); + targetDisplayType = 'block'; + } + + if (samplerName === 'mirostat_mode') { + relatedDOMElement = $('#mirostat_block_ooba'); + targetDisplayType = 'block'; + } + + if (samplerName === 'dynatemp') { + relatedDOMElement = $('#dynatemp_block_ooba'); + targetDisplayType = 'block'; + } + + if (samplerName === 'banned_tokens') { + relatedDOMElement = $('#banned_tokens_block_ooba'); + targetDisplayType = 'block'; + } + + if (samplerName === 'sampler_order') { + relatedDOMElement = $('#sampler_order_block'); + targetDisplayType = 'flex'; + } + + // Get the current state of the custom data attribute + const previousState = relatedDOMElement.data('selectsampler'); + + if ($(this).prop('checked') === false) { + //console.log('saw clicking checkbox from on to off...'); + if (previousState === 'shown') { + console.log('saw previously custom shown sampler'); + //console.log('removing from custom force show list'); + relatedDOMElement.removeData('selectsampler'); + $(this).parent().find('.sampler_name').removeAttr('style'); + power_user?.selectSamplers?.forceShown.splice(power_user?.selectSamplers?.forceShown.indexOf(samplerName), 1); + console.log(power_user?.selectSamplers?.forceShown); + } else { + console.log('saw previous untouched sampler'); + //console.log(`adding ${samplerName} to force hide list`); + relatedDOMElement.data('selectsampler', 'hidden'); + console.log(relatedDOMElement.data('selectsampler')); + power_user.selectSamplers.forceHidden.push(samplerName); + $(this).parent().find('.sampler_name').attr('style', forcedOffColoring); + console.log(power_user.selectSamplers.forceHidden); + } + } else { // going from unchecked to checked + //console.log('saw clicking checkbox from off to on...'); + if (previousState === 'hidden') { + console.log('saw previously custom hidden sampler'); + //console.log('removing from custom force hide list'); + relatedDOMElement.removeData('selectsampler'); + $(this).parent().find('.sampler_name').removeAttr('style'); + power_user?.selectSamplers?.forceHidden.splice(power_user?.selectSamplers?.forceHidden.indexOf(samplerName), 1); + console.log(power_user?.selectSamplers?.forceHidden); + } else { + console.log('saw previous untouched sampler'); + //console.log(`adding ${samplerName} to force shown list`); + relatedDOMElement.data('selectsampler', 'shown'); + console.log(relatedDOMElement.data('selectsampler')); + power_user.selectSamplers.forceShown.push(samplerName); + $(this).parent().find('.sampler_name').attr('style', forcedOnColoring); + console.log(power_user.selectSamplers.forceShown); + } + } + await saveSettingsDebounced(); + + const shouldDisplay = $(this).prop('checked') ? targetDisplayType : 'none'; + relatedDOMElement.css('display', shouldDisplay); + + console.log(samplerName, relatedDOMElement.data('selectsampler'), shouldDisplay); + }); + +} + +function isElementVisibleInDOM(element) { + while (element && element !== document.body) { + if (window.getComputedStyle(element).display === 'none') { + return false; + } + element = element.parentElement; + } + return true; +} + + +async function listSamplers(main_api, arrayOnly = false) { + let availableSamplers; + if (main_api === 'textgenerationwebui') { + availableSamplers = TGsamplerNames; + const valuesToRemove = new Set(['streaming', 'seed', 'bypass_status_check', 'custom_model', 'legacy_api', 'samplers']); + availableSamplers = availableSamplers.filter(sampler => !valuesToRemove.has(sampler)); + availableSamplers.sort(); + } + + if (arrayOnly) { + console.log('returning full samplers array'); + return availableSamplers; + } + + const samplersListHTML = availableSamplers.reduce((html, sampler) => { + let customColor; + const targetDOMelement = $(`#${sampler}_${main_api}`); + + const isInForceHiddenArray = userDisabledSamplers.includes(sampler); + const isInForceShownArray = userShownSamplers.includes(sampler); + let isVisibleInDOM = isElementVisibleInDOM(targetDOMelement[0]); + const isInDefaultState = () => { + if (isVisibleInDOM && isInForceShownArray) { return false; } + else if (!isVisibleInDOM && isInForceHiddenArray) { return false; } + else { return true; } + }; + + const shouldBeChecked = () => { + if (isInForceHiddenArray) { + customColor = forcedOffColoring; + return false; + } + else if (isInForceShownArray) { + customColor = forcedOnColoring; + return true; + } + else { return isVisibleInDOM; } + }; + console.log(sampler, isInDefaultState(), isInForceHiddenArray, shouldBeChecked()); + return html + ` +
+ + ${sampler} +
+ `; + }, ''); + + return samplersListHTML; +} + +// Goal 3: make "sampler is hidden/disabled" status persistent (save settings) +// this runs on initial getSettings as well as after API changes + +export async function validateDisabledSamplers(redraw = false) { + const APISamplers = await listSamplers(main_api, true); + + if (!Array.isArray(APISamplers)) { + return; + } + + for (const sampler of APISamplers) { + let relatedDOMElement = $(`#${sampler}_${main_api}`).parent(); + let targetDisplayType = 'flex'; + + if (sampler === 'json_schema') { + relatedDOMElement = $('#json_schema_block'); + targetDisplayType = 'block'; + } + + if (sampler === 'grammar_string') { + relatedDOMElement = $('#grammar_block_ooba'); + targetDisplayType = 'block'; + } + + if (sampler === 'guidance_scale') { + relatedDOMElement = $('#cfg_block_ooba'); + targetDisplayType = 'block'; + } + + if (sampler === 'mirostat_mode') { + relatedDOMElement = $('#mirostat_block_ooba'); + targetDisplayType = 'block'; + } + + if (sampler === 'dynatemp') { + relatedDOMElement = $('#dynatemp_block_ooba'); + targetDisplayType = 'block'; + } + + if (sampler === 'banned_tokens') { + relatedDOMElement = $('#banned_tokens_block_ooba'); + targetDisplayType = 'block'; + } + + if (sampler === 'sampler_order') { + relatedDOMElement = $('#sampler_order_block'); + } + + if (power_user?.selectSamplers?.forceHidden.includes(sampler)) { + //default handling for standard sliders + relatedDOMElement.data('selectsampler', 'hidden'); + relatedDOMElement.css('display', 'none'); + } else if (power_user?.selectSamplers?.forceShown.includes(sampler)) { + relatedDOMElement.data('selectsampler', 'shown'); + relatedDOMElement.css('display', targetDisplayType); + } else { + if (relatedDOMElement.data('selectsampler') === 'hidden') { + relatedDOMElement.removeAttr('selectsampler'); + relatedDOMElement.css('display', targetDisplayType); + } + if (relatedDOMElement.data('selectsampler') === 'shown') { + relatedDOMElement.removeAttr('selectsampler'); + relatedDOMElement.css('display', 'none'); + } + } + if (redraw) { + let samplersHTML = await listSamplers(main_api); + $('#apiSamplersList').empty().append(samplersHTML); + setSamplerListListeners(); + } + + + } +} + + +export async function initCustomSelectedSamplers() { + + userDisabledSamplers = power_user?.selectSamplers?.forceHidden || []; + userShownSamplers = power_user?.selectSamplers?.forceShown || []; + power_user.selectSamplers = {}; + power_user.selectSamplers.forceHidden = userDisabledSamplers; + power_user.selectSamplers.forceShown = userShownSamplers; + await saveSettingsDebounced(); + $('#samplerSelectButton').off('click').on('click', showSamplerSelectPopup); + +} + +// Goal 4: filter hidden samplers from API output + +// Goal 5: allow addition of custom samplers to be displayed +// Goal 6: send custom sampler values into prompt diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 166c72596..be75a9561 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -447,8 +447,9 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'unhide', ], helpString: 'Unhides a message from the prompt.', })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'disable', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-disable', callback: disableGroupMemberCallback, + aliases: ['disable', 'disablemember', 'memberdisable'], unnamedArgumentList: [ new SlashCommandArgument( 'member index or name', [ARGUMENT_TYPE.NUMBER, ARGUMENT_TYPE.STRING], true, @@ -456,7 +457,8 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'disable', ], helpString: 'Disables a group member from being drafted for replies.', })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'enable', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-enable', + aliases: ['enable', 'enablemember', 'memberenable'], callback: enableGroupMemberCallback, unnamedArgumentList: [ new SlashCommandArgument( @@ -465,9 +467,9 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'enable', ], helpString: 'Enables a group member to be drafted for replies.', })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberadd', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-add', callback: addGroupMemberCallback, - aliases: ['addmember'], + aliases: ['addmember', 'memberadd'], unnamedArgumentList: [ new SlashCommandArgument( 'character name', [ARGUMENT_TYPE.STRING], true, @@ -481,15 +483,15 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberadd', Example:
  • - 
    /memberadd John Doe
    + 
    /member-add John Doe
`, })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberremove', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-remove', callback: removeGroupMemberCallback, - aliases: ['removemember'], + aliases: ['removemember', 'memberremove'], unnamedArgumentList: [ new SlashCommandArgument( 'member index or name', [ARGUMENT_TYPE.NUMBER, ARGUMENT_TYPE.STRING], true, @@ -503,16 +505,16 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberremove Example:
  • - 
    /memberremove 2
    - 
    /memberremove John Doe
    + 
    /member-remove 2
    + 
    /member-remove John Doe
`, })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberup', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-up', callback: moveGroupMemberUpCallback, - aliases: ['upmember'], + aliases: ['upmember', 'memberup'], unnamedArgumentList: [ new SlashCommandArgument( 'member index or name', [ARGUMENT_TYPE.NUMBER, ARGUMENT_TYPE.STRING], true, @@ -520,9 +522,9 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberup', ], helpString: 'Moves a group member up in the group chat list.', })); -SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'memberdown', +SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'member-down', callback: moveGroupMemberDownCallback, - aliases: ['downmember'], + aliases: ['downmember', 'memberdown'], unnamedArgumentList: [ new SlashCommandArgument( 'member index or name', [ARGUMENT_TYPE.NUMBER, ARGUMENT_TYPE.STRING], true, @@ -702,6 +704,19 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'addswipe', })); SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'abort', callback: abortCallback, + namedArgumentList: [ + SlashCommandNamedArgument.fromProps({ name: 'quiet', + description: 'Whether to suppress the toast message notifying about the /abort call.', + typeList: [ARGUMENT_TYPE.BOOLEAN], + defaultValue: 'true', + enumList: ['true', 'false'], + }), + ], + unnamedArgumentList: [ + SlashCommandArgument.fromProps({ description: 'The reason for aborting command execution. Shown when quiet=false', + typeList: [ARGUMENT_TYPE.STRING], + }), + ], helpString: 'Aborts the slash command batch execution.', })); SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'fuzzy', @@ -847,6 +862,12 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'messages', new SlashCommandNamedArgument( 'names', 'show message author names', [ARGUMENT_TYPE.BOOLEAN], false, false, 'off', ['off', 'on'], ), + new SlashCommandNamedArgument( + 'hidden', 'include hidden messages', [ARGUMENT_TYPE.BOOLEAN], false, false, 'on', ['off', 'on'], + ), + new SlashCommandNamedArgument( + 'role', 'filter messages by role' , [ARGUMENT_TYPE.STRING], false, false, null, ['system', 'assistant', 'user'], + ), ], unnamedArgumentList: [ new SlashCommandArgument( @@ -858,6 +879,12 @@ SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'messages',
Returns the specified message or range of messages as a string.
+
+ Use the hidden=off argument to exclude hidden messages. +
+
+ Use the role argument to filter messages by role. Possible values are: system, assistant, user. +
Examples:
    @@ -1310,13 +1337,37 @@ async function popupCallback(args, value) { function getMessagesCallback(args, value) { const includeNames = !isFalseBoolean(args?.names); + const includeHidden = isTrueBoolean(args?.hidden); + const role = args?.role; const range = stringToRange(value, 0, chat.length - 1); if (!range) { - console.warn(`WARN: Invalid range provided for /getmessages command: ${value}`); + console.warn(`WARN: Invalid range provided for /messages command: ${value}`); return ''; } + const filterByRole = (mes) => { + if (!role) { + return true; + } + + const isNarrator = mes.extra?.type === system_message_types.NARRATOR; + + if (role === 'system') { + return isNarrator && !mes.is_user; + } + + if (role === 'assistant') { + return !isNarrator && !mes.is_user; + } + + if (role === 'user') { + return !isNarrator && mes.is_user; + } + + throw new Error(`Invalid role provided. Expected one of: system, assistant, user. Got: ${role}`); + }; + const messages = []; for (let messageId = range.start; messageId <= range.end; messageId++) { @@ -1326,7 +1377,13 @@ function getMessagesCallback(args, value) { continue; } - if (message.is_system) { + if (role && !filterByRole(message)) { + console.debug(`/messages: Skipping message with ID ${messageId} due to role filter`); + continue; + } + + if (!includeHidden && message.is_system) { + console.debug(`/messages: Skipping hidden message with ID ${messageId}`); continue; } @@ -1377,9 +1434,15 @@ async function runCallback(args, name) { } } -function abortCallback() { - $('#send_textarea').val('')[0].dispatchEvent(new Event('input', { bubbles:true })); - throw new Error('/abort command executed'); +/** + * + * @param {object} param0 + * @param {SlashCommandAbortController} param0._abortController + * @param {string} [param0.quiet] + * @param {string} [reason] + */ +function abortCallback({ _abortController, quiet }, reason) { + _abortController.abort((reason ?? '').toString().length == 0 ? '/abort command executed' : reason, !isFalseBoolean(quiet ?? 'true')); } async function delayCallback(_, amount) { @@ -2645,7 +2708,7 @@ const clearCommandProgressDebounced = debounce(clearCommandProgress); * @prop {boolean} [handleParserErrors] (true) Whether to handle parser errors (show toast on error) or throw. * @prop {SlashCommandScope} [scope] (null) The scope to be used when executing the commands. * @prop {boolean} [handleExecutionErrors] (false) Whether to handle execution errors (show toast on error) or throw - * @prop {PARSER_FLAG[]} [parserFlags] (null) Parser flags to apply + * @prop {{[id:PARSER_FLAG]:boolean}} [parserFlags] (null) Parser flags to apply * @prop {SlashCommandAbortController} [abortController] (null) Controller used to abort or pause command execution * @prop {(done:number, total:number)=>void} [onProgress] (null) Callback to handle progress events */ @@ -2653,7 +2716,7 @@ const clearCommandProgressDebounced = debounce(clearCommandProgress); /** * @typedef ExecuteSlashCommandsOnChatInputOptions * @prop {SlashCommandScope} [scope] (null) The scope to be used when executing the commands. - * @prop {PARSER_FLAG[]} [parserFlags] (null) Parser flags to apply + * @prop {{[id:PARSER_FLAG]:boolean}} [parserFlags] (null) Parser flags to apply * @prop {boolean} [clearChatInput] (false) Whether to clear the chat input textarea */ @@ -2705,10 +2768,12 @@ export async function executeSlashCommandsOnChatInput(text, options = {}) { } } catch (e) { document.querySelector('#form_sheld').classList.add('script_error'); - toastr.error(e.message); result = new SlashCommandClosureResult(); result.isError = true; result.errorMessage = e.message; + if (e.cause !== 'abort') { + toastr.error(e.message); + } } finally { delay(1000).then(()=>clearCommandProgressDebounced()); @@ -2740,7 +2805,7 @@ async function executeSlashCommandsWithOptions(text, options = {}) { let closure; try { - closure = parser.parse(text, true, options.parserFlags, options.abortController); + closure = parser.parse(text, true, options.parserFlags, options.abortController ?? new SlashCommandAbortController()); closure.scope.parent = options.scope; closure.onProgress = options.onProgress; } catch (e) { @@ -2767,8 +2832,9 @@ async function executeSlashCommandsWithOptions(text, options = {}) { try { const result = await closure.execute(); - if (result.isAborted) { + if (result.isAborted && !result.isQuietlyAborted) { toastr.warning(result.abortReason, 'Command execution aborted'); + closure.abortController.signal.isQuiet = true; } return result; } catch (e) { diff --git a/public/scripts/slash-commands/SlashCommand.js b/public/scripts/slash-commands/SlashCommand.js index 905967373..182c31882 100644 --- a/public/scripts/slash-commands/SlashCommand.js +++ b/public/scripts/slash-commands/SlashCommand.js @@ -1,5 +1,25 @@ +import { SlashCommandAbortController } from './SlashCommandAbortController.js'; import { SlashCommandArgument, SlashCommandNamedArgument } from './SlashCommandArgument.js'; import { SlashCommandClosure } from './SlashCommandClosure.js'; +import { PARSER_FLAG } from './SlashCommandParser.js'; +import { SlashCommandScope } from './SlashCommandScope.js'; + + + + +/** + * @typedef {{ + * _pipe:string|SlashCommandClosure, + * _scope:SlashCommandScope, + * _parserFlags:{[id:PARSER_FLAG]:boolean}, + * _abortController:SlashCommandAbortController, + * [id:string]:string|SlashCommandClosure, + * }} NamedArguments + */ + +/** + * @typedef {string|SlashCommandClosure|(string|SlashCommandClosure)[]} UnnamedArguments +*/ @@ -8,7 +28,7 @@ export class SlashCommand { * Creates a SlashCommand from a properties object. * @param {Object} props * @param {string} [props.name] - * @param {(namedArguments:Object., unnamedArguments:string|SlashCommandClosure|(string|SlashCommandClosure)[])=>string|SlashCommandClosure|void|Promise} [props.callback] + * @param {(namedArguments:NamedArguments, unnamedArguments:string|SlashCommandClosure|(string|SlashCommandClosure)[])=>string|SlashCommandClosure|void|Promise} [props.callback] * @param {string} [props.helpString] * @param {boolean} [props.splitUnnamedArgument] * @param {string[]} [props.aliases] @@ -25,7 +45,7 @@ export class SlashCommand { /**@type {string}*/ name; - /**@type {(namedArguments:Object, unnamedArguments:string|SlashCommandClosure|(string|SlashCommandClosure)[])=>string|SlashCommandClosure|Promise}*/ callback; + /**@type {(namedArguments:{_pipe:string|SlashCommandClosure, _scope:SlashCommandScope, _abortController:SlashCommandAbortController, [id:string]:string|SlashCommandClosure}, unnamedArguments:string|SlashCommandClosure|(string|SlashCommandClosure)[])=>string|SlashCommandClosure|Promise}*/ callback; /**@type {string}*/ helpString; /**@type {boolean}*/ splitUnnamedArgument = false; /**@type {string[]}*/ aliases = []; diff --git a/public/scripts/slash-commands/SlashCommandAbortController.js b/public/scripts/slash-commands/SlashCommandAbortController.js index f65bbb14d..7ed919f8c 100644 --- a/public/scripts/slash-commands/SlashCommandAbortController.js +++ b/public/scripts/slash-commands/SlashCommandAbortController.js @@ -5,7 +5,8 @@ export class SlashCommandAbortController { constructor() { this.signal = new SlashCommandAbortSignal(); } - abort(reason = 'No reason.') { + abort(reason = 'No reason.', isQuiet = false) { + this.signal.isQuiet = isQuiet; this.signal.aborted = true; this.signal.reason = reason; } @@ -20,8 +21,8 @@ export class SlashCommandAbortController { } export class SlashCommandAbortSignal { + /**@type {boolean}*/ isQuiet = false; /**@type {boolean}*/ paused = false; /**@type {boolean}*/ aborted = false; /**@type {string}*/ reason = null; - } diff --git a/public/scripts/slash-commands/SlashCommandAutoCompleteNameResult.js b/public/scripts/slash-commands/SlashCommandAutoCompleteNameResult.js index c6bd0d1fb..8a9c08653 100644 --- a/public/scripts/slash-commands/SlashCommandAutoCompleteNameResult.js +++ b/public/scripts/slash-commands/SlashCommandAutoCompleteNameResult.js @@ -50,6 +50,17 @@ export class SlashCommandAutoCompleteNameResult extends AutoCompleteNameResult { } getNamedArgumentAt(text, index, isSelect) { + function getSplitRegex() { + try { + return new RegExp('(?<==)'); + } catch { + // For browsers that don't support lookbehind + return new RegExp('=(.*)'); + } + } + if (!Array.isArray(this.executor.command?.namedArgumentList)) { + return null; + } const notProvidedNamedArguments = this.executor.command.namedArgumentList.filter(arg=>!this.executor.namedArgumentList.find(it=>it.name == arg.name)); let name; let value; @@ -62,7 +73,7 @@ export class SlashCommandAutoCompleteNameResult extends AutoCompleteNameResult { // cursor is somewhere within the named arguments (including final space) argAssign = this.executor.namedArgumentList.find(it=>it.start <= index && it.end >= index); if (argAssign) { - const [argName, ...v] = text.slice(argAssign.start, index).split(/(?<==)/); + const [argName, ...v] = text.slice(argAssign.start, index).split(getSplitRegex()); name = argName; value = v.join(''); start = argAssign.start; @@ -99,7 +110,7 @@ export class SlashCommandAutoCompleteNameResult extends AutoCompleteNameResult { const result = new AutoCompleteSecondaryNameResult( value, start + name.length, - cmdArg.enumList.map(it=>new SlashCommandEnumAutoCompleteOption(it)), + cmdArg.enumList.map(it=>new SlashCommandEnumAutoCompleteOption(this.executor.command, it)), true, ); result.isRequired = true; @@ -122,6 +133,9 @@ export class SlashCommandAutoCompleteNameResult extends AutoCompleteNameResult { } getUnnamedArgumentAt(text, index, isSelect) { + if (!Array.isArray(this.executor.command?.unnamedArgumentList)) { + return null; + } const lastArgIsBlank = this.executor.unnamedArgumentList.slice(-1)[0]?.value == ''; const notProvidedArguments = this.executor.command.unnamedArgumentList.slice(this.executor.unnamedArgumentList.length - (lastArgIsBlank ? 1 : 0)); let value; @@ -154,7 +168,7 @@ export class SlashCommandAutoCompleteNameResult extends AutoCompleteNameResult { const result = new AutoCompleteSecondaryNameResult( value, start, - cmdArg.enumList.map(it=>new SlashCommandEnumAutoCompleteOption(it)), + cmdArg.enumList.map(it=>new SlashCommandEnumAutoCompleteOption(this.executor.command, it)), false, ); const isCompleteValue = cmdArg.enumList.find(it=>it.value == value); diff --git a/public/scripts/slash-commands/SlashCommandClosure.js b/public/scripts/slash-commands/SlashCommandClosure.js index ce3ec783d..27f81f38d 100644 --- a/public/scripts/slash-commands/SlashCommandClosure.js +++ b/public/scripts/slash-commands/SlashCommandClosure.js @@ -161,6 +161,7 @@ export class SlashCommandClosure { let args = { _scope: this.scope, _parserFlags: executor.parserFlags, + _abortController: this.abortController, }; let value; // substitute named arguments @@ -223,6 +224,15 @@ export class SlashCommandClosure { ?.replace(/\\\{/g, '{') ?.replace(/\\\}/g, '}') ; + } else if (Array.isArray(value)) { + value = value.map(v=>{ + if (typeof v == 'string') { + return v + ?.replace(/\\\{/g, '{') + ?.replace(/\\\}/g, '}'); + } + return v; + }); } let abortResult = await this.testAbortController(); @@ -254,6 +264,7 @@ export class SlashCommandClosure { if (this.abortController?.signal?.aborted) { const result = new SlashCommandClosureResult(); result.isAborted = true; + result.isQuietlyAborted = this.abortController.signal.isQuiet; result.abortReason = this.abortController.signal.reason.toString(); return result; } diff --git a/public/scripts/slash-commands/SlashCommandClosureResult.js b/public/scripts/slash-commands/SlashCommandClosureResult.js index 4de4926df..740d09a9d 100644 --- a/public/scripts/slash-commands/SlashCommandClosureResult.js +++ b/public/scripts/slash-commands/SlashCommandClosureResult.js @@ -2,6 +2,7 @@ export class SlashCommandClosureResult { /**@type {boolean}*/ interrupt = false; /**@type {string}*/ pipe; /**@type {boolean}*/ isAborted = false; + /**@type {boolean}*/ isQuietlyAborted = false; /**@type {string}*/ abortReason; /**@type {boolean}*/ isError = false; /**@type {string}*/ errorMessage; diff --git a/public/scripts/slash-commands/SlashCommandEnumAutoCompleteOption.js b/public/scripts/slash-commands/SlashCommandEnumAutoCompleteOption.js index 9d9a5d40d..12d9c64ad 100644 --- a/public/scripts/slash-commands/SlashCommandEnumAutoCompleteOption.js +++ b/public/scripts/slash-commands/SlashCommandEnumAutoCompleteOption.js @@ -1,16 +1,20 @@ import { AutoCompleteOption } from '../autocomplete/AutoCompleteOption.js'; +import { SlashCommand } from './SlashCommand.js'; import { SlashCommandEnumValue } from './SlashCommandEnumValue.js'; export class SlashCommandEnumAutoCompleteOption extends AutoCompleteOption { + /**@type {SlashCommand}*/ cmd; /**@type {SlashCommandEnumValue}*/ enumValue; /** + * @param {SlashCommand} cmd * @param {SlashCommandEnumValue} enumValue */ - constructor(enumValue) { + constructor(cmd, enumValue) { super(enumValue.value, '◊'); + this.cmd = cmd; this.enumValue = enumValue; } @@ -25,22 +29,6 @@ export class SlashCommandEnumAutoCompleteOption extends AutoCompleteOption { renderDetails() { - const frag = document.createDocumentFragment(); - const specs = document.createElement('div'); { - specs.classList.add('specs'); - const name = document.createElement('div'); { - name.classList.add('name'); - name.classList.add('monospace'); - name.textContent = this.name; - specs.append(name); - } - frag.append(specs); - } - const help = document.createElement('span'); { - help.classList.add('help'); - help.textContent = this.enumValue.description; - frag.append(help); - } - return frag; + return this.cmd.renderHelpDetails(); } } diff --git a/public/scripts/slash-commands/SlashCommandExecutor.js b/public/scripts/slash-commands/SlashCommandExecutor.js index 2cf0c4647..f64c37aff 100644 --- a/public/scripts/slash-commands/SlashCommandExecutor.js +++ b/public/scripts/slash-commands/SlashCommandExecutor.js @@ -20,7 +20,7 @@ export class SlashCommandExecutor { // @ts-ignore /**@type {SlashCommandNamedArgumentAssignment[]}*/ namedArgumentList = []; /**@type {SlashCommandUnnamedArgumentAssignment[]}*/ unnamedArgumentList = []; - /**@type {Object} */ parserFlags; + /**@type {{[id:PARSER_FLAG]:boolean}} */ parserFlags; get commandCount() { return 1 diff --git a/public/scripts/slash-commands/SlashCommandNamedArgumentAutoCompleteOption.js b/public/scripts/slash-commands/SlashCommandNamedArgumentAutoCompleteOption.js index f6b114a25..29f4867fd 100644 --- a/public/scripts/slash-commands/SlashCommandNamedArgumentAutoCompleteOption.js +++ b/public/scripts/slash-commands/SlashCommandNamedArgumentAutoCompleteOption.js @@ -27,22 +27,6 @@ export class SlashCommandNamedArgumentAutoCompleteOption extends AutoCompleteOpt renderDetails() { - const frag = document.createDocumentFragment(); - const specs = document.createElement('div'); { - specs.classList.add('specs'); - const name = document.createElement('div'); { - name.classList.add('name'); - name.classList.add('monospace'); - name.textContent = this.name; - specs.append(name); - } - frag.append(specs); - } - const help = document.createElement('span'); { - help.classList.add('help'); - help.innerHTML = `${this.arg.isRequired ? '' : '(optional) '}${this.arg.description ?? ''}`; - frag.append(help); - } - return frag; + return this.cmd.renderHelpDetails(); } } diff --git a/public/scripts/slash-commands/SlashCommandParser.js b/public/scripts/slash-commands/SlashCommandParser.js index e886df0db..171a38491 100644 --- a/public/scripts/slash-commands/SlashCommandParser.js +++ b/public/scripts/slash-commands/SlashCommandParser.js @@ -114,7 +114,6 @@ export class SlashCommandParser { constructor() { - //TODO should not be re-registered from every instance // add dummy commands for help strings / autocomplete if (!Object.keys(this.commands).includes('parser-flag')) { const help = {}; @@ -186,6 +185,15 @@ export class SlashCommandParser { relevance: 0, }; + function getQuotedRunRegex() { + try { + return new RegExp('(".+?(?it.start <= index && it.end >= index); - console.log(macro); if (macro) { const frag = document.createRange().createContextualFragment(await (await fetch('/scripts/templates/macros.html')).text()); const options = [...frag.querySelectorAll('ul:nth-of-type(2n+1) > li')].map(li=>new MacroAutoCompleteOption( @@ -821,6 +828,7 @@ export class SlashCommandParser { assignment.start = this.index; value = ''; } + assignment.start = this.index; assignment.value = this.parseClosure(); assignment.end = this.index; listValues.push(assignment); diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index 9e335600d..8d7fe9299 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -174,7 +174,7 @@ async function* parseStreamData(json) { else if (Array.isArray(json.choices)) { const isNotPrimary = json?.choices?.[0]?.index > 0; if (isNotPrimary || json.choices.length === 0) { - return null; + throw new Error('Not a primary swipe'); } if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { @@ -271,7 +271,7 @@ export class SmoothEventSourceStream extends EventSourceStream { hasFocus && await eventSource.emit(event_types.SMOOTH_STREAM_TOKEN_RECEIVED, parsed.chunk); } } catch (error) { - console.error('Smooth Streaming parsing error', error); + console.debug('Smooth Streaming parsing error', error); controller.enqueue(event); } }, diff --git a/public/scripts/tags.js b/public/scripts/tags.js index 64e83428c..f39e1e926 100644 --- a/public/scripts/tags.js +++ b/public/scripts/tags.js @@ -14,9 +14,12 @@ import { // eslint-disable-next-line no-unused-vars import { FILTER_TYPES, FILTER_STATES, DEFAULT_FILTER_STATE, isFilterState, FilterHelper } from './filters.js'; -import { groupCandidatesFilter, groups, selected_group } from './group-chats.js'; +import { groupCandidatesFilter, groups, select_group_chats, selected_group } from './group-chats.js'; import { download, onlyUnique, parseJsonFile, uuidv4, getSortableDelay, flashHighlight } from './utils.js'; import { power_user } from './power-user.js'; +import { SlashCommandParser } from './slash-commands/SlashCommandParser.js'; +import { SlashCommand } from './slash-commands/SlashCommand.js'; +import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js'; export { TAG_FOLDER_TYPES, @@ -63,7 +66,7 @@ export const tag_filter_types = { const ACTIONABLE_TAGS = { FAV: { id: '1', sort_order: 1, name: 'Show only favorites', color: 'rgba(255, 255, 0, 0.5)', action: filterByFav, icon: 'fa-solid fa-star', class: 'filterByFavorites' }, GROUP: { id: '0', sort_order: 2, name: 'Show only groups', color: 'rgba(100, 100, 100, 0.5)', action: filterByGroups, icon: 'fa-solid fa-users', class: 'filterByGroups' }, - FOLDER: { id: '4', sort_order: 3, name: 'Always show folders', color: 'rgba(120, 120, 120, 0.5)', action: filterByFolder, icon: 'fa-solid fa-folder-plus', class: 'filterByFolder' }, + FOLDER: { id: '4', sort_order: 3, name: 'Show only folders', color: 'rgba(120, 120, 120, 0.5)', action: filterByFolder, icon: 'fa-solid fa-folder-plus', class: 'filterByFolder' }, VIEW: { id: '2', sort_order: 4, name: 'Manage tags', color: 'rgba(150, 100, 100, 0.5)', action: onViewTagsListClick, icon: 'fa-solid fa-gear', class: 'manageTags' }, HINT: { id: '3', sort_order: 5, name: 'Show Tag List', color: 'rgba(150, 100, 100, 0.5)', action: onTagListHintClick, icon: 'fa-solid fa-tags', class: 'showTagList' }, UNFILTER: { id: '5', sort_order: 6, name: 'Clear all filters', action: onClearAllFiltersClick, icon: 'fa-solid fa-filter-circle-xmark', class: 'clearAllFilters' }, @@ -174,9 +177,8 @@ function filterByTagState(entities, { globalDisplayFilters = false, subForEntity } // Hide folders that have 0 visible sub entities after the first filtering round - const alwaysFolder = isFilterState(entitiesFilter.getFilterData(FILTER_TYPES.FOLDER), FILTER_STATES.SELECTED); if (entity.type === 'tag') { - return alwaysFolder || entity.entities.length > 0; + return entity.entities.length > 0; } return true; @@ -322,6 +324,13 @@ function filterByGroups(filterHelper) { * @param {FilterHelper} filterHelper Instance of FilterHelper class. */ function filterByFolder(filterHelper) { + if (!power_user.bogus_folders) { + $('#bogus_folders').prop('checked', true).trigger('input'); + onViewTagsListClick(); + flashHighlight($('#dialogue_popup .tag_as_folder, #dialogue_popup .tag_folder_indicator')); + return; + } + const state = toggleTagThreeState($(this)); ACTIONABLE_TAGS.FOLDER.filter_state = state; filterHelper.setFilterData(FILTER_TYPES.FOLDER, state); @@ -350,7 +359,7 @@ function createTagMapFromList(listElement, key) { * If you have an entity, you can get it's key easily via `getTagKeyForEntity(entity)`. * * @param {string} key - The key for which to get tags via the tag map - * @param {boolean} [sort=true] - + * @param {boolean} [sort=true] - Whether the tag list should be sorted * @returns {Tag[]} A list of tags */ function getTagsList(key, sort = true) { @@ -456,35 +465,122 @@ export function getTagKeyForEntityElement(element) { return undefined; } +/** + * Adds a tag to a given entity + * @param {Tag} tag - The tag to add + * @param {string|string[]} entityId - The entity to add this tag to. Has to be the entity key (e.g. `addTagToEntity`). (Also allows multiple entities to be passed in) + * @param {object} [options={}] - Optional arguments + * @param {JQuery|string?} [options.tagListSelector=null] - An optional selector if a specific list should be updated with the new tag too (for example because the add was triggered for that function) + * @param {PrintTagListOptions} [options.tagListOptions] - Optional parameters for printing the tag list. Can be set to be consistent with the expected behavior of tags in the list that was defined before. + * @returns {boolean} Whether at least one tag was added + */ +export function addTagToEntity(tag, entityId, { tagListSelector = null, tagListOptions = {} } = {}) { + let result = false; + // Add tags to the map + if (Array.isArray(entityId)) { + entityId.forEach((id) => result = addTagToMap(tag.id, id) || result); + } else { + result = addTagToMap(tag.id, entityId); + } + + // Save and redraw + printCharactersDebounced(); + saveSettingsDebounced(); + + // We should manually add the selected tag to the print tag function, so we cover places where the tag list did not automatically include it + tagListOptions.addTag = tag; + + // add tag to the UI and internal map - we reprint so sorting and new markup is done correctly + if (tagListSelector) printTagList(tagListSelector, tagListOptions); + const inlineSelector = getInlineListSelector(); + if (inlineSelector) { + printTagList($(inlineSelector), tagListOptions); + } + + return result; +} + +/** + * Removes a tag from a given entity + * @param {Tag} tag - The tag to remove + * @param {string|string[]} entityId - The entity to remove this tag from. Has to be the entity key (e.g. `addTagToEntity`). (Also allows multiple entities to be passed in) + * @param {object} [options={}] - Optional arguments + * @param {JQuery|string?} [options.tagListSelector=null] - An optional selector if a specific list should be updated with the tag removed too (for example because the add was triggered for that function) + * @param {JQuery?} [options.tagElement=null] - Optionally a direct html element of the tag to be removed, so it can be removed from the UI + * @returns {boolean} Whether at least one tag was removed + */ +export function removeTagFromEntity(tag, entityId, { tagListSelector = null, tagElement = null } = {}) { + let result = false; + // Remove tag from the map + if (Array.isArray(entityId)) { + entityId.forEach((id) => result = removeTagFromMap(tag.id, id) || result); + } else { + result = removeTagFromMap(tag.id, entityId); + } + + // Save and redraw + printCharactersDebounced(); + saveSettingsDebounced(); + + // We don't reprint the lists, we can just remove the html elements from them. + if (tagListSelector) { + const $selector = (typeof tagListSelector === 'string') ? $(tagListSelector) : tagListSelector; + $selector.find(`.tag[id="${tag.id}"]`).remove(); + } + if (tagElement) tagElement.remove(); + $(`${getInlineListSelector()} .tag[id="${tag.id}"]`).remove(); + + return result; +} + +/** + * Adds a tag from a given character. If no character is provided, adds it from the currently active one. + * @param {string} tagId - The id of the tag + * @param {string} characterId - The id/key of the character or group + * @returns {boolean} Whether the tag was added or not + */ function addTagToMap(tagId, characterId = null) { const key = characterId !== null && characterId !== undefined ? getTagKeyForEntity(characterId) : getTagKey(); if (!key) { - return; + return false; } if (!Array.isArray(tag_map[key])) { tag_map[key] = [tagId]; + return true; } else { + if (tag_map[key].includes(tagId)) + return false; + tag_map[key].push(tagId); tag_map[key] = tag_map[key].filter(onlyUnique); + return true; } } +/** + * Removes a tag from a given character. If no character is provided, removes it from the currently active one. + * @param {string} tagId - The id of the tag + * @param {string} characterId - The id/key of the character or group + * @returns {boolean} Whether the tag was removed or not + */ function removeTagFromMap(tagId, characterId = null) { const key = characterId !== null && characterId !== undefined ? getTagKeyForEntity(characterId) : getTagKey(); if (!key) { - return; + return false; } if (!Array.isArray(tag_map[key])) { tag_map[key] = []; + return false; } else { const indexOf = tag_map[key].indexOf(tagId); tag_map[key].splice(indexOf, 1); + return indexOf !== -1; } } @@ -528,24 +624,7 @@ function selectTag(event, ui, listSelector, { tagListOptions = {} } = {}) { const characterData = event.target.closest('#bulk_tags_div')?.dataset.characters; const characterIds = characterData ? JSON.parse(characterData).characterIds : null; - if (characterIds) { - characterIds.forEach((characterId) => addTagToMap(tag.id, characterId)); - } else { - addTagToMap(tag.id); - } - - printCharactersDebounced(); - saveSettingsDebounced(); - - // We should manually add the selected tag to the print tag function, so we cover places where the tag list did not automatically include it - tagListOptions.addTag = tag; - - // add tag to the UI and internal map - we reprint so sorting and new markup is done correctly - printTagList(listSelector, tagListOptions); - const inlineSelector = getInlineListSelector(); - if (inlineSelector) { - printTagList($(inlineSelector), tagListOptions); - } + addTagToEntity(tag, characterIds, { tagListSelector: listSelector, tagListOptions: tagListOptions }); // need to return false to keep the input clear return false; @@ -628,6 +707,7 @@ function createNewTag(tagName) { create_date: Date.now(), }; tags.push(tag); + console.debug('Created new tag', tag.name, 'with id', tag.id); return tag; } @@ -883,8 +963,9 @@ function printTagFilters(type = tag_filter_types.character) { const FILTER_SELECTOR = type === tag_filter_types.character ? CHARACTER_FILTER_SELECTOR : GROUP_FILTER_SELECTOR; $(FILTER_SELECTOR).empty(); - // Print all action tags. (Exclude folder if that setting isn't chosen) - const actionTags = Object.values(ACTIONABLE_TAGS).filter(tag => power_user.bogus_folders || tag.id != ACTIONABLE_TAGS.FOLDER.id); + // Print all action tags. (Rework 'Folder' button to some kind of onboarding if no folders are enabled yet) + const actionTags = Object.values(ACTIONABLE_TAGS); + actionTags.find(x => x == ACTIONABLE_TAGS.FOLDER).name = power_user.bogus_folders ? 'Show only folders' : 'Enable \'Tags as Folder\'\n\nAllows characters to be grouped in folders by their assigned tags.\nTags have to be explicitly chosen as folder to show up.\n\nClick here to start'; printTagList($(FILTER_SELECTOR), { empty: false, sort: false, tags: actionTags, tagActionSelector: tag => tag.action, tagOptions: { isGeneralList: true } }); const inListActionTags = Object.values(InListActionable); @@ -924,8 +1005,8 @@ function updateTagFilterIndicator() { function onTagRemoveClick(event) { event.stopPropagation(); - const tag = $(this).closest('.tag'); - const tagId = tag.attr('id'); + const tagElement = $(this).closest('.tag'); + const tagId = tagElement.attr('id'); // Check if we are inside the drilldown. If so, we call remove on the bogus folder if ($(this).closest('.rm_tag_bogus_drilldown').length > 0) { @@ -934,24 +1015,13 @@ function onTagRemoveClick(event) { return; } + const tag = tags.find(t => t.id === tagId); + // Optional, check for multiple character ids being present. const characterData = event.target.closest('#bulk_tags_div')?.dataset.characters; const characterIds = characterData ? JSON.parse(characterData).characterIds : null; - tag.remove(); - - if (characterIds) { - characterIds.forEach((characterId) => removeTagFromMap(tagId, characterId)); - } else { - removeTagFromMap(tagId); - } - - $(`${getInlineListSelector()} .tag[id="${tagId}"]`).remove(); - - printCharactersDebounced(); - saveSettingsDebounced(); - - + removeTagFromEntity(tag, characterIds, { tagElement: tagElement }); } // @ts-ignore @@ -977,7 +1047,7 @@ function onGroupCreateClick() { export function applyTagsOnCharacterSelect() { //clearTagsFilter(); - const chid = Number($(this).attr('chid')); + const chid = Number(this_chid); printTagList($('#tagList'), { forEntityOrKey: chid, tagOptions: { removable: true } }); } @@ -1453,14 +1523,200 @@ function printViewTagList(empty = true) { } } +function registerTagsSlashCommands() { + /** + * Gets the key for char/group for a slash command. If none can be found, a toastr will be shown and null returned. + * @param {string?} [charName] The optionally provided char name + * @returns {string?} - The char/group key, or null if none found + */ + function paraGetCharKey(charName) { + const entity = charName + ? (characters.find(x => x.name === charName) || groups.find(x => x.name == charName)) + : (selected_group ? groups.find(x => x.id == selected_group) : characters[this_chid]); + const key = getTagKeyForEntity(entity); + if (!key) { + toastr.warning(`Character ${charName} not found.`); + return null; + } + return key; + } + /** + * Gets a tag by its name. Optionally can create the tag if it does not exist. + * @param {string} tagName - The name of the tag + * @param {object} options - Optional arguments + * @param {boolean} [options.allowCreate=false] - Whether a new tag should be created if no tag with the name exists + * @returns {Tag?} The tag, or null if not found + */ + function paraGetTag(tagName, { allowCreate = false } = {}) { + if (!tagName) { + toastr.warning('Tag name must be provided.'); + return null; + } + let tag = tags.find(t => t.name === tagName); + if (allowCreate && !tag) { + tag = createNewTag(tagName); + } + if (!tag) { + toastr.warning(`Tag ${tagName} not found.`); + return null; + } + return tag; + } + + function updateTagsList() { + switch (menu_type) { + case 'characters': + printTagFilters(tag_filter_types.character); + printTagFilters(tag_filter_types.group_member); + break; + case 'character_edit': + applyTagsOnCharacterSelect(); + break; + case 'group_edit': + select_group_chats(selected_group, true); + break; + } + } + + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tag-add', + returns: 'true/false - Whether the tag was added or was assigned already', + /** @param {{name: string}} namedArgs @param {string} tagName @returns {string} */ + callback: ({ name }, tagName) => { + const key = paraGetCharKey(name); + if (!key) return 'false'; + const tag = paraGetTag(tagName, { allowCreate: true }); + if (!tag) return 'false'; + const result = addTagToEntity(tag, key); + updateTagsList(); + return String(result); + }, + namedArgumentList: [ + new SlashCommandNamedArgument('name', 'Character name', [ARGUMENT_TYPE.STRING], false, false, '{{char}}'), + ], + unnamedArgumentList: [ + new SlashCommandArgument('tag name', [ARGUMENT_TYPE.STRING], true), + ], + helpString: ` +
    + Adds a tag to the character. If no character is provided, it adds it to the current character ({{char}}). + If the tag doesn't exist, it is created. +
    +
    + Example: +
      +
    • + 
      /tag-add name="Chloe" scenario
      + will add the tag "scenario" to the character named Chloe. +
      • +
    +
    + `, + })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tag-remove', + returns: 'true/false - Whether the tag was removed or wasn\'t assigned already', + /** @param {{name: string}} namedArgs @param {string} tagName @returns {string} */ + callback: ({ name }, tagName) => { + const key = paraGetCharKey(name); + if (!key) return 'false'; + const tag = paraGetTag(tagName); + if (!tag) return 'false'; + const result = removeTagFromEntity(tag, key); + updateTagsList(); + return String(result); + }, + namedArgumentList: [ + new SlashCommandNamedArgument('name', 'Character name', [ARGUMENT_TYPE.STRING], false, false, '{{char}}'), + ], + unnamedArgumentList: [ + new SlashCommandArgument('tag name', [ARGUMENT_TYPE.STRING], true), + ], + helpString: ` +
    + Removes a tag from the character. If no character is provided, it removes it from the current character ({{char}}). +
    +
    + Example: +
      +
    • + 
      /tag-remove name="Chloe" scenario
      + will remove the tag "scenario" from the character named Chloe. +
      • +
    +
    + `, + })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tag-exists', + returns: 'true/false - Whether the given tag name is assigned to the character', + /** @param {{name: string}} namedArgs @param {string} tagName @returns {string} */ + callback: ({ name }, tagName) => { + const key = paraGetCharKey(name); + if (!key) return 'false'; + const tag = paraGetTag(tagName); + if (!tag) return 'false'; + return String(tag_map[key].includes(tag.id)); + }, + namedArgumentList: [ + new SlashCommandNamedArgument('name', 'Character name', [ARGUMENT_TYPE.STRING], false, false, '{{char}}'), + ], + unnamedArgumentList: [ + new SlashCommandArgument('tag name', [ARGUMENT_TYPE.STRING], true), + ], + helpString: ` +
    + Checks whether the given tag is assigned to the character. If no character is provided, it checks the current character ({{char}}). +
    +
    + Example: +
      +
    • + 
      /tag-exists name="Chloe" scenario
      + will return true if the character named Chloe has the tag "scenario". +
      • +
    +
    + `, + })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tag-list', + returns: 'Comma-separated list of all assigned tags', + /** @param {{name: string}} namedArgs @returns {string} */ + callback: ({ name }) => { + const key = paraGetCharKey(name); + if (!key) return ''; + const tags = getTagsList(key); + return tags.map(x => x.name).join(', '); + }, + namedArgumentList: [ + new SlashCommandNamedArgument('name', 'Character name', [ARGUMENT_TYPE.STRING], false, false, '{{char}}'), + ], + helpString: ` +
    + Lists all assigned tags of the character. If no character is provided, it uses the current character ({{char}}). +
    + Note that there is no special handling for tags containing commas, they will be printed as-is. +
    +
    + Example: +
      +
    • + 
      /tag-list name="Chloe"
      + could return something like OC, scenario, edited, funny +
      • +
    +
    + `, + })); +} + export function initTags() { createTagInput('#tagInput', '#tagList', { tagOptions: { removable: true } }); createTagInput('#groupTagInput', '#groupTagList', { tagOptions: { removable: true } }); $(document).on('click', '#rm_button_create', onCharacterCreateClick); $(document).on('click', '#rm_button_group_chats', onGroupCreateClick); - $(document).on('click', '.character_select', applyTagsOnCharacterSelect); - $(document).on('click', '.group_select', applyTagsOnGroupSelect); $(document).on('click', '.tag_remove', onTagRemoveClick); $(document).on('input', '.tag_input', onTagInput); $(document).on('click', '.tags_view', onViewTagsListClick); @@ -1471,6 +1727,7 @@ export function initTags() { $(document).on('click', '.tag_view_backup', onTagsBackupClick); $(document).on('click', '.tag_view_restore', onBackupRestoreClick); eventSource.on(event_types.CHARACTER_DUPLICATED, copyTags); + eventSource.makeFirst(event_types.CHAT_CHANGED, () => selected_group ? applyTagsOnGroupSelect() : applyTagsOnCharacterSelect()); $(document).on('input', '#dialogue_popup input[name="auto_sort_tags"]', (evt) => { const toggle = $(evt.target).is(':checked'); @@ -1498,4 +1755,6 @@ export function initTags() { printCharactersDebounced(); } } + + registerTagsSlashCommands(); } diff --git a/public/scripts/templates/welcome.html b/public/scripts/templates/welcome.html index 6082b2cde..a950c5a11 100644 --- a/public/scripts/templates/welcome.html +++ b/public/scripts/templates/welcome.html @@ -11,9 +11,10 @@ Click and select a Chat API.
  • - Click and pick a character + Click and pick a character.
    • +You can browse a list of bundled characters in the Download Extensions & Assets menu within .

    Confused or lost?

      diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index 293dffe43..d8f36cf45 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -237,7 +237,7 @@ function onMancerModelSelect() { $('#api_button_textgenerationwebui').trigger('click'); const limits = mancerModels.find(x => x.id === modelId)?.limits; - setGenerationParamsFromPreset({ max_length: limits.context, genamt: limits.completion }, true); + setGenerationParamsFromPreset({ max_length: limits.context }); } function onTogetherModelSelect() { diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index 55b4f1c19..8b7a438d6 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -166,7 +166,7 @@ export let textgenerationwebui_banned_in_macros = []; export let textgenerationwebui_presets = []; export let textgenerationwebui_preset_names = []; -const setting_names = [ +export const setting_names = [ 'temp', 'temperature_last', 'rep_pen', @@ -659,7 +659,7 @@ jQuery(function () { 'no_repeat_ngram_size_textgenerationwebui': 0, 'min_length_textgenerationwebui': 0, 'num_beams_textgenerationwebui': 1, - 'length_penalty_textgenerationwebui': 0, + 'length_penalty_textgenerationwebui': 1, 'penalty_alpha_textgenerationwebui': 0, 'typical_p_textgenerationwebui': 1, // Added entry 'guidance_scale_textgenerationwebui': 1, @@ -991,7 +991,7 @@ export function getTextGenModel() { } export function isJsonSchemaSupported() { - return settings.type === TABBY && main_api === 'textgenerationwebui'; + return [TABBY, LLAMACPP].includes(settings.type) && main_api === 'textgenerationwebui'; } export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) { @@ -1065,7 +1065,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, 'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1, 'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(settings.negative_prompt) ?? '', 'grammar_string': settings.grammar_string, - 'json_schema': settings.type === TABBY ? settings.json_schema : undefined, + 'json_schema': [TABBY, LLAMACPP].includes(settings.type) ? settings.json_schema : undefined, // llama.cpp aliases. In case someone wants to use LM Studio as Text Completion API 'repeat_penalty': settings.rep_pen, 'tfs_z': settings.tfs, @@ -1150,5 +1150,15 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, eventSource.emitAndWait(event_types.TEXT_COMPLETION_SETTINGS_READY, params); + // Grammar conflicts with with json_schema + if (settings.type === LLAMACPP) { + if (params.json_schema && Object.keys(params.json_schema).length > 0) { + delete params.grammar_string; + delete params.grammar; + } else { + delete params.json_schema; + } + } + return params; } diff --git a/public/scripts/utils.js b/public/scripts/utils.js index 2a5ab9752..d62310b3c 100644 --- a/public/scripts/utils.js +++ b/public/scripts/utils.js @@ -732,6 +732,24 @@ export function isDataURL(str) { return regex.test(str); } +/** + * Gets the size of an image from a data URL. + * @param {string} dataUrl Image data URL + * @returns {Promise<{ width: number, height: number }>} Image size + */ +export function getImageSizeFromDataURL(dataUrl) { + const image = new Image(); + image.src = dataUrl; + return new Promise((resolve, reject) => { + image.onload = function () { + resolve({ width: image.width, height: image.height }); + }; + image.onerror = function () { + reject(new Error('Failed to load image')); + }; + }); +} + export function getCharaFilename(chid) { const context = getContext(); const fileName = context.characters[chid ?? context.characterId].avatar; @@ -773,6 +791,29 @@ export function escapeRegex(string) { return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&'); } +/** + * Instantiates a regular expression from a string. + * @param {string} input The input string. + * @returns {RegExp} The regular expression instance. + * @copyright Originally from: https://github.com/IonicaBizau/regex-parser.js/blob/master/lib/index.js + */ +export function regexFromString(input) { + try { + // Parse input + var m = input.match(/(\/?)(.+)\1([a-z]*)/i); + + // Invalid flags + if (m[3] && !/^(?!.*?(.).*?\1)[gmixXsuUAJ]+$/.test(m[3])) { + return RegExp(input); + } + + // Create the regular expression + return new RegExp(m[2], m[3]); + } catch { + return; + } +} + export class Stopwatch { /** * Initializes a Stopwatch class. @@ -1449,3 +1490,166 @@ export function includesIgnoreCaseAndAccents(text, searchTerm) { // Check if the normalized text includes the normalized search term return normalizedText.includes(normalizedSearchTerm); } + +/** + * @typedef {object} Select2Option The option object for select2 controls + * @property {string} id - The unique ID inside this select + * @property {string} text - The text for this option + * @property {number?} [count] - Optionally show the count how often that option was chosen already + */ + +/** + * Returns a unique hash as ID for a select2 option text + * + * @param {string} option - The option + * @returns {string} A hashed version of that option + */ +export function getSelect2OptionId(option) { + return String(getStringHash(option)); +} + +/** + * Modifies the select2 options by adding not existing one and optionally selecting them + * + * @param {JQuery} element - The "select" element to add the options to + * @param {string[]|Select2Option[]} items - The option items to build, add or select + * @param {object} [options] - Optional arguments + * @param {boolean} [options.select=false] - Whether the options should be selected right away + * @param {object} [options.changeEventArgs=null] - Optional event args being passed into the "change" event when its triggered because a new options is selected + */ +export function select2ModifyOptions(element, items, { select = false, changeEventArgs = null } = {}) { + if (!items.length) return; + /** @type {Select2Option[]} */ + const dataItems = items.map(x => typeof x === 'string' ? { id: getSelect2OptionId(x), text: x } : x); + + const existingValues = []; + dataItems.forEach(item => { + // Set the value, creating a new option if necessary + if (element.find('option[value=\'' + item.id + '\']').length) { + if (select) existingValues.push(item.id); + } else { + // Create a DOM Option and optionally pre-select by default + var newOption = new Option(item.text, item.id, select, select); + // Append it to the select + element.append(newOption); + if (select) element.trigger('change', changeEventArgs); + } + if (existingValues.length) element.val(existingValues).trigger('change', changeEventArgs); + }); +} + +/** + * Returns the ajax settings that can be used on the select2 ajax property to dynamically get the data. + * Can be used on a single global array, querying data from the server or anything similar. + * + * @param {function():Select2Option[]} dataProvider - The provider/function to retrieve the data - can be as simple as "() => myData" for arrays + * @return {{transport: (params, success, failure) => any}} The ajax object with the transport function to use on the select2 ajax property + */ +export function dynamicSelect2DataViaAjax(dataProvider) { + function dynamicSelect2DataTransport(params, success, failure) { + var items = dataProvider(); + // fitering if params.data.q available + if (params.data && params.data.q) { + items = items.filter(function (item) { + return includesIgnoreCaseAndAccents(item.text, params.data.q); + }); + } + var promise = new Promise(function (resolve, reject) { + resolve({ results: items }); + }); + promise.then(success); + promise.catch(failure); + } + const ajax = { + transport: dynamicSelect2DataTransport, + }; + return ajax; +} + +/** + * Checks whether a given control is a select2 choice element - meaning one of the results being displayed in the select multi select box + * @param {JQuery|HTMLElement} element - The element to check + * @returns {boolean} Whether this is a choice element + */ +export function isSelect2ChoiceElement(element) { + const $element = $(element); + return ($element.hasClass('select2-selection__choice__display') || $element.parents('.select2-selection__choice__display').length > 0); +} + +/** + * Subscribes a 'click' event handler to the choice elements of a select2 multi-select control + * + * @param {JQuery} control The original control the select2 was applied to + * @param {function(HTMLElement):void} action - The action to execute when a choice element is clicked + * @param {object} options - Optional parameters + * @param {boolean} [options.buttonStyle=false] - Whether the choices should be styles as a clickable button with color and hover transition, instead of just changed cursor + * @param {boolean} [options.closeDrawer=false] - Whether the drawer should be closed and focus removed after the choice item was clicked + * @param {boolean} [options.openDrawer=false] - Whether the drawer should be opened, even if this click would normally close it + */ +export function select2ChoiceClickSubscribe(control, action, { buttonStyle = false, closeDrawer = false, openDrawer = false } = {}) { + // Add class for styling (hover color, changed cursor, etc) + control.addClass('select2_choice_clickable'); + if (buttonStyle) control.addClass('select2_choice_clickable_buttonstyle'); + + // Get the real container below and create a click handler on that one + const select2Container = control.next('span.select2-container'); + select2Container.on('click', function (event) { + const isChoice = isSelect2ChoiceElement(event.target); + if (isChoice) { + event.preventDefault(); + + // select2 still bubbles the event to open the dropdown. So we close it here and remove focus if we want that + if (closeDrawer) { + control.select2('close'); + setTimeout(() => select2Container.find('textarea').trigger('blur'), debounce_timeout.quick); + } + if (openDrawer) { + control.select2('open'); + } + + // Now execute the actual action that was subscribed + action(event.target); + } + }); +} + +/** + * Applies syntax highlighting to a given regex string by generating HTML with classes + * + * @param {string} regexStr - The javascript compatible regex string + * @returns {string} The html representation of the highlighted regex + */ +export function highlightRegex(regexStr) { + // Function to escape HTML special characters for safety + const escapeHtml = (str) => str.replace(/[&<>"']/g, match => ({ + '&': '&', '<': '<', '>': '>', '"': '"', '\'': ''', + })[match]); + + // Replace special characters with their HTML-escaped forms + regexStr = escapeHtml(regexStr); + + // Patterns that we want to highlight only if they are not escaped + const patterns = { + brackets: /(? { + regexStr = regexStr.replace(pattern, match => `${match}`); + }; + + // Apply highlighting patterns + wrapPattern(patterns.brackets, 'regex-brackets'); + wrapPattern(patterns.quantifiers, 'regex-quantifier'); + wrapPattern(patterns.operators, 'regex-operator'); + wrapPattern(patterns.specialChars, 'regex-special'); + wrapPattern(patterns.flags, 'regex-flags'); + wrapPattern(patterns.delimiters, 'regex-delimiter'); + + return `${regexStr}`; +} diff --git a/public/scripts/variables.js b/public/scripts/variables.js index bfaaab2af..5f3229692 100644 --- a/public/scripts/variables.js +++ b/public/scripts/variables.js @@ -1,11 +1,13 @@ import { chat_metadata, getCurrentChatId, saveSettingsDebounced, sendSystemMessage, system_message_types } from '../script.js'; import { extension_settings, saveMetadataDebounced } from './extensions.js'; -import { executeSlashCommands } from './slash-commands.js'; +import { executeSlashCommands, executeSlashCommandsWithOptions } from './slash-commands.js'; import { SlashCommand } from './slash-commands/SlashCommand.js'; +import { SlashCommandAbortController } from './slash-commands/SlashCommandAbortController.js'; import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js'; import { SlashCommandClosure } from './slash-commands/SlashCommandClosure.js'; +import { SlashCommandClosureResult } from './slash-commands/SlashCommandClosureResult.js'; import { SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js'; -import { SlashCommandParser } from './slash-commands/SlashCommandParser.js'; +import { PARSER_FLAG, SlashCommandParser } from './slash-commands/SlashCommandParser.js'; import { SlashCommandScope } from './slash-commands/SlashCommandScope.js'; import { isFalseBoolean } from './utils.js'; @@ -314,61 +316,117 @@ function listVariablesCallback() { sendSystemMessage(system_message_types.GENERIC, htmlMessage); } -async function whileCallback(args, command) { +/** + * + * @param {import('./slash-commands/SlashCommand.js').NamedArguments} args + * @param {(string|SlashCommandClosure)[]} value + */ +async function whileCallback(args, value) { const isGuardOff = isFalseBoolean(args.guard); const iterations = isGuardOff ? Number.MAX_SAFE_INTEGER : MAX_LOOPS; + /**@type {string|SlashCommandClosure} */ + let command; + if (value) { + if (value[0] instanceof SlashCommandClosure) { + command = value[0]; + } else { + command = value.join(' '); + } + } + let commandResult; for (let i = 0; i < iterations; i++) { const { a, b, rule } = parseBooleanOperands(args); const result = evalBoolean(rule, a, b); if (result && command) { - if (command instanceof SlashCommandClosure) await command.execute(); - else await executeSubCommands(command, args._scope, args._parserFlags); + if (command instanceof SlashCommandClosure) { + commandResult = await command.execute(); + } else { + commandResult = await executeSubCommands(command, args._scope, args._parserFlags, args._abortController); + } + if (commandResult.isAborted) break; } else { break; } } + if (commandResult) { + return commandResult.pipe; + } + return ''; } +/** + * + * @param {import('./slash-commands/SlashCommand.js').NamedArguments} args + * @param {import('./slash-commands/SlashCommand.js').UnnamedArguments} value + * @returns + */ async function timesCallback(args, value) { let repeats; let command; if (Array.isArray(value)) { - [repeats, command] = value; + [repeats, ...command] = value; + if (command[0] instanceof SlashCommandClosure) { + command = command[0]; + } else { + command = command.join(' '); + } } else { - [repeats, ...command] = value.split(' '); + [repeats, ...command] = /**@type {string}*/(value).split(' '); command = command.join(' '); } const isGuardOff = isFalseBoolean(args.guard); const iterations = Math.min(Number(repeats), isGuardOff ? Number.MAX_SAFE_INTEGER : MAX_LOOPS); + let result; for (let i = 0; i < iterations; i++) { + /**@type {SlashCommandClosureResult}*/ if (command instanceof SlashCommandClosure) { command.scope.setMacro('timesIndex', i); - await command.execute(); + result = await command.execute(); } else { - await executeSubCommands(command.replace(/\{\{timesIndex\}\}/g, i), args._scope, args._parserFlags); + result = await executeSubCommands(command.replace(/\{\{timesIndex\}\}/g, i.toString()), args._scope, args._parserFlags, args._abortController); } + if (result.isAborted) break; } - return ''; + return result?.pipe ?? ''; } -async function ifCallback(args, command) { +/** + * + * @param {import('./slash-commands/SlashCommand.js').NamedArguments} args + * @param {(string|SlashCommandClosure)[]} value + */ +async function ifCallback(args, value) { const { a, b, rule } = parseBooleanOperands(args); const result = evalBoolean(rule, a, b); - if (result && command) { - if (command instanceof SlashCommandClosure) return (await command.execute()).pipe; - return await executeSubCommands(command, args._scope, args._parserFlags); - } else if (!result && args.else && ((typeof args.else === 'string' && args.else !== '') || args.else instanceof SlashCommandClosure)) { - if (args.else instanceof SlashCommandClosure) return (await args.else.execute(args._scope)).pipe; - return await executeSubCommands(args.else, args._scope, args._parserFlags); + /**@type {string|SlashCommandClosure} */ + let command; + if (value) { + if (value[0] instanceof SlashCommandClosure) { + command = value[0]; + } else { + command = value.join(' '); + } } + let commandResult; + if (result && command) { + if (command instanceof SlashCommandClosure) return (await command.execute()).pipe; + commandResult = await executeSubCommands(command, args._scope, args._parserFlags, args._abortController); + } else if (!result && args.else && ((typeof args.else === 'string' && args.else !== '') || args.else instanceof SlashCommandClosure)) { + if (args.else instanceof SlashCommandClosure) return (await args.else.execute()).pipe; + commandResult = await executeSubCommands(args.else, args._scope, args._parserFlags, args._abortController); + } + + if (commandResult) { + return commandResult.pipe; + } return ''; } @@ -511,20 +569,25 @@ function evalBoolean(rule, a, b) { /** * Executes a slash command from a string (may be enclosed in quotes) and returns the result. * @param {string} command Command to execute. May contain escaped macro and batch separators. - * @returns {Promise} Pipe result + * @param {SlashCommandScope} [scope] The scope to use. + * @param {{[id:PARSER_FLAG]:boolean}} [parserFlags] The parser flags to use. + * @param {SlashCommandAbortController} [abortController] The abort controller to use. + * @returns {Promise} Closure execution result */ -async function executeSubCommands(command, scope = null, parserFlags = null) { +async function executeSubCommands(command, scope = null, parserFlags = null, abortController = null) { if (command.startsWith('"') && command.endsWith('"')) { command = command.slice(1, -1); } - const result = await executeSlashCommands(command, true, scope, true, parserFlags); + const result = await executeSlashCommandsWithOptions(command, { + handleExecutionErrors: false, + handleParserErrors: false, + parserFlags, + scope, + abortController: abortController ?? new SlashCommandAbortController(), + }); - if (!result || typeof result !== 'object') { - return ''; - } - - return result?.pipe || ''; + return result; } /** @@ -1066,6 +1129,7 @@ export function registerVariableCommands() { 'command to execute if true', [ARGUMENT_TYPE.CLOSURE, ARGUMENT_TYPE.SUBCOMMAND], true, ), ], + splitUnnamedArgument: true, helpString: `
      Compares the value of the left operand a with the value of the right operand b, @@ -1132,6 +1196,7 @@ export function registerVariableCommands() { 'command to execute while true', [ARGUMENT_TYPE.CLOSURE, ARGUMENT_TYPE.SUBCOMMAND], true, ), ], + splitUnnamedArgument: true, helpString: `
      Compares the value of the left operand a with the value of the right operand b, @@ -1158,7 +1223,7 @@ export function registerVariableCommands() { Examples:
      • - 
        /setvar key=i 0 | /while left=i right=10 rule=let "/addvar key=i 1"
        + 
        /setvar key=i 0 | /while left=i right=10 rule=lte "/addvar key=i 1"
        adds 1 to the value of "i" until it reaches 10.
      @@ -1184,6 +1249,7 @@ export function registerVariableCommands() { true, ), ], + splitUnnamedArgument: true, helpString: `
      Execute any valid slash command enclosed in quotes repeats number of times. @@ -1592,7 +1658,7 @@ export function registerVariableCommands() { returns: 'length of the provided value', unnamedArgumentList: [ new SlashCommandArgument( - 'value', [ARGUMENT_TYPE.STRING, ARGUMENT_TYPE.VARIABLE_NAME], true + 'value', [ARGUMENT_TYPE.STRING, ARGUMENT_TYPE.VARIABLE_NAME], true, ), ], helpString: ` diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index af102a45c..10a9cc47e 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -1,5 +1,5 @@ import { saveSettings, callPopup, substituteParams, getRequestHeaders, chat_metadata, this_chid, characters, saveCharacterDebounced, menu_type, eventSource, event_types, getExtensionPromptByName, saveMetadata, getCurrentChatId, extension_prompt_roles } from '../script.js'; -import { download, debounce, initScrollHeight, resetScrollHeight, parseJsonFile, extractDataFromPng, getFileBuffer, getCharaFilename, getSortableDelay, escapeRegex, PAGINATION_TEMPLATE, navigation_option, waitUntilCondition, isTrueBoolean, setValueByPath, flashHighlight } from './utils.js'; +import { download, debounce, initScrollHeight, resetScrollHeight, parseJsonFile, extractDataFromPng, getFileBuffer, getCharaFilename, getSortableDelay, escapeRegex, PAGINATION_TEMPLATE, navigation_option, waitUntilCondition, isTrueBoolean, setValueByPath, flashHighlight, select2ModifyOptions, getSelect2OptionId, dynamicSelect2DataViaAjax, highlightRegex, select2ChoiceClickSubscribe, isFalseBoolean } from './utils.js'; import { extension_settings, getContext } from './extensions.js'; import { NOTE_MODULE_NAME, metadata_keys, shouldWIAddPrompt } from './authors-note.js'; import { isMobile } from './RossAscends-mods.js'; @@ -69,7 +69,7 @@ const saveSettingsDebounced = debounce(() => { saveSettings(); }, debounce_timeout.relaxed); const sortFn = (a, b) => b.order - a.order; -let updateEditor = (navigation) => { console.debug('Triggered WI navigation', navigation); }; +let updateEditor = (navigation, flashOnNav = true) => { console.debug('Triggered WI navigation', navigation, flashOnNav); }; // Do not optimize. updateEditor is a function that is updated by the displayWorldEntries with new data. const worldInfoFilter = new FilterHelper(() => updateEditor()); @@ -197,6 +197,13 @@ class WorldInfoBuffer { * @returns {boolean} True if the string was found in the buffer */ matchKeys(haystack, needle, entry) { + // If the needle is a regex, we do regex pattern matching and override all the other options + const keyRegex = parseRegexFromString(needle); + if (keyRegex) { + return keyRegex.test(haystack); + } + + // Otherwise we do normal matching of plaintext with the chosen entry settings const transformedString = this.#transformString(needle, entry); const matchWholeWords = entry.matchWholeWords ?? world_info_match_whole_words; @@ -541,6 +548,19 @@ function registerWorldInfoSlashCommands() { return ''; } + if (typeof newEntryTemplate[field] === 'boolean') { + const isTrue = isTrueBoolean(value); + const isFalse = isFalseBoolean(value); + + if (isTrue) { + value = String(true); + } + + if (isFalse) { + value = String(false); + } + } + const fuse = new Fuse(entries, { keys: [{ name: field, weight: 1 }], includeScore: true, @@ -984,8 +1004,39 @@ function nullWorldInfo() { toastr.info('Create or import a new World Info file first.', 'World Info is not set', { timeOut: 10000, preventDuplicates: true }); } -function displayWorldEntries(name, data, navigation = navigation_option.none) { - updateEditor = (navigation) => displayWorldEntries(name, data, navigation); +/** @type {Select2Option[]} Cache all keys as selectable dropdown option */ +const worldEntryKeyOptionsCache = []; + +/** + * Update the cache and all select options for the keys with new values to display + * @param {string[]|Select2Option[]} keyOptions - An array of options to update + * @param {object} options - Optional arguments + * @param {boolean?} [options.remove=false] - Whether the option was removed, so the count should be reduced - otherwise it'll be increased + * @param {boolean?} [options.reset=false] - Whether the cache should be reset. Reset will also not trigger update of the controls, as we expect them to be redrawn anyway + */ +function updateWorldEntryKeyOptionsCache(keyOptions, { remove = false, reset = false } = {}) { + if (!keyOptions.length) return; + /** @type {Select2Option[]} */ + const options = keyOptions.map(x => typeof x === 'string' ? { id: getSelect2OptionId(x), text: x } : x); + if (reset) worldEntryKeyOptionsCache.length = 0; + options.forEach(option => { + // Update the cache list + let cachedEntry = worldEntryKeyOptionsCache.find(x => x.id == option.id); + if (cachedEntry) { + cachedEntry.count += !remove ? 1 : -1; + } else if (!remove) { + worldEntryKeyOptionsCache.push(option); + cachedEntry = option; + cachedEntry.count = 1; + } + }); + + // Sort by count DESC and then alphabetically + worldEntryKeyOptionsCache.sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)); +} + +function displayWorldEntries(name, data, navigation = navigation_option.none, flashOnNav = true) { + updateEditor = (navigation, flashOnNav = true) => displayWorldEntries(name, data, navigation, flashOnNav); const worldEntriesList = $('#world_popup_entries_list'); @@ -1020,6 +1071,10 @@ function displayWorldEntries(name, data, navigation = navigation_option.none) { entriesArray = worldInfoFilter.applyFilters(entriesArray); entriesArray = sortEntries(entriesArray); + // Cache keys + const keys = entriesArray.flatMap(entry => [...entry.key, ...entry.keysecondary]); + updateWorldEntryKeyOptionsCache(keys, { reset: true }); + // Run the callback for printing this typeof callback === 'function' && callback(entriesArray); return entriesArray; @@ -1036,7 +1091,7 @@ function displayWorldEntries(name, data, navigation = navigation_option.none) { $('#world_info_pagination').pagination({ dataSource: getDataArray, pageSize: Number(localStorage.getItem(storageKey)) || perPageDefault, - sizeChangerOptions: [10, 25, 50, 100], + sizeChangerOptions: [10, 25, 50, 100, 500, 1000], showSizeChanger: true, pageRange: 1, pageNumber: startPage, @@ -1114,7 +1169,7 @@ function displayWorldEntries(name, data, navigation = navigation_option.none) { const parentOffset = element.parent().offset(); const scrollOffset = elementOffset.top - parentOffset.top; $('#WorldInfo').scrollTop(scrollOffset); - flashHighlight(element); + if (flashOnNav) flashHighlight(element); }); } @@ -1202,9 +1257,10 @@ function displayWorldEntries(name, data, navigation = navigation_option.none) { } worldEntriesList.sortable({ + items: '.world_entry', delay: getSortableDelay(), handle: '.drag-handle', - stop: async function (event, ui) { + stop: async function (_event, _ui) { const firstEntryUid = $('#world_popup_entries_list .world_entry').first().data('uid'); const minDisplayIndex = data?.entries[firstEntryUid]?.displayIndex ?? 0; $('#world_popup_entries_list .world_entry').each(function (index) { @@ -1234,6 +1290,7 @@ const originalDataKeyMap = { 'displayIndex': 'extensions.display_index', 'excludeRecursion': 'extensions.exclude_recursion', 'preventRecursion': 'extensions.prevent_recursion', + 'delayUntilRecursion': 'extensions.delay_until_recursion', 'selectiveLogic': 'selectiveLogic', 'comment': 'comment', 'constant': 'constant', @@ -1299,6 +1356,139 @@ function deleteOriginalDataValue(data, uid) { } } +/** @typedef {import('./utils.js').Select2Option} Select2Option */ + +/** + * Splits a given input string that contains one or more keywords or regexes, separated by commas. + * + * Each part can be a valid regex following the pattern `/myregex/flags` with optional flags. Commmas inside the regex are allowed, slashes have to be escaped like this: `\/` + * If a regex doesn't stand alone, it is not treated as a regex. + * + * @param {string} input - One or multiple keywords or regexes, separated by commas + * @returns {string[]} An array of keywords and regexes + */ +function splitKeywordsAndRegexes(input) { + /** @type {string[]} */ + let keywordsAndRegexes = []; + + // We can make this easy. Instead of writing another function to find and parse regexes, + // we gonna utilize the custom tokenizer that also handles the input. + // No need for validation here + const addFindCallback = (/** @type {Select2Option} */ item) => { + keywordsAndRegexes.push(item.text); + }; + + const { term } = customTokenizer({ _type: 'custom_call', term: input }, undefined, addFindCallback); + const finalTerm = term.trim(); + if (finalTerm) { + addFindCallback({ id: getSelect2OptionId(finalTerm), text: finalTerm }); + } + + return keywordsAndRegexes; +} + +/** + * Tokenizer parsing input and splitting it into keywords and regexes + * + * @param {{_type: string, term: string}} input - The typed input + * @param {{options: object}} _selection - The selection even object (?) + * @param {function(Select2Option):void} callback - The original callback function to call if an item should be inserted + * @returns {{term: string}} - The remaining part that is untokenized in the textbox + */ +function customTokenizer(input, _selection, callback) { + let current = input.term; + + // Go over the input and check the current state, if we can get a token + for (let i = 0; i < current.length; i++) { + let char = current[i]; + + // If a comma is typed, we tokenize the input. + // unless we are inside a possible regex, which would allow commas inside + if (char === ',') { + // We take everything up till now and consider this a token + const token = current.slice(0, i).trim(); + + // Now how we test if this is a valid regex? And not a finished one, but a half-finished one? + // Easy, if someone typed a comma it can't be a delimiter escape. + // So we just check if this opening with a slash, and if so, we "close" the regex and try to parse it. + // So if we are inside a valid regex, we can't take the token now, we continue processing until the regex is closed, + // or this is not a valid regex anymore + if (token.startsWith('/') && isValidRegex(token + '/')) { + continue; + } + + // So now the comma really means the token is done. + // We take the token up till now, and insert it. Empty will be skipped. + if (token) { + const isRegex = isValidRegex(token); + + // Last chance to check for valid regex again. Because it might have been valid while typing, but now is not valid anymore and contains commas we need to split. + if (token.startsWith('/') && !isRegex) { + const tokens = token.split(',').map(x => x.trim()); + tokens.forEach(x => callback({ id: getSelect2OptionId(x), text: x })); + } else { + callback({ id: getSelect2OptionId(token), text: token }); + } + } + + // Now remove the token from the current input, and the comma too + current = current.slice(i + 1); + i = 0; + } + } + + // At the end, just return the left-over input + return { term: current }; +} + +/** + * Validates if a string is a valid slash-delimited regex, that can be parsed and executed + * + * This is a wrapper around `parseRegexFromString` + * + * @param {string} input - A delimited regex string + * @returns {boolean} Whether this would be a valid regex that can be parsed and executed + */ +function isValidRegex(input) { + return parseRegexFromString(input) !== null; +} + +/** + * Gets a real regex object from a slash-delimited regex string + * + * This function works with `/` as delimiter, and each occurance of it inside the regex has to be escaped. + * Flags are optional, but can only be valid flags supported by JavaScript's `RegExp` (`g`, `i`, `m`, `s`, `u`, `y`). + * + * @param {string} input - A delimited regex string + * @returns {RegExp|null} The regex object, or null if not a valid regex + */ +function parseRegexFromString(input) { + // Extracting the regex pattern and flags + let match = input.match(/^\/([\w\W]+?)\/([gimsuy]*)$/); + if (!match) { + return null; // Not a valid regex format + } + + let [, pattern, flags] = match; + + // If we find any unescaped slash delimiter, we also exit out. + // JS doesn't care about delimiters inside regex patterns, but for this to be a valid regex outside of our implementation, + // we have to make sure that our delimiter is correctly escaped. Or every other engine would fail. + if (pattern.match(/(^|[^\\])\//)) { + return null; + } + + // Now we need to actually unescape the slash delimiters, because JS doesn't care about delimiters + pattern = pattern.replace('\\/', '/'); + + // Then we return the regex. If it fails, it was invalid syntax. + try { + return new RegExp(pattern, flags); + } catch (e) { + return null; + } +} + function getWorldEntry(name, data, entry) { if (!data.entries[entry.uid]) { return; @@ -1308,28 +1498,125 @@ function getWorldEntry(name, data, entry) { template.data('uid', entry.uid); template.attr('uid', entry.uid); + // Init default state of WI Key toggle (=> true) + if (typeof power_user.wi_key_input_plaintext === 'undefined') power_user.wi_key_input_plaintext = true; + + /** Function to build the keys input controls @param {string} entryPropName @param {string} originalDataValueName */ + function enableKeysInput(entryPropName, originalDataValueName) { + const isFancyInput = !isMobile() && !power_user.wi_key_input_plaintext; + const input = isFancyInput ? template.find(`select[name="${entryPropName}"]`) : template.find(`textarea[name="${entryPropName}"]`); + input.data('uid', entry.uid); + input.on('click', function (event) { + // Prevent closing the drawer on clicking the input + event.stopPropagation(); + }); + + function templateStyling(/** @type {Select2Option} */ item, { searchStyle = false } = {}) { + const content = $('').addClass('item').text(item.text).attr('title', `${item.text}\n\nClick to edit`); + const isRegex = isValidRegex(item.text); + if (isRegex) { + content.html(highlightRegex(item.text)); + content.addClass('regex_item').prepend($('').addClass('regex_icon').text('•*').attr('title', 'Regex')); + } + + if (searchStyle && item.count) { + // Build a wrapping element + const wrapper = $('').addClass('result_block') + .append(content); + wrapper.append($('').addClass('item_count').text(item.count).attr('title', `Used as a key ${item.count} ${item.count != 1 ? 'times' : 'time'} in this lorebook`)); + return wrapper; + } + + return content; + } + + if (isFancyInput) { + input.select2({ + ajax: dynamicSelect2DataViaAjax(() => worldEntryKeyOptionsCache), + tags: true, + tokenSeparators: [','], + tokenizer: customTokenizer, + placeholder: input.attr('placeholder'), + templateResult: item => templateStyling(item, { searchStyle: true }), + templateSelection: item => templateStyling(item), + }); + input.on('change', function (_, { skipReset, noSave } = {}) { + const uid = $(this).data('uid'); + /** @type {string[]} */ + const keys = ($(this).select2('data')).map(x => x.text); + + !skipReset && resetScrollHeight(this); + if (!noSave) { + data.entries[uid][entryPropName] = keys; + setOriginalDataValue(data, uid, originalDataValueName, data.entries[uid][entryPropName]); + saveWorldInfo(name, data); + } + }); + input.on('select2:select', /** @type {function(*):void} */ event => updateWorldEntryKeyOptionsCache([event.params.data])); + input.on('select2:unselect', /** @type {function(*):void} */ event => updateWorldEntryKeyOptionsCache([event.params.data], { remove: true })); + + select2ChoiceClickSubscribe(input, target => { + const key = $(target).text(); + console.debug('Editing WI key', key); + + // Remove the current key from the actual selection + const selected = input.val(); + if (!Array.isArray(selected)) return; + var index = selected.indexOf(getSelect2OptionId(key)); + if (index > -1) selected.splice(index, 1); + input.val(selected).trigger('change'); + // Manually update the cache, that change event is not gonna trigger it + updateWorldEntryKeyOptionsCache([key], { remove: true }); + + // We need to "hack" the actual text input into the currently open textarea + input.next('span.select2-container').find('textarea') + .val(key).trigger('input'); + }, { openDrawer: true }); + + select2ModifyOptions(input, entry[entryPropName], { select: true, changeEventArgs: { skipReset: true, noSave: true } }); + } + else { + // Compatibility with mobile devices. On mobile we need a text input field, not a select option control, so we need its own event handlers + template.find(`select[name="${entryPropName}"]`).hide(); + input.show(); + + input.on('input', function (_, { skipReset, noSave } = {}) { + const uid = $(this).data('uid'); + const value = String($(this).val()); + !skipReset && resetScrollHeight(this); + if (!noSave) { + data.entries[uid][entryPropName] = splitKeywordsAndRegexes(value); + setOriginalDataValue(data, uid, originalDataValueName, data.entries[uid][entryPropName]); + saveWorldInfo(name, data); + } + }); + input.val(entry[entryPropName].join(', ')).trigger('input', { skipReset: true }); + } + return { isFancy: isFancyInput, control: input }; + } + // key - const keyInput = template.find('textarea[name="key"]'); - keyInput.data('uid', entry.uid); - keyInput.on('click', function (event) { - // Prevent closing the drawer on clicking the input - event.stopPropagation(); - }); + const keyInput = enableKeysInput('key', 'keys'); - keyInput.on('input', function (_, { skipReset } = {}) { - const uid = $(this).data('uid'); - const value = String($(this).val()); - !skipReset && resetScrollHeight(this); - data.entries[uid].key = value - .split(',') - .map((x) => x.trim()) - .filter((x) => x); + // keysecondary + const keySecondaryInput = enableKeysInput('keysecondary', 'secondary_keys'); - setOriginalDataValue(data, uid, 'keys', data.entries[uid].key); - saveWorldInfo(name, data); + // draw key input switch button + template.find('.switch_input_type_icon').on('click', function () { + power_user.wi_key_input_plaintext = !power_user.wi_key_input_plaintext; + saveSettingsDebounced(); + + // Just redraw the panel + const uid = ($(this).parents('.world_entry')).data('uid'); + updateEditor(uid, false); + + $(`.world_entry[uid="${uid}"] .inline-drawer-icon`).trigger('click'); + // setTimeout(() => { + // }, debounce_timeout.standard); + }).each((_, icon) => { + $(icon).attr('title', $(icon).data(power_user.wi_key_input_plaintext ? 'tooltip-on' : 'tooltip-off')); + $(icon).text($(icon).data(power_user.wi_key_input_plaintext ? 'icon-on' : 'icon-off')); }); - keyInput.val(entry.key.join(', ')).trigger('input', { skipReset: true }); - //initScrollHeight(keyInput); // logic AND/NOT const selectiveLogicDropdown = template.find('select[name="entryLogicType"]'); @@ -1458,25 +1745,6 @@ function getWorldEntry(name, data, entry) { saveWorldInfo(name, data); }); - // keysecondary - const keySecondaryInput = template.find('textarea[name="keysecondary"]'); - keySecondaryInput.data('uid', entry.uid); - keySecondaryInput.on('input', function (_, { skipReset } = {}) { - const uid = $(this).data('uid'); - const value = String($(this).val()); - !skipReset && resetScrollHeight(this); - data.entries[uid].keysecondary = value - .split(',') - .map((x) => x.trim()) - .filter((x) => x); - - setOriginalDataValue(data, uid, 'secondary_keys', data.entries[uid].keysecondary); - saveWorldInfo(name, data); - }); - - keySecondaryInput.val(entry.keysecondary.join(', ')).trigger('input', { skipReset: true }); - //initScrollHeight(keySecondaryInput); - // comment const commentInput = template.find('textarea[name="comment"]'); const commentToggle = template.find('input[name="addMemo"]'); @@ -1539,8 +1807,8 @@ function getWorldEntry(name, data, entry) { if (counter.data('first-run')) { counter.data('first-run', false); countTokensDebounced(counter, contentInput.val()); - initScrollHeight(keyInput); - initScrollHeight(keySecondaryInput); + if (!keyInput.isFancy) initScrollHeight(keyInput.control); + if (!keySecondaryInput.isFancy) initScrollHeight(keySecondaryInput.control); } }); @@ -1563,11 +1831,11 @@ function getWorldEntry(name, data, entry) { .closest('.world_entry') .find('.keysecondarytextpole'); - const keyprimarytextpole = $(this) + const keyprimaryselect = $(this) .closest('.world_entry') - .find('.keyprimarytextpole'); + .find('.keyprimaryselect'); - const keyprimaryHeight = keyprimarytextpole.outerHeight(); + const keyprimaryHeight = keyprimaryselect.outerHeight(); keysecondarytextpole.css('height', keyprimaryHeight + 'px'); value ? keysecondary.show() : keysecondary.hide(); @@ -1619,7 +1887,7 @@ function getWorldEntry(name, data, entry) { saveWorldInfo(name, data); }); groupInput.val(entry.group ?? '').trigger('input'); - setTimeout(() => createEntryInputAutocomplete(groupInput, getInclusionGroupCallback(data)), 1); + setTimeout(() => createEntryInputAutocomplete(groupInput, getInclusionGroupCallback(data), { allowMultiple: true }), 1); // inclusion priority const groupOverrideInput = template.find('input[name="groupOverride"]'); @@ -1891,6 +2159,18 @@ function getWorldEntry(name, data, entry) { }); preventRecursionInput.prop('checked', entry.preventRecursion).trigger('input'); + // delay until recursion + const delayUntilRecursionInput = template.find('input[name="delay_until_recursion"]'); + delayUntilRecursionInput.data('uid', entry.uid); + delayUntilRecursionInput.on('input', function () { + const uid = $(this).data('uid'); + const value = $(this).prop('checked'); + data.entries[uid].delayUntilRecursion = value; + setOriginalDataValue(data, uid, 'extensions.delay_until_recursion', data.entries[uid].delayUntilRecursion); + saveWorldInfo(name, data); + }); + delayUntilRecursionInput.prop('checked', entry.delayUntilRecursion).trigger('input'); + // duplicate button const duplicateButton = template.find('.duplicate_entry_button'); duplicateButton.data('uid', entry.uid); @@ -2029,11 +2309,15 @@ function getWorldEntry(name, data, entry) { * @returns {(input: any, output: any) => any} Callback function for the autocomplete */ function getInclusionGroupCallback(data) { - return function (input, output) { + return function (control, input, output) { + const uid = $(control).data('uid'); + const thisGroups = String($(control).val()).split(/,\s*/).filter(x => x).map(x => x.toLowerCase()); const groups = new Set(); for (const entry of Object.values(data.entries)) { + // Skip the groups of this entry, because auto-complete should only suggest the ones that are already available on other entries + if (entry.uid == uid) continue; if (entry.group) { - groups.add(String(entry.group)); + entry.group.split(/,\s*/).filter(x => x).forEach(x => groups.add(x)); } } @@ -2041,20 +2325,19 @@ function getInclusionGroupCallback(data) { haystack.sort((a, b) => a.localeCompare(b)); const needle = input.term.toLowerCase(); const hasExactMatch = haystack.findIndex(x => x.toLowerCase() == needle) !== -1; - const result = haystack.filter(x => x.toLowerCase().includes(needle)); - - if (input.term && !hasExactMatch) { - result.unshift(input.term); - } + const result = haystack.filter(x => x.toLowerCase().includes(needle) && (!thisGroups.includes(x) || hasExactMatch && thisGroups.filter(g => g == x).length == 1)); output(result); }; } function getAutomationIdCallback(data) { - return function (input, output) { + return function (control, input, output) { + const uid = $(control).data('uid'); const ids = new Set(); for (const entry of Object.values(data.entries)) { + // Skip automation id of this entry, because auto-complete should only suggest the ones that are already available on other entries + if (entry.uid == uid) continue; if (entry.automationId) { ids.add(String(entry.automationId)); } @@ -2070,36 +2353,53 @@ function getAutomationIdCallback(data) { const haystack = Array.from(ids); haystack.sort((a, b) => a.localeCompare(b)); const needle = input.term.toLowerCase(); - const hasExactMatch = haystack.findIndex(x => x.toLowerCase() == needle) !== -1; const result = haystack.filter(x => x.toLowerCase().includes(needle)); - if (input.term && !hasExactMatch) { - result.unshift(input.term); - } - output(result); }; } /** * Create an autocomplete for the inclusion group. - * @param {JQuery} input Input element to attach the autocomplete to - * @param {(input: any, output: any) => any} callback Source data callbacks + * @param {JQuery} input - Input element to attach the autocomplete to + * @param {(control: JQuery, input: any, output: any) => any} callback - Source data callbacks + * @param {object} [options={}] - Optional arguments + * @param {boolean} [options.allowMultiple=false] - Whether to allow multiple comma-separated values */ -function createEntryInputAutocomplete(input, callback) { +function createEntryInputAutocomplete(input, callback, { allowMultiple = false } = {}) { + const handleSelect = (event, ui) => { + // Prevent default autocomplete select, so we can manually set the value + event.preventDefault(); + if (!allowMultiple) { + $(input).val(ui.item.value).trigger('input').trigger('blur'); + } else { + var terms = String($(input).val()).split(/,\s*/); + terms.pop(); // remove the current input + terms.push(ui.item.value); // add the selected item + $(input).val(terms.filter(x => x).join(', ')).trigger('input').trigger('blur'); + } + }; + $(input).autocomplete({ minLength: 0, - source: callback, - select: function (event, ui) { - $(input).val(ui.item.value).trigger('input').trigger('blur'); + source: function (request, response) { + if (!allowMultiple) { + callback(input, request, response); + } else { + const term = request.term.split(/,\s*/).pop(); + request.term = term; + callback(input, request, response); + } }, + select: handleSelect, }); $(input).on('focus click', function () { - $(input).autocomplete('search', String($(input).val())); + $(input).autocomplete('search', allowMultiple ? String($(input).val()).split(/,\s*/).pop() : $(input).val()); }); } + /** * Duplicated a WI entry by copying all of its properties and assigning a new uid * @param {*} data - The data of the book @@ -2152,6 +2452,8 @@ const newEntryTemplate = { position: 0, disable: false, excludeRecursion: false, + preventRecursion: false, + delayUntilRecursion: false, probability: 100, useProbability: true, depth: DEFAULT_DEPTH, @@ -2166,7 +2468,7 @@ const newEntryTemplate = { role: 0, }; -function createWorldInfoEntry(name, data) { +function createWorldInfoEntry(_name, data) { const newUid = getFreeWorldEntryUid(data); if (!Number.isInteger(newUid)) { @@ -2519,7 +2821,7 @@ async function checkWorldInfo(chat, maxContext) { continue; } - if (allActivatedEntries.has(entry) || entry.disable == true || (count > 1 && world_info_recursive && entry.excludeRecursion)) { + if (allActivatedEntries.has(entry) || entry.disable == true || (count > 1 && world_info_recursive && entry.excludeRecursion) || (count == 1 && entry.delayUntilRecursion)) { continue; } @@ -2792,10 +3094,12 @@ function filterGroupsByScoring(groups, buffer, removeEntry) { function filterByInclusionGroups(newEntries, allActivatedEntries, buffer) { console.debug('-- INCLUSION GROUP CHECKS BEGIN --'); const grouped = newEntries.filter(x => x.group).reduce((acc, item) => { - if (!acc[item.group]) { - acc[item.group] = []; - } - acc[item.group].push(item); + item.group.split(/,\s*/).filter(x => x).forEach(group => { + if (!acc[group]) { + acc[group] = []; + } + acc[group].push(item); + }); return acc; }, {}); @@ -2887,9 +3191,10 @@ function convertAgnaiMemoryBook(inputObj) { disable: !entry.enabled, addMemo: !!entry.name, excludeRecursion: false, + delayUntilRecursion: false, displayIndex: index, - probability: null, - useProbability: false, + probability: 100, + useProbability: true, group: '', groupOverride: false, groupWeight: DEFAULT_WEIGHT, @@ -2925,9 +3230,10 @@ function convertRisuLorebook(inputObj) { disable: false, addMemo: true, excludeRecursion: false, + delayUntilRecursion: false, displayIndex: index, - probability: entry.activationPercent ?? null, - useProbability: entry.activationPercent ?? false, + probability: entry.activationPercent ?? 100, + useProbability: entry.activationPercent ?? true, group: '', groupOverride: false, groupWeight: DEFAULT_WEIGHT, @@ -2968,9 +3274,10 @@ function convertNovelLorebook(inputObj) { disable: !entry.enabled, addMemo: addMemo, excludeRecursion: false, + delayUntilRecursion: false, displayIndex: index, - probability: null, - useProbability: false, + probability: 100, + useProbability: true, group: '', groupOverride: false, groupWeight: DEFAULT_WEIGHT, @@ -3008,11 +3315,12 @@ function convertCharacterBook(characterBook) { position: entry.extensions?.position ?? (entry.position === 'before_char' ? world_info_position.before : world_info_position.after), excludeRecursion: entry.extensions?.exclude_recursion ?? false, preventRecursion: entry.extensions?.prevent_recursion ?? false, + delayUntilRecursion: entry.extensions?.delay_until_recursion ?? false, disable: !entry.enabled, addMemo: entry.comment ? true : false, displayIndex: entry.extensions?.display_index ?? index, - probability: entry.extensions?.probability ?? null, - useProbability: entry.extensions?.useProbability ?? false, + probability: entry.extensions?.probability ?? 100, + useProbability: entry.extensions?.useProbability ?? true, depth: entry.extensions?.depth ?? DEFAULT_DEPTH, selectiveLogic: entry.extensions?.selectiveLogic ?? world_info_logic.AND_ANY, group: entry.extensions?.group ?? '', @@ -3261,7 +3569,7 @@ export async function importWorldInfo(file) { toastr.info(`World Info "${data.name}" imported successfully!`); } }, - error: (jqXHR, exception) => { }, + error: (_jqXHR, _exception) => { }, }); } @@ -3468,21 +3776,14 @@ jQuery(() => { }); // Subscribe world loading to the select2 multiselect items (We need to target the specific select2 control) - $('#world_info + span.select2-container').on('click', function (event) { - if ($(event.target).hasClass('select2-selection__choice__display')) { - event.preventDefault(); - - // select2 still bubbles the event to open the dropdown. So we close it here - $('#world_info').select2('close'); - - const name = $(event.target).text(); - const selectedIndex = world_names.indexOf(name); - if (selectedIndex !== -1) { - $('#world_editor_select').val(selectedIndex).trigger('change'); - console.log('Quick selection of world', name); - } + select2ChoiceClickSubscribe($('#world_info'), target => { + const name = $(target).text(); + const selectedIndex = world_names.indexOf(name); + if (selectedIndex !== -1) { + $('#world_editor_select').val(selectedIndex).trigger('change'); + console.log('Quick selection of world', name); } - }); + }, { buttonStyle: true, closeDrawer: true }); } $('#WorldInfo').on('scroll', () => { diff --git a/public/style.css b/public/style.css index cc09a6a01..e7ceb3208 100644 --- a/public/style.css +++ b/public/style.css @@ -129,7 +129,7 @@ body { height: 100vh; height: 100svh; /*defaults as 100%, then reassigned via JS as pixels, will work on PC and Android*/ - height: calc(var(--doc-height) - 1px); + /*height: calc(var(--doc-height) - 1px);*/ background-color: var(--greyCAIbg); background-repeat: no-repeat; background-attachment: fixed; @@ -698,7 +698,7 @@ body .panelControlBar { display: flex; } } - &.paused { + &.script_paused { #rightSendForm > div:not(.mes_send).stscript_btn { &.stscript_pause { display: none; @@ -872,7 +872,8 @@ body .panelControlBar { } #chat .mes.selected{ - background-color: rgb(from var(--SmartThemeQuoteColor) r g b / .5); + /* background-color: rgb(from var(--SmartThemeQuoteColor) r g b / .5); */ + background-color: rgb(102, 0, 0); } .mes q:before, @@ -1112,8 +1113,8 @@ select { } #send_textarea { - min-height: calc(var(--bottomFormBlockSize) + 3px); - height: calc(var(--bottomFormBlockSize) + 3px); + min-height: calc(var(--bottomFormBlockSize) + 2px); + height: calc(var(--bottomFormBlockSize) + 2px); max-height: 50vh; max-height: 50svh; word-wrap: break-word; @@ -2067,6 +2068,7 @@ input[type="file"] { gap: 5px; justify-content: center; align-items: center; + flex-wrap: wrap; } .bulk_select_checkbox { @@ -2373,16 +2375,16 @@ input[type=search]::-webkit-search-cancel-button { height: 1em; width: 1em; border-radius: 50em; - background: url('/img/times-circle.svg') no-repeat 50% 50%; + background-color: var(--SmartThemeBodyColor); + mask: url('/img/times-circle.svg') no-repeat 50% 50%; background-size: contain; - backdrop-filter: invert(1) contrast(9); opacity: 0; pointer-events: none; cursor: pointer; } input[type=search]:focus::-webkit-search-cancel-button { - opacity: .3; + opacity: .5; pointer-events: all; } @@ -2813,7 +2815,7 @@ grammarly-extension { #result_info_text { display: flex; flex-direction: column; - line-height: 1; + line-height: 0.9; text-align: right; } @@ -4883,3 +4885,12 @@ body:not(.movingUI) .drawer-content.maximized { z-index: 9999; } } + +/* CSS styles using a consistent pastel color palette */ +.regex-brackets { color: #FFB347; } /* Pastel Orange */ +.regex-special { color: #B0E0E6; } /* Powder Blue */ +.regex-quantifier { color: #DDA0DD; } /* Plum */ +.regex-operator { color: #FFB6C1; } /* Light Pink */ +.regex-flags { color: #98FB98; } /* Pale Green */ +.regex-delimiter { font-weight: bold; color: #FF6961; } /* Pastel Red */ +.regex-highlight { color: #FAF8F6; } /* Pastel White */ diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 09cc1aa71..508897577 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -75,6 +75,24 @@ function getFiles(dir, files = []) { return files; } +/** + * Ensure that the asset folders exist. + * @param {import('../users').UserDirectoryList} directories - The user's directories + */ +function ensureFoldersExist(directories) { + const folderPath = path.join(directories.assets); + + for (const category of VALID_CATEGORIES) { + const assetCategoryPath = path.join(folderPath, category); + if (fs.existsSync(assetCategoryPath) && !fs.statSync(assetCategoryPath).isDirectory()) { + fs.unlinkSync(assetCategoryPath); + } + if (!fs.existsSync(assetCategoryPath)) { + fs.mkdirSync(assetCategoryPath, { recursive: true }); + } + } +} + const router = express.Router(); /** @@ -92,15 +110,7 @@ router.post('/get', jsonParser, async (request, response) => { try { if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) { - for (const category of VALID_CATEGORIES) { - const assetCategoryPath = path.join(folderPath, category); - if (fs.existsSync(assetCategoryPath) && !fs.statSync(assetCategoryPath).isDirectory()) { - fs.unlinkSync(assetCategoryPath); - } - if (!fs.existsSync(assetCategoryPath)) { - fs.mkdirSync(assetCategoryPath); - } - } + ensureFoldersExist(request.user.directories); const folders = fs.readdirSync(folderPath, { withFileTypes: true }) .filter(file => file.isDirectory()); @@ -193,6 +203,7 @@ router.post('/download', jsonParser, async (request, response) => { } // Validate filename + ensureFoldersExist(request.user.directories); const validation = validateAssetFileName(request.body.filename); if (validation.error) return response.status(400).send(validation.message); diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 5728bcf81..becfc8e07 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -254,7 +254,7 @@ async function sendMakerSuiteRequest(request, response) { }; function getGeminiBody() { - const should_use_system_prompt = model === 'gemini-1.5-pro-latest' && request.body.use_makersuite_sysprompt; + const should_use_system_prompt = ['gemini-1.5-flash-latest', 'gemini-1.5-pro-latest'].includes(model) && request.body.use_makersuite_sysprompt; const prompt = convertGooglePrompt(request.body.messages, model, should_use_system_prompt, request.body.char_name, request.body.user_name); let body = { contents: prompt.contents, diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 2e3171880..fc7ae3ef8 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -436,6 +436,7 @@ function convertWorldInfoToCharacterBook(name, entries) { group_override: entry.groupOverride ?? false, group_weight: entry.groupWeight ?? null, prevent_recursion: entry.preventRecursion ?? false, + delay_until_recursion: entry.delayUntilRecursion ?? false, scan_depth: entry.scanDepth ?? null, match_whole_words: entry.matchWholeWords ?? null, use_group_scoring: entry.useGroupScoring ?? false, diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index a2a81615e..c282dba36 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -7,7 +7,9 @@ const { getConfigValue, color } = require('../util'); const { jsonParser } = require('../express-common'); const writeFileAtomicSync = require('write-file-atomic').sync; const contentDirectory = path.join(process.cwd(), 'default/content'); +const scaffoldDirectory = path.join(process.cwd(), 'default/scaffold'); const contentIndexPath = path.join(contentDirectory, 'index.json'); +const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json'); const characterCardParser = require('../character-card-parser.js'); const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []); @@ -16,6 +18,8 @@ const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDo * @typedef {Object} ContentItem * @property {string} filename * @property {string} type + * @property {string} [name] + * @property {string|null} [folder] */ /** @@ -48,9 +52,7 @@ const CONTENT_TYPES = { */ function getDefaultPresets(directories) { try { - const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); - const contentIndex = JSON.parse(contentIndexText); - + const contentIndex = getContentIndex(); const presets = []; for (const contentItem of contentIndex) { @@ -112,8 +114,12 @@ async function seedContentForUser(contentIndex, directories, forceCategories) { continue; } - contentLog.push(contentItem.filename); - const contentPath = path.join(contentDirectory, contentItem.filename); + if (!contentItem.folder) { + console.log(`Content file ${contentItem.filename} has no parent folder`); + continue; + } + + const contentPath = path.join(contentItem.folder, contentItem.filename); if (!fs.existsSync(contentPath)) { console.log(`Content file ${contentItem.filename} is missing`); @@ -129,6 +135,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) { const basePath = path.parse(contentItem.filename).base; const targetPath = path.join(contentTarget, basePath); + contentLog.push(contentItem.filename); if (fs.existsSync(targetPath)) { console.log(`Content file ${contentItem.filename} already exists in ${contentTarget}`); @@ -157,8 +164,7 @@ async function checkForNewContent(directoriesList, forceCategories = []) { return; } - const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); - const contentIndex = JSON.parse(contentIndexText); + const contentIndex = getContentIndex(); let anyContentAdded = false; for (const directories of directoriesList) { @@ -179,6 +185,38 @@ async function checkForNewContent(directoriesList, forceCategories = []) { } } +/** + * Gets combined content index from the content and scaffold directories. + * @returns {ContentItem[]} Array of content index + */ +function getContentIndex() { + const result = []; + + if (fs.existsSync(scaffoldIndexPath)) { + const scaffoldIndexText = fs.readFileSync(scaffoldIndexPath, 'utf8'); + const scaffoldIndex = JSON.parse(scaffoldIndexText); + if (Array.isArray(scaffoldIndex)) { + scaffoldIndex.forEach((item) => { + item.folder = scaffoldDirectory; + }); + result.push(...scaffoldIndex); + } + } + + if (fs.existsSync(contentIndexPath)) { + const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); + const contentIndex = JSON.parse(contentIndexText); + if (Array.isArray(contentIndex)) { + contentIndex.forEach((item) => { + item.folder = contentDirectory; + }); + result.push(...contentIndex); + } + } + + return result; +} + /** * Gets the target directory for the specified asset type. * @param {ContentType} type Asset type @@ -351,7 +389,7 @@ function parseChubUrl(str) { let domainIndex = -1; splitStr.forEach((part, index) => { - if (part === 'www.chub.ai' || part === 'chub.ai') { + if (part === 'www.chub.ai' || part === 'chub.ai' || part === 'www.characterhub.org' || part === 'characterhub.org') { domainIndex = index; } }); @@ -521,7 +559,7 @@ router.post('/importURL', jsonParser, async (request, response) => { let result; let type; - const isChub = host.includes('chub.ai'); + const isChub = host.includes('chub.ai') || host.includes('characterhub.org'); const isJannnyContent = host.includes('janitorai'); const isPygmalionContent = host.includes('pygmalion.chat'); const isAICharacterCardsContent = host.includes('aicharactercards.com'); diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index 321267d05..82bab1439 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const express = require('express'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); -const tiktoken = require('@dqbd/tiktoken'); +const tiktoken = require('tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); const { convertClaudePrompt, convertGooglePrompt } = require('../prompt-converters'); const { readSecret, SECRET_KEYS } = require('./secrets'); @@ -15,7 +15,7 @@ const { setAdditionalHeaders } = require('../additional-headers'); */ /** - * @type {{[key: string]: import("@dqbd/tiktoken").Tiktoken}} Tokenizers cache + * @type {{[key: string]: import('tiktoken').Tiktoken}} Tokenizers cache */ const tokenizersCache = {}; @@ -262,6 +262,10 @@ function getWebTokenizersChunks(tokenizer, ids) { * @returns {string} Tokenizer model to use */ function getTokenizerModel(requestModel) { + if (requestModel.includes('gpt-4o')) { + return 'gpt-4o'; + } + if (requestModel.includes('gpt-4-32k')) { return 'gpt-4-32k'; }