From 035dbe490119365cedb06cec429bcc75fad3cf41 Mon Sep 17 00:00:00 2001 From: deffcolony <61471128+deffcolony@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:41:43 +0200 Subject: [PATCH 001/140] added issue/pr label workflows 3 months of inactivity Bot posts a comment to remind about it and assigns a stale label No further activity - 5 work days passes bot closes the issue --- .github/close-label.yml | 2 + .github/workflows/add-comment-from-tag.yml | 28 +++++++ .github/workflows/check-merge-conflicts.yml | 16 ++++ .github/workflows/close-stale-issues.yml | 82 +++++++++++++++++++ .github/workflows/get-pr-size.yml | 39 +++++++++ .../manage-pending-labels-closed.yml | 17 ++++ .github/workflows/manage-pending-labels.yml | 42 ++++++++++ 7 files changed, 226 insertions(+) create mode 100644 .github/close-label.yml create mode 100644 .github/workflows/add-comment-from-tag.yml create mode 100644 .github/workflows/check-merge-conflicts.yml create mode 100644 .github/workflows/close-stale-issues.yml create mode 100644 .github/workflows/get-pr-size.yml create mode 100644 .github/workflows/manage-pending-labels-closed.yml create mode 100644 .github/workflows/manage-pending-labels.yml diff --git a/.github/close-label.yml b/.github/close-label.yml new file mode 100644 index 000000000..5f2bfa5ee --- /dev/null +++ b/.github/close-label.yml @@ -0,0 +1,2 @@ +πŸ› Bug: βœ… Fixed +πŸ¦„ Feature Request: βœ… Implemented \ No newline at end of file diff --git a/.github/workflows/add-comment-from-tag.yml b/.github/workflows/add-comment-from-tag.yml new file mode 100644 index 000000000..548f83eac --- /dev/null +++ b/.github/workflows/add-comment-from-tag.yml @@ -0,0 +1,28 @@ +# Based on a label applied to an issue, the bot will add a comment with some additional info + +name: 🎯 Auto-Reply to Labeled Tickets +on: + issues: + types: + - labeled + - unlabeled + pull_request_target: + types: + - labeled + - unlabeled +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + comment: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Label Commenter + uses: peaceiris/actions-label-commenter@v1 + with: + config_file: .github/issue-auto-comments.yml + github_token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/check-merge-conflicts.yml b/.github/workflows/check-merge-conflicts.yml new file mode 100644 index 000000000..e1cbc9d0a --- /dev/null +++ b/.github/workflows/check-merge-conflicts.yml @@ -0,0 +1,16 @@ +# Detect and label pull requests that have merge conflicts +name: πŸ—οΈ Check Merge Conflicts +on: + push: + branches: + - staging +jobs: + check-conflicts: + runs-on: ubuntu-latest + steps: + - uses: mschilde/auto-label-merge-conflicts@master + with: + CONFLICT_LABEL_NAME: "🚫 Merge Conflicts" + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + MAX_RETRIES: 5 + WAIT_MS: 5000 \ No newline at end of file diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml new file mode 100644 index 000000000..1d34a6b1c --- /dev/null +++ b/.github/workflows/close-stale-issues.yml @@ -0,0 +1,82 @@ +# Closes any issues that no longer have user interaction +name: 🎯 Close Stale Issues + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Runs every day at midnight UTC + +jobs: + stale: + runs-on: ubuntu-latest + + steps: + # Comment on, then close issues that haven't been updated for ages + - name: Close Stale Issues + uses: actions/stale@v4 + with: + repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + days-before-stale: 360 + days-before-close: 5 + operations-per-run: 30 + remove-stale-when-updated: true + enable-statistics: true + stale-issue-message: > + This issue has gone 3 months without an update. To keep the ticket open, please indicate that it is still relevant in a comment below. + Otherwise it will be closed in 5 working days. + stale-pr-message: > + This PR is stale because it has been open 6 weeks with no activity. Either remove the stale label or comment below with a short update, + otherwise this PR will be closed in 5 days. + close-issue-message: > + This issue was automatically closed because it has been stalled for over 1 year with no activity. + close-pr-message: > + This pull request was automatically closed because it has been stalled for over 1 year with no activity. + stale-issue-label: '⚰️ Stale' + close-issue-label: 'πŸ•ΈοΈ Inactive' + stale-pr-label: '⚰️ Stale' + close-pr-label: 'πŸ•ΈοΈ Inactive' + exempt-issue-labels: 'πŸ“Œ Keep Open' + exempt-pr-labels: 'πŸ“Œ Keep Open' + labels-to-add-when-unstale: 'πŸ“Œ Keep Open' + + # Comment on, then close issues that required a response from the user, but didn't get one + - name: Close Issues without Response + uses: actions/stale@v4 + with: + repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + days-before-stale: 5 + days-before-close: 3 + operations-per-run: 30 + remove-stale-when-updated: true + stale-issue-message: > + Hi! Looks like additional info is required for this issue to be addressed. + Don't forget to provide this within the next few days to keep your ticket open. + close-issue-message: 'Issue closed due to no response from user.' + only-labels: '🚏 Awaiting User Response' + labels-to-remove-when-unstale: '🚏 Awaiting User Response, πŸ›‘ No Response' + stale-issue-label: 'πŸ›‘ No Response' + close-issue-label: 'πŸ•ΈοΈ Inactive' + exempt-issue-labels: 'πŸ“Œ Keep Open' + exempt-pr-labels: 'πŸ“Œ Keep Open' + + # Comment on issues that we should have replied to + - name: Notify Repo Owner to Respond + uses: actions/stale@v4 + with: + repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + days-before-stale: 7 + days-before-close: 365 + operations-per-run: 30 + remove-stale-when-updated: true + stale-issue-message: Hey SillyTavern, - Don't forget to respond! + stale-pr-message: Hey SillyTavern, - Don't forget to respond! + only-labels: 'πŸ‘€ Awaiting Maintainer Response' + labels-to-remove-when-unstale: 'πŸ‘€ Awaiting Maintainer Response' + close-issue-message: 'Closed due to no response from repo author for over a year' + close-pr-message: 'Closed due to no response from repo author for over a year' + stale-issue-label: 'πŸ‘€ Awaiting Maintainer Response' + stale-pr-label: 'πŸ‘€ Awaiting Maintainer Response' + close-issue-label: 'πŸ•ΈοΈ Inactive' + close-pr-label: 'πŸ•ΈοΈ Inactive' + exempt-issue-labels: 'πŸ“Œ Keep Open' + exempt-pr-labels: 'πŸ“Œ Keep Open' \ No newline at end of file diff --git a/.github/workflows/get-pr-size.yml b/.github/workflows/get-pr-size.yml new file mode 100644 index 000000000..ac5f9d4d3 --- /dev/null +++ b/.github/workflows/get-pr-size.yml @@ -0,0 +1,39 @@ +# Adds a comment to new PRs, showing the compressed size and size difference of new code +# And also labels the PR based on the number of lines changes + +name: 🌈 Check PR Size +on: [pull_request] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + # Find and comment with compressed size + - name: Get Compressed Size + uses: preactjs/compressed-size-action@v2 + with: + repo-token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + pattern: './dist/**/*.{js,css,html}' + strip-hash: '\\b\\w{8}\\.' + exclude: '**/node_modules/**' + minimum-change-threshold: 100 + # Check number of lines of code added + - name: Label based on Lines of Code + uses: codelytv/pr-size-labeler@v1 + with: + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + xs_max_size: '10' + s_max_size: '100' + m_max_size: '500' + l_max_size: '1000' + s_label: '🟩 PR - Small' + m_label: '🟨 PR - Medium' + l_label: '🟧 PR - Large' + xl_label: 'πŸŸ₯ PR - XL' + fail_if_xl: 'false' + message_if_xl: > + It looks like this PR is very large (over 1000 lines). + Try to avoid addressing multiple issues in a single PR, and + in the future consider breaking large tasks down into smaller steps. + This it to make reviewing, testing, reverting and general quality management easier. \ No newline at end of file diff --git a/.github/workflows/manage-pending-labels-closed.yml b/.github/workflows/manage-pending-labels-closed.yml new file mode 100644 index 000000000..892f2f8f9 --- /dev/null +++ b/.github/workflows/manage-pending-labels-closed.yml @@ -0,0 +1,17 @@ +# When a new comment is added to an issue, if it had the Stale or Awaiting User Response labels, then those labels will be removed + +name: 🎯 Remove Pending Labels on Close +on: + issues: + types: [closed] +jobs: + remove-labels: + runs-on: ubuntu-latest + steps: + - name: Remove Labels when Closed + uses: actions-cool/issues-helper@v2 + with: + actions: remove-labels + token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.issue.number }} + labels: '🚏 Awaiting User Response,⚰️ Stale,πŸ‘€ Awaiting Maintainer Response' \ No newline at end of file diff --git a/.github/workflows/manage-pending-labels.yml b/.github/workflows/manage-pending-labels.yml new file mode 100644 index 000000000..44d91b523 --- /dev/null +++ b/.github/workflows/manage-pending-labels.yml @@ -0,0 +1,42 @@ +# When a new comment is added to an issue, if it had the Stale or Awaiting User Response labels, then those labels will be removed + +name: 🎯 Add/ Remove Awaiting Response Labels +on: + issue_comment: + types: [created] +jobs: + remove-stale: + runs-on: ubuntu-latest + if: ${{ github.event.comment.author_association != 'COLLABORATOR' && github.event.comment.author_association != 'OWNER' }} + steps: + - name: Remove Stale labels when Updated + uses: actions-cool/issues-helper@v2 + with: + actions: remove-labels + token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.issue.number }} + labels: '🚏 Awaiting User Response,⚰️ Stale' + + add-awaiting-author: + runs-on: ubuntu-latest + if: ${{!github.event.issue.pull_request && github.event.comment.author_association != 'COLLABORATOR' && github.event.comment.author_association != 'OWNER' && github.event.issue.state == 'open' }} + steps: + - name: Add Awaiting Author labels when Updated + uses: actions-cool/issues-helper@v2 + with: + actions: add-labels + token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.issue.number }} + labels: 'πŸ‘€ Awaiting Maintainer Response' + + remove-awaiting-author: + runs-on: ubuntu-latest + if: ${{ github.event.comment.author_association == 'OWNER' }} + steps: + - name: Remove Awaiting Author labels when Updated + uses: actions-cool/issues-helper@v2 + with: + actions: remove-labels + token: ${{ secrets.BOT_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.issue.number }} + labels: 'πŸ‘€ Awaiting Maintainer Response' \ No newline at end of file From 1c9b89fdcceee5ebadd17e9b34f23eb9f0b37eab Mon Sep 17 00:00:00 2001 From: deffcolony <61471128+deffcolony@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:48:29 +0200 Subject: [PATCH 002/140] Create issue-auto-comments.yml --- .github/issue-auto-comments.yml | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/issue-auto-comments.yml diff --git a/.github/issue-auto-comments.yml b/.github/issue-auto-comments.yml new file mode 100644 index 000000000..8947d4e1a --- /dev/null +++ b/.github/issue-auto-comments.yml @@ -0,0 +1,62 @@ +comment: + footer: | + --- + > I am a bot, and this is an automated message πŸ€– +labels: + - name: βœ–οΈ Invalid + labeled: + issue: + action: close + body: > + Hello @{{ issue.user.login }} your ticket has been marked as invalid. + Please ensure you follow the issue template, provide all requested info, + and be sure to check the docs + previous issues prior to raising tickets. + pr: + body: Thank you @{{ pull_request.user.login }} for suggesting this. Please follow the pull request templates. + action: close + + - name: πŸ‘©β€πŸ’» Good First Issue + labeled: + issue: + body: > + This issue has been marked as a good first issue for first-time contributors to implement! + This is a great way to support the project, while also improving your skills, you'll also be credited as a contributor once your PR is merged. + If you're new to SillyTavern [here are a collection of resources](https://docs.sillytavern.app/) + If you need any support at all, feel free to reach out via [Discord](https://discord.gg/sillytavern). + + - name: ❌ wontfix + labeled: + issue: + action: close + body: > + This ticked has been marked as 'wontfix', which usually means it is out-of-scope, or not feasible at this time. + You can still fork the project and make the changes yourself. + + - name: βœ… Fixed + labeled: + issue: + body: > + Hello @{{ issue.user.login }}! It looks like all or part of this issue has now been implemented. + + + - name: ‼️ High Priority + labeled: + issue: + body: > + This ticket has been marked as high priority, and has been bumped to the top of the priority list. + You should expect an implementation to be pushed out soon. Thank you for your patience. + + - name: πŸ’€ Spam + labeled: + issue: + action: close + locking: lock + lock_reason: spam + body: > + This issue has been identified as spam, and is now locked. + Users who repeatedly raise spam issues may be blocked or reported. + + - name: β›” Don't Merge + labeled: + pr: + body: This PR has been temporarily blocked from merging. \ No newline at end of file From f75daba6c0206f80e1d1a048df9546de5b847b9a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:38:18 +0300 Subject: [PATCH 003/140] Image inlining hint always visible --- public/css/toggle-dependent.css | 8 -------- 1 file changed, 8 deletions(-) diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 30fb33bcf..834e31c31 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -433,14 +433,6 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint { display: none !important; } -#openai_image_inlining:not(:checked)~#image_inlining_hint { - display: none; -} - -#openai_image_inlining:checked~#image_inlining_hint { - display: block; -} - #smooth_streaming:not(:checked)~#smooth_streaming_speed_control { display: none; } From bc9c70556e8e3ff8c07aaf7a0aed5dc307c0cecb Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:53:46 +0300 Subject: [PATCH 004/140] Clean-up mentions of /public/ --- .github/workflows/update-docs.yml | 43 ------------------- Update-Instructions.txt | 18 +++++--- .../extensions/expressions/settings.html | 2 +- src/util.js | 6 +-- 4 files changed, 16 insertions(+), 53 deletions(-) delete mode 100644 .github/workflows/update-docs.yml diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml deleted file mode 100644 index 567cac607..000000000 --- a/.github/workflows/update-docs.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Update SillyTavern-Docs - -on: - push: - branches: - - main - -jobs: - update_docs: - runs-on: ubuntu-latest - - steps: - - name: Checkout current repository - uses: actions/checkout@v2 - - - name: Checkout SillyTavern-Docs repository - uses: actions/checkout@v2 - with: - repository: SillyTavern/SillyTavern-Docs - path: SillyTavern-Docs - - - name: Clone SillyTavern wiki into SillyTavern-Docs/extensions - run: rm -rf SillyTavern-Docs/extensions && git clone https://github.com/SillyTavern/SillyTavern.wiki.git SillyTavern-Docs/extensions && rm -rf SillyTavern-Docs/extensions/.git - - - name: Copy files - run: | - cp public/notes/content.md SillyTavern-Docs/guidebook.md - cp faq.md SillyTavern-Docs/faq.md - cp readme.md SillyTavern-Docs/readme.md - cp public/notes/update.md SillyTavern-Docs/update.md - - - name: Deploy to external repository - uses: cpina/github-action-push-to-another-repository@main - env: - SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} - with: - # GitHub Action output files - source-directory: SillyTavern-Docs/ - destination-github-username: SillyTavern - destination-repository-name: SillyTavern-Docs - user-email: github-actions[bot]@users.noreply.github.com - user-name: "GitHub Actions" - target-branch: "main" diff --git a/Update-Instructions.txt b/Update-Instructions.txt index f153660b2..6e7071184 100644 --- a/Update-Instructions.txt +++ b/Update-Instructions.txt @@ -33,7 +33,14 @@ If you insist on installing via a zip, here is the tedious process for doing the 2. Unzip it into a folder OUTSIDE of your current ST installation. 3. Do the usual setup procedure for your OS to install the NodeJS requirements. -4. Copy the following files/folders as necessary(*) from your old ST installation: +4a. Updating 1.12.0 and above + +Copy the user data directory from your data root into the data root of the new install. + +By default: /data/default-user + +4a. Migrating from <1.12.0 to >=1.20.0 +Copy the following files/folders as necessary(*) from your old ST installation: - Assets - Backgrounds @@ -54,16 +61,15 @@ If you insist on installing via a zip, here is the tedious process for doing the - Worlds - User - settings.json - - secrets.json <---- this one is in the base folder, not /public/ + - secrets.json <---- This one is in the base folder, not /public/ (*) 'As necessary' = "If you made any custom content related to those folders". None of the folders are mandatory, so only copy what you need. **NB: DO NOT COPY THE ENTIRE /PUBLIC/ FOLDER.** Doing so could break the new install and prevent new features from being present. + Paste those items into the /data/default-user folder of the new install. -5. Paste those items into the /Public/ folder of the new install. +5. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. -6. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. - -7. If everything shows up, you can safely delete the old ST folder. +6. If everything shows up, you can safely delete the old ST folder. diff --git a/public/scripts/extensions/expressions/settings.html b/public/scripts/extensions/expressions/settings.html index 4a7347a74..e8b1484b2 100644 --- a/public/scripts/extensions/expressions/settings.html +++ b/public/scripts/extensions/expressions/settings.html @@ -78,7 +78,7 @@ Remove all image overrides -

Hint: Create new folder in the public/characters/ folder and name it as the name of the character. +

Hint: Create new folder in the /characters/ folder of your user data directory and name it as the name of the character. Put images with expressions there. File names should follow the pattern: [expression_label].[image_format]

diff --git a/src/util.js b/src/util.js index e1410eee8..ab19f3ccf 100644 --- a/src/util.js +++ b/src/util.js @@ -311,9 +311,9 @@ function tryParse(str) { } /** - * Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the - * client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator. - * @param {string} root The root directory of the public folder. + * Takes a path to a client-accessible file in the data folder and converts it to a relative URL segment that the + * client can fetch it from. This involves stripping the data root path prefix and always using `/` as the separator. + * @param {string} root The root directory of the user data folder. * @param {string} inputPath The path to be converted. * @returns The relative URL path from which the client can access the file. */ From df93d43c36de5ffe4cbb28930c6e68ad05f09388 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 00:02:48 +0300 Subject: [PATCH 005/140] Remove obnoxious mobile padding on right panel --- public/css/mobile-styles.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/css/mobile-styles.css b/public/css/mobile-styles.css index 68534d8a2..8456d54a0 100644 --- a/public/css/mobile-styles.css +++ b/public/css/mobile-styles.css @@ -231,9 +231,11 @@ backdrop-filter: blur(calc(var(--SmartThemeBlurStrength) * 2)); } + /* #right-nav-panel { padding-right: 15px; } + */ #floatingPrompt, #cfgConfig, From 41ad7c5d266d33b9432ab94b5cded10ffab29f1a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 02:34:50 +0300 Subject: [PATCH 006/140] Verify data bank attachments --- public/scripts/chats.js | 52 ++++++++++++++++++++++++++++++++++++++--- src/endpoints/files.js | 25 ++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/public/scripts/chats.js b/public/scripts/chats.js index a4082f6f0..19e316fa5 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -592,9 +592,10 @@ async function deleteMessageImage() { /** * Deletes file from the server. * @param {string} url Path to the file on the server + * @param {boolean} [silent=false] If true, do not show error messages * @returns {Promise} True if file was deleted, false otherwise. */ -async function deleteFileFromServer(url) { +async function deleteFileFromServer(url, silent = false) { try { const result = await fetch('/api/files/delete', { method: 'POST', @@ -602,7 +603,7 @@ async function deleteFileFromServer(url) { body: JSON.stringify({ path: url }), }); - if (!result.ok) { + if (!result.ok && !silent) { const error = await result.text(); throw new Error(error); } @@ -702,7 +703,8 @@ async function deleteAttachment(attachment, source, callback, confirm = true) { break; } - await deleteFileFromServer(attachment.url); + const silent = confirm === false; + await deleteFileFromServer(attachment.url, silent); callback(); } @@ -756,6 +758,7 @@ async function openAttachmentManager() { for (const attachment of sortedAttachmentList) { const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone(); + attachmentTemplate.find('.attachmentFileIcon').attr('title', attachment.url); attachmentTemplate.find('.attachmentListItemName').text(attachment.name); attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size)); attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString()); @@ -883,6 +886,7 @@ async function openAttachmentManager() { }); const cleanupFn = await renderButtons(); + await verifyAttachments(); await renderAttachments(); await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' }); @@ -1039,6 +1043,48 @@ export function getDataBankAttachmentsForSource(source) { } } +/** + * Verifies all attachments in the Data Bank. + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachments() { + for (const source of Object.values(ATTACHMENT_SOURCE)) { + await verifyAttachmentsForSource(source); + } +} + +/** + * Verifies all attachments for a specific source. + * @param {string} source Attachment source + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachmentsForSource(source) { + try { + const attachments = getDataBankAttachmentsForSource(source); + const urls = attachments.map(a => a.url); + const response = await fetch('/api/files/verify', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ urls }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error); + } + + const verifiedUrls = await response.json(); + for (const attachment of attachments) { + if (verifiedUrls[attachment.url] === false) { + console.log('Deleting orphaned attachment', attachment); + await deleteAttachment(attachment, source, () => { }, false); + } + } + } catch (error) { + console.error('Attachment verification failed', error); + } +} + /** * Registers a file converter function. * @param {string} mimeType MIME type diff --git a/src/endpoints/files.js b/src/endpoints/files.js index 1c66273bd..371381c21 100644 --- a/src/endpoints/files.js +++ b/src/endpoints/files.js @@ -57,4 +57,29 @@ router.post('/delete', jsonParser, async (request, response) => { } }); +router.post('/verify', jsonParser, async (request, response) => { + try { + if (!Array.isArray(request.body.urls)) { + return response.status(400).send('No URLs specified'); + } + + const verified = {}; + + for (const url of request.body.urls) { + const pathToVerify = path.join(request.user.directories.root, url); + if (!pathToVerify.startsWith(request.user.directories.files)) { + console.debug(`File verification: Invalid path: ${pathToVerify}`); + continue; + } + const fileExists = fs.existsSync(pathToVerify); + verified[url] = fileExists; + } + + return response.send(verified); + } catch (error) { + console.log(error); + return response.sendStatus(500); + } +}); + module.exports = { router }; From 2f45f50d370afc8de4bf3cc2be61c7161e66790c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:52:59 +0300 Subject: [PATCH 007/140] Add config value for forwarded IPs whitelisting --- default/config.yaml | 2 ++ src/middleware/whitelist.js | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/default/config.yaml b/default/config.yaml index 8966447d0..355573d96 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -9,6 +9,8 @@ port: 8000 # -- SECURITY CONFIGURATION -- # Toggle whitelist mode whitelistMode: true +# Whitelist will also verify IP in X-Forwarded-For / X-Real-IP headers +enableForwardedWhitelist: true # Whitelist of allowed IP addresses whitelist: - 127.0.0.1 diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index def408650..24c1af8e5 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -6,6 +6,7 @@ const { getIpFromRequest } = require('../express-common'); const { color, getConfigValue } = require('../util'); const whitelistPath = path.join(process.cwd(), './whitelist.txt'); +const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false); let whitelist = getConfigValue('whitelist', []); let knownIPs = new Set(); @@ -24,14 +25,18 @@ if (fs.existsSync(whitelistPath)) { * @returns {string|undefined} The client IP address */ function getForwardedIp(req) { + if (!enableForwardedWhitelist) { + return undefined; + } + // Check if X-Real-IP is available if (req.headers['x-real-ip']) { - return req.headers['x-real-ip']; + return req.headers['x-real-ip'].toString(); } // Check for X-Forwarded-For and parse if available if (req.headers['x-forwarded-for']) { - const ipList = req.headers['x-forwarded-for'].split(',').map(ip => ip.trim()); + const ipList = req.headers['x-forwarded-for'].toString().split(',').map(ip => ip.trim()); return ipList[0]; } From 5a5463bd5d0123b1d82090ddc08b19db0196f15d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:02:50 +0300 Subject: [PATCH 008/140] #2095 Suppress auto-execution on streamed swiped generations. --- public/scripts/extensions/quick-reply/index.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index 7b58f4aaa..74cdbeb78 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -1,4 +1,4 @@ -import { chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; +import { chat, chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; import { extension_settings } from '../../extensions.js'; import { QuickReplyApi } from './api/QuickReplyApi.js'; import { AutoExecuteHandler } from './src/AutoExecuteHandler.js'; @@ -238,7 +238,12 @@ const onUserMessage = async () => { }; eventSource.on(event_types.USER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onUserMessage, args)); -const onAiMessage = async () => { +const onAiMessage = async (messageId) => { + if (['...'].includes(chat[messageId]?.mes)) { + log('QR auto-execution suppressed for swiped message'); + return; + } + await autoExec.handleAi(); }; eventSource.on(event_types.CHARACTER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onAiMessage, args)); From 776260c85ad4c7d731e9f3059f868fba3e6106da Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:25:46 +0300 Subject: [PATCH 009/140] Add Data Bank to attachments extension display name --- public/scripts/extensions/attachments/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/extensions/attachments/manifest.json b/public/scripts/extensions/attachments/manifest.json index 2037168c2..27f55f77c 100644 --- a/public/scripts/extensions/attachments/manifest.json +++ b/public/scripts/extensions/attachments/manifest.json @@ -1,5 +1,5 @@ { - "display_name": "Chat Attachments", + "display_name": "Data Bank (Chat Attachments)", "loading_order": 3, "requires": [], "optional": [], From 6d1933c8f35c7702cb04a1904b1e12f9a3198d05 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:35:42 +0300 Subject: [PATCH 010/140] Escape name regex in message formatting function --- public/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 9bbb8dfa0..3f28b2e7b 100644 --- a/public/script.js +++ b/public/script.js @@ -1840,7 +1840,7 @@ function messageFormatting(mes, ch_name, isSystem, isUser, messageId) { */ if (!power_user.allow_name2_display && ch_name && !isUser && !isSystem) { - mes = mes.replace(new RegExp(`(^|\n)${ch_name}:`, 'g'), '$1'); + mes = mes.replace(new RegExp(`(^|\n)${escapeRegex(ch_name)}:`, 'g'), '$1'); } /** @type {any} */ From 0f0895f345eb14c60840d382370c66f339145aa9 Mon Sep 17 00:00:00 2001 From: Bronya-Rand Date: Mon, 22 Apr 2024 19:11:00 +0100 Subject: [PATCH 011/140] feat: implement miHoYo scraper --- public/img/mihoyo.svg | 3 + public/scripts/chats.js | 8 +- .../extensions/attachments/mihoyo-scrape.html | 27 +++ public/scripts/scrapers.js | 157 +++++++++++++++++- 4 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 public/img/mihoyo.svg create mode 100644 public/scripts/extensions/attachments/mihoyo-scrape.html diff --git a/public/img/mihoyo.svg b/public/img/mihoyo.svg new file mode 100644 index 000000000..37b266529 --- /dev/null +++ b/public/img/mihoyo.svg @@ -0,0 +1,3 @@ + + + diff --git a/public/scripts/chats.js b/public/scripts/chats.js index 19e316fa5..e9d402a1d 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -789,7 +789,13 @@ async function openAttachmentManager() { } const buttonTemplate = template.find('.actionButtonTemplate .actionButton').clone(); - buttonTemplate.find('.actionButtonIcon').addClass(scraper.iconClass); + if (scraper.iconAvailable) { + buttonTemplate.find('.actionButtonIcon').addClass(scraper.iconClass); + buttonTemplate.find('.actionButtonImg').remove(); + } else { + buttonTemplate.find('.actionButtonImg').attr('src', scraper.iconClass); + buttonTemplate.find('.actionButtonIcon').remove(); + } buttonTemplate.find('.actionButtonText').text(scraper.name); buttonTemplate.attr('title', scraper.description); buttonTemplate.on('click', () => { diff --git a/public/scripts/extensions/attachments/mihoyo-scrape.html b/public/scripts/extensions/attachments/mihoyo-scrape.html new file mode 100644 index 000000000..edd097545 --- /dev/null +++ b/public/scripts/extensions/attachments/mihoyo-scrape.html @@ -0,0 +1,27 @@ +
+
+

miHoYo/HoYoverse HoYoLAB Scraper

+
+

Select a Wiki to parse through.

+
+ +
+
+

+ Enter the Wiki Page ID. +

+
+
+ This is the last digit in the HoYoLAB URL i.e. + https://wiki.hoyolab.com/pc/hsr/entry/X + + Example: + 14 + +
+ +
diff --git a/public/scripts/scrapers.js b/public/scripts/scrapers.js index 10529ee89..4a7311e88 100644 --- a/public/scripts/scrapers.js +++ b/public/scripts/scrapers.js @@ -9,6 +9,7 @@ import { isValidUrl } from './utils.js'; * @property {string} name * @property {string} description * @property {string} iconClass + * @property {boolean} iconAvailable * @property {() => Promise} isAvailable * @property {() => Promise} scrape */ @@ -19,6 +20,7 @@ import { isValidUrl } from './utils.js'; * @property {string} name * @property {string} description * @property {string} iconClass + * @property {boolean} iconAvailable */ export class ScraperManager { @@ -45,7 +47,7 @@ export class ScraperManager { * @returns {ScraperInfo[]} List of scrapers available for the Data Bank */ static getDataBankScrapers() { - return ScraperManager.#scrapers.map(s => ({ id: s.id, name: s.name, description: s.description, iconClass: s.iconClass })); + return ScraperManager.#scrapers.map(s => ({ id: s.id, name: s.name, description: s.description, iconClass: s.iconClass, iconAvailable: s.iconAvailable})); } /** @@ -87,6 +89,7 @@ class Notepad { this.name = 'Notepad'; this.description = 'Create a text file from scratch.'; this.iconClass = 'fa-solid fa-note-sticky'; + this.iconAvailable = true; } /** @@ -133,6 +136,7 @@ class WebScraper { this.name = 'Web'; this.description = 'Download a page from the web.'; this.iconClass = 'fa-solid fa-globe'; + this.iconAvailable = true; } /** @@ -207,6 +211,7 @@ class FileScraper { this.name = 'File'; this.description = 'Upload a file from your computer.'; this.iconClass = 'fa-solid fa-upload'; + this.iconAvailable = true; } /** @@ -243,6 +248,7 @@ class FandomScraper { this.name = 'Fandom'; this.description = 'Download a page from the Fandom wiki.'; this.iconClass = 'fa-solid fa-fire'; + this.iconAvailable = true; } /** @@ -339,6 +345,153 @@ class FandomScraper { } } +/** + * Scrapes data from the miHoYo/HoYoverse HoYoLAB wiki. + * @implements {Scraper} + */ +class miHoYoScraper { + constructor() { + this.id = 'mihoyo'; + this.name = 'miHoYo'; + this.description = 'Scrapes a page from the miHoYo/HoYoverse HoYoLAB wiki.'; + this.iconClass = 'img/mihoyo.svg'; + this.iconAvailable = false; // There is no miHoYo icon in Font Awesome + } + + /** + * Check if the scraper is available. + * @returns {Promise} + */ + async isAvailable() { + try { + const result = await fetch('/api/plugins/hoyoverse/probe', { + method: 'POST', + headers: getRequestHeaders(), + }); + + return result.ok; + } catch (error) { + console.debug('Could not probe miHoYo plugin', error); + return false; + } + } + + /** + * Outputs Data Information in a human-readable format. + * @param {Object} m Data to be parsed + * @returns {string} Human-readable format of the data + */ + parseOutput(m) { + let temp = ''; + for (const d in m) { + if (m[d].key === "") { + temp += `- ${m[d].value}\n`; + continue; + } + temp += `- ${m[d].key}: ${m[d].value}\n`; + } + return temp; + } + + /** Scrape data from the miHoYo/HoYoverse HoYoLAB wiki. + * @returns {Promise} File attachments scraped from the wiki. + */ + + async scrape() { + let miHoYoWiki = ''; + let miHoYoWikiID = ''; + + const template = $(await renderExtensionTemplateAsync('attachments', 'mihoyo-scrape', {})); + + template.find('select[name="mihoyoScrapeWikiDropdown"]').on('change', function () { + miHoYoWiki = String($(this).val()); + }); + template.find('input[name="mihoyoScrapeWikiID"]').on('input', function () { + miHoYoWikiID = String($(this).val()); + }); + + const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false }); + + if (confirm !== POPUP_RESULT.AFFIRMATIVE) { + return; + } + + if (!miHoYoWiki) { + toastr.error('A specific HoYoLab wiki is required'); + return; + } + + if (!miHoYoWikiID) { + toastr.error('A specific HoYoLab wiki ID is required'); + return; + } + + if (miHoYoWiki === 'genshin') { + toastr.error('The Genshin Impact parser has not been implemented *yet*'); + return; + } + + let toast; + if (miHoYoWiki === 'hsr') { + toast = toastr.info(`Scraping the Honkai: Star Rail HoYoLAB wiki for Wiki Entry ID: ${miHoYoWikiID}`); + } else { + toast = toastr.info(`Scraping the Genshin Impact wiki for Wiki Entry ID: ${miHoYoWikiID}`); + } + + let result; + if (miHoYoWiki === 'hsr') { + result = await fetch('/api/plugins/hoyoverse/silver-wolf', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ miHoYoWiki, miHoYoWikiID }), + }); + } else if (miHoYoWiki === 'genshin') { + result = await fetch('/api/plugins/hoyoverse/furina', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ miHoYoWiki, miHoYoWikiID }), + }); + } else { + throw new Error('Unknown wiki name identifier'); + } + + if (!result.ok) { + const error = await result.text(); + throw new Error(error); + } + + const data = await result.json(); + toastr.clear(toast); + + const fileName = data[0].name; + const dataContent = data[0].content; + + //parse the data as a long string of data + let combinedContent = ''; + combinedContent += `Name: ${data[0].name}\n`; + + if (dataContent.description !== "") { + combinedContent += `Description: ${dataContent.description}\n\n`; + } + + if (dataContent.modules != []) { + for (const m in dataContent.modules) { + if (dataContent.modules[m].data.length === 0) { + continue; + } + combinedContent += dataContent.modules[m].name + '\n'; + combinedContent += this.parseOutput(dataContent.modules[m].data); + combinedContent += '\n'; + } + } + + const file = new File([combinedContent], `${fileName}.txt`, { type: 'text/plain' }); + + return [file]; + } +} + + /** * Scrape transcript from a YouTube video. * @implements {Scraper} @@ -349,6 +502,7 @@ class YouTubeScraper { this.name = 'YouTube'; this.description = 'Download a transcript from a YouTube video.'; this.iconClass = 'fa-solid fa-closed-captioning'; + this.iconAvailable = true; } /** @@ -413,4 +567,5 @@ ScraperManager.registerDataBankScraper(new FileScraper()); ScraperManager.registerDataBankScraper(new Notepad()); ScraperManager.registerDataBankScraper(new WebScraper()); ScraperManager.registerDataBankScraper(new FandomScraper()); +ScraperManager.registerDataBankScraper(new miHoYoScraper()); ScraperManager.registerDataBankScraper(new YouTubeScraper()); From 770f3e5da3bd70139146248372c089f12fc1f61e Mon Sep 17 00:00:00 2001 From: Bronya-Rand Date: Mon, 22 Apr 2024 19:12:02 +0100 Subject: [PATCH 012/140] chore: apply align-items center and img sample for img only scraper icons --- public/scripts/extensions/attachments/manager.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/scripts/extensions/attachments/manager.html b/public/scripts/extensions/attachments/manager.html index ecfbe2fbb..8cff50fe0 100644 --- a/public/scripts/extensions/attachments/manager.html +++ b/public/scripts/extensions/attachments/manager.html @@ -108,8 +108,9 @@
-
+
+
From 4370db6bdc7f6959caa378c27add3d0c16508eb8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:09:52 +0300 Subject: [PATCH 013/140] Implement World Info activation using Vector Storage --- public/index.html | 11 +- public/script.js | 1 + public/scripts/extensions/vectors/index.js | 135 ++++++++++++++++++ .../scripts/extensions/vectors/settings.html | 40 ++++++ public/scripts/world-info.js | 77 ++++++++-- src/endpoints/characters.js | 1 + 6 files changed, 250 insertions(+), 15 deletions(-) diff --git a/public/index.html b/public/index.html index ab7329d76..75aaf4eac 100644 --- a/public/index.html +++ b/public/index.html @@ -4955,10 +4955,11 @@
- + + + +
@@ -6065,4 +6066,4 @@ - \ No newline at end of file + diff --git a/public/script.js b/public/script.js index 3f28b2e7b..a1df1eeb2 100644 --- a/public/script.js +++ b/public/script.js @@ -449,6 +449,7 @@ export const event_types = { CHARACTER_DUPLICATED: 'character_duplicated', SMOOTH_STREAM_TOKEN_RECEIVED: 'smooth_stream_token_received', FILE_ATTACHMENT_DELETED: 'file_attachment_deleted', + WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate', }; export const eventSource = new EventEmitter(); diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 6ff5c6af5..5cdd0830b 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -23,6 +23,7 @@ import { collapseNewlines } from '../../power-user.js'; import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js'; import { getDataBankAttachments, getFileAttachment } from '../../chats.js'; import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js'; +import { getSortedEntries } from '../../world-info.js'; const MODULE_NAME = 'vectors'; @@ -66,6 +67,11 @@ const settings = { file_position_db: extension_prompt_types.IN_PROMPT, file_depth_db: 4, file_depth_role_db: extension_prompt_roles.SYSTEM, + + // For World Info + enabled_world_info: false, + enabled_for_all: false, + max_entries: 5, }; const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); @@ -472,6 +478,10 @@ async function rearrangeChat(chat) { await processFiles(chat); } + if (settings.enabled_world_info) { + await activateWorldInfo(chat); + } + if (!settings.enabled_chats) { return; } @@ -845,6 +855,7 @@ async function purgeVectorIndex(collectionId) { function toggleSettings() { $('#vectors_files_settings').toggle(!!settings.enabled_files); $('#vectors_chats_settings').toggle(!!settings.enabled_chats); + $('#vectors_world_info_settings').toggle(!!settings.enabled_world_info); $('#together_vectorsModel').toggle(settings.source === 'togetherai'); $('#openai_vectorsModel').toggle(settings.source === 'openai'); $('#cohere_vectorsModel').toggle(settings.source === 'cohere'); @@ -934,6 +945,111 @@ async function onPurgeFilesClick() { } } +async function activateWorldInfo(chat) { + if (!settings.enabled_world_info) { + console.debug('Vectors: Disabled for World Info'); + return; + } + + const entries = await getSortedEntries(); + + if (!Array.isArray(entries) || entries.length === 0) { + console.debug('Vectors: No WI entries found'); + return; + } + + // Group entries by "world" field + const groupedEntries = {}; + + for (const entry of entries) { + // Skip orphaned entries. Is it even possible? + if (!entry.world) { + console.debug('Vectors: Skipped orphaned WI entry', entry); + continue; + } + + // Skip disabled entries + if (entry.disable) { + console.debug('Vectors: Skipped disabled WI entry', entry); + continue; + } + + // Skip entries without content + if (!entry.content) { + console.debug('Vectors: Skipped WI entry without content', entry); + continue; + } + + // Skip non-vectorized entries + if (!entry.vectorized && !settings.enabled_for_all) { + console.debug('Vectors: Skipped non-vectorized WI entry', entry); + continue; + } + + if (!Object.hasOwn(groupedEntries, entry.world)) { + groupedEntries[entry.world] = []; + } + + groupedEntries[entry.world].push(entry); + } + + const collectionIds = []; + + if (Object.keys(groupedEntries).length === 0) { + console.debug('Vectors: No WI entries to synchronize'); + return; + } + + // Synchronize collections + for (const world in groupedEntries) { + const collectionId = `world_${getStringHash(world)}`; + const hashesInCollection = await getSavedHashes(collectionId); + const newEntries = groupedEntries[world].filter(x => !hashesInCollection.includes(getStringHash(x.content))); + const deletedHashes = hashesInCollection.filter(x => !groupedEntries[world].some(y => getStringHash(y.content) === x)); + + if (newEntries.length > 0) { + console.log(`Vectors: Found ${newEntries.length} new WI entries for world ${world}`); + await insertVectorItems(collectionId, newEntries.map(x => ({ hash: getStringHash(x.content), text: x.content, index: x.uid }))); + } + + if (deletedHashes.length > 0) { + console.log(`Vectors: Deleted ${deletedHashes.length} old hashes for world ${world}`); + await deleteVectorItems(collectionId, deletedHashes); + } + + collectionIds.push(collectionId); + } + + // Perform a multi-query + const queryText = await getQueryText(chat); + + if (queryText.length === 0) { + console.debug('Vectors: No text to query for WI'); + return; + } + + const queryResults = await queryMultipleCollections(collectionIds, queryText, settings.max_entries); + const activatedHashes = Object.values(queryResults).flatMap(x => x.hashes).filter(onlyUnique); + const activatedEntries = []; + + // Activate entries found in the query results + for (const entry of entries) { + const hash = getStringHash(entry.content); + + if (activatedHashes.includes(hash)) { + activatedEntries.push(entry); + } + } + + if (activatedEntries.length === 0) { + console.debug('Vectors: No activated WI entries found'); + return; + } + + console.log(`Vectors: Activated ${activatedEntries.length} WI entries`, activatedEntries); + await eventSource.emit(event_types.WORLDINFO_FORCE_ACTIVATE, activatedEntries); +} + jQuery(async () => { if (!extension_settings.vectors) { extension_settings.vectors = settings; @@ -1134,6 +1250,25 @@ jQuery(async () => { saveSettingsDebounced(); }); + $('#vectors_enabled_world_info').prop('checked', settings.enabled_world_info).on('input', () => { + settings.enabled_world_info = !!$('#vectors_enabled_world_info').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + toggleSettings(); + }); + + $('#vectors_enabled_for_all').prop('checked', settings.enabled_for_all).on('input', () => { + settings.enabled_for_all = !!$('#vectors_enabled_for_all').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + + $('#vectors_max_entries').val(settings.max_entries).on('input', () => { + settings.max_entries = Number($('#vectors_max_entries').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + const validSecret = !!secret_state[SECRET_KEYS.NOMICAI]; const placeholder = validSecret ? 'βœ”οΈ Key saved' : '❌ Missing key'; $('#api_key_nomicai').attr('placeholder', placeholder); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index 02499d120..bcaf1c06e 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -97,6 +97,46 @@
+

+ World Info settings +

+ + + +
+
+ +
    +
  • + Checked: all entries except ❌ status can be activated. +
  • +
  • + Unchecked: only entries with πŸ”— status can be activated. +
  • +
+
+
+
+ +
+
+ + +
+
+ +
+
+
+

File vectorization settings

diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 2932e9050..767f923d0 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -82,6 +82,11 @@ class WorldInfoBuffer { /** @typedef {{scanDepth?: number, caseSensitive?: boolean, matchWholeWords?: boolean}} WIScanEntry The entry that triggered the scan */ // End typedef area + /** + * @type {object[]} Array of entries that need to be activated no matter what + */ + static externalActivations = []; + /** * @type {string[]} Array of messages sorted by ascending depth */ @@ -220,6 +225,23 @@ class WorldInfoBuffer { getDepth() { return world_info_depth + this.#skew; } + + /** + * Check if the current entry is externally activated. + * @param {object} entry WI entry to check + * @returns {boolean} True if the entry is forcefully activated + */ + isExternallyActivated(entry) { + // Entries could be copied with structuredClone, so we need to compare them by string representation + return WorldInfoBuffer.externalActivations.some(x => JSON.stringify(x) === JSON.stringify(entry)); + } + + /** + * Clears the force activations buffer. + */ + cleanExternalActivations() { + WorldInfoBuffer.externalActivations.splice(0, WorldInfoBuffer.externalActivations.length); + } } export function getWorldInfoSettings() { @@ -362,6 +384,10 @@ function setWorldInfoSettings(settings, data) { $('.chat_lorebook_button').toggleClass('world_set', hasWorldInfo); }); + eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => { + WorldInfoBuffer.externalActivations.push(...entries); + }); + // Add slash commands registerWorldInfoSlashCommands(); } @@ -564,6 +590,7 @@ function registerWorldInfoSlashCommands() { return ''; } + registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); registerSlashCommand('getchatbook', getChatBookCallback, ['getchatlore', 'getchatwi'], '– get a name of the chat-bound lorebook or create a new one if was unbound, and pass it down the pipe', true, true); registerSlashCommand('findentry', findBookEntryCallback, ['findlore', 'findwi'], '(file=bookName field=field [texts]) – find a UID of the record from the specified book using the fuzzy match of a field value (default: key) and pass it down the pipe, e.g. /findentry file=chatLore field=key Shadowfang', true, true); registerSlashCommand('getentryfield', getEntryFieldCallback, ['getlorefield', 'getwifield'], '(file=bookName field=field [UID]) – get a field value (default: content) of the record with the UID from the specified book and pass it down the pipe, e.g. /getentryfield file=chatLore field=content 123', true, true); @@ -964,6 +991,7 @@ const originalDataKeyMap = { 'caseSensitive': 'extensions.case_sensitive', 'scanDepth': 'extensions.scan_depth', 'automationId': 'extensions.automation_id', + 'vectorized': 'extensions.vectorized', }; function setOriginalDataValue(data, uid, key, value) { @@ -1071,6 +1099,16 @@ function getWorldEntry(name, data, entry) { ); } + // Verify names to exist in the system + if (data.entries[uid]?.characterFilter?.names?.length > 0) { + for (const name of [...data.entries[uid].characterFilter.names]) { + if (!getContext().characters.find(x => x.avatar.replace(/\.[^/.]+$/, '') === name)) { + console.warn(`World Info: Character ${name} not found. Removing from the entry filter.`, entry); + data.entries[uid].characterFilter.names = data.entries[uid].characterFilter.names.filter(x => x !== name); + } + } + } + setOriginalDataValue(data, uid, 'character_filter', data.entries[uid].characterFilter); saveWorldInfo(name, data); }); @@ -1454,22 +1492,37 @@ function getWorldEntry(name, data, entry) { case 'constant': data.entries[uid].constant = true; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', true); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.removeClass('disabledWIEntry'); break; case 'normal': data.entries[uid].constant = false; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); + template.removeClass('disabledWIEntry'); + break; + case 'vectorized': + data.entries[uid].constant = false; + data.entries[uid].disable = false; + data.entries[uid].vectorized = true; + setOriginalDataValue(data, uid, 'enabled', true); + setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', true); template.removeClass('disabledWIEntry'); break; case 'disabled': data.entries[uid].constant = false; data.entries[uid].disable = true; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', false); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.addClass('disabledWIEntry'); break; } @@ -1480,6 +1533,8 @@ function getWorldEntry(name, data, entry) { const entryState = function () { if (entry.constant === true) { return 'constant'; + } else if (entry.vectorized === true) { + return 'vectorized'; } else if (entry.disable === true) { return 'disabled'; } else { @@ -1719,6 +1774,7 @@ const newEntryTemplate = { comment: '', content: '', constant: false, + vectorized: false, selective: true, selectiveLogic: world_info_logic.AND_ANY, addMemo: false, @@ -1925,7 +1981,7 @@ async function getCharacterLore() { } const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1941,7 +1997,7 @@ async function getGlobalLore() { let entries = []; for (const worldName of selected_world_info) { const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1963,14 +2019,14 @@ async function getChatLore() { } const data = await loadWorldInfoData(chatWorld); - const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: chatWorld })) : []; console.debug(`Chat lore has ${entries.length} entries`); return entries; } -async function getSortedEntries() { +export async function getSortedEntries() { try { const globalLore = await getGlobalLore(); const characterLore = await getCharacterLore(); @@ -2098,7 +2154,7 @@ async function checkWorldInfo(chat, maxContext) { continue; } - if (entry.constant) { + if (entry.constant || buffer.isExternallyActivated(entry)) { entry.content = substituteParams(entry.content); activatedNow.add(entry); continue; @@ -2295,6 +2351,8 @@ async function checkWorldInfo(chat, maxContext) { context.setExtensionPrompt(NOTE_MODULE_NAME, ANWithWI, chat_metadata[metadata_keys.position], chat_metadata[metadata_keys.depth], extension_settings.note.allowWIScan, chat_metadata[metadata_keys.role]); } + buffer.cleanExternalActivations(); + return { worldInfoBefore, worldInfoAfter, WIDepthEntries, allActivatedEntries }; } @@ -2381,6 +2439,7 @@ function convertAgnaiMemoryBook(inputObj) { content: entry.entry, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.weight, position: 0, @@ -2415,6 +2474,7 @@ function convertRisuLorebook(inputObj) { content: entry.content, constant: entry.alwaysActive, selective: entry.selective, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.insertorder, position: world_info_position.before, @@ -2454,6 +2514,7 @@ function convertNovelLorebook(inputObj) { content: entry.text, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.contextConfig?.budgetPriority ?? 0, position: 0, @@ -2510,6 +2571,7 @@ function convertCharacterBook(characterBook) { matchWholeWords: entry.extensions?.match_whole_words ?? null, automationId: entry.extensions?.automation_id ?? '', role: entry.extensions?.role ?? extension_prompt_roles.SYSTEM, + vectorized: entry.extensions?.vectorized ?? false, }; }); @@ -2785,11 +2847,6 @@ function assignLorebookToChat() { jQuery(() => { - $(document).ready(function () { - registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); - }); - - $('#world_info').on('mousedown change', async function (e) { // If there's no world names, don't do anything if (world_names.length === 0) { diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 556a8fecc..571eec8c0 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -439,6 +439,7 @@ function convertWorldInfoToCharacterBook(name, entries) { case_sensitive: entry.caseSensitive ?? null, automation_id: entry.automationId ?? '', role: entry.role ?? 0, + vectorized: entry.vectorized ?? false, }, }; From d97f0a4c4d875d09591b8f4e63251020c3c3da5d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:18:45 +0300 Subject: [PATCH 014/140] Add new NAI Diffusion model --- public/scripts/extensions/stable-diffusion/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index d7b0cfc0b..88fdbad40 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -1657,6 +1657,10 @@ async function loadNovelModels() { value: 'safe-diffusion', text: 'NAI Diffusion Anime V1 (Curated)', }, + { + value: 'nai-diffusion-furry-3', + text: 'NAI Diffusion Furry V3', + }, { value: 'nai-diffusion-furry', text: 'NAI Diffusion Furry', From 890cf8162781ed1e007dc86e506cc4cfb55387eb Mon Sep 17 00:00:00 2001 From: joenunezb Date: Tue, 23 Apr 2024 03:56:50 -0700 Subject: [PATCH 015/140] Fix: InformaticAI response without message in choices --- src/endpoints/backends/text-completions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 22806cbf0..0e9598827 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -325,7 +325,7 @@ router.post('/generate', jsonParser, async function (request, response) { // Map InfermaticAI response to OAI completions format if (apiType === TEXTGEN_TYPES.INFERMATICAI) { - data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content })); + data['choices'] = (data?.choices || []).map(choice => ({ text: choice?.message?.content || choice.text })); } return response.send(data); From 75372ad0ccbfcc3b60010e3177ebd27a2afd8835 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:15:54 +0300 Subject: [PATCH 016/140] Use Map for caches instead of objects --- public/scripts/extensions/vectors/index.js | 12 +++++++----- src/endpoints/classify.js | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 5cdd0830b..ca2ddbdeb 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -287,8 +287,10 @@ async function synchronizeChat(batchSize = 5) { } } -// Cache object for storing hash values -const hashCache = {}; +/** + * @type {Map} Cache object for storing hash values + */ +const hashCache = new Map(); /** * Gets the hash value for a given string @@ -297,15 +299,15 @@ const hashCache = {}; */ function getStringHash(str) { // Check if the hash is already in the cache - if (Object.hasOwn(hashCache, str)) { - return hashCache[str]; + if (hashCache.has(str)) { + return hashCache.get(str); } // Calculate the hash value const hash = calculateHash(str); // Store the hash in the cache - hashCache[str] = hash; + hashCache.set(str, hash); return hash; } diff --git a/src/endpoints/classify.js b/src/endpoints/classify.js index 5a9772e1d..758b95247 100644 --- a/src/endpoints/classify.js +++ b/src/endpoints/classify.js @@ -5,7 +5,10 @@ const TASK = 'text-classification'; const router = express.Router(); -const cacheObject = {}; +/** + * @type {Map} Cache for classification results + */ +const cacheObject = new Map(); router.post('/labels', jsonParser, async (req, res) => { try { @@ -23,15 +26,20 @@ router.post('/', jsonParser, async (req, res) => { try { const { text } = req.body; + /** + * Get classification result for a given text + * @param {string} text Text to classify + * @returns {Promise} Classification result + */ async function getResult(text) { - if (Object.hasOwn(cacheObject, text)) { - return cacheObject[text]; + if (cacheObject.has(text)) { + return cacheObject.get(text); } else { const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); - cacheObject[text] = result; + cacheObject.set(text, result); return result; } } From a421af9ea9c8a2b9246feb37cdff5fc79d63faba Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:06:59 +0300 Subject: [PATCH 017/140] Increase max attachment size --- public/scripts/chats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/chats.js b/public/scripts/chats.js index 19e316fa5..f5fa6a851 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -53,7 +53,7 @@ import { ScraperManager } from './scrapers.js'; * @returns {Promise} Converted file text */ -const fileSizeLimit = 1024 * 1024 * 10; // 10 MB +const fileSizeLimit = 1024 * 1024 * 100; // 100 MB const ATTACHMENT_SOURCE = { GLOBAL: 'global', CHAT: 'chat', From 71f41d52330f5b4e9993e00e01803638d500688b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:11:47 +0300 Subject: [PATCH 018/140] Fix server crash in auto login --- src/users.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/users.js b/src/users.js index 831ff09d5..023ddca07 100644 --- a/src/users.js +++ b/src/users.js @@ -511,7 +511,7 @@ async function tryAutoLogin(request) { const userHandles = await getAllUserHandles(); if (userHandles.length === 1) { const user = await storage.getItem(toKey(userHandles[0])); - if (!user.password) { + if (user && !user.password) { request.session.handle = userHandles[0]; return true; } From b6b9b542d7a10f8a129321967db24b3dbabd1366 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 01:51:54 +0300 Subject: [PATCH 019/140] Add drag&drop to data bank --- public/scripts/chats.js | 57 +++++++++++++++++++ .../extensions/attachments/files-dropped.html | 8 +++ public/style.css | 7 ++- 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 public/scripts/extensions/attachments/files-dropped.html diff --git a/public/scripts/chats.js b/public/scripts/chats.js index f5fa6a851..778ee7b6a 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -863,6 +863,61 @@ async function openAttachmentManager() { template.find('.chatAttachmentsName').text(chatName); } + function addDragAndDrop() { + $(document.body).on('dragover', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').addClass('dragover'); + }); + + $(document.body).on('dragleave', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + }); + + $(document.body).on('drop', '.dialogue_popup', async (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + + const files = Array.from(event.originalEvent.dataTransfer.files); + const targets = Object.values(ATTACHMENT_SOURCE); + + const isNotCharacter = this_chid === undefined || selected_group; + const isNotInChat = getCurrentChatId() === undefined; + let selectedTarget = ATTACHMENT_SOURCE.GLOBAL; + + if (isNotCharacter) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); + } + + if (isNotInChat) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); + } + + const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets })); + targetSelectTemplate.find('.droppedFilesTarget').on('input', function () { + selectedTarget = String($(this).val()); + }); + const result = await callGenericPopup(targetSelectTemplate, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Upload', cancelButton: 'Cancel' }); + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.log('File upload cancelled'); + return; + } + for (const file of files) { + await uploadFileAttachmentToServer(file, selectedTarget); + } + renderAttachments(); + }); + } + + function removeDragAndDrop() { + $(document.body).off('dragover', '.shadow_popup'); + $(document.body).off('dragleave', '.shadow_popup'); + $(document.body).off('drop', '.shadow_popup'); + } + let sortField = localStorage.getItem('DataBank_sortField') || 'created'; let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc'; let filterString = ''; @@ -888,9 +943,11 @@ async function openAttachmentManager() { const cleanupFn = await renderButtons(); await verifyAttachments(); await renderAttachments(); + addDragAndDrop(); await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' }); cleanupFn(); + removeDragAndDrop(); } /** diff --git a/public/scripts/extensions/attachments/files-dropped.html b/public/scripts/extensions/attachments/files-dropped.html new file mode 100644 index 000000000..7295c4994 --- /dev/null +++ b/public/scripts/extensions/attachments/files-dropped.html @@ -0,0 +1,8 @@ +
+ Save {{count}} file(s) to... + +
diff --git a/public/style.css b/public/style.css index 1535c0b4d..9cde87423 100644 --- a/public/style.css +++ b/public/style.css @@ -2262,6 +2262,11 @@ grammarly-extension { } } +.dialogue_popup.dragover { + filter: brightness(1.1) saturate(1.1); + outline: 3px dashed var(--SmartThemeBorderColor); +} + #bgtest { display: none; width: 100vw; @@ -4015,4 +4020,4 @@ body:not(.movingUI) .drawer-content.maximized { height: 100vh; z-index: 9999; } -} \ No newline at end of file +} From 61241df0d4a7025b5cca8515245937c0d260fcd5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:33:16 +0300 Subject: [PATCH 020/140] Add download and move for DB attachments --- public/scripts/chats.js | 87 ++++++++++++++++--- .../extensions/attachments/files-dropped.html | 2 +- .../extensions/attachments/manager.html | 2 + .../attachments/move-attachment.html | 8 ++ 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 public/scripts/extensions/attachments/move-attachment.html diff --git a/public/scripts/chats.js b/public/scripts/chats.js index 778ee7b6a..0d2a9706a 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -56,8 +56,8 @@ import { ScraperManager } from './scrapers.js'; const fileSizeLimit = 1024 * 1024 * 100; // 100 MB const ATTACHMENT_SOURCE = { GLOBAL: 'global', - CHAT: 'chat', CHARACTER: 'character', + CHAT: 'chat', }; /** @@ -670,6 +670,55 @@ async function editAttachment(attachment, source, callback) { callback(); } +/** + * Downloads an attachment to the user's device. + * @param {FileAttachment} attachment Attachment to download + */ +async function downloadAttachment(attachment) { + const fileText = attachment.text || (await getFileAttachment(attachment.url)); + const blob = new Blob([fileText], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = attachment.name; + a.click(); + URL.revokeObjectURL(url); +} + +/** + * Moves a file attachment to a different source. + * @param {FileAttachment} attachment Attachment to moves + * @param {string} source Source of the attachment + * @param {function} callback Success callback + * @returns {Promise} A promise that resolves when the attachment is moved. + */ +async function moveAttachment(attachment, source, callback) { + let selectedTarget = source; + const targets = getAvailableTargets(); + const template = $(await renderExtensionTemplateAsync('attachments', 'move-attachment', { name: attachment.name, targets })); + template.find('.moveAttachmentTarget').val(source).on('input', function () { + selectedTarget = String($(this).val()); + }); + + const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Move', cancelButton: 'Cancel' }); + + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.debug('Move attachment cancelled'); + return; + } + + if (selectedTarget === source) { + console.debug('Move attachment cancelled: same source and target'); + return; + } + + const content = await getFileAttachment(attachment.url); + const file = new File([content], attachment.name, { type: 'text/plain' }); + await deleteAttachment(attachment, source, () => { }, false); + await uploadFileAttachmentToServer(file, selectedTarget); + callback(); +} + /** * Deletes an attachment from the server and the chat. * @param {FileAttachment} attachment Attachment to delete @@ -765,6 +814,8 @@ async function openAttachmentManager() { attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment)); attachmentTemplate.find('.editAttachmentButton').on('click', () => editAttachment(attachment, source, renderAttachments)); attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments)); + attachmentTemplate.find('.downloadAttachmentButton').on('click', () => downloadAttachment(attachment)); + attachmentTemplate.find('.moveAttachmentButton').on('click', () => moveAttachment(attachment, source, renderAttachments)); template.find(sources[source]).append(attachmentTemplate); } } @@ -882,19 +933,8 @@ async function openAttachmentManager() { $(event.target).closest('.dialogue_popup').removeClass('dragover'); const files = Array.from(event.originalEvent.dataTransfer.files); - const targets = Object.values(ATTACHMENT_SOURCE); - - const isNotCharacter = this_chid === undefined || selected_group; - const isNotInChat = getCurrentChatId() === undefined; let selectedTarget = ATTACHMENT_SOURCE.GLOBAL; - - if (isNotCharacter) { - targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); - } - - if (isNotInChat) { - targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); - } + const targets = getAvailableTargets(); const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets })); targetSelectTemplate.find('.droppedFilesTarget').on('input', function () { @@ -950,6 +990,27 @@ async function openAttachmentManager() { removeDragAndDrop(); } +/** + * Gets a list of available targets for attachments. + * @returns {string[]} List of available targets + */ +function getAvailableTargets() { + const targets = Object.values(ATTACHMENT_SOURCE); + + const isNotCharacter = this_chid === undefined || selected_group; + const isNotInChat = getCurrentChatId() === undefined; + + if (isNotCharacter) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); + } + + if (isNotInChat) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); + } + + return targets; +} + /** * Runs a known scraper on a source and saves the result as an attachment. * @param {string} scraperId Id of the scraper diff --git a/public/scripts/extensions/attachments/files-dropped.html b/public/scripts/extensions/attachments/files-dropped.html index 7295c4994..9fd014ded 100644 --- a/public/scripts/extensions/attachments/files-dropped.html +++ b/public/scripts/extensions/attachments/files-dropped.html @@ -1,4 +1,4 @@ -
+
Save {{count}} file(s) to... + {{#each targets}} + + {{/each}} + +
From 2bba186c9efa420df544fec0422a9d24c04059e5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:37:57 +0300 Subject: [PATCH 021/140] Add slash command and d&d hint for data bank --- public/scripts/extensions/attachments/index.js | 3 +++ public/scripts/extensions/attachments/manager.html | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/attachments/index.js b/public/scripts/extensions/attachments/index.js index a7f58dbc7..ab3db9419 100644 --- a/public/scripts/extensions/attachments/index.js +++ b/public/scripts/extensions/attachments/index.js @@ -1,6 +1,9 @@ import { renderExtensionTemplateAsync } from '../../extensions.js'; +import { registerSlashCommand } from '../../slash-commands.js'; jQuery(async () => { const buttons = await renderExtensionTemplateAsync('attachments', 'buttons', {}); $('#extensionsMenu').prepend(buttons); + + registerSlashCommand('db', () => document.getElementById('manageAttachments')?.click(), ['databank', 'data-bank'], '– open the data bank', true, true); }); diff --git a/public/scripts/extensions/attachments/manager.html b/public/scripts/extensions/attachments/manager.html index 8ad45a98e..68ad732a2 100644 --- a/public/scripts/extensions/attachments/manager.html +++ b/public/scripts/extensions/attachments/manager.html @@ -7,8 +7,13 @@
These files will be available for extensions that support attachments (e.g. Vector Storage).
-
- Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. +
+ + Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. + + + Drag and drop files here to upload. +
From 530bf81940d0d0ad4d082b701736f9bb235ac4fa Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:48:08 +0300 Subject: [PATCH 022/140] #2127 Encode export PNG name --- src/endpoints/characters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 571eec8c0..929818ade 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -1098,7 +1098,7 @@ router.post('/export', jsonParser, async function (request, response) { const fileContent = await fsPromises.readFile(filename); const contentType = mime.lookup(filename) || 'image/png'; response.setHeader('Content-Type', contentType); - response.setHeader('Content-Disposition', `attachment; filename=${path.basename(filename)}`); + response.setHeader('Content-Disposition', `attachment; filename="${encodeURI(path.basename(filename))}"`); return response.send(fileContent); } case 'json': { From 51014e7a8d2aa40f633dbe8fab8995ad2ae49f5e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:54:55 +0300 Subject: [PATCH 023/140] Fix VRM assets console spam --- src/endpoints/assets.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 78f5270a7..09cc1aa71 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -56,6 +56,8 @@ function validateAssetFileName(inputFilename) { * @returns {string[]} - The array of files */ function getFiles(dir, files = []) { + if (!fs.existsSync(dir)) return files; + // Get an array of all files and directories in the passed directory using fs.readdirSync const fileList = fs.readdirSync(dir, { withFileTypes: true }); // Create the full path of the file/directory by concatenating the passed directory and file/directory name From b1c199e650171cb76c8765d586d7a86903808fe2 Mon Sep 17 00:00:00 2001 From: Yokayo <52032299+Yokayo@users.noreply.github.com> Date: Wed, 24 Apr 2024 19:02:00 +0700 Subject: [PATCH 024/140] Add more localizable strings --- public/index.html | 97 ++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/public/index.html b/public/index.html index 75aaf4eac..184901e53 100644 --- a/public/index.html +++ b/public/index.html @@ -2810,7 +2810,7 @@ - View hidden API keys + View hidden API keys
@@ -3714,7 +3714,7 @@ Smooth Streaming - +
Speed @@ -4062,7 +4062,7 @@ Chat Backgrounds
- Chat backgrounds generated with the  Image Generation extension will appear here. + Chat backgrounds generated with the  Image Generation extension will appear here.
@@ -4266,7 +4266,7 @@
- Tokens: counting... + Tokens: counting...
@@ -4368,7 +4368,7 @@
- Tokens: counting... + Tokens: counting...
@@ -4388,7 +4388,7 @@
- Tokens: counting... + Tokens: counting...
@@ -4626,8 +4626,8 @@
-

- Prompt Overrides (For OpenAI/Claude/Scale APIs, Window/OpenRouter, and Instruct Mode) +

+ Prompt Overrides (For OpenAI/Claude/Scale APIs, Window/OpenRouter, and Instruct Mode)

@@ -4637,14 +4637,14 @@

Main Prompt

- Tokens: counting... + Tokens: counting...

Jailbreak

- Tokens: counting... + Tokens: counting...
@@ -4652,9 +4652,9 @@
-

- Creator's Metadata - (Not sent with the AI Prompt) +

+ Creator's Metadata + (Not sent with the AI Prompt)

@@ -4690,7 +4690,7 @@
- Tokens: counting... + Tokens: counting...
@@ -4702,7 +4702,7 @@
- Tokens: counting... + Tokens: counting...
@@ -4712,7 +4712,7 @@ Character's Note - +

@@ -4732,13 +4732,14 @@
- Tokens: counting... + Tokens: counting...

Talkativeness

-
How often the character speaks in  group chats! +
+ How often the character speaks in group chats!
@@ -4755,7 +4756,7 @@
- Tokens: counting... + Tokens: counting...
+ Chat Lorebook for +
@@ -5074,7 +5077,7 @@ Content - (Tokens:  counting...)  + (Tokens:  counting...
@@ -5558,17 +5561,17 @@
- Author's Note + Author's Note
- Unique to this chat.
- Checkpoints inherit the Note from their parent, and can be changed individually after that.
+ Unique to this chat.
+ Checkpoints inherit the Note from their parent, and can be changed individually after that.
- Tokens: 0 + Tokens: 0

- User inputs until next insertion: (disabled) + User inputs until next insertion: (disabled)

@@ -5621,11 +5624,11 @@
- Will be automatically added as the author's note for this character. Will be used in groups, but + Will be automatically added as the author's note for this character. Will be used in groups, but can't be modified when a group chat is open.
- Tokens: 0 + Tokens: 0
@@ -5650,14 +5653,14 @@
- Default Author's Note + Default Author's Note
- Will be automatically added as the Author's Note for all new chats. + Will be automatically added as the Author's Note for all new chats.
- Tokens: 0 + Tokens: 0