Implement data clean-up dialog (#4072)

* [wip] Add user data cleanup service

* Add clean-up report viewer

* Fix review comments

* Add function comments

* Implement item actions

* Fix UI styles

* Add placeholder for empty results, update category description view

* Add displayEmptyPlaceholder method to show message when results list is empty

* Adjust menu buttons row

* Delete char-scoped data bank attachments on character deletion

* Data Bank: Handle character attachments on rename

* Remove line breaks in description strings

* Drop the category when the last item is deleted

* Skip invalid hashes instead of bailing

public/css/data-maid.css

@@ -0,0 +1,149 @@
+.dataMaidDialogContainer {
+    height: 100%;
+    overflow: hidden;
+}
+
+.dataMaidDialog {
+    display: flex;
+    flex-direction: column;
+    gap: 5px;
+    height: 100%;
+    overflow: hidden;
+}
+
+.dataMaidDialogHeader {
+    display: flex;
+    gap: 10px;
+    align-items: center;
+    text-align: left;
+}
+
+.dataMaidHeaderInfo {
+    flex: 1;
+    margin: 0;
+    padding: 5px 10px;
+}
+
+.dataMaidTextView {
+    width: 100%;
+    height: 100%;
+    font-family: var(--monoFontFamily);
+    resize: none;
+    font-size: 0.95em;
+}
+
+.dataMaidImageView {
+    width: 100%;
+    height: 100%;
+    object-fit: contain;
+}
+
+.dataMaidSpinner {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    height: 100%;
+}
+
+.dataMaidPlaceholder {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    font-size: 1.05em;
+}
+
+.dataMaidResultsList:empty {
+    display: none;
+}
+
+.dataMaidResultsList {
+    text-align: left;
+    display: flex;
+    flex-direction: column;
+    gap: 2px;
+    overflow-y: auto;
+    height: 100%;
+    flex-grow: 1;
+}
+
+.dataMaidCategory {
+    border: 1px solid var(--SmartThemeBorderColor);
+    border-radius: 10px;
+    padding: 0 10px;
+}
+
+.dataMaidCategoryHeader {
+    width: 100%;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    margin-right: 5px;
+    padding: 0 5px;
+}
+
+.dataMaidCategoryDetails {
+    display: flex;
+    flex-direction: column;
+    gap: 1px;
+}
+
+.dataMaidCategoryName {
+    flex: 3;
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+.dataMaidCategoryInfo {
+    flex: 1;
+    display: flex;
+    align-items: baseline;
+    gap: 5px;
+}
+
+.dataMaidCategoryContent {
+    border: 1px solid var(--SmartThemeBorderColor);
+    padding: 5px;
+    border-radius: 10px;
+    background-color: var(--black30a);
+    margin: 10px 0;
+}
+
+.dataMaidCategoryContent>.info-block {
+    white-space: pre-wrap;
+}
+
+.dataMaidItem {
+    display: flex;
+    flex-direction: column;
+    padding: 5px;
+    width: 100%;
+    border-bottom: 1px solid var(--SmartThemeBorderColor);
+}
+
+.dataMaidItem:last-child {
+    border-bottom: none;
+}
+
+.dataMaidItemHeader {
+    display: flex;
+    align-items: center;
+    gap: 5px;
+}
+
+.dataMaidItemName {
+    display: flex;
+    flex: 1;
+    align-items: baseline;
+    gap: 2px;
+    word-break: break-all;
+}
+
+.dataMaidItemActions {
+    display: flex;
+    align-items: center;
+    gap: 5px;
+}
+
+.dataMaidItemActions>button {
+    font-size: 0.9em;
+}

public/css/st-tailwind.css

@@ -556,6 +556,10 @@ textarea:disabled {
     min-width: fit-content;
 }
 
+.flexGap2 {
+    gap: 2px;
+}
+
 .flexGap5 {
     gap: 5px;
 }

public/index.html

@@ -4738,13 +4738,16 @@
 
                         <div name="MiscellaneousToggles">
                             <h4><span data-i18n="Miscellaneous">Miscellaneous</span></h4>
-                            <div class="flex-container">
+                            <div class="flex-container flexGap2">
                                 <div id="reload_chat" class="menu_button whitespacenowrap" data-i18n="[title]Reload and redraw the currently open chat" title="Reload and redraw the currently open chat.">
                                     <small data-i18n="Reload Chat">Reload Chat</small>
                                 </div>
                                 <div id="debug_menu" class="menu_button whitespacenowrap">
                                     <small data-i18n="Debug Menu">Debug Menu</small>
                                 </div>
+                                <div id="data_maid_button" class="menu_button whitespacenowrap" title="Find and delete backups, unused chats, files, images, etc." data-i18n="[title]Find and delete backups, unused chats, files, images, etc.">
+                                    <small data-i18n="Clean-Up">Clean-Up</small>
+                                </div>
                             </div>
                             <label class="checkbox_label flexWrap" for="smooth_streaming">
                                 <input id="smooth_streaming" type="checkbox" />

public/script.js

@@ -283,6 +283,7 @@ import { getContext } from './scripts/st-context.js';
 import { extractReasoningFromData, initReasoning, parseReasoningInSwipes, PromptReasoning, ReasoningHandler, removeReasoningFromString, updateReasoningUI } from './scripts/reasoning.js';
 import { accountStorage } from './scripts/util/AccountStorage.js';
 import { initWelcomeScreen, openPermanentAssistantChat, openPermanentAssistantCard, getPermanentAssistantAvatar } from './scripts/welcome-screen.js';
+import { initDataMaid } from './scripts/data-maid.js';
 
 // API OBJECT FOR EXTERNAL WIRING
 globalThis.SillyTavern = {
@@ -1026,6 +1027,7 @@ async function firstLoadInit() {
     initWelcomeScreen();
     await initScrapers();
     initCustomSelectedSamplers();
+    initDataMaid();
     addDebugFunctions();
     doDailyExtensionUpdatesCheck();
     await hideLoader();

public/scripts/data-maid.js

@@ -0,0 +1,393 @@
+import { getRequestHeaders } from '../script.js';
+import { t } from './i18n.js';
+import { callGenericPopup, Popup, POPUP_TYPE } from './popup.js';
+import { renderTemplateAsync } from './templates.js';
+import { humanFileSize, timestampToMoment } from './utils.js';
+
+/**
+ * @typedef {object} DataMaidReportResult
+ * @property {import('../../src/endpoints/data-maid.js').DataMaidSanitizedReport} report - The sanitized report of the Data Maid.
+ * @property {string} token - The token to use for the Data Maid report.
+ */
+
+/**
+ * Data Maid Dialog class for managing the cleanup dialog interface.
+ */
+class DataMaidDialog {
+    constructor() {
+        this.token = null;
+        this.container = null;
+        this.isScanning = false;
+
+        this.DATA_MAID_CATEGORIES = {
+            files: {
+                name: t`Files`,
+                description: t`Files that are not associated with chat messages or Data Bank. WILL DELETE MANUAL UPLOADS!`,
+            },
+            images: {
+                name: t`Images`,
+                description: t`Images that are not associated with chat messages. WILL DELETE MANUAL UPLOADS!`,
+            },
+            chats: {
+                name: t`Chats`,
+                description: t`Chat files associated with deleted characters.`,
+            },
+            groupChats: {
+                name: t`Group Chats`,
+                description: t`Chat files associated with deleted groups.`,
+            },
+            avatarThumbnails: {
+                name: t`Avatar Thumbnails`,
+                description: t`Thumbnails for avatars of missing or deleted characters.`,
+            },
+            backgroundThumbnails: {
+                name: t`Background Thumbnails`,
+                description: t`Thumbnails for missing or deleted backgrounds.`,
+            },
+            chatBackups: {
+                name: t`Chat Backups`,
+                description: t`Automatically generated chat backups.`,
+            },
+            settingsBackups: {
+                name: t`Settings Backups`,
+                description: t`Automatically generated settings backups.`,
+            },
+        };
+    }
+
+    /**
+     * Returns a promise that resolves to the Data Maid report.
+     * @returns {Promise<DataMaidReportResult>}
+     * @private
+     */
+    async getReport() {
+        const response = await fetch('/api/data-maid/report', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Error fetching Data Maid report: ${response.statusText}`);
+        }
+
+        return await response.json();
+    }
+
+    /**
+     * Finalizes the Data Maid process by sending a request to the server.
+     * @returns {Promise<void>}
+     * @private
+     */
+    async finalize() {
+        const response = await fetch('/api/data-maid/finalize', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({ token: this.token }),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Error finalizing Data Maid: ${response.statusText}`);
+        }
+    }
+
+    /**
+     * Sets up the dialog UI elements and event listeners.
+     * @private
+     */
+    async setupDialogUI() {
+        const template = await renderTemplateAsync('dataMaidDialog');
+        this.container = document.createElement('div');
+        this.container.classList.add('dataMaidDialogContainer');
+        this.container.innerHTML = template;
+
+        const startButton = this.container.querySelector('.dataMaidStartButton');
+        startButton.addEventListener('click', () => this.handleScanClick());
+    }
+
+    /**
+     * Handles the scan button click event.
+     * @private
+     */
+    async handleScanClick() {
+        if (this.isScanning) {
+            toastr.warning(t`The scan is already running. Please wait for it to finish.`);
+            return;
+        }
+
+        try {
+            const resultsList = this.container.querySelector('.dataMaidResultsList');
+            resultsList.innerHTML = '';
+            this.showSpinner();
+            this.isScanning = true;
+
+            const report = await this.getReport();
+
+            this.hideSpinner();
+            await this.renderReport(report, resultsList);
+            this.token = report.token;
+        } catch (error) {
+            this.hideSpinner();
+            toastr.error(t`An error has occurred. Check the console for details.`);
+            console.error('Error generating Data Maid report:', error);
+        } finally {
+            this.isScanning = false;
+        }
+    }
+
+    /**
+     * Shows the loading spinner and hides the placeholder.
+     * @private
+     */
+    showSpinner() {
+        const spinner = this.container.querySelector('.dataMaidSpinner');
+        const placeholder = this.container.querySelector('.dataMaidPlaceholder');
+        placeholder.classList.add('displayNone');
+        spinner.classList.remove('displayNone');
+    }
+
+    /**
+     * Hides the loading spinner.
+     * @private
+     */
+    hideSpinner() {
+        const spinner = this.container.querySelector('.dataMaidSpinner');
+        spinner.classList.add('displayNone');
+    }
+
+    /**
+     * Renders the Data Maid report into the results list.
+     * @param {DataMaidReportResult} report
+     * @param {Element} resultsList
+     * @private
+     */
+    async renderReport(report, resultsList) {
+        for (const [prop, data] of Object.entries(this.DATA_MAID_CATEGORIES)) {
+            const category = await this.renderCategory(prop, data.name, data.description, report.report[prop]);
+            if (!category) {
+                continue;
+            }
+            resultsList.appendChild(category);
+        }
+        this.displayEmptyPlaceholder();
+    }
+
+    /**
+     * Displays a placeholder message if no items are found in the results list.
+     * @private
+     */
+    displayEmptyPlaceholder() {
+        const resultsList = this.container.querySelector('.dataMaidResultsList');
+        if (resultsList.children.length === 0) {
+            const placeholder = this.container.querySelector('.dataMaidPlaceholder');
+            placeholder.classList.remove('displayNone');
+            placeholder.textContent = t`No items found to clean up. Come back later!`;
+        }
+    }
+
+    /**
+     * Renders a single Data Maid category into a DOM element.
+     * @param {string} prop Property name for the category
+     * @param {string} name Name of the category
+     * @param {string} description Description of the category
+     * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
+     * @return {Promise<Element|null>} A promise that resolves to a DOM element containing the rendered category
+     * @private
+     */
+    async renderCategory(prop, name, description, items) {
+        if (!Array.isArray(items) || items.length === 0) {
+            return null;
+        }
+
+        const viewModel = {
+            name: name,
+            description: description,
+            totalSize: humanFileSize(items.reduce((sum, item) => sum + item.size, 0)),
+            totalItems: items.length,
+            items: items.sort((a, b) => b.mtime - a.mtime).map(item => ({
+                ...item,
+                size: humanFileSize(item.size),
+                date: timestampToMoment(item.mtime).format('L LT'),
+            })),
+        };
+
+        const template = await renderTemplateAsync('dataMaidCategory', viewModel);
+        const categoryElement = document.createElement('div');
+        categoryElement.innerHTML = template;
+        categoryElement.querySelectorAll('.dataMaidItemView').forEach(button => {
+            button.addEventListener('click', async () => {
+                const item = button.closest('.dataMaidItem');
+                const hash = item?.getAttribute('data-hash');
+                if (hash) {
+                    await this.view(prop, hash);
+                }
+            });
+        });
+        categoryElement.querySelectorAll('.dataMaidItemDownload').forEach(button => {
+            button.addEventListener('click', async () => {
+                const item = button.closest('.dataMaidItem');
+                const hash = item?.getAttribute('data-hash');
+                if (hash) {
+                    await this.download(items, hash);
+                }
+            });
+        });
+        categoryElement.querySelectorAll('.dataMaidDeleteAll').forEach(button => {
+            button.addEventListener('click', async (event) => {
+                event.stopPropagation();
+                const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete all files in this category. THIS CANNOT BE UNDONE!`);
+                if (!confirm) {
+                    return;
+                }
+
+                const hashes = items.map(item => item.hash).filter(hash => hash);
+                await this.delete(hashes);
+
+                categoryElement.remove();
+                this.displayEmptyPlaceholder();
+            });
+
+        });
+        categoryElement.querySelectorAll('.dataMaidItemDelete').forEach(button => {
+            button.addEventListener('click', async () => {
+                const item = button.closest('.dataMaidItem');
+                const hash = item?.getAttribute('data-hash');
+                if (hash) {
+                    const confirm = await Popup.show.confirm(t`Are you sure?`, t`This will permanently delete the file. THIS CANNOT BE UNDONE!`);
+                    if (!confirm) {
+                        return;
+                    }
+                    if (await this.delete([hash])) {
+                        item.remove();
+                        items.splice(items.findIndex(i => i.hash === hash), 1);
+                        if (items.length === 0) {
+                            categoryElement.remove();
+                            this.displayEmptyPlaceholder();
+                        }
+                    }
+                }
+            });
+        });
+        return categoryElement;
+    }
+
+    /**
+     * Constructs the URL for viewing an item by its hash.
+     * @param {string} hash Hash of the item to view
+     * @returns {string} URL to view the item
+     * @private
+     */
+    getViewUrl(hash) {
+        return `/api/data-maid/view?hash=${encodeURIComponent(hash)}&token=${encodeURIComponent(this.token)}`;
+    }
+
+    /**
+     * Downloads an item by its hash.
+     * @param {import('../../src/endpoints/data-maid.js').DataMaidSanitizedRecord[]} items List of items in the category
+     * @param {string} hash Hash of the item to download
+     * @private
+     */
+    async download(items, hash) {
+        const item = items.find(i => i.hash === hash);
+        if (!item) {
+            return;
+        }
+        const url = this.getViewUrl(hash);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = item?.name || hash;
+        document.body.appendChild(a);
+        a.click();
+        document.body.removeChild(a);
+    }
+
+    /**
+     * Opens the item view for a specific hash.
+     * @param {string} prop Property name for the category
+     * @param {string} hash Item hash to view
+     * @private
+     */
+    async view(prop, hash) {
+        const url = this.getViewUrl(hash);
+        const isImage = ['images', 'avatarThumbnails', 'backgroundThumbnails'].includes(prop);
+        const element = isImage
+            ? await this.getViewElement(url)
+            : await this.getTextViewElement(url);
+        await callGenericPopup(element, POPUP_TYPE.DISPLAY, '', { large: true, wide: true });
+    }
+
+    /**
+     * Deletes an item by its file path hash.
+     * @param {string[]} hashes Hashes of items to delete
+     * @return {Promise<boolean>} True if the deletion was successful, false otherwise
+     * @private
+     */
+    async delete(hashes) {
+        try {
+            const response = await fetch('/api/data-maid/delete', {
+                method: 'POST',
+                headers: getRequestHeaders(),
+                body: JSON.stringify({ hashes: hashes, token: this.token }),
+            });
+
+            if (!response.ok) {
+                throw new Error(`Error deleting item: ${response.statusText}`);
+            }
+
+            return true;
+        } catch (error) {
+            console.error('Error deleting item:', error);
+            return false;
+        }
+    }
+
+    /**
+     * Gets an image element for viewing images.
+     * @param {string} url View URL
+     * @returns {Promise<HTMLElement>} Image element
+     * @private
+     */
+    async getViewElement(url) {
+        const img = document.createElement('img');
+        img.src = url;
+        img.classList.add('dataMaidImageView');
+        return img;
+    }
+
+    /**
+     * Gets an iframe element for viewing text content.
+     * @param {string} url View URL
+     * @returns {Promise<HTMLTextAreaElement>} Frame element
+     * @private
+     */
+    async getTextViewElement(url) {
+        const response = await fetch(url);
+        const text = await response.text();
+        const element = document.createElement('textarea');
+        element.classList.add('dataMaidTextView');
+        element.readOnly = true;
+        element.textContent = text;
+        return element;
+    }
+
+    /**
+     * Opens the Data Maid dialog and handles the interaction.
+     */
+    async open() {
+        await this.setupDialogUI();
+        await callGenericPopup(this.container, POPUP_TYPE.TEXT, '', { wide: true, large: true });
+
+        if (this.token) {
+            await this.finalize();
+        }
+    }
+}
+
+export function initDataMaid() {
+    const dataMaidButton = document.getElementById('data_maid_button');
+    if (!dataMaidButton) {
+        console.warn('Data Maid button not found');
+        return;
+    }
+
+    dataMaidButton.addEventListener('click', () => new DataMaidDialog().open());
+}

public/scripts/extensions/attachments/index.js

@@ -216,8 +216,37 @@ function cleanUpAttachments() {
     }
 }
 
+/**
+ * Clean up character attachments when a character is deleted.
+ * @param {{character: import('../../char-data.js').v1CharData}} data Event data
+ */
+function cleanUpCharacterAttachments(data) {
+    const avatar = data?.character?.avatar;
+    if (!avatar) return;
+    if (Array.isArray(extension_settings?.character_attachments?.[avatar])) {
+        delete extension_settings.character_attachments[avatar];
+        saveSettingsDebounced();
+    }
+}
+
+/**
+ * Handle character rename event to update character attachments.
+ * @param {string} oldAvatar Old avatar name
+ * @param {string} newAvatar New avatar name
+ */
+function handleCharacterRename(oldAvatar, newAvatar) {
+    if (!oldAvatar || !newAvatar) return;
+    if (Array.isArray(extension_settings?.character_attachments?.[oldAvatar])) {
+        extension_settings.character_attachments[newAvatar] = extension_settings.character_attachments[oldAvatar];
+        delete extension_settings.character_attachments[oldAvatar];
+        saveSettingsDebounced();
+    }
+}
+
 jQuery(async () => {
     eventSource.on(event_types.APP_READY, cleanUpAttachments);
+    eventSource.on(event_types.CHARACTER_DELETED, cleanUpCharacterAttachments);
+    eventSource.on(event_types.CHARACTER_RENAMED, handleCharacterRename);
     const manageButton = await renderExtensionTemplateAsync('attachments', 'manage-button', {});
     const attachButton = await renderExtensionTemplateAsync('attachments', 'attach-button', {});
     $('#data_bank_wand_container').append(manageButton);

public/scripts/templates/dataMaidCategory.html

@@ -0,0 +1,68 @@
+<div class="dataMaidCategory inline-drawer">
+    <div class="inline-drawer-toggle inline-drawer-header">
+        <div class="dataMaidCategoryHeader">
+            <div class="dataMaidCategoryDetails">
+                <div class="dataMaidCategoryName" data-i18n="{{name}}">
+                    {{name}}
+                </div>
+                <small>{{description}}</small>
+                <div class="dataMaidCategoryInfo">
+                    <small>
+                        <i class="fa-solid fa-file-alt fa-sm"></i>
+                        {{totalItems}}
+                    </small>
+                    <span>&VerticalBar;</span>
+                    <small>
+                        <i class="fa-solid fa-hdd fa-sm"></i>
+                        {{totalSize}}
+                    </small>
+                </div>
+            </div>
+            <div class="dataMaidDeleteAll right_menu_button" title="Delete all items in this category" data-i18n="[title]Delete all items in this category">
+                <i class="fa-solid fa-fw fa-broom"></i>
+            </div>
+        </div>
+        <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
+    </div>
+    <div class="dataMaidCategoryContent inline-drawer-content">
+        <div class="flex-container">
+            {{#each items}}
+            {{#with this}}
+            <div class="dataMaidItem" data-hash="{{hash}}">
+                <div class="dataMaidItemHeader">
+                    <div class="dataMaidItemName">
+                        {{#if parent}}
+                        <span class="dataMaidItemParent">({{parent}})</span>
+                        <span>/</span>
+                        {{/if}}
+                        <b>{{name}}</b>
+                    </div>
+                    <div class="dataMaidItemActions">
+                        <button class="dataMaidItemView menu_button menu_button_icon margin0" title="View item content" data-i18n="[title]View item content">
+                            <i class="fa-solid fa-fw fa-eye"></i>
+                        </button>
+                        <button class="dataMaidItemDownload menu_button menu_button_icon margin0" title="Download item" data-i18n="[title]Download item">
+                            <i class="fa-solid fa-fw fa-download"></i>
+                        </button>
+                        <button class="dataMaidItemDelete menu_button menu_button_icon margin0" title="Delete this item" data-i18n="[title]Delete this item">
+                            <i class="fa-solid fa-fw fa-trash-alt"></i>
+                        </button>
+                    </div>
+                </div>
+                <div class="dataMaidItemInfo">
+                    <small>
+                        <i class="fa-solid fa-file fa-sm"></i>
+                        {{size}}
+                    </small>
+                    <span>&VerticalBar;</span>
+                    <small>
+                        <i class="fa-solid fa-calendar fa-sm"></i>
+                        {{date}}
+                    </small>
+                </div>
+            </div>
+            {{/with}}
+            {{/each}}
+        </div>
+    </div>
+</div>

public/scripts/templates/dataMaidDialog.html

@@ -0,0 +1,25 @@
+<div class="dataMaidDialog">
+    <div class="dataMaidDialogHeader">
+        <div class="dataMaidHeaderInfo info-block warning margin0">
+            <small data-i18n="Once deleted, the files will be gone forever!">
+                Once deleted, the files will be gone forever!
+            </small>
+            <br>
+            <small data-i18n="Make sure to back up your data in advance.">
+                Make sure to back up your data in advance.
+            </small>
+        </div>
+        <button class="menu_button menu_button_icon dataMaidStartButton">
+            <i class="fa fa-cog"></i>
+            <span data-i18n="Scan">Scan</span>
+        </button>
+    </div>
+    <hr>
+    <div class="dataMaidPlaceholder" data-i18n="No results yet. Tap 'Scan' to start scanning.">
+        No results yet. Tap 'Scan' to start scanning.
+    </div>
+    <div class="displayNone dataMaidSpinner">
+        <i class="fa-solid fa-spinner fa-spin fa-3x"></i>
+    </div>
+    <div class="dataMaidResultsList"></div>
+</div>

public/style.css

@@ -11,6 +11,7 @@
 @import url(css/tags.css);
 @import url(css/scrollable-button.css);
 @import url(css/welcome.css);
+@import url(css/data-maid.css);
 
 :root {
     --doc-height: 100%;

src/endpoints/chats.js

@@ -23,6 +23,8 @@ const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups'
 const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000, 'number'));
 const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'boolean');
 
+export const CHAT_BACKUPS_PREFIX = 'chat_';
+
 /**
  * Saves a chat to the backups directory.
  * @param {string} directory The user's backups directory.
@@ -31,7 +33,6 @@ const checkIntegrity = !!getConfigValue('backups.chat.checkIntegrity', true, 'bo
  */
 function backupChat(directory, name, chat) {
     try {
-
         if (!isBackupEnabled) {
             return;
         }
@@ -39,16 +40,16 @@ function backupChat(directory, name, chat) {
         // replace non-alphanumeric characters with underscores
         name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
 
-        const backupFile = path.join(directory, `chat_${name}_${generateTimestamp()}.jsonl`);
+        const backupFile = path.join(directory, `${CHAT_BACKUPS_PREFIX}${name}_${generateTimestamp()}.jsonl`);
         writeFileAtomicSync(backupFile, chat, 'utf-8');
 
-        removeOldBackups(directory, `chat_${name}_`);
+        removeOldBackups(directory, `${CHAT_BACKUPS_PREFIX}${name}_`);
 
         if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
             return;
         }
 
-        removeOldBackups(directory, 'chat_', maxTotalChatBackups);
+        removeOldBackups(directory, CHAT_BACKUPS_PREFIX, maxTotalChatBackups);
     } catch (err) {
         console.error(`Could not backup chat for ${name}`, err);
     }

src/endpoints/data-maid.js

@@ -0,0 +1,735 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import express from 'express';
+import mime from 'mime-types';
+import { getSettingsBackupFilePrefix } from './settings.js';
+import { CHAT_BACKUPS_PREFIX } from './chats.js';
+import { isPathUnderParent, tryParse } from '../util.js';
+import { SETTINGS_FILE } from '../constants.js';
+
+const sha256 = str => crypto.createHash('sha256').update(str).digest('hex');
+
+/**
+ * @typedef {object} DataMaidRawReport
+ * @property {string[]} images - List of loose user images
+ * @property {string[]} files - List of loose user files
+ * @property {string[]} chats - List of loose character chats
+ * @property {string[]} groupChats - List of loose group chats
+ * @property {string[]} avatarThumbnails - List of loose avatar thumbnails
+ * @property {string[]} backgroundThumbnails - List of loose background thumbnails
+ * @property {string[]} chatBackups - List of chat backups
+ * @property {string[]} settingsBackups - List of settings backups
+ */
+
+/**
+ * @typedef {object} DataMaidSanitizedRecord - The entry excluding the sensitive paths.
+ * @property {string} name - The name of the file.
+ * @property {string} hash - The SHA-256 hash of the file path.
+ * @property {string} [parent] - The name of the parent directory, if applicable.
+ * @property {number} [size] - The size of the file in bytes, if available.
+ * @property {number} [mtime] - The last modification time of the file, if available.
+ */
+
+/**
+ * @typedef {object} DataMaidSanitizedReport - The report containing loose user data.
+ * @property {DataMaidSanitizedRecord[]} images - List of sanitized loose user images
+ * @property {DataMaidSanitizedRecord[]} files - List of sanitized loose user files
+ * @property {DataMaidSanitizedRecord[]} chats - List of sanitized loose character chats
+ * @property {DataMaidSanitizedRecord[]} groupChats - List of sanitized loose group chats
+ * @property {DataMaidSanitizedRecord[]} avatarThumbnails - List of sanitized loose avatar thumbnails
+ * @property {DataMaidSanitizedRecord[]} backgroundThumbnails - List of sanitized loose background thumbnails
+ * @property {DataMaidSanitizedRecord[]} chatBackups - List of sanitized chat backups
+ * @property {DataMaidSanitizedRecord[]} settingsBackups - List of sanitized settings backups
+ */
+
+/**
+ * @typedef {object} DataMaidMessage - The chat message object.
+ * @property {DataMaidMessageExtra} [extra] - The extra data object.
+ * @property {DataMaidChatMetadata} [chat_metadata] - The chat metadata object.
+ */
+
+/**
+ * @typedef {object} DataMaidFile - The file object.
+ * @property {string} url - The file URL
+ */
+
+/**
+ * @typedef {object} DataMaidChatMetadata - The chat metadata object.
+ * @property {DataMaidFile[]} [attachments] - The array of attachments, if any.
+ */
+
+/**
+ * @typedef {object} DataMaidMessageExtra - The extra data object.
+ * @property {string} [image] - The link to the image, if any.
+ * @property {string[]} [image_swipes] - The links to the image swipes, if any.
+ * @property {DataMaidFile} [file] - The file object, if any.
+ */
+
+/**
+ * @typedef {object} DataMaidTokenEntry
+ * @property {string} handle - The user's handle or identifier.
+ * @property {{path: string, hash: string}[]} paths - The list of file paths and their hashes that can be cleaned up.
+ */
+
+/**
+ * Service for detecting and managing loose user data files.
+ * Helps identify orphaned files that are no longer referenced by the application.
+ */
+export class DataMaidService {
+    /**
+     * @type {Map<string, DataMaidTokenEntry>} Map clean-up tokens to user IDs
+     */
+    static TOKENS = new Map();
+
+    /**
+     * Creates a new DataMaidService instance for a specific user.
+     * @param {string} handle - The user's handle.
+     * @param {import('../users.js').UserDirectoryList} directories - List of user directories to scan for loose data.
+     */
+    constructor(handle, directories) {
+        this.handle = handle;
+        this.directories = directories;
+    }
+
+    /**
+     * Generates a report of loose user data.
+     * @returns {Promise<DataMaidRawReport>} A report containing lists of loose user data.
+     */
+    async generateReport() {
+        /** @type {DataMaidRawReport} */
+        const report = {
+            images: await this.#collectImages(),
+            files: await this.#collectFiles(),
+            chats: await this.#collectChats(),
+            groupChats: await this.#collectGroupChats(),
+            avatarThumbnails: await this.#collectAvatarThumbnails(),
+            backgroundThumbnails: await this.#collectBackgroundThumbnails(),
+            chatBackups: await this.#collectChatBackups(),
+            settingsBackups: await this.#collectSettingsBackups(),
+        };
+
+        return report;
+    }
+
+
+    /**
+     * Sanitizes a record by hashing the file name and removing sensitive information.
+     * Additionally, adds metadata like size and modification time.
+     * @param {string} name The file or directory name to sanitize.
+     * @param {boolean} withParent If the model should include the parent directory name.
+     * @returns {Promise<DataMaidSanitizedRecord>} A sanitized record with the file name, hash, parent directory name, size, and modification time.
+     */
+    async #sanitizeRecord(name, withParent) {
+        const stat = fs.existsSync(name) ? await fs.promises.stat(name) : null;
+        return {
+            name: path.basename(name),
+            hash: sha256(name),
+            parent: withParent ? path.basename(path.dirname(name)) : void 0,
+            size: stat?.size,
+            mtime: stat?.mtimeMs,
+        };
+    }
+
+    /**
+     * Sanitizes the report by hashing the file paths and removing sensitive information.
+     * @param {DataMaidRawReport} report - The raw report containing loose user data.
+     * @returns {Promise<DataMaidSanitizedReport>} A sanitized report with sensitive paths removed.
+     */
+    async sanitizeReport(report) {
+        const sanitizedReport = {
+            images: await Promise.all(report.images.map(i => this.#sanitizeRecord(i, true))),
+            files: await Promise.all(report.files.map(i => this.#sanitizeRecord(i, false))),
+            chats: await Promise.all(report.chats.map(i => this.#sanitizeRecord(i, true))),
+            groupChats: await Promise.all(report.groupChats.map(i => this.#sanitizeRecord(i, false))),
+            avatarThumbnails: await Promise.all(report.avatarThumbnails.map(i => this.#sanitizeRecord(i, false))),
+            backgroundThumbnails: await Promise.all(report.backgroundThumbnails.map(i => this.#sanitizeRecord(i, false))),
+            chatBackups: await Promise.all(report.chatBackups.map(i => this.#sanitizeRecord(i, false))),
+            settingsBackups: await Promise.all(report.settingsBackups.map(i => this.#sanitizeRecord(i, false))),
+        };
+
+        return sanitizedReport;
+    }
+
+    /**
+     * Collects loose user images from the provided directories.
+     * Images are considered loose if they exist in the user images directory
+     * but are not referenced in any chat messages.
+     * @returns {Promise<string[]>} List of paths to loose user images
+     */
+    async #collectImages() {
+        const result = [];
+
+        try {
+            const messages = await this.#parseAllChats(x => !!x?.extra?.image || Array.isArray(x?.extra?.image_swipes));
+            const knownImages = new Set();
+            for (const message of messages) {
+                if (message?.extra?.image) {
+                    knownImages.add(message.extra.image);
+                }
+                if (Array.isArray(message?.extra?.image_swipes)) {
+                    for (const swipe of message.extra.image_swipes) {
+                        knownImages.add(swipe);
+                    }
+                }
+            }
+            const knownImageFullPaths = new Set();
+            knownImages.forEach(image => {
+                if (image.startsWith('http') || image.startsWith('data:')) {
+                    return; // Skip URLs and data URIs
+                }
+                knownImageFullPaths.add(path.normalize(path.join(this.directories.root, image)));
+            });
+            const images = await fs.promises.readdir(this.directories.userImages, { withFileTypes: true });
+            for (const dirent of images) {
+                const direntPath = path.join(dirent.parentPath, dirent.name);
+                if (dirent.isFile() && !knownImageFullPaths.has(direntPath)) {
+                    result.push(direntPath);
+                }
+                if (dirent.isDirectory()) {
+                    const subdirFiles = await fs.promises.readdir(direntPath, { withFileTypes: true });
+                    for (const file of subdirFiles) {
+                        const subdirFilePath = path.join(direntPath, file.name);
+                        if (file.isFile() && !knownImageFullPaths.has(subdirFilePath)) {
+                            result.push(subdirFilePath);
+                        }
+                    }
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting user images:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects loose user files from the provided directories.
+     * Files are considered loose if they exist in the files directory
+     * but are not referenced in chat messages, metadata, or settings.
+     * @returns {Promise<string[]>} List of paths to loose user files
+     */
+    async #collectFiles() {
+        const result = [];
+
+        try {
+            const messages = await this.#parseAllChats(x => !!x?.extra?.file?.url);
+            const knownFiles = new Set();
+            for (const message of messages) {
+                if (message?.extra?.file?.url) {
+                    knownFiles.add(message.extra.file.url);
+                }
+            }
+            const metadata = await this.#parseAllMetadata(x => Array.isArray(x?.attachments) && x.attachments.length > 0);
+            for (const meta of metadata) {
+                if (Array.isArray(meta?.attachments)) {
+                    for (const attachment of meta.attachments) {
+                        if (attachment?.url) {
+                            knownFiles.add(attachment.url);
+                        }
+                    }
+                }
+            }
+            const pathToSettings = path.join(this.directories.root, SETTINGS_FILE);
+            if (fs.existsSync(pathToSettings)) {
+                try {
+                    const settingsContent = await fs.promises.readFile(pathToSettings, 'utf-8');
+                    const settings = tryParse(settingsContent);
+                    if (Array.isArray(settings?.extension_settings?.attachments)) {
+                        for (const file of settings.extension_settings.attachments) {
+                            if (file?.url) {
+                                knownFiles.add(file.url);
+                            }
+                        }
+                    }
+                    if (typeof settings?.extension_settings?.character_attachments === 'object') {
+                        for (const files of Object.values(settings.extension_settings.character_attachments)) {
+                            if (!Array.isArray(files)) {
+                                continue;
+                            }
+                            for (const file of files) {
+                                if (file?.url) {
+                                    knownFiles.add(file.url);
+                                }
+                            }
+                        }
+                    }
+                } catch (error) {
+                    console.error('[Data Maid] Error reading settings file:', error);
+                }
+            }
+            const knownFileFullPaths = new Set();
+            knownFiles.forEach(file => {
+                knownFileFullPaths.add(path.normalize(path.join(this.directories.root, file)));
+            });
+            const files = await fs.promises.readdir(this.directories.files, { withFileTypes: true });
+            for (const file of files) {
+                const filePath = path.join(this.directories.files, file.name);
+                if (file.isFile() && !knownFileFullPaths.has(filePath)) {
+                    result.push(filePath);
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting user files:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects loose character chats from the provided directories.
+     * Chat folders are considered loose if they don't have corresponding character files.
+     * @returns {Promise<string[]>} List of paths to loose character chats
+     */
+    async #collectChats() {
+        const result = [];
+
+        try {
+            const knownChatFolders = new Set();
+            const characters = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
+            for (const file of characters) {
+                if (file.isFile() && path.parse(file.name).ext === '.png') {
+                    knownChatFolders.add(file.name.replace('.png', ''));
+                }
+            }
+            const chatFolders = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
+            for (const folder of chatFolders) {
+                if (folder.isDirectory() && !knownChatFolders.has(folder.name)) {
+                    const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, folder.name), { withFileTypes: true });
+                    for (const file of chatFiles) {
+                        if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
+                            result.push(path.join(this.directories.chats, folder.name, file.name));
+                        }
+                    }
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting character chats:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects loose group chats from the provided directories.
+     * Group chat files are considered loose if they're not referenced by any group definition.
+     * @returns {Promise<string[]>} List of paths to loose group chats
+     */
+    async #collectGroupChats() {
+        const result = [];
+
+        try {
+            const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
+            const knownGroupChats = new Set();
+            for (const file of groups) {
+                if (file.isFile() && path.parse(file.name).ext === '.json') {
+                    try {
+                        const pathToFile = path.join(this.directories.groups, file.name);
+                        const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
+                        const groupData = tryParse(fileContent);
+                        if (groupData?.chat_id) {
+                            knownGroupChats.add(groupData.chat_id);
+                        }
+                        if (Array.isArray(groupData?.chats)) {
+                            for (const chat of groupData.chats) {
+                                knownGroupChats.add(chat);
+                            }
+                        }
+                    } catch (error) {
+                        console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
+                    }
+                }
+            }
+            const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
+            for (const file of groupChats) {
+                if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
+                    if (!knownGroupChats.has(path.parse(file.name).name)) {
+                        result.push(path.join(this.directories.groupChats, file.name));
+                    }
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting group chats:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects loose avatar thumbnails from the provided directories.
+     * @returns {Promise<string[]>} List of paths to loose avatar thumbnails
+     */
+    async #collectAvatarThumbnails() {
+        const result = [];
+
+        try {
+            const knownAvatars = new Set();
+            const avatars = await fs.promises.readdir(this.directories.characters, { withFileTypes: true });
+            for (const file of avatars) {
+                if (file.isFile()) {
+                    knownAvatars.add(file.name);
+                }
+            }
+            const avatarThumbnails = await fs.promises.readdir(this.directories.thumbnailsAvatar, { withFileTypes: true });
+            for (const file of avatarThumbnails) {
+                if (file.isFile() && !knownAvatars.has(file.name)) {
+                    result.push(path.join(this.directories.thumbnailsAvatar, file.name));
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting avatar thumbnails:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects loose background thumbnails from the provided directories.
+     * @returns {Promise<string[]>} List of paths to loose background thumbnails
+     */
+    async #collectBackgroundThumbnails() {
+        const result = [];
+
+        try {
+            const knownBackgrounds = new Set();
+            const backgrounds = await fs.promises.readdir(this.directories.backgrounds, { withFileTypes: true });
+            for (const file of backgrounds) {
+                if (file.isFile()) {
+                    knownBackgrounds.add(file.name);
+                }
+            }
+            const backgroundThumbnails = await fs.promises.readdir(this.directories.thumbnailsBg, { withFileTypes: true });
+            for (const file of backgroundThumbnails) {
+                if (file.isFile() && !knownBackgrounds.has(file.name)) {
+                    result.push(path.join(this.directories.thumbnailsBg, file.name));
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting background thumbnails:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects chat backups from the provided directories.
+     * @returns {Promise<string[]>} List of paths to chat backups
+     */
+    async #collectChatBackups() {
+        const result = [];
+
+        try {
+            const prefix = CHAT_BACKUPS_PREFIX;
+            const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
+            for (const file of backups) {
+                if (file.isFile() && file.name.startsWith(prefix)) {
+                    result.push(path.join(this.directories.backups, file.name));
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting chat backups:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Collects settings backups from the provided directories.
+     * @returns {Promise<string[]>} List of paths to settings backups
+     */
+    async #collectSettingsBackups() {
+        const result = [];
+
+        try {
+            const prefix = getSettingsBackupFilePrefix(this.handle);
+            const backups = await fs.promises.readdir(this.directories.backups, { withFileTypes: true });
+            for (const file of backups) {
+                if (file.isFile() && file.name.startsWith(prefix)) {
+                    result.push(path.join(this.directories.backups, file.name));
+                }
+            }
+        } catch (error) {
+            console.error('[Data Maid] Error collecting settings backups:', error);
+        }
+
+        return result;
+    }
+
+    /**
+     * Parses all chat files and returns an array of chat messages.
+     * Searches both individual character chats and group chats.
+     * @param {function(DataMaidMessage): boolean} filterFn - Filter function to apply to each message.
+     * @returns {Promise<DataMaidMessage[]>} Array of chat messages
+     */
+    async #parseAllChats(filterFn) {
+        try {
+            const allChats = [];
+
+            const groupChats = await fs.promises.readdir(this.directories.groupChats, { withFileTypes: true });
+            for (const file of groupChats) {
+                if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
+                    const chatMessages = await this.#parseChatFile(path.join(this.directories.groupChats, file.name));
+                    allChats.push(...chatMessages.filter(filterFn));
+                }
+            }
+
+            const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
+            for (const directory of chatDirectories) {
+                if (directory.isDirectory()) {
+                    const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
+                    for (const file of chatFiles) {
+                        if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
+                            const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
+                            allChats.push(...chatMessages.filter(filterFn));
+                        }
+                    }
+                }
+            }
+
+            return allChats;
+        } catch (error) {
+            console.error('[Data Maid] Error parsing chats:', error);
+            return [];
+        }
+    }
+
+    /**
+     * Parses all metadata from chat files and group definitions.
+     * Extracts metadata from both active and historical chat data.
+     * @param {function(DataMaidChatMetadata): boolean} filterFn - Filter function to apply to each metadata entry.
+     * @returns {Promise<DataMaidChatMetadata[]>} Parsed chat metadata as an array.
+     */
+    async #parseAllMetadata(filterFn) {
+        try {
+            const allMetadata = [];
+
+            const groups = await fs.promises.readdir(this.directories.groups, { withFileTypes: true });
+            for (const file of groups) {
+                if (file.isFile() && path.parse(file.name).ext === '.json') {
+                    try {
+                        const pathToFile = path.join(this.directories.groups, file.name);
+                        const fileContent = await fs.promises.readFile(pathToFile, 'utf-8');
+                        const groupData = tryParse(fileContent);
+                        if (groupData?.chat_metadata && filterFn(groupData.chat_metadata)) {
+                            allMetadata.push(groupData.chat_metadata);
+                        }
+                        if (groupData?.past_metadata) {
+                            allMetadata.push(...Object.values(groupData.past_metadata).filter(filterFn));
+                        }
+                    } catch (error) {
+                        console.error(`[Data Maid] Error parsing group chat file ${file.name}:`, error);
+                    }
+                }
+            }
+
+            const chatDirectories = await fs.promises.readdir(this.directories.chats, { withFileTypes: true });
+            for (const directory of chatDirectories) {
+                if (directory.isDirectory()) {
+                    const chatFiles = await fs.promises.readdir(path.join(this.directories.chats, directory.name), { withFileTypes: true });
+                    for (const file of chatFiles) {
+                        if (file.isFile() && path.parse(file.name).ext === '.jsonl') {
+                            const chatMessages = await this.#parseChatFile(path.join(this.directories.chats, directory.name, file.name));
+                            const chatMetadata = chatMessages?.[0]?.chat_metadata;
+                            if (chatMetadata && filterFn(chatMetadata)) {
+                                allMetadata.push(chatMetadata);
+                            }
+                        }
+                    }
+                }
+            }
+
+            return allMetadata;
+        } catch (error) {
+            console.error('[Data Maid] Error parsing chats:', error);
+            return [];
+        }
+    }
+
+    /**
+     * Parses a single chat file and returns an array of chat messages.
+     * Each line in the JSONL file represents one message.
+     * @param {string} filePath Path to the chat file to parse.
+     * @returns {Promise<DataMaidMessage[]>} Parsed chat messages as an array.
+     */
+    async #parseChatFile(filePath) {
+        try {
+            const content = await fs.promises.readFile(filePath, 'utf-8');
+            const chatData = content.split('\n').map(tryParse).filter(Boolean);
+            return chatData;
+        } catch (error) {
+            console.error(`[Data Maid] Error reading chat file ${filePath}:`, error);
+            return [];
+        }
+    }
+
+    /**
+     * Generates a unique token for the user to clean up their data.
+     * Replaces any existing token for the same user.
+     * @param {string} handle - The user's handle or identifier.
+     * @param {DataMaidRawReport} report - The report containing loose user data.
+     * @returns {string} A unique token.
+     */
+    static generateToken(handle, report) {
+        // Remove any existing token for this user
+        for (const [token, entry] of this.TOKENS.entries()) {
+            if (entry.handle === handle) {
+                this.TOKENS.delete(token);
+            }
+        }
+
+        const token = crypto.randomBytes(32).toString('hex');
+        const tokenEntry = {
+            handle,
+            paths: Object.values(report).filter(v => Array.isArray(v)).flat().map(x => ({ path: x, hash: sha256(x) })),
+        };
+        this.TOKENS.set(token, tokenEntry);
+        return token;
+    }
+}
+
+export const router = express.Router();
+
+router.post('/report', async (req, res) => {
+    try {
+        if (!req.user || !req.user.directories) {
+            return res.sendStatus(403);
+        }
+
+        const dataMaid = new DataMaidService(req.user.profile.handle, req.user.directories);
+        const rawReport = await dataMaid.generateReport();
+
+        const report = await dataMaid.sanitizeReport(rawReport);
+        const token = DataMaidService.generateToken(req.user.profile.handle, rawReport);
+
+        return res.json({ report, token });
+    } catch (error) {
+        console.error('[Data Maid] Error generating data maid report:', error);
+        return res.sendStatus(500);
+    }
+});
+
+router.post('/finalize', async (req, res) => {
+    try {
+        if (!req.user || !req.user.directories) {
+            return res.sendStatus(403);
+        }
+
+        if (!req.body.token) {
+            return res.sendStatus(400);
+        }
+
+        const token = req.body.token.toString();
+        if (!DataMaidService.TOKENS.has(token)) {
+            return res.sendStatus(403);
+        }
+
+        const tokenEntry = DataMaidService.TOKENS.get(token);
+        if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
+            return res.sendStatus(403);
+        }
+
+        // Remove the token after finalization
+        DataMaidService.TOKENS.delete(token);
+        return res.sendStatus(204);
+    } catch (error) {
+        console.error('[Data Maid] Error finalizing the token:', error);
+        return res.sendStatus(500);
+    }
+});
+
+router.get('/view', async (req, res) => {
+    try {
+        if (!req.user || !req.user.directories) {
+            return res.sendStatus(403);
+        }
+
+        if (!req.query.token || !req.query.hash) {
+            return res.sendStatus(400);
+        }
+
+        const token = req.query.token.toString();
+        const hash = req.query.hash.toString();
+
+        if (!DataMaidService.TOKENS.has(token)) {
+            return res.sendStatus(403);
+        }
+
+        const tokenEntry = DataMaidService.TOKENS.get(token);
+        if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
+            return res.sendStatus(403);
+        }
+
+        const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
+        if (!fileEntry) {
+            return res.sendStatus(404);
+        }
+
+        if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
+            console.warn('[Data Maid] Attempted access to a file outside of the user directory:', fileEntry.path);
+            return res.sendStatus(403);
+        }
+
+        const pathToFile = fileEntry.path;
+        const fileExists = fs.existsSync(pathToFile);
+
+        if (!fileExists) {
+            return res.sendStatus(404);
+        }
+
+        const fileBuffer = await fs.promises.readFile(pathToFile);
+        const mimeType = mime.lookup(pathToFile) || 'text/plain';
+        res.setHeader('Content-Type', mimeType);
+        return res.send(fileBuffer);
+    } catch (error) {
+        console.error('[Data Maid] Error viewing file:', error);
+        return res.sendStatus(500);
+    }
+});
+
+router.post('/delete', async (req, res) => {
+    try {
+        if (!req.user || !req.user.directories) {
+            return res.sendStatus(403);
+        }
+
+        const { token, hashes } = req.body;
+        if (!token || !Array.isArray(hashes) || hashes.length === 0) {
+            return res.sendStatus(400);
+        }
+
+        if (!DataMaidService.TOKENS.has(token)) {
+            return res.sendStatus(403);
+        }
+
+        const tokenEntry = DataMaidService.TOKENS.get(token);
+        if (!tokenEntry || tokenEntry.handle !== req.user.profile.handle) {
+            return res.sendStatus(403);
+        }
+
+        for (const hash of hashes) {
+            const fileEntry = tokenEntry.paths.find(entry => entry.hash === hash);
+            if (!fileEntry) {
+                continue;
+            }
+
+            if (!isPathUnderParent(req.user.directories.root, fileEntry.path)) {
+                console.warn('[Data Maid] Attempted deletion of a file outside of the user directory:', fileEntry.path);
+                continue;
+            }
+
+            const pathToFile = fileEntry.path;
+            const fileExists = fs.existsSync(pathToFile);
+
+            if (!fileExists) {
+                continue;
+            }
+
+            await fs.promises.unlink(pathToFile);
+        }
+
+        return res.sendStatus(204);
+    } catch (error) {
+        console.error('[Data Maid] Error deleting files:', error);
+        return res.sendStatus(500);
+    }
+});

src/endpoints/settings.js

@@ -81,7 +81,7 @@ function sortByName(_) {
  * @param {string} handle User handle
  * @returns {string} File prefix
  */
-function getFilePrefix(handle) {
+export function getSettingsBackupFilePrefix(handle) {
     return `settings_${handle}_`;
 }
 
@@ -131,7 +131,7 @@ async function backupSettings() {
  */
 function backupUserSettings(handle, preventDuplicates) {
     const userDirectories = getUserDirectories(handle);
-    const backupFile = path.join(userDirectories.backups, `${getFilePrefix(handle)}${generateTimestamp()}.json`);
+    const backupFile = path.join(userDirectories.backups, `${getSettingsBackupFilePrefix(handle)}${generateTimestamp()}.json`);
     const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);
 
     if (preventDuplicates && isDuplicateBackup(handle, sourceFile)) {
@@ -183,7 +183,7 @@ function areFilesEqual(file1, file2) {
 function getLatestBackup(handle) {
     const userDirectories = getUserDirectories(handle);
     const backupFiles = fs.readdirSync(userDirectories.backups)
-        .filter(x => x.startsWith(getFilePrefix(handle)))
+        .filter(x => x.startsWith(getSettingsBackupFilePrefix(handle)))
         .map(x => ({ name: x, ctime: fs.statSync(path.join(userDirectories.backups, x)).ctimeMs }));
     const latestBackup = backupFiles.sort((a, b) => b.ctime - a.ctime)[0]?.name;
     if (!latestBackup) {
@@ -283,7 +283,7 @@ router.post('/get', (request, response) => {
 router.post('/get-snapshots', async (request, response) => {
     try {
         const snapshots = fs.readdirSync(request.user.directories.backups);
-        const userFilesPattern = getFilePrefix(request.user.profile.handle);
+        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
         const userSnapshots = snapshots.filter(x => x.startsWith(userFilesPattern));
 
         const result = userSnapshots.map(x => {
@@ -300,7 +300,7 @@ router.post('/get-snapshots', async (request, response) => {
 
 router.post('/load-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
     try {
-        const userFilesPattern = getFilePrefix(request.user.profile.handle);
+        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
 
         if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
             return response.status(400).send({ error: 'Invalid snapshot name' });
@@ -334,7 +334,7 @@ router.post('/make-snapshot', async (request, response) => {
 
 router.post('/restore-snapshot', getFileNameValidationFunction('name'), async (request, response) => {
     try {
-        const userFilesPattern = getFilePrefix(request.user.profile.handle);
+        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);
 
         if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {
             return response.status(400).send({ error: 'Invalid snapshot name' });

src/fetch-patch.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import mime from 'mime-types';
 import { serverDirectory } from './server-directory.js';
+import { getRequestURL, isFileURL, isPathUnderParent } from './util.js';
 
 const originalFetch = globalThis.fetch;
 
@@ -10,57 +11,6 @@ const ALLOWED_EXTENSIONS = [
     '.wasm',
 ];
 
-/**
- * Checks if a child path is under a parent path.
- * @param {string} parentPath Parent path
- * @param {string} childPath Child path
- * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
- */
-function isPathUnderParent(parentPath, childPath) {
-    const normalizedParent = path.normalize(parentPath);
-    const normalizedChild = path.normalize(childPath);
-
-    const relativePath = path.relative(normalizedParent, normalizedChild);
-
-    return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
-}
-
-/**
- * Checks if the given request is a file URL.
- * @param {string | URL | Request} request The request to check
- * @return {boolean} Returns true if the request is a file URL, false otherwise
- */
-function isFileURL(request) {
-    if (typeof request === 'string') {
-        return request.startsWith('file://');
-    }
-    if (request instanceof URL) {
-        return request.protocol === 'file:';
-    }
-    if (request instanceof Request) {
-        return request.url.startsWith('file://');
-    }
-    return false;
-}
-
-/**
- * Gets the URL from the request.
- * @param {string | URL | Request} request The request to get the URL from
- * @return {string} The URL of the request
- */
-function getRequestURL(request) {
-    if (typeof request === 'string') {
-        return request;
-    }
-    if (request instanceof URL) {
-        return request.href;
-    }
-    if (request instanceof Request) {
-        return request.url;
-    }
-    throw new TypeError('Invalid request type');
-}
-
 // Patched fetch function that handles file URLs
 globalThis.fetch = async (/** @type {string | URL | Request} */ request, /** @type {RequestInit | undefined} */ options) => {
     if (!isFileURL(request)) {

src/server-startup.js

@@ -46,6 +46,7 @@ import { router as textCompletionsRouter } from './endpoints/backends/text-compl
 import { router as scaleAltRouter } from './endpoints/backends/scale-alt.js';
 import { router as speechRouter } from './endpoints/speech.js';
 import { router as azureRouter } from './endpoints/azure.js';
+import { router as dataMaidRouter } from './endpoints/data-maid.js';
 
 /**
  * @typedef {object} ServerStartupResult
@@ -173,6 +174,7 @@ export function setupPrivateEndpoints(app) {
     app.use('/api/backends/scale-alt', scaleAltRouter);
     app.use('/api/speech', speechRouter);
     app.use('/api/azure', azureRouter);
+    app.use('/api/data-maid', dataMaidRouter);
 }
 
 /**

src/util.js

@@ -1146,3 +1146,54 @@ export function setPermissionsSync(targetPath) {
         console.error(`Error setting write permissions for ${targetPath}:`, error);
     }
 }
+
+/**
+ * Checks if a child path is under a parent path.
+ * @param {string} parentPath Parent path
+ * @param {string} childPath Child path
+ * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
+ */
+export function isPathUnderParent(parentPath, childPath) {
+    const normalizedParent = path.normalize(parentPath);
+    const normalizedChild = path.normalize(childPath);
+
+    const relativePath = path.relative(normalizedParent, normalizedChild);
+
+    return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
+}
+
+/**
+ * Checks if the given request is a file URL.
+ * @param {string | URL | Request} request The request to check
+ * @return {boolean} Returns true if the request is a file URL, false otherwise
+ */
+export function isFileURL(request) {
+    if (typeof request === 'string') {
+        return request.startsWith('file://');
+    }
+    if (request instanceof URL) {
+        return request.protocol === 'file:';
+    }
+    if (request instanceof Request) {
+        return request.url.startsWith('file://');
+    }
+    return false;
+}
+
+/**
+ * Gets the URL from the request.
+ * @param {string | URL | Request} request The request to get the URL from
+ * @return {string} The URL of the request
+ */
+export function getRequestURL(request) {
+    if (typeof request === 'string') {
+        return request;
+    }
+    if (request instanceof URL) {
+        return request.href;
+    }
+    if (request instanceof Request) {
+        return request.url;
+    }
+    throw new TypeError('Invalid request type');
+}