#1954 Remove backtick wrapping for inserted files

public/scripts/chats.js

@@ -41,6 +41,7 @@ import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from './popup.js';
  * @property {string} url File URL
  * @property {number} size File size
  * @property {string} name File name
+ * @property {number} created Timestamp
  * @property {string} [text] File text
  */
 
@@ -168,6 +169,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
                 url: fileUrl,
                 size: file.size,
                 name: file.name,
+                created: Date.now(),
             };
         }
 
@@ -383,7 +385,7 @@ export async function appendFileContent(message, messageText) {
         const fileText = message.extra.file.text || (await getFileAttachment(message.extra.file.url));
 
         if (fileText) {
-            const fileWrapped = `\`\`\`\n${fileText}\n\`\`\`\n\n`;
+            const fileWrapped = `${fileText}\n\n`;
             message.extra.fileLength = fileWrapped.length;
             messageText = fileWrapped + messageText;
         }
@@ -618,11 +620,38 @@ async function deleteAttachment(attachment, source, callback) {
  */
 async function openAttachmentManager() {
     /**
-     *
+     * Renders a list of attachments.
      * @param {FileAttachment[]} attachments List of attachments
      * @param {string} source Source of the attachments
      */
     async function renderList(attachments, source) {
+        /**
+         * Sorts attachments by sortField and sortOrder.
+         * @param {FileAttachment} a First attachment
+         * @param {FileAttachment} b Second attachment
+         * @returns {number} Sort order
+         */
+        function sortFn(a, b) {
+            const sortValueA = a[sortField];
+            const sortValueB = b[sortField];
+            if (typeof sortValueA === 'string' && typeof sortValueB === 'string') {
+                return sortValueA.localeCompare(sortValueB) * (sortOrder === 'asc' ? 1 : -1);
+            }
+            return (sortValueA - sortValueB) * (sortOrder === 'asc' ? 1 : -1);
+        }
+
+        /**
+         * Filters attachments by name.
+         * @param {FileAttachment} a Attachment
+         * @returns {boolean} True if attachment matches the filter, false otherwise.
+         */
+        function filterFn(a) {
+            if (!filterString) {
+                return true;
+            }
+
+            return a.name.toLowerCase().includes(filterString.toLowerCase());
+        }
         const sources = {
             [ATTACHMENT_SOURCE.GLOBAL]: '.globalAttachmentsList',
             [ATTACHMENT_SOURCE.CHARACTER]: '.characterAttachmentsList',
@@ -630,16 +659,37 @@ async function openAttachmentManager() {
         };
 
         template.find(sources[source]).empty();
-        for (const attachment of attachments) {
+
+        // Sort attachments by sortField and sortOrder, and apply filter
+        const sortedAttachmentList = attachments.slice().filter(filterFn).sort(sortFn);
+
+        for (const attachment of sortedAttachmentList) {
             const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone();
             attachmentTemplate.find('.attachmentListItemName').text(attachment.name);
             attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size));
+            attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString());
             attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment));
             attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments));
             template.find(sources[source]).append(attachmentTemplate);
         }
     }
 
+    /**
+     * Renders buttons for the attachment manager.
+     * @param {string} source Source of the buttons
+     */
+    function renderButtons(source) {
+        const sources = {
+            [ATTACHMENT_SOURCE.GLOBAL]: '.globalAttachmentsTitle',
+            [ATTACHMENT_SOURCE.CHARACTER]: '.characterAttachmentsTitle',
+            [ATTACHMENT_SOURCE.CHAT]: '.chatAttachmentsTitle',
+        };
+
+        const buttonsList = template.find('.actionButtonsTemplate .actionButtons').clone();
+        buttonsList.find('.menu_button').data('attachment-manager-target', source);
+        template.find(sources[source]).append(buttonsList);
+    }
+
     async function renderAttachments() {
         /** @type {FileAttachment[]} */
         const globalAttachments = extension_settings.attachments ?? [];
@@ -664,8 +714,14 @@ async function openAttachmentManager() {
         template.find('.chatAttachmentsName').text(chatName);
     }
 
+    let sortField = localStorage.getItem('DataBank_sortField') || 'created';
+    let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc';
+    let filterString = '';
     const hasFandomPlugin = await isFandomPluginAvailable();
     const template = $(await renderExtensionTemplateAsync('attachments', 'manager', {}));
+    renderButtons(ATTACHMENT_SOURCE.GLOBAL);
+    renderButtons(ATTACHMENT_SOURCE.CHARACTER);
+    renderButtons(ATTACHMENT_SOURCE.CHAT);
     template.find('.scrapeWebpageButton').on('click', function () {
         openWebpageScraper(String($(this).data('attachment-manager-target')), renderAttachments);
     });
@@ -675,6 +731,21 @@ async function openAttachmentManager() {
     template.find('.uploadFileButton').on('click', function () {
         openFileUploader(String($(this).data('attachment-manager-target')), renderAttachments);
     });
+    template.find('.attachmentSearch').on('input', function () {
+        filterString = String($(this).val());
+        renderAttachments();
+    });
+    template.find('.attachmentSort').on('change', function () {
+        if (!(this instanceof HTMLSelectElement) || this.selectedOptions.length === 0)  {
+            return;
+        }
+
+        sortField = this.selectedOptions[0].dataset.sortField;
+        sortOrder = this.selectedOptions[0].dataset.sortOrder;
+        localStorage.setItem('DataBank_sortField', sortField);
+        localStorage.setItem('DataBank_sortOrder', sortOrder);
+        renderAttachments();
+    });
     await renderAttachments();
     callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' });
 }
@@ -734,8 +805,85 @@ async function getTitleFromHtmlBlob(blob) {
  * @param {function} callback Callback function
  */
 async function openFandomScraper(target, callback) {
-    toastr.info('Not implemented yet', target);
-    callback();
+    if (!await isFandomPluginAvailable()) {
+        toastr.error('Fandom scraper plugin is not available');
+        return;
+    }
+
+    let fandom = '';
+    let filter = '';
+    let output = 'single';
+
+    const template = $(await renderExtensionTemplateAsync('attachments', 'fandom-scrape', {}));
+    template.find('input[name="fandomScrapeInput"]').on('input', function () {
+        fandom = String($(this).val());
+    });
+    template.find('input[name="fandomScrapeFilter"]').on('input', function () {
+        filter = String($(this).val());
+    });
+    template.find('input[name="fandomScrapeOutput"]').on('input', function () {
+        output = String($(this).val());
+    });
+
+    const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false });
+
+    if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
+        return;
+    }
+
+    if (!fandom) {
+        toastr.error('Fandom name is required');
+        return;
+    }
+
+    try {
+        const result = await fetch('/api/plugins/fandom/scrape', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({ fandom, filter }),
+        });
+
+        if (!result.ok) {
+            const error = await result.text();
+            throw new Error(error);
+        }
+
+        // Get domain name part if it's a URL
+        try {
+            const url = new URL(fandom);
+            const fandomId = url.hostname.split('.')[0] || fandom;
+            fandom = fandomId;
+        } catch {
+            // Ignore
+        }
+
+        const data = await result.json();
+        let numberOfAttachments;
+
+        if (output === 'multi') {
+            numberOfAttachments = data.length;
+            for (const attachment of data) {
+                const file = new File([String(attachment.content).trim()], `${String(attachment.title).trim()}.txt`, { type: 'text/plain' });
+                await uploadFileAttachmentToServer(file, target);
+            }
+        }
+
+        if (output === 'single') {
+            numberOfAttachments = 1;
+            const combinedContent = data.map((a) => String(a.title).trim() + '\n\n' + String(a.content).trim()).join('\n\n\n\n');
+            const file = new File([combinedContent], `${fandom}.txt`, { type: 'text/plain' });
+            await uploadFileAttachmentToServer(file, target);
+        }
+
+        if (numberOfAttachments) {
+            toastr.success(`Scraped ${numberOfAttachments} attachments from ${fandom}`);
+        }
+
+        callback();
+    } catch (error) {
+        console.error('Fandom scraping failed', error);
+        toastr.error('Check browser console for details.', 'Fandom scraping failed');
+    }
 }
 
 /**
@@ -791,6 +939,7 @@ async function uploadFileAttachmentToServer(file, target) {
     }
 
     const fileUrl = await uploadFileAttachment(uniqueFileName, base64Data);
+    const convertedSize = Math.round(base64Data.length * 0.75);
 
     if (!fileUrl) {
         return;
@@ -798,8 +947,9 @@ async function uploadFileAttachmentToServer(file, target) {
 
     const attachment = {
         url: fileUrl,
-        size: file.size,
+        size: convertedSize,
         name: file.name,
+        created: Date.now(),
     };
 
     ensureAttachmentsExist();

public/scripts/extensions/attachments/fandom-scrape.html

@@ -0,0 +1,51 @@
+<div>
+    <div class="flex-container flexFlowColumn">
+        <label for="fandomScrapeInput" data-i18n="Enter a URL or the ID of a Fandom wiki page to scrape:">
+            Enter a URL or the ID of a Fandom wiki page to scrape:
+        </label>
+        <small>
+            <span data-i18n=Examples:">Examples:</span>
+            <code>https://harrypotter.fandom.com/</code>
+            <span data-i18n="or">or</span>
+            <code>harrypotter</code>
+        </small>
+        <input type="text" id="fandomScrapeInput" name="fandomScrapeInput" class="text_pole" placeholder="">
+    </div>
+    <div class="flex-container flexFlowColumn">
+        <label for="fandomScrapeFilter">
+            Optional regex to pick the content by its title:
+        </label>
+        <small>
+            <span data-i18n="Example:">Example:</span>
+            <code>/(Azkaban|Weasley)/gi</code>
+        </small>
+        <input type="text" id="fandomScrapeFilter" name="fandomScrapeFilter" class="text_pole" placeholder="">
+    </div>
+    <div class="flex-container flexFlowColumn">
+        <label>
+            Output format:
+        </label>
+        <label class="checkbox_label justifyLeft" for="fandomScrapeOutputSingle">
+            <input id="fandomScrapeOutputSingle" type="radio" name="fandomScrapeOutput" value="single" checked>
+            <div class="flex-container flexFlowColumn flexNoGap">
+                <span data-i18n="Single file">
+                    Single file
+                </span>
+                <small data-i18n="All articles will be concatenated into a single file.">
+                    All articles will be concatenated into a single file.
+                </small>
+            </div>
+        </label>
+        <label class="checkbox_label justifyLeft" for="fandomScrapeOutputMulti">
+            <input id="fandomScrapeOutputMulti" type="radio" name="fandomScrapeOutput" value="multi">
+            <div class="flex-container flexFlowColumn flexNoGap">
+                <span data-i18n="File per article">
+                    File per article
+                </span>
+                <small data-i18n="Each article will be saved as a separate file.">
+                    Not recommended. Each article will be saved as a separate file.
+                </small>
+            </div>
+        </label>
+    </div>
+</div>

public/scripts/extensions/attachments/manager.html

@@ -1,4 +1,4 @@
-<div class="wide100p paddingTopBot5">
+<div class="wide100p padding5">
     <h2 class="marginBot5">
         <span data-i18n="Data Bank">
             Data Bank
@@ -10,31 +10,35 @@
     <div data-i18n="Supported file types: Plain Text, PDF, Markdown, HTML." class="marginTopBot5">
         Supported file types: Plain Text, PDF, Markdown, HTML.
     </div>
+    <div class="flex-container marginTopBot5">
+        <input type="search" id="attachmentSearch" class="attachmentSearch text_pole margin0 flex1" placeholder="Search...">
+        <select id="attachmentSort" class="attachmentSort text_pole margin0 flex1">
+            <option data-sort-field="created" data-sort-order="desc" data-i18n="Date (Newest First)">
+                Date (Newest First)
+            </option>
+            <option data-sort-field="created" data-sort-order="asc" data-i18n="Date (Oldest First)">
+                Date (Oldest First)
+            </option>
+            <option data-sort-field="name" data-sort-order="asc" data-i18n="Name (A-Z)">
+                Name (A-Z)
+            </option>
+            <option data-sort-field="name" data-sort-order="desc" data-i18n="Name (Z-A)">
+                Name (Z-A)
+            </option>
+            <option data-sort-field="size" data-sort-order="asc" data-i18n="Size (Smallest First)">
+                Size (Smallest First)
+            </option>
+            <option data-sort-field="size" data-sort-order="desc" data-i18n="Size (Largest First)">
+                Size (Largest First)
+            </option>
+        </select>
+
+    </div>
     <div class="justifyLeft globalAttachmentsBlock marginBot10">
-        <h3 class="margin0 title_restorable">
+        <h3 class="globalAttachmentsTitle margin0 title_restorable">
             <span data-i18n="Global Attachments">
                 Global Attachments
             </span>
-            <div class="flex-container flexGap10">
-                <div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Download a page from the web.">
-                    <i class="fa-fw fa-solid fa-globe"></i>
-                    <span data-i18n="From Web">
-                        From Web
-                    </span>
-                </div>
-                <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
-                    <span data-i18n="From Fandom">
-                        From Fandom
-                    </span>
-                </div>
-                <div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Upload a file from your computer.">
-                    <i class="fa-fw fa-solid fa-upload"></i>
-                    <span data-i18n="From File">
-                        From File
-                    </span>
-                </div>
-            </div>
         </h3>
         <small data-i18n="These files are available for all characters in all chats.">
             These files are available for all characters in all chats.
@@ -43,30 +47,10 @@
         <hr>
     </div>
     <div class="justifyLeft characterAttachmentsBlock marginBot10">
-        <h3 class="margin0 title_restorable">
+        <h3 class="characterAttachmentsTitle margin0 title_restorable">
             <span data-i18n="Character Attachments">
                 Character Attachments
             </span>
-            <div class="flex-container flexGap10">
-                <div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Download a page from the web.">
-                    <i class="fa-fw fa-solid fa-globe"></i>
-                    <span data-i18n="From Web">
-                        From Web
-                    </span>
-                </div>
-                <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
-                    <span data-i18n="From Fandom">
-                        From Fandom
-                    </span>
-                </div>
-                <div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Upload a file from your computer.">
-                    <i class="fa-fw fa-solid fa-upload"></i>
-                    <span data-i18n="From File">
-                        From File
-                    </span>
-                </div>
-            </div>
         </h3>
         <div class="flex-container flexFlowColumn">
             <strong><small class="characterAttachmentsName"></small></strong>
@@ -78,30 +62,10 @@
         <hr>
     </div>
     <div class="justifyLeft chatAttachmentsBlock marginBot10">
-        <h3 class="margin0 title_restorable">
+        <h3 class="chatAttachmentsTitle margin0 title_restorable">
             <span data-i18n="Chat Attachments">
                 Chat Attachments
             </span>
-            <div class="flex-container flexGap10">
-                <div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Download a page from the web.">
-                    <i class="fa-fw fa-solid fa-globe"></i>
-                    <span data-i18n="From Web">
-                        From Web
-                    </span>
-                </div>
-                <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
-                    <span data-i18n="From Fandom">
-                        From Fandom
-                    </span>
-                </div>
-                <div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Upload a file from your computer.">
-                    <i class="fa-fw fa-solid fa-upload"></i>
-                    <span data-i18n="From File">
-                        From File
-                    </span>
-                </div>
-            </div>
         </h3>
         <div class="flex-container flexFlowColumn">
             <strong><small class="chatAttachmentsName"></small></strong>
@@ -116,9 +80,33 @@
         <div class="attachmentListItem flex-container alignItemsCenter flexGap10">
             <div class="attachmentFileIcon fa-solid fa-file-alt"></div>
             <div class="attachmentListItemName flex1"></div>
+            <small class="attachmentListItemCreated"></small>
             <small class="attachmentListItemSize"></small>
             <div class="viewAttachmentButton right_menu_button fa-solid fa-magnifying-glass" title="View attachment content"></div>
             <div class="deleteAttachmentButton right_menu_button fa-solid fa-trash" title="Delete attachment"></div>
         </div>
     </div>
+
+    <div class="actionButtonsTemplate template_element">
+        <div class="actionButtons flex-container flexGap10">
+            <div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="" title="Download a page from the web.">
+                <i class="fa-fw fa-solid fa-globe"></i>
+                <span data-i18n="From Web">
+                    From Web
+                </span>
+            </div>
+            <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="" title="Download a page from the Fandom wiki.">
+            <i class="fa-fw fa-solid fa-fire"></i>
+                <span data-i18n="From Fandom">
+                    From Fandom
+                </span>
+            </div>
+            <div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="" title="Upload a file from your computer.">
+                <i class="fa-fw fa-solid fa-upload"></i>
+                <span data-i18n="From File">
+                    From File
+                </span>
+            </div>
+        </div>
+    </div>
 </div>

public/scripts/extensions/attachments/style.css

@@ -18,3 +18,12 @@
 .attachmentListItem {
     padding: 10px;
 }
+
+.attachmentListItemSize {
+    min-width: 4em;
+    text-align: right;
+}
+
+.attachmentListItemCreated {
+    text-align: right;
+}

public/scripts/extensions/vectors/index.js

@@ -222,8 +222,7 @@ async function processFiles(chat) {
 
             // Trim file inserted by the script
             const fileText = String(message.mes)
-                .substring(0, message.extra.fileLength).trim()
-                .replace(/^```/, '').replace(/```$/, '').trim();
+                .substring(0, message.extra.fileLength).trim();
 
             // Convert kilobytes to string length
             const thresholdLength = settings.size_threshold * 1024;
@@ -247,8 +246,7 @@ async function processFiles(chat) {
             const queryText = getQueryText(chat);
             const fileChunks = await retrieveFileChunks(queryText, collectionId);
 
-            // Wrap it back in a code block
-            message.mes = `\`\`\`\n${fileChunks}\n\`\`\`\n\n${message.mes}`;
+            message.mes = `${fileChunks}\n\n${message.mes}`;
         }
     } catch (error) {
         console.error('Vectors: Failed to retrieve files', error);

public/scripts/login.js

@@ -177,9 +177,10 @@ function displayError(message) {
 
 /**
  * Redirects the user to the home page.
+ * Preserves the query string.
  */
 function redirectToHome() {
-    window.location.href = '/';
+    window.location.href = '/' + window.location.search;
 }
 
 /**

public/scripts/openai.js

@@ -1594,6 +1594,11 @@ async function sendAltScaleRequest(messages, logit_bias, signal, type) {
         signal: signal,
     });
 
+    if (!response.ok) {
+        tryParseStreamingError(response, await response.text());
+        throw new Error('Scale response does not indicate success.');
+    }
+
     const data = await response.json();
     return data.output;
 }

server.js

@@ -248,7 +248,9 @@ if (!disableCsrf) {
 // Host index page
 app.get('/', (request, response) => {
     if (userModule.shouldRedirectToLogin(request)) {
-        return response.redirect('/login');
+        const query = request.url.split('?')[1];
+        const redirectUrl = query ? `/login?${query}` : '/login';
+        return response.redirect(redirectUrl);
     }
 
     return response.sendFile('index.html', { root: path.join(process.cwd(), 'public') });

src/endpoints/backends/scale-alt.js

@@ -7,16 +7,18 @@ const { readSecret, SECRET_KEYS } = require('../secrets');
 
 const router = express.Router();
 
-router.post('/generate', jsonParser, function (request, response) {
+router.post('/generate', jsonParser, async function (request, response) {
     if (!request.body) return response.sendStatus(400);
 
-    fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', {
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-            'cookie': `_jwt=${readSecret(request.user.directories, SECRET_KEYS.SCALE_COOKIE)}`,
-        },
-        body: JSON.stringify({
+    try {
+        const cookie = readSecret(request.user.directories, SECRET_KEYS.SCALE_COOKIE);
+
+        if (!cookie) {
+            console.log('No Scale cookie found');
+            return response.sendStatus(400);
+        }
+
+        const body = {
             json: {
                 variant: {
                     name: 'New Variant',
@@ -59,18 +61,41 @@ router.post('/generate', jsonParser, function (request, response) {
                     'modelParameters.logprobs': ['undefined'],
                 },
             },
-        }),
-    })
-        .then(res => res.json())
-        .then(data => {
-            console.log(data.result.data.json.outputs[0]);
-            return response.send({ output: data.result.data.json.outputs[0] });
-        })
-        .catch((error) => {
-            console.error('Error:', error);
-            return response.send({ error: true });
+        };
+
+        console.log('Scale request:', body);
+
+        const result = await fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'cookie': `_jwt=${cookie}`,
+            },
+            timeout: 0,
+            body: JSON.stringify(body),
         });
 
+        if (!result.ok) {
+            const text = await result.text();
+            console.log('Scale request failed', result.statusText, text);
+            return response.status(500).send({ error: { message: result.statusText } });
+        }
+
+        const data = await result.json();
+        const output = data?.result?.data?.json?.outputs?.[0] || '';
+
+        console.log('Scale response:', data);
+
+        if (!output) {
+            console.warn('Scale response is empty');
+            return response.sendStatus(500).send({ error: { message: 'Empty response' } });
+        }
+
+        return response.json({ output });
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
 });
 
 module.exports = { router };

src/users.js

@@ -326,7 +326,7 @@ function toAvatarKey(handle) {
 }
 
 /**
- * Initializes the user storage. Currently a no-op.
+ * Initializes the user storage.
  * @param {string} dataRoot The root directory for user data
  * @returns {Promise<void>}
  */
@@ -655,6 +655,10 @@ async function createBackupArchive(handle, response) {
     archive.finalize();
 }
 
+/**
+ * Checks if any admin users are not password protected. If so, logs a warning.
+ * @returns {Promise<void>}
+ */
 async function checkAccountsProtection() {
     if (!ENABLE_ACCOUNTS) {
         return;