Integrate data bank with Fandom plugin

public/scripts/chats.js

@@ -41,6 +41,7 @@ import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from './popup.js';
  * @property {string} url File URL
  * @property {number} size File size
  * @property {string} name File name
+ * @property {number} created Timestamp
  * @property {string} [text] File text
  */
 
@@ -168,6 +169,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
                 url: fileUrl,
                 size: file.size,
                 name: file.name,
+                created: Date.now(),
             };
         }
 
@@ -734,8 +736,85 @@ async function getTitleFromHtmlBlob(blob) {
  * @param {function} callback Callback function
  */
 async function openFandomScraper(target, callback) {
-    toastr.info('Not implemented yet', target);
+    if (!await isFandomPluginAvailable()) {
-    callback();
+        toastr.error('Fandom scraper plugin is not available');
+        return;
+    }
+
+    let fandom = '';
+    let filter = '';
+    let output = 'single';
+
+    const template = $(await renderExtensionTemplateAsync('attachments', 'fandom-scrape', {}));
+    template.find('input[name="fandomScrapeInput"]').on('input', function () {
+        fandom = String($(this).val());
+    });
+    template.find('input[name="fandomScrapeFilter"]').on('input', function () {
+        filter = String($(this).val());
+    });
+    template.find('input[name="fandomScrapeOutput"]').on('input', function () {
+        output = String($(this).val());
+    });
+
+    const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false });
+
+    if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
+        return;
+    }
+
+    if (!fandom) {
+        toastr.error('Fandom name is required');
+        return;
+    }
+
+    try {
+        const result = await fetch('/api/plugins/fandom/scrape', {
+            method: 'POST',
+            headers: getRequestHeaders(),
+            body: JSON.stringify({ fandom, filter }),
+        });
+
+        if (!result.ok) {
+            const error = await result.text();
+            throw new Error(error);
+        }
+
+        // Get domain name part if it's a URL
+        try {
+            const url = new URL(fandom);
+            const fandomId = url.hostname.split('.')[0] || fandom;
+            fandom = fandomId;
+        } catch {
+            // Ignore
+        }
+
+        const data = await result.json();
+        let numberOfAttachments;
+
+        if (output === 'multi') {
+            numberOfAttachments = data.length;
+            for (const attachment of data) {
+                const file = new File([String(attachment.content).trim()], `${String(attachment.title).trim()}.txt`, { type: 'text/plain' });
+                await uploadFileAttachmentToServer(file, target);
+            }
+        }
+
+        if (output === 'single') {
+            numberOfAttachments = 1;
+            const combinedContent = data.map((a) => String(a.title).trim() + '\n\n' + String(a.content).trim()).join('\n\n\n\n');
+            const file = new File([combinedContent], `${fandom}.txt`, { type: 'text/plain' });
+            await uploadFileAttachmentToServer(file, target);
+        }
+
+        if (numberOfAttachments) {
+            toastr.success(`Scraped ${numberOfAttachments} attachments from ${fandom}`);
+        }
+
+        callback();
+    } catch (error) {
+        console.error('Fandom scraping failed', error);
+        toastr.error('Check browser console for details.', 'Fandom scraping failed');
+    }
 }
 
 /**
@@ -791,6 +870,7 @@ async function uploadFileAttachmentToServer(file, target) {
     }
 
     const fileUrl = await uploadFileAttachment(uniqueFileName, base64Data);
+    const convertedSize = Math.round(base64Data.length * 0.75);
 
     if (!fileUrl) {
         return;
@@ -798,8 +878,9 @@ async function uploadFileAttachmentToServer(file, target) {
 
     const attachment = {
         url: fileUrl,
-        size: file.size,
+        size: convertedSize,
         name: file.name,
+        created: Date.now(),
     };
 
     ensureAttachmentsExist();

public/scripts/extensions/attachments/fandom-scrape.html

@@ -0,0 +1,51 @@
+<div>
+    <div class="flex-container flexFlowColumn">
+        <label for="fandomScrapeInput" data-i18n="Enter a URL or the ID of a Fandom wiki page to scrape:">
+            Enter a URL or the ID of a Fandom wiki page to scrape:
+        </label>
+        <small>
+            <span data-i18n=Examples:">Examples:</span>
+            <code>https://harrypotter.fandom.com/</code>
+            <span data-i18n="or">or</span>
+            <code>harrypotter</code>
+        </small>
+        <input type="text" id="fandomScrapeInput" name="fandomScrapeInput" class="text_pole" placeholder="">
+    </div>
+    <div class="flex-container flexFlowColumn">
+        <label for="fandomScrapeFilter">
+            Optional regex to pick the content by its title:
+        </label>
+        <small>
+            <span data-i18n="Example:">Example:</span>
+            <code>/(Azkaban|Weasley)/gi</code>
+        </small>
+        <input type="text" id="fandomScrapeFilter" name="fandomScrapeFilter" class="text_pole" placeholder="">
+    </div>
+    <div class="flex-container flexFlowColumn">
+        <label>
+            Output format:
+        </label>
+        <label class="checkbox_label justifyLeft" for="fandomScrapeOutputSingle">
+            <input id="fandomScrapeOutputSingle" type="radio" name="fandomScrapeOutput" value="single" checked>
+            <div class="flex-container flexFlowColumn flexNoGap">
+                <span data-i18n="Single file">
+                    Single file
+                </span>
+                <small data-i18n="All articles will be concatenated into a single file.">
+                    All articles will be concatenated into a single file.
+                </small>
+            </div>
+        </label>
+        <label class="checkbox_label justifyLeft" for="fandomScrapeOutputMulti">
+            <input id="fandomScrapeOutputMulti" type="radio" name="fandomScrapeOutput" value="multi">
+            <div class="flex-container flexFlowColumn flexNoGap">
+                <span data-i18n="File per article">
+                    File per article
+                </span>
+                <small data-i18n="Each article will be saved as a separate file.">
+                    Not recommended. Each article will be saved as a separate file.
+                </small>
+            </div>
+        </label>
+    </div>
+</div>

public/scripts/extensions/attachments/manager.html

@@ -23,7 +23,7 @@
                     </span>
                 </div>
                 <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
+                    <i class="fa-fw fa-solid fa-fire"></i>
                     <span data-i18n="From Fandom">
                         From Fandom
                     </span>
@@ -55,7 +55,7 @@
                     </span>
                 </div>
                 <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
+                    <i class="fa-fw fa-solid fa-fire"></i>
                     <span data-i18n="From Fandom">
                         From Fandom
                     </span>
@@ -90,7 +90,7 @@
                     </span>
                 </div>
                 <div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Download a page from the Fandom wiki.">
-                    <i class="fa-fw fa-solid fa-fan"></i>
+                    <i class="fa-fw fa-solid fa-fire"></i>
                     <span data-i18n="From Fandom">
                         From Fandom
                     </span>