Add prerequisites for websearch extension

public/script.js

@@ -190,7 +190,7 @@ import { getBackgrounds, initBackgrounds } from './scripts/backgrounds.js';
 import { hideLoader, showLoader } from './scripts/loader.js';
 import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
 import { loadMancerModels } from './scripts/mancer-settings.js';
-import { getFileAttachment, hasPendingFileAttachment, populateFileAttachment } from './scripts/chats.js';
+import { appendFileContent, hasPendingFileAttachment, populateFileAttachment } from './scripts/chats.js';
 import { replaceVariableMacros } from './scripts/variables.js';
 
 //exporting functions and vars for mods
@@ -3098,26 +3098,18 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
             coreChat.pop();
         }
 
-        coreChat = await Promise.all(coreChat.map(async (chatItem) => {
+        coreChat = await Promise.all(coreChat.map(async (chatItem, index) => {
             let message = chatItem.mes;
             let regexType = chatItem.is_user ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT;
             let options = { isPrompt: true };
 
             let regexedMessage = getRegexedString(message, regexType, options);
-
-            if (chatItem.extra?.file) {
-                const fileText = chatItem.extra.file.text || (await getFileAttachment(chatItem.extra.file.url));
-
-                if (fileText) {
-                    const fileWrapped = `\`\`\`\n${fileText}\n\`\`\`\n\n`;
-                    chatItem.extra.fileLength = fileWrapped.length;
-                    regexedMessage = fileWrapped + regexedMessage;
-                }
-            }
+            regexedMessage = await appendFileContent(chatItem, regexedMessage);
 
             return {
                 ...chatItem,
                 mes: regexedMessage,
+                index,
             };
         }));
 

public/scripts/chats.js

@@ -341,6 +341,25 @@ function embedMessageFile(messageId, messageBlock) {
     }
 }
 
+/**
+ * Appends file content to the message text.
+ * @param {object} message Message object
+ * @param {string} messageText Message text
+ * @returns {Promise<string>} Message text with file content appended.
+ */
+export async function appendFileContent(message, messageText) {
+    if (message.extra?.file) {
+        const fileText = message.extra.file.text || (await getFileAttachment(message.extra.file.url));
+
+        if (fileText) {
+            const fileWrapped = `\`\`\`\n${fileText}\n\`\`\`\n\n`;
+            message.extra.fileLength = fileWrapped.length;
+            messageText = fileWrapped + messageText;
+        }
+    }
+    return messageText;
+}
+
 jQuery(function () {
     $(document).on('click', '.mes_hide', async function () {
         const messageBlock = $(this).closest('.mes');

public/scripts/extensions.js

@@ -879,7 +879,7 @@ async function runGenerationInterceptors(chat, contextSize) {
         exitImmediately = immediately;
     };
 
-    for (const manifest of Object.values(manifests)) {
+    for (const manifest of Object.values(manifests).sort((a, b) => a.loading_order - b.loading_order)) {
         const interceptorKey = manifest.generate_interceptor;
         if (typeof window[interceptorKey] === 'function') {
             try {

public/scripts/utils.js

@@ -1143,11 +1143,13 @@ export async function extractTextFromPDF(blob) {
  * @param {Blob} blob HTML content blob
  * @returns {Promise<string>} A promise that resolves to the parsed text.
  */
-export async function extractTextFromHTML(blob) {
+export async function extractTextFromHTML(blob, textSelector = 'body') {
     const html = await blob.text();
     const domParser = new DOMParser();
     const document = domParser.parseFromString(DOMPurify.sanitize(html), 'text/html');
-    const text = postProcessText(document.body.textContent);
+    const elements = document.querySelectorAll(textSelector);
+    const rawText = Array.from(elements).map(e => e.textContent).join('\n');
+    const text = postProcessText(rawText);
     return text;
 }
 

src/endpoints/serpapi.js

@@ -5,6 +5,23 @@ const { jsonParser } = require('../express-common');
 
 const router = express.Router();
 
+// Cosplay as Firefox
+const visitHeaders = {
+    'Accept': '*/*',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Cache-Control': 'no-cache',
+    'Pragma': 'no-cache',
+    'TE': 'trailers',
+    'DNT': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+};
+
 router.post('/search', jsonParser, async (request, response) => {
     try {
         const key = readSecret(SECRET_KEYS.SERPAPI);
@@ -31,4 +48,28 @@ router.post('/search', jsonParser, async (request, response) => {
     }
 });
 
+router.post('/visit', jsonParser, async (request, response) => {
+    try {
+        const url = request.body.url;
+
+        if (!url) {
+            console.log('No url provided for /visit');
+            return response.sendStatus(400);
+        }
+
+        const result = await fetch(url, { headers: visitHeaders });
+
+        if (!result.ok) {
+            console.log(`Visit failed ${result.status} ${result.statusText}`);
+            return response.sendStatus(500);
+        }
+
+        const text = await result.text();
+        return response.send(text);
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
 module.exports = { router };