Merge pull request #1452 from valadaptive/assets-router

Use Express router for assets + "files" endpoints

public/script.js

@@ -2914,7 +2914,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
         let textareaText;
         if (type !== 'regenerate' && type !== 'swipe' && type !== 'quiet' && !isImpersonate && !dryRun) {
             is_send_press = true;
-            textareaText = $('#send_textarea').val();
+            textareaText = String($('#send_textarea').val());
             $('#send_textarea').val('').trigger('input');
         } else {
             textareaText = '';
@@ -2960,7 +2960,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
         //*********************************
 
         //for normal messages sent from user..
-        if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet') {
+        if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet' && !dryRun) {
             // If user message contains no text other than bias - send as a system message
             if (messageBias && replaceBiasMarkup(textareaText).trim().length === 0) {
                 sendSystemMessage(system_message_types.GENERIC, ' ', { bias: messageBias });
@@ -2969,7 +2969,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
                 await sendMessageAsUser(textareaText, messageBias);
             }
         }
-        else if (textareaText == '' && !automatic_trigger && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
+        else if (textareaText == '' && !automatic_trigger && !dryRun && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
             // Use send_if_empty if set and the user message is empty. Only when sending messages normally
             await sendMessageAsUser(oai_settings.send_if_empty.trim(), messageBias);
         }

public/scripts/chats.js

@@ -152,7 +152,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
  */
 export async function uploadFileAttachment(fileName, base64Data) {
     try {
-        const result = await fetch('/api/file/upload', {
+        const result = await fetch('/api/files/upload', {
             method: 'POST',
             headers: getRequestHeaders(),
             body: JSON.stringify({
@@ -302,7 +302,7 @@ async function viewMessageFile(messageId) {
     modalTemplate.addClass('file_modal');
     addCopyToCodeBlocks(modalTemplate);
 
-    callPopup(modalTemplate, 'text');
+    callPopup(modalTemplate, 'text', '', { wide: true, large: true });
 }
 
 /**

server.js

@@ -3597,7 +3597,10 @@ require('./src/endpoints/novelai').registerEndpoints(app, jsonParser);
 require('./src/endpoints/extensions').registerEndpoints(app, jsonParser);
 
 // Asset management
-require('./src/endpoints/assets').registerEndpoints(app, jsonParser);
+app.use('/api/assets', require('./src/endpoints/assets').router);
+
+// File management
+app.use('/api/files', require('./src/endpoints/files').router);
 
 // Character sprite management
 require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser);

src/endpoints/assets.js

@@ -1,10 +1,12 @@
 const path = require('path');
 const fs = require('fs');
+const express = require('express');
 const sanitize = require('sanitize-filename');
 const fetch = require('node-fetch').default;
 const { finished } = require('stream/promises');
 const writeFileSyncAtomic = require('write-file-atomic').sync;
 const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
+const { jsonParser } = require('../express-common');
 
 const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
 
@@ -57,273 +59,266 @@ function getFiles(dir, files = []) {
     return files;
 }
 
+const router = express.Router();
+
 /**
- * Registers the endpoints for the asset management.
+ * HTTP POST handler function to retrieve name of all files of a given folder path.
- * @param {import('express').Express} app Express app
+ *
- * @param {any} jsonParser JSON parser middleware
+ * @param {Object} request - HTTP Request object. Require folder path in query
+ * @param {Object} response - HTTP Response object will contain a list of file path.
+ *
+ * @returns {void}
  */
-function registerEndpoints(app, jsonParser) {
+router.post('/get', jsonParser, async (_, response) => {
-    /**
+    const folderPath = path.join(DIRECTORIES.assets);
-     * HTTP POST handler function to retrieve name of all files of a given folder path.
+    let output = {};
-     *
+    //console.info("Checking files into",folderPath);
-     * @param {Object} request - HTTP Request object. Require folder path in query
-     * @param {Object} response - HTTP Response object will contain a list of file path.
-     *
-     * @returns {void}
-     */
-    app.post('/api/assets/get', jsonParser, async (_, response) => {
-        const folderPath = path.join(DIRECTORIES.assets);
-        let output = {};
-        //console.info("Checking files into",folderPath);
 
-        try {
+    try {
-            if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
+        if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
-                const folders = fs.readdirSync(folderPath)
+            const folders = fs.readdirSync(folderPath)
-                    .filter(filename => {
+                .filter(filename => {
-                        return fs.statSync(path.join(folderPath, filename)).isDirectory();
+                    return fs.statSync(path.join(folderPath, filename)).isDirectory();
-                    });
-
-                for (const folder of folders) {
-                    if (folder == 'temp')
-                        continue;
-
-                    // Live2d assets
-                    if (folder == 'live2d') {
-                        output[folder] = [];
-                        const live2d_folder = path.normalize(path.join(folderPath, folder));
-                        const files = getFiles(live2d_folder);
-                        //console.debug("FILE FOUND:",files)
-                        for (let file of files) {
-                            file = path.normalize(file.replace('public' + path.sep, ''));
-                            if (file.includes('model') && file.endsWith('.json')) {
-                                //console.debug("Asset live2d model found:",file)
-                                output[folder].push(path.normalize(path.join(file)));
-                            }
-                        }
-                        continue;
-                    }
-
-                    // Other assets (bgm/ambient/blip)
-                    const files = fs.readdirSync(path.join(folderPath, folder))
-                        .filter(filename => {
-                            return filename != '.placeholder';
-                        });
-                    output[folder] = [];
-                    for (const file of files) {
-                        output[folder].push(path.join('assets', folder, file));
-                    }
-                }
-            }
-        }
-        catch (err) {
-            console.log(err);
-        }
-        return response.send(output);
-    });
-
-    /**
-     * HTTP POST handler function to download the requested asset.
-     *
-     * @param {Object} request - HTTP Request object, expects a url, a category and a filename.
-     * @param {Object} response - HTTP Response only gives status.
-     *
-     * @returns {void}
-     */
-    app.post('/api/assets/download', jsonParser, async (request, response) => {
-        const url = request.body.url;
-        const inputCategory = request.body.category;
-        const inputFilename = sanitize(request.body.filename);
-
-        // Check category
-        let category = null;
-        for (let i of VALID_CATEGORIES)
-            if (i == inputCategory)
-                category = i;
-
-        if (category === null) {
-            console.debug('Bad request: unsuported asset category.');
-            return response.sendStatus(400);
-        }
-
-        // Sanitize filename
-        const safe_input = checkAssetFileName(inputFilename);
-        if (safe_input == '')
-            return response.sendStatus(400);
-
-        const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
-        const file_path = path.join(DIRECTORIES.assets, category, safe_input);
-        console.debug('Request received to download', url, 'to', file_path);
-
-        try {
-            // Download to temp
-            const res = await fetch(url);
-            if (!res.ok || res.body === null) {
-                throw new Error(`Unexpected response ${res.statusText}`);
-            }
-            const destination = path.resolve(temp_path);
-            // Delete if previous download failed
-            if (fs.existsSync(temp_path)) {
-                fs.unlink(temp_path, (err) => {
-                    if (err) throw err;
                 });
-            }
-            const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
-            await finished(res.body.pipe(fileStream));
 
-            // Move into asset place
+            for (const folder of folders) {
-            console.debug('Download finished, moving file from', temp_path, 'to', file_path);
+                if (folder == 'temp')
-            fs.renameSync(temp_path, file_path);
+                    continue;
-            response.sendStatus(200);
-        }
-        catch (error) {
-            console.log(error);
-            response.sendStatus(500);
-        }
-    });
-
-    /**
-     * HTTP POST handler function to delete the requested asset.
-     *
-     * @param {Object} request - HTTP Request object, expects a category and a filename
-     * @param {Object} response - HTTP Response only gives stats.
-     *
-     * @returns {void}
-     */
-    app.post('/api/assets/delete', jsonParser, async (request, response) => {
-        const inputCategory = request.body.category;
-        const inputFilename = sanitize(request.body.filename);
-
-        // Check category
-        let category = null;
-        for (let i of VALID_CATEGORIES)
-            if (i == inputCategory)
-                category = i;
-
-        if (category === null) {
-            console.debug('Bad request: unsuported asset category.');
-            return response.sendStatus(400);
-        }
-
-        // Sanitize filename
-        const safe_input = checkAssetFileName(inputFilename);
-        if (safe_input == '')
-            return response.sendStatus(400);
-
-        const file_path = path.join(DIRECTORIES.assets, category, safe_input);
-        console.debug('Request received to delete', category, file_path);
-
-        try {
-            // Delete if previous download failed
-            if (fs.existsSync(file_path)) {
-                fs.unlink(file_path, (err) => {
-                    if (err) throw err;
-                });
-                console.debug('Asset deleted.');
-            }
-            else {
-                console.debug('Asset not found.');
-                response.sendStatus(400);
-            }
-            // Move into asset place
-            response.sendStatus(200);
-        }
-        catch (error) {
-            console.log(error);
-            response.sendStatus(500);
-        }
-    });
-
-    ///////////////////////////////
-    /**
-     * HTTP POST handler function to retrieve a character background music list.
-     *
-     * @param {Object} request - HTTP Request object, expects a character name in the query.
-     * @param {Object} response - HTTP Response object will contain a list of audio file path.
-     *
-     * @returns {void}
-     */
-    app.post('/api/assets/character', jsonParser, async (request, response) => {
-        if (request.query.name === undefined) return response.sendStatus(400);
-        const name = sanitize(request.query.name.toString());
-        const inputCategory = request.query.category;
-
-        // Check category
-        let category = null;
-        for (let i of VALID_CATEGORIES)
-            if (i == inputCategory)
-                category = i;
-
-        if (category === null) {
-            console.debug('Bad request: unsuported asset category.');
-            return response.sendStatus(400);
-        }
-
-        const folderPath = path.join(DIRECTORIES.characters, name, category);
-
-        let output = [];
-        try {
-            if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
 
                 // Live2d assets
-                if (category == 'live2d') {
+                if (folder == 'live2d') {
-                    const folders = fs.readdirSync(folderPath);
+                    output[folder] = [];
-                    for (let modelFolder of folders) {
+                    const live2d_folder = path.normalize(path.join(folderPath, folder));
-                        const live2dModelPath = path.join(folderPath, modelFolder);
+                    const files = getFiles(live2d_folder);
-                        if (fs.statSync(live2dModelPath).isDirectory()) {
+                    //console.debug("FILE FOUND:",files)
-                            for (let file of fs.readdirSync(live2dModelPath)) {
+                    for (let file of files) {
-                                //console.debug("Character live2d model found:", file)
+                        file = path.normalize(file.replace('public' + path.sep, ''));
-                                if (file.includes('model') && file.endsWith('.json'))
+                        if (file.includes('model') && file.endsWith('.json')) {
-                                    output.push(path.join('characters', name, category, modelFolder, file));
+                            //console.debug("Asset live2d model found:",file)
-                            }
+                            output[folder].push(path.normalize(path.join(file)));
                         }
                     }
-                    return response.send(output);
+                    continue;
                 }
 
-                // Other assets
+                // Other assets (bgm/ambient/blip)
-                const files = fs.readdirSync(folderPath)
+                const files = fs.readdirSync(path.join(folderPath, folder))
                     .filter(filename => {
                         return filename != '.placeholder';
                     });
-
+                output[folder] = [];
-                for (let i of files)
+                for (const file of files) {
-                    output.push(`/characters/${name}/${category}/${i}`);
+                    output[folder].push(path.join('assets', folder, file));
+                }
             }
-            return response.send(output);
         }
-        catch (err) {
+    }
-            console.log(err);
+    catch (err) {
-            return response.sendStatus(500);
+        console.log(err);
+    }
+    return response.send(output);
+});
+
+/**
+ * HTTP POST handler function to download the requested asset.
+ *
+ * @param {Object} request - HTTP Request object, expects a url, a category and a filename.
+ * @param {Object} response - HTTP Response only gives status.
+ *
+ * @returns {void}
+ */
+router.post('/download', jsonParser, async (request, response) => {
+    const url = request.body.url;
+    const inputCategory = request.body.category;
+    const inputFilename = sanitize(request.body.filename);
+
+    // Check category
+    let category = null;
+    for (let i of VALID_CATEGORIES)
+        if (i == inputCategory)
+            category = i;
+
+    if (category === null) {
+        console.debug('Bad request: unsuported asset category.');
+        return response.sendStatus(400);
+    }
+
+    // Sanitize filename
+    const safe_input = checkAssetFileName(inputFilename);
+    if (safe_input == '')
+        return response.sendStatus(400);
+
+    const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
+    const file_path = path.join(DIRECTORIES.assets, category, safe_input);
+    console.debug('Request received to download', url, 'to', file_path);
+
+    try {
+        // Download to temp
+        const res = await fetch(url);
+        if (!res.ok || res.body === null) {
+            throw new Error(`Unexpected response ${res.statusText}`);
         }
-    });
+        const destination = path.resolve(temp_path);
-
+        // Delete if previous download failed
-    app.post('/api/file/upload', jsonParser, async (request, response) => {
+        if (fs.existsSync(temp_path)) {
-        try {
+            fs.unlink(temp_path, (err) => {
-            if (!request.body.name) {
+                if (err) throw err;
-                return response.status(400).send('No upload name specified');
+            });
-            }
-
-            if (!request.body.data) {
-                return response.status(400).send('No upload data specified');
-            }
-
-            const safeInput = checkAssetFileName(request.body.name);
-
-            if (!safeInput) {
-                return response.status(400).send('Invalid upload name');
-            }
-
-            const pathToUpload = path.join(DIRECTORIES.files, safeInput);
-            writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
-            const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
-            return response.send({ path: url });
-        } catch (error) {
-            console.log(error);
-            return response.sendStatus(500);
         }
-    });
+        const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
-}
+        await finished(res.body.pipe(fileStream));
 
-module.exports = {
+        // Move into asset place
-    registerEndpoints,
+        console.debug('Download finished, moving file from', temp_path, 'to', file_path);
-};
+        fs.renameSync(temp_path, file_path);
+        response.sendStatus(200);
+    }
+    catch (error) {
+        console.log(error);
+        response.sendStatus(500);
+    }
+});
+
+/**
+ * HTTP POST handler function to delete the requested asset.
+ *
+ * @param {Object} request - HTTP Request object, expects a category and a filename
+ * @param {Object} response - HTTP Response only gives stats.
+ *
+ * @returns {void}
+ */
+router.post('/delete', jsonParser, async (request, response) => {
+    const inputCategory = request.body.category;
+    const inputFilename = sanitize(request.body.filename);
+
+    // Check category
+    let category = null;
+    for (let i of VALID_CATEGORIES)
+        if (i == inputCategory)
+            category = i;
+
+    if (category === null) {
+        console.debug('Bad request: unsuported asset category.');
+        return response.sendStatus(400);
+    }
+
+    // Sanitize filename
+    const safe_input = checkAssetFileName(inputFilename);
+    if (safe_input == '')
+        return response.sendStatus(400);
+
+    const file_path = path.join(DIRECTORIES.assets, category, safe_input);
+    console.debug('Request received to delete', category, file_path);
+
+    try {
+        // Delete if previous download failed
+        if (fs.existsSync(file_path)) {
+            fs.unlink(file_path, (err) => {
+                if (err) throw err;
+            });
+            console.debug('Asset deleted.');
+        }
+        else {
+            console.debug('Asset not found.');
+            response.sendStatus(400);
+        }
+        // Move into asset place
+        response.sendStatus(200);
+    }
+    catch (error) {
+        console.log(error);
+        response.sendStatus(500);
+    }
+});
+
+///////////////////////////////
+/**
+ * HTTP POST handler function to retrieve a character background music list.
+ *
+ * @param {Object} request - HTTP Request object, expects a character name in the query.
+ * @param {Object} response - HTTP Response object will contain a list of audio file path.
+ *
+ * @returns {void}
+ */
+router.post('/character', jsonParser, async (request, response) => {
+    if (request.query.name === undefined) return response.sendStatus(400);
+    const name = sanitize(request.query.name.toString());
+    const inputCategory = request.query.category;
+
+    // Check category
+    let category = null;
+    for (let i of VALID_CATEGORIES)
+        if (i == inputCategory)
+            category = i;
+
+    if (category === null) {
+        console.debug('Bad request: unsuported asset category.');
+        return response.sendStatus(400);
+    }
+
+    const folderPath = path.join(DIRECTORIES.characters, name, category);
+
+    let output = [];
+    try {
+        if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
+
+            // Live2d assets
+            if (category == 'live2d') {
+                const folders = fs.readdirSync(folderPath);
+                for (let modelFolder of folders) {
+                    const live2dModelPath = path.join(folderPath, modelFolder);
+                    if (fs.statSync(live2dModelPath).isDirectory()) {
+                        for (let file of fs.readdirSync(live2dModelPath)) {
+                            //console.debug("Character live2d model found:", file)
+                            if (file.includes('model') && file.endsWith('.json'))
+                                output.push(path.join('characters', name, category, modelFolder, file));
+                        }
+                    }
+                }
+                return response.send(output);
+            }
+
+            // Other assets
+            const files = fs.readdirSync(folderPath)
+                .filter(filename => {
+                    return filename != '.placeholder';
+                });
+
+            for (let i of files)
+                output.push(`/characters/${name}/${category}/${i}`);
+        }
+        return response.send(output);
+    }
+    catch (err) {
+        console.log(err);
+        return response.sendStatus(500);
+    }
+});
+
+router.post('/upload', jsonParser, async (request, response) => {
+    try {
+        if (!request.body.name) {
+            return response.status(400).send('No upload name specified');
+        }
+
+        if (!request.body.data) {
+            return response.status(400).send('No upload data specified');
+        }
+
+        const safeInput = checkAssetFileName(request.body.name);
+
+        if (!safeInput) {
+            return response.status(400).send('Invalid upload name');
+        }
+
+        const pathToUpload = path.join(DIRECTORIES.files, safeInput);
+        writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
+        const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
+        return response.send({ path: url });
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
+module.exports = { router, checkAssetFileName };

src/endpoints/files.js

@@ -0,0 +1,35 @@
+const path = require('path');
+const writeFileSyncAtomic = require('write-file-atomic').sync;
+const express = require('express');
+const router = express.Router();
+const { checkAssetFileName } = require('./assets');
+const { jsonParser } = require('../express-common');
+const { DIRECTORIES } = require('../constants');
+
+router.post('/upload', jsonParser, async (request, response) => {
+    try {
+        if (!request.body.name) {
+            return response.status(400).send('No upload name specified');
+        }
+
+        if (!request.body.data) {
+            return response.status(400).send('No upload data specified');
+        }
+
+        const safeInput = checkAssetFileName(request.body.name);
+
+        if (!safeInput) {
+            return response.status(400).send('Invalid upload name');
+        }
+
+        const pathToUpload = path.join(DIRECTORIES.files, safeInput);
+        writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
+        const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
+        return response.send({ path: url });
+    } catch (error) {
+        console.log(error);
+        return response.sendStatus(500);
+    }
+});
+
+module.exports = { router };