diff --git a/public/scripts/extensions/assets/index.js b/public/scripts/extensions/assets/index.js index 8bab38633..7728ba1b4 100644 --- a/public/scripts/extensions/assets/index.js +++ b/public/scripts/extensions/assets/index.js @@ -150,7 +150,7 @@ async function installAsset(url, assetType, filename) { const category = assetType; try { const body = { url, category, filename }; - const result = await fetch('/asset_download', { + const result = await fetch('/api/assets/download', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify(body), @@ -171,7 +171,7 @@ async function deleteAsset(assetType, filename) { const category = assetType; try { const body = { category, filename }; - const result = await fetch('/asset_delete', { + const result = await fetch('/api/assets/delete', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify(body), @@ -194,7 +194,7 @@ async function deleteAsset(assetType, filename) { async function updateCurrentAssets() { console.debug(DEBUG_PREFIX, "Checking installed assets...") try { - const result = await fetch(`/get_assets`, { + const result = await fetch(`/api/assets/get`, { method: 'POST', headers: getRequestHeaders(), }); diff --git a/public/scripts/extensions/audio/index.js b/public/scripts/extensions/audio/index.js index 97d0757e7..02052382c 100644 --- a/public/scripts/extensions/audio/index.js +++ b/public/scripts/extensions/audio/index.js @@ -191,7 +191,7 @@ async function getAssetsList(type) { console.debug(DEBUG_PREFIX, "getting assets of type", type); try { - const result = await fetch(`/get_assets`, { + const result = await fetch(`/api/assets/get`, { method: 'POST', headers: getRequestHeaders(), }); @@ -209,7 +209,7 @@ async function getCharacterBgmList(name) { console.debug(DEBUG_PREFIX, "getting bgm list for", name); try { - const result = await fetch(`/get_character_assets_list?name=${encodeURIComponent(name)}&category=${CHARACTER_BGM_FOLDER}`, { + const result = await fetch(`/api/assets/character?name=${encodeURIComponent(name)}&category=${CHARACTER_BGM_FOLDER}`, { method: 'POST', headers: getRequestHeaders(), }); diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index c695ca88c..c29cd2067 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -73,7 +73,7 @@ export let secret_state = {}; export async function writeSecret(key, value) { try { - const response = await fetch('/writesecret', { + const response = await fetch('/api/secrets/write', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ key, value }), @@ -94,7 +94,7 @@ export async function writeSecret(key, value) { export async function readSecretState() { try { - const response = await fetch('/readsecretstate', { + const response = await fetch('/api/secrets/read', { method: 'POST', headers: getRequestHeaders(), }); diff --git a/server.js b/server.js index f2b71c223..d9fbcedf1 100644 --- a/server.js +++ b/server.js @@ -9,7 +9,6 @@ const path = require('path'); const readline = require('readline'); const util = require('util'); const { Readable } = require('stream'); -const { finished } = require('stream/promises'); const { TextDecoder } = require('util'); // cli/fs related library imports @@ -62,7 +61,7 @@ const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); const contentManager = require('./src/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { writeSecret, readSecret, readSecretState, migrateSecrets, SECRET_KEYS, getAllSecrets } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); const { delay, getVersion, getImageBuffers } = require('./src/util'); const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); @@ -125,7 +124,6 @@ const whitelistMode = config.whitelistMode; const autorun = config.autorun && !cliArguments.ssl; const enableExtensions = config.enableExtensions; const listen = config.listen; -const allowKeysExposure = config.allowKeysExposure; const API_OPENAI = "https://api.openai.com/v1"; const API_CLAUDE = "https://api.anthropic.com/v1"; @@ -3791,45 +3789,6 @@ function ensurePublicDirectoriesExist() { } } -app.post('/writesecret', jsonParser, (request, response) => { - const key = request.body.key; - const value = request.body.value; - - writeSecret(key, value); - return response.send('ok'); -}); - -app.post('/readsecretstate', jsonParser, (_, response) => { - - try { - const state = readSecretState(); - return response.send(state); - } catch (error) { - console.error(error); - return response.send({}); - } -}); - -app.post('/viewsecrets', jsonParser, async (_, response) => { - if (!allowKeysExposure) { - console.error('secrets.json could not be viewed unless the value of allowKeysExposure in config.conf is set to true'); - return response.sendStatus(403); - } - - try { - const secrets = getAllSecrets(); - - if (!secrets) { - return response.sendStatus(404); - } - - return response.send(secrets); - } catch (error) { - console.error(error); - return response.sendStatus(500); - } -}); - app.post('/delete_sprite', jsonParser, async (request, response) => { const label = request.body.label; const name = request.body.name; @@ -4134,237 +4093,9 @@ function importRisuSprites(data) { } } -/** - * HTTP POST handler function to retrieve name of all files of a given folder path. - * - * @param {Object} request - HTTP Request object. Require folder path in query - * @param {Object} response - HTTP Response object will contain a list of file path. - * - * @returns {void} - */ -app.post('/get_assets', jsonParser, async (request, response) => { - const folderPath = path.join(directories.assets); - let output = {} - //console.info("Checking files into",folderPath); - try { - if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) { - const folders = fs.readdirSync(folderPath) - .filter(filename => { - return fs.statSync(path.join(folderPath, filename)).isDirectory(); - }); - - for (const folder of folders) { - if (folder == "temp") - continue; - const files = fs.readdirSync(path.join(folderPath, folder)) - .filter(filename => { - return filename != ".placeholder"; - }); - output[folder] = []; - for (const file of files) { - output[folder].push(path.join("assets", folder, file)); - } - } - } - } - catch (err) { - console.log(err); - } - finally { - return response.send(output); - } -}); - - -function checkAssetFileName(inputFilename) { - // Sanitize filename - if (inputFilename.indexOf('\0') !== -1) { - console.debug("Bad request: poisong null bytes in filename."); - return ''; - } - - if (!/^[a-zA-Z0-9_\-\.]+$/.test(inputFilename)) { - console.debug("Bad request: illegal character in filename, only alphanumeric, '_', '-' are accepted."); - return ''; - } - - if (contentManager.unsafeExtensions.some(ext => inputFilename.toLowerCase().endsWith(ext))) { - console.debug("Bad request: forbidden file extension."); - return ''; - } - - if (inputFilename.startsWith('.')) { - console.debug("Bad request: filename cannot start with '.'"); - return ''; - } - - return path.normalize(inputFilename).replace(/^(\.\.(\/|\\|$))+/, '');; -} - -/** - * HTTP POST handler function to download the requested asset. - * - * @param {Object} request - HTTP Request object, expects a url, a category and a filename. - * @param {Object} response - HTTP Response only gives status. - * - * @returns {void} - */ -app.post('/asset_download', jsonParser, async (request, response) => { - const url = request.body.url; - const inputCategory = request.body.category; - const inputFilename = sanitize(request.body.filename); - const validCategories = ["bgm", "ambient"]; - const fetch = require('node-fetch').default; - - // Check category - let category = null; - for (let i of validCategories) - if (i == inputCategory) - category = i; - - if (category === null) { - console.debug("Bad request: unsuported asset category."); - return response.sendStatus(400); - } - - // Sanitize filename - const safe_input = checkAssetFileName(inputFilename); - if (safe_input == '') - return response.sendStatus(400); - - const temp_path = path.join(directories.assets, "temp", safe_input) - const file_path = path.join(directories.assets, category, safe_input) - console.debug("Request received to download", url, "to", file_path); - - try { - // Download to temp - const res = await fetch(url); - if (!res.ok || res.body === null) { - throw new Error(`Unexpected response ${res.statusText}`); - } - const destination = path.resolve(temp_path); - // Delete if previous download failed - if (fs.existsSync(temp_path)) { - fs.unlink(temp_path, (err) => { - if (err) throw err; - }); - } - const fileStream = fs.createWriteStream(destination, { flags: 'wx' }); - await finished(res.body.pipe(fileStream)); - - // Move into asset place - console.debug("Download finished, moving file from", temp_path, "to", file_path); - fs.renameSync(temp_path, file_path); - response.sendStatus(200); - } - catch (error) { - console.log(error); - response.sendStatus(500); - } -}); - -/** - * HTTP POST handler function to delete the requested asset. - * - * @param {Object} request - HTTP Request object, expects a category and a filename - * @param {Object} response - HTTP Response only gives stats. - * - * @returns {void} - */ -app.post('/asset_delete', jsonParser, async (request, response) => { - const inputCategory = request.body.category; - const inputFilename = sanitize(request.body.filename); - const validCategories = ["bgm", "ambient"]; - - // Check category - let category = null; - for (let i of validCategories) - if (i == inputCategory) - category = i; - - if (category === null) { - console.debug("Bad request: unsuported asset category."); - return response.sendStatus(400); - } - - // Sanitize filename - const safe_input = checkAssetFileName(inputFilename); - if (safe_input == '') - return response.sendStatus(400); - - const file_path = path.join(directories.assets, category, safe_input) - console.debug("Request received to delete", category, file_path); - - try { - // Delete if previous download failed - if (fs.existsSync(file_path)) { - fs.unlink(file_path, (err) => { - if (err) throw err; - }); - console.debug("Asset deleted."); - } - else { - console.debug("Asset not found."); - response.sendStatus(400); - } - // Move into asset place - response.sendStatus(200); - } - catch (error) { - console.log(error); - response.sendStatus(500); - } -}); - - -/////////////////////////////// -/** - * HTTP POST handler function to retrieve a character background music list. - * - * @param {Object} request - HTTP Request object, expects a character name in the query. - * @param {Object} response - HTTP Response object will contain a list of audio file path. - * - * @returns {void} - */ -app.post('/get_character_assets_list', jsonParser, async (request, response) => { - if (request.query.name === undefined) return response.sendStatus(400); - const name = sanitize(request.query.name.toString()); - const inputCategory = request.query.category; - const validCategories = ["bgm", "ambient"] - - // Check category - let category = null - for (let i of validCategories) - if (i == inputCategory) - category = i - - if (category === null) { - console.debug("Bad request: unsuported asset category."); - return response.sendStatus(400); - } - - const folderPath = path.join(directories.characters, name, category); - - let output = []; - try { - if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) { - const files = fs.readdirSync(folderPath) - .filter(filename => { - return filename != ".placeholder"; - }); - - for (let i of files) - output.push(`/characters/${name}/${category}/${i}`); - - } - return response.send(output); - } - catch (err) { - console.log(err); - return response.sendStatus(500); - } -}); +// Secrets managemenet +require('./src/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation require('./src/thumbnails').registerEndpoints(app, jsonParser); @@ -4375,6 +4106,9 @@ require('./src/novelai').registerEndpoints(app, jsonParser); // Third-party extensions require('./src/extensions').registerEndpoints(app, jsonParser); +// Asset management +require('./src/assets').registerEndpoints(app, jsonParser); + // Stable Diffusion generation require('./src/stable-diffusion').registerEndpoints(app, jsonParser); diff --git a/src/assets.js b/src/assets.js new file mode 100644 index 000000000..d2e022444 --- /dev/null +++ b/src/assets.js @@ -0,0 +1,250 @@ +const path = require('path'); +const fs = require('fs'); +const sanitize = require('sanitize-filename'); +const fetch = require('node-fetch').default; +const { finished } = require('stream/promises'); +const { directories, UNSAFE_EXTENSIONS } = require('./constants'); + +const VALID_CATEGORIES = ["bgm", "ambient"]; + +/** + * Sanitizes the input filename for theasset. + * @param {string} inputFilename Input filename + * @returns {string} Normalized or empty path if invalid + */ +function checkAssetFileName(inputFilename) { + // Sanitize filename + if (inputFilename.indexOf('\0') !== -1) { + console.debug("Bad request: poisong null bytes in filename."); + return ''; + } + + if (!/^[a-zA-Z0-9_\-\.]+$/.test(inputFilename)) { + console.debug("Bad request: illegal character in filename, only alphanumeric, '_', '-' are accepted."); + return ''; + } + + if (UNSAFE_EXTENSIONS.some(ext => inputFilename.toLowerCase().endsWith(ext))) { + console.debug("Bad request: forbidden file extension."); + return ''; + } + + if (inputFilename.startsWith('.')) { + console.debug("Bad request: filename cannot start with '.'"); + return ''; + } + + return path.normalize(inputFilename).replace(/^(\.\.(\/|\\|$))+/, '');; +} + +/** + * Registers the endpoints for the asset management. + * @param {import('express').Express} app Express app + * @param {any} jsonParser JSON parser middleware + */ +function registerEndpoints(app, jsonParser) { + /** + * HTTP POST handler function to retrieve name of all files of a given folder path. + * + * @param {Object} request - HTTP Request object. Require folder path in query + * @param {Object} response - HTTP Response object will contain a list of file path. + * + * @returns {void} + */ + app.post('/api/assets/get', jsonParser, async (_, response) => { + const folderPath = path.join(directories.assets); + let output = {} + //console.info("Checking files into",folderPath); + + try { + if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) { + const folders = fs.readdirSync(folderPath) + .filter(filename => { + return fs.statSync(path.join(folderPath, filename)).isDirectory(); + }); + + for (const folder of folders) { + if (folder == "temp") + continue; + const files = fs.readdirSync(path.join(folderPath, folder)) + .filter(filename => { + return filename != ".placeholder"; + }); + output[folder] = []; + for (const file of files) { + output[folder].push(path.join("assets", folder, file)); + } + } + } + } + catch (err) { + console.log(err); + } + finally { + return response.send(output); + } + }); + + /** + * HTTP POST handler function to download the requested asset. + * + * @param {Object} request - HTTP Request object, expects a url, a category and a filename. + * @param {Object} response - HTTP Response only gives status. + * + * @returns {void} + */ + app.post('/api/assets/download', jsonParser, async (request, response) => { + const url = request.body.url; + const inputCategory = request.body.category; + const inputFilename = sanitize(request.body.filename); + + // Check category + let category = null; + for (let i of VALID_CATEGORIES) + if (i == inputCategory) + category = i; + + if (category === null) { + console.debug("Bad request: unsuported asset category."); + return response.sendStatus(400); + } + + // Sanitize filename + const safe_input = checkAssetFileName(inputFilename); + if (safe_input == '') + return response.sendStatus(400); + + const temp_path = path.join(directories.assets, "temp", safe_input) + const file_path = path.join(directories.assets, category, safe_input) + console.debug("Request received to download", url, "to", file_path); + + try { + // Download to temp + const res = await fetch(url); + if (!res.ok || res.body === null) { + throw new Error(`Unexpected response ${res.statusText}`); + } + const destination = path.resolve(temp_path); + // Delete if previous download failed + if (fs.existsSync(temp_path)) { + fs.unlink(temp_path, (err) => { + if (err) throw err; + }); + } + const fileStream = fs.createWriteStream(destination, { flags: 'wx' }); + await finished(res.body.pipe(fileStream)); + + // Move into asset place + console.debug("Download finished, moving file from", temp_path, "to", file_path); + fs.renameSync(temp_path, file_path); + response.sendStatus(200); + } + catch (error) { + console.log(error); + response.sendStatus(500); + } + }); + + /** + * HTTP POST handler function to delete the requested asset. + * + * @param {Object} request - HTTP Request object, expects a category and a filename + * @param {Object} response - HTTP Response only gives stats. + * + * @returns {void} + */ + app.post('/api/assets/delete', jsonParser, async (request, response) => { + const inputCategory = request.body.category; + const inputFilename = sanitize(request.body.filename); + + // Check category + let category = null; + for (let i of VALID_CATEGORIES) + if (i == inputCategory) + category = i; + + if (category === null) { + console.debug("Bad request: unsuported asset category."); + return response.sendStatus(400); + } + + // Sanitize filename + const safe_input = checkAssetFileName(inputFilename); + if (safe_input == '') + return response.sendStatus(400); + + const file_path = path.join(directories.assets, category, safe_input) + console.debug("Request received to delete", category, file_path); + + try { + // Delete if previous download failed + if (fs.existsSync(file_path)) { + fs.unlink(file_path, (err) => { + if (err) throw err; + }); + console.debug("Asset deleted."); + } + else { + console.debug("Asset not found."); + response.sendStatus(400); + } + // Move into asset place + response.sendStatus(200); + } + catch (error) { + console.log(error); + response.sendStatus(500); + } + }); + + /////////////////////////////// + /** + * HTTP POST handler function to retrieve a character background music list. + * + * @param {Object} request - HTTP Request object, expects a character name in the query. + * @param {Object} response - HTTP Response object will contain a list of audio file path. + * + * @returns {void} + */ + app.post('/api/assets/character', jsonParser, async (request, response) => { + if (request.query.name === undefined) return response.sendStatus(400); + const name = sanitize(request.query.name.toString()); + const inputCategory = request.query.category; + + // Check category + let category = null + for (let i of VALID_CATEGORIES) + if (i == inputCategory) + category = i + + if (category === null) { + console.debug("Bad request: unsuported asset category."); + return response.sendStatus(400); + } + + const folderPath = path.join(directories.characters, name, category); + + let output = []; + try { + if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) { + const files = fs.readdirSync(folderPath) + .filter(filename => { + return filename != ".placeholder"; + }); + + for (let i of files) + output.push(`/characters/${name}/${category}/${i}`); + + } + return response.send(output); + } + catch (err) { + console.log(err); + return response.sendStatus(500); + } + }); +} + +module.exports = { + registerEndpoints, +} diff --git a/src/constants.js b/src/constants.js index 51e7879b5..a9c6cb87d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -25,6 +25,84 @@ const directories = { assets: 'public/assets', }; +const UNSAFE_EXTENSIONS = [ + ".php", + ".exe", + ".com", + ".dll", + ".pif", + ".application", + ".gadget", + ".msi", + ".jar", + ".cmd", + ".bat", + ".reg", + ".sh", + ".py", + ".js", + ".jse", + ".jsp", + ".pdf", + ".html", + ".htm", + ".hta", + ".vb", + ".vbs", + ".vbe", + ".cpl", + ".msc", + ".scr", + ".sql", + ".iso", + ".img", + ".dmg", + ".ps1", + ".ps1xml", + ".ps2", + ".ps2xml", + ".psc1", + ".psc2", + ".msh", + ".msh1", + ".msh2", + ".mshxml", + ".msh1xml", + ".msh2xml", + ".scf", + ".lnk", + ".inf", + ".reg", + ".doc", + ".docm", + ".docx", + ".dot", + ".dotm", + ".dotx", + ".xls", + ".xlsm", + ".xlsx", + ".xlt", + ".xltm", + ".xltx", + ".xlam", + ".ppt", + ".pptm", + ".pptx", + ".pot", + ".potm", + ".potx", + ".ppam", + ".ppsx", + ".ppsm", + ".pps", + ".ppam", + ".sldx", + ".sldm", + ".ws", +]; + module.exports = { directories, + UNSAFE_EXTENSIONS, } diff --git a/src/content-manager.js b/src/content-manager.js index e453d5f4a..999b2a3d1 100644 --- a/src/content-manager.js +++ b/src/content-manager.js @@ -5,83 +5,6 @@ const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const unsafeExtensions = [ - ".php", - ".exe", - ".com", - ".dll", - ".pif", - ".application", - ".gadget", - ".msi", - ".jar", - ".cmd", - ".bat", - ".reg", - ".sh", - ".py", - ".js", - ".jse", - ".jsp", - ".pdf", - ".html", - ".htm", - ".hta", - ".vb", - ".vbs", - ".vbe", - ".cpl", - ".msc", - ".scr", - ".sql", - ".iso", - ".img", - ".dmg", - ".ps1", - ".ps1xml", - ".ps2", - ".ps2xml", - ".psc1", - ".psc2", - ".msh", - ".msh1", - ".msh2", - ".mshxml", - ".msh1xml", - ".msh2xml", - ".scf", - ".lnk", - ".inf", - ".reg", - ".doc", - ".docm", - ".docx", - ".dot", - ".dotm", - ".dotx", - ".xls", - ".xlsm", - ".xlsx", - ".xlt", - ".xltm", - ".xltx", - ".xlam", - ".ppt", - ".pptm", - ".pptx", - ".pot", - ".potm", - ".potx", - ".ppam", - ".ppsx", - ".ppsm", - ".pps", - ".ppam", - ".sldx", - ".sldm", - ".ws", -]; - function checkForNewContent() { try { if (config.skipContentCheck) { @@ -162,5 +85,4 @@ function getContentLog() { module.exports = { checkForNewContent, - unsafeExtensions, } diff --git a/src/secrets.js b/src/secrets.js index b09a33f08..a422f36a7 100644 --- a/src/secrets.js +++ b/src/secrets.js @@ -1,5 +1,6 @@ const fs = require('fs'); const path = require('path'); +const { getConfigValue } = require('./util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); @@ -34,7 +35,7 @@ function writeSecret(key, value) { const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); const secrets = JSON.parse(fileContents); secrets[key] = value; - writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8"); + writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), "utf-8"); } /** @@ -114,7 +115,7 @@ function migrateSecrets(settingsFile) { if (modified) { console.log('Writing updated settings.json...'); - const settingsContent = JSON.stringify(settings); + const settingsContent = JSON.stringify(settings, null, 4); writeFileAtomicSync(settingsFile, settingsContent, "utf-8"); } } @@ -138,11 +139,61 @@ function getAllSecrets() { return secrets; } +/** + * Registers endpoints for the secret management API + * @param {import('express').Express} app Express app + * @param {any} jsonParser JSON parser middleware + */ +function registerEndpoints(app, jsonParser) { + + app.post('/api/secrets/write', jsonParser, (request, response) => { + const key = request.body.key; + const value = request.body.value; + + writeSecret(key, value); + return response.send('ok'); + }); + + app.post('/api/secrets/read', jsonParser, (_, response) => { + + try { + const state = readSecretState(); + return response.send(state); + } catch (error) { + console.error(error); + return response.send({}); + } + }); + + app.post('/viewsecrets', jsonParser, async (_, response) => { + const allowKeysExposure = getConfigValue('allowKeysExposure', false); + + if (!allowKeysExposure) { + console.error('secrets.json could not be viewed unless the value of allowKeysExposure in config.conf is set to true'); + return response.sendStatus(403); + } + + try { + const secrets = getAllSecrets(); + + if (!secrets) { + return response.sendStatus(404); + } + + return response.send(secrets); + } catch (error) { + console.error(error); + return response.sendStatus(500); + } + }); +} + module.exports = { writeSecret, readSecret, readSecretState, migrateSecrets, getAllSecrets, + registerEndpoints, SECRET_KEYS, };