From 0f105e030024397b417f18550cac2fdfd209c63d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 7 Apr 2024 18:11:23 +0300 Subject: [PATCH] Fix circular deps, add Helmet https://helmetjs.github.io/ --- package-lock.json | 9 +++++++ package.json | 1 + server.js | 9 +++++-- src/endpoints/content-manager.js | 45 +++++--------------------------- src/users.js | 30 ++++++++++++++++++--- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index d9aaf7959..b4e5d82c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "form-data": "^4.0.0", "google-translate-api-browser": "^3.0.1", "gpt3-tokenizer": "^1.1.5", + "helmet": "^7.1.0", "ip-matching": "^2.1.2", "ipaddr.js": "^2.0.1", "jimp": "^0.22.10", @@ -2176,6 +2177,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/helmet": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz", + "integrity": "sha512-g+HZqgfbpXdCkme/Cd/mZkV0aV3BZZZSugecH03kl38m/Kmdx8jKjBikpDj2cr+Iynv4KpYEviojNdTJActJAg==", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/htmlparser2": { "version": "8.0.2", "funding": [ diff --git a/package.json b/package.json index fcc4fb039..d358ef5d4 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "form-data": "^4.0.0", "google-translate-api-browser": "^3.0.1", "gpt3-tokenizer": "^1.1.5", + "helmet": "^7.1.0", "ip-matching": "^2.1.2", "ipaddr.js": "^2.0.1", "jimp": "^0.22.10", diff --git a/server.js b/server.js index 9e3497d03..c20bd30b0 100644 --- a/server.js +++ b/server.js @@ -20,6 +20,7 @@ const compression = require('compression'); const cookieParser = require('cookie-parser'); const multer = require('multer'); const responseTime = require('response-time'); +const helmet = require('helmet').default; // net related library imports const net = require('net'); @@ -34,6 +35,7 @@ util.inspect.defaultOptions.depth = 4; // local library imports const { initUserStorage, + ensurePublicDirectoriesExist, userDataMiddleware, migrateUserData, getCsrfSecret, @@ -109,6 +111,9 @@ const serverDirectory = __dirname; process.chdir(serverDirectory); const app = express(); +app.use(helmet({ + contentSecurityPolicy: false, +})); app.use(compression()); app.use(responseTime()); @@ -474,9 +479,9 @@ const setupTasks = async function () { // in any order for encapsulation reasons, but right now it's unknown if that would break anything. await initUserStorage(); await settingsEndpoint.init(); - await contentManager.ensurePublicDirectoriesExist(); + const directories = await ensurePublicDirectoriesExist(); await migrateUserData(); - contentManager.checkForNewContent(); + await contentManager.checkForNewContent(directories); await ensureThumbnailCache(); cleanUploads(); diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 42aa45576..905d8f1bc 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -7,9 +7,7 @@ const { getConfigValue } = require('../util'); const { jsonParser } = require('../express-common'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { getAllUserHandles, getUserDirectories } = require('../users'); const characterCardParser = require('../character-card-parser.js'); -const { PUBLIC_DIRECTORIES } = require('../constants'); /** * @typedef {Object} ContentItem @@ -17,28 +15,6 @@ const { PUBLIC_DIRECTORIES } = require('../constants'); * @property {string} type */ -/** - * Ensures that the content directories exist. - * @returns {Promise} - */ -async function ensurePublicDirectoriesExist() { - for (const dir of Object.values(PUBLIC_DIRECTORIES)) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - } - - const userHandles = await getAllUserHandles(); - for (const handle of userHandles) { - const userDirectories = getUserDirectories(handle); - for (const dir of Object.values(userDirectories)) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - } - } -} - /** * Gets the default presets from the content directory. * @param {import('../users').UserDirectoryList} directories User directories @@ -90,11 +66,9 @@ function getDefaultPresetFile(filename) { /** * Seeds content for a user. * @param {ContentItem[]} contentIndex Content index - * @param {string} userHandle User handle + * @param {import('../users').UserDirectoryList} directories User directories */ -async function seedContentForUser(contentIndex, userHandle) { - const directories = getUserDirectories(userHandle); - +async function seedContentForUser(contentIndex, directories) { if (!fs.existsSync(directories.root)) { fs.mkdirSync(directories.root, { recursive: true }); } @@ -140,10 +114,10 @@ async function seedContentForUser(contentIndex, userHandle) { /** * Checks for new content and seeds it for all users. - * @param {string} [userHandle] User to check the content for (optional) + * @param {import('../users').UserDirectoryList[]} directoriesList List of user directories * @returns {Promise} */ -async function checkForNewContent(userHandle) { +async function checkForNewContent(directoriesList) { try { if (getConfigValue('skipContentCheck', false)) { return; @@ -151,15 +125,9 @@ async function checkForNewContent(userHandle) { const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); const contentIndex = JSON.parse(contentIndexText); - const userHandles = await getAllUserHandles(); - if (userHandle && userHandles.includes(userHandle)) { - await seedContentForUser(contentIndex, userHandle); - return; - } - - for (const userHandle of userHandles) { - await seedContentForUser(contentIndex, userHandle); + for (const directories of directoriesList) { + await seedContentForUser(contentIndex, directories); } } catch (err) { console.log('Content check failed', err); @@ -509,7 +477,6 @@ router.post('/importUUID', jsonParser, async (request, response) => { }); module.exports = { - ensurePublicDirectoriesExist, checkForNewContent, getDefaultPresets, getDefaultPresetFile, diff --git a/src/users.js b/src/users.js index 47b366575..41f69c0f4 100644 --- a/src/users.js +++ b/src/users.js @@ -10,7 +10,7 @@ const { getConfigValue, color, delay, setConfigValue, Cache } = require('./util' const express = require('express'); const { readSecret, writeSecret } = require('./endpoints/secrets'); const { jsonParser } = require('./express-common'); -const contentManager = require('./endpoints/content-manager'); +const { checkForNewContent } = require('./endpoints/content-manager'); const DATA_ROOT = getConfigValue('dataRoot', './data'); const MFA_CACHE = new Cache(5 * 60 * 1000); @@ -73,6 +73,29 @@ const STORAGE_KEYS = { * @property {string} vectors - The directory where the vectors are stored */ +/** + * Ensures that the content directories exist. + * @returns {Promise} - The list of user directories + */ +async function ensurePublicDirectoriesExist() { + for (const dir of Object.values(PUBLIC_DIRECTORIES)) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + + const userHandles = await getAllUserHandles(); + const directoriesList = userHandles.map(handle => getUserDirectories(handle)); + for (const userDirectories of directoriesList) { + for (const dir of Object.values(userDirectories)) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + } + return directoriesList; +} + /** * Perform migration from the old user data format to the new one. */ @@ -607,8 +630,8 @@ endpoints.post('/create', requireAdminMiddleware, jsonParser, async (request, re // Create user directories console.log('Creating data directories for', newUser.handle); - await contentManager.ensurePublicDirectoriesExist(); - await contentManager.checkForNewContent(newUser.handle); + const directories = await ensurePublicDirectoriesExist(); + await checkForNewContent(directories); return response.json({ handle: newUser.handle }); }); @@ -616,6 +639,7 @@ router.use('/api/users', endpoints); module.exports = { initUserStorage, + ensurePublicDirectoriesExist, getCurrentUserDirectories, getCurrentUserHandle, getAllUserHandles,