Fix circular deps, add Helmet https://helmetjs.github.io/

package-lock.json

@@ -25,6 +25,7 @@
                 "form-data": "^4.0.0",
                 "google-translate-api-browser": "^3.0.1",
                 "gpt3-tokenizer": "^1.1.5",
+                "helmet": "^7.1.0",
                 "ip-matching": "^2.1.2",
                 "ipaddr.js": "^2.0.1",
                 "jimp": "^0.22.10",
@@ -2176,6 +2177,14 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/helmet": {
+            "version": "7.1.0",
+            "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",
+            "integrity": "sha512-g+HZqgfbpXdCkme/Cd/mZkV0aV3BZZZSugecH03kl38m/Kmdx8jKjBikpDj2cr+Iynv4KpYEviojNdTJActJAg==",
+            "engines": {
+                "node": ">=16.0.0"
+            }
+        },
         "node_modules/htmlparser2": {
             "version": "8.0.2",
             "funding": [

package.json

@@ -15,6 +15,7 @@
         "form-data": "^4.0.0",
         "google-translate-api-browser": "^3.0.1",
         "gpt3-tokenizer": "^1.1.5",
+        "helmet": "^7.1.0",
         "ip-matching": "^2.1.2",
         "ipaddr.js": "^2.0.1",
         "jimp": "^0.22.10",

server.js

@@ -20,6 +20,7 @@ const compression = require('compression');
 const cookieParser = require('cookie-parser');
 const multer = require('multer');
 const responseTime = require('response-time');
+const helmet = require('helmet').default;
 
 // net related library imports
 const net = require('net');
@@ -34,6 +35,7 @@ util.inspect.defaultOptions.depth = 4;
 // local library imports
 const {
     initUserStorage,
+    ensurePublicDirectoriesExist,
     userDataMiddleware,
     migrateUserData,
     getCsrfSecret,
@@ -109,6 +111,9 @@ const serverDirectory = __dirname;
 process.chdir(serverDirectory);
 
 const app = express();
+app.use(helmet({
+    contentSecurityPolicy: false,
+}));
 app.use(compression());
 app.use(responseTime());
 
@@ -474,9 +479,9 @@ const setupTasks = async function () {
     // in any order for encapsulation reasons, but right now it's unknown if that would break anything.
     await initUserStorage();
     await settingsEndpoint.init();
-    await contentManager.ensurePublicDirectoriesExist();
+    const directories = await ensurePublicDirectoriesExist();
     await migrateUserData();
-    contentManager.checkForNewContent();
+    await contentManager.checkForNewContent(directories);
     await ensureThumbnailCache();
     cleanUploads();
 

src/endpoints/content-manager.js

@@ -7,9 +7,7 @@ const { getConfigValue } = require('../util');
 const { jsonParser } = require('../express-common');
 const contentDirectory = path.join(process.cwd(), 'default/content');
 const contentIndexPath = path.join(contentDirectory, 'index.json');
-const { getAllUserHandles, getUserDirectories } = require('../users');
 const characterCardParser = require('../character-card-parser.js');
-const { PUBLIC_DIRECTORIES } = require('../constants');
 
 /**
  * @typedef {Object} ContentItem
@@ -17,28 +15,6 @@ const { PUBLIC_DIRECTORIES } = require('../constants');
  * @property {string} type
  */
 
-/**
- * Ensures that the content directories exist.
- * @returns {Promise<void>}
- */
-async function ensurePublicDirectoriesExist() {
-    for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
-        if (!fs.existsSync(dir)) {
-            fs.mkdirSync(dir, { recursive: true });
-        }
-    }
-
-    const userHandles = await getAllUserHandles();
-    for (const handle of userHandles) {
-        const userDirectories = getUserDirectories(handle);
-        for (const dir of Object.values(userDirectories)) {
-            if (!fs.existsSync(dir)) {
-                fs.mkdirSync(dir, { recursive: true });
-            }
-        }
-    }
-}
-
 /**
  * Gets the default presets from the content directory.
  * @param {import('../users').UserDirectoryList} directories User directories
@@ -90,11 +66,9 @@ function getDefaultPresetFile(filename) {
 /**
  * Seeds content for a user.
  * @param {ContentItem[]} contentIndex Content index
- * @param {string} userHandle User handle
+ * @param {import('../users').UserDirectoryList} directories User directories
  */
-async function seedContentForUser(contentIndex, userHandle) {
+async function seedContentForUser(contentIndex, directories) {
-    const directories = getUserDirectories(userHandle);
-
     if (!fs.existsSync(directories.root)) {
         fs.mkdirSync(directories.root, { recursive: true });
     }
@@ -140,10 +114,10 @@ async function seedContentForUser(contentIndex, userHandle) {
 
 /**
  * Checks for new content and seeds it for all users.
- * @param {string} [userHandle] User to check the content for (optional)
+ * @param {import('../users').UserDirectoryList[]} directoriesList List of user directories
  * @returns {Promise<void>}
  */
-async function checkForNewContent(userHandle) {
+async function checkForNewContent(directoriesList) {
     try {
         if (getConfigValue('skipContentCheck', false)) {
             return;
@@ -151,15 +125,9 @@ async function checkForNewContent(userHandle) {
 
         const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8');
         const contentIndex = JSON.parse(contentIndexText);
-        const userHandles = await getAllUserHandles();
 
-        if (userHandle && userHandles.includes(userHandle)) {
+        for (const directories of directoriesList) {
-            await seedContentForUser(contentIndex, userHandle);
+            await seedContentForUser(contentIndex, directories);
-            return;
-        }
-
-        for (const userHandle of userHandles) {
-            await seedContentForUser(contentIndex, userHandle);
         }
     } catch (err) {
         console.log('Content check failed', err);
@@ -509,7 +477,6 @@ router.post('/importUUID', jsonParser, async (request, response) => {
 });
 
 module.exports = {
-    ensurePublicDirectoriesExist,
     checkForNewContent,
     getDefaultPresets,
     getDefaultPresetFile,

src/users.js

@@ -10,7 +10,7 @@ const { getConfigValue, color, delay, setConfigValue, Cache } = require('./util'
 const express = require('express');
 const { readSecret, writeSecret } = require('./endpoints/secrets');
 const { jsonParser } = require('./express-common');
-const contentManager = require('./endpoints/content-manager');
+const { checkForNewContent } = require('./endpoints/content-manager');
 
 const DATA_ROOT = getConfigValue('dataRoot', './data');
 const MFA_CACHE = new Cache(5 * 60 * 1000);
@@ -73,6 +73,29 @@ const STORAGE_KEYS = {
  * @property {string} vectors - The directory where the vectors are stored
  */
 
+/**
+ * Ensures that the content directories exist.
+ * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
+ */
+async function ensurePublicDirectoriesExist() {
+    for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+    }
+
+    const userHandles = await getAllUserHandles();
+    const directoriesList = userHandles.map(handle => getUserDirectories(handle));
+    for (const userDirectories of directoriesList) {
+        for (const dir of Object.values(userDirectories)) {
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+        }
+    }
+    return directoriesList;
+}
+
 /**
  * Perform migration from the old user data format to the new one.
  */
@@ -607,8 +630,8 @@ endpoints.post('/create', requireAdminMiddleware, jsonParser, async (request, re
 
     // Create user directories
     console.log('Creating data directories for', newUser.handle);
-    await contentManager.ensurePublicDirectoriesExist();
+    const directories = await ensurePublicDirectoriesExist();
-    await contentManager.checkForNewContent(newUser.handle);
+    await checkForNewContent(directories);
     return response.json({ handle: newUser.handle });
 });
 
@@ -616,6 +639,7 @@ router.use('/api/users', endpoints);
 
 module.exports = {
     initUserStorage,
+    ensurePublicDirectoriesExist,
     getCurrentUserDirectories,
     getCurrentUserHandle,
     getAllUserHandles,