Fix circular deps, add Helmet https://helmetjs.github.io/

This commit is contained in:
Cohee 2024-04-07 18:11:23 +03:00
parent c6ffe4502a
commit 0f105e0300
5 changed files with 50 additions and 44 deletions

9
package-lock.json generated
View File

@ -25,6 +25,7 @@
"form-data": "^4.0.0",
"google-translate-api-browser": "^3.0.1",
"gpt3-tokenizer": "^1.1.5",
"helmet": "^7.1.0",
"ip-matching": "^2.1.2",
"ipaddr.js": "^2.0.1",
"jimp": "^0.22.10",
@ -2176,6 +2177,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/helmet": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",
"integrity": "sha512-g+HZqgfbpXdCkme/Cd/mZkV0aV3BZZZSugecH03kl38m/Kmdx8jKjBikpDj2cr+Iynv4KpYEviojNdTJActJAg==",
"engines": {
"node": ">=16.0.0"
}
},
"node_modules/htmlparser2": {
"version": "8.0.2",
"funding": [

View File

@ -15,6 +15,7 @@
"form-data": "^4.0.0",
"google-translate-api-browser": "^3.0.1",
"gpt3-tokenizer": "^1.1.5",
"helmet": "^7.1.0",
"ip-matching": "^2.1.2",
"ipaddr.js": "^2.0.1",
"jimp": "^0.22.10",

View File

@ -20,6 +20,7 @@ const compression = require('compression');
const cookieParser = require('cookie-parser');
const multer = require('multer');
const responseTime = require('response-time');
const helmet = require('helmet').default;
// net related library imports
const net = require('net');
@ -34,6 +35,7 @@ util.inspect.defaultOptions.depth = 4;
// local library imports
const {
initUserStorage,
ensurePublicDirectoriesExist,
userDataMiddleware,
migrateUserData,
getCsrfSecret,
@ -109,6 +111,9 @@ const serverDirectory = __dirname;
process.chdir(serverDirectory);
const app = express();
app.use(helmet({
contentSecurityPolicy: false,
}));
app.use(compression());
app.use(responseTime());
@ -474,9 +479,9 @@ const setupTasks = async function () {
// in any order for encapsulation reasons, but right now it's unknown if that would break anything.
await initUserStorage();
await settingsEndpoint.init();
await contentManager.ensurePublicDirectoriesExist();
const directories = await ensurePublicDirectoriesExist();
await migrateUserData();
contentManager.checkForNewContent();
await contentManager.checkForNewContent(directories);
await ensureThumbnailCache();
cleanUploads();

View File

@ -7,9 +7,7 @@ const { getConfigValue } = require('../util');
const { jsonParser } = require('../express-common');
const contentDirectory = path.join(process.cwd(), 'default/content');
const contentIndexPath = path.join(contentDirectory, 'index.json');
const { getAllUserHandles, getUserDirectories } = require('../users');
const characterCardParser = require('../character-card-parser.js');
const { PUBLIC_DIRECTORIES } = require('../constants');
/**
* @typedef {Object} ContentItem
@ -17,28 +15,6 @@ const { PUBLIC_DIRECTORIES } = require('../constants');
* @property {string} type
*/
/**
* Ensures that the content directories exist.
* @returns {Promise<void>}
*/
async function ensurePublicDirectoriesExist() {
for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
const userHandles = await getAllUserHandles();
for (const handle of userHandles) {
const userDirectories = getUserDirectories(handle);
for (const dir of Object.values(userDirectories)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
}
}
/**
* Gets the default presets from the content directory.
* @param {import('../users').UserDirectoryList} directories User directories
@ -90,11 +66,9 @@ function getDefaultPresetFile(filename) {
/**
* Seeds content for a user.
* @param {ContentItem[]} contentIndex Content index
* @param {string} userHandle User handle
* @param {import('../users').UserDirectoryList} directories User directories
*/
async function seedContentForUser(contentIndex, userHandle) {
const directories = getUserDirectories(userHandle);
async function seedContentForUser(contentIndex, directories) {
if (!fs.existsSync(directories.root)) {
fs.mkdirSync(directories.root, { recursive: true });
}
@ -140,10 +114,10 @@ async function seedContentForUser(contentIndex, userHandle) {
/**
* Checks for new content and seeds it for all users.
* @param {string} [userHandle] User to check the content for (optional)
* @param {import('../users').UserDirectoryList[]} directoriesList List of user directories
* @returns {Promise<void>}
*/
async function checkForNewContent(userHandle) {
async function checkForNewContent(directoriesList) {
try {
if (getConfigValue('skipContentCheck', false)) {
return;
@ -151,15 +125,9 @@ async function checkForNewContent(userHandle) {
const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8');
const contentIndex = JSON.parse(contentIndexText);
const userHandles = await getAllUserHandles();
if (userHandle && userHandles.includes(userHandle)) {
await seedContentForUser(contentIndex, userHandle);
return;
}
for (const userHandle of userHandles) {
await seedContentForUser(contentIndex, userHandle);
for (const directories of directoriesList) {
await seedContentForUser(contentIndex, directories);
}
} catch (err) {
console.log('Content check failed', err);
@ -509,7 +477,6 @@ router.post('/importUUID', jsonParser, async (request, response) => {
});
module.exports = {
ensurePublicDirectoriesExist,
checkForNewContent,
getDefaultPresets,
getDefaultPresetFile,

View File

@ -10,7 +10,7 @@ const { getConfigValue, color, delay, setConfigValue, Cache } = require('./util'
const express = require('express');
const { readSecret, writeSecret } = require('./endpoints/secrets');
const { jsonParser } = require('./express-common');
const contentManager = require('./endpoints/content-manager');
const { checkForNewContent } = require('./endpoints/content-manager');
const DATA_ROOT = getConfigValue('dataRoot', './data');
const MFA_CACHE = new Cache(5 * 60 * 1000);
@ -73,6 +73,29 @@ const STORAGE_KEYS = {
* @property {string} vectors - The directory where the vectors are stored
*/
/**
* Ensures that the content directories exist.
* @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
*/
async function ensurePublicDirectoriesExist() {
for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
const userHandles = await getAllUserHandles();
const directoriesList = userHandles.map(handle => getUserDirectories(handle));
for (const userDirectories of directoriesList) {
for (const dir of Object.values(userDirectories)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
}
return directoriesList;
}
/**
* Perform migration from the old user data format to the new one.
*/
@ -607,8 +630,8 @@ endpoints.post('/create', requireAdminMiddleware, jsonParser, async (request, re
// Create user directories
console.log('Creating data directories for', newUser.handle);
await contentManager.ensurePublicDirectoriesExist();
await contentManager.checkForNewContent(newUser.handle);
const directories = await ensurePublicDirectoriesExist();
await checkForNewContent(directories);
return response.json({ handle: newUser.handle });
});
@ -616,6 +639,7 @@ router.use('/api/users', endpoints);
module.exports = {
initUserStorage,
ensurePublicDirectoriesExist,
getCurrentUserDirectories,
getCurrentUserHandle,
getAllUserHandles,