#2422 Move uploads under the data root

.dockerignore

@@ -8,3 +8,5 @@ Start.bat
 cloudflared.exe
 access.log
 /data
+/cache
+.DS_Store

.npmignore

@@ -5,4 +5,6 @@ node_modules/
 secrets.json
 /dist
 /backups/
+/data
+/cache
 access.log

server.js

@@ -136,7 +136,7 @@ const disableCsrf = cliArguments.disableCsrf ?? getConfigValue('disableCsrfProte
 const basicAuthMode = cliArguments.basicAuthMode ?? getConfigValue('basicAuthMode', DEFAULT_BASIC_AUTH);
 const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS);
 
-const { UPLOADS_PATH } = require('./src/constants');
+const uploadsPath = path.join(dataRoot, require('./src/constants').UPLOADS_DIRECTORY);
 
 // CORS Settings //
 const CORS = cors({
@@ -286,7 +286,7 @@ app.use(userModule.requireLoginMiddleware);
 app.get('/api/ping', (_, response) => response.sendStatus(204));
 
 // File uploads
-app.use(multer({ dest: UPLOADS_PATH, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));
+app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));
 app.use(require('./src/middleware/multerMonkeyPatch'));
 
 // User data mount
@@ -303,8 +303,8 @@ app.get('/version', async function (_, response) {
 
 function cleanUploads() {
     try {
-        if (fs.existsSync(UPLOADS_PATH)) {
+        if (fs.existsSync(uploadsPath)) {
-            const uploads = fs.readdirSync(UPLOADS_PATH);
+            const uploads = fs.readdirSync(uploadsPath);
 
             if (!uploads.length) {
                 return;
@@ -312,7 +312,7 @@ function cleanUploads() {
 
             console.debug(`Cleaning uploads folder (${uploads.length} files)`);
             uploads.forEach(file => {
-                const pathToFile = path.join(UPLOADS_PATH, file);
+                const pathToFile = path.join(uploadsPath, file);
                 fs.unlinkSync(pathToFile);
             });
         }

src/constants.js

@@ -196,7 +196,10 @@ const CHAT_COMPLETION_SOURCES = {
     GROQ: 'groq',
 };
 
-const UPLOADS_PATH = './uploads';
+/**
+ * Path to multer file uploads under the data root.
+ */
+const UPLOADS_DIRECTORY = '_uploads';
 
 // TODO: this is copied from the client code; there should be a way to de-duplicate it eventually
 const TEXTGEN_TYPES = {
@@ -364,7 +367,7 @@ module.exports = {
     PUBLIC_DIRECTORIES,
     USER_DIRECTORY_TEMPLATE,
     UNSAFE_EXTENSIONS,
-    UPLOADS_PATH,
+    UPLOADS_DIRECTORY,
     GEMINI_SAFETY,
     BISON_SAFETY,
     TEXTGEN_TYPES,

src/endpoints/avatars.js

@@ -4,7 +4,7 @@ const fs = require('fs');
 const sanitize = require('sanitize-filename');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 const { jsonParser, urlencodedParser } = require('../express-common');
-const { AVATAR_WIDTH, AVATAR_HEIGHT, UPLOADS_PATH } = require('../constants');
+const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
 const { getImages, tryParse } = require('../util');
 
 // image processing related library imports
@@ -39,7 +39,7 @@ router.post('/upload', urlencodedParser, async (request, response) => {
     if (!request.file) return response.sendStatus(400);
 
     try {
-        const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
+        const pathToUpload = path.join(request.file.destination, request.file.filename);
         const crop = tryParse(request.query.crop);
         let rawImg = await jimp.read(pathToUpload);
 

src/endpoints/backgrounds.js

@@ -4,7 +4,6 @@ const express = require('express');
 const sanitize = require('sanitize-filename');
 
 const { jsonParser, urlencodedParser } = require('../express-common');
-const { UPLOADS_PATH } = require('../constants');
 const { invalidateThumbnail } = require('./thumbnails');
 const { getImages } = require('../util');
 
@@ -60,7 +59,7 @@ router.post('/rename', jsonParser, function (request, response) {
 router.post('/upload', urlencodedParser, function (request, response) {
     if (!request.body || !request.file) return response.sendStatus(400);
 
-    const img_path = path.join(UPLOADS_PATH, request.file.filename);
+    const img_path = path.join(request.file.destination, request.file.filename);
     const filename = request.file.originalname;
 
     try {

src/endpoints/characters.js

@@ -11,7 +11,7 @@ const mime = require('mime-types');
 
 const jimp = require('jimp');
 
-const { UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
+const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
 const { jsonParser, urlencodedParser } = require('../express-common');
 const { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } = require('../util');
 const { TavernCardValidator } = require('../validator/TavernCardValidator');
@@ -729,7 +729,7 @@ router.post('/create', urlencodedParser, async function (request, response) {
             return response.send(avatarName);
         } else {
             const crop = tryParse(request.query.crop);
-            const uploadPath = path.join(UPLOADS_PATH, request.file.filename);
+            const uploadPath = path.join(request.file.destination, request.file.filename);
             await writeCharacterData(uploadPath, char, internalName, request, crop);
             fs.unlinkSync(uploadPath);
             return response.send(avatarName);
@@ -812,7 +812,7 @@ router.post('/edit', urlencodedParser, async function (request, response) {
             await writeCharacterData(avatarPath, char, targetFile, request);
         } else {
             const crop = tryParse(request.query.crop);
-            const newAvatarPath = path.join(UPLOADS_PATH, request.file.filename);
+            const newAvatarPath = path.join(request.file.destination, request.file.filename);
             invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);
             await writeCharacterData(newAvatarPath, char, targetFile, request, crop);
             fs.unlinkSync(newAvatarPath);
@@ -1096,7 +1096,7 @@ function getPreservedName(request) {
 router.post('/import', urlencodedParser, async function (request, response) {
     if (!request.body || !request.file) return response.sendStatus(400);
 
-    const uploadPath = path.join(UPLOADS_PATH, request.file.filename);
+    const uploadPath = path.join(request.file.destination, request.file.filename);
     const format = request.body.file_type;
     const preservedFileName = getPreservedName(request);
 

src/endpoints/chats.js

@@ -6,7 +6,6 @@ const sanitize = require('sanitize-filename');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 
 const { jsonParser, urlencodedParser } = require('../express-common');
-const { UPLOADS_PATH } = require('../constants');
 const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util');
 
 /**
@@ -323,7 +322,7 @@ router.post('/group/import', urlencodedParser, function (request, response) {
         }
 
         const chatname = humanizedISO8601DateTime();
-        const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
+        const pathToUpload = path.join(filedata.destination, filedata.filename);
         const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`);
         fs.copyFileSync(pathToUpload, pathToNewFile);
         fs.unlinkSync(pathToUpload);
@@ -347,7 +346,7 @@ router.post('/import', urlencodedParser, function (request, response) {
     }
 
     try {
-        const data = fs.readFileSync(path.join(UPLOADS_PATH, request.file.filename), 'utf8');
+        const data = fs.readFileSync(path.join(request.file.destination, request.file.filename), 'utf8');
 
         if (format === 'json') {
             const jsonData = JSON.parse(data);
@@ -388,7 +387,7 @@ router.post('/import', urlencodedParser, function (request, response) {
             if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
                 const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
                 const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
-                fs.copyFileSync(path.join(UPLOADS_PATH, request.file.filename), filePath);
+                fs.copyFileSync(path.join(request.file.destination, request.file.filename), filePath);
                 response.send({ res: true });
             } else {
                 console.log('Incorrect chat format .jsonl');

src/endpoints/sprites.js

@@ -5,7 +5,6 @@ const express = require('express');
 const mime = require('mime-types');
 const sanitize = require('sanitize-filename');
 const writeFileAtomicSync = require('write-file-atomic').sync;
-const { UPLOADS_PATH } = require('../constants');
 const { getImageBuffers } = require('../util');
 const { jsonParser, urlencodedParser } = require('../express-common');
 
@@ -190,7 +189,7 @@ router.post('/upload-zip', urlencodedParser, async (request, response) => {
             return response.sendStatus(404);
         }
 
-        const spritePackPath = path.join(UPLOADS_PATH, file.filename);
+        const spritePackPath = path.join(file.destination, file.filename);
         const sprites = await getImageBuffers(spritePackPath);
         const files = fs.readdirSync(spritesPath);
 
@@ -248,7 +247,7 @@ router.post('/upload', urlencodedParser, async (request, response) => {
         }
 
         const filename = label + path.parse(file.originalname).ext;
-        const spritePath = path.join(UPLOADS_PATH, file.filename);
+        const spritePath = path.join(file.destination, file.filename);
         const pathToFile = path.join(spritesPath, filename);
         // Copy uploaded file to sprites folder
         fs.cpSync(spritePath, pathToFile);

src/endpoints/worldinfo.js

@@ -5,7 +5,6 @@ const sanitize = require('sanitize-filename');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 
 const { jsonParser, urlencodedParser } = require('../express-common');
-const { UPLOADS_PATH } = require('../constants');
 
 /**
  * Reads a World Info file and returns its contents
@@ -74,7 +73,7 @@ router.post('/import', urlencodedParser, (request, response) => {
     if (request.body.convertedData) {
         fileContents = request.body.convertedData;
     } else {
-        const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
+        const pathToUpload = path.join(request.file.destination, request.file.filename);
         fileContents = fs.readFileSync(pathToUpload, 'utf8');
         fs.unlinkSync(pathToUpload);
     }