From 5b002c6e46dcc638d2c2b86d394952d3dfc9da04 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 26 Jun 2024 23:22:42 +0300 Subject: [PATCH] #2422 Move uploads under the data root --- .dockerignore | 2 ++ .npmignore | 2 ++ server.js | 10 +++++----- src/constants.js | 7 +++++-- src/endpoints/avatars.js | 4 ++-- src/endpoints/backgrounds.js | 3 +-- src/endpoints/characters.js | 8 ++++---- src/endpoints/chats.js | 7 +++---- src/endpoints/sprites.js | 5 ++--- src/endpoints/worldinfo.js | 3 +-- 10 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.dockerignore b/.dockerignore index e4995fe58..9f6dc7953 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,5 @@ Start.bat cloudflared.exe access.log /data +/cache +.DS_Store diff --git a/.npmignore b/.npmignore index 10082773f..ccee41fb6 100644 --- a/.npmignore +++ b/.npmignore @@ -5,4 +5,6 @@ node_modules/ secrets.json /dist /backups/ +/data +/cache access.log diff --git a/server.js b/server.js index ab0460ac0..9f1bd81b4 100644 --- a/server.js +++ b/server.js @@ -136,7 +136,7 @@ const disableCsrf = cliArguments.disableCsrf ?? getConfigValue('disableCsrfProte const basicAuthMode = cliArguments.basicAuthMode ?? getConfigValue('basicAuthMode', DEFAULT_BASIC_AUTH); const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS); -const { UPLOADS_PATH } = require('./src/constants'); +const uploadsPath = path.join(dataRoot, require('./src/constants').UPLOADS_DIRECTORY); // CORS Settings // const CORS = cors({ @@ -286,7 +286,7 @@ app.use(userModule.requireLoginMiddleware); app.get('/api/ping', (_, response) => response.sendStatus(204)); // File uploads -app.use(multer({ dest: UPLOADS_PATH, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar')); +app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar')); app.use(require('./src/middleware/multerMonkeyPatch')); // User data mount @@ -303,8 +303,8 @@ app.get('/version', async function (_, response) { function cleanUploads() { try { - if (fs.existsSync(UPLOADS_PATH)) { - const uploads = fs.readdirSync(UPLOADS_PATH); + if (fs.existsSync(uploadsPath)) { + const uploads = fs.readdirSync(uploadsPath); if (!uploads.length) { return; @@ -312,7 +312,7 @@ function cleanUploads() { console.debug(`Cleaning uploads folder (${uploads.length} files)`); uploads.forEach(file => { - const pathToFile = path.join(UPLOADS_PATH, file); + const pathToFile = path.join(uploadsPath, file); fs.unlinkSync(pathToFile); }); } diff --git a/src/constants.js b/src/constants.js index bb992530b..89af4f739 100644 --- a/src/constants.js +++ b/src/constants.js @@ -196,7 +196,10 @@ const CHAT_COMPLETION_SOURCES = { GROQ: 'groq', }; -const UPLOADS_PATH = './uploads'; +/** + * Path to multer file uploads under the data root. + */ +const UPLOADS_DIRECTORY = '_uploads'; // TODO: this is copied from the client code; there should be a way to de-duplicate it eventually const TEXTGEN_TYPES = { @@ -364,7 +367,7 @@ module.exports = { PUBLIC_DIRECTORIES, USER_DIRECTORY_TEMPLATE, UNSAFE_EXTENSIONS, - UPLOADS_PATH, + UPLOADS_DIRECTORY, GEMINI_SAFETY, BISON_SAFETY, TEXTGEN_TYPES, diff --git a/src/endpoints/avatars.js b/src/endpoints/avatars.js index 58571bae9..5f8509f0f 100644 --- a/src/endpoints/avatars.js +++ b/src/endpoints/avatars.js @@ -4,7 +4,7 @@ const fs = require('fs'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser, urlencodedParser } = require('../express-common'); -const { AVATAR_WIDTH, AVATAR_HEIGHT, UPLOADS_PATH } = require('../constants'); +const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); const { getImages, tryParse } = require('../util'); // image processing related library imports @@ -39,7 +39,7 @@ router.post('/upload', urlencodedParser, async (request, response) => { if (!request.file) return response.sendStatus(400); try { - const pathToUpload = path.join(UPLOADS_PATH, request.file.filename); + const pathToUpload = path.join(request.file.destination, request.file.filename); const crop = tryParse(request.query.crop); let rawImg = await jimp.read(pathToUpload); diff --git a/src/endpoints/backgrounds.js b/src/endpoints/backgrounds.js index b8965ab5f..6ef59f923 100644 --- a/src/endpoints/backgrounds.js +++ b/src/endpoints/backgrounds.js @@ -4,7 +4,6 @@ const express = require('express'); const sanitize = require('sanitize-filename'); const { jsonParser, urlencodedParser } = require('../express-common'); -const { UPLOADS_PATH } = require('../constants'); const { invalidateThumbnail } = require('./thumbnails'); const { getImages } = require('../util'); @@ -60,7 +59,7 @@ router.post('/rename', jsonParser, function (request, response) { router.post('/upload', urlencodedParser, function (request, response) { if (!request.body || !request.file) return response.sendStatus(400); - const img_path = path.join(UPLOADS_PATH, request.file.filename); + const img_path = path.join(request.file.destination, request.file.filename); const filename = request.file.originalname; try { diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 17dcaa6b8..5b6246f52 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -11,7 +11,7 @@ const mime = require('mime-types'); const jimp = require('jimp'); -const { UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); +const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); const { jsonParser, urlencodedParser } = require('../express-common'); const { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } = require('../util'); const { TavernCardValidator } = require('../validator/TavernCardValidator'); @@ -729,7 +729,7 @@ router.post('/create', urlencodedParser, async function (request, response) { return response.send(avatarName); } else { const crop = tryParse(request.query.crop); - const uploadPath = path.join(UPLOADS_PATH, request.file.filename); + const uploadPath = path.join(request.file.destination, request.file.filename); await writeCharacterData(uploadPath, char, internalName, request, crop); fs.unlinkSync(uploadPath); return response.send(avatarName); @@ -812,7 +812,7 @@ router.post('/edit', urlencodedParser, async function (request, response) { await writeCharacterData(avatarPath, char, targetFile, request); } else { const crop = tryParse(request.query.crop); - const newAvatarPath = path.join(UPLOADS_PATH, request.file.filename); + const newAvatarPath = path.join(request.file.destination, request.file.filename); invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url); await writeCharacterData(newAvatarPath, char, targetFile, request, crop); fs.unlinkSync(newAvatarPath); @@ -1096,7 +1096,7 @@ function getPreservedName(request) { router.post('/import', urlencodedParser, async function (request, response) { if (!request.body || !request.file) return response.sendStatus(400); - const uploadPath = path.join(UPLOADS_PATH, request.file.filename); + const uploadPath = path.join(request.file.destination, request.file.filename); const format = request.body.file_type; const preservedFileName = getPreservedName(request); diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js index d88b27fa3..b6f675c04 100644 --- a/src/endpoints/chats.js +++ b/src/endpoints/chats.js @@ -6,7 +6,6 @@ const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser, urlencodedParser } = require('../express-common'); -const { UPLOADS_PATH } = require('../constants'); const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util'); /** @@ -323,7 +322,7 @@ router.post('/group/import', urlencodedParser, function (request, response) { } const chatname = humanizedISO8601DateTime(); - const pathToUpload = path.join(UPLOADS_PATH, filedata.filename); + const pathToUpload = path.join(filedata.destination, filedata.filename); const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`); fs.copyFileSync(pathToUpload, pathToNewFile); fs.unlinkSync(pathToUpload); @@ -347,7 +346,7 @@ router.post('/import', urlencodedParser, function (request, response) { } try { - const data = fs.readFileSync(path.join(UPLOADS_PATH, request.file.filename), 'utf8'); + const data = fs.readFileSync(path.join(request.file.destination, request.file.filename), 'utf8'); if (format === 'json') { const jsonData = JSON.parse(data); @@ -388,7 +387,7 @@ router.post('/import', urlencodedParser, function (request, response) { if (jsonData.user_name !== undefined || jsonData.name !== undefined) { const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`; const filePath = path.join(request.user.directories.chats, avatarUrl, fileName); - fs.copyFileSync(path.join(UPLOADS_PATH, request.file.filename), filePath); + fs.copyFileSync(path.join(request.file.destination, request.file.filename), filePath); response.send({ res: true }); } else { console.log('Incorrect chat format .jsonl'); diff --git a/src/endpoints/sprites.js b/src/endpoints/sprites.js index 88a577b3b..43bfdb145 100644 --- a/src/endpoints/sprites.js +++ b/src/endpoints/sprites.js @@ -5,7 +5,6 @@ const express = require('express'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { UPLOADS_PATH } = require('../constants'); const { getImageBuffers } = require('../util'); const { jsonParser, urlencodedParser } = require('../express-common'); @@ -190,7 +189,7 @@ router.post('/upload-zip', urlencodedParser, async (request, response) => { return response.sendStatus(404); } - const spritePackPath = path.join(UPLOADS_PATH, file.filename); + const spritePackPath = path.join(file.destination, file.filename); const sprites = await getImageBuffers(spritePackPath); const files = fs.readdirSync(spritesPath); @@ -248,7 +247,7 @@ router.post('/upload', urlencodedParser, async (request, response) => { } const filename = label + path.parse(file.originalname).ext; - const spritePath = path.join(UPLOADS_PATH, file.filename); + const spritePath = path.join(file.destination, file.filename); const pathToFile = path.join(spritesPath, filename); // Copy uploaded file to sprites folder fs.cpSync(spritePath, pathToFile); diff --git a/src/endpoints/worldinfo.js b/src/endpoints/worldinfo.js index f8ab2d498..4f125bcc4 100644 --- a/src/endpoints/worldinfo.js +++ b/src/endpoints/worldinfo.js @@ -5,7 +5,6 @@ const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser, urlencodedParser } = require('../express-common'); -const { UPLOADS_PATH } = require('../constants'); /** * Reads a World Info file and returns its contents @@ -74,7 +73,7 @@ router.post('/import', urlencodedParser, (request, response) => { if (request.body.convertedData) { fileContents = request.body.convertedData; } else { - const pathToUpload = path.join(UPLOADS_PATH, request.file.filename); + const pathToUpload = path.join(request.file.destination, request.file.filename); fileContents = fs.readFileSync(pathToUpload, 'utf8'); fs.unlinkSync(pathToUpload); }