diff --git a/.gitignore b/.gitignore index 72a123ef..64b33ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ access.log /cache/ public/css/user.css /plugins/ +/data diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/default/config.yaml b/default/config.yaml index dedb5ac5..506a270e 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -1,4 +1,6 @@ # -- NETWORK CONFIGURATION -- +# Root directory for user data storage +dataRoot: ./data # Listen for incoming connections listen: false # Server port diff --git a/default/content/index.json b/default/content/index.json index 80726f2f..d81d3755 100644 --- a/default/content/index.json +++ b/default/content/index.json @@ -1,4 +1,8 @@ [ + { + "filename": "settings.json", + "type": "settings" + }, { "filename": "themes/Dark Lite.json", "type": "theme" diff --git a/default/settings.json b/default/content/settings.json similarity index 100% rename from default/settings.json rename to default/content/settings.json diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 00000000..417bf315 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,12 @@ +import { UserDirectoryList, User } from "./src/users"; + +declare global { + namespace Express { + export interface Request { + user: { + profile: User; + directories: UserDirectoryList; + }; + } + } +} diff --git a/jsconfig.json b/jsconfig.json index 652e04b1..6f8f1a02 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -14,4 +14,4 @@ "node_modules", "**/node_modules/*" ] -} \ No newline at end of file +} diff --git a/post-install.js b/post-install.js index 406154aa..645085a8 100644 --- a/post-install.js +++ b/post-install.js @@ -106,7 +106,6 @@ function addMissingConfigValues() { */ function createDefaultFiles() { const files = { - settings: './public/settings.json', config: './config.yaml', user: './public/css/user.css', }; @@ -167,29 +166,6 @@ function copyWasmFiles() { } } -/** - * Moves the custom background into settings.json. - */ -function migrateBackground() { - if (!fs.existsSync('./public/css/bg_load.css')) return; - - const bgCSS = fs.readFileSync('./public/css/bg_load.css', 'utf-8'); - const bgMatch = /url\('([^']*)'\)/.exec(bgCSS); - if (!bgMatch) return; - const bgFilename = bgMatch[1].replace('../backgrounds/', ''); - - const settings = fs.readFileSync('./public/settings.json', 'utf-8'); - const settingsJSON = JSON.parse(settings); - if (Object.hasOwn(settingsJSON, 'background')) { - console.log(color.yellow('Both bg_load.css and the "background" setting exist. Please delete bg_load.css manually.')); - return; - } - - settingsJSON.background = { name: bgFilename, url: `url('backgrounds/${bgFilename}')` }; - fs.writeFileSync('./public/settings.json', JSON.stringify(settingsJSON, null, 4)); - fs.rmSync('./public/css/bg_load.css'); -} - try { // 0. Convert config.conf to config.yaml convertConfig(); @@ -199,8 +175,6 @@ try { copyWasmFiles(); // 3. Add missing config values addMissingConfigValues(); - // 4. Migrate bg_load.css to settings.json - migrateBackground(); } catch (error) { console.error(error); } diff --git a/server.js b/server.js index 85ebf555..4edabbf7 100644 --- a/server.js +++ b/server.js @@ -33,6 +33,7 @@ util.inspect.defaultOptions.maxStringLength = null; util.inspect.defaultOptions.depth = 4; // local library imports +const { initUserStorage, userDataMiddleware, getUserDirectories, getAllUserHandles } = require('./src/users'); const basicAuthMiddleware = require('./src/middleware/basicAuth'); const whitelistMiddleware = require('./src/middleware/whitelist'); const contentManager = require('./src/endpoints/content-manager'); @@ -112,7 +113,7 @@ const listen = cliArguments.listen ?? getConfigValue('listen', DEFAULT_LISTEN); const enableCorsProxy = cliArguments.corsProxy ?? getConfigValue('enableCorsProxy', DEFAULT_CORS_PROXY); const basicAuthMode = getConfigValue('basicAuthMode', false); -const { DIRECTORIES, UPLOADS_PATH } = require('./src/constants'); +const { UPLOADS_PATH, PUBLIC_DIRECTORIES } = require('./src/constants'); // CORS Settings // const CORS = cors({ @@ -211,29 +212,8 @@ if (enableCorsProxy) { } app.use(express.static(process.cwd() + '/public', {})); +app.use(userDataMiddleware(app)); -app.use('/backgrounds', (req, res) => { - const filePath = decodeURIComponent(path.join(process.cwd(), DIRECTORIES.backgrounds, req.url.replace(/%20/g, ' '))); - fs.readFile(filePath, (err, data) => { - if (err) { - res.status(404).send('File not found'); - return; - } - //res.contentType('image/jpeg'); - res.send(data); - }); -}); - -app.use('/characters', (req, res) => { - const filePath = decodeURIComponent(path.join(process.cwd(), DIRECTORIES.characters, req.url.replace(/%20/g, ' '))); - fs.readFile(filePath, (err, data) => { - if (err) { - res.status(404).send('File not found'); - return; - } - res.send(data); - }); -}); app.use(multer({ dest: UPLOADS_PATH, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar')); app.get('/', function (request, response) { response.sendFile(process.cwd() + '/public/index.html'); @@ -487,6 +467,7 @@ const setupTasks = async function () { // TODO: do endpoint init functions depend on certain directories existing or not existing? They should be callable // in any order for encapsulation reasons, but right now it's unknown if that would break anything. + await initUserStorage(); await settingsEndpoint.init(); ensurePublicDirectoriesExist(); contentManager.checkForNewContent(); @@ -579,10 +560,20 @@ if (cliArguments.ssl) { ); } -function ensurePublicDirectoriesExist() { - for (const dir of Object.values(DIRECTORIES)) { +async function ensurePublicDirectoriesExist() { + for (const dir of Object.values(PUBLIC_DIRECTORIES)) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } } + + const userHandles = await getAllUserHandles(); + for (const handle of userHandles) { + const userDirectories = getUserDirectories(handle); + for (const dir of Object.values(userDirectories)) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + } } diff --git a/src/constants.js b/src/constants.js index 918374ea..d6a639ea 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,34 +1,51 @@ -const DIRECTORIES = { - worlds: 'public/worlds/', - user: 'public/user', - avatars: 'public/User Avatars', +const PUBLIC_DIRECTORIES = { images: 'public/img/', - userImages: 'public/user/images/', - groups: 'public/groups/', - groupChats: 'public/group chats', - chats: 'public/chats/', - characters: 'public/characters/', - backgrounds: 'public/backgrounds', - novelAI_Settings: 'public/NovelAI Settings', - koboldAI_Settings: 'public/KoboldAI Settings', - openAI_Settings: 'public/OpenAI Settings', - textGen_Settings: 'public/TextGen Settings', - thumbnails: 'thumbnails/', - thumbnailsBg: 'thumbnails/bg/', - thumbnailsAvatar: 'thumbnails/avatar/', - themes: 'public/themes', - movingUI: 'public/movingUI', - extensions: 'public/scripts/extensions', - instruct: 'public/instruct', - context: 'public/context', backups: 'backups/', - quickreplies: 'public/QuickReplies', - assets: 'public/assets', - comfyWorkflows: 'public/user/workflows', - files: 'public/user/files', sounds: 'public/sounds', }; +/** + * @type {import('./users').UserDirectoryList} + * @readonly + * @enum {string} + */ +const USER_DIRECTORY_TEMPLATE = Object.freeze({ + root: '', + thumbnails: 'thumbnails', + thumbnailsBg: 'thumbnails/bg', + thumbnailsAvatar: 'thumbnails/avatar', + worlds: 'worlds', + user: 'user', + avatars: 'User Avatars', + userImages: 'user/images', + groups: 'groups', + groupChats: 'group chats', + chats: 'chats', + characters: 'characters', + backgrounds: 'backgrounds', + novelAI_Settings: 'NovelAI Settings', + koboldAI_Settings: 'KoboldAI Settings', + openAI_Settings: 'OpenAI Settings', + textGen_Settings: 'TextGen Settings', + themes: 'themes', + movingUI: 'movingUI', + extensions: 'scripts/extensions', + instruct: 'instruct', + context: 'context', + quickreplies: 'QuickReplies', + assets: 'assets', + comfyWorkflows: 'user/workflows', + files: 'user/files', +}); + +const DEFAULT_USER = Object.freeze({ + uuid: '00000000-0000-0000-0000-000000000000', + handle: 'user0', + name: 'User', + created: 0, + password: '', +}); + const UNSAFE_EXTENSIONS = [ '.php', '.exe', @@ -270,7 +287,9 @@ const OPENROUTER_KEYS = [ ]; module.exports = { - DIRECTORIES, + DEFAULT_USER, + PUBLIC_DIRECTORIES, + USER_DIRECTORY_TEMPLATE, UNSAFE_EXTENSIONS, UPLOADS_PATH, GEMINI_SAFETY, diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index ea9b5f5c..20241104 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -6,17 +6,16 @@ const sanitize = require('sanitize-filename'); const { getConfigValue } = require('../util'); const { jsonParser } = require('../express-common'); const contentDirectory = path.join(process.cwd(), 'default/content'); -const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('../constants'); -const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; +const { getAllUserHandles, getUserDirectories } = require('../users'); const characterCardParser = require('../character-card-parser.js'); /** * Gets the default presets from the content directory. + * @param {import('../users').UserDirectoryList} directories User directories * @returns {object[]} Array of default presets */ -function getDefaultPresets() { +function getDefaultPresets(directories) { try { const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); const contentIndex = JSON.parse(contentIndexText); @@ -26,7 +25,7 @@ function getDefaultPresets() { for (const contentItem of contentIndex) { if (contentItem.type.endsWith('_preset') || contentItem.type === 'instruct' || contentItem.type === 'context') { contentItem.name = path.parse(contentItem.filename).name; - contentItem.folder = getTargetByType(contentItem.type); + contentItem.folder = getTargetByType(contentItem.type, directories); presets.push(contentItem); } } @@ -59,120 +58,117 @@ function getDefaultPresetFile(filename) { } } -function migratePresets() { - for (const presetFolder of presetFolders) { - const presetPath = path.join(process.cwd(), presetFolder); - const presetFiles = fs.readdirSync(presetPath); - - for (const presetFile of presetFiles) { - const presetFilePath = path.join(presetPath, presetFile); - const newFileName = presetFile.replace('.settings', '.json'); - const newFilePath = path.join(presetPath, newFileName); - const backupFileName = presetFolder.replace('/', '_') + '_' + presetFile; - const backupFilePath = path.join(DIRECTORIES.backups, backupFileName); - - if (presetFilePath.endsWith('.settings')) { - if (!fs.existsSync(newFilePath)) { - fs.cpSync(presetFilePath, backupFilePath); - fs.cpSync(presetFilePath, newFilePath); - console.log(`Migrated ${presetFilePath} to ${newFilePath}`); - } - } - } - } -} - -function checkForNewContent() { +async function checkForNewContent() { try { - migratePresets(); - if (getConfigValue('skipContentCheck', false)) { return; } - const contentLog = getContentLog(); const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8'); const contentIndex = JSON.parse(contentIndexText); + const userHandles = await getAllUserHandles(); - for (const contentItem of contentIndex) { - // If the content item is already in the log, skip it - if (contentLog.includes(contentItem.filename)) { - continue; + for (const userHandle of userHandles) { + const directories = getUserDirectories(userHandle); + + if (!fs.existsSync(directories.root)) { + fs.mkdirSync(directories.root, { recursive: true }); } - contentLog.push(contentItem.filename); - const contentPath = path.join(contentDirectory, contentItem.filename); + const contentLogPath = path.join(directories.root, 'content.log'); + const contentLog = getContentLog(contentLogPath); - if (!fs.existsSync(contentPath)) { - console.log(`Content file ${contentItem.filename} is missing`); - continue; + for (const contentItem of contentIndex) { + // If the content item is already in the log, skip it + if (contentLog.includes(contentItem.filename)) { + continue; + } + + contentLog.push(contentItem.filename); + const contentPath = path.join(contentDirectory, contentItem.filename); + + if (!fs.existsSync(contentPath)) { + console.log(`Content file ${contentItem.filename} is missing`); + continue; + } + + const contentTarget = getTargetByType(contentItem.type, directories); + + if (!contentTarget) { + console.log(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`); + continue; + } + + const basePath = path.parse(contentItem.filename).base; + const targetPath = path.join(process.cwd(), contentTarget, basePath); + + if (fs.existsSync(targetPath)) { + console.log(`Content file ${contentItem.filename} already exists in ${contentTarget}`); + continue; + } + + fs.cpSync(contentPath, targetPath, { recursive: true, force: false }); + console.log(`Content file ${contentItem.filename} copied to ${contentTarget}`); } - const contentTarget = getTargetByType(contentItem.type); - - if (!contentTarget) { - console.log(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`); - continue; - } - - const basePath = path.parse(contentItem.filename).base; - const targetPath = path.join(process.cwd(), contentTarget, basePath); - - if (fs.existsSync(targetPath)) { - console.log(`Content file ${contentItem.filename} already exists in ${contentTarget}`); - continue; - } - - fs.cpSync(contentPath, targetPath, { recursive: true, force: false }); - console.log(`Content file ${contentItem.filename} copied to ${contentTarget}`); + fs.writeFileSync(contentLogPath, contentLog.join('\n')); } - - fs.writeFileSync(contentLogPath, contentLog.join('\n')); } catch (err) { console.log('Content check failed', err); } } -function getTargetByType(type) { +/** + * Gets the target directory for the specified asset type. + * @param {string} type Asset type + * @param {import('../users').UserDirectoryList} directories User directories + * @returns {string | null} Target directory + */ +function getTargetByType(type, directories) { switch (type) { + case 'settings': + return directories.root; case 'character': - return DIRECTORIES.characters; + return directories.characters; case 'sprites': - return DIRECTORIES.characters; + return directories.characters; case 'background': - return DIRECTORIES.backgrounds; + return directories.backgrounds; case 'world': - return DIRECTORIES.worlds; - case 'sound': - return DIRECTORIES.sounds; + return directories.worlds; case 'avatar': - return DIRECTORIES.avatars; + return directories.avatars; case 'theme': - return DIRECTORIES.themes; + return directories.themes; case 'workflow': - return DIRECTORIES.comfyWorkflows; + return directories.comfyWorkflows; case 'kobold_preset': - return DIRECTORIES.koboldAI_Settings; + return directories.koboldAI_Settings; case 'openai_preset': - return DIRECTORIES.openAI_Settings; + return directories.openAI_Settings; case 'novel_preset': - return DIRECTORIES.novelAI_Settings; + return directories.novelAI_Settings; case 'textgen_preset': - return DIRECTORIES.textGen_Settings; + return directories.textGen_Settings; case 'instruct': - return DIRECTORIES.instruct; + return directories.instruct; case 'context': - return DIRECTORIES.context; + return directories.context; case 'moving_ui': - return DIRECTORIES.movingUI; + return directories.movingUI; case 'quick_replies': - return DIRECTORIES.quickreplies; + return directories.quickreplies; default: return null; } } -function getContentLog() { +/** + * Gets the content log from the content log file. + * @param {string} contentLogPath Path to the content log file + * @returns {string[]} Array of content log lines + */ +function getContentLog(contentLogPath) { if (!fs.existsSync(contentLogPath)) { return []; } diff --git a/src/endpoints/presets.js b/src/endpoints/presets.js index 0c2f15a2..ed90275c 100644 --- a/src/endpoints/presets.js +++ b/src/endpoints/presets.js @@ -3,30 +3,30 @@ const path = require('path'); const express = require('express'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); const { jsonParser } = require('../express-common'); /** * Gets the folder and extension for the preset settings based on the API source ID. * @param {string} apiId API source ID + * @param {import('../users').UserDirectoryList} directories User directories * @returns {object} Object containing the folder and extension for the preset settings */ -function getPresetSettingsByAPI(apiId) { +function getPresetSettingsByAPI(apiId, directories) { switch (apiId) { case 'kobold': case 'koboldhorde': - return { folder: DIRECTORIES.koboldAI_Settings, extension: '.json' }; + return { folder: directories.koboldAI_Settings, extension: '.json' }; case 'novel': - return { folder: DIRECTORIES.novelAI_Settings, extension: '.json' }; + return { folder: directories.novelAI_Settings, extension: '.json' }; case 'textgenerationwebui': - return { folder: DIRECTORIES.textGen_Settings, extension: '.json' }; + return { folder: directories.textGen_Settings, extension: '.json' }; case 'openai': - return { folder: DIRECTORIES.openAI_Settings, extension: '.json' }; + return { folder: directories.openAI_Settings, extension: '.json' }; case 'instruct': - return { folder: DIRECTORIES.instruct, extension: '.json' }; + return { folder: directories.instruct, extension: '.json' }; case 'context': - return { folder: DIRECTORIES.context, extension: '.json' }; + return { folder: directories.context, extension: '.json' }; default: return { folder: null, extension: null }; } @@ -40,7 +40,7 @@ router.post('/save', jsonParser, function (request, response) { return response.sendStatus(400); } - const settings = getPresetSettingsByAPI(request.body.apiId); + const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories); const filename = name + settings.extension; if (!settings.folder) { @@ -58,7 +58,7 @@ router.post('/delete', jsonParser, function (request, response) { return response.sendStatus(400); } - const settings = getPresetSettingsByAPI(request.body.apiId); + const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories); const filename = name + settings.extension; if (!settings.folder) { @@ -77,9 +77,9 @@ router.post('/delete', jsonParser, function (request, response) { router.post('/restore', jsonParser, function (request, response) { try { - const settings = getPresetSettingsByAPI(request.body.apiId); + const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories); const name = sanitize(request.body.name); - const defaultPresets = getDefaultPresets(); + const defaultPresets = getDefaultPresets(request.user.directories); const defaultPreset = defaultPresets.find(p => p.name === name && p.folder === settings.folder); @@ -104,7 +104,7 @@ router.post('/save-openai', jsonParser, function (request, response) { if (!name) return response.sendStatus(400); const filename = `${name}.json`; - const fullpath = path.join(DIRECTORIES.openAI_Settings, filename); + const fullpath = path.join(request.user.directories.openAI_Settings, filename); writeFileAtomicSync(fullpath, JSON.stringify(request.body, null, 4), 'utf-8'); return response.send({ name }); }); @@ -116,7 +116,7 @@ router.post('/delete-openai', jsonParser, function (request, response) { } const name = request.body.name; - const pathToFile = path.join(DIRECTORIES.openAI_Settings, `${name}.json`); + const pathToFile = path.join(request.user.directories.openAI_Settings, `${name}.json`); if (fs.existsSync(pathToFile)) { fs.rmSync(pathToFile); diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index afd41a1f..980c5eb7 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -109,64 +109,6 @@ function readSecretState() { return state; } -/** - * Migrates secrets from settings.json to secrets.json - * @param {string} settingsFile Path to settings.json - * @returns {void} - */ -function migrateSecrets(settingsFile) { - const palmKey = readSecret('api_key_palm'); - if (palmKey) { - console.log('Migrating Palm key...'); - writeSecret(SECRET_KEYS.MAKERSUITE, palmKey); - deleteSecret('api_key_palm'); - } - - if (!fs.existsSync(settingsFile)) { - console.log('Settings file does not exist'); - return; - } - - try { - let modified = false; - const fileContents = fs.readFileSync(settingsFile, 'utf8'); - const settings = JSON.parse(fileContents); - const oaiKey = settings?.api_key_openai; - const hordeKey = settings?.horde_settings?.api_key; - const novelKey = settings?.api_key_novel; - - if (typeof oaiKey === 'string') { - console.log('Migrating OpenAI key...'); - writeSecret(SECRET_KEYS.OPENAI, oaiKey); - delete settings.api_key_openai; - modified = true; - } - - if (typeof hordeKey === 'string') { - console.log('Migrating Horde key...'); - writeSecret(SECRET_KEYS.HORDE, hordeKey); - delete settings.horde_settings.api_key; - modified = true; - } - - if (typeof novelKey === 'string') { - console.log('Migrating Novel key...'); - writeSecret(SECRET_KEYS.NOVEL, novelKey); - delete settings.api_key_novel; - modified = true; - } - - if (modified) { - console.log('Writing updated settings.json...'); - const settingsContent = JSON.stringify(settings, null, 4); - writeFileAtomicSync(settingsFile, settingsContent, 'utf-8'); - } - } - catch (error) { - console.error('Could not migrate secrets file. Proceed with caution.'); - } -} - /** * Reads all secrets from the secrets file * @returns {Record | undefined} Secrets @@ -251,7 +193,6 @@ module.exports = { writeSecret, readSecret, readSecretState, - migrateSecrets, getAllSecrets, SECRET_KEYS, router, diff --git a/src/endpoints/settings.js b/src/endpoints/settings.js index aae32b84..ad591491 100644 --- a/src/endpoints/settings.js +++ b/src/endpoints/settings.js @@ -2,13 +2,13 @@ const fs = require('fs'); const path = require('path'); const express = require('express'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('../constants'); +const { PUBLIC_DIRECTORIES } = require('../constants'); const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util'); const { jsonParser } = require('../express-common'); -const { migrateSecrets } = require('./secrets'); +const { getAllUserHandles, getUserDirectories } = require('../users'); +const SETTINGS_FILE = 'settings.json'; const enableExtensions = getConfigValue('enableExtensions', true); -const SETTINGS_FILE = './public/settings.json'; function readAndParseFromDirectory(directoryPath, fileExtension = '.json') { const files = fs @@ -61,16 +61,22 @@ function readPresetsFromDirectory(directoryPath, options = {}) { return { fileContents, fileNames }; } -function backupSettings() { +async function backupSettings() { try { - if (!fs.existsSync(DIRECTORIES.backups)) { - fs.mkdirSync(DIRECTORIES.backups); + if (!fs.existsSync(PUBLIC_DIRECTORIES.backups)) { + fs.mkdirSync(PUBLIC_DIRECTORIES.backups); } - const backupFile = path.join(DIRECTORIES.backups, `settings_${generateTimestamp()}.json`); - fs.copyFileSync(SETTINGS_FILE, backupFile); + const userHandles = await getAllUserHandles(); - removeOldBackups('settings_'); + for (const handle of userHandles) { + const userDirectories = getUserDirectories(handle); + const backupFile = path.join(PUBLIC_DIRECTORIES.backups, `settings_${handle}_${generateTimestamp()}.json`); + const sourceFile = path.join(userDirectories.root, SETTINGS_FILE); + fs.copyFileSync(sourceFile, backupFile); + + removeOldBackups(`settings_${handle}`); + } } catch (err) { console.log('Could not backup settings file', err); } @@ -80,7 +86,8 @@ const router = express.Router(); router.post('/save', jsonParser, function (request, response) { try { - writeFileAtomicSync('public/settings.json', JSON.stringify(request.body, null, 4), 'utf8'); + const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE); + writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8'); response.send({ result: 'ok' }); } catch (err) { console.log(err); @@ -92,48 +99,49 @@ router.post('/save', jsonParser, function (request, response) { router.post('/get', jsonParser, (request, response) => { let settings; try { - settings = fs.readFileSync('public/settings.json', 'utf8'); + const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE); + settings = fs.readFileSync(pathToSettings, 'utf8'); } catch (e) { return response.sendStatus(500); } // NovelAI Settings const { fileContents: novelai_settings, fileNames: novelai_setting_names } - = readPresetsFromDirectory(DIRECTORIES.novelAI_Settings, { - sortFunction: sortByName(DIRECTORIES.novelAI_Settings), + = readPresetsFromDirectory(request.user.directories.novelAI_Settings, { + sortFunction: sortByName(request.user.directories.novelAI_Settings), removeFileExtension: true, }); // OpenAI Settings const { fileContents: openai_settings, fileNames: openai_setting_names } - = readPresetsFromDirectory(DIRECTORIES.openAI_Settings, { - sortFunction: sortByName(DIRECTORIES.openAI_Settings), removeFileExtension: true, + = readPresetsFromDirectory(request.user.directories.openAI_Settings, { + sortFunction: sortByName(request.user.directories.openAI_Settings), removeFileExtension: true, }); // TextGenerationWebUI Settings const { fileContents: textgenerationwebui_presets, fileNames: textgenerationwebui_preset_names } - = readPresetsFromDirectory(DIRECTORIES.textGen_Settings, { - sortFunction: sortByName(DIRECTORIES.textGen_Settings), removeFileExtension: true, + = readPresetsFromDirectory(request.user.directories.textGen_Settings, { + sortFunction: sortByName(request.user.directories.textGen_Settings), removeFileExtension: true, }); //Kobold const { fileContents: koboldai_settings, fileNames: koboldai_setting_names } - = readPresetsFromDirectory(DIRECTORIES.koboldAI_Settings, { - sortFunction: sortByName(DIRECTORIES.koboldAI_Settings), removeFileExtension: true, + = readPresetsFromDirectory(request.user.directories.koboldAI_Settings, { + sortFunction: sortByName(request.user.directories.koboldAI_Settings), removeFileExtension: true, }); const worldFiles = fs - .readdirSync(DIRECTORIES.worlds) + .readdirSync(request.user.directories.worlds) .filter(file => path.extname(file).toLowerCase() === '.json') .sort((a, b) => a.localeCompare(b)); const world_names = worldFiles.map(item => path.parse(item).name); - const themes = readAndParseFromDirectory(DIRECTORIES.themes); - const movingUIPresets = readAndParseFromDirectory(DIRECTORIES.movingUI); - const quickReplyPresets = readAndParseFromDirectory(DIRECTORIES.quickreplies); + const themes = readAndParseFromDirectory(request.user.directories.themes); + const movingUIPresets = readAndParseFromDirectory(request.user.directories.movingUI); + const quickReplyPresets = readAndParseFromDirectory(request.user.directories.quickreplies); - const instruct = readAndParseFromDirectory(DIRECTORIES.instruct); - const context = readAndParseFromDirectory(DIRECTORIES.context); + const instruct = readAndParseFromDirectory(request.user.directories.instruct); + const context = readAndParseFromDirectory(request.user.directories.context); response.send({ settings, @@ -155,10 +163,11 @@ router.post('/get', jsonParser, (request, response) => { }); }); -// Sync for now, but should probably be migrated to async file APIs +/** + * Initializes the settings endpoint + */ async function init() { - backupSettings(); - migrateSecrets(SETTINGS_FILE); + await backupSettings(); } module.exports = { router, init }; diff --git a/src/endpoints/themes.js b/src/endpoints/themes.js index 4815c5c3..72f874b8 100644 --- a/src/endpoints/themes.js +++ b/src/endpoints/themes.js @@ -4,7 +4,6 @@ const fs = require('fs'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser } = require('../express-common'); -const { DIRECTORIES } = require('../constants'); const router = express.Router(); @@ -13,7 +12,7 @@ router.post('/save', jsonParser, (request, response) => { return response.sendStatus(400); } - const filename = path.join(DIRECTORIES.themes, sanitize(request.body.name) + '.json'); + const filename = path.join(request.user.directories.themes, sanitize(request.body.name) + '.json'); writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); return response.sendStatus(200); @@ -25,7 +24,7 @@ router.post('/delete', jsonParser, function (request, response) { } try { - const filename = path.join(DIRECTORIES.themes, sanitize(request.body.name) + '.json'); + const filename = path.join(request.user.directories.themes, sanitize(request.body.name) + '.json'); if (!fs.existsSync(filename)) { console.error('Theme file not found:', filename); return response.sendStatus(404); diff --git a/src/endpoints/thumbnails.js b/src/endpoints/thumbnails.js index ad898db1..6815efa4 100644 --- a/src/endpoints/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -4,24 +4,25 @@ const express = require('express'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('../constants'); +const { getAllUserHandles, getUserDirectories } = require('../users'); const { getConfigValue } = require('../util'); const { jsonParser } = require('../express-common'); /** * Gets a path to thumbnail folder based on the type. + * @param {import('../users').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Thumbnail type * @returns {string} Path to the thumbnails folder */ -function getThumbnailFolder(type) { +function getThumbnailFolder(directories, type) { let thumbnailFolder; switch (type) { case 'bg': - thumbnailFolder = DIRECTORIES.thumbnailsBg; + thumbnailFolder = directories.thumbnailsBg; break; case 'avatar': - thumbnailFolder = DIRECTORIES.thumbnailsAvatar; + thumbnailFolder = directories.thumbnailsAvatar; break; } @@ -30,18 +31,19 @@ function getThumbnailFolder(type) { /** * Gets a path to the original images folder based on the type. + * @param {import('../users').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Thumbnail type * @returns {string} Path to the original images folder */ -function getOriginalFolder(type) { +function getOriginalFolder(directories, type) { let originalFolder; switch (type) { case 'bg': - originalFolder = DIRECTORIES.backgrounds; + originalFolder = directories.backgrounds; break; case 'avatar': - originalFolder = DIRECTORIES.characters; + originalFolder = directories.characters; break; } @@ -50,11 +52,12 @@ function getOriginalFolder(type) { /** * Removes the generated thumbnail from the disk. + * @param {import('../users').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Type of the thumbnail * @param {string} file Name of the file */ -function invalidateThumbnail(type, file) { - const folder = getThumbnailFolder(type); +function invalidateThumbnail(directories, type, file) { + const folder = getThumbnailFolder(directories, type); if (folder === undefined) throw new Error('Invalid thumbnail type'); const pathToThumbnail = path.join(folder, file); @@ -66,13 +69,14 @@ function invalidateThumbnail(type, file) { /** * Generates a thumbnail for the given file. + * @param {import('../users').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Type of the thumbnail * @param {string} file Name of the file * @returns */ -async function generateThumbnail(type, file) { - let thumbnailFolder = getThumbnailFolder(type); - let originalFolder = getOriginalFolder(type); +async function generateThumbnail(directories, type, file) { + let thumbnailFolder = getThumbnailFolder(directories, type); + let originalFolder = getOriginalFolder(directories, type); if (thumbnailFolder === undefined || originalFolder === undefined) throw new Error('Invalid thumbnail type'); const pathToCachedFile = path.join(thumbnailFolder, file); @@ -133,24 +137,28 @@ async function generateThumbnail(type, file) { * @returns {Promise} Promise that resolves when the cache is validated */ async function ensureThumbnailCache() { - const cacheFiles = fs.readdirSync(DIRECTORIES.thumbnailsBg); + const userHandles = await getAllUserHandles(); + for (const handle of userHandles) { + const directories = getUserDirectories(handle); + const cacheFiles = fs.readdirSync(directories.thumbnailsBg); - // files exist, all ok - if (cacheFiles.length) { - return; + // files exist, all ok + if (cacheFiles.length) { + return; + } + + console.log('Generating thumbnails cache. Please wait...'); + + const bgFiles = fs.readdirSync(directories.backgrounds); + const tasks = []; + + for (const file of bgFiles) { + tasks.push(generateThumbnail(directories, 'bg', file)); + } + + await Promise.all(tasks); + console.log(`Done! Generated: ${bgFiles.length} preview images`); } - - console.log('Generating thumbnails cache. Please wait...'); - - const bgFiles = fs.readdirSync(DIRECTORIES.backgrounds); - const tasks = []; - - for (const file of bgFiles) { - tasks.push(generateThumbnail('bg', file)); - } - - await Promise.all(tasks); - console.log(`Done! Generated: ${bgFiles.length} preview images`); } const router = express.Router(); @@ -176,13 +184,13 @@ router.get('/', jsonParser, async function (request, response) { } if (getConfigValue('disableThumbnails', false) == true) { - let folder = getOriginalFolder(type); + let folder = getOriginalFolder(request.user.directories, type); if (folder === undefined) return response.sendStatus(400); const pathToOriginalFile = path.join(folder, file); return response.sendFile(pathToOriginalFile, { root: process.cwd() }); } - const pathToCachedFile = await generateThumbnail(type, file); + const pathToCachedFile = await generateThumbnail(request.user.directories, type, file); if (!pathToCachedFile) { return response.sendStatus(404); diff --git a/src/users.js b/src/users.js new file mode 100644 index 00000000..dab9f139 --- /dev/null +++ b/src/users.js @@ -0,0 +1,155 @@ +const fsPromises = require('fs').promises; +const path = require('path'); +const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants'); +const { getConfigValue } = require('./util'); + +const DATA_ROOT = getConfigValue('dataRoot', './data'); + +/** + * @typedef {Object} User + * @property {string} uuid - The user's id + * @property {string} handle - The user's short handle. Used for directories and other references + * @property {string} name - The user's name. Displayed in the UI + * @property {number} created - The timestamp when the user was created + * @property {string} password - SHA256 hash of the user's password + */ + +/** + * @typedef {Object} UserDirectoryList + * @property {string} root - The root directory for the user + * @property {string} thumbnails - The directory where the thumbnails are stored + * @property {string} thumbnailsBg - The directory where the background thumbnails are stored + * @property {string} thumbnailsAvatar - The directory where the avatar thumbnails are stored + * @property {string} worlds - The directory where the WI are stored + * @property {string} user - The directory where the user's public data is stored + * @property {string} avatars - The directory where the avatars are stored + * @property {string} userImages - The directory where the images are stored + * @property {string} groups - The directory where the groups are stored + * @property {string} groupChats - The directory where the group chats are stored + * @property {string} chats - The directory where the chats are stored + * @property {string} characters - The directory where the characters are stored + * @property {string} backgrounds - The directory where the backgrounds are stored + * @property {string} novelAI_Settings - The directory where the NovelAI settings are stored + * @property {string} koboldAI_Settings - The directory where the KoboldAI settings are stored + * @property {string} openAI_Settings - The directory where the OpenAI settings are stored + * @property {string} textGen_Settings - The directory where the TextGen settings are stored + * @property {string} themes - The directory where the themes are stored + * @property {string} movingUI - The directory where the moving UI data is stored + * @property {string} extensions - The directory where the extensions are stored + * @property {string} instruct - The directory where the instruct templates is stored + * @property {string} context - The directory where the context templates is stored + * @property {string} quickreplies - The directory where the quick replies are stored + * @property {string} assets - The directory where the assets are stored + * @property {string} comfyWorkflows - The directory where the ComfyUI workflows are stored + * @property {string} files - The directory where the uploaded files are stored + */ + +/** + * Initializes the user storage. Currently a no-op. + * @returns {Promise} + */ +async function initUserStorage() { + return Promise.resolve(); +} + +/** + * Gets a user for the current request. Hard coded to return the default user. + * @param {import('express').Request} _req - The request object. Currently unused. + * @returns {Promise} - The user's handle + */ +async function getCurrentUserHandle(_req) { + return DEFAULT_USER.handle; +} + +/** + * Gets a list of all user handles. Currently hard coded to return the default user's handle. + * @returns {Promise} - The list of user handles + */ +async function getAllUserHandles() { + return [DEFAULT_USER.handle]; +} + +/** + * Gets the directories listing for the provided user. + * @param {import('express').Request} req - The request object + * @returns {Promise} - The user's directories like {worlds: 'data/user0/worlds/', ... + */ +async function getCurrentUserDirectories(req) { + const handle = await getCurrentUserHandle(req); + return getUserDirectories(handle); +} + +/** + * Gets the directories listing for the provided user. + * @param {string} handle User handle + * @returns {UserDirectoryList} User directories + */ +function getUserDirectories(handle) { + const directories = structuredClone(USER_DIRECTORY_TEMPLATE); + for (const key in directories) { + directories[key] = path.join(DATA_ROOT, handle, USER_DIRECTORY_TEMPLATE[key]); + } + return directories; +} + +/** + * Middleware to add user data to the request object. + * @param {import('express').Application} app - The express app + * @returns {import('express').RequestHandler} + */ +function userDataMiddleware(app) { + app.use('/backgrounds/:path', async (req, res) => { + try { + const filePath = path.join(process.cwd(), req.user.directories.backgrounds, decodeURIComponent(req.params.path)); + const data = await fsPromises.readFile(filePath); + return res.send(data); + } + catch { + return res.sendStatus(404); + } + }); + + app.use('/characters/:path', async (req, res) => { + try { + const filePath = path.join(process.cwd(), req.user.directories.characters, decodeURIComponent(req.params.path)); + const data = await fsPromises.readFile(filePath); + return res.send(data); + } + catch { + return res.sendStatus(404); + } + }); + + app.use('/User Avatars/:path', async (req, res) => { + try { + const filePath = path.join(process.cwd(), req.user.directories.avatars, decodeURIComponent(req.params.path)); + const data = await fsPromises.readFile(filePath); + return res.send(data); + } + catch { + return res.sendStatus(404); + } + }); + + /** + * Middleware to add user data to the request object. + * @param {import('express').Request} req Request object + * @param {import('express').Response} res Response object + * @param {import('express').NextFunction} next Next function + */ + return async (req, res, next) => { + const directories = await getCurrentUserDirectories(req); + req.user.profile = DEFAULT_USER; + req.user.directories = directories; + next(); + }; +} + +module.exports = { + initUserStorage, + getCurrentUserDirectories, + getCurrentUserHandle, + getAllUserHandles, + getUserDirectories, + userDataMiddleware, +}; diff --git a/src/util.js b/src/util.js index e23acb68..dfbc1e6d 100644 --- a/src/util.js +++ b/src/util.js @@ -8,7 +8,7 @@ const yaml = require('yaml'); const { default: simpleGit } = require('simple-git'); const { Readable } = require('stream'); -const { DIRECTORIES } = require('./constants'); +const { PUBLIC_DIRECTORIES } = require('./constants'); /** * Returns the config object from the config.yaml file. @@ -355,9 +355,9 @@ function generateTimestamp() { function removeOldBackups(prefix) { const MAX_BACKUPS = 25; - let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix)); + let files = fs.readdirSync(PUBLIC_DIRECTORIES.backups).filter(f => f.startsWith(prefix)); if (files.length > MAX_BACKUPS) { - files = files.map(f => path.join(DIRECTORIES.backups, f)); + files = files.map(f => path.join(PUBLIC_DIRECTORIES.backups, f)); files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); fs.rmSync(files[0]);