// Native Node Modules import path from 'node:path'; import fs from 'node:fs'; import crypto from 'node:crypto'; import os from 'node:os'; import process from 'node:process'; import { Buffer } from 'node:buffer'; // Express and other dependencies import storage from 'node-persist'; import express from 'express'; import mime from 'mime-types'; import archiver from 'archiver'; import _ from 'lodash'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } from './constants.js'; import { getConfigValue, color, delay, setConfigValue, generateTimestamp } from './util.js'; import { readSecret, writeSecret } from './endpoints/secrets.js'; import { getContentOfType } from './endpoints/content-manager.js'; export const KEY_PREFIX = 'user:'; const AVATAR_PREFIX = 'avatar:'; const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false); const AUTHELIA_AUTH = getConfigValue('autheliaAuth', false); const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false); const ANON_CSRF_SECRET = crypto.randomBytes(64).toString('base64'); /** * Cache for user directories. * @type {Map} */ const DIRECTORIES_CACHE = new Map(); const PUBLIC_USER_AVATAR = '/img/default-user.png'; const STORAGE_KEYS = { csrfSecret: 'csrfSecret', cookieSecret: 'cookieSecret', }; /** * @typedef {Object} User * @property {string} handle - The user's short handle. Used for directories and other references * @property {string} name - The user's name. Displayed in the UI * @property {number} created - The timestamp when the user was created * @property {string} password - Scrypt hash of the user's password * @property {string} salt - Salt used for hashing the password * @property {boolean} enabled - Whether the user is enabled * @property {boolean} admin - Whether the user is an admin (can manage other users) */ /** * @typedef {Object} UserViewModel * @property {string} handle - The user's short handle. Used for directories and other references * @property {string} name - The user's name. Displayed in the UI * @property {string} avatar - The user's avatar image * @property {boolean} [admin] - Whether the user is an admin (can manage other users) * @property {boolean} password - Whether the user is password protected * @property {boolean} [enabled] - Whether the user is enabled * @property {number} [created] - The timestamp when the user was created */ /** * @typedef {Object} UserDirectoryList * @property {string} root - The root directory for the user * @property {string} thumbnails - The directory where the thumbnails are stored * @property {string} thumbnailsBg - The directory where the background thumbnails are stored * @property {string} thumbnailsAvatar - The directory where the avatar thumbnails are stored * @property {string} worlds - The directory where the WI are stored * @property {string} user - The directory where the user's public data is stored * @property {string} avatars - The directory where the avatars are stored * @property {string} userImages - The directory where the images are stored * @property {string} groups - The directory where the groups are stored * @property {string} groupChats - The directory where the group chats are stored * @property {string} chats - The directory where the chats are stored * @property {string} characters - The directory where the characters are stored * @property {string} backgrounds - The directory where the backgrounds are stored * @property {string} novelAI_Settings - The directory where the NovelAI settings are stored * @property {string} koboldAI_Settings - The directory where the KoboldAI settings are stored * @property {string} openAI_Settings - The directory where the OpenAI settings are stored * @property {string} textGen_Settings - The directory where the TextGen settings are stored * @property {string} themes - The directory where the themes are stored * @property {string} movingUI - The directory where the moving UI data is stored * @property {string} extensions - The directory where the extensions are stored * @property {string} instruct - The directory where the instruct templates is stored * @property {string} context - The directory where the context templates is stored * @property {string} quickreplies - The directory where the quick replies are stored * @property {string} assets - The directory where the assets are stored * @property {string} comfyWorkflows - The directory where the ComfyUI workflows are stored * @property {string} files - The directory where the uploaded files are stored * @property {string} vectors - The directory where the vectors are stored * @property {string} backups - The directory where the backups are stored * @property {string} sysprompt - The directory where the system prompt data is stored */ /** * Ensures that the content directories exist. * @returns {Promise} - The list of user directories */ export async function ensurePublicDirectoriesExist() { for (const dir of Object.values(PUBLIC_DIRECTORIES)) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } } const userHandles = await getAllUserHandles(); const directoriesList = userHandles.map(handle => getUserDirectories(handle)); for (const userDirectories of directoriesList) { for (const dir of Object.values(userDirectories)) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } } } return directoriesList; } /** * Gets a list of all user directories. * @returns {Promise} - The list of user directories */ export async function getUserDirectoriesList() { const userHandles = await getAllUserHandles(); const directoriesList = userHandles.map(handle => getUserDirectories(handle)); return directoriesList; } /** * Perform migration from the old user data format to the new one. */ export async function migrateUserData() { const publicDirectory = path.join(process.cwd(), 'public'); // No need to migrate if the characters directory doesn't exists if (!fs.existsSync(path.join(publicDirectory, 'characters'))) { return; } const TIMEOUT = 10; console.log(); console.log(color.magenta('Preparing to migrate user data...')); console.log(`All public data will be moved to the ${globalThis.DATA_ROOT} directory.`); console.log('This process may take a while depending on the amount of data to move.'); console.log(`Backups will be placed in the ${PUBLIC_DIRECTORIES.backups} directory.`); console.log(`The process will start in ${TIMEOUT} seconds. Press Ctrl+C to cancel.`); for (let i = TIMEOUT; i > 0; i--) { console.log(`${i}...`); await delay(1000); } console.log(color.magenta('Starting migration... Do not interrupt the process!')); const userDirectories = getUserDirectories(DEFAULT_USER.handle); const dataMigrationMap = [ { old: path.join(publicDirectory, 'assets'), new: userDirectories.assets, file: false, }, { old: path.join(publicDirectory, 'backgrounds'), new: userDirectories.backgrounds, file: false, }, { old: path.join(publicDirectory, 'characters'), new: userDirectories.characters, file: false, }, { old: path.join(publicDirectory, 'chats'), new: userDirectories.chats, file: false, }, { old: path.join(publicDirectory, 'context'), new: userDirectories.context, file: false, }, { old: path.join(publicDirectory, 'group chats'), new: userDirectories.groupChats, file: false, }, { old: path.join(publicDirectory, 'groups'), new: userDirectories.groups, file: false, }, { old: path.join(publicDirectory, 'instruct'), new: userDirectories.instruct, file: false, }, { old: path.join(publicDirectory, 'KoboldAI Settings'), new: userDirectories.koboldAI_Settings, file: false, }, { old: path.join(publicDirectory, 'movingUI'), new: userDirectories.movingUI, file: false, }, { old: path.join(publicDirectory, 'NovelAI Settings'), new: userDirectories.novelAI_Settings, file: false, }, { old: path.join(publicDirectory, 'OpenAI Settings'), new: userDirectories.openAI_Settings, file: false, }, { old: path.join(publicDirectory, 'QuickReplies'), new: userDirectories.quickreplies, file: false, }, { old: path.join(publicDirectory, 'TextGen Settings'), new: userDirectories.textGen_Settings, file: false, }, { old: path.join(publicDirectory, 'themes'), new: userDirectories.themes, file: false, }, { old: path.join(publicDirectory, 'user'), new: userDirectories.user, file: false, }, { old: path.join(publicDirectory, 'User Avatars'), new: userDirectories.avatars, file: false, }, { old: path.join(publicDirectory, 'worlds'), new: userDirectories.worlds, file: false, }, { old: path.join(publicDirectory, 'scripts/extensions/third-party'), new: userDirectories.extensions, file: false, }, { old: path.join(process.cwd(), 'thumbnails'), new: userDirectories.thumbnails, file: false, }, { old: path.join(process.cwd(), 'vectors'), new: userDirectories.vectors, file: false, }, { old: path.join(process.cwd(), 'secrets.json'), new: path.join(userDirectories.root, 'secrets.json'), file: true, }, { old: path.join(publicDirectory, 'settings.json'), new: path.join(userDirectories.root, 'settings.json'), file: true, }, { old: path.join(publicDirectory, 'stats.json'), new: path.join(userDirectories.root, 'stats.json'), file: true, }, ]; const currentDate = new Date().toISOString().split('T')[0]; const backupDirectory = path.join(process.cwd(), PUBLIC_DIRECTORIES.backups, '_migration', currentDate); if (!fs.existsSync(backupDirectory)) { fs.mkdirSync(backupDirectory, { recursive: true }); } const errors = []; for (const migration of dataMigrationMap) { console.log(`Migrating ${migration.old} to ${migration.new}...`); try { if (!fs.existsSync(migration.old)) { console.log(color.yellow(`Skipping migration of ${migration.old} as it does not exist.`)); continue; } if (migration.file) { // Copy the file to the new location fs.cpSync(migration.old, migration.new, { force: true }); // Move the file to the backup location fs.cpSync( migration.old, path.join(backupDirectory, path.basename(migration.old)), { recursive: true, force: true }, ); fs.rmSync(migration.old, { recursive: true, force: true }); } else { // Copy the directory to the new location fs.cpSync(migration.old, migration.new, { recursive: true, force: true }); // Move the directory to the backup location fs.cpSync( migration.old, path.join(backupDirectory, path.basename(migration.old)), { recursive: true, force: true }, ); fs.rmSync(migration.old, { recursive: true, force: true }); } } catch (error) { console.error(color.red(`Error migrating ${migration.old} to ${migration.new}:`), error.message); errors.push(migration.old); } } if (errors.length > 0) { console.log(color.red('Migration completed with errors. Move the following files manually:')); errors.forEach(error => console.error(error)); } console.log(color.green('Migration completed!')); } export async function migrateSystemPrompts() { /** * Gets the default system prompts. * @returns {Promise} - The list of default system prompts */ async function getDefaultSystemPrompts() { try { return getContentOfType('sysprompt', 'json'); } catch { return []; } } const directories = await getUserDirectoriesList(); for (const directory of directories) { try { const migrateMarker = path.join(directory.sysprompt, '.migrated'); if (fs.existsSync(migrateMarker)) { continue; } const backupsPath = path.join(directory.backups, '_sysprompt'); fs.mkdirSync(backupsPath, { recursive: true }); const defaultPrompts = await getDefaultSystemPrompts(); const instucts = fs.readdirSync(directory.instruct); let migratedPrompts = []; for (const instruct of instucts) { const instructPath = path.join(directory.instruct, instruct); const sysPromptPath = path.join(directory.sysprompt, instruct); if (path.extname(instruct) === '.json' && !fs.existsSync(sysPromptPath)) { const instructData = JSON.parse(fs.readFileSync(instructPath, 'utf8')); if ('system_prompt' in instructData && 'name' in instructData) { const backupPath = path.join(backupsPath, `${instructData.name}.json`); fs.cpSync(instructPath, backupPath, { force: true }); const syspromptData = { name: instructData.name, content: instructData.system_prompt }; migratedPrompts.push(syspromptData); delete instructData.system_prompt; writeFileAtomicSync(instructPath, JSON.stringify(instructData, null, 4)); } } } // Only leave unique contents migratedPrompts = _.uniqBy(migratedPrompts, 'content'); // Only leave contents that are not in the default prompts migratedPrompts = migratedPrompts.filter(x => !defaultPrompts.some(y => y.content === x.content)); for (const sysPromptData of migratedPrompts) { sysPromptData.name = `[Migrated] ${sysPromptData.name}`; const syspromptPath = path.join(directory.sysprompt, `${sysPromptData.name}.json`); writeFileAtomicSync(syspromptPath, JSON.stringify(sysPromptData, null, 4)); console.log(`Migrated system prompt ${sysPromptData.name} for ${directory.root.split(path.sep).pop()}`); } writeFileAtomicSync(migrateMarker, ''); } catch (error) { console.error('Error migrating system prompts:', error); } } } /** * Converts a user handle to a storage key. * @param {string} handle User handle * @returns {string} The key for the user storage */ export function toKey(handle) { return `${KEY_PREFIX}${handle}`; } /** * Converts a user handle to a storage key for avatars. * @param {string} handle User handle * @returns {string} The key for the avatar storage */ export function toAvatarKey(handle) { return `${AVATAR_PREFIX}${handle}`; } /** * Initializes the user storage. * @param {string} dataRoot The root directory for user data * @returns {Promise} */ export async function initUserStorage(dataRoot) { globalThis.DATA_ROOT = dataRoot; console.log('Using data root:', color.green(globalThis.DATA_ROOT)); console.log(); await storage.init({ dir: path.join(globalThis.DATA_ROOT, '_storage'), ttl: false, // Never expire }); const keys = await getAllUserHandles(); // If there are no users, create the default user if (keys.length === 0) { await storage.setItem(toKey(DEFAULT_USER.handle), DEFAULT_USER); } } /** * Get the cookie secret from the config. If it doesn't exist, generate a new one. * @returns {string} The cookie secret */ export function getCookieSecret() { let secret = getConfigValue(STORAGE_KEYS.cookieSecret); if (!secret) { console.warn(color.yellow('Cookie secret is missing from config.yaml. Generating a new one...')); secret = crypto.randomBytes(64).toString('base64'); setConfigValue(STORAGE_KEYS.cookieSecret, secret); } return secret; } /** * Generates a random password salt. * @returns {string} The password salt */ export function getPasswordSalt() { return crypto.randomBytes(16).toString('base64'); } /** * Get the session name for the current server. * @returns {string} The session name */ export function getCookieSessionName() { // Get server hostname and hash it to generate a session suffix const suffix = crypto.createHash('sha256').update(os.hostname()).digest('hex').slice(0, 8); return `session-${suffix}`; } /** * Hashes a password using scrypt with the provided salt. * @param {string} password Password to hash * @param {string} salt Salt to use for hashing * @returns {string} Hashed password */ export function getPasswordHash(password, salt) { return crypto.scryptSync(password.normalize(), salt, 64).toString('base64'); } /** * Get the CSRF secret from the storage. * @param {import('express').Request} [request] HTTP request object * @returns {string} The CSRF secret */ export function getCsrfSecret(request) { if (!request || !request.user) { return ANON_CSRF_SECRET; } let csrfSecret = readSecret(request.user.directories, STORAGE_KEYS.csrfSecret); if (!csrfSecret) { csrfSecret = crypto.randomBytes(64).toString('base64'); writeSecret(request.user.directories, STORAGE_KEYS.csrfSecret, csrfSecret); } return csrfSecret; } /** * Gets a list of all user handles. * @returns {Promise} - The list of user handles */ export async function getAllUserHandles() { const keys = await storage.keys(x => x.key.startsWith(KEY_PREFIX)); const handles = keys.map(x => x.replace(KEY_PREFIX, '')); return handles; } /** * Gets the directories listing for the provided user. * @param {string} handle User handle * @returns {UserDirectoryList} User directories */ export function getUserDirectories(handle) { if (DIRECTORIES_CACHE.has(handle)) { const cache = DIRECTORIES_CACHE.get(handle); if (cache) { return cache; } } const directories = structuredClone(USER_DIRECTORY_TEMPLATE); for (const key in directories) { directories[key] = path.join(globalThis.DATA_ROOT, handle, USER_DIRECTORY_TEMPLATE[key]); } DIRECTORIES_CACHE.set(handle, directories); return directories; } /** * Gets the avatar URL for the provided user. * @param {string} handle User handle * @returns {Promise} User avatar URL */ export async function getUserAvatar(handle) { try { // Check if the user has a custom avatar const avatarKey = toAvatarKey(handle); const avatar = await storage.getItem(avatarKey); if (avatar) { return avatar; } // Fallback to reading from files if custom avatar is not set const directory = getUserDirectories(handle); const pathToSettings = path.join(directory.root, SETTINGS_FILE); const settings = fs.existsSync(pathToSettings) ? JSON.parse(fs.readFileSync(pathToSettings, 'utf8')) : {}; const avatarFile = settings?.power_user?.default_persona || settings?.user_avatar; if (!avatarFile) { return PUBLIC_USER_AVATAR; } const avatarPath = path.join(directory.avatars, avatarFile); if (!fs.existsSync(avatarPath)) { return PUBLIC_USER_AVATAR; } const mimeType = mime.lookup(avatarPath); const base64Content = fs.readFileSync(avatarPath, 'base64'); return `data:${mimeType};base64,${base64Content}`; } catch { // Ignore errors return PUBLIC_USER_AVATAR; } } /** * Checks if the user should be redirected to the login page. * @param {import('express').Request} request Request object * @returns {boolean} Whether the user should be redirected to the login page */ export function shouldRedirectToLogin(request) { return ENABLE_ACCOUNTS && !request.user; } /** * Tries auto-login if there is only one user and it's not password protected. * or another configured method such authlia or basic * @param {import('express').Request} request Request object * @param {boolean} basicAuthMode If Basic auth mode is enabled * @returns {Promise} Whether auto-login was performed */ export async function tryAutoLogin(request, basicAuthMode) { if (!ENABLE_ACCOUNTS || request.user || !request.session) { return false; } if (!request.query.noauto) { if (await singleUserLogin(request)) { return true; } if (AUTHELIA_AUTH && await autheliaUserLogin(request)) { return true; } if (basicAuthMode && PER_USER_BASIC_AUTH && await basicUserLogin(request)) { return true; } } return false; } /** * Tries auto-login if there is only one user and it's not password protected. * @param {import('express').Request} request Request object * @returns {Promise} Whether auto-login was performed */ async function singleUserLogin(request) { if (!request.session) { return false; } const userHandles = await getAllUserHandles(); if (userHandles.length === 1) { const user = await storage.getItem(toKey(userHandles[0])); if (user && !user.password) { request.session.handle = userHandles[0]; return true; } } return false; } /** * Tries auto-login with authlia trusted headers. * https://www.authelia.com/integration/trusted-header-sso/introduction/ * @param {import('express').Request} request Request object * @returns {Promise} Whether auto-login was performed */ async function autheliaUserLogin(request) { if (!request.session) { return false; } const remoteUser = request.get('Remote-User'); if (!remoteUser) { return false; } const userHandles = await getAllUserHandles(); for (const userHandle of userHandles) { if (remoteUser === userHandle) { const user = await storage.getItem(toKey(userHandle)); if (user && user.enabled) { request.session.handle = userHandle; return true; } } } return false; } /** * Tries auto-login with basic auth username. * @param {import('express').Request} request Request object * @returns {Promise} Whether auto-login was performed */ async function basicUserLogin(request) { if (!request.session) { return false; } const authHeader = request.headers.authorization; if (!authHeader) { return false; } const [scheme, credentials] = authHeader.split(' '); if (scheme !== 'Basic' || !credentials) { return false; } const [username, password] = Buffer.from(credentials, 'base64') .toString('utf8') .split(':'); const userHandles = await getAllUserHandles(); for (const userHandle of userHandles) { if (username === userHandle) { const user = await storage.getItem(toKey(userHandle)); // Verify pass again here just to be sure if (user && user.enabled && user.password && user.password === getPasswordHash(password, user.salt)) { request.session.handle = userHandle; return true; } } } return false; } /** * Middleware to add user data to the request object. * @param {import('express').Request} request Request object * @param {import('express').Response} response Response object * @param {import('express').NextFunction} next Next function */ export async function setUserDataMiddleware(request, response, next) { // If user accounts are disabled, use the default user if (!ENABLE_ACCOUNTS) { const handle = DEFAULT_USER.handle; const directories = getUserDirectories(handle); request.user = { profile: DEFAULT_USER, directories: directories, }; return next(); } if (!request.session) { console.error('Session not available'); return response.sendStatus(500); } // If user accounts are enabled, get the user from the session let handle = request.session?.handle; // If we have the only user and it's not password protected, use it if (!handle) { return next(); } /** @type {User} */ const user = await storage.getItem(toKey(handle)); if (!user) { console.error('User not found:', handle); return next(); } if (!user.enabled) { console.error('User is disabled:', handle); return next(); } const directories = getUserDirectories(handle); request.user = { profile: user, directories: directories, }; // Touch the session if loading the home page if (request.method === 'GET' && request.path === '/') { request.session.touch = Date.now(); } return next(); } /** * Middleware to add user data to the request object. * @param {import('express').Request} request Request object * @param {import('express').Response} response Response object * @param {import('express').NextFunction} next Next function */ export function requireLoginMiddleware(request, response, next) { if (!request.user) { return response.sendStatus(403); } return next(); } /** * Creates a route handler for serving files from a specific directory. * @param {(req: import('express').Request) => string} directoryFn A function that returns the directory path to serve files from * @returns {import('express').RequestHandler} */ function createRouteHandler(directoryFn) { return async (req, res) => { try { const directory = directoryFn(req); const filePath = decodeURIComponent(req.params[0]); const exists = fs.existsSync(path.join(directory, filePath)); if (!exists) { return res.sendStatus(404); } return res.sendFile(filePath, { root: directory }); } catch (error) { return res.sendStatus(500); } }; } /** * Verifies that the current user is an admin. * @param {import('express').Request} request Request object * @param {import('express').Response} response Response object * @param {import('express').NextFunction} next Next function * @returns {any} */ export function requireAdminMiddleware(request, response, next) { if (!request.user) { return response.sendStatus(403); } if (request.user.profile.admin) { return next(); } console.warn('Unauthorized access to admin endpoint:', request.originalUrl); return response.sendStatus(403); } /** * Creates an archive of the user's data root directory. * @param {string} handle User handle * @param {import('express').Response} response Express response object to write to * @returns {Promise} Promise that resolves when the archive is created */ export async function createBackupArchive(handle, response) { const directories = getUserDirectories(handle); console.log('Backup requested for', handle); const archive = archiver('zip'); archive.on('error', function (err) { response.status(500).send({ error: err.message }); }); // On stream closed we can end the request archive.on('end', function () { console.log('Archive wrote %d bytes', archive.pointer()); response.end(); // End the Express response }); const timestamp = generateTimestamp(); // Set the archive name response.attachment(`${handle}-${timestamp}.zip`); // This is the streaming magic // @ts-ignore archive.pipe(response); // Append files from a sub-directory, putting its contents at the root of archive archive.directory(directories.root, false); archive.finalize(); } /** * Gets all of the users. * @returns {Promise} */ async function getAllUsers() { if (!ENABLE_ACCOUNTS) { return []; } /** * @type {User[]} */ const users = await storage.values(); return users; } /** * Gets all of the enabled users. * @returns {Promise} */ export async function getAllEnabledUsers() { const users = await getAllUsers(); return users.filter(x => x.enabled); } /** * Express router for serving files from the user's directories. */ export const router = express.Router(); router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds)); router.use('/characters/*', createRouteHandler(req => req.user.directories.characters)); router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars)); router.use('/assets/*', createRouteHandler(req => req.user.directories.assets)); router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages)); router.use('/user/files/*', createRouteHandler(req => req.user.directories.files)); router.use('/scripts/extensions/third-party/*', createRouteHandler(req => req.user.directories.extensions));