rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-01 20:06:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Split user directories from public, part 1

Browse Source
This commit is contained in:
Cohee 2024-04-06 20:09:39 +03:00
parent b3b7017bf2
commit cd5aec7368
18 changed files with 406 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -45,3 +45,4 @@ access.log
/cache/
public/css/user.css
/plugins/
/data

0
data/.gitkeep Normal file
View File

2
default/config.yaml
View File

@ -1,4 +1,6 @@
# -- NETWORK CONFIGURATION --
# Root directory for user data storage
dataRoot: ./data
# Listen for incoming connections
listen: false
# Server port

4
default/content/index.json
View File

@ -1,4 +1,8 @@
[
{
"filename": "settings.json",
"type": "settings"
},
{
"filename": "themes/Dark Lite.json",
"type": "theme"

0
default/settings.json → default/content/settings.json
View File

12
index.d.ts vendored Normal file
View File

@ -0,0 +1,12 @@
import { UserDirectoryList, User } from "./src/users";
declare global {
namespace Express {
export interface Request {
user: {
profile: User;
directories: UserDirectoryList;
};
}
}
}

2
jsconfig.json
View File

@ -14,4 +14,4 @@
"node_modules",
"**/node_modules/*"
]
}
}

26
post-install.js
View File

@ -106,7 +106,6 @@ function addMissingConfigValues() {
*/
function createDefaultFiles() {
const files = {
settings: './public/settings.json',
config: './config.yaml',
user: './public/css/user.css',
};
@ -167,29 +166,6 @@ function copyWasmFiles() {
}
}
/**
* Moves the custom background into settings.json.
*/
function migrateBackground() {
if (!fs.existsSync('./public/css/bg_load.css')) return;
const bgCSS = fs.readFileSync('./public/css/bg_load.css', 'utf-8');
const bgMatch = /url\('([^']*)'\)/.exec(bgCSS);
if (!bgMatch) return;
const bgFilename = bgMatch[1].replace('../backgrounds/', '');
const settings = fs.readFileSync('./public/settings.json', 'utf-8');
const settingsJSON = JSON.parse(settings);
if (Object.hasOwn(settingsJSON, 'background')) {
console.log(color.yellow('Both bg_load.css and the "background" setting exist. Please delete bg_load.css manually.'));
return;
}
settingsJSON.background = { name: bgFilename, url: `url('backgrounds/${bgFilename}')` };
fs.writeFileSync('./public/settings.json', JSON.stringify(settingsJSON, null, 4));
fs.rmSync('./public/css/bg_load.css');
}
try {
// 0. Convert config.conf to config.yaml
convertConfig();
@ -199,8 +175,6 @@ try {
copyWasmFiles();
// 3. Add missing config values
addMissingConfigValues();
// 4. Migrate bg_load.css to settings.json
migrateBackground();
} catch (error) {
console.error(error);
}

41
server.js
View File

@ -33,6 +33,7 @@ util.inspect.defaultOptions.maxStringLength = null;
util.inspect.defaultOptions.depth = 4;
// local library imports
const { initUserStorage, userDataMiddleware, getUserDirectories, getAllUserHandles } = require('./src/users');
const basicAuthMiddleware = require('./src/middleware/basicAuth');
const whitelistMiddleware = require('./src/middleware/whitelist');
const contentManager = require('./src/endpoints/content-manager');
@ -112,7 +113,7 @@ const listen = cliArguments.listen ?? getConfigValue('listen', DEFAULT_LISTEN);
const enableCorsProxy = cliArguments.corsProxy ?? getConfigValue('enableCorsProxy', DEFAULT_CORS_PROXY);
const basicAuthMode = getConfigValue('basicAuthMode', false);
const { DIRECTORIES, UPLOADS_PATH } = require('./src/constants');
const { UPLOADS_PATH, PUBLIC_DIRECTORIES } = require('./src/constants');
// CORS Settings //
const CORS = cors({
@ -211,29 +212,8 @@ if (enableCorsProxy) {
}
app.use(express.static(process.cwd() + '/public', {}));
app.use(userDataMiddleware(app));
app.use('/backgrounds', (req, res) => {
const filePath = decodeURIComponent(path.join(process.cwd(), DIRECTORIES.backgrounds, req.url.replace(/%20/g, ' ')));
fs.readFile(filePath, (err, data) => {
if (err) {
res.status(404).send('File not found');
return;
}
//res.contentType('image/jpeg');
res.send(data);
});
});
app.use('/characters', (req, res) => {
const filePath = decodeURIComponent(path.join(process.cwd(), DIRECTORIES.characters, req.url.replace(/%20/g, ' ')));
fs.readFile(filePath, (err, data) => {
if (err) {
res.status(404).send('File not found');
return;
}
res.send(data);
});
});
app.use(multer({ dest: UPLOADS_PATH, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));
app.get('/', function (request, response) {
response.sendFile(process.cwd() + '/public/index.html');
@ -487,6 +467,7 @@ const setupTasks = async function () {
// TODO: do endpoint init functions depend on certain directories existing or not existing? They should be callable
// in any order for encapsulation reasons, but right now it's unknown if that would break anything.
await initUserStorage();
await settingsEndpoint.init();
ensurePublicDirectoriesExist();
contentManager.checkForNewContent();
@ -579,10 +560,20 @@ if (cliArguments.ssl) {
);
}
function ensurePublicDirectoriesExist() {
for (const dir of Object.values(DIRECTORIES)) {
async function ensurePublicDirectoriesExist() {
for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
const userHandles = await getAllUserHandles();
for (const handle of userHandles) {
const userDirectories = getUserDirectories(handle);
for (const dir of Object.values(userDirectories)) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
}
}
}

73
src/constants.js
View File

@ -1,34 +1,51 @@
const DIRECTORIES = {
worlds: 'public/worlds/',
user: 'public/user',
avatars: 'public/User Avatars',
const PUBLIC_DIRECTORIES = {
images: 'public/img/',
userImages: 'public/user/images/',
groups: 'public/groups/',
groupChats: 'public/group chats',
chats: 'public/chats/',
characters: 'public/characters/',
backgrounds: 'public/backgrounds',
novelAI_Settings: 'public/NovelAI Settings',
koboldAI_Settings: 'public/KoboldAI Settings',
openAI_Settings: 'public/OpenAI Settings',
textGen_Settings: 'public/TextGen Settings',
thumbnails: 'thumbnails/',
thumbnailsBg: 'thumbnails/bg/',
thumbnailsAvatar: 'thumbnails/avatar/',
themes: 'public/themes',
movingUI: 'public/movingUI',
extensions: 'public/scripts/extensions',
instruct: 'public/instruct',
context: 'public/context',
backups: 'backups/',
quickreplies: 'public/QuickReplies',
assets: 'public/assets',
comfyWorkflows: 'public/user/workflows',
files: 'public/user/files',
sounds: 'public/sounds',
};
/**
* @type {import('./users').UserDirectoryList}
* @readonly
* @enum {string}
*/
const USER_DIRECTORY_TEMPLATE = Object.freeze({
root: '',
thumbnails: 'thumbnails',
thumbnailsBg: 'thumbnails/bg',
thumbnailsAvatar: 'thumbnails/avatar',
worlds: 'worlds',
user: 'user',
avatars: 'User Avatars',
userImages: 'user/images',
groups: 'groups',
groupChats: 'group chats',
chats: 'chats',
characters: 'characters',
backgrounds: 'backgrounds',
novelAI_Settings: 'NovelAI Settings',
koboldAI_Settings: 'KoboldAI Settings',
openAI_Settings: 'OpenAI Settings',
textGen_Settings: 'TextGen Settings',
themes: 'themes',
movingUI: 'movingUI',
extensions: 'scripts/extensions',
instruct: 'instruct',
context: 'context',
quickreplies: 'QuickReplies',
assets: 'assets',
comfyWorkflows: 'user/workflows',
files: 'user/files',
});
const DEFAULT_USER = Object.freeze({
uuid: '00000000-0000-0000-0000-000000000000',
handle: 'user0',
name: 'User',
created: 0,
password: '',
});
const UNSAFE_EXTENSIONS = [
'.php',
'.exe',
@ -270,7 +287,9 @@ const OPENROUTER_KEYS = [
];
module.exports = {
DIRECTORIES,
DEFAULT_USER,
PUBLIC_DIRECTORIES,
USER_DIRECTORY_TEMPLATE,
UNSAFE_EXTENSIONS,
UPLOADS_PATH,
GEMINI_SAFETY,

154
src/endpoints/content-manager.js
View File

@ -6,17 +6,16 @@ const sanitize = require('sanitize-filename');
const { getConfigValue } = require('../util');
const { jsonParser } = require('../express-common');
const contentDirectory = path.join(process.cwd(), 'default/content');
const contentLogPath = path.join(contentDirectory, 'content.log');
const contentIndexPath = path.join(contentDirectory, 'index.json');
const { DIRECTORIES } = require('../constants');
const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings];
const { getAllUserHandles, getUserDirectories } = require('../users');
const characterCardParser = require('../character-card-parser.js');
/**
* Gets the default presets from the content directory.
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {object[]} Array of default presets
*/
function getDefaultPresets() {
function getDefaultPresets(directories) {
try {
const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8');
const contentIndex = JSON.parse(contentIndexText);
@ -26,7 +25,7 @@ function getDefaultPresets() {
for (const contentItem of contentIndex) {
if (contentItem.type.endsWith('_preset') || contentItem.type === 'instruct' || contentItem.type === 'context') {
contentItem.name = path.parse(contentItem.filename).name;
contentItem.folder = getTargetByType(contentItem.type);
contentItem.folder = getTargetByType(contentItem.type, directories);
presets.push(contentItem);
}
}
@ -59,120 +58,117 @@ function getDefaultPresetFile(filename) {
}
}
function migratePresets() {
for (const presetFolder of presetFolders) {
const presetPath = path.join(process.cwd(), presetFolder);
const presetFiles = fs.readdirSync(presetPath);
for (const presetFile of presetFiles) {
const presetFilePath = path.join(presetPath, presetFile);
const newFileName = presetFile.replace('.settings', '.json');
const newFilePath = path.join(presetPath, newFileName);
const backupFileName = presetFolder.replace('/', '_') + '_' + presetFile;
const backupFilePath = path.join(DIRECTORIES.backups, backupFileName);
if (presetFilePath.endsWith('.settings')) {
if (!fs.existsSync(newFilePath)) {
fs.cpSync(presetFilePath, backupFilePath);
fs.cpSync(presetFilePath, newFilePath);
console.log(`Migrated ${presetFilePath} to ${newFilePath}`);
}
}
}
}
}
function checkForNewContent() {
async function checkForNewContent() {
try {
migratePresets();
if (getConfigValue('skipContentCheck', false)) {
return;
}
const contentLog = getContentLog();
const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8');
const contentIndex = JSON.parse(contentIndexText);
const userHandles = await getAllUserHandles();
for (const contentItem of contentIndex) {
// If the content item is already in the log, skip it
if (contentLog.includes(contentItem.filename)) {
continue;
for (const userHandle of userHandles) {
const directories = getUserDirectories(userHandle);
if (!fs.existsSync(directories.root)) {
fs.mkdirSync(directories.root, { recursive: true });
}
contentLog.push(contentItem.filename);
const contentPath = path.join(contentDirectory, contentItem.filename);
const contentLogPath = path.join(directories.root, 'content.log');
const contentLog = getContentLog(contentLogPath);
if (!fs.existsSync(contentPath)) {
console.log(`Content file ${contentItem.filename} is missing`);
continue;
for (const contentItem of contentIndex) {
// If the content item is already in the log, skip it
if (contentLog.includes(contentItem.filename)) {
continue;
}
contentLog.push(contentItem.filename);
const contentPath = path.join(contentDirectory, contentItem.filename);
if (!fs.existsSync(contentPath)) {
console.log(`Content file ${contentItem.filename} is missing`);
continue;
}
const contentTarget = getTargetByType(contentItem.type, directories);
if (!contentTarget) {
console.log(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`);
continue;
}
const basePath = path.parse(contentItem.filename).base;
const targetPath = path.join(process.cwd(), contentTarget, basePath);
if (fs.existsSync(targetPath)) {
console.log(`Content file ${contentItem.filename} already exists in ${contentTarget}`);
continue;
}
fs.cpSync(contentPath, targetPath, { recursive: true, force: false });
console.log(`Content file ${contentItem.filename} copied to ${contentTarget}`);
}
const contentTarget = getTargetByType(contentItem.type);
if (!contentTarget) {
console.log(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`);
continue;
}
const basePath = path.parse(contentItem.filename).base;
const targetPath = path.join(process.cwd(), contentTarget, basePath);
if (fs.existsSync(targetPath)) {
console.log(`Content file ${contentItem.filename} already exists in ${contentTarget}`);
continue;
}
fs.cpSync(contentPath, targetPath, { recursive: true, force: false });
console.log(`Content file ${contentItem.filename} copied to ${contentTarget}`);
fs.writeFileSync(contentLogPath, contentLog.join('\n'));
}
fs.writeFileSync(contentLogPath, contentLog.join('\n'));
} catch (err) {
console.log('Content check failed', err);
}
}
function getTargetByType(type) {
/**
* Gets the target directory for the specified asset type.
* @param {string} type Asset type
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {string | null} Target directory
*/
function getTargetByType(type, directories) {
switch (type) {
case 'settings':
return directories.root;
case 'character':
return DIRECTORIES.characters;
return directories.characters;
case 'sprites':
return DIRECTORIES.characters;
return directories.characters;
case 'background':
return DIRECTORIES.backgrounds;
return directories.backgrounds;
case 'world':
return DIRECTORIES.worlds;
case 'sound':
return DIRECTORIES.sounds;
return directories.worlds;
case 'avatar':
return DIRECTORIES.avatars;
return directories.avatars;
case 'theme':
return DIRECTORIES.themes;
return directories.themes;
case 'workflow':
return DIRECTORIES.comfyWorkflows;
return directories.comfyWorkflows;
case 'kobold_preset':
return DIRECTORIES.koboldAI_Settings;
return directories.koboldAI_Settings;
case 'openai_preset':
return DIRECTORIES.openAI_Settings;
return directories.openAI_Settings;
case 'novel_preset':
return DIRECTORIES.novelAI_Settings;
return directories.novelAI_Settings;
case 'textgen_preset':
return DIRECTORIES.textGen_Settings;
return directories.textGen_Settings;
case 'instruct':
return DIRECTORIES.instruct;
return directories.instruct;
case 'context':
return DIRECTORIES.context;
return directories.context;
case 'moving_ui':
return DIRECTORIES.movingUI;
return directories.movingUI;
case 'quick_replies':
return DIRECTORIES.quickreplies;
return directories.quickreplies;
default:
return null;
}
}
function getContentLog() {
/**
* Gets the content log from the content log file.
* @param {string} contentLogPath Path to the content log file
* @returns {string[]} Array of content log lines
*/
function getContentLog(contentLogPath) {
if (!fs.existsSync(contentLogPath)) {
return [];
}

28
src/endpoints/presets.js
View File

@ -3,30 +3,30 @@ const path = require('path');
const express = require('express');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES } = require('../constants');
const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager');
const { jsonParser } = require('../express-common');
/**
* Gets the folder and extension for the preset settings based on the API source ID.
* @param {string} apiId API source ID
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {object} Object containing the folder and extension for the preset settings
*/
function getPresetSettingsByAPI(apiId) {
function getPresetSettingsByAPI(apiId, directories) {
switch (apiId) {
case 'kobold':
case 'koboldhorde':
return { folder: DIRECTORIES.koboldAI_Settings, extension: '.json' };
return { folder: directories.koboldAI_Settings, extension: '.json' };
case 'novel':
return { folder: DIRECTORIES.novelAI_Settings, extension: '.json' };
return { folder: directories.novelAI_Settings, extension: '.json' };
case 'textgenerationwebui':
return { folder: DIRECTORIES.textGen_Settings, extension: '.json' };
return { folder: directories.textGen_Settings, extension: '.json' };
case 'openai':
return { folder: DIRECTORIES.openAI_Settings, extension: '.json' };
return { folder: directories.openAI_Settings, extension: '.json' };
case 'instruct':
return { folder: DIRECTORIES.instruct, extension: '.json' };
return { folder: directories.instruct, extension: '.json' };
case 'context':
return { folder: DIRECTORIES.context, extension: '.json' };
return { folder: directories.context, extension: '.json' };
default:
return { folder: null, extension: null };
}
@ -40,7 +40,7 @@ router.post('/save', jsonParser, function (request, response) {
return response.sendStatus(400);
}
const settings = getPresetSettingsByAPI(request.body.apiId);
const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories);
const filename = name + settings.extension;
if (!settings.folder) {
@ -58,7 +58,7 @@ router.post('/delete', jsonParser, function (request, response) {
return response.sendStatus(400);
}
const settings = getPresetSettingsByAPI(request.body.apiId);
const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories);
const filename = name + settings.extension;
if (!settings.folder) {
@ -77,9 +77,9 @@ router.post('/delete', jsonParser, function (request, response) {
router.post('/restore', jsonParser, function (request, response) {
try {
const settings = getPresetSettingsByAPI(request.body.apiId);
const settings = getPresetSettingsByAPI(request.body.apiId, request.user.directories);
const name = sanitize(request.body.name);
const defaultPresets = getDefaultPresets();
const defaultPresets = getDefaultPresets(request.user.directories);
const defaultPreset = defaultPresets.find(p => p.name === name && p.folder === settings.folder);
@ -104,7 +104,7 @@ router.post('/save-openai', jsonParser, function (request, response) {
if (!name) return response.sendStatus(400);
const filename = `${name}.json`;
const fullpath = path.join(DIRECTORIES.openAI_Settings, filename);
const fullpath = path.join(request.user.directories.openAI_Settings, filename);
writeFileAtomicSync(fullpath, JSON.stringify(request.body, null, 4), 'utf-8');
return response.send({ name });
});
@ -116,7 +116,7 @@ router.post('/delete-openai', jsonParser, function (request, response) {
}
const name = request.body.name;
const pathToFile = path.join(DIRECTORIES.openAI_Settings, `${name}.json`);
const pathToFile = path.join(request.user.directories.openAI_Settings, `${name}.json`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);

59
src/endpoints/secrets.js
View File

@ -109,64 +109,6 @@ function readSecretState() {
return state;
}
/**
* Migrates secrets from settings.json to secrets.json
* @param {string} settingsFile Path to settings.json
* @returns {void}
*/
function migrateSecrets(settingsFile) {
const palmKey = readSecret('api_key_palm');
if (palmKey) {
console.log('Migrating Palm key...');
writeSecret(SECRET_KEYS.MAKERSUITE, palmKey);
deleteSecret('api_key_palm');
}
if (!fs.existsSync(settingsFile)) {
console.log('Settings file does not exist');
return;
}
try {
let modified = false;
const fileContents = fs.readFileSync(settingsFile, 'utf8');
const settings = JSON.parse(fileContents);
const oaiKey = settings?.api_key_openai;
const hordeKey = settings?.horde_settings?.api_key;
const novelKey = settings?.api_key_novel;
if (typeof oaiKey === 'string') {
console.log('Migrating OpenAI key...');
writeSecret(SECRET_KEYS.OPENAI, oaiKey);
delete settings.api_key_openai;
modified = true;
}
if (typeof hordeKey === 'string') {
console.log('Migrating Horde key...');
writeSecret(SECRET_KEYS.HORDE, hordeKey);
delete settings.horde_settings.api_key;
modified = true;
}
if (typeof novelKey === 'string') {
console.log('Migrating Novel key...');
writeSecret(SECRET_KEYS.NOVEL, novelKey);
delete settings.api_key_novel;
modified = true;
}
if (modified) {
console.log('Writing updated settings.json...');
const settingsContent = JSON.stringify(settings, null, 4);
writeFileAtomicSync(settingsFile, settingsContent, 'utf-8');
}
}
catch (error) {
console.error('Could not migrate secrets file. Proceed with caution.');
}
}
/**
* Reads all secrets from the secrets file
* @returns {Record<string, string> | undefined} Secrets
@ -251,7 +193,6 @@ module.exports = {
writeSecret,
readSecret,
readSecretState,
migrateSecrets,
getAllSecrets,
SECRET_KEYS,
router,

65
src/endpoints/settings.js
View File

@ -2,13 +2,13 @@ const fs = require('fs');
const path = require('path');
const express = require('express');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES } = require('../constants');
const { PUBLIC_DIRECTORIES } = require('../constants');
const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util');
const { jsonParser } = require('../express-common');
const { migrateSecrets } = require('./secrets');
const { getAllUserHandles, getUserDirectories } = require('../users');
const SETTINGS_FILE = 'settings.json';
const enableExtensions = getConfigValue('enableExtensions', true);
const SETTINGS_FILE = './public/settings.json';
function readAndParseFromDirectory(directoryPath, fileExtension = '.json') {
const files = fs
@ -61,16 +61,22 @@ function readPresetsFromDirectory(directoryPath, options = {}) {
return { fileContents, fileNames };
}
function backupSettings() {
async function backupSettings() {
try {
if (!fs.existsSync(DIRECTORIES.backups)) {
fs.mkdirSync(DIRECTORIES.backups);
if (!fs.existsSync(PUBLIC_DIRECTORIES.backups)) {
fs.mkdirSync(PUBLIC_DIRECTORIES.backups);
}
const backupFile = path.join(DIRECTORIES.backups, `settings_${generateTimestamp()}.json`);
fs.copyFileSync(SETTINGS_FILE, backupFile);
const userHandles = await getAllUserHandles();
removeOldBackups('settings_');
for (const handle of userHandles) {
const userDirectories = getUserDirectories(handle);
const backupFile = path.join(PUBLIC_DIRECTORIES.backups, `settings_${handle}_${generateTimestamp()}.json`);
const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);
fs.copyFileSync(sourceFile, backupFile);
removeOldBackups(`settings_${handle}`);
}
} catch (err) {
console.log('Could not backup settings file', err);
}
@ -80,7 +86,8 @@ const router = express.Router();
router.post('/save', jsonParser, function (request, response) {
try {
writeFileAtomicSync('public/settings.json', JSON.stringify(request.body, null, 4), 'utf8');
const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);
writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8');
response.send({ result: 'ok' });
} catch (err) {
console.log(err);
@ -92,48 +99,49 @@ router.post('/save', jsonParser, function (request, response) {
router.post('/get', jsonParser, (request, response) => {
let settings;
try {
settings = fs.readFileSync('public/settings.json', 'utf8');
const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);
settings = fs.readFileSync(pathToSettings, 'utf8');
} catch (e) {
return response.sendStatus(500);
}
// NovelAI Settings
const { fileContents: novelai_settings, fileNames: novelai_setting_names }
= readPresetsFromDirectory(DIRECTORIES.novelAI_Settings, {
sortFunction: sortByName(DIRECTORIES.novelAI_Settings),
= readPresetsFromDirectory(request.user.directories.novelAI_Settings, {
sortFunction: sortByName(request.user.directories.novelAI_Settings),
removeFileExtension: true,
});
// OpenAI Settings
const { fileContents: openai_settings, fileNames: openai_setting_names }
= readPresetsFromDirectory(DIRECTORIES.openAI_Settings, {
sortFunction: sortByName(DIRECTORIES.openAI_Settings), removeFileExtension: true,
= readPresetsFromDirectory(request.user.directories.openAI_Settings, {
sortFunction: sortByName(request.user.directories.openAI_Settings), removeFileExtension: true,
});
// TextGenerationWebUI Settings
const { fileContents: textgenerationwebui_presets, fileNames: textgenerationwebui_preset_names }
= readPresetsFromDirectory(DIRECTORIES.textGen_Settings, {
sortFunction: sortByName(DIRECTORIES.textGen_Settings), removeFileExtension: true,
= readPresetsFromDirectory(request.user.directories.textGen_Settings, {
sortFunction: sortByName(request.user.directories.textGen_Settings), removeFileExtension: true,
});
//Kobold
const { fileContents: koboldai_settings, fileNames: koboldai_setting_names }
= readPresetsFromDirectory(DIRECTORIES.koboldAI_Settings, {
sortFunction: sortByName(DIRECTORIES.koboldAI_Settings), removeFileExtension: true,
= readPresetsFromDirectory(request.user.directories.koboldAI_Settings, {
sortFunction: sortByName(request.user.directories.koboldAI_Settings), removeFileExtension: true,
});
const worldFiles = fs
.readdirSync(DIRECTORIES.worlds)
.readdirSync(request.user.directories.worlds)
.filter(file => path.extname(file).toLowerCase() === '.json')
.sort((a, b) => a.localeCompare(b));
const world_names = worldFiles.map(item => path.parse(item).name);
const themes = readAndParseFromDirectory(DIRECTORIES.themes);
const movingUIPresets = readAndParseFromDirectory(DIRECTORIES.movingUI);
const quickReplyPresets = readAndParseFromDirectory(DIRECTORIES.quickreplies);
const themes = readAndParseFromDirectory(request.user.directories.themes);
const movingUIPresets = readAndParseFromDirectory(request.user.directories.movingUI);
const quickReplyPresets = readAndParseFromDirectory(request.user.directories.quickreplies);
const instruct = readAndParseFromDirectory(DIRECTORIES.instruct);
const context = readAndParseFromDirectory(DIRECTORIES.context);
const instruct = readAndParseFromDirectory(request.user.directories.instruct);
const context = readAndParseFromDirectory(request.user.directories.context);
response.send({
settings,
@ -155,10 +163,11 @@ router.post('/get', jsonParser, (request, response) => {
});
});
// Sync for now, but should probably be migrated to async file APIs
/**
* Initializes the settings endpoint
*/
async function init() {
backupSettings();
migrateSecrets(SETTINGS_FILE);
await backupSettings();
}
module.exports = { router, init };

5
src/endpoints/themes.js
View File

@ -4,7 +4,6 @@ const fs = require('fs');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const router = express.Router();
@ -13,7 +12,7 @@ router.post('/save', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const filename = path.join(DIRECTORIES.themes, sanitize(request.body.name) + '.json');
const filename = path.join(request.user.directories.themes, sanitize(request.body.name) + '.json');
writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8');
return response.sendStatus(200);
@ -25,7 +24,7 @@ router.post('/delete', jsonParser, function (request, response) {
}
try {
const filename = path.join(DIRECTORIES.themes, sanitize(request.body.name) + '.json');
const filename = path.join(request.user.directories.themes, sanitize(request.body.name) + '.json');
if (!fs.existsSync(filename)) {
console.error('Theme file not found:', filename);
return response.sendStatus(404);

68
src/endpoints/thumbnails.js
View File

@ -4,24 +4,25 @@ const express = require('express');
const sanitize = require('sanitize-filename');
const jimp = require('jimp');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES } = require('../constants');
const { getAllUserHandles, getUserDirectories } = require('../users');
const { getConfigValue } = require('../util');
const { jsonParser } = require('../express-common');
/**
* Gets a path to thumbnail folder based on the type.
* @param {import('../users').UserDirectoryList} directories User directories
* @param {'bg' | 'avatar'} type Thumbnail type
* @returns {string} Path to the thumbnails folder
*/
function getThumbnailFolder(type) {
function getThumbnailFolder(directories, type) {
let thumbnailFolder;
switch (type) {
case 'bg':
thumbnailFolder = DIRECTORIES.thumbnailsBg;
thumbnailFolder = directories.thumbnailsBg;
break;
case 'avatar':
thumbnailFolder = DIRECTORIES.thumbnailsAvatar;
thumbnailFolder = directories.thumbnailsAvatar;
break;
}
@ -30,18 +31,19 @@ function getThumbnailFolder(type) {
/**
* Gets a path to the original images folder based on the type.
* @param {import('../users').UserDirectoryList} directories User directories
* @param {'bg' | 'avatar'} type Thumbnail type
* @returns {string} Path to the original images folder
*/
function getOriginalFolder(type) {
function getOriginalFolder(directories, type) {
let originalFolder;
switch (type) {
case 'bg':
originalFolder = DIRECTORIES.backgrounds;
originalFolder = directories.backgrounds;
break;
case 'avatar':
originalFolder = DIRECTORIES.characters;
originalFolder = directories.characters;
break;
}
@ -50,11 +52,12 @@ function getOriginalFolder(type) {
/**
* Removes the generated thumbnail from the disk.
* @param {import('../users').UserDirectoryList} directories User directories
* @param {'bg' | 'avatar'} type Type of the thumbnail
* @param {string} file Name of the file
*/
function invalidateThumbnail(type, file) {
const folder = getThumbnailFolder(type);
function invalidateThumbnail(directories, type, file) {
const folder = getThumbnailFolder(directories, type);
if (folder === undefined) throw new Error('Invalid thumbnail type');
const pathToThumbnail = path.join(folder, file);
@ -66,13 +69,14 @@ function invalidateThumbnail(type, file) {
/**
* Generates a thumbnail for the given file.
* @param {import('../users').UserDirectoryList} directories User directories
* @param {'bg' | 'avatar'} type Type of the thumbnail
* @param {string} file Name of the file
* @returns
*/
async function generateThumbnail(type, file) {
let thumbnailFolder = getThumbnailFolder(type);
let originalFolder = getOriginalFolder(type);
async function generateThumbnail(directories, type, file) {
let thumbnailFolder = getThumbnailFolder(directories, type);
let originalFolder = getOriginalFolder(directories, type);
if (thumbnailFolder === undefined || originalFolder === undefined) throw new Error('Invalid thumbnail type');
const pathToCachedFile = path.join(thumbnailFolder, file);
@ -133,24 +137,28 @@ async function generateThumbnail(type, file) {
* @returns {Promise<void>} Promise that resolves when the cache is validated
*/
async function ensureThumbnailCache() {
const cacheFiles = fs.readdirSync(DIRECTORIES.thumbnailsBg);
const userHandles = await getAllUserHandles();
for (const handle of userHandles) {
const directories = getUserDirectories(handle);
const cacheFiles = fs.readdirSync(directories.thumbnailsBg);
// files exist, all ok
if (cacheFiles.length) {
return;
// files exist, all ok
if (cacheFiles.length) {
return;
}
console.log('Generating thumbnails cache. Please wait...');
const bgFiles = fs.readdirSync(directories.backgrounds);
const tasks = [];
for (const file of bgFiles) {
tasks.push(generateThumbnail(directories, 'bg', file));
}
await Promise.all(tasks);
console.log(`Done! Generated: ${bgFiles.length} preview images`);
}
console.log('Generating thumbnails cache. Please wait...');
const bgFiles = fs.readdirSync(DIRECTORIES.backgrounds);
const tasks = [];
for (const file of bgFiles) {
tasks.push(generateThumbnail('bg', file));
}
await Promise.all(tasks);
console.log(`Done! Generated: ${bgFiles.length} preview images`);
}
const router = express.Router();
@ -176,13 +184,13 @@ router.get('/', jsonParser, async function (request, response) {
}
if (getConfigValue('disableThumbnails', false) == true) {
let folder = getOriginalFolder(type);
let folder = getOriginalFolder(request.user.directories, type);
if (folder === undefined) return response.sendStatus(400);
const pathToOriginalFile = path.join(folder, file);
return response.sendFile(pathToOriginalFile, { root: process.cwd() });
}
const pathToCachedFile = await generateThumbnail(type, file);
const pathToCachedFile = await generateThumbnail(request.user.directories, type, file);
if (!pathToCachedFile) {
return response.sendStatus(404);

155
src/users.js Normal file
View File

@ -0,0 +1,155 @@
const fsPromises = require('fs').promises;
const path = require('path');
const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants');
const { getConfigValue } = require('./util');
const DATA_ROOT = getConfigValue('dataRoot', './data');
/**
* @typedef {Object} User
* @property {string} uuid - The user's id
* @property {string} handle - The user's short handle. Used for directories and other references
* @property {string} name - The user's name. Displayed in the UI
* @property {number} created - The timestamp when the user was created
* @property {string} password - SHA256 hash of the user's password
*/
/**
* @typedef {Object} UserDirectoryList
* @property {string} root - The root directory for the user
* @property {string} thumbnails - The directory where the thumbnails are stored
* @property {string} thumbnailsBg - The directory where the background thumbnails are stored
* @property {string} thumbnailsAvatar - The directory where the avatar thumbnails are stored
* @property {string} worlds - The directory where the WI are stored
* @property {string} user - The directory where the user's public data is stored
* @property {string} avatars - The directory where the avatars are stored
* @property {string} userImages - The directory where the images are stored
* @property {string} groups - The directory where the groups are stored
* @property {string} groupChats - The directory where the group chats are stored
* @property {string} chats - The directory where the chats are stored
* @property {string} characters - The directory where the characters are stored
* @property {string} backgrounds - The directory where the backgrounds are stored
* @property {string} novelAI_Settings - The directory where the NovelAI settings are stored
* @property {string} koboldAI_Settings - The directory where the KoboldAI settings are stored
* @property {string} openAI_Settings - The directory where the OpenAI settings are stored
* @property {string} textGen_Settings - The directory where the TextGen settings are stored
* @property {string} themes - The directory where the themes are stored
* @property {string} movingUI - The directory where the moving UI data is stored
* @property {string} extensions - The directory where the extensions are stored
* @property {string} instruct - The directory where the instruct templates is stored
* @property {string} context - The directory where the context templates is stored
* @property {string} quickreplies - The directory where the quick replies are stored
* @property {string} assets - The directory where the assets are stored
* @property {string} comfyWorkflows - The directory where the ComfyUI workflows are stored
* @property {string} files - The directory where the uploaded files are stored
*/
/**
* Initializes the user storage. Currently a no-op.
* @returns {Promise<void>}
*/
async function initUserStorage() {
return Promise.resolve();
}
/**
* Gets a user for the current request. Hard coded to return the default user.
* @param {import('express').Request} _req - The request object. Currently unused.
* @returns {Promise<string>} - The user's handle
*/
async function getCurrentUserHandle(_req) {
return DEFAULT_USER.handle;
}
/**
* Gets a list of all user handles. Currently hard coded to return the default user's handle.
* @returns {Promise<string[]>} - The list of user handles
*/
async function getAllUserHandles() {
return [DEFAULT_USER.handle];
}
/**
* Gets the directories listing for the provided user.
* @param {import('express').Request} req - The request object
* @returns {Promise<UserDirectoryList>} - The user's directories like {worlds: 'data/user0/worlds/', ...
*/
async function getCurrentUserDirectories(req) {
const handle = await getCurrentUserHandle(req);
return getUserDirectories(handle);
}
/**
* Gets the directories listing for the provided user.
* @param {string} handle User handle
* @returns {UserDirectoryList} User directories
*/
function getUserDirectories(handle) {
const directories = structuredClone(USER_DIRECTORY_TEMPLATE);
for (const key in directories) {
directories[key] = path.join(DATA_ROOT, handle, USER_DIRECTORY_TEMPLATE[key]);
}
return directories;
}
/**
* Middleware to add user data to the request object.
* @param {import('express').Application} app - The express app
* @returns {import('express').RequestHandler}
*/
function userDataMiddleware(app) {
app.use('/backgrounds/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.backgrounds, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
app.use('/characters/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.characters, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
app.use('/User Avatars/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.avatars, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
/**
* Middleware to add user data to the request object.
* @param {import('express').Request} req Request object
* @param {import('express').Response} res Response object
* @param {import('express').NextFunction} next Next function
*/
return async (req, res, next) => {
const directories = await getCurrentUserDirectories(req);
req.user.profile = DEFAULT_USER;
req.user.directories = directories;
next();
};
}
module.exports = {
initUserStorage,
getCurrentUserDirectories,
getCurrentUserHandle,
getAllUserHandles,
getUserDirectories,
userDataMiddleware,
};

6
src/util.js
View File

@ -8,7 +8,7 @@ const yaml = require('yaml');
const { default: simpleGit } = require('simple-git');
const { Readable } = require('stream');
const { DIRECTORIES } = require('./constants');
const { PUBLIC_DIRECTORIES } = require('./constants');
/**
* Returns the config object from the config.yaml file.
@ -355,9 +355,9 @@ function generateTimestamp() {
function removeOldBackups(prefix) {
const MAX_BACKUPS = 25;
let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix));
let files = fs.readdirSync(PUBLIC_DIRECTORIES.backups).filter(f => f.startsWith(prefix));
if (files.length > MAX_BACKUPS) {
files = files.map(f => path.join(DIRECTORIES.backups, f));
files = files.map(f => path.join(PUBLIC_DIRECTORIES.backups, f));
files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
fs.rmSync(files[0]);