Add data migration procedure
This commit is contained in:
parent
b07a6a9a78
commit
11193896b2
|
@ -1 +0,0 @@
|
|||
# Put images here to select them as a user persona avatar.
|
|
@ -1 +0,0 @@
|
|||
Put ambient audio files here.
|
|
@ -1 +0,0 @@
|
|||
Put bgm audio files here
|
|
@ -1 +0,0 @@
|
|||
Put blip audio files here
|
|
@ -1 +0,0 @@
|
|||
Put live2d model folders here
|
|
@ -1 +0,0 @@
|
|||
Put VRM animation files here
|
|
@ -1 +0,0 @@
|
|||
Put VRM model files here
|
|
@ -1,8 +0,0 @@
|
|||
# Put PNG character cards here.
|
||||
|
||||
To create a sprites folder, name it the same as your character (NOT the PNG file).
|
||||
|
||||
For example:
|
||||
|
||||
- Character: /characters/Asuka Langley.png
|
||||
- Sprite: /characters/Asuka Langley/joy.png
|
|
@ -1,5 +0,0 @@
|
|||
# Put Chat JSONL files here in subfolders corresponding to character names
|
||||
|
||||
For example:
|
||||
|
||||
- /chats/Robot/chat.jsonl
|
|
@ -1 +0,0 @@
|
|||
# Put Group Chat JSONL files here
|
|
@ -1 +0,0 @@
|
|||
# Put Group JSON files here
|
|
@ -1 +0,0 @@
|
|||
# Put World Info / Lorebook JSON files here
|
|
@ -33,7 +33,13 @@ util.inspect.defaultOptions.maxStringLength = null;
|
|||
util.inspect.defaultOptions.depth = 4;
|
||||
|
||||
// local library imports
|
||||
const { initUserStorage, userDataMiddleware, getUserDirectories, getAllUserHandles } = require('./src/users');
|
||||
const {
|
||||
initUserStorage,
|
||||
userDataMiddleware,
|
||||
getUserDirectories,
|
||||
getAllUserHandles,
|
||||
migrateUserData,
|
||||
} = require('./src/users');
|
||||
const basicAuthMiddleware = require('./src/middleware/basicAuth');
|
||||
const whitelistMiddleware = require('./src/middleware/whitelist');
|
||||
const contentManager = require('./src/endpoints/content-manager');
|
||||
|
@ -471,6 +477,7 @@ const setupTasks = async function () {
|
|||
await initUserStorage();
|
||||
await settingsEndpoint.init();
|
||||
ensurePublicDirectoriesExist();
|
||||
await migrateUserData();
|
||||
contentManager.checkForNewContent();
|
||||
await ensureThumbnailCache();
|
||||
cleanUploads();
|
||||
|
|
203
src/users.js
203
src/users.js
|
@ -1,6 +1,7 @@
|
|||
const path = require('path');
|
||||
const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants');
|
||||
const { getConfigValue } = require('./util');
|
||||
const fs = require('fs');
|
||||
const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES } = require('./constants');
|
||||
const { getConfigValue, color, delay } = require('./util');
|
||||
const express = require('express');
|
||||
|
||||
const DATA_ROOT = getConfigValue('dataRoot', './data');
|
||||
|
@ -45,6 +46,201 @@ const DATA_ROOT = getConfigValue('dataRoot', './data');
|
|||
* @property {string} vectors - The directory where the vectors are stored
|
||||
*/
|
||||
|
||||
/**
|
||||
* Perform migration from the old user data format to the new one.
|
||||
*/
|
||||
async function migrateUserData() {
|
||||
const publicDirectory = path.join(process.cwd(), 'public');
|
||||
|
||||
// No need to migrate if the characters directory doesn't exists
|
||||
if (!fs.existsSync(path.join(publicDirectory, 'characters'))) {
|
||||
return;
|
||||
}
|
||||
|
||||
const TIMEOUT = 10;
|
||||
|
||||
console.log();
|
||||
console.log(color.magenta('Preparing to migrate user data...'));
|
||||
console.log(`All public data will be moved to the ${DATA_ROOT} directory.`);
|
||||
console.log('This process may take a while depending on the amount of data to move.');
|
||||
console.log(`Backups will be placed in the ${PUBLIC_DIRECTORIES.backups} directory.`);
|
||||
console.log(`The process will start in ${TIMEOUT} seconds. Press Ctrl+C to cancel.`);
|
||||
|
||||
for (let i = TIMEOUT; i > 0; i--) {
|
||||
console.log(`${i}...`);
|
||||
await delay(1000);
|
||||
}
|
||||
|
||||
console.log(color.magenta('Starting migration... Do not interrupt the process!'));
|
||||
|
||||
const userDirectories = getUserDirectories(DEFAULT_USER.handle);
|
||||
|
||||
const dataMigrationMap = [
|
||||
{
|
||||
old: path.join(publicDirectory, 'assets'),
|
||||
new: userDirectories.assets,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'backgrounds'),
|
||||
new: userDirectories.backgrounds,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'characters'),
|
||||
new: userDirectories.characters,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'chats'),
|
||||
new: userDirectories.chats,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'context'),
|
||||
new: userDirectories.context,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'group chats'),
|
||||
new: userDirectories.groupChats,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'groups'),
|
||||
new: userDirectories.groups,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'instruct'),
|
||||
new: userDirectories.instruct,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'KoboldAI Settings'),
|
||||
new: userDirectories.koboldAI_Settings,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'movingUI'),
|
||||
new: userDirectories.movingUI,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'NovelAI Settings'),
|
||||
new: userDirectories.novelAI_Settings,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'OpenAI Settings'),
|
||||
new: userDirectories.openAI_Settings,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'QuickReplies'),
|
||||
new: userDirectories.quickreplies,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'TextGen Settings'),
|
||||
new: userDirectories.textGen_Settings,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'themes'),
|
||||
new: userDirectories.themes,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'user'),
|
||||
new: userDirectories.user,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'User Avatars'),
|
||||
new: userDirectories.avatars,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'worlds'),
|
||||
new: userDirectories.worlds,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'scripts/extensions/third-party'),
|
||||
new: userDirectories.extensions,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(process.cwd(), 'thumbnails'),
|
||||
new: userDirectories.thumbnails,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(process.cwd(), 'vectors'),
|
||||
new: userDirectories.vectors,
|
||||
file: false,
|
||||
},
|
||||
{
|
||||
old: path.join(process.cwd(), 'secrets.json'),
|
||||
new: path.join(userDirectories.root, 'secrets.json'),
|
||||
file: true,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'settings.json'),
|
||||
new: path.join(userDirectories.root, 'settings.json'),
|
||||
file: true,
|
||||
},
|
||||
{
|
||||
old: path.join(publicDirectory, 'stats.json'),
|
||||
new: path.join(userDirectories.root, 'stats.json'),
|
||||
file: true,
|
||||
},
|
||||
];
|
||||
|
||||
const currentDate = new Date().toISOString().split('T')[0];
|
||||
const backupDirectory = path.join(process.cwd(), PUBLIC_DIRECTORIES.backups, '_migration', currentDate);
|
||||
|
||||
if (!fs.existsSync(backupDirectory)) {
|
||||
fs.mkdirSync(backupDirectory, { recursive: true });
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
|
||||
for (const migration of dataMigrationMap) {
|
||||
console.log(`Migrating ${migration.old} to ${migration.new}...`);
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(migration.old)) {
|
||||
console.log(color.yellow(`Skipping migration of ${migration.old} as it does not exist.`));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (migration.file) {
|
||||
// Copy the file to the new location
|
||||
fs.cpSync(migration.old, migration.new, { force: true });
|
||||
// Move the file to the backup location
|
||||
fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old)));
|
||||
} else {
|
||||
// Copy the directory to the new location
|
||||
fs.cpSync(migration.old, migration.new, { recursive: true, force: true });
|
||||
// Move the directory to the backup location
|
||||
fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old)));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(color.red(`Error migrating ${migration.old} to ${migration.new}:`), error.message);
|
||||
errors.push(migration.old);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log(color.red('Migration completed with errors. Move the following files manually:'));
|
||||
errors.forEach(error => console.error(error));
|
||||
}
|
||||
|
||||
console.log(color.green('Migration completed!'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the user storage. Currently a no-op.
|
||||
* @returns {Promise<void>}
|
||||
|
@ -138,7 +334,7 @@ function createRouteHandler(directoryFn) {
|
|||
const router = express.Router();
|
||||
router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds));
|
||||
router.use('/characters/*', createRouteHandler(req => req.user.directories.characters));
|
||||
router.use('/User Avatars/*', createRouteHandler(req => req.user.directories.avatars));
|
||||
router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars));
|
||||
router.use('/assets/*', createRouteHandler(req => req.user.directories.assets));
|
||||
router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages));
|
||||
router.use('/user/files/*', createRouteHandler(req => req.user.directories.files));
|
||||
|
@ -151,5 +347,6 @@ module.exports = {
|
|||
getAllUserHandles,
|
||||
getUserDirectories,
|
||||
userDataMiddleware,
|
||||
migrateUserData,
|
||||
router,
|
||||
};
|
||||
|
|
Loading…
Reference in New Issue