From 11193896b225a79eb017049c828eea63e0f23863 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 7 Apr 2024 03:01:55 +0300 Subject: [PATCH] Add data migration procedure --- public/KoboldAI Settings/.gitkeep | 0 public/NovelAI Settings/.gitkeep | 0 public/OpenAI Settings/.gitkeep | 0 public/QuickReplies/.gitkeep | 0 public/TextGen Settings/.gitkeep | 0 public/User Avatars/README.md | 1 - public/assets/ambient/.placeholder | 1 - public/assets/bgm/.placeholder | 1 - public/assets/blip/.placeholder | 1 - public/assets/live2d/.placeholder | 1 - public/assets/temp/.placeholder | 0 public/assets/vrm/animation/.placeholder | 1 - public/assets/vrm/model/.placeholder | 1 - public/backgrounds/.gitkeep | 0 public/characters/.gitkeep | 8 - public/chats/.gitkeep | 5 - public/context/.gitkeep | 0 public/group chats/.gitkeep | 1 - public/groups/.gitkeep | 1 - public/instruct/.gitkeep | 0 public/movingUI/.gitkeep | 0 public/themes/.gitkeep | 0 public/user/.gitkeep | 0 public/worlds/README.md | 1 - server.js | 9 +- src/users.js | 203 ++++++++++++++++++++++- 26 files changed, 208 insertions(+), 27 deletions(-) delete mode 100644 public/KoboldAI Settings/.gitkeep delete mode 100644 public/NovelAI Settings/.gitkeep delete mode 100644 public/OpenAI Settings/.gitkeep delete mode 100644 public/QuickReplies/.gitkeep delete mode 100644 public/TextGen Settings/.gitkeep delete mode 100644 public/User Avatars/README.md delete mode 100644 public/assets/ambient/.placeholder delete mode 100644 public/assets/bgm/.placeholder delete mode 100644 public/assets/blip/.placeholder delete mode 100644 public/assets/live2d/.placeholder delete mode 100644 public/assets/temp/.placeholder delete mode 100644 public/assets/vrm/animation/.placeholder delete mode 100644 public/assets/vrm/model/.placeholder delete mode 100644 public/backgrounds/.gitkeep delete mode 100644 public/characters/.gitkeep delete mode 100644 public/chats/.gitkeep delete mode 100644 public/context/.gitkeep delete mode 100644 public/group chats/.gitkeep delete mode 100644 public/groups/.gitkeep delete mode 100644 public/instruct/.gitkeep delete mode 100644 public/movingUI/.gitkeep delete mode 100644 public/themes/.gitkeep delete mode 100644 public/user/.gitkeep delete mode 100644 public/worlds/README.md diff --git a/public/KoboldAI Settings/.gitkeep b/public/KoboldAI Settings/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/NovelAI Settings/.gitkeep b/public/NovelAI Settings/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/OpenAI Settings/.gitkeep b/public/OpenAI Settings/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/QuickReplies/.gitkeep b/public/QuickReplies/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/TextGen Settings/.gitkeep b/public/TextGen Settings/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/User Avatars/README.md b/public/User Avatars/README.md deleted file mode 100644 index 17c7b8bff..000000000 --- a/public/User Avatars/README.md +++ /dev/null @@ -1 +0,0 @@ -# Put images here to select them as a user persona avatar. diff --git a/public/assets/ambient/.placeholder b/public/assets/ambient/.placeholder deleted file mode 100644 index a4faa1166..000000000 --- a/public/assets/ambient/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put ambient audio files here. \ No newline at end of file diff --git a/public/assets/bgm/.placeholder b/public/assets/bgm/.placeholder deleted file mode 100644 index 95839f44e..000000000 --- a/public/assets/bgm/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put bgm audio files here diff --git a/public/assets/blip/.placeholder b/public/assets/blip/.placeholder deleted file mode 100644 index 1ebb7122e..000000000 --- a/public/assets/blip/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put blip audio files here diff --git a/public/assets/live2d/.placeholder b/public/assets/live2d/.placeholder deleted file mode 100644 index c76c79ab1..000000000 --- a/public/assets/live2d/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put live2d model folders here diff --git a/public/assets/temp/.placeholder b/public/assets/temp/.placeholder deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/assets/vrm/animation/.placeholder b/public/assets/vrm/animation/.placeholder deleted file mode 100644 index c7a29571f..000000000 --- a/public/assets/vrm/animation/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put VRM animation files here diff --git a/public/assets/vrm/model/.placeholder b/public/assets/vrm/model/.placeholder deleted file mode 100644 index 14ce3cf88..000000000 --- a/public/assets/vrm/model/.placeholder +++ /dev/null @@ -1 +0,0 @@ -Put VRM model files here diff --git a/public/backgrounds/.gitkeep b/public/backgrounds/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/characters/.gitkeep b/public/characters/.gitkeep deleted file mode 100644 index 0fa956051..000000000 --- a/public/characters/.gitkeep +++ /dev/null @@ -1,8 +0,0 @@ -# Put PNG character cards here. - -To create a sprites folder, name it the same as your character (NOT the PNG file). - -For example: - -- Character: /characters/Asuka Langley.png -- Sprite: /characters/Asuka Langley/joy.png diff --git a/public/chats/.gitkeep b/public/chats/.gitkeep deleted file mode 100644 index ac488e9b9..000000000 --- a/public/chats/.gitkeep +++ /dev/null @@ -1,5 +0,0 @@ -# Put Chat JSONL files here in subfolders corresponding to character names - -For example: - -- /chats/Robot/chat.jsonl diff --git a/public/context/.gitkeep b/public/context/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/group chats/.gitkeep b/public/group chats/.gitkeep deleted file mode 100644 index 535c6dab6..000000000 --- a/public/group chats/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# Put Group Chat JSONL files here diff --git a/public/groups/.gitkeep b/public/groups/.gitkeep deleted file mode 100644 index 0f56f2da9..000000000 --- a/public/groups/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# Put Group JSON files here diff --git a/public/instruct/.gitkeep b/public/instruct/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/movingUI/.gitkeep b/public/movingUI/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/themes/.gitkeep b/public/themes/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/user/.gitkeep b/public/user/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/public/worlds/README.md b/public/worlds/README.md deleted file mode 100644 index 4e5570780..000000000 --- a/public/worlds/README.md +++ /dev/null @@ -1 +0,0 @@ -# Put World Info / Lorebook JSON files here diff --git a/server.js b/server.js index c9615896b..de2123cd2 100644 --- a/server.js +++ b/server.js @@ -33,7 +33,13 @@ util.inspect.defaultOptions.maxStringLength = null; util.inspect.defaultOptions.depth = 4; // local library imports -const { initUserStorage, userDataMiddleware, getUserDirectories, getAllUserHandles } = require('./src/users'); +const { + initUserStorage, + userDataMiddleware, + getUserDirectories, + getAllUserHandles, + migrateUserData, +} = require('./src/users'); const basicAuthMiddleware = require('./src/middleware/basicAuth'); const whitelistMiddleware = require('./src/middleware/whitelist'); const contentManager = require('./src/endpoints/content-manager'); @@ -471,6 +477,7 @@ const setupTasks = async function () { await initUserStorage(); await settingsEndpoint.init(); ensurePublicDirectoriesExist(); + await migrateUserData(); contentManager.checkForNewContent(); await ensureThumbnailCache(); cleanUploads(); diff --git a/src/users.js b/src/users.js index 10409e2f2..6c9a62e53 100644 --- a/src/users.js +++ b/src/users.js @@ -1,6 +1,7 @@ const path = require('path'); -const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants'); -const { getConfigValue } = require('./util'); +const fs = require('fs'); +const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES } = require('./constants'); +const { getConfigValue, color, delay } = require('./util'); const express = require('express'); const DATA_ROOT = getConfigValue('dataRoot', './data'); @@ -45,6 +46,201 @@ const DATA_ROOT = getConfigValue('dataRoot', './data'); * @property {string} vectors - The directory where the vectors are stored */ +/** + * Perform migration from the old user data format to the new one. + */ +async function migrateUserData() { + const publicDirectory = path.join(process.cwd(), 'public'); + + // No need to migrate if the characters directory doesn't exists + if (!fs.existsSync(path.join(publicDirectory, 'characters'))) { + return; + } + + const TIMEOUT = 10; + + console.log(); + console.log(color.magenta('Preparing to migrate user data...')); + console.log(`All public data will be moved to the ${DATA_ROOT} directory.`); + console.log('This process may take a while depending on the amount of data to move.'); + console.log(`Backups will be placed in the ${PUBLIC_DIRECTORIES.backups} directory.`); + console.log(`The process will start in ${TIMEOUT} seconds. Press Ctrl+C to cancel.`); + + for (let i = TIMEOUT; i > 0; i--) { + console.log(`${i}...`); + await delay(1000); + } + + console.log(color.magenta('Starting migration... Do not interrupt the process!')); + + const userDirectories = getUserDirectories(DEFAULT_USER.handle); + + const dataMigrationMap = [ + { + old: path.join(publicDirectory, 'assets'), + new: userDirectories.assets, + file: false, + }, + { + old: path.join(publicDirectory, 'backgrounds'), + new: userDirectories.backgrounds, + file: false, + }, + { + old: path.join(publicDirectory, 'characters'), + new: userDirectories.characters, + file: false, + }, + { + old: path.join(publicDirectory, 'chats'), + new: userDirectories.chats, + file: false, + }, + { + old: path.join(publicDirectory, 'context'), + new: userDirectories.context, + file: false, + }, + { + old: path.join(publicDirectory, 'group chats'), + new: userDirectories.groupChats, + file: false, + }, + { + old: path.join(publicDirectory, 'groups'), + new: userDirectories.groups, + file: false, + }, + { + old: path.join(publicDirectory, 'instruct'), + new: userDirectories.instruct, + file: false, + }, + { + old: path.join(publicDirectory, 'KoboldAI Settings'), + new: userDirectories.koboldAI_Settings, + file: false, + }, + { + old: path.join(publicDirectory, 'movingUI'), + new: userDirectories.movingUI, + file: false, + }, + { + old: path.join(publicDirectory, 'NovelAI Settings'), + new: userDirectories.novelAI_Settings, + file: false, + }, + { + old: path.join(publicDirectory, 'OpenAI Settings'), + new: userDirectories.openAI_Settings, + file: false, + }, + { + old: path.join(publicDirectory, 'QuickReplies'), + new: userDirectories.quickreplies, + file: false, + }, + { + old: path.join(publicDirectory, 'TextGen Settings'), + new: userDirectories.textGen_Settings, + file: false, + }, + { + old: path.join(publicDirectory, 'themes'), + new: userDirectories.themes, + file: false, + }, + { + old: path.join(publicDirectory, 'user'), + new: userDirectories.user, + file: false, + }, + { + old: path.join(publicDirectory, 'User Avatars'), + new: userDirectories.avatars, + file: false, + }, + { + old: path.join(publicDirectory, 'worlds'), + new: userDirectories.worlds, + file: false, + }, + { + old: path.join(publicDirectory, 'scripts/extensions/third-party'), + new: userDirectories.extensions, + file: false, + }, + { + old: path.join(process.cwd(), 'thumbnails'), + new: userDirectories.thumbnails, + file: false, + }, + { + old: path.join(process.cwd(), 'vectors'), + new: userDirectories.vectors, + file: false, + }, + { + old: path.join(process.cwd(), 'secrets.json'), + new: path.join(userDirectories.root, 'secrets.json'), + file: true, + }, + { + old: path.join(publicDirectory, 'settings.json'), + new: path.join(userDirectories.root, 'settings.json'), + file: true, + }, + { + old: path.join(publicDirectory, 'stats.json'), + new: path.join(userDirectories.root, 'stats.json'), + file: true, + }, + ]; + + const currentDate = new Date().toISOString().split('T')[0]; + const backupDirectory = path.join(process.cwd(), PUBLIC_DIRECTORIES.backups, '_migration', currentDate); + + if (!fs.existsSync(backupDirectory)) { + fs.mkdirSync(backupDirectory, { recursive: true }); + } + + const errors = []; + + for (const migration of dataMigrationMap) { + console.log(`Migrating ${migration.old} to ${migration.new}...`); + + try { + if (!fs.existsSync(migration.old)) { + console.log(color.yellow(`Skipping migration of ${migration.old} as it does not exist.`)); + continue; + } + + if (migration.file) { + // Copy the file to the new location + fs.cpSync(migration.old, migration.new, { force: true }); + // Move the file to the backup location + fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old))); + } else { + // Copy the directory to the new location + fs.cpSync(migration.old, migration.new, { recursive: true, force: true }); + // Move the directory to the backup location + fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old))); + } + } catch (error) { + console.error(color.red(`Error migrating ${migration.old} to ${migration.new}:`), error.message); + errors.push(migration.old); + } + } + + if (errors.length > 0) { + console.log(color.red('Migration completed with errors. Move the following files manually:')); + errors.forEach(error => console.error(error)); + } + + console.log(color.green('Migration completed!')); +} + /** * Initializes the user storage. Currently a no-op. * @returns {Promise} @@ -138,7 +334,7 @@ function createRouteHandler(directoryFn) { const router = express.Router(); router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds)); router.use('/characters/*', createRouteHandler(req => req.user.directories.characters)); -router.use('/User Avatars/*', createRouteHandler(req => req.user.directories.avatars)); +router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars)); router.use('/assets/*', createRouteHandler(req => req.user.directories.assets)); router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages)); router.use('/user/files/*', createRouteHandler(req => req.user.directories.files)); @@ -151,5 +347,6 @@ module.exports = { getAllUserHandles, getUserDirectories, userDataMiddleware, + migrateUserData, router, };