Add data migration procedure

public/User Avatars/README.md

@@ -1 +0,0 @@
-# Put images here to select them as a user persona avatar.

public/assets/ambient/.placeholder

@@ -1 +0,0 @@
-Put ambient audio files here.

public/assets/bgm/.placeholder

@@ -1 +0,0 @@
-Put bgm audio files here

public/assets/blip/.placeholder

@@ -1 +0,0 @@
-Put blip audio files here

public/assets/live2d/.placeholder

@@ -1 +0,0 @@
-Put live2d model folders here

public/assets/vrm/animation/.placeholder

@@ -1 +0,0 @@
-Put VRM animation files here

public/assets/vrm/model/.placeholder

@@ -1 +0,0 @@
-Put VRM model files here

public/characters/.gitkeep

@@ -1,8 +0,0 @@
-# Put PNG character cards here.
-
-To create a sprites folder, name it the same as your character (NOT the PNG file).
-
-For example:
-
-- Character: /characters/Asuka Langley.png
-- Sprite: /characters/Asuka Langley/joy.png

public/chats/.gitkeep

@@ -1,5 +0,0 @@
-# Put Chat JSONL files here in subfolders corresponding to character names
-
-For example:
-
-- /chats/Robot/chat.jsonl

public/group chats/.gitkeep

@@ -1 +0,0 @@
-# Put Group Chat JSONL files here

public/groups/.gitkeep

@@ -1 +0,0 @@
-# Put Group JSON files here

public/worlds/README.md

@@ -1 +0,0 @@
-# Put World Info / Lorebook JSON files here

server.js

@@ -33,7 +33,13 @@ util.inspect.defaultOptions.maxStringLength = null;
 util.inspect.defaultOptions.depth = 4;
 
 // local library imports
-const { initUserStorage, userDataMiddleware, getUserDirectories, getAllUserHandles } = require('./src/users');
+const {
+    initUserStorage,
+    userDataMiddleware,
+    getUserDirectories,
+    getAllUserHandles,
+    migrateUserData,
+} = require('./src/users');
 const basicAuthMiddleware = require('./src/middleware/basicAuth');
 const whitelistMiddleware = require('./src/middleware/whitelist');
 const contentManager = require('./src/endpoints/content-manager');
@@ -471,6 +477,7 @@ const setupTasks = async function () {
     await initUserStorage();
     await settingsEndpoint.init();
     ensurePublicDirectoriesExist();
+    await migrateUserData();
     contentManager.checkForNewContent();
     await ensureThumbnailCache();
     cleanUploads();

src/users.js

@@ -1,6 +1,7 @@
 const path = require('path');
-const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants');
+const fs = require('fs');
-const { getConfigValue } = require('./util');
+const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES } = require('./constants');
+const { getConfigValue, color, delay } = require('./util');
 const express = require('express');
 
 const DATA_ROOT = getConfigValue('dataRoot', './data');
@@ -45,6 +46,201 @@ const DATA_ROOT = getConfigValue('dataRoot', './data');
  * @property {string} vectors - The directory where the vectors are stored
  */
 
+/**
+ * Perform migration from the old user data format to the new one.
+ */
+async function migrateUserData() {
+    const publicDirectory = path.join(process.cwd(), 'public');
+
+    // No need to migrate if the characters directory doesn't exists
+    if (!fs.existsSync(path.join(publicDirectory, 'characters'))) {
+        return;
+    }
+
+    const TIMEOUT = 10;
+
+    console.log();
+    console.log(color.magenta('Preparing to migrate user data...'));
+    console.log(`All public data will be moved to the ${DATA_ROOT} directory.`);
+    console.log('This process may take a while depending on the amount of data to move.');
+    console.log(`Backups will be placed in the ${PUBLIC_DIRECTORIES.backups} directory.`);
+    console.log(`The process will start in ${TIMEOUT} seconds. Press Ctrl+C to cancel.`);
+
+    for (let i = TIMEOUT; i > 0; i--) {
+        console.log(`${i}...`);
+        await delay(1000);
+    }
+
+    console.log(color.magenta('Starting migration... Do not interrupt the process!'));
+
+    const userDirectories = getUserDirectories(DEFAULT_USER.handle);
+
+    const dataMigrationMap = [
+        {
+            old: path.join(publicDirectory, 'assets'),
+            new: userDirectories.assets,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'backgrounds'),
+            new: userDirectories.backgrounds,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'characters'),
+            new: userDirectories.characters,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'chats'),
+            new: userDirectories.chats,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'context'),
+            new: userDirectories.context,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'group chats'),
+            new: userDirectories.groupChats,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'groups'),
+            new: userDirectories.groups,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'instruct'),
+            new: userDirectories.instruct,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'KoboldAI Settings'),
+            new: userDirectories.koboldAI_Settings,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'movingUI'),
+            new: userDirectories.movingUI,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'NovelAI Settings'),
+            new: userDirectories.novelAI_Settings,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'OpenAI Settings'),
+            new: userDirectories.openAI_Settings,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'QuickReplies'),
+            new: userDirectories.quickreplies,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'TextGen Settings'),
+            new: userDirectories.textGen_Settings,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'themes'),
+            new: userDirectories.themes,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'user'),
+            new: userDirectories.user,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'User Avatars'),
+            new: userDirectories.avatars,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'worlds'),
+            new: userDirectories.worlds,
+            file: false,
+        },
+        {
+            old: path.join(publicDirectory, 'scripts/extensions/third-party'),
+            new: userDirectories.extensions,
+            file: false,
+        },
+        {
+            old: path.join(process.cwd(), 'thumbnails'),
+            new: userDirectories.thumbnails,
+            file: false,
+        },
+        {
+            old: path.join(process.cwd(), 'vectors'),
+            new: userDirectories.vectors,
+            file: false,
+        },
+        {
+            old: path.join(process.cwd(), 'secrets.json'),
+            new: path.join(userDirectories.root, 'secrets.json'),
+            file: true,
+        },
+        {
+            old: path.join(publicDirectory, 'settings.json'),
+            new: path.join(userDirectories.root, 'settings.json'),
+            file: true,
+        },
+        {
+            old: path.join(publicDirectory, 'stats.json'),
+            new: path.join(userDirectories.root, 'stats.json'),
+            file: true,
+        },
+    ];
+
+    const currentDate = new Date().toISOString().split('T')[0];
+    const backupDirectory = path.join(process.cwd(), PUBLIC_DIRECTORIES.backups, '_migration', currentDate);
+
+    if (!fs.existsSync(backupDirectory)) {
+        fs.mkdirSync(backupDirectory, { recursive: true });
+    }
+
+    const errors = [];
+
+    for (const migration of dataMigrationMap) {
+        console.log(`Migrating ${migration.old} to ${migration.new}...`);
+
+        try {
+            if (!fs.existsSync(migration.old)) {
+                console.log(color.yellow(`Skipping migration of ${migration.old} as it does not exist.`));
+                continue;
+            }
+
+            if (migration.file) {
+                // Copy the file to the new location
+                fs.cpSync(migration.old, migration.new, { force: true });
+                // Move the file to the backup location
+                fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old)));
+            } else {
+                // Copy the directory to the new location
+                fs.cpSync(migration.old, migration.new, { recursive: true, force: true });
+                // Move the directory to the backup location
+                fs.renameSync(migration.old, path.join(backupDirectory, path.basename(migration.old)));
+            }
+        } catch (error) {
+            console.error(color.red(`Error migrating ${migration.old} to ${migration.new}:`), error.message);
+            errors.push(migration.old);
+        }
+    }
+
+    if (errors.length > 0) {
+        console.log(color.red('Migration completed with errors. Move the following files manually:'));
+        errors.forEach(error => console.error(error));
+    }
+
+    console.log(color.green('Migration completed!'));
+}
+
 /**
  * Initializes the user storage. Currently a no-op.
  * @returns {Promise<void>}
@@ -138,7 +334,7 @@ function createRouteHandler(directoryFn) {
 const router = express.Router();
 router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds));
 router.use('/characters/*', createRouteHandler(req => req.user.directories.characters));
-router.use('/User Avatars/*', createRouteHandler(req => req.user.directories.avatars));
+router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars));
 router.use('/assets/*', createRouteHandler(req => req.user.directories.assets));
 router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages));
 router.use('/user/files/*', createRouteHandler(req => req.user.directories.files));
@@ -151,5 +347,6 @@ module.exports = {
     getAllUserHandles,
     getUserDirectories,
     userDataMiddleware,
+    migrateUserData,
     router,
 };