Merge branch 'staging' into feat/slash-commands-missing-args

public/script.js

@@ -1249,7 +1249,7 @@ async function getCharacters() {
 }
 
 async function delChat(chatfile) {
-    const response = await fetch('/delchat', {
+    const response = await fetch('/api/chats/delete', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({
@@ -3883,7 +3883,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
                         toastr.error(data.response, 'API Error');
                     }
                 }
-                console.debug('/savechat called by /Generate');
+                console.debug('/api/chats/save called by /Generate');
 
                 await saveChatConditional();
                 is_send_press = false;
@@ -4903,7 +4903,7 @@ async function renamePastChats(newAvatar, newValue) {
     for (const { file_name } of pastChats) {
         try {
             const fileNameWithoutExtension = file_name.replace('.jsonl', '');
-            const getChatResponse = await fetch('/getchat', {
+            const getChatResponse = await fetch('/api/chats/get', {
                 method: 'POST',
                 headers: getRequestHeaders(),
                 body: JSON.stringify({
@@ -4927,7 +4927,7 @@ async function renamePastChats(newAvatar, newValue) {
                     }
                 }
 
-                const saveChatResponse = await fetch('/savechat', {
+                const saveChatResponse = await fetch('/api/chats/save', {
                     method: 'POST',
                     headers: getRequestHeaders(),
                     body: JSON.stringify({
@@ -5018,7 +5018,7 @@ async function saveChat(chat_name, withMetadata, mesId) {
     ];
     return jQuery.ajax({
         type: 'POST',
-        url: '/savechat',
+        url: '/api/chats/save',
         data: JSON.stringify({
             ch_name: characters[this_chid].name,
             file_name: file_name,
@@ -5110,11 +5110,11 @@ function getThumbnailUrl(type, file) {
 }
 
 async function getChat() {
-    //console.log('/getchat -- entered for -- ' + characters[this_chid].name);
+    //console.log('/api/chats/get -- entered for -- ' + characters[this_chid].name);
     try {
         const response = await $.ajax({
             type: 'POST',
-            url: '/getchat',
+            url: '/api/chats/get',
             data: JSON.stringify({
                 ch_name: characters[this_chid].name,
                 file_name: characters[this_chid].chat,
@@ -5875,7 +5875,7 @@ export async function getChatsFromFiles(data, isGroupChat) {
     let chat_promise = chat_list.map(({ file_name }) => {
         return new Promise(async (res, rej) => {
             try {
-                const endpoint = isGroupChat ? '/getgroupchat' : '/getchat';
+                const endpoint = isGroupChat ? '/api/chats/group/get' : '/api/chats/get';
                 const requestBody = isGroupChat
                     ? JSON.stringify({ id: file_name })
                     : JSON.stringify({
@@ -6562,7 +6562,7 @@ export async function saveChatConditional() {
 async function importCharacterChat(formData) {
     await jQuery.ajax({
         type: 'POST',
-        url: '/importchat',
+        url: '/api/chats/import',
         data: formData,
         beforeSend: function () {
         },
@@ -8161,7 +8161,8 @@ jQuery(async function () {
         };
 
         try {
-            const response = await fetch('/renamechat', {
+            showLoader();
+            const response = await fetch('/api/chats/rename', {
                 method: 'POST',
                 body: JSON.stringify(body),
                 headers: getRequestHeaders(),
@@ -8193,8 +8194,11 @@ jQuery(async function () {
             $('#option_select_chat').trigger('click');
             $('#options').hide();
         } catch {
+            hideLoader();
             await delay(500);
             await callPopup('An error has occurred. Chat was not renamed.', 'text');
+        } finally {
+            hideLoader();
         }
     });
 
@@ -8215,7 +8219,7 @@ jQuery(async function () {
         };
         console.log(body);
         try {
-            const response = await fetch('/exportchat', {
+            const response = await fetch('/api/chats/export', {
                 method: 'POST',
                 body: JSON.stringify(body),
                 headers: getRequestHeaders(),

public/scripts/bookmarks.js

@@ -251,7 +251,7 @@ async function convertSoloToGroupChat() {
     const metadata = Object.assign({}, chat_metadata);
     delete metadata.main_chat;
 
-    const createGroupResponse = await fetch('/creategroup', {
+    const createGroupResponse = await fetch('/api/groups/create', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({
@@ -320,7 +320,7 @@ async function convertSoloToGroupChat() {
     }
 
     // Save group chat
-    const createChatResponse = await fetch('/savegroupchat', {
+    const createChatResponse = await fetch('/api/chats/group/save', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: chatName, chat: groupChat }),

public/scripts/group-chats.js

@@ -116,7 +116,7 @@ setInterval(groupChatAutoModeWorker, 5000);
 const saveGroupDebounced = debounce(async (group, reload) => await _save(group, reload), 500);
 
 async function _save(group, reload = true) {
-    await fetch('/editgroup', {
+    await fetch('/api/groups/edit', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify(group),
@@ -152,7 +152,7 @@ async function regenerateGroup() {
 }
 
 async function loadGroupChat(chatId) {
-    const response = await fetch('/getgroupchat', {
+    const response = await fetch('/api/chats/group/get', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: chatId }),
@@ -401,7 +401,7 @@ async function saveGroupChat(groupId, shouldSaveGroup) {
     const group = groups.find(x => x.id == groupId);
     const chat_id = group.chat_id;
     group['date_last_chat'] = Date.now();
-    const response = await fetch('/savegroupchat', {
+    const response = await fetch('/api/chats/group/save', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: chat_id, chat: [...chat] }),
@@ -455,7 +455,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
                     }
 
                     if (hadChanges) {
-                        const saveChatResponse = await fetch('/savegroupchat', {
+                        const saveChatResponse = await fetch('/api/chats/group/save', {
                             method: 'POST',
                             headers: getRequestHeaders(),
                             body: JSON.stringify({ id: chatId, chat: [...messages] }),
@@ -476,7 +476,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
 }
 
 async function getGroups() {
-    const response = await fetch('/getgroups', {
+    const response = await fetch('/api/groups/all', {
         method: 'POST',
         headers: getRequestHeaders(),
     });
@@ -968,7 +968,7 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i
 async function deleteGroup(id) {
     const group = groups.find((x) => x.id === id);
 
-    const response = await fetch('/deletegroup', {
+    const response = await fetch('/api/groups/delete', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: id }),
@@ -1521,7 +1521,7 @@ async function createGroup() {
     const chatName = humanizedDateTime();
     const chats = [chatName];
 
-    const createGroupResponse = await fetch('/creategroup', {
+    const createGroupResponse = await fetch('/api/groups/create', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({
@@ -1659,7 +1659,7 @@ export async function deleteGroupChat(groupId, chatId) {
     delete group.past_metadata[chatId];
     updateChatMetadata(group.chat_metadata, true);
 
-    const response = await fetch('/deletegroupchat', {
+    const response = await fetch('/api/chats/group/delete', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: chatId }),
@@ -1679,7 +1679,7 @@ export async function deleteGroupChat(groupId, chatId) {
 export async function importGroupChat(formData) {
     await jQuery.ajax({
         type: 'POST',
-        url: '/importgroupchat',
+        url: '/api/chats/group/import',
         data: formData,
         beforeSend: function () {
         },
@@ -1720,7 +1720,7 @@ export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) {
 
     await editGroup(groupId, true, false);
 
-    await fetch('/savegroupchat', {
+    await fetch('/api/chats/group/save', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ id: name, chat: [...trimmed_chat] }),

public/scripts/power-user.js

@@ -813,6 +813,7 @@ async function CreateZenSliders(elmnt) {
             if (numVal === offVal) { handle.text('Off').css('color', 'rgba(128,128,128,0.5'); }
             else if (numVal === allVal) { handle.text('All'); }
             else { handle.css('color', ''); }
+            numVal = steps[stepNumber];
         }
         //everything else uses the flat slider value
         //also note: the above sliders are not custom inputtable due to the array aliasing

public/scripts/world-info.js

@@ -424,7 +424,7 @@ async function loadWorldInfoData(name) {
         return worldInfoCache[name];
     }
 
-    const response = await fetch('/getworldinfo', {
+    const response = await fetch('/api/worldinfo/get', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ name: name }),
@@ -1402,7 +1402,7 @@ function createWorldInfoEntry(name, data, fromSlashCommand = false) {
 }
 
 async function _save(name, data) {
-    await fetch('/editworldinfo', {
+    await fetch('/api/worldinfo/edit', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ name: name, data: data }),
@@ -1464,7 +1464,7 @@ async function deleteWorldInfo(worldInfoName) {
         return;
     }
 
-    const response = await fetch('/deleteworldinfo', {
+    const response = await fetch('/api/worldinfo/delete', {
         method: 'POST',
         headers: getRequestHeaders(),
         body: JSON.stringify({ name: worldInfoName }),
@@ -2269,7 +2269,7 @@ export async function importWorldInfo(file) {
 
     jQuery.ajax({
         type: 'POST',
-        url: '/importworldinfo',
+        url: '/api/worldinfo/import',
         data: formData,
         beforeSend: () => { },
         cache: false,

server.js

@@ -6,7 +6,6 @@ const fs = require('fs');
 const http = require('http');
 const https = require('https');
 const path = require('path');
-const readline = require('readline');
 const util = require('util');
 const { Readable } = require('stream');
 
@@ -48,11 +47,10 @@ const { jsonParser, urlencodedParser } = require('./src/express-common.js');
 const contentManager = require('./src/endpoints/content-manager');
 const statsHelpers = require('./statsHelpers.js');
 const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets');
-const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension } = require('./src/util');
+const { delay, getVersion, getConfigValue, color, uuidv4, tryParse, clientRelativePath, removeFileExtension, generateTimestamp, removeOldBackups } = require('./src/util');
 const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails');
 const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers');
 const { convertClaudePrompt } = require('./src/chat-completion');
-const { readWorldInfoFile } = require('./src/worldinfo');
 
 // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
 // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
@@ -723,55 +721,6 @@ app.post('/api/textgenerationwebui/generate', jsonParser, async function (reques
     }
 });
 
-app.post('/savechat', jsonParser, function (request, response) {
-    try {
-        var dir_name = String(request.body.avatar_url).replace('.png', '');
-        let chat_data = request.body.chat;
-        let jsonlData = chat_data.map(JSON.stringify).join('\n');
-        writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8');
-        backupChat(dir_name, jsonlData);
-        return response.send({ result: 'ok' });
-    } catch (error) {
-        response.send(error);
-        return console.log(error);
-    }
-});
-
-app.post('/getchat', jsonParser, function (request, response) {
-    try {
-        const dirName = String(request.body.avatar_url).replace('.png', '');
-        const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName);
-
-        //if no chat dir for the character is found, make one with the character name
-        if (!chatDirExists) {
-            fs.mkdirSync(DIRECTORIES.chats + dirName);
-            return response.send({});
-        }
-
-
-        if (!request.body.file_name) {
-            return response.send({});
-        }
-
-        const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`;
-        const chatFileExists = fs.existsSync(fileName);
-
-        if (!chatFileExists) {
-            return response.send({});
-        }
-
-        const data = fs.readFileSync(fileName, 'utf8');
-        const lines = data.split('\n');
-
-        // Iterate through the array of strings and parse each line as JSON
-        const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
-        return response.send(jsonData);
-    } catch (error) {
-        console.error(error);
-        return response.send({});
-    }
-});
-
 // Only called for kobold
 app.post('/getstatus', jsonParser, async function (request, response) {
     if (!request.body) return response.sendStatus(400);
@@ -829,30 +778,6 @@ app.post('/getstatus', jsonParser, async function (request, response) {
     }
 });
 
-
-app.post('/renamechat', jsonParser, async function (request, response) {
-    if (!request.body || !request.body.original_file || !request.body.renamed_file) {
-        return response.sendStatus(400);
-    }
-
-    const pathToFolder = request.body.is_group
-        ? DIRECTORIES.groupChats
-        : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
-    const pathToOriginalFile = path.join(pathToFolder, request.body.original_file);
-    const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file);
-    console.log('Old chat name', pathToOriginalFile);
-    console.log('New chat name', pathToRenamedFile);
-
-    if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {
-        console.log('Either Source or Destination files are not available');
-        return response.status(400).send({ error: true });
-    }
-
-    console.log('Successfully renamed.');
-    fs.renameSync(pathToOriginalFile, pathToRenamedFile);
-    return response.send({ ok: true });
-});
-
 /**
  * Handle a POST request to get the stats object
  *
@@ -967,37 +892,6 @@ app.post('/delbackground', jsonParser, function (request, response) {
     return response.send('ok');
 });
 
-app.post('/delchat', jsonParser, function (request, response) {
-    console.log('/delchat entered');
-    if (!request.body) {
-        console.log('no request body seen');
-        return response.sendStatus(400);
-    }
-
-    if (request.body.chatfile !== sanitize(request.body.chatfile)) {
-        console.error('Malicious chat name prevented');
-        return response.sendStatus(403);
-    }
-
-    const dirName = String(request.body.avatar_url).replace('.png', '');
-    const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`;
-    const chatFileExists = fs.existsSync(fileName);
-
-    if (!chatFileExists) {
-        console.log(`Chat file not found '${fileName}'`);
-        return response.sendStatus(400);
-    } else {
-        console.log('found the chat file: ' + fileName);
-        /* fs.unlinkSync(fileName); */
-        fs.rmSync(fileName);
-        console.log('deleted chat file: ' + fileName);
-
-    }
-
-
-    return response.send('ok');
-});
-
 app.post('/renamebackground', jsonParser, function (request, response) {
     if (!request.body) return response.sendStatus(400);
 
@@ -1165,34 +1059,6 @@ app.post('/getsettings', jsonParser, (request, response) => {
     });
 });
 
-app.post('/getworldinfo', jsonParser, (request, response) => {
-    if (!request.body?.name) {
-        return response.sendStatus(400);
-    }
-
-    const file = readWorldInfoFile(request.body.name);
-
-    return response.send(file);
-});
-
-app.post('/deleteworldinfo', jsonParser, (request, response) => {
-    if (!request.body?.name) {
-        return response.sendStatus(400);
-    }
-
-    const worldInfoName = request.body.name;
-    const filename = sanitize(`${worldInfoName}.json`);
-    const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
-
-    if (!fs.existsSync(pathToWorldInfo)) {
-        throw new Error(`World info file ${filename} doesn't exist.`);
-    }
-
-    fs.rmSync(pathToWorldInfo);
-
-    return response.sendStatus(200);
-});
-
 app.post('/savetheme', jsonParser, (request, response) => {
     if (!request.body || !request.body.name) {
         return response.sendStatus(400);
@@ -1237,275 +1103,6 @@ function getImages(path) {
         .sort(Intl.Collator().compare);
 }
 
-app.post('/exportchat', jsonParser, async function (request, response) {
-    if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
-        return response.sendStatus(400);
-    }
-    const pathToFolder = request.body.is_group
-        ? DIRECTORIES.groupChats
-        : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
-    let filename = path.join(pathToFolder, request.body.file);
-    let exportfilename = request.body.exportfilename;
-    if (!fs.existsSync(filename)) {
-        const errorMessage = {
-            message: `Could not find JSONL file to export. Source chat file: ${filename}.`,
-        };
-        console.log(errorMessage.message);
-        return response.status(404).json(errorMessage);
-    }
-    try {
-        // Short path for JSONL files
-        if (request.body.format == 'jsonl') {
-            try {
-                const rawFile = fs.readFileSync(filename, 'utf8');
-                const successMessage = {
-                    message: `Chat saved to ${exportfilename}`,
-                    result: rawFile,
-                };
-
-                console.log(`Chat exported as ${exportfilename}`);
-                return response.status(200).json(successMessage);
-            }
-            catch (err) {
-                console.error(err);
-                const errorMessage = {
-                    message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
-                };
-                console.log(errorMessage.message);
-                return response.status(500).json(errorMessage);
-            }
-        }
-
-        const readStream = fs.createReadStream(filename);
-        const rl = readline.createInterface({
-            input: readStream,
-        });
-        let buffer = '';
-        rl.on('line', (line) => {
-            const data = JSON.parse(line);
-            if (data.mes) {
-                const name = data.name;
-                const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
-                buffer += (`${name}: ${message}\n\n`);
-            }
-        });
-        rl.on('close', () => {
-            const successMessage = {
-                message: `Chat saved to ${exportfilename}`,
-                result: buffer,
-            };
-            console.log(`Chat exported as ${exportfilename}`);
-            return response.status(200).json(successMessage);
-        });
-    }
-    catch (err) {
-        console.log('chat export failed.');
-        console.log(err);
-        return response.sendStatus(400);
-    }
-});
-
-app.post('/importgroupchat', urlencodedParser, function (request, response) {
-    try {
-        const filedata = request.file;
-
-        if (!filedata) {
-            return response.sendStatus(400);
-        }
-
-        const chatname = humanizedISO8601DateTime();
-        const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
-        const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`);
-        fs.copyFileSync(pathToUpload, pathToNewFile);
-        fs.unlinkSync(pathToUpload);
-        return response.send({ res: chatname });
-    } catch (error) {
-        console.error(error);
-        return response.send({ error: true });
-    }
-});
-
-app.post('/importchat', urlencodedParser, function (request, response) {
-    if (!request.body) return response.sendStatus(400);
-
-    var format = request.body.file_type;
-    let filedata = request.file;
-    let avatar_url = (request.body.avatar_url).replace('.png', '');
-    let ch_name = request.body.character_name;
-    let user_name = request.body.user_name || 'You';
-
-    if (!filedata) {
-        return response.sendStatus(400);
-    }
-
-    try {
-        const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8');
-
-        if (format === 'json') {
-            const jsonData = JSON.parse(data);
-            if (jsonData.histories !== undefined) {
-                //console.log('/importchat confirms JSON histories are defined');
-                const chat = {
-                    from(history) {
-                        return [
-                            {
-                                user_name: user_name,
-                                character_name: ch_name,
-                                create_date: humanizedISO8601DateTime(),
-                            },
-                            ...history.msgs.map(
-                                (message) => ({
-                                    name: message.src.is_human ? user_name : ch_name,
-                                    is_user: message.src.is_human,
-                                    send_date: humanizedISO8601DateTime(),
-                                    mes: message.text,
-                                }),
-                            )];
-                    },
-                };
-
-                const newChats = [];
-                (jsonData.histories.histories ?? []).forEach((history) => {
-                    newChats.push(chat.from(history));
-                });
-
-                const errors = [];
-
-                for (const chat of newChats) {
-                    const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`;
-                    const fileContent = chat.map(tryParse).filter(x => x).join('\n');
-
-                    try {
-                        writeFileAtomicSync(filePath, fileContent, 'utf8');
-                    } catch (err) {
-                        errors.push(err);
-                    }
-                }
-
-                if (0 < errors.length) {
-                    response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors));
-                }
-
-                response.send({ res: true });
-            } else if (Array.isArray(jsonData.data_visible)) {
-                // oobabooga's format
-                /** @type {object[]} */
-                const chat = [{
-                    user_name: user_name,
-                    character_name: ch_name,
-                    create_date: humanizedISO8601DateTime(),
-                }];
-
-                for (const arr of jsonData.data_visible) {
-                    if (arr[0]) {
-                        const userMessage = {
-                            name: user_name,
-                            is_user: true,
-                            send_date: humanizedISO8601DateTime(),
-                            mes: arr[0],
-                        };
-                        chat.push(userMessage);
-                    }
-                    if (arr[1]) {
-                        const charMessage = {
-                            name: ch_name,
-                            is_user: false,
-                            send_date: humanizedISO8601DateTime(),
-                            mes: arr[1],
-                        };
-                        chat.push(charMessage);
-                    }
-                }
-
-                const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
-                writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
-
-                response.send({ res: true });
-            } else {
-                console.log('Incorrect chat format .json');
-                return response.send({ error: true });
-            }
-        }
-
-        if (format === 'jsonl') {
-            const line = data.split('\n')[0];
-
-            let jsonData = JSON.parse(line);
-
-            if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
-                fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`));
-                response.send({ res: true });
-            } else {
-                console.log('Incorrect chat format .jsonl');
-                return response.send({ error: true });
-            }
-        }
-    } catch (error) {
-        console.error(error);
-        return response.send({ error: true });
-    }
-});
-
-app.post('/importworldinfo', urlencodedParser, (request, response) => {
-    if (!request.file) return response.sendStatus(400);
-
-    const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`;
-
-    let fileContents = null;
-
-    if (request.body.convertedData) {
-        fileContents = request.body.convertedData;
-    } else {
-        const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
-        fileContents = fs.readFileSync(pathToUpload, 'utf8');
-        fs.unlinkSync(pathToUpload);
-    }
-
-    try {
-        const worldContent = JSON.parse(fileContents);
-        if (!('entries' in worldContent)) {
-            throw new Error('File must contain a world info entries list');
-        }
-    } catch (err) {
-        return response.status(400).send('Is not a valid world info file');
-    }
-
-    const pathToNewFile = path.join(DIRECTORIES.worlds, filename);
-    const worldName = path.parse(pathToNewFile).name;
-
-    if (!worldName) {
-        return response.status(400).send('World file must have a name');
-    }
-
-    writeFileAtomicSync(pathToNewFile, fileContents);
-    return response.send({ name: worldName });
-});
-
-app.post('/editworldinfo', jsonParser, (request, response) => {
-    if (!request.body) {
-        return response.sendStatus(400);
-    }
-
-    if (!request.body.name) {
-        return response.status(400).send('World file must have a name');
-    }
-
-    try {
-        if (!('entries' in request.body.data)) {
-            throw new Error('World info must contain an entries list');
-        }
-    } catch (err) {
-        return response.status(400).send('Is not a valid world info file');
-    }
-
-    const filename = `${sanitize(request.body.name)}.json`;
-    const pathToFile = path.join(DIRECTORIES.worlds, filename);
-
-    writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4));
-
-    return response.send({ ok: true });
-});
-
 app.post('/uploaduseravatar', urlencodedParser, async (request, response) => {
     if (!request.file) return response.sendStatus(400);
 
@@ -1614,181 +1211,6 @@ app.post('/listimgfiles/:folder', (req, res) => {
 });
 
 
-app.post('/getgroups', jsonParser, (_, response) => {
-    const groups = [];
-
-    if (!fs.existsSync(DIRECTORIES.groups)) {
-        fs.mkdirSync(DIRECTORIES.groups);
-    }
-
-    const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json');
-    const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl');
-
-    files.forEach(function (file) {
-        try {
-            const filePath = path.join(DIRECTORIES.groups, file);
-            const fileContents = fs.readFileSync(filePath, 'utf8');
-            const group = JSON.parse(fileContents);
-            const groupStat = fs.statSync(filePath);
-            group['date_added'] = groupStat.birthtimeMs;
-            group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs);
-
-            let chat_size = 0;
-            let date_last_chat = 0;
-
-            if (Array.isArray(group.chats) && Array.isArray(chats)) {
-                for (const chat of chats) {
-                    if (group.chats.includes(path.parse(chat).name)) {
-                        const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat));
-                        chat_size += chatStat.size;
-                        date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
-                    }
-                }
-            }
-
-            group['date_last_chat'] = date_last_chat;
-            group['chat_size'] = chat_size;
-            groups.push(group);
-        }
-        catch (error) {
-            console.error(error);
-        }
-    });
-
-    return response.send(groups);
-});
-
-app.post('/creategroup', jsonParser, (request, response) => {
-    if (!request.body) {
-        return response.sendStatus(400);
-    }
-
-    const id = String(Date.now());
-    const groupMetadata = {
-        id: id,
-        name: request.body.name ?? 'New Group',
-        members: request.body.members ?? [],
-        avatar_url: request.body.avatar_url,
-        allow_self_responses: !!request.body.allow_self_responses,
-        activation_strategy: request.body.activation_strategy ?? 0,
-        generation_mode: request.body.generation_mode ?? 0,
-        disabled_members: request.body.disabled_members ?? [],
-        chat_metadata: request.body.chat_metadata ?? {},
-        fav: request.body.fav,
-        chat_id: request.body.chat_id ?? id,
-        chats: request.body.chats ?? [id],
-    };
-    const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
-    const fileData = JSON.stringify(groupMetadata);
-
-    if (!fs.existsSync(DIRECTORIES.groups)) {
-        fs.mkdirSync(DIRECTORIES.groups);
-    }
-
-    writeFileAtomicSync(pathToFile, fileData);
-    return response.send(groupMetadata);
-});
-
-app.post('/editgroup', jsonParser, (request, response) => {
-    if (!request.body || !request.body.id) {
-        return response.sendStatus(400);
-    }
-    const id = request.body.id;
-    const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
-    const fileData = JSON.stringify(request.body);
-
-    writeFileAtomicSync(pathToFile, fileData);
-    return response.send({ ok: true });
-});
-
-app.post('/getgroupchat', jsonParser, (request, response) => {
-    if (!request.body || !request.body.id) {
-        return response.sendStatus(400);
-    }
-
-    const id = request.body.id;
-    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
-
-    if (fs.existsSync(pathToFile)) {
-        const data = fs.readFileSync(pathToFile, 'utf8');
-        const lines = data.split('\n');
-
-        // Iterate through the array of strings and parse each line as JSON
-        const jsonData = lines.map(line => tryParse(line)).filter(x => x);
-        return response.send(jsonData);
-    } else {
-        return response.send([]);
-    }
-});
-
-app.post('/deletegroupchat', jsonParser, (request, response) => {
-    if (!request.body || !request.body.id) {
-        return response.sendStatus(400);
-    }
-
-    const id = request.body.id;
-    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
-
-    if (fs.existsSync(pathToFile)) {
-        fs.rmSync(pathToFile);
-        return response.send({ ok: true });
-    }
-
-    return response.send({ error: true });
-});
-
-app.post('/savegroupchat', jsonParser, (request, response) => {
-    if (!request.body || !request.body.id) {
-        return response.sendStatus(400);
-    }
-
-    const id = request.body.id;
-    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
-
-    if (!fs.existsSync(DIRECTORIES.groupChats)) {
-        fs.mkdirSync(DIRECTORIES.groupChats);
-    }
-
-    let chat_data = request.body.chat;
-    let jsonlData = chat_data.map(JSON.stringify).join('\n');
-    writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
-    backupChat(String(id), jsonlData);
-    return response.send({ ok: true });
-});
-
-app.post('/deletegroup', jsonParser, async (request, response) => {
-    if (!request.body || !request.body.id) {
-        return response.sendStatus(400);
-    }
-
-    const id = request.body.id;
-    const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`));
-
-    try {
-        // Delete group chats
-        const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8'));
-
-        if (group && Array.isArray(group.chats)) {
-            for (const chat of group.chats) {
-                console.log('Deleting group chat', chat);
-                const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
-
-                if (fs.existsSync(pathToFile)) {
-                    fs.rmSync(pathToFile);
-                }
-            }
-        }
-    } catch (error) {
-        console.error('Could not delete group chats. Clean them up manually.', error);
-    }
-
-    if (fs.existsSync(pathToGroup)) {
-        fs.rmSync(pathToGroup);
-    }
-
-    return response.send({ ok: true });
-});
-
 function cleanUploads() {
     try {
         if (fs.existsSync(UPLOADS_PATH)) {
@@ -2605,6 +2027,30 @@ redirect('/importcharacter', '/api/characters/import');
 redirect('/dupecharacter', '/api/characters/duplicate');
 redirect('/exportcharacter', '/api/characters/export');
 
+// Redirect deprecated chat API endpoints
+redirect('/savechat', '/api/chats/save');
+redirect('/getchat', '/api/chats/get');
+redirect('/renamechat', '/api/chats/rename');
+redirect('/delchat', '/api/chats/delete');
+redirect('/exportchat', '/api/chats/export');
+redirect('/importgroupchat', '/api/chats/group/import');
+redirect('/importchat', '/api/chats/import');
+redirect('/getgroupchat', '/api/chats/group/get');
+redirect('/deletegroupchat', '/api/chats/group/delete');
+redirect('/savegroupchat', '/api/chats/group/save');
+
+// Redirect deprecated group API endpoints
+redirect('/getgroups', '/api/groups/all');
+redirect('/creategroup', '/api/groups/create');
+redirect('/editgroup', '/api/groups/edit');
+redirect('/deletegroup', '/api/groups/delete');
+
+// Redirect deprecated worldinfo API endpoints
+redirect('/getworldinfo', '/api/worldinfo/get');
+redirect('/deleteworldinfo', '/api/worldinfo/delete');
+redirect('/importworldinfo', '/api/worldinfo/import');
+redirect('/editworldinfo', '/api/worldinfo/edit');
+
 // ** REST CLIENT ASYNC WRAPPERS **
 
 /**
@@ -2656,6 +2102,15 @@ app.use('/api/files', require('./src/endpoints/files').router);
 // Character management
 app.use('/api/characters', require('./src/endpoints/characters').router);
 
+// Chat management
+app.use('/api/chats', require('./src/endpoints/chats').router);
+
+// Group management
+app.use('/api/groups', require('./src/endpoints/groups').router);
+
+// World info management
+app.use('/api/worldinfo', require('./src/endpoints/worldinfo').router);
+
 // Character sprite management
 app.use('/api/sprites', require('./src/endpoints/sprites').router);
 
@@ -2760,47 +2215,6 @@ if (cliArguments.ssl) {
     );
 }
 
-function generateTimestamp() {
-    const now = new Date();
-    const year = now.getFullYear();
-    const month = String(now.getMonth() + 1).padStart(2, '0');
-    const day = String(now.getDate()).padStart(2, '0');
-    const hours = String(now.getHours()).padStart(2, '0');
-    const minutes = String(now.getMinutes()).padStart(2, '0');
-    const seconds = String(now.getSeconds()).padStart(2, '0');
-
-    return `${year}${month}${day}-${hours}${minutes}${seconds}`;
-}
-
-/**
- *
- * @param {string} name
- * @param {string} chat
- */
-function backupChat(name, chat) {
-    try {
-        const isBackupDisabled = getConfigValue('disableChatBackup', false);
-
-        if (isBackupDisabled) {
-            return;
-        }
-
-        if (!fs.existsSync(DIRECTORIES.backups)) {
-            fs.mkdirSync(DIRECTORIES.backups);
-        }
-
-        // replace non-alphanumeric characters with underscores
-        name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
-
-        const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
-        writeFileAtomicSync(backupFile, chat, 'utf-8');
-
-        removeOldBackups(`chat_${name}_`);
-    } catch (err) {
-        console.log(`Could not backup chat for ${name}`, err);
-    }
-}
-
 function backupSettings() {
     try {
         if (!fs.existsSync(DIRECTORIES.backups)) {
@@ -2816,21 +2230,6 @@ function backupSettings() {
     }
 }
 
-/**
- * @param {string} prefix
- */
-function removeOldBackups(prefix) {
-    const MAX_BACKUPS = 25;
-
-    let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix));
-    if (files.length > MAX_BACKUPS) {
-        files = files.map(f => path.join(DIRECTORIES.backups, f));
-        files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
-
-        fs.rmSync(files[0]);
-    }
-}
-
 function ensurePublicDirectoriesExist() {
     for (const dir of Object.values(DIRECTORIES)) {
         if (!fs.existsSync(dir)) {

src/endpoints/characters.js

@@ -16,7 +16,7 @@ const { jsonParser, urlencodedParser } = require('../express-common');
 const { deepMerge, humanizedISO8601DateTime, tryParse } = require('../util');
 const { TavernCardValidator } = require('../validator/TavernCardValidator');
 const characterCardParser = require('../character-card-parser.js');
-const { readWorldInfoFile, convertWorldInfoToCharacterBook } = require('../worldinfo');
+const { readWorldInfoFile } = require('./worldinfo');
 const { invalidateThumbnail } = require('./thumbnails');
 const { importRisuSprites } = require('./sprites');
 
@@ -330,6 +330,46 @@ function charaFormatData(data) {
     return char;
 }
 
+/**
+ * @param {string} name Name of World Info file
+ * @param {object} entries Entries object
+ */
+function convertWorldInfoToCharacterBook(name, entries) {
+    /** @type {{ entries: object[]; name: string }} */
+    const result = { entries: [], name };
+
+    for (const index in entries) {
+        const entry = entries[index];
+
+        const originalEntry = {
+            id: entry.uid,
+            keys: entry.key,
+            secondary_keys: entry.keysecondary,
+            comment: entry.comment,
+            content: entry.content,
+            constant: entry.constant,
+            selective: entry.selective,
+            insertion_order: entry.order,
+            enabled: !entry.disable,
+            position: entry.position == 0 ? 'before_char' : 'after_char',
+            extensions: {
+                position: entry.position,
+                exclude_recursion: entry.excludeRecursion,
+                display_index: entry.displayIndex,
+                probability: entry.probability ?? null,
+                useProbability: entry.useProbability ?? false,
+                depth: entry.depth ?? 4,
+                selectiveLogic: entry.selectiveLogic ?? 0,
+                group: entry.group ?? '',
+            },
+        };
+
+        result.entries.push(originalEntry);
+    }
+
+    return result;
+}
+
 const router = express.Router();
 
 router.post('/create', urlencodedParser, async function (request, response) {

src/endpoints/chats.js

@@ -0,0 +1,411 @@
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+const express = require('express');
+const sanitize = require('sanitize-filename');
+const writeFileAtomicSync = require('write-file-atomic').sync;
+
+const { jsonParser, urlencodedParser } = require('../express-common');
+const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
+const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util');
+
+/**
+ *
+ * @param {string} name
+ * @param {string} chat
+ */
+function backupChat(name, chat) {
+    try {
+        const isBackupDisabled = getConfigValue('disableChatBackup', false);
+
+        if (isBackupDisabled) {
+            return;
+        }
+
+        if (!fs.existsSync(DIRECTORIES.backups)) {
+            fs.mkdirSync(DIRECTORIES.backups);
+        }
+
+        // replace non-alphanumeric characters with underscores
+        name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
+
+        const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
+        writeFileAtomicSync(backupFile, chat, 'utf-8');
+
+        removeOldBackups(`chat_${name}_`);
+    } catch (err) {
+        console.log(`Could not backup chat for ${name}`, err);
+    }
+}
+
+const router = express.Router();
+
+router.post('/save', jsonParser, function (request, response) {
+    try {
+        var dir_name = String(request.body.avatar_url).replace('.png', '');
+        let chat_data = request.body.chat;
+        let jsonlData = chat_data.map(JSON.stringify).join('\n');
+        writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8');
+        backupChat(dir_name, jsonlData);
+        return response.send({ result: 'ok' });
+    } catch (error) {
+        response.send(error);
+        return console.log(error);
+    }
+});
+
+router.post('/get', jsonParser, function (request, response) {
+    try {
+        const dirName = String(request.body.avatar_url).replace('.png', '');
+        const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName);
+
+        //if no chat dir for the character is found, make one with the character name
+        if (!chatDirExists) {
+            fs.mkdirSync(DIRECTORIES.chats + dirName);
+            return response.send({});
+        }
+
+
+        if (!request.body.file_name) {
+            return response.send({});
+        }
+
+        const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`;
+        const chatFileExists = fs.existsSync(fileName);
+
+        if (!chatFileExists) {
+            return response.send({});
+        }
+
+        const data = fs.readFileSync(fileName, 'utf8');
+        const lines = data.split('\n');
+
+        // Iterate through the array of strings and parse each line as JSON
+        const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
+        return response.send(jsonData);
+    } catch (error) {
+        console.error(error);
+        return response.send({});
+    }
+});
+
+
+router.post('/rename', jsonParser, async function (request, response) {
+    if (!request.body || !request.body.original_file || !request.body.renamed_file) {
+        return response.sendStatus(400);
+    }
+
+    const pathToFolder = request.body.is_group
+        ? DIRECTORIES.groupChats
+        : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
+    const pathToOriginalFile = path.join(pathToFolder, request.body.original_file);
+    const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file);
+    console.log('Old chat name', pathToOriginalFile);
+    console.log('New chat name', pathToRenamedFile);
+
+    if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {
+        console.log('Either Source or Destination files are not available');
+        return response.status(400).send({ error: true });
+    }
+
+    console.log('Successfully renamed.');
+    fs.renameSync(pathToOriginalFile, pathToRenamedFile);
+    return response.send({ ok: true });
+});
+
+router.post('/delete', jsonParser, function (request, response) {
+    console.log('/api/chats/delete entered');
+    if (!request.body) {
+        console.log('no request body seen');
+        return response.sendStatus(400);
+    }
+
+    if (request.body.chatfile !== sanitize(request.body.chatfile)) {
+        console.error('Malicious chat name prevented');
+        return response.sendStatus(403);
+    }
+
+    const dirName = String(request.body.avatar_url).replace('.png', '');
+    const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`;
+    const chatFileExists = fs.existsSync(fileName);
+
+    if (!chatFileExists) {
+        console.log(`Chat file not found '${fileName}'`);
+        return response.sendStatus(400);
+    } else {
+        console.log('found the chat file: ' + fileName);
+        /* fs.unlinkSync(fileName); */
+        fs.rmSync(fileName);
+        console.log('deleted chat file: ' + fileName);
+
+    }
+
+
+    return response.send('ok');
+});
+
+router.post('/export', jsonParser, async function (request, response) {
+    if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
+        return response.sendStatus(400);
+    }
+    const pathToFolder = request.body.is_group
+        ? DIRECTORIES.groupChats
+        : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
+    let filename = path.join(pathToFolder, request.body.file);
+    let exportfilename = request.body.exportfilename;
+    if (!fs.existsSync(filename)) {
+        const errorMessage = {
+            message: `Could not find JSONL file to export. Source chat file: ${filename}.`,
+        };
+        console.log(errorMessage.message);
+        return response.status(404).json(errorMessage);
+    }
+    try {
+        // Short path for JSONL files
+        if (request.body.format == 'jsonl') {
+            try {
+                const rawFile = fs.readFileSync(filename, 'utf8');
+                const successMessage = {
+                    message: `Chat saved to ${exportfilename}`,
+                    result: rawFile,
+                };
+
+                console.log(`Chat exported as ${exportfilename}`);
+                return response.status(200).json(successMessage);
+            }
+            catch (err) {
+                console.error(err);
+                const errorMessage = {
+                    message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
+                };
+                console.log(errorMessage.message);
+                return response.status(500).json(errorMessage);
+            }
+        }
+
+        const readStream = fs.createReadStream(filename);
+        const rl = readline.createInterface({
+            input: readStream,
+        });
+        let buffer = '';
+        rl.on('line', (line) => {
+            const data = JSON.parse(line);
+            if (data.mes) {
+                const name = data.name;
+                const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
+                buffer += (`${name}: ${message}\n\n`);
+            }
+        });
+        rl.on('close', () => {
+            const successMessage = {
+                message: `Chat saved to ${exportfilename}`,
+                result: buffer,
+            };
+            console.log(`Chat exported as ${exportfilename}`);
+            return response.status(200).json(successMessage);
+        });
+    }
+    catch (err) {
+        console.log('chat export failed.');
+        console.log(err);
+        return response.sendStatus(400);
+    }
+});
+
+router.post('/group/import', urlencodedParser, function (request, response) {
+    try {
+        const filedata = request.file;
+
+        if (!filedata) {
+            return response.sendStatus(400);
+        }
+
+        const chatname = humanizedISO8601DateTime();
+        const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
+        const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`);
+        fs.copyFileSync(pathToUpload, pathToNewFile);
+        fs.unlinkSync(pathToUpload);
+        return response.send({ res: chatname });
+    } catch (error) {
+        console.error(error);
+        return response.send({ error: true });
+    }
+});
+
+router.post('/import', urlencodedParser, function (request, response) {
+    if (!request.body) return response.sendStatus(400);
+
+    var format = request.body.file_type;
+    let filedata = request.file;
+    let avatar_url = (request.body.avatar_url).replace('.png', '');
+    let ch_name = request.body.character_name;
+    let user_name = request.body.user_name || 'You';
+
+    if (!filedata) {
+        return response.sendStatus(400);
+    }
+
+    try {
+        const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8');
+
+        if (format === 'json') {
+            const jsonData = JSON.parse(data);
+            if (jsonData.histories !== undefined) {
+                //console.log('/api/chats/import confirms JSON histories are defined');
+                const chat = {
+                    from(history) {
+                        return [
+                            {
+                                user_name: user_name,
+                                character_name: ch_name,
+                                create_date: humanizedISO8601DateTime(),
+                            },
+                            ...history.msgs.map(
+                                (message) => ({
+                                    name: message.src.is_human ? user_name : ch_name,
+                                    is_user: message.src.is_human,
+                                    send_date: humanizedISO8601DateTime(),
+                                    mes: message.text,
+                                }),
+                            )];
+                    },
+                };
+
+                const newChats = [];
+                (jsonData.histories.histories ?? []).forEach((history) => {
+                    newChats.push(chat.from(history));
+                });
+
+                const errors = [];
+
+                for (const chat of newChats) {
+                    const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`;
+                    const fileContent = chat.map(tryParse).filter(x => x).join('\n');
+
+                    try {
+                        writeFileAtomicSync(filePath, fileContent, 'utf8');
+                    } catch (err) {
+                        errors.push(err);
+                    }
+                }
+
+                if (0 < errors.length) {
+                    response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors));
+                }
+
+                response.send({ res: true });
+            } else if (Array.isArray(jsonData.data_visible)) {
+                // oobabooga's format
+                /** @type {object[]} */
+                const chat = [{
+                    user_name: user_name,
+                    character_name: ch_name,
+                    create_date: humanizedISO8601DateTime(),
+                }];
+
+                for (const arr of jsonData.data_visible) {
+                    if (arr[0]) {
+                        const userMessage = {
+                            name: user_name,
+                            is_user: true,
+                            send_date: humanizedISO8601DateTime(),
+                            mes: arr[0],
+                        };
+                        chat.push(userMessage);
+                    }
+                    if (arr[1]) {
+                        const charMessage = {
+                            name: ch_name,
+                            is_user: false,
+                            send_date: humanizedISO8601DateTime(),
+                            mes: arr[1],
+                        };
+                        chat.push(charMessage);
+                    }
+                }
+
+                const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
+                writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
+
+                response.send({ res: true });
+            } else {
+                console.log('Incorrect chat format .json');
+                return response.send({ error: true });
+            }
+        }
+
+        if (format === 'jsonl') {
+            const line = data.split('\n')[0];
+
+            let jsonData = JSON.parse(line);
+
+            if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
+                fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`));
+                response.send({ res: true });
+            } else {
+                console.log('Incorrect chat format .jsonl');
+                return response.send({ error: true });
+            }
+        }
+    } catch (error) {
+        console.error(error);
+        return response.send({ error: true });
+    }
+});
+
+router.post('/group/get', jsonParser, (request, response) => {
+    if (!request.body || !request.body.id) {
+        return response.sendStatus(400);
+    }
+
+    const id = request.body.id;
+    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
+
+    if (fs.existsSync(pathToFile)) {
+        const data = fs.readFileSync(pathToFile, 'utf8');
+        const lines = data.split('\n');
+
+        // Iterate through the array of strings and parse each line as JSON
+        const jsonData = lines.map(line => tryParse(line)).filter(x => x);
+        return response.send(jsonData);
+    } else {
+        return response.send([]);
+    }
+});
+
+router.post('/group/delete', jsonParser, (request, response) => {
+    if (!request.body || !request.body.id) {
+        return response.sendStatus(400);
+    }
+
+    const id = request.body.id;
+    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
+
+    if (fs.existsSync(pathToFile)) {
+        fs.rmSync(pathToFile);
+        return response.send({ ok: true });
+    }
+
+    return response.send({ error: true });
+});
+
+router.post('/group/save', jsonParser, (request, response) => {
+    if (!request.body || !request.body.id) {
+        return response.sendStatus(400);
+    }
+
+    const id = request.body.id;
+    const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
+
+    if (!fs.existsSync(DIRECTORIES.groupChats)) {
+        fs.mkdirSync(DIRECTORIES.groupChats);
+    }
+
+    let chat_data = request.body.chat;
+    let jsonlData = chat_data.map(JSON.stringify).join('\n');
+    writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
+    backupChat(String(id), jsonlData);
+    return response.send({ ok: true });
+});
+
+module.exports = { router };

src/endpoints/groups.js

@@ -0,0 +1,133 @@
+const fs = require('fs');
+const path = require('path');
+const express = require('express');
+const sanitize = require('sanitize-filename');
+const writeFileAtomicSync = require('write-file-atomic').sync;
+
+const { jsonParser } = require('../express-common');
+const { DIRECTORIES } = require('../constants');
+const { humanizedISO8601DateTime } = require('../util');
+
+const router = express.Router();
+
+router.post('/all', jsonParser, (_, response) => {
+    const groups = [];
+
+    if (!fs.existsSync(DIRECTORIES.groups)) {
+        fs.mkdirSync(DIRECTORIES.groups);
+    }
+
+    const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json');
+    const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl');
+
+    files.forEach(function (file) {
+        try {
+            const filePath = path.join(DIRECTORIES.groups, file);
+            const fileContents = fs.readFileSync(filePath, 'utf8');
+            const group = JSON.parse(fileContents);
+            const groupStat = fs.statSync(filePath);
+            group['date_added'] = groupStat.birthtimeMs;
+            group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs);
+
+            let chat_size = 0;
+            let date_last_chat = 0;
+
+            if (Array.isArray(group.chats) && Array.isArray(chats)) {
+                for (const chat of chats) {
+                    if (group.chats.includes(path.parse(chat).name)) {
+                        const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat));
+                        chat_size += chatStat.size;
+                        date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
+                    }
+                }
+            }
+
+            group['date_last_chat'] = date_last_chat;
+            group['chat_size'] = chat_size;
+            groups.push(group);
+        }
+        catch (error) {
+            console.error(error);
+        }
+    });
+
+    return response.send(groups);
+});
+
+router.post('/create', jsonParser, (request, response) => {
+    if (!request.body) {
+        return response.sendStatus(400);
+    }
+
+    const id = String(Date.now());
+    const groupMetadata = {
+        id: id,
+        name: request.body.name ?? 'New Group',
+        members: request.body.members ?? [],
+        avatar_url: request.body.avatar_url,
+        allow_self_responses: !!request.body.allow_self_responses,
+        activation_strategy: request.body.activation_strategy ?? 0,
+        generation_mode: request.body.generation_mode ?? 0,
+        disabled_members: request.body.disabled_members ?? [],
+        chat_metadata: request.body.chat_metadata ?? {},
+        fav: request.body.fav,
+        chat_id: request.body.chat_id ?? id,
+        chats: request.body.chats ?? [id],
+    };
+    const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
+    const fileData = JSON.stringify(groupMetadata);
+
+    if (!fs.existsSync(DIRECTORIES.groups)) {
+        fs.mkdirSync(DIRECTORIES.groups);
+    }
+
+    writeFileAtomicSync(pathToFile, fileData);
+    return response.send(groupMetadata);
+});
+
+router.post('/edit', jsonParser, (request, response) => {
+    if (!request.body || !request.body.id) {
+        return response.sendStatus(400);
+    }
+    const id = request.body.id;
+    const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
+    const fileData = JSON.stringify(request.body);
+
+    writeFileAtomicSync(pathToFile, fileData);
+    return response.send({ ok: true });
+});
+
+router.post('/delete', jsonParser, async (request, response) => {
+    if (!request.body || !request.body.id) {
+        return response.sendStatus(400);
+    }
+
+    const id = request.body.id;
+    const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`));
+
+    try {
+        // Delete group chats
+        const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8'));
+
+        if (group && Array.isArray(group.chats)) {
+            for (const chat of group.chats) {
+                console.log('Deleting group chat', chat);
+                const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
+
+                if (fs.existsSync(pathToFile)) {
+                    fs.rmSync(pathToFile);
+                }
+            }
+        }
+    } catch (error) {
+        console.error('Could not delete group chats. Clean them up manually.', error);
+    }
+
+    if (fs.existsSync(pathToGroup)) {
+        fs.rmSync(pathToGroup);
+    }
+
+    return response.send({ ok: true });
+});
+
+module.exports = { router };

src/endpoints/worldinfo.js

@@ -0,0 +1,120 @@
+const fs = require('fs');
+const path = require('path');
+const express = require('express');
+const sanitize = require('sanitize-filename');
+const writeFileAtomicSync = require('write-file-atomic').sync;
+
+const { jsonParser, urlencodedParser } = require('../express-common');
+const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
+
+function readWorldInfoFile(worldInfoName) {
+    const dummyObject = { entries: {} };
+
+    if (!worldInfoName) {
+        return dummyObject;
+    }
+
+    const filename = `${worldInfoName}.json`;
+    const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
+
+    if (!fs.existsSync(pathToWorldInfo)) {
+        console.log(`World info file ${filename} doesn't exist.`);
+        return dummyObject;
+    }
+
+    const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
+    const worldInfo = JSON.parse(worldInfoText);
+    return worldInfo;
+}
+
+const router = express.Router();
+
+router.post('/get', jsonParser, (request, response) => {
+    if (!request.body?.name) {
+        return response.sendStatus(400);
+    }
+
+    const file = readWorldInfoFile(request.body.name);
+
+    return response.send(file);
+});
+
+router.post('/delete', jsonParser, (request, response) => {
+    if (!request.body?.name) {
+        return response.sendStatus(400);
+    }
+
+    const worldInfoName = request.body.name;
+    const filename = sanitize(`${worldInfoName}.json`);
+    const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
+
+    if (!fs.existsSync(pathToWorldInfo)) {
+        throw new Error(`World info file ${filename} doesn't exist.`);
+    }
+
+    fs.rmSync(pathToWorldInfo);
+
+    return response.sendStatus(200);
+});
+
+router.post('/import', urlencodedParser, (request, response) => {
+    if (!request.file) return response.sendStatus(400);
+
+    const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`;
+
+    let fileContents = null;
+
+    if (request.body.convertedData) {
+        fileContents = request.body.convertedData;
+    } else {
+        const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
+        fileContents = fs.readFileSync(pathToUpload, 'utf8');
+        fs.unlinkSync(pathToUpload);
+    }
+
+    try {
+        const worldContent = JSON.parse(fileContents);
+        if (!('entries' in worldContent)) {
+            throw new Error('File must contain a world info entries list');
+        }
+    } catch (err) {
+        return response.status(400).send('Is not a valid world info file');
+    }
+
+    const pathToNewFile = path.join(DIRECTORIES.worlds, filename);
+    const worldName = path.parse(pathToNewFile).name;
+
+    if (!worldName) {
+        return response.status(400).send('World file must have a name');
+    }
+
+    writeFileAtomicSync(pathToNewFile, fileContents);
+    return response.send({ name: worldName });
+});
+
+router.post('/edit', jsonParser, (request, response) => {
+    if (!request.body) {
+        return response.sendStatus(400);
+    }
+
+    if (!request.body.name) {
+        return response.status(400).send('World file must have a name');
+    }
+
+    try {
+        if (!('entries' in request.body.data)) {
+            throw new Error('World info must contain an entries list');
+        }
+    } catch (err) {
+        return response.status(400).send('Is not a valid world info file');
+    }
+
+    const filename = `${sanitize(request.body.name)}.json`;
+    const pathToFile = path.join(DIRECTORIES.worlds, filename);
+
+    writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4));
+
+    return response.send({ ok: true });
+});
+
+module.exports = { router, readWorldInfoFile };

src/util.js

@@ -7,6 +7,8 @@ const mime = require('mime-types');
 const yaml = require('yaml');
 const { default: simpleGit } = require('simple-git');
 
+const { DIRECTORIES } = require('./constants');
+
 /**
  * Returns the config object from the config.yaml file.
  * @returns {object} Config object
@@ -307,6 +309,34 @@ function removeFileExtension(filename) {
     return filename.replace(/\.[^.]+$/, '');
 }
 
+function generateTimestamp() {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+    const hours = String(now.getHours()).padStart(2, '0');
+    const minutes = String(now.getMinutes()).padStart(2, '0');
+    const seconds = String(now.getSeconds()).padStart(2, '0');
+
+    return `${year}${month}${day}-${hours}${minutes}${seconds}`;
+}
+
+/**
+ * @param {string} prefix
+ */
+function removeOldBackups(prefix) {
+    const MAX_BACKUPS = 25;
+
+    let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix));
+    if (files.length > MAX_BACKUPS) {
+        files = files.map(f => path.join(DIRECTORIES.backups, f));
+        files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
+
+        fs.rmSync(files[0]);
+    }
+}
+
+
 module.exports = {
     getConfig,
     getConfigValue,
@@ -323,4 +353,6 @@ module.exports = {
     tryParse,
     clientRelativePath,
     removeFileExtension,
+    generateTimestamp,
+    removeOldBackups,
 };

src/worldinfo.js

@@ -1,68 +0,0 @@
-const path = require('path');
-const fs = require('fs');
-const { DIRECTORIES } = require('./constants');
-
-function readWorldInfoFile(worldInfoName) {
-    const dummyObject = { entries: {} };
-
-    if (!worldInfoName) {
-        return dummyObject;
-    }
-
-    const filename = `${worldInfoName}.json`;
-    const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
-
-    if (!fs.existsSync(pathToWorldInfo)) {
-        console.log(`World info file ${filename} doesn't exist.`);
-        return dummyObject;
-    }
-
-    const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
-    const worldInfo = JSON.parse(worldInfoText);
-    return worldInfo;
-}
-
-/**
- * @param {string} name Name of World Info file
- * @param {object} entries Entries object
- */
-function convertWorldInfoToCharacterBook(name, entries) {
-    /** @type {{ entries: object[]; name: string }} */
-    const result = { entries: [], name };
-
-    for (const index in entries) {
-        const entry = entries[index];
-
-        const originalEntry = {
-            id: entry.uid,
-            keys: entry.key,
-            secondary_keys: entry.keysecondary,
-            comment: entry.comment,
-            content: entry.content,
-            constant: entry.constant,
-            selective: entry.selective,
-            insertion_order: entry.order,
-            enabled: !entry.disable,
-            position: entry.position == 0 ? 'before_char' : 'after_char',
-            extensions: {
-                position: entry.position,
-                exclude_recursion: entry.excludeRecursion,
-                display_index: entry.displayIndex,
-                probability: entry.probability ?? null,
-                useProbability: entry.useProbability ?? false,
-                depth: entry.depth ?? 4,
-                selectiveLogic: entry.selectiveLogic ?? 0,
-                group: entry.group ?? '',
-            },
-        };
-
-        result.entries.push(originalEntry);
-    }
-
-    return result;
-}
-
-module.exports = {
-    readWorldInfoFile,
-    convertWorldInfoToCharacterBook,
-};