diff --git a/public/script.js b/public/script.js index ee4311ab4..6ab1ddccc 100644 --- a/public/script.js +++ b/public/script.js @@ -1249,7 +1249,7 @@ async function getCharacters() { } async function delChat(chatfile) { - const response = await fetch('/delchat', { + const response = await fetch('/api/chats/delete', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ @@ -3883,7 +3883,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject, toastr.error(data.response, 'API Error'); } } - console.debug('/savechat called by /Generate'); + console.debug('/api/chats/save called by /Generate'); await saveChatConditional(); is_send_press = false; @@ -4903,7 +4903,7 @@ async function renamePastChats(newAvatar, newValue) { for (const { file_name } of pastChats) { try { const fileNameWithoutExtension = file_name.replace('.jsonl', ''); - const getChatResponse = await fetch('/getchat', { + const getChatResponse = await fetch('/api/chats/get', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ @@ -4927,7 +4927,7 @@ async function renamePastChats(newAvatar, newValue) { } } - const saveChatResponse = await fetch('/savechat', { + const saveChatResponse = await fetch('/api/chats/save', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ @@ -5018,7 +5018,7 @@ async function saveChat(chat_name, withMetadata, mesId) { ]; return jQuery.ajax({ type: 'POST', - url: '/savechat', + url: '/api/chats/save', data: JSON.stringify({ ch_name: characters[this_chid].name, file_name: file_name, @@ -5110,11 +5110,11 @@ function getThumbnailUrl(type, file) { } async function getChat() { - //console.log('/getchat -- entered for -- ' + characters[this_chid].name); + //console.log('/api/chats/get -- entered for -- ' + characters[this_chid].name); try { const response = await $.ajax({ type: 'POST', - url: '/getchat', + url: '/api/chats/get', data: JSON.stringify({ ch_name: characters[this_chid].name, file_name: characters[this_chid].chat, @@ -5875,7 +5875,7 @@ export async function getChatsFromFiles(data, isGroupChat) { let chat_promise = chat_list.map(({ file_name }) => { return new Promise(async (res, rej) => { try { - const endpoint = isGroupChat ? '/getgroupchat' : '/getchat'; + const endpoint = isGroupChat ? '/api/chats/group/get' : '/api/chats/get'; const requestBody = isGroupChat ? JSON.stringify({ id: file_name }) : JSON.stringify({ @@ -6562,7 +6562,7 @@ export async function saveChatConditional() { async function importCharacterChat(formData) { await jQuery.ajax({ type: 'POST', - url: '/importchat', + url: '/api/chats/import', data: formData, beforeSend: function () { }, @@ -8161,7 +8161,8 @@ jQuery(async function () { }; try { - const response = await fetch('/renamechat', { + showLoader(); + const response = await fetch('/api/chats/rename', { method: 'POST', body: JSON.stringify(body), headers: getRequestHeaders(), @@ -8193,8 +8194,11 @@ jQuery(async function () { $('#option_select_chat').trigger('click'); $('#options').hide(); } catch { + hideLoader(); await delay(500); await callPopup('An error has occurred. Chat was not renamed.', 'text'); + } finally { + hideLoader(); } }); @@ -8215,7 +8219,7 @@ jQuery(async function () { }; console.log(body); try { - const response = await fetch('/exportchat', { + const response = await fetch('/api/chats/export', { method: 'POST', body: JSON.stringify(body), headers: getRequestHeaders(), diff --git a/public/scripts/bookmarks.js b/public/scripts/bookmarks.js index 0711aafaa..131f95961 100644 --- a/public/scripts/bookmarks.js +++ b/public/scripts/bookmarks.js @@ -251,7 +251,7 @@ async function convertSoloToGroupChat() { const metadata = Object.assign({}, chat_metadata); delete metadata.main_chat; - const createGroupResponse = await fetch('/creategroup', { + const createGroupResponse = await fetch('/api/groups/create', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ @@ -320,7 +320,7 @@ async function convertSoloToGroupChat() { } // Save group chat - const createChatResponse = await fetch('/savegroupchat', { + const createChatResponse = await fetch('/api/chats/group/save', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: chatName, chat: groupChat }), diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index fe3298723..b906cf2cf 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -116,7 +116,7 @@ setInterval(groupChatAutoModeWorker, 5000); const saveGroupDebounced = debounce(async (group, reload) => await _save(group, reload), 500); async function _save(group, reload = true) { - await fetch('/editgroup', { + await fetch('/api/groups/edit', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify(group), @@ -152,7 +152,7 @@ async function regenerateGroup() { } async function loadGroupChat(chatId) { - const response = await fetch('/getgroupchat', { + const response = await fetch('/api/chats/group/get', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: chatId }), @@ -401,7 +401,7 @@ async function saveGroupChat(groupId, shouldSaveGroup) { const group = groups.find(x => x.id == groupId); const chat_id = group.chat_id; group['date_last_chat'] = Date.now(); - const response = await fetch('/savegroupchat', { + const response = await fetch('/api/chats/group/save', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: chat_id, chat: [...chat] }), @@ -455,7 +455,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) { } if (hadChanges) { - const saveChatResponse = await fetch('/savegroupchat', { + const saveChatResponse = await fetch('/api/chats/group/save', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: chatId, chat: [...messages] }), @@ -476,7 +476,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) { } async function getGroups() { - const response = await fetch('/getgroups', { + const response = await fetch('/api/groups/all', { method: 'POST', headers: getRequestHeaders(), }); @@ -968,7 +968,7 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i async function deleteGroup(id) { const group = groups.find((x) => x.id === id); - const response = await fetch('/deletegroup', { + const response = await fetch('/api/groups/delete', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: id }), @@ -1521,7 +1521,7 @@ async function createGroup() { const chatName = humanizedDateTime(); const chats = [chatName]; - const createGroupResponse = await fetch('/creategroup', { + const createGroupResponse = await fetch('/api/groups/create', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ @@ -1659,7 +1659,7 @@ export async function deleteGroupChat(groupId, chatId) { delete group.past_metadata[chatId]; updateChatMetadata(group.chat_metadata, true); - const response = await fetch('/deletegroupchat', { + const response = await fetch('/api/chats/group/delete', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: chatId }), @@ -1679,7 +1679,7 @@ export async function deleteGroupChat(groupId, chatId) { export async function importGroupChat(formData) { await jQuery.ajax({ type: 'POST', - url: '/importgroupchat', + url: '/api/chats/group/import', data: formData, beforeSend: function () { }, @@ -1720,7 +1720,7 @@ export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) { await editGroup(groupId, true, false); - await fetch('/savegroupchat', { + await fetch('/api/chats/group/save', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ id: name, chat: [...trimmed_chat] }), diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 0ca7b336e..21806cb99 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -813,6 +813,7 @@ async function CreateZenSliders(elmnt) { if (numVal === offVal) { handle.text('Off').css('color', 'rgba(128,128,128,0.5'); } else if (numVal === allVal) { handle.text('All'); } else { handle.css('color', ''); } + numVal = steps[stepNumber]; } //everything else uses the flat slider value //also note: the above sliders are not custom inputtable due to the array aliasing diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index cf1c45426..d29cbe222 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -424,7 +424,7 @@ async function loadWorldInfoData(name) { return worldInfoCache[name]; } - const response = await fetch('/getworldinfo', { + const response = await fetch('/api/worldinfo/get', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ name: name }), @@ -1402,7 +1402,7 @@ function createWorldInfoEntry(name, data, fromSlashCommand = false) { } async function _save(name, data) { - await fetch('/editworldinfo', { + await fetch('/api/worldinfo/edit', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ name: name, data: data }), @@ -1464,7 +1464,7 @@ async function deleteWorldInfo(worldInfoName) { return; } - const response = await fetch('/deleteworldinfo', { + const response = await fetch('/api/worldinfo/delete', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify({ name: worldInfoName }), @@ -2269,7 +2269,7 @@ export async function importWorldInfo(file) { jQuery.ajax({ type: 'POST', - url: '/importworldinfo', + url: '/api/worldinfo/import', data: formData, beforeSend: () => { }, cache: false, diff --git a/server.js b/server.js index eb5c1c1a4..6a82378ae 100644 --- a/server.js +++ b/server.js @@ -6,7 +6,6 @@ const fs = require('fs'); const http = require('http'); const https = require('https'); const path = require('path'); -const readline = require('readline'); const util = require('util'); const { Readable } = require('stream'); @@ -48,11 +47,10 @@ const { jsonParser, urlencodedParser } = require('./src/express-common.js'); const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); -const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension } = require('./src/util'); +const { delay, getVersion, getConfigValue, color, uuidv4, tryParse, clientRelativePath, removeFileExtension, generateTimestamp, removeOldBackups } = require('./src/util'); const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); -const { readWorldInfoFile } = require('./src/worldinfo'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 @@ -723,55 +721,6 @@ app.post('/api/textgenerationwebui/generate', jsonParser, async function (reques } }); -app.post('/savechat', jsonParser, function (request, response) { - try { - var dir_name = String(request.body.avatar_url).replace('.png', ''); - let chat_data = request.body.chat; - let jsonlData = chat_data.map(JSON.stringify).join('\n'); - writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); - backupChat(dir_name, jsonlData); - return response.send({ result: 'ok' }); - } catch (error) { - response.send(error); - return console.log(error); - } -}); - -app.post('/getchat', jsonParser, function (request, response) { - try { - const dirName = String(request.body.avatar_url).replace('.png', ''); - const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName); - - //if no chat dir for the character is found, make one with the character name - if (!chatDirExists) { - fs.mkdirSync(DIRECTORIES.chats + dirName); - return response.send({}); - } - - - if (!request.body.file_name) { - return response.send({}); - } - - const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`; - const chatFileExists = fs.existsSync(fileName); - - if (!chatFileExists) { - return response.send({}); - } - - const data = fs.readFileSync(fileName, 'utf8'); - const lines = data.split('\n'); - - // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x); - return response.send(jsonData); - } catch (error) { - console.error(error); - return response.send({}); - } -}); - // Only called for kobold app.post('/getstatus', jsonParser, async function (request, response) { if (!request.body) return response.sendStatus(400); @@ -829,30 +778,6 @@ app.post('/getstatus', jsonParser, async function (request, response) { } }); - -app.post('/renamechat', jsonParser, async function (request, response) { - if (!request.body || !request.body.original_file || !request.body.renamed_file) { - return response.sendStatus(400); - } - - const pathToFolder = request.body.is_group - ? DIRECTORIES.groupChats - : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); - const pathToOriginalFile = path.join(pathToFolder, request.body.original_file); - const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file); - console.log('Old chat name', pathToOriginalFile); - console.log('New chat name', pathToRenamedFile); - - if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) { - console.log('Either Source or Destination files are not available'); - return response.status(400).send({ error: true }); - } - - console.log('Successfully renamed.'); - fs.renameSync(pathToOriginalFile, pathToRenamedFile); - return response.send({ ok: true }); -}); - /** * Handle a POST request to get the stats object * @@ -967,37 +892,6 @@ app.post('/delbackground', jsonParser, function (request, response) { return response.send('ok'); }); -app.post('/delchat', jsonParser, function (request, response) { - console.log('/delchat entered'); - if (!request.body) { - console.log('no request body seen'); - return response.sendStatus(400); - } - - if (request.body.chatfile !== sanitize(request.body.chatfile)) { - console.error('Malicious chat name prevented'); - return response.sendStatus(403); - } - - const dirName = String(request.body.avatar_url).replace('.png', ''); - const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`; - const chatFileExists = fs.existsSync(fileName); - - if (!chatFileExists) { - console.log(`Chat file not found '${fileName}'`); - return response.sendStatus(400); - } else { - console.log('found the chat file: ' + fileName); - /* fs.unlinkSync(fileName); */ - fs.rmSync(fileName); - console.log('deleted chat file: ' + fileName); - - } - - - return response.send('ok'); -}); - app.post('/renamebackground', jsonParser, function (request, response) { if (!request.body) return response.sendStatus(400); @@ -1165,34 +1059,6 @@ app.post('/getsettings', jsonParser, (request, response) => { }); }); -app.post('/getworldinfo', jsonParser, (request, response) => { - if (!request.body?.name) { - return response.sendStatus(400); - } - - const file = readWorldInfoFile(request.body.name); - - return response.send(file); -}); - -app.post('/deleteworldinfo', jsonParser, (request, response) => { - if (!request.body?.name) { - return response.sendStatus(400); - } - - const worldInfoName = request.body.name; - const filename = sanitize(`${worldInfoName}.json`); - const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); - - if (!fs.existsSync(pathToWorldInfo)) { - throw new Error(`World info file ${filename} doesn't exist.`); - } - - fs.rmSync(pathToWorldInfo); - - return response.sendStatus(200); -}); - app.post('/savetheme', jsonParser, (request, response) => { if (!request.body || !request.body.name) { return response.sendStatus(400); @@ -1237,275 +1103,6 @@ function getImages(path) { .sort(Intl.Collator().compare); } -app.post('/exportchat', jsonParser, async function (request, response) { - if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) { - return response.sendStatus(400); - } - const pathToFolder = request.body.is_group - ? DIRECTORIES.groupChats - : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); - let filename = path.join(pathToFolder, request.body.file); - let exportfilename = request.body.exportfilename; - if (!fs.existsSync(filename)) { - const errorMessage = { - message: `Could not find JSONL file to export. Source chat file: ${filename}.`, - }; - console.log(errorMessage.message); - return response.status(404).json(errorMessage); - } - try { - // Short path for JSONL files - if (request.body.format == 'jsonl') { - try { - const rawFile = fs.readFileSync(filename, 'utf8'); - const successMessage = { - message: `Chat saved to ${exportfilename}`, - result: rawFile, - }; - - console.log(`Chat exported as ${exportfilename}`); - return response.status(200).json(successMessage); - } - catch (err) { - console.error(err); - const errorMessage = { - message: `Could not read JSONL file to export. Source chat file: ${filename}.`, - }; - console.log(errorMessage.message); - return response.status(500).json(errorMessage); - } - } - - const readStream = fs.createReadStream(filename); - const rl = readline.createInterface({ - input: readStream, - }); - let buffer = ''; - rl.on('line', (line) => { - const data = JSON.parse(line); - if (data.mes) { - const name = data.name; - const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n'); - buffer += (`${name}: ${message}\n\n`); - } - }); - rl.on('close', () => { - const successMessage = { - message: `Chat saved to ${exportfilename}`, - result: buffer, - }; - console.log(`Chat exported as ${exportfilename}`); - return response.status(200).json(successMessage); - }); - } - catch (err) { - console.log('chat export failed.'); - console.log(err); - return response.sendStatus(400); - } -}); - -app.post('/importgroupchat', urlencodedParser, function (request, response) { - try { - const filedata = request.file; - - if (!filedata) { - return response.sendStatus(400); - } - - const chatname = humanizedISO8601DateTime(); - const pathToUpload = path.join(UPLOADS_PATH, filedata.filename); - const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`); - fs.copyFileSync(pathToUpload, pathToNewFile); - fs.unlinkSync(pathToUpload); - return response.send({ res: chatname }); - } catch (error) { - console.error(error); - return response.send({ error: true }); - } -}); - -app.post('/importchat', urlencodedParser, function (request, response) { - if (!request.body) return response.sendStatus(400); - - var format = request.body.file_type; - let filedata = request.file; - let avatar_url = (request.body.avatar_url).replace('.png', ''); - let ch_name = request.body.character_name; - let user_name = request.body.user_name || 'You'; - - if (!filedata) { - return response.sendStatus(400); - } - - try { - const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8'); - - if (format === 'json') { - const jsonData = JSON.parse(data); - if (jsonData.histories !== undefined) { - //console.log('/importchat confirms JSON histories are defined'); - const chat = { - from(history) { - return [ - { - user_name: user_name, - character_name: ch_name, - create_date: humanizedISO8601DateTime(), - }, - ...history.msgs.map( - (message) => ({ - name: message.src.is_human ? user_name : ch_name, - is_user: message.src.is_human, - send_date: humanizedISO8601DateTime(), - mes: message.text, - }), - )]; - }, - }; - - const newChats = []; - (jsonData.histories.histories ?? []).forEach((history) => { - newChats.push(chat.from(history)); - }); - - const errors = []; - - for (const chat of newChats) { - const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`; - const fileContent = chat.map(tryParse).filter(x => x).join('\n'); - - try { - writeFileAtomicSync(filePath, fileContent, 'utf8'); - } catch (err) { - errors.push(err); - } - } - - if (0 < errors.length) { - response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors)); - } - - response.send({ res: true }); - } else if (Array.isArray(jsonData.data_visible)) { - // oobabooga's format - /** @type {object[]} */ - const chat = [{ - user_name: user_name, - character_name: ch_name, - create_date: humanizedISO8601DateTime(), - }]; - - for (const arr of jsonData.data_visible) { - if (arr[0]) { - const userMessage = { - name: user_name, - is_user: true, - send_date: humanizedISO8601DateTime(), - mes: arr[0], - }; - chat.push(userMessage); - } - if (arr[1]) { - const charMessage = { - name: ch_name, - is_user: false, - send_date: humanizedISO8601DateTime(), - mes: arr[1], - }; - chat.push(charMessage); - } - } - - const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n'); - writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8'); - - response.send({ res: true }); - } else { - console.log('Incorrect chat format .json'); - return response.send({ error: true }); - } - } - - if (format === 'jsonl') { - const line = data.split('\n')[0]; - - let jsonData = JSON.parse(line); - - if (jsonData.user_name !== undefined || jsonData.name !== undefined) { - fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`)); - response.send({ res: true }); - } else { - console.log('Incorrect chat format .jsonl'); - return response.send({ error: true }); - } - } - } catch (error) { - console.error(error); - return response.send({ error: true }); - } -}); - -app.post('/importworldinfo', urlencodedParser, (request, response) => { - if (!request.file) return response.sendStatus(400); - - const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`; - - let fileContents = null; - - if (request.body.convertedData) { - fileContents = request.body.convertedData; - } else { - const pathToUpload = path.join(UPLOADS_PATH, request.file.filename); - fileContents = fs.readFileSync(pathToUpload, 'utf8'); - fs.unlinkSync(pathToUpload); - } - - try { - const worldContent = JSON.parse(fileContents); - if (!('entries' in worldContent)) { - throw new Error('File must contain a world info entries list'); - } - } catch (err) { - return response.status(400).send('Is not a valid world info file'); - } - - const pathToNewFile = path.join(DIRECTORIES.worlds, filename); - const worldName = path.parse(pathToNewFile).name; - - if (!worldName) { - return response.status(400).send('World file must have a name'); - } - - writeFileAtomicSync(pathToNewFile, fileContents); - return response.send({ name: worldName }); -}); - -app.post('/editworldinfo', jsonParser, (request, response) => { - if (!request.body) { - return response.sendStatus(400); - } - - if (!request.body.name) { - return response.status(400).send('World file must have a name'); - } - - try { - if (!('entries' in request.body.data)) { - throw new Error('World info must contain an entries list'); - } - } catch (err) { - return response.status(400).send('Is not a valid world info file'); - } - - const filename = `${sanitize(request.body.name)}.json`; - const pathToFile = path.join(DIRECTORIES.worlds, filename); - - writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4)); - - return response.send({ ok: true }); -}); - app.post('/uploaduseravatar', urlencodedParser, async (request, response) => { if (!request.file) return response.sendStatus(400); @@ -1614,181 +1211,6 @@ app.post('/listimgfiles/:folder', (req, res) => { }); -app.post('/getgroups', jsonParser, (_, response) => { - const groups = []; - - if (!fs.existsSync(DIRECTORIES.groups)) { - fs.mkdirSync(DIRECTORIES.groups); - } - - const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json'); - const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl'); - - files.forEach(function (file) { - try { - const filePath = path.join(DIRECTORIES.groups, file); - const fileContents = fs.readFileSync(filePath, 'utf8'); - const group = JSON.parse(fileContents); - const groupStat = fs.statSync(filePath); - group['date_added'] = groupStat.birthtimeMs; - group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs); - - let chat_size = 0; - let date_last_chat = 0; - - if (Array.isArray(group.chats) && Array.isArray(chats)) { - for (const chat of chats) { - if (group.chats.includes(path.parse(chat).name)) { - const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat)); - chat_size += chatStat.size; - date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs); - } - } - } - - group['date_last_chat'] = date_last_chat; - group['chat_size'] = chat_size; - groups.push(group); - } - catch (error) { - console.error(error); - } - }); - - return response.send(groups); -}); - -app.post('/creategroup', jsonParser, (request, response) => { - if (!request.body) { - return response.sendStatus(400); - } - - const id = String(Date.now()); - const groupMetadata = { - id: id, - name: request.body.name ?? 'New Group', - members: request.body.members ?? [], - avatar_url: request.body.avatar_url, - allow_self_responses: !!request.body.allow_self_responses, - activation_strategy: request.body.activation_strategy ?? 0, - generation_mode: request.body.generation_mode ?? 0, - disabled_members: request.body.disabled_members ?? [], - chat_metadata: request.body.chat_metadata ?? {}, - fav: request.body.fav, - chat_id: request.body.chat_id ?? id, - chats: request.body.chats ?? [id], - }; - const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`); - const fileData = JSON.stringify(groupMetadata); - - if (!fs.existsSync(DIRECTORIES.groups)) { - fs.mkdirSync(DIRECTORIES.groups); - } - - writeFileAtomicSync(pathToFile, fileData); - return response.send(groupMetadata); -}); - -app.post('/editgroup', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`); - const fileData = JSON.stringify(request.body); - - writeFileAtomicSync(pathToFile, fileData); - return response.send({ ok: true }); -}); - -app.post('/getgroupchat', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (fs.existsSync(pathToFile)) { - const data = fs.readFileSync(pathToFile, 'utf8'); - const lines = data.split('\n'); - - // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map(line => tryParse(line)).filter(x => x); - return response.send(jsonData); - } else { - return response.send([]); - } -}); - -app.post('/deletegroupchat', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (fs.existsSync(pathToFile)) { - fs.rmSync(pathToFile); - return response.send({ ok: true }); - } - - return response.send({ error: true }); -}); - -app.post('/savegroupchat', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (!fs.existsSync(DIRECTORIES.groupChats)) { - fs.mkdirSync(DIRECTORIES.groupChats); - } - - let chat_data = request.body.chat; - let jsonlData = chat_data.map(JSON.stringify).join('\n'); - writeFileAtomicSync(pathToFile, jsonlData, 'utf8'); - backupChat(String(id), jsonlData); - return response.send({ ok: true }); -}); - -app.post('/deletegroup', jsonParser, async (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`)); - - try { - // Delete group chats - const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8')); - - if (group && Array.isArray(group.chats)) { - for (const chat of group.chats) { - console.log('Deleting group chat', chat); - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (fs.existsSync(pathToFile)) { - fs.rmSync(pathToFile); - } - } - } - } catch (error) { - console.error('Could not delete group chats. Clean them up manually.', error); - } - - if (fs.existsSync(pathToGroup)) { - fs.rmSync(pathToGroup); - } - - return response.send({ ok: true }); -}); - function cleanUploads() { try { if (fs.existsSync(UPLOADS_PATH)) { @@ -2605,6 +2027,30 @@ redirect('/importcharacter', '/api/characters/import'); redirect('/dupecharacter', '/api/characters/duplicate'); redirect('/exportcharacter', '/api/characters/export'); +// Redirect deprecated chat API endpoints +redirect('/savechat', '/api/chats/save'); +redirect('/getchat', '/api/chats/get'); +redirect('/renamechat', '/api/chats/rename'); +redirect('/delchat', '/api/chats/delete'); +redirect('/exportchat', '/api/chats/export'); +redirect('/importgroupchat', '/api/chats/group/import'); +redirect('/importchat', '/api/chats/import'); +redirect('/getgroupchat', '/api/chats/group/get'); +redirect('/deletegroupchat', '/api/chats/group/delete'); +redirect('/savegroupchat', '/api/chats/group/save'); + +// Redirect deprecated group API endpoints +redirect('/getgroups', '/api/groups/all'); +redirect('/creategroup', '/api/groups/create'); +redirect('/editgroup', '/api/groups/edit'); +redirect('/deletegroup', '/api/groups/delete'); + +// Redirect deprecated worldinfo API endpoints +redirect('/getworldinfo', '/api/worldinfo/get'); +redirect('/deleteworldinfo', '/api/worldinfo/delete'); +redirect('/importworldinfo', '/api/worldinfo/import'); +redirect('/editworldinfo', '/api/worldinfo/edit'); + // ** REST CLIENT ASYNC WRAPPERS ** /** @@ -2656,6 +2102,15 @@ app.use('/api/files', require('./src/endpoints/files').router); // Character management app.use('/api/characters', require('./src/endpoints/characters').router); +// Chat management +app.use('/api/chats', require('./src/endpoints/chats').router); + +// Group management +app.use('/api/groups', require('./src/endpoints/groups').router); + +// World info management +app.use('/api/worldinfo', require('./src/endpoints/worldinfo').router); + // Character sprite management app.use('/api/sprites', require('./src/endpoints/sprites').router); @@ -2760,47 +2215,6 @@ if (cliArguments.ssl) { ); } -function generateTimestamp() { - const now = new Date(); - const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, '0'); - const day = String(now.getDate()).padStart(2, '0'); - const hours = String(now.getHours()).padStart(2, '0'); - const minutes = String(now.getMinutes()).padStart(2, '0'); - const seconds = String(now.getSeconds()).padStart(2, '0'); - - return `${year}${month}${day}-${hours}${minutes}${seconds}`; -} - -/** - * - * @param {string} name - * @param {string} chat - */ -function backupChat(name, chat) { - try { - const isBackupDisabled = getConfigValue('disableChatBackup', false); - - if (isBackupDisabled) { - return; - } - - if (!fs.existsSync(DIRECTORIES.backups)) { - fs.mkdirSync(DIRECTORIES.backups); - } - - // replace non-alphanumeric characters with underscores - name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase(); - - const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`); - writeFileAtomicSync(backupFile, chat, 'utf-8'); - - removeOldBackups(`chat_${name}_`); - } catch (err) { - console.log(`Could not backup chat for ${name}`, err); - } -} - function backupSettings() { try { if (!fs.existsSync(DIRECTORIES.backups)) { @@ -2816,21 +2230,6 @@ function backupSettings() { } } -/** - * @param {string} prefix - */ -function removeOldBackups(prefix) { - const MAX_BACKUPS = 25; - - let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix)); - if (files.length > MAX_BACKUPS) { - files = files.map(f => path.join(DIRECTORIES.backups, f)); - files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); - - fs.rmSync(files[0]); - } -} - function ensurePublicDirectoriesExist() { for (const dir of Object.values(DIRECTORIES)) { if (!fs.existsSync(dir)) { diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 5f858cb11..f21c29a5f 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -16,7 +16,7 @@ const { jsonParser, urlencodedParser } = require('../express-common'); const { deepMerge, humanizedISO8601DateTime, tryParse } = require('../util'); const { TavernCardValidator } = require('../validator/TavernCardValidator'); const characterCardParser = require('../character-card-parser.js'); -const { readWorldInfoFile, convertWorldInfoToCharacterBook } = require('../worldinfo'); +const { readWorldInfoFile } = require('./worldinfo'); const { invalidateThumbnail } = require('./thumbnails'); const { importRisuSprites } = require('./sprites'); @@ -330,6 +330,46 @@ function charaFormatData(data) { return char; } +/** + * @param {string} name Name of World Info file + * @param {object} entries Entries object + */ +function convertWorldInfoToCharacterBook(name, entries) { + /** @type {{ entries: object[]; name: string }} */ + const result = { entries: [], name }; + + for (const index in entries) { + const entry = entries[index]; + + const originalEntry = { + id: entry.uid, + keys: entry.key, + secondary_keys: entry.keysecondary, + comment: entry.comment, + content: entry.content, + constant: entry.constant, + selective: entry.selective, + insertion_order: entry.order, + enabled: !entry.disable, + position: entry.position == 0 ? 'before_char' : 'after_char', + extensions: { + position: entry.position, + exclude_recursion: entry.excludeRecursion, + display_index: entry.displayIndex, + probability: entry.probability ?? null, + useProbability: entry.useProbability ?? false, + depth: entry.depth ?? 4, + selectiveLogic: entry.selectiveLogic ?? 0, + group: entry.group ?? '', + }, + }; + + result.entries.push(originalEntry); + } + + return result; +} + const router = express.Router(); router.post('/create', urlencodedParser, async function (request, response) { diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js new file mode 100644 index 000000000..b8063b72f --- /dev/null +++ b/src/endpoints/chats.js @@ -0,0 +1,411 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const express = require('express'); +const sanitize = require('sanitize-filename'); +const writeFileAtomicSync = require('write-file-atomic').sync; + +const { jsonParser, urlencodedParser } = require('../express-common'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util'); + +/** + * + * @param {string} name + * @param {string} chat + */ +function backupChat(name, chat) { + try { + const isBackupDisabled = getConfigValue('disableChatBackup', false); + + if (isBackupDisabled) { + return; + } + + if (!fs.existsSync(DIRECTORIES.backups)) { + fs.mkdirSync(DIRECTORIES.backups); + } + + // replace non-alphanumeric characters with underscores + name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase(); + + const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`); + writeFileAtomicSync(backupFile, chat, 'utf-8'); + + removeOldBackups(`chat_${name}_`); + } catch (err) { + console.log(`Could not backup chat for ${name}`, err); + } +} + +const router = express.Router(); + +router.post('/save', jsonParser, function (request, response) { + try { + var dir_name = String(request.body.avatar_url).replace('.png', ''); + let chat_data = request.body.chat; + let jsonlData = chat_data.map(JSON.stringify).join('\n'); + writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); + backupChat(dir_name, jsonlData); + return response.send({ result: 'ok' }); + } catch (error) { + response.send(error); + return console.log(error); + } +}); + +router.post('/get', jsonParser, function (request, response) { + try { + const dirName = String(request.body.avatar_url).replace('.png', ''); + const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName); + + //if no chat dir for the character is found, make one with the character name + if (!chatDirExists) { + fs.mkdirSync(DIRECTORIES.chats + dirName); + return response.send({}); + } + + + if (!request.body.file_name) { + return response.send({}); + } + + const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`; + const chatFileExists = fs.existsSync(fileName); + + if (!chatFileExists) { + return response.send({}); + } + + const data = fs.readFileSync(fileName, 'utf8'); + const lines = data.split('\n'); + + // Iterate through the array of strings and parse each line as JSON + const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x); + return response.send(jsonData); + } catch (error) { + console.error(error); + return response.send({}); + } +}); + + +router.post('/rename', jsonParser, async function (request, response) { + if (!request.body || !request.body.original_file || !request.body.renamed_file) { + return response.sendStatus(400); + } + + const pathToFolder = request.body.is_group + ? DIRECTORIES.groupChats + : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); + const pathToOriginalFile = path.join(pathToFolder, request.body.original_file); + const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file); + console.log('Old chat name', pathToOriginalFile); + console.log('New chat name', pathToRenamedFile); + + if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) { + console.log('Either Source or Destination files are not available'); + return response.status(400).send({ error: true }); + } + + console.log('Successfully renamed.'); + fs.renameSync(pathToOriginalFile, pathToRenamedFile); + return response.send({ ok: true }); +}); + +router.post('/delete', jsonParser, function (request, response) { + console.log('/api/chats/delete entered'); + if (!request.body) { + console.log('no request body seen'); + return response.sendStatus(400); + } + + if (request.body.chatfile !== sanitize(request.body.chatfile)) { + console.error('Malicious chat name prevented'); + return response.sendStatus(403); + } + + const dirName = String(request.body.avatar_url).replace('.png', ''); + const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`; + const chatFileExists = fs.existsSync(fileName); + + if (!chatFileExists) { + console.log(`Chat file not found '${fileName}'`); + return response.sendStatus(400); + } else { + console.log('found the chat file: ' + fileName); + /* fs.unlinkSync(fileName); */ + fs.rmSync(fileName); + console.log('deleted chat file: ' + fileName); + + } + + + return response.send('ok'); +}); + +router.post('/export', jsonParser, async function (request, response) { + if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) { + return response.sendStatus(400); + } + const pathToFolder = request.body.is_group + ? DIRECTORIES.groupChats + : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); + let filename = path.join(pathToFolder, request.body.file); + let exportfilename = request.body.exportfilename; + if (!fs.existsSync(filename)) { + const errorMessage = { + message: `Could not find JSONL file to export. Source chat file: ${filename}.`, + }; + console.log(errorMessage.message); + return response.status(404).json(errorMessage); + } + try { + // Short path for JSONL files + if (request.body.format == 'jsonl') { + try { + const rawFile = fs.readFileSync(filename, 'utf8'); + const successMessage = { + message: `Chat saved to ${exportfilename}`, + result: rawFile, + }; + + console.log(`Chat exported as ${exportfilename}`); + return response.status(200).json(successMessage); + } + catch (err) { + console.error(err); + const errorMessage = { + message: `Could not read JSONL file to export. Source chat file: ${filename}.`, + }; + console.log(errorMessage.message); + return response.status(500).json(errorMessage); + } + } + + const readStream = fs.createReadStream(filename); + const rl = readline.createInterface({ + input: readStream, + }); + let buffer = ''; + rl.on('line', (line) => { + const data = JSON.parse(line); + if (data.mes) { + const name = data.name; + const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n'); + buffer += (`${name}: ${message}\n\n`); + } + }); + rl.on('close', () => { + const successMessage = { + message: `Chat saved to ${exportfilename}`, + result: buffer, + }; + console.log(`Chat exported as ${exportfilename}`); + return response.status(200).json(successMessage); + }); + } + catch (err) { + console.log('chat export failed.'); + console.log(err); + return response.sendStatus(400); + } +}); + +router.post('/group/import', urlencodedParser, function (request, response) { + try { + const filedata = request.file; + + if (!filedata) { + return response.sendStatus(400); + } + + const chatname = humanizedISO8601DateTime(); + const pathToUpload = path.join(UPLOADS_PATH, filedata.filename); + const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`); + fs.copyFileSync(pathToUpload, pathToNewFile); + fs.unlinkSync(pathToUpload); + return response.send({ res: chatname }); + } catch (error) { + console.error(error); + return response.send({ error: true }); + } +}); + +router.post('/import', urlencodedParser, function (request, response) { + if (!request.body) return response.sendStatus(400); + + var format = request.body.file_type; + let filedata = request.file; + let avatar_url = (request.body.avatar_url).replace('.png', ''); + let ch_name = request.body.character_name; + let user_name = request.body.user_name || 'You'; + + if (!filedata) { + return response.sendStatus(400); + } + + try { + const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8'); + + if (format === 'json') { + const jsonData = JSON.parse(data); + if (jsonData.histories !== undefined) { + //console.log('/api/chats/import confirms JSON histories are defined'); + const chat = { + from(history) { + return [ + { + user_name: user_name, + character_name: ch_name, + create_date: humanizedISO8601DateTime(), + }, + ...history.msgs.map( + (message) => ({ + name: message.src.is_human ? user_name : ch_name, + is_user: message.src.is_human, + send_date: humanizedISO8601DateTime(), + mes: message.text, + }), + )]; + }, + }; + + const newChats = []; + (jsonData.histories.histories ?? []).forEach((history) => { + newChats.push(chat.from(history)); + }); + + const errors = []; + + for (const chat of newChats) { + const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`; + const fileContent = chat.map(tryParse).filter(x => x).join('\n'); + + try { + writeFileAtomicSync(filePath, fileContent, 'utf8'); + } catch (err) { + errors.push(err); + } + } + + if (0 < errors.length) { + response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors)); + } + + response.send({ res: true }); + } else if (Array.isArray(jsonData.data_visible)) { + // oobabooga's format + /** @type {object[]} */ + const chat = [{ + user_name: user_name, + character_name: ch_name, + create_date: humanizedISO8601DateTime(), + }]; + + for (const arr of jsonData.data_visible) { + if (arr[0]) { + const userMessage = { + name: user_name, + is_user: true, + send_date: humanizedISO8601DateTime(), + mes: arr[0], + }; + chat.push(userMessage); + } + if (arr[1]) { + const charMessage = { + name: ch_name, + is_user: false, + send_date: humanizedISO8601DateTime(), + mes: arr[1], + }; + chat.push(charMessage); + } + } + + const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n'); + writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8'); + + response.send({ res: true }); + } else { + console.log('Incorrect chat format .json'); + return response.send({ error: true }); + } + } + + if (format === 'jsonl') { + const line = data.split('\n')[0]; + + let jsonData = JSON.parse(line); + + if (jsonData.user_name !== undefined || jsonData.name !== undefined) { + fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`)); + response.send({ res: true }); + } else { + console.log('Incorrect chat format .jsonl'); + return response.send({ error: true }); + } + } + } catch (error) { + console.error(error); + return response.send({ error: true }); + } +}); + +router.post('/group/get', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (fs.existsSync(pathToFile)) { + const data = fs.readFileSync(pathToFile, 'utf8'); + const lines = data.split('\n'); + + // Iterate through the array of strings and parse each line as JSON + const jsonData = lines.map(line => tryParse(line)).filter(x => x); + return response.send(jsonData); + } else { + return response.send([]); + } +}); + +router.post('/group/delete', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (fs.existsSync(pathToFile)) { + fs.rmSync(pathToFile); + return response.send({ ok: true }); + } + + return response.send({ error: true }); +}); + +router.post('/group/save', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (!fs.existsSync(DIRECTORIES.groupChats)) { + fs.mkdirSync(DIRECTORIES.groupChats); + } + + let chat_data = request.body.chat; + let jsonlData = chat_data.map(JSON.stringify).join('\n'); + writeFileAtomicSync(pathToFile, jsonlData, 'utf8'); + backupChat(String(id), jsonlData); + return response.send({ ok: true }); +}); + +module.exports = { router }; diff --git a/src/endpoints/groups.js b/src/endpoints/groups.js new file mode 100644 index 000000000..d7341ff04 --- /dev/null +++ b/src/endpoints/groups.js @@ -0,0 +1,133 @@ +const fs = require('fs'); +const path = require('path'); +const express = require('express'); +const sanitize = require('sanitize-filename'); +const writeFileAtomicSync = require('write-file-atomic').sync; + +const { jsonParser } = require('../express-common'); +const { DIRECTORIES } = require('../constants'); +const { humanizedISO8601DateTime } = require('../util'); + +const router = express.Router(); + +router.post('/all', jsonParser, (_, response) => { + const groups = []; + + if (!fs.existsSync(DIRECTORIES.groups)) { + fs.mkdirSync(DIRECTORIES.groups); + } + + const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json'); + const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl'); + + files.forEach(function (file) { + try { + const filePath = path.join(DIRECTORIES.groups, file); + const fileContents = fs.readFileSync(filePath, 'utf8'); + const group = JSON.parse(fileContents); + const groupStat = fs.statSync(filePath); + group['date_added'] = groupStat.birthtimeMs; + group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs); + + let chat_size = 0; + let date_last_chat = 0; + + if (Array.isArray(group.chats) && Array.isArray(chats)) { + for (const chat of chats) { + if (group.chats.includes(path.parse(chat).name)) { + const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat)); + chat_size += chatStat.size; + date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs); + } + } + } + + group['date_last_chat'] = date_last_chat; + group['chat_size'] = chat_size; + groups.push(group); + } + catch (error) { + console.error(error); + } + }); + + return response.send(groups); +}); + +router.post('/create', jsonParser, (request, response) => { + if (!request.body) { + return response.sendStatus(400); + } + + const id = String(Date.now()); + const groupMetadata = { + id: id, + name: request.body.name ?? 'New Group', + members: request.body.members ?? [], + avatar_url: request.body.avatar_url, + allow_self_responses: !!request.body.allow_self_responses, + activation_strategy: request.body.activation_strategy ?? 0, + generation_mode: request.body.generation_mode ?? 0, + disabled_members: request.body.disabled_members ?? [], + chat_metadata: request.body.chat_metadata ?? {}, + fav: request.body.fav, + chat_id: request.body.chat_id ?? id, + chats: request.body.chats ?? [id], + }; + const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`); + const fileData = JSON.stringify(groupMetadata); + + if (!fs.existsSync(DIRECTORIES.groups)) { + fs.mkdirSync(DIRECTORIES.groups); + } + + writeFileAtomicSync(pathToFile, fileData); + return response.send(groupMetadata); +}); + +router.post('/edit', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`); + const fileData = JSON.stringify(request.body); + + writeFileAtomicSync(pathToFile, fileData); + return response.send({ ok: true }); +}); + +router.post('/delete', jsonParser, async (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`)); + + try { + // Delete group chats + const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8')); + + if (group && Array.isArray(group.chats)) { + for (const chat of group.chats) { + console.log('Deleting group chat', chat); + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (fs.existsSync(pathToFile)) { + fs.rmSync(pathToFile); + } + } + } + } catch (error) { + console.error('Could not delete group chats. Clean them up manually.', error); + } + + if (fs.existsSync(pathToGroup)) { + fs.rmSync(pathToGroup); + } + + return response.send({ ok: true }); +}); + +module.exports = { router }; diff --git a/src/endpoints/worldinfo.js b/src/endpoints/worldinfo.js new file mode 100644 index 000000000..16a1db8eb --- /dev/null +++ b/src/endpoints/worldinfo.js @@ -0,0 +1,120 @@ +const fs = require('fs'); +const path = require('path'); +const express = require('express'); +const sanitize = require('sanitize-filename'); +const writeFileAtomicSync = require('write-file-atomic').sync; + +const { jsonParser, urlencodedParser } = require('../express-common'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); + +function readWorldInfoFile(worldInfoName) { + const dummyObject = { entries: {} }; + + if (!worldInfoName) { + return dummyObject; + } + + const filename = `${worldInfoName}.json`; + const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); + + if (!fs.existsSync(pathToWorldInfo)) { + console.log(`World info file ${filename} doesn't exist.`); + return dummyObject; + } + + const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8'); + const worldInfo = JSON.parse(worldInfoText); + return worldInfo; +} + +const router = express.Router(); + +router.post('/get', jsonParser, (request, response) => { + if (!request.body?.name) { + return response.sendStatus(400); + } + + const file = readWorldInfoFile(request.body.name); + + return response.send(file); +}); + +router.post('/delete', jsonParser, (request, response) => { + if (!request.body?.name) { + return response.sendStatus(400); + } + + const worldInfoName = request.body.name; + const filename = sanitize(`${worldInfoName}.json`); + const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); + + if (!fs.existsSync(pathToWorldInfo)) { + throw new Error(`World info file ${filename} doesn't exist.`); + } + + fs.rmSync(pathToWorldInfo); + + return response.sendStatus(200); +}); + +router.post('/import', urlencodedParser, (request, response) => { + if (!request.file) return response.sendStatus(400); + + const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`; + + let fileContents = null; + + if (request.body.convertedData) { + fileContents = request.body.convertedData; + } else { + const pathToUpload = path.join(UPLOADS_PATH, request.file.filename); + fileContents = fs.readFileSync(pathToUpload, 'utf8'); + fs.unlinkSync(pathToUpload); + } + + try { + const worldContent = JSON.parse(fileContents); + if (!('entries' in worldContent)) { + throw new Error('File must contain a world info entries list'); + } + } catch (err) { + return response.status(400).send('Is not a valid world info file'); + } + + const pathToNewFile = path.join(DIRECTORIES.worlds, filename); + const worldName = path.parse(pathToNewFile).name; + + if (!worldName) { + return response.status(400).send('World file must have a name'); + } + + writeFileAtomicSync(pathToNewFile, fileContents); + return response.send({ name: worldName }); +}); + +router.post('/edit', jsonParser, (request, response) => { + if (!request.body) { + return response.sendStatus(400); + } + + if (!request.body.name) { + return response.status(400).send('World file must have a name'); + } + + try { + if (!('entries' in request.body.data)) { + throw new Error('World info must contain an entries list'); + } + } catch (err) { + return response.status(400).send('Is not a valid world info file'); + } + + const filename = `${sanitize(request.body.name)}.json`; + const pathToFile = path.join(DIRECTORIES.worlds, filename); + + writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4)); + + return response.send({ ok: true }); +}); + +module.exports = { router, readWorldInfoFile }; diff --git a/src/util.js b/src/util.js index c8079e6b8..d4c416473 100644 --- a/src/util.js +++ b/src/util.js @@ -7,6 +7,8 @@ const mime = require('mime-types'); const yaml = require('yaml'); const { default: simpleGit } = require('simple-git'); +const { DIRECTORIES } = require('./constants'); + /** * Returns the config object from the config.yaml file. * @returns {object} Config object @@ -307,6 +309,34 @@ function removeFileExtension(filename) { return filename.replace(/\.[^.]+$/, ''); } +function generateTimestamp() { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + + return `${year}${month}${day}-${hours}${minutes}${seconds}`; +} + +/** + * @param {string} prefix + */ +function removeOldBackups(prefix) { + const MAX_BACKUPS = 25; + + let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix)); + if (files.length > MAX_BACKUPS) { + files = files.map(f => path.join(DIRECTORIES.backups, f)); + files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); + + fs.rmSync(files[0]); + } +} + + module.exports = { getConfig, getConfigValue, @@ -323,4 +353,6 @@ module.exports = { tryParse, clientRelativePath, removeFileExtension, + generateTimestamp, + removeOldBackups, }; diff --git a/src/worldinfo.js b/src/worldinfo.js deleted file mode 100644 index 31fdd21e4..000000000 --- a/src/worldinfo.js +++ /dev/null @@ -1,68 +0,0 @@ -const path = require('path'); -const fs = require('fs'); -const { DIRECTORIES } = require('./constants'); - -function readWorldInfoFile(worldInfoName) { - const dummyObject = { entries: {} }; - - if (!worldInfoName) { - return dummyObject; - } - - const filename = `${worldInfoName}.json`; - const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); - - if (!fs.existsSync(pathToWorldInfo)) { - console.log(`World info file ${filename} doesn't exist.`); - return dummyObject; - } - - const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8'); - const worldInfo = JSON.parse(worldInfoText); - return worldInfo; -} - -/** - * @param {string} name Name of World Info file - * @param {object} entries Entries object - */ -function convertWorldInfoToCharacterBook(name, entries) { - /** @type {{ entries: object[]; name: string }} */ - const result = { entries: [], name }; - - for (const index in entries) { - const entry = entries[index]; - - const originalEntry = { - id: entry.uid, - keys: entry.key, - secondary_keys: entry.keysecondary, - comment: entry.comment, - content: entry.content, - constant: entry.constant, - selective: entry.selective, - insertion_order: entry.order, - enabled: !entry.disable, - position: entry.position == 0 ? 'before_char' : 'after_char', - extensions: { - position: entry.position, - exclude_recursion: entry.excludeRecursion, - display_index: entry.displayIndex, - probability: entry.probability ?? null, - useProbability: entry.useProbability ?? false, - depth: entry.depth ?? 4, - selectiveLogic: entry.selectiveLogic ?? 0, - group: entry.group ?? '', - }, - }; - - result.entries.push(originalEntry); - } - - return result; -} - -module.exports = { - readWorldInfoFile, - convertWorldInfoToCharacterBook, -};