From e870c35613afc2d79a01397064d11e40ba55409d Mon Sep 17 00:00:00 2001 From: valadaptive Date: Tue, 5 Dec 2023 23:01:23 -0500 Subject: [PATCH] Move chat endpoints into separate file --- server.js | 430 +---------------------------------------- src/endpoints/chats.js | 411 +++++++++++++++++++++++++++++++++++++++ src/util.js | 32 +++ 3 files changed, 447 insertions(+), 426 deletions(-) create mode 100644 src/endpoints/chats.js diff --git a/server.js b/server.js index 719a2d4f3..ef95504a4 100644 --- a/server.js +++ b/server.js @@ -6,7 +6,6 @@ const fs = require('fs'); const http = require('http'); const https = require('https'); const path = require('path'); -const readline = require('readline'); const util = require('util'); const { Readable } = require('stream'); @@ -48,7 +47,7 @@ const { jsonParser, urlencodedParser } = require('./src/express-common.js'); const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); -const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension } = require('./src/util'); +const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension, generateTimestamp, removeOldBackups } = require('./src/util'); const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); @@ -723,55 +722,6 @@ app.post('/api/textgenerationwebui/generate', jsonParser, async function (reques } }); -app.post('/api/chats/save', jsonParser, function (request, response) { - try { - var dir_name = String(request.body.avatar_url).replace('.png', ''); - let chat_data = request.body.chat; - let jsonlData = chat_data.map(JSON.stringify).join('\n'); - writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); - backupChat(dir_name, jsonlData); - return response.send({ result: 'ok' }); - } catch (error) { - response.send(error); - return console.log(error); - } -}); - -app.post('/api/chats/get', jsonParser, function (request, response) { - try { - const dirName = String(request.body.avatar_url).replace('.png', ''); - const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName); - - //if no chat dir for the character is found, make one with the character name - if (!chatDirExists) { - fs.mkdirSync(DIRECTORIES.chats + dirName); - return response.send({}); - } - - - if (!request.body.file_name) { - return response.send({}); - } - - const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`; - const chatFileExists = fs.existsSync(fileName); - - if (!chatFileExists) { - return response.send({}); - } - - const data = fs.readFileSync(fileName, 'utf8'); - const lines = data.split('\n'); - - // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x); - return response.send(jsonData); - } catch (error) { - console.error(error); - return response.send({}); - } -}); - // Only called for kobold app.post('/getstatus', jsonParser, async function (request, response) { if (!request.body) return response.sendStatus(400); @@ -829,30 +779,6 @@ app.post('/getstatus', jsonParser, async function (request, response) { } }); - -app.post('/api/chats/rename', jsonParser, async function (request, response) { - if (!request.body || !request.body.original_file || !request.body.renamed_file) { - return response.sendStatus(400); - } - - const pathToFolder = request.body.is_group - ? DIRECTORIES.groupChats - : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); - const pathToOriginalFile = path.join(pathToFolder, request.body.original_file); - const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file); - console.log('Old chat name', pathToOriginalFile); - console.log('New chat name', pathToRenamedFile); - - if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) { - console.log('Either Source or Destination files are not available'); - return response.status(400).send({ error: true }); - } - - console.log('Successfully renamed.'); - fs.renameSync(pathToOriginalFile, pathToRenamedFile); - return response.send({ ok: true }); -}); - /** * Handle a POST request to get the stats object * @@ -967,37 +893,6 @@ app.post('/delbackground', jsonParser, function (request, response) { return response.send('ok'); }); -app.post('/api/chats/delete', jsonParser, function (request, response) { - console.log('/api/chats/delete entered'); - if (!request.body) { - console.log('no request body seen'); - return response.sendStatus(400); - } - - if (request.body.chatfile !== sanitize(request.body.chatfile)) { - console.error('Malicious chat name prevented'); - return response.sendStatus(403); - } - - const dirName = String(request.body.avatar_url).replace('.png', ''); - const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`; - const chatFileExists = fs.existsSync(fileName); - - if (!chatFileExists) { - console.log(`Chat file not found '${fileName}'`); - return response.sendStatus(400); - } else { - console.log('found the chat file: ' + fileName); - /* fs.unlinkSync(fileName); */ - fs.rmSync(fileName); - console.log('deleted chat file: ' + fileName); - - } - - - return response.send('ok'); -}); - app.post('/renamebackground', jsonParser, function (request, response) { if (!request.body) return response.sendStatus(400); @@ -1237,215 +1132,6 @@ function getImages(path) { .sort(Intl.Collator().compare); } -app.post('/api/chats/export', jsonParser, async function (request, response) { - if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) { - return response.sendStatus(400); - } - const pathToFolder = request.body.is_group - ? DIRECTORIES.groupChats - : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); - let filename = path.join(pathToFolder, request.body.file); - let exportfilename = request.body.exportfilename; - if (!fs.existsSync(filename)) { - const errorMessage = { - message: `Could not find JSONL file to export. Source chat file: ${filename}.`, - }; - console.log(errorMessage.message); - return response.status(404).json(errorMessage); - } - try { - // Short path for JSONL files - if (request.body.format == 'jsonl') { - try { - const rawFile = fs.readFileSync(filename, 'utf8'); - const successMessage = { - message: `Chat saved to ${exportfilename}`, - result: rawFile, - }; - - console.log(`Chat exported as ${exportfilename}`); - return response.status(200).json(successMessage); - } - catch (err) { - console.error(err); - const errorMessage = { - message: `Could not read JSONL file to export. Source chat file: ${filename}.`, - }; - console.log(errorMessage.message); - return response.status(500).json(errorMessage); - } - } - - const readStream = fs.createReadStream(filename); - const rl = readline.createInterface({ - input: readStream, - }); - let buffer = ''; - rl.on('line', (line) => { - const data = JSON.parse(line); - if (data.mes) { - const name = data.name; - const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n'); - buffer += (`${name}: ${message}\n\n`); - } - }); - rl.on('close', () => { - const successMessage = { - message: `Chat saved to ${exportfilename}`, - result: buffer, - }; - console.log(`Chat exported as ${exportfilename}`); - return response.status(200).json(successMessage); - }); - } - catch (err) { - console.log('chat export failed.'); - console.log(err); - return response.sendStatus(400); - } -}); - -app.post('/api/chats/group/import', urlencodedParser, function (request, response) { - try { - const filedata = request.file; - - if (!filedata) { - return response.sendStatus(400); - } - - const chatname = humanizedISO8601DateTime(); - const pathToUpload = path.join(UPLOADS_PATH, filedata.filename); - const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`); - fs.copyFileSync(pathToUpload, pathToNewFile); - fs.unlinkSync(pathToUpload); - return response.send({ res: chatname }); - } catch (error) { - console.error(error); - return response.send({ error: true }); - } -}); - -app.post('/api/chats/import', urlencodedParser, function (request, response) { - if (!request.body) return response.sendStatus(400); - - var format = request.body.file_type; - let filedata = request.file; - let avatar_url = (request.body.avatar_url).replace('.png', ''); - let ch_name = request.body.character_name; - let user_name = request.body.user_name || 'You'; - - if (!filedata) { - return response.sendStatus(400); - } - - try { - const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8'); - - if (format === 'json') { - const jsonData = JSON.parse(data); - if (jsonData.histories !== undefined) { - //console.log('/api/chats/import confirms JSON histories are defined'); - const chat = { - from(history) { - return [ - { - user_name: user_name, - character_name: ch_name, - create_date: humanizedISO8601DateTime(), - }, - ...history.msgs.map( - (message) => ({ - name: message.src.is_human ? user_name : ch_name, - is_user: message.src.is_human, - send_date: humanizedISO8601DateTime(), - mes: message.text, - }), - )]; - }, - }; - - const newChats = []; - (jsonData.histories.histories ?? []).forEach((history) => { - newChats.push(chat.from(history)); - }); - - const errors = []; - - for (const chat of newChats) { - const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`; - const fileContent = chat.map(tryParse).filter(x => x).join('\n'); - - try { - writeFileAtomicSync(filePath, fileContent, 'utf8'); - } catch (err) { - errors.push(err); - } - } - - if (0 < errors.length) { - response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors)); - } - - response.send({ res: true }); - } else if (Array.isArray(jsonData.data_visible)) { - // oobabooga's format - /** @type {object[]} */ - const chat = [{ - user_name: user_name, - character_name: ch_name, - create_date: humanizedISO8601DateTime(), - }]; - - for (const arr of jsonData.data_visible) { - if (arr[0]) { - const userMessage = { - name: user_name, - is_user: true, - send_date: humanizedISO8601DateTime(), - mes: arr[0], - }; - chat.push(userMessage); - } - if (arr[1]) { - const charMessage = { - name: ch_name, - is_user: false, - send_date: humanizedISO8601DateTime(), - mes: arr[1], - }; - chat.push(charMessage); - } - } - - const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n'); - writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8'); - - response.send({ res: true }); - } else { - console.log('Incorrect chat format .json'); - return response.send({ error: true }); - } - } - - if (format === 'jsonl') { - const line = data.split('\n')[0]; - - let jsonData = JSON.parse(line); - - if (jsonData.user_name !== undefined || jsonData.name !== undefined) { - fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`)); - response.send({ res: true }); - } else { - console.log('Incorrect chat format .jsonl'); - return response.send({ error: true }); - } - } - } catch (error) { - console.error(error); - return response.send({ error: true }); - } -}); - app.post('/importworldinfo', urlencodedParser, (request, response) => { if (!request.file) return response.sendStatus(400); @@ -1701,61 +1387,6 @@ app.post('/editgroup', jsonParser, (request, response) => { return response.send({ ok: true }); }); -app.post('/api/chats/group/get', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (fs.existsSync(pathToFile)) { - const data = fs.readFileSync(pathToFile, 'utf8'); - const lines = data.split('\n'); - - // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map(line => tryParse(line)).filter(x => x); - return response.send(jsonData); - } else { - return response.send([]); - } -}); - -app.post('/api/chats/group/delete', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (fs.existsSync(pathToFile)) { - fs.rmSync(pathToFile); - return response.send({ ok: true }); - } - - return response.send({ error: true }); -}); - -app.post('/api/chats/group/save', jsonParser, (request, response) => { - if (!request.body || !request.body.id) { - return response.sendStatus(400); - } - - const id = request.body.id; - const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); - - if (!fs.existsSync(DIRECTORIES.groupChats)) { - fs.mkdirSync(DIRECTORIES.groupChats); - } - - let chat_data = request.body.chat; - let jsonlData = chat_data.map(JSON.stringify).join('\n'); - writeFileAtomicSync(pathToFile, jsonlData, 'utf8'); - backupChat(String(id), jsonlData); - return response.send({ ok: true }); -}); - app.post('/deletegroup', jsonParser, async (request, response) => { if (!request.body || !request.body.id) { return response.sendStatus(400); @@ -2656,6 +2287,9 @@ app.use('/api/files', require('./src/endpoints/files').router); // Character management app.use('/api/characters', require('./src/endpoints/characters').router); +// Chat management +app.use('/api/chats', require('./src/endpoints/chats').router); + // Character sprite management app.use('/api/sprites', require('./src/endpoints/sprites').router); @@ -2760,47 +2394,6 @@ if (cliArguments.ssl) { ); } -function generateTimestamp() { - const now = new Date(); - const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, '0'); - const day = String(now.getDate()).padStart(2, '0'); - const hours = String(now.getHours()).padStart(2, '0'); - const minutes = String(now.getMinutes()).padStart(2, '0'); - const seconds = String(now.getSeconds()).padStart(2, '0'); - - return `${year}${month}${day}-${hours}${minutes}${seconds}`; -} - -/** - * - * @param {string} name - * @param {string} chat - */ -function backupChat(name, chat) { - try { - const isBackupDisabled = getConfigValue('disableChatBackup', false); - - if (isBackupDisabled) { - return; - } - - if (!fs.existsSync(DIRECTORIES.backups)) { - fs.mkdirSync(DIRECTORIES.backups); - } - - // replace non-alphanumeric characters with underscores - name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase(); - - const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`); - writeFileAtomicSync(backupFile, chat, 'utf-8'); - - removeOldBackups(`chat_${name}_`); - } catch (err) { - console.log(`Could not backup chat for ${name}`, err); - } -} - function backupSettings() { try { if (!fs.existsSync(DIRECTORIES.backups)) { @@ -2816,21 +2409,6 @@ function backupSettings() { } } -/** - * @param {string} prefix - */ -function removeOldBackups(prefix) { - const MAX_BACKUPS = 25; - - let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix)); - if (files.length > MAX_BACKUPS) { - files = files.map(f => path.join(DIRECTORIES.backups, f)); - files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); - - fs.rmSync(files[0]); - } -} - function ensurePublicDirectoriesExist() { for (const dir of Object.values(DIRECTORIES)) { if (!fs.existsSync(dir)) { diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js new file mode 100644 index 000000000..b8063b72f --- /dev/null +++ b/src/endpoints/chats.js @@ -0,0 +1,411 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const express = require('express'); +const sanitize = require('sanitize-filename'); +const writeFileAtomicSync = require('write-file-atomic').sync; + +const { jsonParser, urlencodedParser } = require('../express-common'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util'); + +/** + * + * @param {string} name + * @param {string} chat + */ +function backupChat(name, chat) { + try { + const isBackupDisabled = getConfigValue('disableChatBackup', false); + + if (isBackupDisabled) { + return; + } + + if (!fs.existsSync(DIRECTORIES.backups)) { + fs.mkdirSync(DIRECTORIES.backups); + } + + // replace non-alphanumeric characters with underscores + name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase(); + + const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`); + writeFileAtomicSync(backupFile, chat, 'utf-8'); + + removeOldBackups(`chat_${name}_`); + } catch (err) { + console.log(`Could not backup chat for ${name}`, err); + } +} + +const router = express.Router(); + +router.post('/save', jsonParser, function (request, response) { + try { + var dir_name = String(request.body.avatar_url).replace('.png', ''); + let chat_data = request.body.chat; + let jsonlData = chat_data.map(JSON.stringify).join('\n'); + writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); + backupChat(dir_name, jsonlData); + return response.send({ result: 'ok' }); + } catch (error) { + response.send(error); + return console.log(error); + } +}); + +router.post('/get', jsonParser, function (request, response) { + try { + const dirName = String(request.body.avatar_url).replace('.png', ''); + const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName); + + //if no chat dir for the character is found, make one with the character name + if (!chatDirExists) { + fs.mkdirSync(DIRECTORIES.chats + dirName); + return response.send({}); + } + + + if (!request.body.file_name) { + return response.send({}); + } + + const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`; + const chatFileExists = fs.existsSync(fileName); + + if (!chatFileExists) { + return response.send({}); + } + + const data = fs.readFileSync(fileName, 'utf8'); + const lines = data.split('\n'); + + // Iterate through the array of strings and parse each line as JSON + const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x); + return response.send(jsonData); + } catch (error) { + console.error(error); + return response.send({}); + } +}); + + +router.post('/rename', jsonParser, async function (request, response) { + if (!request.body || !request.body.original_file || !request.body.renamed_file) { + return response.sendStatus(400); + } + + const pathToFolder = request.body.is_group + ? DIRECTORIES.groupChats + : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); + const pathToOriginalFile = path.join(pathToFolder, request.body.original_file); + const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file); + console.log('Old chat name', pathToOriginalFile); + console.log('New chat name', pathToRenamedFile); + + if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) { + console.log('Either Source or Destination files are not available'); + return response.status(400).send({ error: true }); + } + + console.log('Successfully renamed.'); + fs.renameSync(pathToOriginalFile, pathToRenamedFile); + return response.send({ ok: true }); +}); + +router.post('/delete', jsonParser, function (request, response) { + console.log('/api/chats/delete entered'); + if (!request.body) { + console.log('no request body seen'); + return response.sendStatus(400); + } + + if (request.body.chatfile !== sanitize(request.body.chatfile)) { + console.error('Malicious chat name prevented'); + return response.sendStatus(403); + } + + const dirName = String(request.body.avatar_url).replace('.png', ''); + const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`; + const chatFileExists = fs.existsSync(fileName); + + if (!chatFileExists) { + console.log(`Chat file not found '${fileName}'`); + return response.sendStatus(400); + } else { + console.log('found the chat file: ' + fileName); + /* fs.unlinkSync(fileName); */ + fs.rmSync(fileName); + console.log('deleted chat file: ' + fileName); + + } + + + return response.send('ok'); +}); + +router.post('/export', jsonParser, async function (request, response) { + if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) { + return response.sendStatus(400); + } + const pathToFolder = request.body.is_group + ? DIRECTORIES.groupChats + : path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', '')); + let filename = path.join(pathToFolder, request.body.file); + let exportfilename = request.body.exportfilename; + if (!fs.existsSync(filename)) { + const errorMessage = { + message: `Could not find JSONL file to export. Source chat file: ${filename}.`, + }; + console.log(errorMessage.message); + return response.status(404).json(errorMessage); + } + try { + // Short path for JSONL files + if (request.body.format == 'jsonl') { + try { + const rawFile = fs.readFileSync(filename, 'utf8'); + const successMessage = { + message: `Chat saved to ${exportfilename}`, + result: rawFile, + }; + + console.log(`Chat exported as ${exportfilename}`); + return response.status(200).json(successMessage); + } + catch (err) { + console.error(err); + const errorMessage = { + message: `Could not read JSONL file to export. Source chat file: ${filename}.`, + }; + console.log(errorMessage.message); + return response.status(500).json(errorMessage); + } + } + + const readStream = fs.createReadStream(filename); + const rl = readline.createInterface({ + input: readStream, + }); + let buffer = ''; + rl.on('line', (line) => { + const data = JSON.parse(line); + if (data.mes) { + const name = data.name; + const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n'); + buffer += (`${name}: ${message}\n\n`); + } + }); + rl.on('close', () => { + const successMessage = { + message: `Chat saved to ${exportfilename}`, + result: buffer, + }; + console.log(`Chat exported as ${exportfilename}`); + return response.status(200).json(successMessage); + }); + } + catch (err) { + console.log('chat export failed.'); + console.log(err); + return response.sendStatus(400); + } +}); + +router.post('/group/import', urlencodedParser, function (request, response) { + try { + const filedata = request.file; + + if (!filedata) { + return response.sendStatus(400); + } + + const chatname = humanizedISO8601DateTime(); + const pathToUpload = path.join(UPLOADS_PATH, filedata.filename); + const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`); + fs.copyFileSync(pathToUpload, pathToNewFile); + fs.unlinkSync(pathToUpload); + return response.send({ res: chatname }); + } catch (error) { + console.error(error); + return response.send({ error: true }); + } +}); + +router.post('/import', urlencodedParser, function (request, response) { + if (!request.body) return response.sendStatus(400); + + var format = request.body.file_type; + let filedata = request.file; + let avatar_url = (request.body.avatar_url).replace('.png', ''); + let ch_name = request.body.character_name; + let user_name = request.body.user_name || 'You'; + + if (!filedata) { + return response.sendStatus(400); + } + + try { + const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8'); + + if (format === 'json') { + const jsonData = JSON.parse(data); + if (jsonData.histories !== undefined) { + //console.log('/api/chats/import confirms JSON histories are defined'); + const chat = { + from(history) { + return [ + { + user_name: user_name, + character_name: ch_name, + create_date: humanizedISO8601DateTime(), + }, + ...history.msgs.map( + (message) => ({ + name: message.src.is_human ? user_name : ch_name, + is_user: message.src.is_human, + send_date: humanizedISO8601DateTime(), + mes: message.text, + }), + )]; + }, + }; + + const newChats = []; + (jsonData.histories.histories ?? []).forEach((history) => { + newChats.push(chat.from(history)); + }); + + const errors = []; + + for (const chat of newChats) { + const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`; + const fileContent = chat.map(tryParse).filter(x => x).join('\n'); + + try { + writeFileAtomicSync(filePath, fileContent, 'utf8'); + } catch (err) { + errors.push(err); + } + } + + if (0 < errors.length) { + response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors)); + } + + response.send({ res: true }); + } else if (Array.isArray(jsonData.data_visible)) { + // oobabooga's format + /** @type {object[]} */ + const chat = [{ + user_name: user_name, + character_name: ch_name, + create_date: humanizedISO8601DateTime(), + }]; + + for (const arr of jsonData.data_visible) { + if (arr[0]) { + const userMessage = { + name: user_name, + is_user: true, + send_date: humanizedISO8601DateTime(), + mes: arr[0], + }; + chat.push(userMessage); + } + if (arr[1]) { + const charMessage = { + name: ch_name, + is_user: false, + send_date: humanizedISO8601DateTime(), + mes: arr[1], + }; + chat.push(charMessage); + } + } + + const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n'); + writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8'); + + response.send({ res: true }); + } else { + console.log('Incorrect chat format .json'); + return response.send({ error: true }); + } + } + + if (format === 'jsonl') { + const line = data.split('\n')[0]; + + let jsonData = JSON.parse(line); + + if (jsonData.user_name !== undefined || jsonData.name !== undefined) { + fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`)); + response.send({ res: true }); + } else { + console.log('Incorrect chat format .jsonl'); + return response.send({ error: true }); + } + } + } catch (error) { + console.error(error); + return response.send({ error: true }); + } +}); + +router.post('/group/get', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (fs.existsSync(pathToFile)) { + const data = fs.readFileSync(pathToFile, 'utf8'); + const lines = data.split('\n'); + + // Iterate through the array of strings and parse each line as JSON + const jsonData = lines.map(line => tryParse(line)).filter(x => x); + return response.send(jsonData); + } else { + return response.send([]); + } +}); + +router.post('/group/delete', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (fs.existsSync(pathToFile)) { + fs.rmSync(pathToFile); + return response.send({ ok: true }); + } + + return response.send({ error: true }); +}); + +router.post('/group/save', jsonParser, (request, response) => { + if (!request.body || !request.body.id) { + return response.sendStatus(400); + } + + const id = request.body.id; + const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`); + + if (!fs.existsSync(DIRECTORIES.groupChats)) { + fs.mkdirSync(DIRECTORIES.groupChats); + } + + let chat_data = request.body.chat; + let jsonlData = chat_data.map(JSON.stringify).join('\n'); + writeFileAtomicSync(pathToFile, jsonlData, 'utf8'); + backupChat(String(id), jsonlData); + return response.send({ ok: true }); +}); + +module.exports = { router }; diff --git a/src/util.js b/src/util.js index c8079e6b8..d4c416473 100644 --- a/src/util.js +++ b/src/util.js @@ -7,6 +7,8 @@ const mime = require('mime-types'); const yaml = require('yaml'); const { default: simpleGit } = require('simple-git'); +const { DIRECTORIES } = require('./constants'); + /** * Returns the config object from the config.yaml file. * @returns {object} Config object @@ -307,6 +309,34 @@ function removeFileExtension(filename) { return filename.replace(/\.[^.]+$/, ''); } +function generateTimestamp() { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + + return `${year}${month}${day}-${hours}${minutes}${seconds}`; +} + +/** + * @param {string} prefix + */ +function removeOldBackups(prefix) { + const MAX_BACKUPS = 25; + + let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix)); + if (files.length > MAX_BACKUPS) { + files = files.map(f => path.join(DIRECTORIES.backups, f)); + files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); + + fs.rmSync(files[0]); + } +} + + module.exports = { getConfig, getConfigValue, @@ -323,4 +353,6 @@ module.exports = { tryParse, clientRelativePath, removeFileExtension, + generateTimestamp, + removeOldBackups, };