mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
749 lines
26 KiB
JavaScript
749 lines
26 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import readline from 'node:readline';
|
|
import process from 'node:process';
|
|
|
|
import express from 'express';
|
|
import sanitize from 'sanitize-filename';
|
|
import { sync as writeFileAtomicSync } from 'write-file-atomic';
|
|
import _ from 'lodash';
|
|
|
|
import { jsonParser, urlencodedParser } from '../express-common.js';
|
|
import validateAvatarUrlMiddleware from '../middleware/validateFileName.js';
|
|
import {
|
|
getConfigValue,
|
|
humanizedISO8601DateTime,
|
|
tryParse,
|
|
generateTimestamp,
|
|
removeOldBackups,
|
|
formatBytes,
|
|
} from '../util.js';
|
|
|
|
const isBackupEnabled = !!getConfigValue('backups.chat.enabled', true);
|
|
const maxTotalChatBackups = Number(getConfigValue('backups.chat.maxTotalBackups', -1));
|
|
const throttleInterval = Number(getConfigValue('backups.chat.throttleInterval', 10_000));
|
|
|
|
/**
|
|
* Saves a chat to the backups directory.
|
|
* @param {string} directory The user's backups directory.
|
|
* @param {string} name The name of the chat.
|
|
* @param {string} chat The serialized chat to save.
|
|
*/
|
|
function backupChat(directory, name, chat) {
|
|
try {
|
|
|
|
if (!isBackupEnabled) {
|
|
return;
|
|
}
|
|
|
|
// replace non-alphanumeric characters with underscores
|
|
name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
|
|
|
const backupFile = path.join(directory, `chat_${name}_${generateTimestamp()}.jsonl`);
|
|
writeFileAtomicSync(backupFile, chat, 'utf-8');
|
|
|
|
removeOldBackups(directory, `chat_${name}_`);
|
|
|
|
if (isNaN(maxTotalChatBackups) || maxTotalChatBackups < 0) {
|
|
return;
|
|
}
|
|
|
|
removeOldBackups(directory, 'chat_', maxTotalChatBackups);
|
|
} catch (err) {
|
|
console.log(`Could not backup chat for ${name}`, err);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @type {Map<string, import('lodash').DebouncedFunc<function(string, string, string): void>>}
|
|
*/
|
|
const backupFunctions = new Map();
|
|
|
|
/**
|
|
* Gets a backup function for a user.
|
|
* @param {string} handle User handle
|
|
* @returns {function(string, string, string): void} Backup function
|
|
*/
|
|
function getBackupFunction(handle) {
|
|
if (!backupFunctions.has(handle)) {
|
|
backupFunctions.set(handle, _.throttle(backupChat, throttleInterval, { leading: true, trailing: true }));
|
|
}
|
|
return backupFunctions.get(handle) || (() => {});
|
|
}
|
|
|
|
/**
|
|
* Gets a preview message from an array of chat messages
|
|
* @param {Array<Object>} messages - Array of chat messages, each with a 'mes' property
|
|
* @returns {string} A truncated preview of the last message or empty string if no messages
|
|
*/
|
|
function getPreviewMessage(messages) {
|
|
const strlen = 400;
|
|
const lastMessage = messages[messages.length - 1]?.mes;
|
|
|
|
if (!lastMessage) {
|
|
return '';
|
|
}
|
|
|
|
return lastMessage.length > strlen
|
|
? '...' + lastMessage.substring(lastMessage.length - strlen)
|
|
: lastMessage;
|
|
}
|
|
|
|
process.on('exit', () => {
|
|
for (const func of backupFunctions.values()) {
|
|
func.flush();
|
|
}
|
|
});
|
|
|
|
/**
|
|
* Imports a chat from Ooba's format.
|
|
* @param {string} userName User name
|
|
* @param {string} characterName Character name
|
|
* @param {object} jsonData JSON data
|
|
* @returns {string} Chat data
|
|
*/
|
|
function importOobaChat(userName, characterName, jsonData) {
|
|
/** @type {object[]} */
|
|
const chat = [{
|
|
user_name: userName,
|
|
character_name: characterName,
|
|
create_date: humanizedISO8601DateTime(),
|
|
}];
|
|
|
|
for (const arr of jsonData.data_visible) {
|
|
if (arr[0]) {
|
|
const userMessage = {
|
|
name: userName,
|
|
is_user: true,
|
|
send_date: humanizedISO8601DateTime(),
|
|
mes: arr[0],
|
|
};
|
|
chat.push(userMessage);
|
|
}
|
|
if (arr[1]) {
|
|
const charMessage = {
|
|
name: characterName,
|
|
is_user: false,
|
|
send_date: humanizedISO8601DateTime(),
|
|
mes: arr[1],
|
|
};
|
|
chat.push(charMessage);
|
|
}
|
|
}
|
|
|
|
return chat.map(obj => JSON.stringify(obj)).join('\n');
|
|
}
|
|
|
|
/**
|
|
* Imports a chat from Agnai's format.
|
|
* @param {string} userName User name
|
|
* @param {string} characterName Character name
|
|
* @param {object} jsonData Chat data
|
|
* @returns {string} Chat data
|
|
*/
|
|
function importAgnaiChat(userName, characterName, jsonData) {
|
|
/** @type {object[]} */
|
|
const chat = [{
|
|
user_name: userName,
|
|
character_name: characterName,
|
|
create_date: humanizedISO8601DateTime(),
|
|
}];
|
|
|
|
for (const message of jsonData.messages) {
|
|
const isUser = !!message.userId;
|
|
chat.push({
|
|
name: isUser ? userName : characterName,
|
|
is_user: isUser,
|
|
send_date: humanizedISO8601DateTime(),
|
|
mes: message.msg,
|
|
});
|
|
}
|
|
|
|
return chat.map(obj => JSON.stringify(obj)).join('\n');
|
|
}
|
|
|
|
/**
|
|
* Imports a chat from CAI Tools format.
|
|
* @param {string} userName User name
|
|
* @param {string} characterName Character name
|
|
* @param {object} jsonData JSON data
|
|
* @returns {string[]} Converted data
|
|
*/
|
|
function importCAIChat(userName, characterName, jsonData) {
|
|
/**
|
|
* Converts the chat data to suitable format.
|
|
* @param {object} history Imported chat data
|
|
* @returns {object[]} Converted chat data
|
|
*/
|
|
function convert(history) {
|
|
const starter = {
|
|
user_name: userName,
|
|
character_name: characterName,
|
|
create_date: humanizedISO8601DateTime(),
|
|
};
|
|
|
|
const historyData = history.msgs.map((msg) => ({
|
|
name: msg.src.is_human ? userName : characterName,
|
|
is_user: msg.src.is_human,
|
|
send_date: humanizedISO8601DateTime(),
|
|
mes: msg.text,
|
|
}));
|
|
|
|
return [starter, ...historyData];
|
|
}
|
|
|
|
const newChats = (jsonData.histories.histories ?? []).map(history => newChats.push(convert(history).map(obj => JSON.stringify(obj)).join('\n')));
|
|
return newChats;
|
|
}
|
|
|
|
/**
|
|
* Imports a chat from Kobold Lite format.
|
|
* @param {string} _userName User name
|
|
* @param {string} _characterName Character name
|
|
* @param {object} data JSON data
|
|
* @returns {string} Chat data
|
|
*/
|
|
function importKoboldLiteChat(_userName, _characterName, data) {
|
|
const inputToken = '{{[INPUT]}}';
|
|
const outputToken = '{{[OUTPUT]}}';
|
|
|
|
/** @type {function(string): object} */
|
|
function processKoboldMessage(msg) {
|
|
const isUser = msg.includes(inputToken);
|
|
return {
|
|
name: isUser ? header.user_name : header.character_name,
|
|
is_user: isUser,
|
|
mes: msg.replaceAll(inputToken, '').replaceAll(outputToken, '').trim(),
|
|
send_date: Date.now(),
|
|
};
|
|
}
|
|
|
|
// Create the header
|
|
const header = {
|
|
user_name: String(data.savedsettings.chatname),
|
|
character_name: String(data.savedsettings.chatopponent).split('||$||')[0],
|
|
};
|
|
// Format messages
|
|
const formattedMessages = data.actions.map(processKoboldMessage);
|
|
// Add prompt if available
|
|
if (data.prompt) {
|
|
formattedMessages.unshift(processKoboldMessage(data.prompt));
|
|
}
|
|
// Combine header and messages
|
|
const chatData = [header, ...formattedMessages];
|
|
return chatData.map(obj => JSON.stringify(obj)).join('\n');
|
|
}
|
|
|
|
/**
|
|
* Flattens `msg` and `swipes` data from Chub Chat format.
|
|
* Only changes enough to make it compatible with the standard chat serialization format.
|
|
* @param {string} userName User name
|
|
* @param {string} characterName Character name
|
|
* @param {string[]} lines serialised JSONL data
|
|
* @returns {string} Converted data
|
|
*/
|
|
function flattenChubChat(userName, characterName, lines) {
|
|
function flattenSwipe(swipe) {
|
|
return swipe.message ? swipe.message : swipe;
|
|
}
|
|
|
|
function convert(line) {
|
|
const lineData = tryParse(line);
|
|
if (!lineData) return line;
|
|
|
|
if (lineData.mes && lineData.mes.message) {
|
|
lineData.mes = lineData?.mes.message;
|
|
}
|
|
|
|
if (lineData?.swipes && Array.isArray(lineData.swipes)) {
|
|
lineData.swipes = lineData.swipes.map(swipe => flattenSwipe(swipe));
|
|
}
|
|
|
|
return JSON.stringify(lineData);
|
|
}
|
|
|
|
return (lines ?? []).map(convert).join('\n');
|
|
}
|
|
|
|
/**
|
|
* Imports a chat from RisuAI format.
|
|
* @param {string} userName User name
|
|
* @param {string} characterName Character name
|
|
* @param {object} jsonData Imported chat data
|
|
* @returns {string} Chat data
|
|
*/
|
|
function importRisuChat(userName, characterName, jsonData) {
|
|
/** @type {object[]} */
|
|
const chat = [{
|
|
user_name: userName,
|
|
character_name: characterName,
|
|
create_date: humanizedISO8601DateTime(),
|
|
}];
|
|
|
|
for (const message of jsonData.data.message) {
|
|
const isUser = message.role === 'user';
|
|
chat.push({
|
|
name: message.name ?? (isUser ? userName : characterName),
|
|
is_user: isUser,
|
|
send_date: Number(message.time ?? Date.now()),
|
|
mes: message.data ?? '',
|
|
});
|
|
}
|
|
|
|
return chat.map(obj => JSON.stringify(obj)).join('\n');
|
|
}
|
|
|
|
export const router = express.Router();
|
|
|
|
router.post('/save', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
|
|
try {
|
|
const directoryName = String(request.body.avatar_url).replace('.png', '');
|
|
const chatData = request.body.chat;
|
|
const jsonlData = chatData.map(JSON.stringify).join('\n');
|
|
const fileName = `${String(request.body.file_name)}.jsonl`;
|
|
const filePath = path.join(request.user.directories.chats, directoryName, sanitize(fileName));
|
|
writeFileAtomicSync(filePath, jsonlData, 'utf8');
|
|
getBackupFunction(request.user.profile.handle)(request.user.directories.backups, directoryName, jsonlData);
|
|
return response.send({ result: 'ok' });
|
|
} catch (error) {
|
|
response.send(error);
|
|
return console.log(error);
|
|
}
|
|
});
|
|
|
|
router.post('/get', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
|
|
try {
|
|
const dirName = String(request.body.avatar_url).replace('.png', '');
|
|
const directoryPath = path.join(request.user.directories.chats, dirName);
|
|
const chatDirExists = fs.existsSync(directoryPath);
|
|
|
|
//if no chat dir for the character is found, make one with the character name
|
|
if (!chatDirExists) {
|
|
fs.mkdirSync(directoryPath);
|
|
return response.send({});
|
|
}
|
|
|
|
if (!request.body.file_name) {
|
|
return response.send({});
|
|
}
|
|
|
|
const fileName = `${String(request.body.file_name)}.jsonl`;
|
|
const filePath = path.join(directoryPath, sanitize(fileName));
|
|
const chatFileExists = fs.existsSync(filePath);
|
|
|
|
if (!chatFileExists) {
|
|
return response.send({});
|
|
}
|
|
|
|
const data = fs.readFileSync(filePath, 'utf8');
|
|
const lines = data.split('\n');
|
|
|
|
// Iterate through the array of strings and parse each line as JSON
|
|
const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
|
|
return response.send(jsonData);
|
|
} catch (error) {
|
|
console.error(error);
|
|
return response.send({});
|
|
}
|
|
});
|
|
|
|
|
|
router.post('/rename', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
|
|
if (!request.body || !request.body.original_file || !request.body.renamed_file) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
const pathToFolder = request.body.is_group
|
|
? request.user.directories.groupChats
|
|
: path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));
|
|
const pathToOriginalFile = path.join(pathToFolder, sanitize(request.body.original_file));
|
|
const pathToRenamedFile = path.join(pathToFolder, sanitize(request.body.renamed_file));
|
|
const sanitizedFileName = path.parse(pathToRenamedFile).name;
|
|
console.log('Old chat name', pathToOriginalFile);
|
|
console.log('New chat name', pathToRenamedFile);
|
|
|
|
if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {
|
|
console.log('Either Source or Destination files are not available');
|
|
return response.status(400).send({ error: true });
|
|
}
|
|
|
|
fs.copyFileSync(pathToOriginalFile, pathToRenamedFile);
|
|
fs.rmSync(pathToOriginalFile);
|
|
console.log('Successfully renamed.');
|
|
return response.send({ ok: true, sanitizedFileName });
|
|
});
|
|
|
|
router.post('/delete', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
|
|
const dirName = String(request.body.avatar_url).replace('.png', '');
|
|
const fileName = String(request.body.chatfile);
|
|
const filePath = path.join(request.user.directories.chats, dirName, sanitize(fileName));
|
|
const chatFileExists = fs.existsSync(filePath);
|
|
|
|
if (!chatFileExists) {
|
|
console.log(`Chat file not found '${filePath}'`);
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
fs.rmSync(filePath);
|
|
console.log('Deleted chat file: ' + filePath);
|
|
return response.send('ok');
|
|
});
|
|
|
|
router.post('/export', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
|
|
if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
|
|
return response.sendStatus(400);
|
|
}
|
|
const pathToFolder = request.body.is_group
|
|
? request.user.directories.groupChats
|
|
: path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));
|
|
let filename = path.join(pathToFolder, request.body.file);
|
|
let exportfilename = request.body.exportfilename;
|
|
if (!fs.existsSync(filename)) {
|
|
const errorMessage = {
|
|
message: `Could not find JSONL file to export. Source chat file: ${filename}.`,
|
|
};
|
|
console.log(errorMessage.message);
|
|
return response.status(404).json(errorMessage);
|
|
}
|
|
try {
|
|
// Short path for JSONL files
|
|
if (request.body.format === 'jsonl') {
|
|
try {
|
|
const rawFile = fs.readFileSync(filename, 'utf8');
|
|
const successMessage = {
|
|
message: `Chat saved to ${exportfilename}`,
|
|
result: rawFile,
|
|
};
|
|
|
|
console.log(`Chat exported as ${exportfilename}`);
|
|
return response.status(200).json(successMessage);
|
|
} catch (err) {
|
|
console.error(err);
|
|
const errorMessage = {
|
|
message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
|
|
};
|
|
console.log(errorMessage.message);
|
|
return response.status(500).json(errorMessage);
|
|
}
|
|
}
|
|
|
|
const readStream = fs.createReadStream(filename);
|
|
const rl = readline.createInterface({
|
|
input: readStream,
|
|
});
|
|
let buffer = '';
|
|
rl.on('line', (line) => {
|
|
const data = JSON.parse(line);
|
|
// Skip non-printable/prompt-hidden messages
|
|
if (data.is_system) {
|
|
return;
|
|
}
|
|
if (data.mes) {
|
|
const name = data.name;
|
|
const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
|
|
buffer += (`${name}: ${message}\n\n`);
|
|
}
|
|
});
|
|
rl.on('close', () => {
|
|
const successMessage = {
|
|
message: `Chat saved to ${exportfilename}`,
|
|
result: buffer,
|
|
};
|
|
console.log(`Chat exported as ${exportfilename}`);
|
|
return response.status(200).json(successMessage);
|
|
});
|
|
} catch (err) {
|
|
console.log('chat export failed.');
|
|
console.log(err);
|
|
return response.sendStatus(400);
|
|
}
|
|
});
|
|
|
|
router.post('/group/import', urlencodedParser, function (request, response) {
|
|
try {
|
|
const filedata = request.file;
|
|
|
|
if (!filedata) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
const chatname = humanizedISO8601DateTime();
|
|
const pathToUpload = path.join(filedata.destination, filedata.filename);
|
|
const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`);
|
|
fs.copyFileSync(pathToUpload, pathToNewFile);
|
|
fs.unlinkSync(pathToUpload);
|
|
return response.send({ res: chatname });
|
|
} catch (error) {
|
|
console.error(error);
|
|
return response.send({ error: true });
|
|
}
|
|
});
|
|
|
|
router.post('/import', urlencodedParser, validateAvatarUrlMiddleware, function (request, response) {
|
|
if (!request.body) return response.sendStatus(400);
|
|
|
|
const format = request.body.file_type;
|
|
const avatarUrl = (request.body.avatar_url).replace('.png', '');
|
|
const characterName = request.body.character_name;
|
|
const userName = request.body.user_name || 'User';
|
|
|
|
if (!request.file) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
try {
|
|
const pathToUpload = path.join(request.file.destination, request.file.filename);
|
|
const data = fs.readFileSync(pathToUpload, 'utf8');
|
|
|
|
if (format === 'json') {
|
|
fs.unlinkSync(pathToUpload);
|
|
const jsonData = JSON.parse(data);
|
|
|
|
/** @type {function(string, string, object): string|string[]} */
|
|
let importFunc;
|
|
|
|
if (jsonData.savedsettings !== undefined) { // Kobold Lite format
|
|
importFunc = importKoboldLiteChat;
|
|
} else if (jsonData.histories !== undefined) { // CAI Tools format
|
|
importFunc = importCAIChat;
|
|
} else if (Array.isArray(jsonData.data_visible)) { // oobabooga's format
|
|
importFunc = importOobaChat;
|
|
} else if (Array.isArray(jsonData.messages)) { // Agnai's format
|
|
importFunc = importAgnaiChat;
|
|
} else if (jsonData.type === 'risuChat') { // RisuAI format
|
|
importFunc = importRisuChat;
|
|
} else { // Unknown format
|
|
console.log('Incorrect chat format .json');
|
|
return response.send({ error: true });
|
|
}
|
|
|
|
const handleChat = (chat) => {
|
|
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
|
|
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
|
|
writeFileAtomicSync(filePath, chat, 'utf8');
|
|
};
|
|
|
|
const chat = importFunc(userName, characterName, jsonData);
|
|
|
|
if (Array.isArray(chat)) {
|
|
chat.forEach(handleChat);
|
|
} else {
|
|
handleChat(chat);
|
|
}
|
|
|
|
return response.send({ res: true });
|
|
}
|
|
|
|
if (format === 'jsonl') {
|
|
let lines = data.split('\n');
|
|
const header = lines[0];
|
|
|
|
const jsonData = JSON.parse(header);
|
|
|
|
if (!(jsonData.user_name !== undefined || jsonData.name !== undefined)) {
|
|
console.log('Incorrect chat format .jsonl');
|
|
return response.send({ error: true });
|
|
}
|
|
|
|
// Do a tiny bit of work to import Chub Chat data
|
|
// Processing the entire file is so fast that it's not worth checking if it's a Chub chat first
|
|
let flattenedChat = data;
|
|
try {
|
|
// flattening is unlikely to break, but it's not worth failing to
|
|
// import normal chats in an attempt to import a Chub chat
|
|
flattenedChat = flattenChubChat(userName, characterName, lines);
|
|
} catch (error) {
|
|
console.warn('Failed to flatten Chub Chat data: ', error);
|
|
}
|
|
|
|
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
|
|
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
|
|
if (flattenedChat !== data) {
|
|
writeFileAtomicSync(filePath, flattenedChat, 'utf8');
|
|
} else {
|
|
fs.copyFileSync(pathToUpload, filePath);
|
|
}
|
|
fs.unlinkSync(pathToUpload);
|
|
response.send({ res: true });
|
|
}
|
|
} catch (error) {
|
|
console.error(error);
|
|
return response.send({ error: true });
|
|
}
|
|
});
|
|
|
|
router.post('/group/get', jsonParser, (request, response) => {
|
|
if (!request.body || !request.body.id) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
const id = request.body.id;
|
|
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
|
|
|
|
if (fs.existsSync(pathToFile)) {
|
|
const data = fs.readFileSync(pathToFile, 'utf8');
|
|
const lines = data.split('\n');
|
|
|
|
// Iterate through the array of strings and parse each line as JSON
|
|
const jsonData = lines.map(line => tryParse(line)).filter(x => x);
|
|
return response.send(jsonData);
|
|
} else {
|
|
return response.send([]);
|
|
}
|
|
});
|
|
|
|
router.post('/group/delete', jsonParser, (request, response) => {
|
|
if (!request.body || !request.body.id) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
const id = request.body.id;
|
|
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
|
|
|
|
if (fs.existsSync(pathToFile)) {
|
|
fs.rmSync(pathToFile);
|
|
return response.send({ ok: true });
|
|
}
|
|
|
|
return response.send({ error: true });
|
|
});
|
|
|
|
router.post('/group/save', jsonParser, (request, response) => {
|
|
if (!request.body || !request.body.id) {
|
|
return response.sendStatus(400);
|
|
}
|
|
|
|
const id = request.body.id;
|
|
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
|
|
|
|
if (!fs.existsSync(request.user.directories.groupChats)) {
|
|
fs.mkdirSync(request.user.directories.groupChats);
|
|
}
|
|
|
|
let chat_data = request.body.chat;
|
|
let jsonlData = chat_data.map(JSON.stringify).join('\n');
|
|
writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
|
|
getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
|
|
return response.send({ ok: true });
|
|
});
|
|
|
|
router.post('/search', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
|
|
try {
|
|
const { query, avatar_url, group_id } = request.body;
|
|
let chatFiles = [];
|
|
|
|
if (group_id) {
|
|
// Find group's chat IDs first
|
|
const groupDir = path.join(request.user.directories.groups);
|
|
const groupFiles = fs.readdirSync(groupDir)
|
|
.filter(file => file.endsWith('.json'));
|
|
|
|
let targetGroup;
|
|
for (const groupFile of groupFiles) {
|
|
try {
|
|
const groupData = JSON.parse(fs.readFileSync(path.join(groupDir, groupFile), 'utf8'));
|
|
if (groupData.id === group_id) {
|
|
targetGroup = groupData;
|
|
break;
|
|
}
|
|
} catch (error) {
|
|
console.error(groupFile, 'group file is corrupted:', error);
|
|
}
|
|
}
|
|
|
|
if (!targetGroup?.chats) {
|
|
return response.send([]);
|
|
}
|
|
|
|
// Find group chat files for given group ID
|
|
const groupChatsDir = path.join(request.user.directories.groupChats);
|
|
chatFiles = targetGroup.chats
|
|
.map(chatId => {
|
|
const filePath = path.join(groupChatsDir, `${chatId}.jsonl`);
|
|
if (!fs.existsSync(filePath)) return null;
|
|
const stats = fs.statSync(filePath);
|
|
return {
|
|
file_name: chatId,
|
|
file_size: formatBytes(stats.size),
|
|
path: filePath,
|
|
};
|
|
})
|
|
.filter(x => x);
|
|
} else {
|
|
// Regular character chat directory
|
|
const character_name = avatar_url.replace('.png', '');
|
|
const directoryPath = path.join(request.user.directories.chats, character_name);
|
|
|
|
if (!fs.existsSync(directoryPath)) {
|
|
return response.send([]);
|
|
}
|
|
|
|
chatFiles = fs.readdirSync(directoryPath)
|
|
.filter(file => file.endsWith('.jsonl'))
|
|
.map(fileName => {
|
|
const filePath = path.join(directoryPath, fileName);
|
|
const stats = fs.statSync(filePath);
|
|
return {
|
|
file_name: fileName,
|
|
file_size: formatBytes(stats.size),
|
|
path: filePath,
|
|
};
|
|
});
|
|
}
|
|
|
|
const results = [];
|
|
|
|
// Search logic
|
|
for (const chatFile of chatFiles) {
|
|
const data = fs.readFileSync(chatFile.path, 'utf8');
|
|
const messages = data.split('\n')
|
|
.map(line => { try { return JSON.parse(line); } catch (_) { return null; } })
|
|
.filter(x => x && typeof x.mes === 'string');
|
|
|
|
if (query && messages.length === 0) {
|
|
continue;
|
|
}
|
|
|
|
const lastMessage = messages[messages.length - 1];
|
|
const lastMesDate = lastMessage?.send_date || Math.round(fs.statSync(chatFile.path).mtimeMs);
|
|
|
|
// If no search query, just return metadata
|
|
if (!query) {
|
|
results.push({
|
|
file_name: chatFile.file_name,
|
|
file_size: chatFile.file_size,
|
|
message_count: messages.length,
|
|
last_mes: lastMesDate,
|
|
preview_message: getPreviewMessage(messages),
|
|
});
|
|
continue;
|
|
}
|
|
|
|
// Search through messages
|
|
const fragments = query.trim().toLowerCase().split(/\s+/).filter(x => x);
|
|
const hasMatch = messages.some(message => {
|
|
const text = message?.mes?.toLowerCase();
|
|
return text && fragments.every(fragment => text.includes(fragment));
|
|
});
|
|
|
|
if (hasMatch) {
|
|
results.push({
|
|
file_name: chatFile.file_name,
|
|
file_size: chatFile.file_size,
|
|
message_count: messages.length,
|
|
last_mes: lastMesDate,
|
|
preview_message: getPreviewMessage(messages),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Sort by last message date descending
|
|
results.sort((a, b) => new Date(b.last_mes).getTime() - new Date(a.last_mes).getTime());
|
|
return response.send(results);
|
|
|
|
} catch (error) {
|
|
console.error('Chat search error:', error);
|
|
return response.status(500).json({ error: 'Search failed' });
|
|
}
|
|
});
|