rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Join recent chat/group queries

Browse Source
This commit is contained in:
Cohee
2025-05-14 10:25:38 +03:00
parent 155172a2b4
commit 587cecb12c
4 changed files with 152 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
public/scripts/welcome-screen.js
View File

@ -31,7 +31,7 @@ const assistantAvatarKey = 'assistant';
const defaultAssistantAvatar = 'default_Assistant.png';
const DEFAULT_DISPLAYED = 3;
const MAX_DISPLAYED = 20;
const MAX_DISPLAYED = 15;
export function getPermanentAssistantAvatar() {
const assistantAvatar = accountStorage.getItem(assistantAvatarKey);
@ -247,32 +247,19 @@ async function openRecentGroupChat(groupId, fileName) {
* @property {boolean} hidden Chat will be hidden by default
*/
async function getRecentChats() {
const charData = async () => {
const response = await fetch('/api/characters/recent', {
const response = await fetch('/api/chats/recent', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ max: MAX_DISPLAYED }),
});
if (!response.ok) {
console.warn('Failed to fetch recent character chats');
return [];
}
return await response.json();
};
const groupData = async () => {
const response = await fetch('/api/groups/recent', {
method: 'POST',
headers: getRequestHeaders(),
});
if (!response.ok) {
console.warn('Failed to fetch recent group chats');
return [];
}
return await response.json();
};
/** @type {RecentChat[]} */
const data = await Promise.all([charData(), groupData()]).then(res => res.flat());
const data = await response.json();
data.sort((a, b) => sortMoments(timestampToMoment(a.last_mes), timestampToMoment(b.last_mes)))
.map(chat => ({ chat, character: characters.find(x => x.avatar === chat.avatar), group: groups.find(x => x.id === chat.group) }))
@ -290,7 +277,7 @@ async function getRecentChats() {
chat.group = chat.group || '';
});
return data.slice(0, MAX_DISPLAYED);
return data;
}
export async function openPermanentAssistantChat({ tryCreate = true, created = false } = {}) {

106
src/endpoints/characters.js
View File

@ -1,7 +1,6 @@
import path from 'node:path';
import fs from 'node:fs';
import { promises as fsPromises } from 'node:fs';
import readline from 'node:readline';
import { Buffer } from 'node:buffer';
import express from 'express';
@ -22,6 +21,7 @@ import { readWorldInfoFile } from './worldinfo.js';
import { invalidateThumbnail } from './thumbnails.js';
import { importRisuSprites } from './sprites.js';
import { getUserDirectories } from '../users.js';
import { getChatInfo } from './chats.js';
const defaultAvatarPath = './public/img/ai4.png';
// With 100 MB limit it would take roughly 3000 characters to reach this limit
@ -934,69 +934,6 @@ async function importFromPng(uploadPath, { request }, preservedFileName) {
return '';
}
/**
* @typedef {Object} ChatInfo
* @property {string} [file_name] - The name of the chat file
* @property {string} [file_size] - The size of the chat file
* @property {number} [chat_items] - The number of chat items in the file
* @property {string} [mes] - The last message in the chat
* @property {number} [last_mes] - The timestamp of the last message
*/
/**
* Reads the information from a chat file.
* @param {string} pathToFile
* @param {object} additionalData
* @returns {Promise<ChatInfo>}
*/
export async function getChatInfo(pathToFile, additionalData = {}, isGroup = false) {
return new Promise(async (res) => {
const fileStream = fs.createReadStream(pathToFile);
const stats = fs.statSync(pathToFile);
const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`;
if (stats.size === 0) {
console.warn(`Found an empty chat file: ${pathToFile}`);
res({});
return;
}
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
let lastLine;
let itemCounter = 0;
rl.on('line', (line) => {
itemCounter++;
lastLine = line;
});
rl.on('close', () => {
rl.close();
if (lastLine) {
const jsonData = tryParse(lastLine);
if (jsonData && (jsonData.name || jsonData.character_name)) {
const chatData = {};
chatData['file_name'] = path.parse(pathToFile).base;
chatData['file_size'] = fileSizeInKB;
chatData['chat_items'] = isGroup ? itemCounter : (itemCounter - 1);
chatData['mes'] = jsonData['mes'] || '[The chat is empty]';
chatData['last_mes'] = jsonData['send_date'] || stats.mtimeMs;
Object.assign(chatData, additionalData);
res(chatData);
} else {
console.warn('Found an invalid or corrupted chat file:', pathToFile);
res({});
}
}
});
});
}
export const router = express.Router();
router.post('/create', async function (request, response) {
@ -1286,47 +1223,6 @@ router.post('/get', validateAvatarUrlMiddleware, async function (request, respon
}
});
router.post('/recent', async function (request, response) {
try {
/** @type {{pngFile: string, filePath: string, mtime: number}[]} */
const allChatFiles = [];
const pngFiles = fs
.readdirSync(request.user.directories.characters, { withFileTypes: true })
.filter(dirent => dirent.isFile() && dirent.name.endsWith('.png'))
.map(dirent => dirent.name);
for (const pngFile of pngFiles) {
const chatsDirectory = pngFile.replace('.png', '');
const pathToChats = path.join(request.user.directories.chats, chatsDirectory);
if (fs.existsSync(pathToChats) && fs.statSync(pathToChats).isDirectory()) {
const chatFiles = fs.readdirSync(pathToChats);
const chatFilesWithDate = chatFiles
.filter(file => file.endsWith('.jsonl'))
.map(file => {
const filePath = path.join(pathToChats, file);
const stats = fs.statSync(filePath);
return { pngFile, filePath, mtime: stats.mtimeMs };
});
allChatFiles.push(...chatFilesWithDate);
}
}
const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, 15);
const jsonFilesPromise = recentChats.map((file) => {
return getChatInfo(file.filePath, { avatar: file.pngFile });
});
const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
const validFiles = chatData.filter(i => i.file_name);
return response.send(validFiles);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
router.post('/chats', validateAvatarUrlMiddleware, async function (request, response) {
try {
if (!request.body) return response.sendStatus(400);

139
src/endpoints/chats.js
View File

@ -351,6 +351,69 @@ async function checkChatIntegrity(filePath, integritySlug) {
return chatIntegrity === integritySlug;
}
/**
* @typedef {Object} ChatInfo
* @property {string} [file_name] - The name of the chat file
* @property {string} [file_size] - The size of the chat file
* @property {number} [chat_items] - The number of chat items in the file
* @property {string} [mes] - The last message in the chat
* @property {number} [last_mes] - The timestamp of the last message
*/
/**
* Reads the information from a chat file.
* @param {string} pathToFile
* @param {object} additionalData
* @returns {Promise<ChatInfo>}
*/
export async function getChatInfo(pathToFile, additionalData = {}, isGroup = false) {
return new Promise(async (res) => {
const fileStream = fs.createReadStream(pathToFile);
const stats = await fs.promises.stat(pathToFile);
const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`;
if (stats.size === 0) {
console.warn(`Found an empty chat file: ${pathToFile}`);
res({});
return;
}
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
let lastLine;
let itemCounter = 0;
rl.on('line', (line) => {
itemCounter++;
lastLine = line;
});
rl.on('close', () => {
rl.close();
if (lastLine) {
const jsonData = tryParse(lastLine);
if (jsonData && (jsonData.name || jsonData.character_name)) {
const chatData = {};
chatData['file_name'] = path.parse(pathToFile).base;
chatData['file_size'] = fileSizeInKB;
chatData['chat_items'] = isGroup ? itemCounter : (itemCounter - 1);
chatData['mes'] = jsonData['mes'] || '[The chat is empty]';
chatData['last_mes'] = jsonData['send_date'] || stats.mtimeMs;
Object.assign(chatData, additionalData);
res(chatData);
} else {
console.warn('Found an invalid or corrupted chat file:', pathToFile);
res({});
}
}
});
});
}
export const router = express.Router();
router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {
@ -809,3 +872,79 @@ router.post('/search', validateAvatarUrlMiddleware, function (request, response)
return response.status(500).json({ error: 'Search failed' });
}
});
router.post('/recent', async function (request, response) {
try {
/** @type {{pngFile?: string, groupId?: string, filePath: string, mtime: number}[]} */
const allChatFiles = [];
const getCharacterChatFiles = async () => {
const pngDirents = await fs.promises.readdir(request.user.directories.characters, { withFileTypes: true });
const pngFiles = pngDirents.filter(e => e.isFile() && path.extname(e.name) === '.png').map(e => e.name);
for (const pngFile of pngFiles) {
const chatsDirectory = pngFile.replace('.png', '');
const pathToChats = path.join(request.user.directories.chats, chatsDirectory);
if (!fs.existsSync(pathToChats)) {
continue;
}
const pathStats = await fs.promises.stat(pathToChats);
if (pathStats.isDirectory()) {
const chatFiles = await fs.promises.readdir(pathToChats);
const jsonlFiles = chatFiles.filter(file => path.extname(file) === '.jsonl');
for (const file of jsonlFiles) {
const filePath = path.join(pathToChats, file);
const stats = await fs.promises.stat(filePath);
allChatFiles.push({ pngFile, filePath, mtime: stats.mtimeMs });
}
}
}
};
const getGroupChatFiles = async () => {
const groupDirents = await fs.promises.readdir(request.user.directories.groups, { withFileTypes: true });
const groups = groupDirents.filter(e => e.isFile() && path.extname(e.name) === '.json').map(e => e.name);
for (const group of groups) {
try {
const groupPath = path.join(request.user.directories.groups, group);
const groupContents = await fs.promises.readFile(groupPath, 'utf8');
const groupData = JSON.parse(groupContents);
if (Array.isArray(groupData.chats)) {
for (const chat of groupData.chats) {
const filePath = path.join(request.user.directories.groupChats, `${chat}.jsonl`);
if (!fs.existsSync(filePath)) {
continue;
}
const stats = await fs.promises.stat(filePath);
allChatFiles.push({ groupId: groupData.id, filePath, mtime: stats.mtimeMs });
}
}
} catch (error) {
// Skip group files that can't be read or parsed
continue;
}
}
};
await Promise.allSettled([getCharacterChatFiles(), getGroupChatFiles()]);
const max = parseInt(request.body.max ?? Number.MAX_SAFE_INTEGER);
const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, max);
const jsonFilesPromise = recentChats.map((file) => {
return file.groupId
? getChatInfo(file.filePath, { group: file.groupId }, true)
: getChatInfo(file.filePath, { avatar: file.pngFile }, false);
});
const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
const validFiles = chatData.filter(i => i.file_name);
return response.send(validFiles);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});

39
src/endpoints/groups.js
View File

@ -6,7 +6,6 @@ import sanitize from 'sanitize-filename';
import { sync as writeFileAtomicSync } from 'write-file-atomic';
import { humanizedISO8601DateTime } from '../util.js';
import { getChatInfo } from './characters.js';
export const router = express.Router();
@ -132,41 +131,3 @@ router.post('/delete', async (request, response) => {
return response.send({ ok: true });
});
router.post('/recent', async (request, response) => {
try {
/** @type {{groupId: string, filePath: string, mtime: number}[]} */
const allChatFiles = [];
const groups = fs.readdirSync(request.user.directories.groups).filter(x => path.extname(x) === '.json');
for (const group of groups) {
const groupPath = path.join(request.user.directories.groups, group);
const groupContents = fs.readFileSync(groupPath, 'utf8');
const groupData = JSON.parse(groupContents);
if (Array.isArray(groupData.chats)) {
for (const chat of groupData.chats) {
const filePath = path.join(request.user.directories.groupChats, `${chat}.jsonl`);
if (!fs.existsSync(filePath)) {
continue;
}
const stats = fs.statSync(filePath);
allChatFiles.push({ groupId: groupData.id, filePath, mtime: stats.mtimeMs });
}
}
}
const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, 15);
const jsonFilesPromise = recentChats.map((file) => {
return getChatInfo(file.filePath, { group: file.groupId }, true);
});
const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
const validFiles = chatData.filter(i => i.file_name);
return response.send(validFiles);
} catch (error) {
console.error('Error while getting recent group chats', error);
return response.sendStatus(500);
}
});