rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor in endpoint changes of user-folders from neo-server

Browse Source
This commit is contained in:
Wolfsblvt
2024-04-22 00:44:15 +02:00
parent 67e57ffd58
commit c5dff7b5d4
1 changed files with 130 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
src/endpoints/stats.js
View File

@@ -5,45 +5,62 @@ const writeFileAtomic = require('write-file-atomic');
const crypto = require('crypto'); const crypto = require('crypto');
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const { jsonParser } = require('../express-common');
const { readAndParseJsonlFile, timestampToMoment, humanizedToDate, calculateDuration, minDate, maxDate, now } = require('../util');
const { getAllUserHandles, getUserDirectories } = require('../users');
const readFile = fs.promises.readFile; const readFile = fs.promises.readFile;
const readdir = fs.promises.readdir; const readdir = fs.promises.readdir;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { readAndParseJsonlFile, timestampToMoment, humanizedToDate, calculateDuration, minDate, maxDate, now } = require('../util');
const statsFilePath = 'public/stats.json';
const MIN_TIMESTAMP = 0; const MIN_TIMESTAMP = 0;
const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime(); const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime();
const MIN_DATE = new Date(MIN_TIMESTAMP); const MIN_DATE = new Date(MIN_TIMESTAMP);
const MAX_DATE = new Date(MAX_TIMESTAMP); const MAX_DATE = new Date(MAX_TIMESTAMP);
const STATS_FILE = 'stats.json';
const CURRENT_STATS_VERSION = '1.1'; const CURRENT_STATS_VERSION = '1.1';
/** @type {StatsCollection} The collection of all stats, accessable via their key - gets set/built on init */ /** @type {Map<string, UserStatsCollection>} The stats collections for each user, accessable via their key - gets set/built on init */
let globalStats; const STATS = new Map();
let lastSaveDate = MIN_DATE; let lastSaveDate = MIN_DATE;
/**
* Gets the user stats collection. Creates a new empty one, if it didn't exist before
* @param {string} userHandle - The user handle
* @returns {UserStatsCollection}
*/
function getUserStats(userHandle) {
return STATS.get(userHandle) ?? createEmptyStats(userHandle)
}
/** /**
* Loads the stats file into memory. If the file doesn't exist or is invalid, * Loads the stats file into memory. If the file doesn't exist or is invalid,
* initializes stats by collecting and creating them for each character. * initializes stats by collecting and creating them for each character.
*/ */
async function init() { async function init() {
try { const userHandles = await getAllUserHandles();
const statsFileContent = await readFile(statsFilePath, 'utf-8'); for (const userHandle of userHandles) {
const obj = JSON.parse(statsFileContent); try {
// Migrate/recreate stats if the version has changed const directories = getUserDirectories(userHandle);
if (obj.version !== CURRENT_STATS_VERSION) { const statsFilePath = path.join(directories.root, STATS_FILE);
console.info(`Found outdated stats of version '${obj.version}'. Recreating stats for current version '${CURRENT_STATS_VERSION}'...`); const statsFileContent = await readFile(statsFilePath, 'utf-8');
await recreateStats(); let userStats = JSON.parse(statsFileContent);
}
globalStats = obj; // Migrate/recreate stats if the version has changed
} catch (err) { if (userStats.version !== CURRENT_STATS_VERSION) {
// If the file doesn't exist or is invalid, initialize stats console.info(`Found outdated stats for user ${userHandle} of version '${userStats.version}'. Recreating stats for current version '${CURRENT_STATS_VERSION}'...`);
if (err.code === 'ENOENT' || err instanceof SyntaxError) { userStats = await recreateStats(userHandle);
recreateStats(); }
} else {
throw err; // Rethrow the error if it's something we didn't expect STATS.set(userHandle, userStats);
} catch (err) {
// If the file doesn't exist or is invalid, initialize stats
if (err.code === 'ENOENT' || err instanceof SyntaxError) {
console.warn(`Error on reading stats file for user ${userHandle}. Trying to recreate it... Error was: ${err.message}`);
recreateStats(userHandle);
} else {
throw err; // Rethrow the error if it's something we didn't expect
}
} }
} }
} }
@@ -65,9 +82,9 @@ async function onExit() {
*/ */
/** /**
* @typedef {object} StatsCollection - An object holding all character stats, and some additional main stats * @typedef {object} UserStatsCollection - An object holding all character stats, and some additional main stats
* @property {string} version - Version number indication the version of this stats data - so it can be automatically migrated/recalculated if any of the calculation logic changes * @property {string} version - Version number indication the version of this stats data - so it can be automatically migrated/recalculated if any of the calculation logic changes
* @property {CharacterStats} global - global characer stats * @property {CharacterStats} global - global user/profile stats
* @property {{[characterKey: string]: CharacterStats}} stats - All the dynamically saved stats objecs * @property {{[characterKey: string]: CharacterStats}} stats - All the dynamically saved stats objecs
* @property {Date} _calculated - * @property {Date} _calculated -
* @property {Date} _recalcualted - * @property {Date} _recalcualted -
@@ -244,109 +261,128 @@ class AggregateStat {
/** /**
* *
* * @param {string} userHandle - User handle
* @returns {Promise<StatsCollection>} The aggregated stats object. * @returns {UserStatsCollection} The aggregated stats object
*/ */
async function recreateStats() { function createEmptyStats(userHandle) {
const EMPTY_USER_STATS = { _calculated: MIN_DATE, _recalcualted: MIN_DATE, version: CURRENT_STATS_VERSION, global: newCharacterStats(userHandle, 'Global'), stats: {} };
// Resetting global user stats
const userStats = { ...EMPTY_USER_STATS };
STATS.set(userHandle, userStats);
return userStats;
}
/**
*
* @param {string} userHandle - User handle
* @returns {Promise<UserStatsCollection>} The aggregated stats object
*/
async function recreateStats(userHandle) {
console.log('Collecting and creating stats...'); console.log('Collecting and creating stats...');
/** @type {StatsCollection} */ const userStats = createEmptyStats(userHandle);
const EMPTY_GLOBAL_STATS = { _calculated: MIN_DATE, _recalcualted: MIN_DATE, version: CURRENT_STATS_VERSION, global: newCharacterStats('global', 'Global'), stats: {} };
// Resetting global stats first
globalStats = { ...EMPTY_GLOBAL_STATS, };
// Load all char files to process their chat folders // Load all char files to process their chat folders
const files = await readdir(DIRECTORIES.characters); const directories = getUserDirectories(userHandle);
const files = await readdir(directories.characters);
const charFiles = files.filter((file) => file.endsWith('.png')); const charFiles = files.filter((file) => file.endsWith('.png'));
let processingPromises = charFiles.map((charFileName, _) => let processingPromises = charFiles.map((charFileName, _) =>
recreateCharacterStats(charFileName.replace('.png', '')), recreateCharacterStats(userHandle, charFileName.replace('.png', '')),
); );
await Promise.all(processingPromises); await Promise.all(processingPromises);
// Remember the date at which those stats were recalculated from the ground up // Remember the date at which those stats were recalculated from the ground up
globalStats._recalcualted = now(); userStats._recalcualted = now();
await saveStatsToFile(); await saveStatsToFile();
console.debug('Stats (re)created and saved to file.'); console.info(`Stats for user ${userHandle} (re)created and saved to file.`);
return globalStats; return userStats;
} }
/** /**
* Recreates stats for a specific character. * Recreates stats for a specific character.
* Should be used very carefully, as it still has to recalculate most of the global stats. * Should be used very carefully, as it still has to recalculate most of the global stats.
* *
* @param {string} characterKey * @param {string} userHandle - User handle
* @param {string} characterKey -
* @return {CharacterStats?} * @return {CharacterStats?}
*/ */
function recreateCharacterStats(characterKey) { function recreateCharacterStats(userHandle, characterKey) {
const userStats = getUserStats(userHandle);
// If we are replacing on a existing global stats, we need to "remove" all old stats // If we are replacing on a existing global stats, we need to "remove" all old stats
if (globalStats.stats[characterKey]) { if (userStats.stats[characterKey]) {
for (const chatStats of globalStats.stats[characterKey].chatsStats) { for (const chatStats of userStats.stats[characterKey].chatsStats) {
removeChatFromCharStats(globalStats.global, chatStats); removeChatFromCharStats(userStats.global, chatStats);
} }
delete globalStats.stats[characterKey]; delete userStats.stats[characterKey];
} }
// Then load chats dir for this character to process // Then load chats dir for this character to process
const charChatsDir = path.join(DIRECTORIES.chats, characterKey); const directories = getUserDirectories(userHandle);
const charChatsDir = path.join(directories.chats, characterKey);
if (!fs.existsSync(charChatsDir)) { if (!fs.existsSync(charChatsDir)) {
return null; return null;
} }
const chatFiles = fs.readdirSync(charChatsDir); const chatFiles = fs.readdirSync(charChatsDir);
chatFiles.forEach(chatName => { chatFiles.forEach(chatName => {
triggerChatUpdate(characterKey, chatName); triggerChatUpdate(userHandle, characterKey, chatName);
}); });
return globalStats[characterKey]; console.info(`(Re)created ${characterKey}'s character stats for user ${userHandle}.`);
return userStats[characterKey];
}; };
/** /**
* *
* @param {string} charChatsDir - The directoy path * @param {string} userHandle - The user handle
* @param {string} characterKey
* @param {string} chatName * @param {string} chatName
* @returns {{chatName: string, lines: object[]}} * @returns {{chatName: string, charName: string, filePath: string, lines: object[]}}
*/ */
function loadChatFile(charChatsDir, chatName) { function loadChatFile(userHandle, characterKey, chatName) {
const fullFilePath = path.join(charChatsDir, sanitize(chatName)); const charName = characterKey.replace('.png', '');
const lines = readAndParseJsonlFile(fullFilePath); const directories = getUserDirectories(userHandle);
return { chatName, lines }; const charChatsDir = path.join(directories.chats, charName);
const filePath = path.join(charChatsDir, sanitize(chatName));
const lines = readAndParseJsonlFile(filePath);
return { chatName, charName, filePath, lines };
} }
/** /**
* *
* * @param {string} userHandle - The user handle
* @param {string} characterKey - The character key * @param {string} characterKey - The character key
* @param {string} chatName - The name of the chat * @param {string} chatName - The name of the chat
* @returns {ChatStats?} * @returns {ChatStats?}
*/ */
function triggerChatUpdate(characterKey, chatName) { function triggerChatUpdate(userHandle, characterKey, chatName) {
const charName = characterKey.replace('.png', '');
const charChatsDir = path.join(DIRECTORIES.chats, charName);
// Load and process chats to get its stats // Load and process chats to get its stats
const loadedChat = loadChatFile(charChatsDir, chatName); const loadedChat = loadChatFile(userHandle, characterKey, chatName);
const fsStats = fs.statSync(path.join(charChatsDir, chatName)); const fsStats = fs.statSync(loadedChat.filePath);
const chatStats = processChat(chatName, loadedChat.lines, { chatSize: fsStats.size }); const chatStats = processChat(chatName, loadedChat.lines, { chatSize: fsStats.size });
if (chatStats === null) { if (chatStats === null) {
return null; return null;
} }
const userStats = getUserStats(userHandle);
// Create empty stats if character stats don't exist yet // Create empty stats if character stats don't exist yet
globalStats.stats[characterKey] ??= newCharacterStats(characterKey, charName); userStats.stats[characterKey] ??= newCharacterStats(characterKey, loadedChat.charName);
// Update char stats with the processed chat stats // Update both the char stats and the global user stats with this chat
updateCharStatsWithChat(globalStats.stats[characterKey], chatStats); updateCharStatsWithChat(userStats.stats[characterKey], chatStats);
updateCharStatsWithChat(userStats.global, chatStats);
// Update the global stats with this chat
updateCharStatsWithChat(globalStats.global, chatStats);
chatStats._calculated = now(); chatStats._calculated = now();
globalStats._calculated = now(); userStats.global._calculated = now()
userStats._calculated = now();
return chatStats; return chatStats;
} }
@@ -702,32 +738,27 @@ function newMessageStats() {
/** /**
* Saves the current state of charStats to a file, only if the data has changed since the last save. * Saves the current state of charStats to a file, only if the data has changed since the last save.
* @param {string?} [userHandle] - Optionally only save file for one user handle
*/ */
async function saveStatsToFile() { async function saveStatsToFile(userHandle = null) {
if (globalStats._calculated > lastSaveDate) { const userHandles = userHandle ? [userHandle] : await getAllUserHandles();
//console.debug("Saving stats to file..."); for (const userHandle of userHandles) {
try { const userStats = getUserStats(userHandle);
await writeFileAtomic(statsFilePath, JSON.stringify(globalStats)); if (userStats._calculated > lastSaveDate) {
lastSaveDate = now(); try {
} catch (error) { const directories = getUserDirectories(userHandle);
console.log('Failed to save stats to file.', error); const statsFilePath = path.join(directories.root, STATS_FILE);
await writeFileAtomic(statsFilePath, JSON.stringify(userStats));
lastSaveDate = now();
} catch (error) {
console.log('Failed to save stats to file.', error);
}
} }
} else {
//console.debug('Stats have not changed since last save. Skipping file write.');
} }
} }
/**
* Returns the current global stats object
* @returns {StatsCollection}
**/
function getGlobalStats() {
return globalStats;
}
const router = express.Router(); const router = express.Router();
/** /**
* @typedef {object} StatsRequestBody * @typedef {object} StatsRequestBody
* @property {boolean?} [global] - Whether the global stats are requested. If true, all other arguments are ignored * @property {boolean?} [global] - Whether the global stats are requested. If true, all other arguments are ignored
@@ -750,21 +781,24 @@ router.post('/get', jsonParser, function (request, response) {
/** @type {StatsRequestBody} */ /** @type {StatsRequestBody} */
const body = request.body; const body = request.body;
const userHandle = request.user.profile.handle;
const userStats = getUserStats(userHandle);
if (!!body.global) { if (!!body.global) {
return send(globalStats.global); return send(userStats.global);
} }
const characterKey = String(body.characterKey); const characterKey = String(body.characterKey);
const chatName = String(body.characterKey); const chatName = String(body.characterKey);
if (characterKey && chatName) { if (characterKey && chatName) {
return send(globalStats.stats[characterKey]?.chatsStats.find(x => x.chatName == chatName)); return send(userStats.stats[characterKey]?.chatsStats.find(x => x.chatName == chatName));
} }
if (characterKey) { if (characterKey) {
return send(globalStats.stats[characterKey]); return send(userStats.stats[characterKey]);
} }
// If no specific filter was requested, we send all stats back // If no specific filter was requested, we send all stats back
return send(globalStats); return send(userStats);
}); });
/** /**
@@ -780,14 +814,16 @@ router.post('/recreate', jsonParser, async function (request, response) {
/** @type {StatsRequestBody} */ /** @type {StatsRequestBody} */
const body = request.body; const body = request.body;
const userHandle = request.user.profile.handle;
try { try {
const characterKey = String(body.characterKey); const characterKey = String(body.characterKey);
if (characterKey) { if (characterKey) {
recreateCharacterStats(characterKey); recreateCharacterStats(userHandle, characterKey);
return send(globalStats.stats[characterKey]); return send(getUserStats(userHandle).stats[characterKey]);
} }
await recreateStats(); await recreateStats(userHandle);
return send(globalStats); return send(getUserStats(userHandle));
} catch (error) { } catch (error) {
console.error(error); console.error(error);
return response.sendStatus(500); return response.sendStatus(500);