rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-03-01 02:17:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

Calculate global stats, prepare stats endpoints

Browse Source
This commit is contained in:
Wolfsblvt 2024-04-20 05:56:26 +02:00
parent 0d4cbf7da6
commit 08f6f8c405
2 changed files with 91 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
src/endpoints/stats.js
View File

@ -19,11 +19,9 @@ const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime();
const MIN_DATE = new Date(MIN_TIMESTAMP);
const MAX_DATE = new Date(MAX_TIMESTAMP);
const CURRENT_STATS_VERSION = '1.1';
const EMPTY_GLOBAL_STATS = { _calculated: MIN_DATE, version: CURRENT_STATS_VERSION, stats: {} };
/** @type {StatsCollection} The collection of all stats, accessable via their key */
let globalStats = { ...EMPTY_GLOBAL_STATS };
/** @type {StatsCollection} The collection of all stats, accessable via their key - gets set/built on init */
let globalStats;
let lastSaveDate = MIN_DATE;
/**
@ -48,8 +46,6 @@ async function init() {
throw err; // Rethrow the error if it's something we didn't expect
}
}
// Save stats every 5 minutes
setInterval(saveStatsToFile, 5 * 60 * 1000);
}
/**
@ -71,8 +67,10 @@ async function onExit() {
/**
* @typedef {object} StatsCollection - An object holding all character stats, and some additional main stats
* @property {string} version - Version number indication the version of this stats data - so it can be automatically migrated/recalculated if any of the calculation logic changes
* @property {CharacterStats} global - global characer stats
* @property {{[characterKey: string]: CharacterStats}} stats - All the dynamically saved stats objecs
* @property {Date} _calculated -
* @property {Date} _recalcualted -
*/
/**
@ -252,39 +250,59 @@ class AggregateStat {
async function recreateStats() {
console.log('Collecting and creating stats...');
/** @type {StatsCollection} */
const EMPTY_GLOBAL_STATS = { _calculated: MIN_DATE, _recalcualted: MIN_DATE, version: CURRENT_STATS_VERSION, global: newCharacterStats('global', 'Global'), stats: {} };
// Resetting global stats first
globalStats = { ...EMPTY_GLOBAL_STATS };
globalStats = { ...EMPTY_GLOBAL_STATS, };
// Load all char files to process their chat folders
const files = await readdir(DIRECTORIES.characters);
const charFiles = files.filter((file) => file.endsWith('.png'));
let processingPromises = charFiles.map((charFileName, _) =>
recreateCharacterStats(charFileName),
recreateCharacterStats(charFileName.replace('.png', '')),
);
await Promise.all(processingPromises);
// Remember the date at which those stats were recalculated from the ground up
globalStats._recalcualted = now();
await saveStatsToFile();
console.debug('Stats (re)created and saved to file.');
return globalStats;
/** @param {string} characterKey @return {CharacterStats?} */
function recreateCharacterStats(characterKey) {
const charName = characterKey.replace('.png', '');
const charChatsDir = path.join(DIRECTORIES.chats, charName);
if (!fs.existsSync(charChatsDir)) {
return null;
}
const chatFiles = fs.readdirSync(charChatsDir);
chatFiles.forEach(chatName => {
triggerChatUpdate(characterKey, chatName);
});
return globalStats[characterKey];
};
}
/**
* Recreates stats for a specific character.
* Should be used very carefully, as it still has to recalculate most of the global stats.
*
* @param {string} characterKey
* @return {CharacterStats?}
*/
function recreateCharacterStats(characterKey) {
// If we are replacing on a existing global stats, we need to "remove" all old stats
if (globalStats.stats[characterKey]) {
for (const chatStats of globalStats.stats[characterKey].chatsStats) {
removeChatFromCharStats(globalStats.global, chatStats);
}
delete globalStats.stats[characterKey];
}
// Then load chats dir for this character to process
const charChatsDir = path.join(DIRECTORIES.chats, characterKey);
if (!fs.existsSync(charChatsDir)) {
return null;
}
const chatFiles = fs.readdirSync(charChatsDir);
chatFiles.forEach(chatName => {
triggerChatUpdate(characterKey, chatName);
});
return globalStats[characterKey];
};
/**
*
@ -324,6 +342,11 @@ function triggerChatUpdate(characterKey, chatName) {
// Update char stats with the processed chat stats
updateCharStatsWithChat(globalStats.stats[characterKey], chatStats);
// Update the global stats with this chat
updateCharStatsWithChat(globalStats.global, chatStats);
chatStats._calculated = now();
globalStats._calculated = now();
return chatStats;
}
@ -339,6 +362,8 @@ function updateCharStatsWithChat(stats, chatStats) {
// Check if we need to remove this chat's previous data first
removeChatFromCharStats(stats, chatStats);
stats.chatsStats.push(chatStats);
stats.chats++;
stats.chatSize += chatStats.chatSize;
stats.firstCreateDate = minDate(chatStats.createDate, stats.firstCreateDate) ?? stats.firstCreateDate;
@ -372,6 +397,7 @@ function updateCharStatsWithChat(stats, chatStats) {
Object.entries(chatStats.genModels).forEach(([model, data]) => addModelUsage(stats.genModels, model, data.tokens, data.count));
stats._calculated = now();
console.debug(`Successfully updated ${stats.name}'s stats with chat ${chatStats.chatName}`);
return true;
}
@ -495,7 +521,6 @@ function processChat(chatName, lines, { chatSize = 0 } = {}) {
// Set up the final values for chat
stats.chattingTime = calculateDuration(stats.createDate, stats.lastInteractionDate);
stats._calculated = now();
return stats;
}
@ -702,18 +727,44 @@ function getGlobalStats() {
const router = express.Router();
/**
* Handle a POST request to get the stats object
*
* This function returns the stats object that was calculated by the `calculateStats` function.
* @typedef {object} StatsRequestBody
* @property {boolean?} [global] - Whether the global stats are requested. If true, all other arguments are ignored
* @property {string?} [characterKey] - The character key for the character to request stats from
* @property {string?} [chatName] - The name of the chat file
*/
/**
* Handle a POST request to get the stats fromm
*
* This function returns the stats object that was calculated and updated based on the chats.
* Depending on the given request filter, it will either return global stats, character stats or chat stats.
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*/
router.post('/get', jsonParser, function (request, response) {
response.send(JSON.stringify(getGlobalStats()));
const send = (data) => response.send(JSON.stringify(data ?? {}));
/** @type {StatsRequestBody} */
const body = request.body;
if (!!body.global) {
return send(globalStats.global);
}
const characterKey = String(body.characterKey);
const chatName = String(body.characterKey);
if (characterKey && chatName) {
return send(globalStats.stats[characterKey]?.chatsStats.find(x => x.chatName == chatName));
}
if (characterKey) {
return send(globalStats.stats[characterKey]);
}
// If no specific filter was requested, we send all stats back
return send(globalStats);
});
/**
@ -725,31 +776,24 @@ router.post('/get', jsonParser, function (request, response) {
* @param {Object} response - Express response object.
*/
router.post('/recreate', jsonParser, async function (request, response) {
const send = (data) => response.send(JSON.stringify(data ?? {}));
/** @type {StatsRequestBody} */
const body = request.body;
try {
const characterKey = String(body.characterKey);
if (characterKey) {
recreateCharacterStats(characterKey);
return send(globalStats.stats[characterKey]);
}
await recreateStats();
return response.sendStatus(200);
return send(globalStats);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
/**
* Handle a POST request to update the stats object
*
* This function updates the stats object with the data from the request body.
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*
*/
router.post('/update', jsonParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
setCharStats(request.body);
return response.sendStatus(200);
});
module.exports = {
router,
init,

2
src/util.js
View File

@ -663,7 +663,7 @@ function calculateDuration(startTime, endTime) {
const startDate = startTime instanceof Date ? startTime : new Date(startTime);
const endDate = endTime instanceof Date ? endTime : new Date(endTime);
return startDate > endDate ? 0 : Math.max(endDate.getDate() - startDate.getDate(), 0);
return startDate > endDate ? 0 : Math.max(endDate.getTime() - startDate.getTime(), 0);
}
/** @param {string} timestamp @returns {Date|null} */