Stats: Implement data retrieval into char popup

- Parse json with Date objects
- Fix char directory
- Add sub info for aggregated stats
- Correctly pull names out of the chat files
- Rework humanized duration, humanized timespan, humanized filesize
- Add smart truncate and sensible round
- Implement/Fix values into the character stat popup
- Implement correct stat API calls on client side

public/css/stats.css

@@ -40,6 +40,7 @@
     flex: 1;
     height: calc(var(--mainFontSize) * 1.33333333333);
     text-align: right;
+    overflow: hidden;
     padding-left: 2px;
     padding-right: 2px;
 }

public/script.js

@@ -1,5 +1,5 @@
 import { humanizedDateTime, favsToHotswap, getMessageTimeStamp, dragElement, isMobile, initRossMods, shouldSendOnEnter } from './scripts/RossAscends-mods.js';
-import { userStatsHandler, statMesProcess, initStats } from './scripts/stats.js';
+import { initStats } from './scripts/stats.js';
 import {
     generateKoboldWithStreaming,
     kai_settings,
@@ -228,6 +228,7 @@ export {
     clearChat,
     getChat,
     getCharacters,
+    getCharacter,
     getGeneratingApi,
     callPopup,
     substituteParams,
@@ -1459,6 +1460,23 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) {
     return entities;
 }
 
+/**
+ * Get one character from the character list via its character key
+ *
+ * To retrieve/refresh it from the API, use `getOneCharacter` to update it first.
+ *
+ * @param {string} characterKey - The character key / avatar url
+ * @returns {object}
+ */
+function getCharacter(characterKey) {
+    return characters.find(x => x.avatar === characterKey);
+}
+
+/**
+ * Gets one character from via API
+ *
+ * @param {string} avatarUrl - The avatar url / character key
+ */
 export async function getOneCharacter(avatarUrl) {
     const response = await fetch('/api/characters/get', {
         method: 'POST',
@@ -4461,7 +4479,7 @@ export async function sendMessageAsUser(messageText, messageBias, insertAt = nul
     }
 
     await populateFileAttachment(message);
-    statMesProcess(message, 'user', characters, this_chid, '');
+    // statMesProcess(message, 'user', characters, this_chid, '');
 
     if (typeof insertAt === 'number' && insertAt >= 0 && insertAt <= chat.length) {
         chat.splice(insertAt, 0, message);
@@ -10504,10 +10522,6 @@ jQuery(async function () {
             }, 2000); */
     //});
 
-    $('.user_stats_button').on('click', function () {
-        userStatsHandler();
-    });
-
     $('#external_import_button').on('click', async () => {
         const html = `<h3>Enter the URL of the content to import</h3>
         Supported sources:<br>

public/scripts/RossAscends-mods.js

@@ -80,19 +80,21 @@ observer.observe(document.documentElement, observerConfig);
 
 
 /**
- * Converts generation time from milliseconds to a human-readable format.
+ * Converts a timespan from milliseconds to a human-readable format.
  *
- * The function takes total generation time as an input, then converts it to a format
+ * The function takes a total timespan as an input, then converts it to a format
  * of "_ Days, _ Hours, _ Minutes, _ Seconds". If the generation time does not exceed a
  * particular measure (like days or hours), that measure will not be included in the output.
  *
- * @param {number} total_gen_time - The total generation time in milliseconds.
+ * @param {number} timespan - The total timespan in milliseconds.
- * @param {boolean} [short=false] - Optional flag indicating whether short form should be used. ('2h' instead of '2 Hours')
+ * @param {object} [options] - Optional parameters
- * @returns {string} - A human-readable string that represents the time spent generating characters.
+ * @param {boolean} [options.short=false] - Flag indicating whether short form should be used. ('2h' instead of '2 Hours')
+ * @param {number} [options.onlyHighest] - Number of maximum blocks to be returned. (If, and daya is the highest matching unit, only returns days and hours, cutting of minutes and seconds)
+ * @returns {string} - A human-readable string that represents the timespan.
  */
-export function humanizeGenTime(total_gen_time, short = false) {
+export function humanizeTimespan(timespan, { short = false, onlyHighest = 2 } = {}) {
     //convert time_spent to humanized format of "_ Hours, _ Minutes, _ Seconds" from milliseconds
-    let time_spent = total_gen_time || 0;
+    let time_spent = timespan || 0;
     time_spent = Math.floor(time_spent / 1000);
     let seconds = time_spent % 60;
     time_spent = Math.floor(time_spent / 60);
@@ -101,13 +103,36 @@ export function humanizeGenTime(total_gen_time, short = false) {
     let hours = time_spent % 24;
     time_spent = Math.floor(time_spent / 24);
     let days = time_spent;
-    let parts = [];
+
-    if (days > 0) { parts.push(short ? `${days}d` : `${days} Days`); }
+    let parts = [
-    if (hours > 0) { parts.push(short ? `${hours}h` : `${hours} Hours`); }
+        { singular: 'Day', plural: 'Days', short: 'd', value: days },
-    if (minutes > 0) { parts.push(short ? `${minutes}m` : `${minutes} Minutes`); }
+        { singular: 'Hour', plural: 'Hours', short: 'h', value: hours },
-    if (seconds > 0) { parts.push(short ? `${seconds}s` : `${seconds} Seconds`); }
+        { singular: 'Minute', plural: 'Minutes', short: 'm', value: minutes },
-    if (!parts.length) { parts.push(short ? '<1s' : 'Instant') }
+        { singular: 'Second', plural: 'Seconds', short: 's', value: seconds },
-    return parts.join(short ? ' ' : ', ');
+    ];
+
+    // Build the final string based on the highest significant units and respecting zeros
+    let resultParts = [];
+    let count = 0;
+    for (let part of parts) {
+        if (part.value > 0) {
+            resultParts.push(part);
+        }
+
+        // If we got a match, we count from there. Take a maximum of X elements
+        if (resultParts.length) count++;
+        if (count >= onlyHighest) {
+            break;
+        }
+    }
+
+    if (!resultParts.length) {
+        return short ? '<1s' : 'Instant';
+    }
+
+    return resultParts.map(part => {
+        return short ? `${part.value}${part.short}` : `${part.value} ${part.value === 1 ? part.singular : part.plural}`;
+    }).join(short ? ' ' : ', ');
 }
 
 /**

public/scripts/stats.js

@@ -1,133 +1,76 @@
 // statsHelper.js
-import { getRequestHeaders, callPopup, characters, this_chid, buildAvatarList, characterToEntity } from '../script.js';
+import { getRequestHeaders, callPopup, characters, this_chid, buildAvatarList, characterToEntity, getOneCharacter, getCharacter } from '../script.js';
-import { humanizeGenTime } from './RossAscends-mods.js';
+import { humanizeTimespan } from './RossAscends-mods.js';
 import { registerDebugFunction } from './power-user.js';
-import { timestampToMoment } from './utils.js';
+import { humanFileSize, humanizedDuration, parseJson, sensibleRound, smartTruncate } from './utils.js';
 
-const MIN_TIMESTAMP = 0;
-const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime();
-const CURRENT_STATS_VERSION = '1.0';
 
 /** @typedef {import('../script.js').Character} Character */
+/** @typedef {import('../../src/endpoints/stats.js').UserStatsCollection} UserStatsCollection */
+/** @typedef {import('../../src/endpoints/stats.js').CharacterStats} CharacterStats */
+/** @typedef {import('../../src/endpoints/stats.js').ChatStats} ChatStats */
+/** @typedef {import('../../src/endpoints/stats.js').MessageStats} MessageStats */
+/** @typedef {import('../../src/endpoints/stats.js').StatsRequestBody} StatsRequestBody */
 
 /**
  * @typedef {object} AggregateStat
- * @property {number[]} list - The values stored, so they can be recalculated
+ * @property {number} count - The number of stats used for this aggregation - used for recalculating avg
  * @property {number} total - Total / Sum
  * @property {number} min - Minimum value
  * @property {number} max - Maximum value
  * @property {number} avg - Average value
+ * @property {number[]} values - All values listed and saved, so the aggregate stats can be updated if needed when elements get removed
+ * @property {number?} subCount - The number of stats used when this is aggregated over the totals of aggregated stats, meaning based on any amount of sub/inner values
  */
 
-/**
- * @typedef {object} Stats - Stats for a chat
- * @property {number} chat_size - The size of the actual chat file
- * @property {number} date_first_chat - Timestamp of the first chat message (made by user)
- * @property {number} date_last_chat - Timestamp of the last chat message in this chat (made by anyone)
- * @property {number} total_gen_time - Total generation time in milliseconds
- * @property {number} total_msg_count - The total messages of user and non-user, not including swipes
- * @property {number} total_swipe_count - The number of swipes in the whole chat
- * @property {number} avg_gen_time - Average generation tme in milliseconds
- * @property {number} avg_swipe_count - Average swipes per non-user message
- *
- * @property {number} avg_chat_msg_count - Average messages per chat
- * @property {number} avg_chat_duration - Average duration of a chat (from first till last message)
- *
- * @property {AggregateStat} msg -
- * @property {AggregateStat} user_msg -
- * @property {AggregateStat} non_user_msg -
- * @property {AggregateStat} words -
- * @property {AggregateStat} user_words -
- * @property {AggregateStat} non_user_words -
- *
- */
-
-
-
-/** @type {StatsCollection} The collection of all stats, accessable via their key */
-let charStats = { timestamp: 0, version: CURRENT_STATS_VERSION, stats: {} };
-
-/**
- * Creates an empty new stats object with default values
- * @returns {Stats} The stats
- */
-function createEmptyStats() {
-    return {
-        total_gen_time: 0,
-        user_word_count: 0,
-        non_user_word_count: 0,
-        user_msg_count: 0,
-        non_user_msg_count: 0,
-        total_swipe_count: 0,
-        chat_size: 0,
-        date_first_chat: MAX_TIMESTAMP,
-        date_last_chat: MIN_TIMESTAMP,
-    };
-}
-
-/**
- * Verifies and returns a numerical stat value. If the provided stat is not a number, returns 0.
- *
- * @param {number|string} stat - The stat value to be checked and returned.
- * @returns {number} - The stat value if it is a number, otherwise 0.
- */
-function verifyStatValue(stat) {
-    return isNaN(Number(stat)) ? 0 : Number(stat);
-}
-
-/**
- * Calculates total stats from character statistics.
- *
- * @returns {Stats} - Object containing total statistics.
- */
-function calculateTotalStats() {
-    let totalStats = createEmptyStats();
-
-    for (let stats of Object.values(charStats.stats)) {
-        totalStats.total_gen_time += verifyStatValue(stats.total_gen_time);
-        totalStats.user_msg_count += verifyStatValue(stats.user_msg_count);
-        totalStats.non_user_msg_count += verifyStatValue(stats.non_user_msg_count);
-        totalStats.user_word_count += verifyStatValue(stats.user_word_count);
-        totalStats.non_user_word_count += verifyStatValue(stats.non_user_word_count);
-        totalStats.total_swipe_count += verifyStatValue(stats.total_swipe_count);
-        totalStats.date_last_chat = Math.max(totalStats.date_last_chat, verifyStatValue(stats.date_last_chat) || MIN_TIMESTAMP);
-        totalStats.date_first_chat = Math.min(totalStats.date_first_chat, verifyStatValue(stats.date_first_chat) || MAX_TIMESTAMP);
-    }
-
-    return totalStats;
-}
-
-/**
- * Build a humanized string for a duration
- * @param {number} start - Start time (in milliseconds)
- * @param {number|null} end - End time (in milliseconds), if null will be replaced with Date.now()
- * @param {object} param2 - Optional parameters
- * @param {string} [param2.fallback='Never'] - Fallback value no duration can be calculated
- * @param {function(string): string} [param2.wrapper=null] - Optional function to wrap/format the resulting humanized duration
- * @returns {string} Humanized duration string
- */
-function humanizedDuration(start, end = null, { fallback = 'Never', wrapper = null } = {}) {
-    end = end ?? Date.now();
-    if (!start || start > end) {
-        return fallback;
-    }
-    // @ts-ignore
-    const humanized = moment.duration(end - start).humanize();
-    return wrapper ? wrapper(humanized) : humanized;
-}
-
 /**
  * @typedef {object} StatField A stat block value to print
  * @property {any} value - The value to print
  * @property {boolean} [isHeader=false] - Flag indicating whether this is a header
  * @property {string|null} [info=null] - Optional text that will be shown as an info icon
- * @property {string|null} [title=null] - Optional title for the value - if null and info is set, info will be used as title too
+ * @property {string|'info'|null} [title=null] - Optional title for the value - if set to 'info', info will be used as title too
  * @property {string[]|null} [classes=null] - Optional list of classes for the stat field
  */
 
+/**
+ * @typedef {object} AggBuildOptions Blah
+ * @property {string | {singular: string, plural: string}} [options.basedOn='chat'] -
+ * @property {string | {singular: string, plural: string}} [options.basedOnSub='message'] -
+ * @property {boolean} [options.excludeTotal=false] - Exclude
+ * @property {((value: *) => string)} [options.transform=null] -
+ */
 
-/** @param {StatField|any} x @returns {StatField} gets the stat field object for any value */
+/** @type {AggBuildOptions} */
-function field(x) { return (typeof x === 'object' && x !== null && Object.hasOwn(x, 'value')) ? x : { value: x }; }
+const DEFAULT_AGG_BUILD_OPTIONS = { basedOn: 'chat', basedOnSub: 'message', excludeTotal: false, transform: null };
+
+/**
+ * Gets the fields for an aggregated value
+ * @param {AggregateStat} agg -
+ * @param {AggBuildOptions} [options=DEFAULT_AGG_BUILD_OPTIONS] -
+ * @returns {StatField[]}
+ */
+function aggregateFields(agg, options = DEFAULT_AGG_BUILD_OPTIONS) {
+    options = { ...DEFAULT_AGG_BUILD_OPTIONS, ...options };
+    const basedOn = (typeof options.basedOn !== 'object' || options.basedOn === null) ? { singular: `${options.basedOn}`, plural: `${options.basedOn}s` } : options.basedOn;
+    const basedOnSub = (typeof options.basedOnSub !== 'object' || options.basedOnSub === null) ? { singular: `${options.basedOnSub}`, plural: `${options.basedOnSub}s` } : options.basedOnSub;
+
+    /** @param {*|number} val @param {string} name @returns {StatField} */
+    const build = (val, name) => {
+        // Apply transform and rounding
+        let value = options.transform ? options.transform(val) : val;
+        value = typeof value === 'number' ? sensibleRound(value) : value;
+
+        // Build title tooltip
+        let title = `${name}, based on ${agg.count} ${agg.count !== 1 ? basedOn.plural : basedOn.singular}`
+        if (agg.subCount) title += ` and ${agg.subCount} ${agg.subCount !== 1 ? basedOnSub.plural : basedOnSub.singular}`;
+
+        return { value: value, title: title };
+    };
+    return [options.excludeTotal ? null : build(agg.total, 'Total'), build(agg.min, 'Minimum'), build(agg.avg, 'Average'), build(agg.max, 'Maximum')];
+}
+
+/** Gets the stat field object for any value @param {StatField|any} x @returns {StatField} */
+function statField(x) { return (typeof x === 'object' && x !== null && Object.hasOwn(x, 'value')) ? x : { value: x }; }
 
 /**
  * Creates an HTML stat block
@@ -140,14 +83,14 @@ function createStatBlock(name, ...values) {
     /** @param {StatField} stat @returns {string} */
     function buildField(stat) {
         const classes = ['rm_stat_field', stat.isHeader ? 'rm_stat_header' : '', ...(stat.classes ?? [])].filter(x => x?.length);
-        return `<div class="${classes.join(' ')}" ${stat.title || stat.info ? `title="${stat.title ?? stat.info}"` : ''}>
+        return `<div class="${classes.join(' ')}" ${stat.title ? `title="${stat.title === 'info' ? stat.info : stat.title}"` : ''}>
             ${stat.value === null || stat.value === '' ? '&zwnj;' : stat.value}
             ${stat.info ? `<small><div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]${stat.info}" title="${stat.info}"></div></small>` : ''}
         </div>`;
     }
 
-    const statName = field(name);
+    const statName = statField(name);
-    const statValues = values.flat(Infinity).map(field);
+    const statValues = values.flat(Infinity).map(statField);
 
     const isDataRow = !statName.isHeader && !statValues.some(x => x.isHeader);
     const isRightSpacing = statValues.slice(-1)[0]?.classes?.includes('rm_stat_right_spacing');
@@ -171,85 +114,84 @@ function createStatBlock(name, ...values) {
  * The stat blocks are tailored depending on the stats type ("User" or "Character").
  *
  * @param {'User'|'Character'} statsType - The type of stats (e.g., "User", "Character")
- * @param {number|null} characterId - Character id for these stats, null if global
+ * @param {CharacterStats} stats - The stats data
- * @param {Stats} stats - The stats data
  */
-function createHtml(statsType, characterId, stats) {
+function createHtml(statsType, stats) {
     const NOW = Date.now();
-    const name = characters[characterId]?.name || 'User';
+    const character = getCharacter(stats.characterKey);
-    const HMTL_STAT_SPACER = '<div class="rm_stat_spacer"></div>';
 
-    /** @param {number} charVal @param {number} userVal @returns {string}  */
+    const HMTL_STAT_SPACER = '<div class="rm_stat_spacer"></div>';
-    function buildBar(userVal, charVal) {
+    const VAL_RIGHT_SPACING = { value: null, classes: ['rm_stat_right_spacing'] };
-        const percentUser = (userVal / (userVal + charVal)) * 100;
+    const BASED_ON_MES_AND_SWIPE = { singular: 'message or swipe', plural: 'messages and swipes' };
-        const percentChar = 100 - percentUser;
+    const HOVER_TOOLTIP_SUFFIX = '\n\nHover over any value to see what it is based on.';
-        return `<div class="rm_stat_bar">
+    const GEN_TOKEN_WARNING = '(Token count is only correct, if setting \'Message Token Count\' was turned on during generation)';
-            <div style="width: ${percentUser}%" title="User: ${userVal}   (${percentUser.toFixed(1)}%)" class="rm_stat_bar_user"></div>
+
-            <div style="width: ${percentChar}%" title="${name}: ${charVal}   (${percentChar.toFixed(1)}%)" class="rm_stat_bar_char"></div>
+    // some pre calculations
-        </div>`;
+    const mostUsedModel = findHighestModel(stats.genModels);
-    }
-    /** @param {any[]} values @returns {StatField[]}  */
-    function buildBarDesc(...values) {
-        return values.flat(Infinity).map(field).map((x, i) => i % 2 == 0 ? { classes: [...(x.classes ?? []), 'rm_stat_field_lefty'], ...x } : x);
-    }
 
     // Create popup HTML with stats
-    let html = `<h3>${statsType} Stats - ${name}</h3>`;
+    let html = `<h3>${statsType} Stats - ${stats.charName}</h3>`;
     html += HMTL_STAT_SPACER;
     html += createStatBlock({ value: 'Character Overview', isHeader: true });
-    html += createStatBlock('Chats', { value: 34 }, { value: null, classes: ['rm_stat_right_spacing'] });
+    html += createStatBlock({ value: 'Chats', info: `The number of existing chats with ${stats.charName}.\nFor the sake of statistics, Branches count as chats and all their messages will be included.` },
-    html += createStatBlock({ value: 'Chats Size', info: 'The chat file sizes calculated and summed.\nThis value is only estimated, and will be refreshed sporadically.' }, { value: `~${'3.54 mb'}` }, { value: null, classes: ['rm_stat_right_spacing'] });
+        stats.chats, VAL_RIGHT_SPACING);
-    html += createStatBlock('Most Used Model', { value: 'Noromaid' }, { value: null, classes: ['rm_stat_right_spacing'] });
+    html += createStatBlock({ value: 'File Size', info: 'The chat file sizes on disk calculated and summed.\nThis value might not represent the exact same value your operating system uses.' },
+        humanFileSize(stats.chatSize), VAL_RIGHT_SPACING);
+    html += createStatBlock({ value: 'Most Used Model', info: 'Most used model for generations, both messages and swipes.\n(Does not include internal generation commands like /gen or /impersonate)\n\nHover over the value to see the numbers behind.' },
+        { value: smartTruncate(mostUsedModel.model, 32), title: 'info', info: `${mostUsedModel.model}\nUsed ${mostUsedModel.count} times to generate ${mostUsedModel.tokens} tokens\n\n${GEN_TOKEN_WARNING}.` }, VAL_RIGHT_SPACING);
     html += HMTL_STAT_SPACER;
     html += createStatBlock('',
-        { value: 'First', isHeader: true, info: `The data corresponding to the first chat with ${name}` },
+        { value: 'First', isHeader: true, info: `Data corresponding to the first chat with ${stats.charName}` },
-        { value: 'Last', isHeader: true, info: `The data corresponding to the last chat with ${name}` },
+        { value: 'Last', isHeader: true, info: `Data corresponding to the last chat with ${stats.charName}` },
-        { value: null, classes: ['rm_stat_right_spacing'] },
+        VAL_RIGHT_SPACING,
     );
-    html += createStatBlock({ value: 'New Chat', info: 'The first/last time when a new chat was started' },
+    html += createStatBlock({ value: 'New Chat', info: 'The first/last time when a new chat was started.' },
-        { value: humanizedDuration(stats.date_first_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_first_chat).format('LL LT') },
+        { value: humanizedDuration(stats.firstCreateDate, NOW, { wrapper: x => `${x} ago` }), title: stats.firstCreateDate },
-        { value: humanizedDuration(stats.date_last_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_last_chat).format('LL LT') },
+        { value: humanizedDuration(stats.lastCreateDate, NOW, { wrapper: x => `${x} ago` }), title: stats.lastCreateDate },
-        { value: null, classes: ['rm_stat_right_spacing'] },
+        VAL_RIGHT_SPACING,
     );
-    html += createStatBlock({ value: 'Chat Ended', info: 'The first/last time when the last message was send to a chat' },
+    html += createStatBlock({ value: 'Chat Ended', info: 'The first/last time when the last message was send to a chat.' },
-        { value: humanizedDuration(stats.date_first_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_first_chat).format('LL LT') },
+        { value: humanizedDuration(stats.firstlastInteractionDate, NOW, { wrapper: x => `${x} ago` }), title: stats.firstlastInteractionDate },
-        { value: humanizedDuration(stats.date_last_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_last_chat).format('LL LT') },
+        { value: humanizedDuration(stats.lastLastInteractionDate, NOW, { wrapper: x => `${x} ago` }), title: stats.lastLastInteractionDate },
-        { value: null, classes: ['rm_stat_right_spacing'] },
+        VAL_RIGHT_SPACING,
     );
 
     html += HMTL_STAT_SPACER;
     html += HMTL_STAT_SPACER;
-    html += createStatBlock({ value: 'Aggregated Stats', isHeader: true, info: 'Values per chat, aggregated over all chats' });
+    html += createStatBlock({ value: 'Aggregated Stats', isHeader: true, info: 'Values per chat, aggregated over all chats\n\n • Total: Total summed value over all chats\n • Min: Minium value for any chat\n • Avg: Average value over all chats\n • Max: Maximum value for any chat' });
     html += createStatBlock(null,
         { value: 'Total', isHeader: true, info: 'Total summed value over all chats' },
         { value: 'Min', isHeader: true, info: 'Minium value for any chat' },
         { value: 'Avg', isHeader: true, info: 'Average value over all chats' },
         { value: 'Max', isHeader: true, info: 'Maximum value for any chat' }
     );
-    html += createStatBlock({ value: 'Chatting Time', info: 'Total chatting time over all chats, and min/avg/max chatting time per chat' },
+    html += createStatBlock({ value: 'Chatting Time', info: 'Chatting time per chat\nCalculated based on chat creation and the last interaction in that chat.' + HOVER_TOOLTIP_SUFFIX },
-        { value: humanizeGenTime(114387009, true) }, { value: humanizeGenTime(7203, true) }, { value: humanizeGenTime(159017, true) }, { value: humanizeGenTime(7884930, true) });
+        ...aggregateFields(stats.chattingTime, { transform: time => humanizeTimespan(time, { short: true }) }));
-    html += createStatBlock({ value: 'Generation Time', info: 'Total generation time over all chats, and min/avg/max generation time per chat' },
+    html += createStatBlock({ value: 'Generation Time', info: 'Generation time per chat\nSummed generation times of all messages and swipes.' + HOVER_TOOLTIP_SUFFIX },
-        humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true));
+        ...aggregateFields(stats.genTime, { transform: time => humanizeTimespan(time, { short: true }) }));
-    html += createStatBlock('Generated Tokens', 2355, 43, 180, 2400);
+    html += createStatBlock({ value: 'Generated Tokens', info: `Generated tokens per chat\nSummed token counts of all messages and swipes.\n${GEN_TOKEN_WARNING}` + HOVER_TOOLTIP_SUFFIX },
+        ...aggregateFields(stats.genTokenCount));
     html += HMTL_STAT_SPACER;
-    html += createStatBlock('Swiping Time', humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true));
+    html += createStatBlock({ value: 'Swiping Time', info: 'Swiping time per chat\nSummed time spend on generation alternative swipes. Excludes the final message that was chosen to continue the chat.' + HOVER_TOOLTIP_SUFFIX },
-    html += createStatBlock({ value: 'Swipes', info: 'Total swipes over all chats, and min/avg/max swipes per chat' },
+        ...aggregateFields(stats.swipeGenTime, { transform: time => humanizeTimespan(time, { short: true }) }));
-        { value: 256 }, { value: 1 }, { value: 4 }, { value: 25 });
+    html += createStatBlock({ value: 'Swipes', info: 'Swipes per chat\nCounts all generated messages/swipes that were not chosen to continue the chat.' + HOVER_TOOLTIP_SUFFIX },
+        ...aggregateFields(stats.swipes));
     html += HMTL_STAT_SPACER;
-    html += createStatBlock('User Response Time', humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true));
+    html += createStatBlock({ value: 'User Response Time', info: 'User response time per chat\nCalculated based on the time between the last action of the message before and the next user message.\nAs \'action\' counts both the message send time and when the last generation of it ended, even if that swipe wasn\'t chosen.' + HOVER_TOOLTIP_SUFFIX },
+        ...aggregateFields(stats.userResponseTime, { transform: time => humanizeTimespan(time, { short: true }) }));
     html += HMTL_STAT_SPACER;
     html += createStatBlock({ value: 'Messages', info: 'Total messages over all chats (excluding swipes), and min/avg/max messages per chat' },
-        512, 2, 12, 100);
+        ...aggregateFields(stats.messages));
-    html += createStatBlock('System Messages', 47, 0, 4, 85);
+    html += createStatBlock('System Messages', ...aggregateFields(stats.systemMessages));
-    html += createStatBlock({ value: 'Messages (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(145, 359, 2, 27, 8, 54, 66, 100));
+    html += createStatBlock({ value: 'Messages (User / Char)', classes: ['rm_stat_field_smaller'] }, ...buildBarDescsFromAggregates(stats.userMessages, stats.charMessages));
     html += createStatBlock({ value: '', info: '' },
-        buildBar(145, 359), buildBar(2, 27), buildBar(8, 54), buildBar(66, 100));
+        ...buildBarsFromAggregates(stats.userMessages, stats.charMessages));
     html += HMTL_STAT_SPACER;
     html += createStatBlock({ value: 'Words', info: 'Total words over all chats, and min/avg/max words per chat' },
-        { value: 5124 }, { value: 26 }, { value: 122 }, { value: 1008 });
+        ...aggregateFields(stats.words));
-    html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(1451, 3594, 22, 279, 84, 625, 762, 2505));
+    html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, ...buildBarDescsFromAggregates(stats.userWords, stats.charWords));
     html += createStatBlock({ value: '', info: '' },
-        buildBar(1451, 3594), buildBar(22, 279), buildBar(84, 625), buildBar(762, 2505));
+        ...buildBarsFromAggregates(stats.userWords, stats.charWords));
 
     html += HMTL_STAT_SPACER;
     html += HMTL_STAT_SPACER;
@@ -261,78 +203,147 @@ function createHtml(statsType, characterId, stats) {
         { value: 'Max', isHeader: true }
     );
     html += createStatBlock({ value: 'Generation Time', info: 'min/avg/max generation time per message' },
-        null, { value: humanizeGenTime(4566, true) }, { value: humanizeGenTime(23523, true) }, { value: humanizeGenTime(286230, true) });
+        ...aggregateFields(stats.perMessageGenTime, { basedOn: 'message', excludeTotal: true, transform: time => humanizeTimespan(time, { short: true }) }));
-    html += createStatBlock('Generated Tokens', null, 43, 180, 2400);
+    html += createStatBlock('Generated Tokens', ...aggregateFields(stats.perMessageGenTokenCount, { basedOn: 'message', excludeTotal: true }));
     html += HMTL_STAT_SPACER;
-    html += createStatBlock('Swiping Time', null, humanizeGenTime(1456, true), humanizeGenTime(2523, true), humanizeGenTime(28230, true));
+    html += createStatBlock('Swiping Time', ...aggregateFields(stats.perMessageSwipeGenTime, { basedOn: 'message', excludeTotal: true, transform: time => humanizeTimespan(time, { short: true }) }));
     html += createStatBlock({ value: 'Swipes', info: 'min/avg/max swipes per <b>non-user</b> message' },
-        null, { value: 1 }, { value: 4 }, { value: 25 });
+        ...aggregateFields(stats.perMessageSwipeCount, { basedOn: 'message', excludeTotal: true }));
     html += HMTL_STAT_SPACER;
-    html += createStatBlock('User Response Time', null, humanizeGenTime(0, true), humanizeGenTime(233, true), humanizeGenTime(13630, true));
+    html += createStatBlock('User Response Time', ...aggregateFields(stats.perMessageUserResponseTime, { basedOn: 'message', excludeTotal: true, transform: time => humanizeTimespan(time, { short: true }) }));
     html += HMTL_STAT_SPACER;
     html += createStatBlock({ value: 'Words', info: 'min/avg/max words per message' },
-        null, { value: 4 }, { value: 145 }, { value: 431 });
+        ...aggregateFields(stats.perMessageWords, { basedOn: 'message', excludeTotal: true }));
-    html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(null, null, 22, 279, 84, 625, 762, 2505));
+    html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, ...buildBarDescsFromAggregates(stats.perMessageUserWords, stats.perMessageCharWords, { basedOn: 'message', excludeTotal: true }));
     html += createStatBlock({ value: '', info: '' },
-        null, buildBar(22, 279), buildBar(84, 625), buildBar(762, 2505));
+        ...buildBarsFromAggregates(stats.perMessageUserWords, stats.perMessageCharWords, { basedOn: 'message', excludeTotal: true }));
 
     html += HMTL_STAT_SPACER;
     html += HMTL_STAT_SPACER;
 
     // Hijack avatar list function to draw the user avatar
-    if (characters[characterId]) {
+    if (character) {
         const placeHolder = $('<div class="rm_stat_avatar_block"></div>');
-        const entity = characterToEntity(characters[characterId], characterId);
+        const cid = characters.indexOf(x => x === character);
+        const entity = characterToEntity(character, cid);
         buildAvatarList(placeHolder, [entity]);
         html = placeHolder.prop('outerHTML') + html;
     }
 
     callPopup(html, 'text', '', { wider: true, allowVerticalScrolling: true });
+
+    return;
+
+    /** @param {AggregateStat} agg1 @param {AggregateStat} agg2 @param {AggBuildOptions} options @returns {StatField[]}  */
+    function buildBarsFromAggregates(agg1, agg2, options = DEFAULT_AGG_BUILD_OPTIONS) {
+        options = { ...DEFAULT_AGG_BUILD_OPTIONS, ...options };
+        const f1 = aggregateFields(agg1, options);
+        const f2 = aggregateFields(agg2, options);
+        const bars = f1.map((_, i) => buildBar(f1[i]?.value, f2[i]?.value));
+        return bars.map(statField);
+    }
+    /** @param {number} charVal @param {number} userVal @returns {string}  */
+    function buildBar(userVal, charVal) {
+        const percentUser = (userVal / (userVal + charVal)) * 100;
+        const percentChar = 100 - percentUser;
+        return `<div class="rm_stat_bar">
+            <div style="width: ${percentUser}%" title="${stats.userName}: ${userVal}   (${percentUser.toFixed(1)}%)" class="rm_stat_bar_user"></div>
+            <div style="width: ${percentChar}%" title="${stats.charName}: ${charVal}   (${percentChar.toFixed(1)}%)" class="rm_stat_bar_char"></div>
+        </div>`;
+    }
+    /** @param {AggregateStat} agg1 @param {AggregateStat} agg2 @param {AggBuildOptions} options @returns {StatField[]}  */
+    function buildBarDescsFromAggregates(agg1, agg2, options = DEFAULT_AGG_BUILD_OPTIONS) {
+        options = { ...DEFAULT_AGG_BUILD_OPTIONS, ...options };
+        const f1 = aggregateFields(agg1, options);
+        const f2 = aggregateFields(agg2, options);
+        const values = [f1[0], f2[0], f1[1], f2[1], f1[2], f2[2], f1[3], f2[3]];
+        return buildBarDescs(values);
+    }
+    /** @param {any[]} values @returns {StatField[]}  */
+    function buildBarDescs(...values) {
+        return values.flat(Infinity).map(statField).map((x, i) => i % 2 == 0 ? { classes: [...(x.classes ?? []), 'rm_stat_field_lefty'], ...x } : x);
+    }
+}
+
+/**
+ * Finds the model with the highest count and returns its name and values
+ *
+ * @param {{[model: string]: { count: number, tokens: number }}} genModels - Object containing model usages
+ * @returns {{ model: string, count: number, tokens: number }} - Object containing the name and values of the model with the highest count
+ */
+function findHighestModel(genModels) {
+    return Object.entries(genModels).reduce((acc, [model, values]) => {
+        return values.count > acc.count ? { model: model, count: values.count, tokens: values.tokens } : acc;
+    }, { model: '<None>', count: 0, tokens: 0 });
 }
 
 /**
  * Handles the user stats by getting them from the server, calculating the total and generating the HTML report.
  */
-async function userStatsHandler() {
+async function showUserStatsPopup() {
     // Get stats from server
-    await getStats();
+    const globalStats = await getGlobalStats();
-
-    // Calculate total stats
-    let totalStats = calculateTotalStats();
 
     // Create HTML with stats
-    createHtml('User', null, totalStats);
+    createHtml('User', globalStats);
 }
 
 /**
  * Handles the character stats by getting them from the server and generating the HTML report.
  *
- * @param {{[characterKey: string]: Character}} characters - Object containing character data.
+ * @param {string} characterKey - The character key.
- * @param {string} this_chid - The character id.
  */
-async function characterStatsHandler(characters, this_chid) {
+async function showCharacterStatsPopup(characterKey) {
     // Get stats from server
-    await getStats();
+    const charStats = await getCharStats(characterKey);
-    // Get character stats
+    if (charStats === null) {
-    let myStats = charStats.stats[characters[this_chid].avatar];
+        toastr.info('No stats exist for the the current character.');
-    if (myStats === undefined) {
+        return;
-        myStats = createEmptyStats();
-        myStats.non_user_word_count = countWords(characters[this_chid].first_mes);
-        charStats.stats[characters[this_chid].avatar] = myStats;
-        updateStats();
     }
+
     // Create HTML with stats
-    createHtml('Character', this_chid, myStats);
+    createHtml('Character', charStats);
+}
+
+
+/**
+ *
+ * @param {string} characterKey - The key of the character
+ * @returns {Promise<CharacterStats?>}
+ */
+async function getCharStats(characterKey) {
+    const stats = await callGetStats({ characterKey: characterKey });
+    return stats;
+}
+
+/**
+ *
+ * @returns {Promise<CharacterStats>}
+ */
+async function getGlobalStats() {
+    const stats = await callGetStats({ global: true });
+    return stats;
+}
+/**
+ *
+ * @returns {Promise<UserStatsCollection?>}
+ */
+async function getFullStatsCollection() {
+    const stats = await callGetStats();
+    return stats;
 }
 
 /**
  * Fetches the character stats from the server.
+ * For retrieving, use the more specific functions `getCharStats`, `getGlobalStats` and `getFullStatsCollection`.
+ * @param {StatsRequestBody} [params={}] Optional parameter for the get request
+ * @returns {Promise<object?>} character stats the full stats collection, depending on what was requested
  */
-async function getStats() {
+async function callGetStats(params = {}) {
     const response = await fetch('/api/stats/get', {
         method: 'POST',
         headers: getRequestHeaders(),
-        body: JSON.stringify({}),
+        body: JSON.stringify(params),
         cache: 'no-cache',
     });
 
@@ -340,7 +351,11 @@ async function getStats() {
         toastr.error('Stats could not be loaded. Try reloading the page.');
         throw new Error('Error getting stats');
     }
-    charStats = await response.json();
+
+    // To use the custom JSON parser, we need to retrieve the body as text first
+    const bodyText = await response.text();
+    const stats = parseJson(bodyText);
+    return stats;
 }
 
 /**
@@ -368,112 +383,15 @@ async function recreateStats() {
     }
 }
 
-
+function initStats() {
-/**
+    $('.rm_stats_button').on('click', async function () {
- * Calculates the generation time based on start and finish times.
+        await showCharacterStatsPopup(characters[this_chid].avatar);
- *
- * @param {string} gen_started - The start time in ISO 8601 format.
- * @param {string} gen_finished - The finish time in ISO 8601 format.
- * @returns {number} - The difference in time in milliseconds.
- */
-function calculateGenTime(gen_started, gen_finished) {
-    if (gen_started === undefined || gen_finished === undefined) {
-        return 0;
-    }
-    let startDate = new Date(gen_started);
-    let endDate = new Date(gen_finished);
-    return endDate.getTime() - startDate.getTime();
-}
-
-/**
- * Sends a POST request to the server to update the statistics.
- */
-async function updateStats() {
-    const response = await fetch('/api/stats/update', {
-        method: 'POST',
-        headers: getRequestHeaders(),
-        body: JSON.stringify(charStats),
     });
-
+    $('.user_stats_button').on('click', async function () {
-    if (response.status !== 200) {
+        await showUserStatsPopup();
-        console.error('Failed to update stats');
-        console.log(response.status);
-    }
-}
-
-/**
- * Returns the count of words in the given string.
- * A word is a sequence of alphanumeric characters (including underscore).
- *
- * @param {string} str - The string to count words in.
- * @returns {number} - Number of words.
- */
-function countWords(str) {
-    const match = str.match(/\b\w+\b/g);
-    return match ? match.length : 0;
-}
-
-/**
- * Handles stat processing for messages.
- *
- * @param {Object} line - Object containing message data.
- * @param {string} type - The type of the message processing (e.g., 'append', 'continue', 'appendFinal', 'swipe').
- * @param {Character[]} characters - Object containing character data.
- * @param {string} this_chid - The character id.
- * @param {string} oldMesssage - The old message that's being processed.
- */
-async function statMesProcess(line, type, characters, this_chid, oldMesssage) {
-    if (this_chid === undefined || characters[this_chid] === undefined) {
-        return;
-    }
-    await getStats();
-
-    let stats = charStats.stats[characters[this_chid].avatar] ?? createEmptyStats();
-
-    stats.total_gen_time += calculateGenTime(line.gen_started, line.gen_finished);
-    if (line.is_user) {
-        if (type != 'append' && type != 'continue' && type != 'appendFinal') {
-            stats.user_msg_count++;
-            stats.user_word_count += countWords(line.mes);
-        } else {
-            let oldLen = oldMesssage.split(' ').length;
-            stats.user_word_count += countWords(line.mes) - oldLen;
-        }
-    } else {
-        // if continue, don't add a message, get the last message and subtract it from the word count of
-        // the new message
-        if (type != 'append' && type != 'continue' && type != 'appendFinal') {
-            stats.non_user_msg_count++;
-            stats.non_user_word_count += countWords(line.mes);
-        } else {
-            let oldLen = oldMesssage.split(' ').length;
-            stats.non_user_word_count += countWords(line.mes) - oldLen;
-        }
-    }
-
-    if (type === 'swipe') {
-        stats.total_swipe_count++;
-    }
-
-    // If this is the first user message, set the first chat time
-    if (line.is_user) {
-        //get min between firstChatTime and timestampToMoment(json.send_date)
-        stats.date_first_chat = Math.min(timestampToMoment(line.send_date) ?? MAX_TIMESTAMP, stats.date_first_chat);
-    }
-
-    // For last chat time, we skip the original first message and then take all user and AI messages
-    if ((stats.user_msg_count + stats.non_user_msg_count) > 1) {
-        stats.date_last_chat = Math.max(timestampToMoment(line.send_date) ?? MIN_TIMESTAMP, stats.date_last_chat);
-    }
-    updateStats();
-}
-
-export function initStats() {
-    $('.rm_stats_button').on('click', function () {
-        characterStatsHandler(characters, this_chid);
     });
     // Wait for debug functions to load, then add the refresh stats function
     registerDebugFunction('refreshStats', 'Refresh Stat File', 'Recreates the stats file based on existing chat files', recreateStats);
 }
 
-export { userStatsHandler, characterStatsHandler, getStats, statMesProcess, charStats };
+export { initStats, showUserStatsPopup, showCharacterStatsPopup, callGetStats };

public/scripts/utils.js

@@ -214,6 +214,49 @@ export function getBase64Async(file) {
     });
 }
 
+
+
+/**
+ * Parse JSON data with optional reviver function.
+ * Converts date strings back to Date objects if found.
+ *
+ * @param {string} json - The JSON data to parse
+ * @param {object} [options] - Optional parameters
+ * @param {Reviver?} [options.reviver=null] - Custom reviver function to customize parsing
+ * @param {boolean} [options.disableDefaultReviver=false] - Flag to disable the default date parsing reviver
+ * @returns {object} - The parsed JSON object
+ */
+export function parseJson(json, { reviver = null, disableDefaultReviver = false } = {}) {
+    /**
+     * @typedef {((this: any, key: string, value: any) => any)} Reviver
+     * @param {object} this -
+     * @param {string} key - The key of the current property being processed
+     * @param {*} value - The value of the current property being processed
+     * @returns {*} - The processed value
+     */
+
+    /** @type {Reviver} The default reviver, that converts Date strings to Date objects */
+    function defaultReviver(key, value) {
+        // Check if the value is a string and can be converted to a Date
+        if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/.test(value)) {
+            return new Date(value);
+        }
+        // Return the original value if it's not a date string
+        return value;
+    }
+
+    // The actual reviver based on the ones given
+    /** @type {Reviver} */
+    function actualReviver(key, value) {
+        if (reviver) value = reviver(key, value);
+        if (!disableDefaultReviver) value = defaultReviver(key, value);
+        return value;
+    };
+
+    // Parse the JSON data using the specified or custom reviver function
+    return JSON.parse(json, actualReviver);
+}
+
 /**
  * Parses a file blob as a JSON object.
  * @param {Blob} file The file to read.
@@ -524,24 +567,44 @@ export function trimToStartSentence(input) {
 }
 
 /**
- * Format bytes as human-readable text.
+ * Build a humanized string for a duration
- *
+ * @param {Date|number} start - Start time (as a Date, or in milliseconds)
- * @param bytes Number of bytes.
+ * @param {Date|number|null} end - End time (as a Date, or in milliseconds), if null will be replaced with Date.now()
- * @param si True to use metric (SI) units, aka powers of 1000. False to use
+ * @param {object} param2 - Optional parameters
- *           binary (IEC), aka powers of 1024.
+ * @param {string} [param2.fallback='Never'] - Fallback value no duration can be calculated
- * @param dp Number of decimal places to display.
+ * @param {function(string): string} [param2.wrapper=null] - Optional function to wrap/format the resulting humanized duration
- *
+ * @returns {string} Humanized duration string
- * @return Formatted string.
  */
-export function humanFileSize(bytes, si = false, dp = 1) {
+export function humanizedDuration(start, end = null, { fallback = 'Never', wrapper = null } = {}) {
+    const startTime = start instanceof Date ? start.getTime() : start;
+    const endTime = end instanceof Date ? end.getTime() : end ?? Date.now();
+    if (!startTime || endTime > endTime) {
+        return fallback;
+    }
+    // @ts-ignore
+    const humanized = moment.duration(endTime - start).humanize();
+    return wrapper ? wrapper(humanized) : humanized;
+}
+
+/**
+ * Format bytes as human-readable text
+ *
+ * @param bytes Number of bytes
+ * @param si True to use metric (SI) units, aka powers of 1000. False to use binary (IEC), aka powers of 1024
+ * @param ibi If `si` is disabled, setting this to True will return unit names as 'KiB', 'MiB' etc. instead of 'KB', 'MB'.
+ * @param dp Number of decimal places to display
+ *
+ * @return Formatted string
+ */
+export function humanFileSize(bytes, si = false, ibi = false, dp = 1) {
     const thresh = si ? 1000 : 1024;
 
     if (Math.abs(bytes) < thresh) {
         return bytes + ' B';
     }
 
-    const units = si
+    const units = si || !ibi
-        ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+        ? ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
         : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
     let u = -1;
     const r = 10 ** dp;
@@ -1419,3 +1482,72 @@ export function setValueByPath(obj, path, value) {
 
     currentObject[keyParts[keyParts.length - 1]] = value;
 }
+
+/**
+ * Rounds a number conditionally in a sensible way, based on thresholds
+ *
+ * @param {number} number - The number to round
+ * @param {object} [options={}] - Optional parameters
+ * @param {{[threshold: number]: number, _: number}} [options.thresholds={ 1: 3, 100: 2, _: 0 }] - Custom rounding thresholds, specified by 'threshold value: rounding decimals'. The default value will be provided with the key '_'.
+ * @returns {number} - The rounded number
+ */
+export function sensibleRound(number, { thresholds = { 1: 3, 100: 2, _: 0 } } = {}) {
+    // Sort thresholds by ascending order of keys
+    const sortedThresholds = Object.keys(thresholds).map(parseFloat).sort((a, b) => a - b);
+
+    // Find the appropriate threshold for rounding
+    let decimalPlaces = thresholds._ ?? 0;
+    for (const threshold of sortedThresholds) {
+        if (number < threshold) {
+            decimalPlaces = thresholds[threshold];
+            break;
+        }
+    }
+
+    return +number.toFixed(decimalPlaces);
+}
+
+/**
+ * Truncates a given text at the end of word boundaries, filling in an ellipsis. The max total length will not exceed the provided value.
+ * If the word boundaries will make this string too short, it'll make a hard truncate instead to preserve contextual information.
+ * @param {string} text - The text to truncate
+ * @param {number} maxLength - max length to truncate to
+ * @param {string} [ellipsis='…'] - Ellipsis to add if the string was truncated
+ * @param {number} [minLength=maxLength/2] - A minimum length below which the word boundary trunaction should not go to. If hit, hard truncation at max length will be used
+ * @returns {string}
+ */
+export function smartTruncate(text, maxLength, ellipsis = '…', minLength = maxLength / 2) {
+    if (text.length <= maxLength) {
+        return text;
+    }
+    if (ellipsis.length > maxLength) {
+        console.warn(`Cannot truncate to length of ${maxLength}, below the length of the ellipsis '${ellipsis}'.`);
+        return text;
+    }
+
+    const isWord = (char) => /^\w$/.test(char);
+    const maxTruncLength = maxLength - ellipsis.length;
+
+    // If the first character beyond the trunc length is non-word, we can cut at the next word char.
+    let cutSoon = !isWord(text[maxTruncLength + 1]);
+
+    let index = maxTruncLength + 1;
+    while (index > 0) {
+        const char = text[index];
+
+        // Look for a non-word character to prepare for a cut
+        if (!isWord(char)) {
+            cutSoon = true;
+        } else if (cutSoon) {
+            break;
+        }
+        index--;
+    }
+
+    // If the cut is too close to start, or no appropriate cut point is found, fall back to hard truncation
+    if (index < minLength) {
+        return text.slice(0, maxLength - ellipsis.length) + ellipsis;
+    }
+
+    return text.slice(0, index + 1) + ellipsis;
+}

src/endpoints/stats.js

@@ -6,7 +6,7 @@ const crypto = require('crypto');
 const sanitize = require('sanitize-filename');
 
 const { jsonParser } = require('../express-common');
-const { readAndParseJsonlFile, timestampToMoment, humanizedToDate, calculateDuration, minDate, maxDate, now } = require('../util');
+const { readAndParseJsonlFile, parseJson, timestampToMoment, humanizedToDate, calculateDuration, minDate, maxDate, now } = require('../util');
 const { getAllUserHandles, getUserDirectories } = require('../users');
 
 const readFile = fs.promises.readFile;
@@ -17,7 +17,7 @@ const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime();
 const MIN_DATE = new Date(MIN_TIMESTAMP);
 const MAX_DATE = new Date(MAX_TIMESTAMP);
 const STATS_FILE = 'stats.json';
-const CURRENT_STATS_VERSION = '1.1';
+const CURRENT_STATS_VERSION = '1.2';
 
 /** @type {Map<string, UserStatsCollection>} The stats collections for each user, accessable via their key - gets set/built on init */
 const STATS = new Map();
@@ -44,7 +44,7 @@ async function init() {
             const directories = getUserDirectories(userHandle);
             const statsFilePath = path.join(directories.root, STATS_FILE);
             const statsFileContent = await readFile(statsFilePath, 'utf-8');
-            let userStats = JSON.parse(statsFileContent);
+            let userStats = parseJson(statsFileContent);
 
             // Migrate/recreate stats if the version has changed
             if (userStats.version !== CURRENT_STATS_VERSION) {
@@ -92,7 +92,8 @@ async function onExit() {
 
 /**
  * @typedef {object} CharacterStats
- * @property {string} name -
+ * @property {string} charName -
+ * @property {string} userName -
  * @property {string} characterKey -
  * @property {number} chats - The creation date of the chat.
  * @property {number} chatSize - The size of all chats
@@ -135,6 +136,8 @@ async function onExit() {
  * @typedef {object} ChatStats
  * @property {string} chatName - The unique identifier for the chat.
  * @property {number} chatId - hash
+ * @property {string} charName - Current character name
+ * @property {string} userName - Current user name
  * @property {number} chatSize -
  * @property {Date} createDate - The creation date of the chat. (time in ISO 8601 format)
  * @property {Date} lastInteractionDate - (time in ISO 8601 format)
@@ -193,10 +196,14 @@ class AggregateStat {
     avg = 0;
     /** @type {number[]} All values listed and saved, so the aggregate stats can be updated if needed when elements get removed */
     values = [];
+    /** @type {number?} The number of stats used when this is aggregated over the totals of aggregated stats, meaning based on any amount of sub/inner values */
+    subCount = null;
+
     constructor() { }
 
     reset() {
-        this.count, this.total, this.min, this.max, this.avg = 0;
+        this.count, this.total, this.min, this.max, this.avg = 0, this.subCount = 0;
+        this.values.length = 0;
     }
 
     /**
@@ -215,6 +222,15 @@ class AggregateStat {
         this.max = Math.max(this.max, value);
     }
 
+    /**
+     * Adds the total of the aggregated value as a single value, and also marks the count as sub values for analysis purposes
+     * @param {AggregateStat} aggregatedValue - The aggregate stat
+     */
+    addAggregatedAsOne(aggregatedValue) {
+        this.add(aggregatedValue.total);
+        this.subCount = (this.subCount ?? 0) + aggregatedValue.count;
+    }
+
     /**
      * Adds all values of a given aggregation as single values
      * @param {AggregateStat} aggregatedValue - The aggregate stat
@@ -257,6 +273,15 @@ class AggregateStat {
     removeAggregated(aggregatedValue) {
         aggregatedValue.values.forEach(x => this.add(x));
     }
+
+    /**
+     * Removes the total of the aggregated value as a single value, and also marks the count as sub values for analysis purposes
+     * @param {AggregateStat} aggregatedValue - The aggregate stat
+     */
+    removeAggregatedAsOne(aggregatedValue) {
+        this.remove(aggregatedValue.total);
+        this.subCount = this.subCount ? this.subCount - aggregatedValue.count : null;
+    }
 }
 
 /**
@@ -288,7 +313,7 @@ async function recreateStats(userHandle) {
     const files = await readdir(directories.characters);
     const charFiles = files.filter((file) => file.endsWith('.png'));
     let processingPromises = charFiles.map((charFileName, _) =>
-        recreateCharacterStats(userHandle, charFileName.replace('.png', '')),
+        recreateCharacterStats(userHandle, charFileName)
     );
     await Promise.all(processingPromises);
 
@@ -321,8 +346,7 @@ function recreateCharacterStats(userHandle, characterKey) {
     }
 
     // Then load chats dir for this character to process
-    const directories = getUserDirectories(userHandle);
+    const charChatsDir = getCharChatsDir(userHandle, characterKey);
-    const charChatsDir = path.join(directories.chats, characterKey);
     if (!fs.existsSync(charChatsDir)) {
         return null;
     }
@@ -336,24 +360,35 @@ function recreateCharacterStats(userHandle, characterKey) {
     return userStats[characterKey];
 };
 
+/**
+ *
+ * @param {string} userHandle - The user handle
+ * @param {string} characterKey - The character key
+ * @returns {string} The chats directory for this specific char
+ */
+function getCharChatsDir(userHandle, characterKey) {
+    const charName = characterKey.replace('.png', '');
+    const directories = getUserDirectories(userHandle);
+    const charChatsDir = path.join(directories.chats, charName);
+    return charChatsDir;
+}
 
 /**
  *
  * @param {string} userHandle - The user handle
  * @param {string} characterKey
  * @param {string} chatName
- * @returns {{chatName: string, charName: string, filePath: string, lines: object[]}}
+ * @returns {{chatName: string, filePath: string, lines: object[]}}
  */
 function loadChatFile(userHandle, characterKey, chatName) {
-    const charName = characterKey.replace('.png', '');
+    const charChatsDir = getCharChatsDir(userHandle, characterKey);
-    const directories = getUserDirectories(userHandle);
-    const charChatsDir = path.join(directories.chats, charName);
 
     const filePath = path.join(charChatsDir, sanitize(chatName));
     const lines = readAndParseJsonlFile(filePath);
-    return { chatName, charName, filePath, lines };
+    return { chatName, filePath, lines };
 }
 
+
 /**
  *
  * @param {string} userHandle - The user handle
@@ -374,14 +409,16 @@ function triggerChatUpdate(userHandle, characterKey, chatName) {
     const userStats = getUserStats(userHandle);
 
     // Create empty stats if character stats don't exist yet
-    userStats.stats[characterKey] ??= newCharacterStats(characterKey, loadedChat.charName);
+    userStats.stats[characterKey] ??= newCharacterStats(characterKey);
 
     // Update both the char stats and the global user stats with this chat
     updateCharStatsWithChat(userStats.stats[characterKey], chatStats);
     updateCharStatsWithChat(userStats.global, chatStats);
 
-    chatStats._calculated = now();
+    // Update name (if it might have changed)
-    userStats.global._calculated = now()
+    userStats.stats[characterKey].charName = chatStats.charName;
+    userStats.stats[characterKey].userName = chatStats.userName;
+
     userStats._calculated = now();
     return chatStats;
 }
@@ -413,14 +450,14 @@ function updateCharStatsWithChat(stats, chatStats) {
     stats.userMessages.add(chatStats.userMessages);
     stats.charMessages.add(chatStats.charMessages);
 
-    stats.genTime.add(chatStats.genTime.total);
+    stats.genTime.addAggregatedAsOne(chatStats.genTime);
-    stats.genTokenCount.add(chatStats.genTokenCount.total);
+    stats.genTokenCount.addAggregatedAsOne(chatStats.genTokenCount);
-    stats.swipeGenTime.add(chatStats.swipeGenTime.total);
+    stats.swipeGenTime.addAggregatedAsOne(chatStats.swipeGenTime);
-    stats.swipes.add(chatStats.swipes.total);
+    stats.swipes.addAggregatedAsOne(chatStats.swipes);
-    stats.userResponseTime.add(chatStats.userResponseTime.total);
+    stats.userResponseTime.addAggregatedAsOne(chatStats.userResponseTime);
-    stats.words.add(chatStats.words.total);
+    stats.words.addAggregatedAsOne(chatStats.words);
-    stats.userWords.add(chatStats.userWords.total);
+    stats.userWords.addAggregatedAsOne(chatStats.userWords);
-    stats.charWords.add(chatStats.charWords.total);
+    stats.charWords.addAggregatedAsOne(chatStats.charWords);
 
     stats.perMessageGenTime.addAggregated(chatStats.genTime);
     stats.perMessageGenTokenCount.addAggregated(chatStats.genTokenCount);
@@ -434,7 +471,7 @@ function updateCharStatsWithChat(stats, chatStats) {
     Object.entries(chatStats.genModels).forEach(([model, data]) => addModelUsage(stats.genModels, model, data.tokens, data.count));
 
     stats._calculated = now();
-    console.debug(`Successfully updated ${stats.name}'s stats with chat ${chatStats.chatName}`);
+    console.debug(`Successfully updated ${stats.charName}'s stats with chat ${chatStats.chatName}`);
     return true;
 }
 
@@ -465,14 +502,14 @@ function removeChatFromCharStats(stats, chatStats) {
     stats.userMessages.remove(chatStats.userMessages);
     stats.charMessages.remove(chatStats.charMessages);
 
-    stats.genTime.remove(chatStats.genTime.total);
+    stats.genTime.removeAggregatedAsOne(chatStats.genTime);
-    stats.genTokenCount.remove(chatStats.genTokenCount.total);
+    stats.genTokenCount.removeAggregatedAsOne(chatStats.genTokenCount);
-    stats.swipeGenTime.remove(chatStats.swipeGenTime.total);
+    stats.swipeGenTime.removeAggregatedAsOne(chatStats.swipeGenTime);
-    stats.swipes.remove(chatStats.swipes.total);
+    stats.swipes.removeAggregatedAsOne(chatStats.swipes);
-    stats.userResponseTime.remove(chatStats.userResponseTime.total);
+    stats.userResponseTime.removeAggregatedAsOne(chatStats.userResponseTime);
-    stats.words.remove(chatStats.words.total);
+    stats.words.removeAggregatedAsOne(chatStats.words);
-    stats.userWords.remove(chatStats.userWords.total);
+    stats.userWords.removeAggregatedAsOne(chatStats.userWords);
-    stats.charWords.remove(chatStats.charWords.total);
+    stats.charWords.removeAggregatedAsOne(chatStats.charWords);
 
     stats.perMessageGenTime.removeAggregated(chatStats.genTime);
     stats.perMessageGenTokenCount.removeAggregated(chatStats.genTokenCount);
@@ -516,6 +553,7 @@ function processChat(chatName, lines, { chatSize = 0 } = {}) {
         // Check if this is the first message, the "data storage"
         if (message.chat_metadata && message.create_date) {
             stats.createDate = humanizedToDate(message.create_date) ?? stats.createDate;
+            stats.lastInteractionDate = stats.createDate;
             stats.chatId = message.chat_metadata['chat_id_hash'];
             continue;
         }
@@ -523,6 +561,10 @@ function processChat(chatName, lines, { chatSize = 0 } = {}) {
         const messageStats = processMessage(message);
         stats.messagesStats.push(messageStats);
 
+        // Update names to the latest message
+        stats.charName = messageStats.isChar ? message.name : stats.charName;
+        stats.userName = messageStats.isUser ? message.name : stats.userName;
+
         stats.lastInteractionDate = maxDate(stats.lastInteractionDate, messageStats.sendDate, ...messageStats.genEndDates) ?? stats.lastInteractionDate;
 
         // Aggregate chat stats for each message
@@ -557,6 +599,7 @@ function processChat(chatName, lines, { chatSize = 0 } = {}) {
     // Set up the final values for chat
     stats.chattingTime = calculateDuration(stats.createDate, stats.lastInteractionDate);
 
+    stats._calculated = now();
     return stats;
 }
 
@@ -641,14 +684,15 @@ function countWordsInString(str) {
  */
 function newCharacterStats(characterKey = '', charName = '') {
     return {
-        name: charName,
+        charName: charName,
+        userName: 'User',
         characterKey: characterKey,
         chats: 0,
         chatSize: 0,
 
-        firstCreateDate: MIN_DATE,
+        firstCreateDate: MAX_DATE,
         lastCreateDate: MIN_DATE,
-        firstlastInteractionDate: MIN_DATE,
+        firstlastInteractionDate: MAX_DATE,
         lastLastInteractionDate: MIN_DATE,
 
         chattingTime: new AggregateStat(),
@@ -690,8 +734,10 @@ function newChatStats(chatName) {
     return {
         chatName: chatName,
         chatId: 0,
+        charName: '',
+        userName: '',
         chatSize: 0,
-        createDate: MIN_DATE,
+        createDate: MAX_DATE,
         lastInteractionDate: MIN_DATE,
 
         chattingTime: 0,
@@ -777,7 +823,7 @@ const router = express.Router();
  * @returns {void}
  */
 router.post('/get', jsonParser, function (request, response) {
-    const send = (data) => response.send(JSON.stringify(data ?? {}));
+    const send = (data) => response.send(JSON.stringify(data ?? null));
     /** @type {StatsRequestBody} */
     const body = request.body;
 
@@ -788,13 +834,11 @@ router.post('/get', jsonParser, function (request, response) {
         return send(userStats.global);
     }
 
-    const characterKey = String(body.characterKey);
+    if (body.characterKey && body.chatName) {
-    const chatName = String(body.characterKey);
+        return send(userStats.stats[body.characterKey]?.chatsStats.find(x => x.chatName == body.chatName));
-    if (characterKey && chatName) {
-        return send(userStats.stats[characterKey]?.chatsStats.find(x => x.chatName == chatName));
     }
-    if (characterKey) {
+    if (body.characterKey) {
-        return send(userStats.stats[characterKey]);
+        return send(userStats.stats[body.characterKey]);
     }
 
     // If no specific filter was requested, we send all stats back

src/util.js

@@ -241,6 +241,47 @@ function readAndParseJsonlFile(filepath) {
     }
 }
 
+/**
+ * Parse JSON data with optional reviver function.
+ * Converts date strings back to Date objects if found.
+ *
+ * @param {string} json - The JSON data to parse
+ * @param {object} [options] - Optional parameters
+ * @param {Reviver?} [options.reviver=null] - Custom reviver function to customize parsing
+ * @param {boolean} [options.disableDefaultReviver=false] - Flag to disable the default date parsing reviver
+ * @returns {object} - The parsed JSON object
+ */
+function parseJson(json, { reviver = null, disableDefaultReviver = false } = {}) {
+    /**
+     * @typedef {((this: any, key: string, value: any) => any)} Reviver
+     * @param {object} this -
+     * @param {string} key - The key of the current property being processed
+     * @param {*} value - The value of the current property being processed
+     * @returns {*} - The processed value
+     */
+
+    /** @type {Reviver} The default reviver, that converts Date strings to Date objects */
+    function defaultReviver(key, value) {
+        // Check if the value is a string and can be converted to a Date
+        if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/.test(value)) {
+            return new Date(value);
+        }
+        // Return the original value if it's not a date string
+        return value;
+    }
+
+    // The actual reviver based on the ones given
+    /** @type {Reviver} */
+    function actualReviver(key, value) {
+        if (reviver) value = reviver(key, value);
+        if (!disableDefaultReviver) value = defaultReviver(key, value);
+        return value;
+    };
+
+    // Parse the JSON data using the specified or custom reviver function
+    return JSON.parse(json, actualReviver);
+}
+
 /**
  * Gets all chunks of data from the given readable stream.
  * @param {any} readableStream Readable stream to read from
@@ -776,6 +817,7 @@ module.exports = {
     extractFileFromZipBuffer,
     getImageBuffers,
     readAndParseJsonlFile,
+    parseJson,
     readAllChunks,
     delay,
     deepMerge,