From 9cef0d8346a8b44a1d069ae04234f671aed1f4ce Mon Sep 17 00:00:00 2001 From: Wolfsblvt Date: Thu, 11 Apr 2024 20:43:20 +0200 Subject: [PATCH] Temp commit - Fixed "old" popup resizing and scroll bars (now actually respecting the chosen setting) --- public/css/stats.css | 96 +++ public/css/toggle-dependent.css | 1 - public/index.html | 1 + public/script.js | 5 +- public/scripts/RossAscends-mods.js | 17 +- public/scripts/stats.js | 396 ++++++++---- public/scripts/tags.js | 2 +- public/style.css | 33 +- src/endpoints/stats.js | 977 +++++++++++++++++++---------- src/util.js | 185 ++++++ 10 files changed, 1215 insertions(+), 498 deletions(-) create mode 100644 public/css/stats.css diff --git a/public/css/stats.css b/public/css/stats.css new file mode 100644 index 000000000..ac86a2152 --- /dev/null +++ b/public/css/stats.css @@ -0,0 +1,96 @@ +.rm_stats_button { + cursor: pointer; +} + +.rm_stat_block { + display: flex; +} + +.rm_stat_block_data_row:hover { + background-color: var(--grey5020a); + filter: drop-shadow(0px 0px 5px var(--SmartThemeShadowColor)); +} + +.rm_stat_name { + flex: 1; +} + +.rm_stat_values { + flex: 2; + display: flex; + align-items: center; +} + +.rm_stat_name .rm_stat_header { + height: calc(var(--mainFontSize) * 1.33333333333 + 3px); + padding-bottom: 3px; + border-bottom: 2px solid; +} + +.rm_stat_name .rm_stat_field { + text-align: left; +} + +.rm_stat_field.rm_stat_field_lefty { + text-align: left; + padding-left: 6px; +} + +.rm_stat_field { + flex: 1; + height: calc(var(--mainFontSize) * 1.33333333333); + text-align: right; + padding-left: 2px; + padding-right: 2px; +} + +.rm_stat_field_smaller { + color: var(--grey70); + font-size: smaller; +} + +.rm_stat_header { + margin-bottom: 3px; + font-weight: bold; +} + +.rm_stat_spacer { + height: 12px; +} + +.rm_stat_bar { + width: 100%; + height: calc(var(--mainFontSize) * 1.33333333333 - 4px); + display: flex; + margin-top: 2px; + margin-bottom: 2px; + padding-left: 6px; +} + +.rm_stat_bar_user { + background-color: rgba(130, 178, 140, 0.9); +} + +.rm_stat_bar_char { + background-color: rgba(178, 140, 130, 0.9); +} + +.rm_stat_block.rm_stat_right_spacing { + margin-right: 33.33333333333%; +} + +.rm_stat_avatar_block { + position: absolute; + top: calc(1.17em + 7px + 20px); + right: 0px; + height: calc(8px + calc(calc(var(--mainFontSize) * 1.33333333333) * 7) + calc(12px * 3)); + width: 33.33333333333%; + display: flex; + justify-content: center; + align-items: center; +} + +.rm_stat_avatar_block .avatar { + scale: 2; + flex: unset; +} diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 3f90edc42..f30501178 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -139,7 +139,6 @@ body.big-avatars .bogus_folder_select .avatar { body.big-avatars .avatar { width: calc(var(--avatar-base-width) * var(--big-avatar-width-factor)); height: calc(var(--avatar-base-height) * var(--big-avatar-height-factor)); - /* width: unset; */ border-style: none; display: flex; justify-content: center; diff --git a/public/index.html b/public/index.html index 352885649..d6c1adf6a 100644 --- a/public/index.html +++ b/public/index.html @@ -64,6 +64,7 @@ + diff --git a/public/script.js b/public/script.js index 18e15e8e2..357eeb379 100644 --- a/public/script.js +++ b/public/script.js @@ -7103,16 +7103,17 @@ function onScenarioOverrideRemoveClick() { * @param {string} type * @param {string} inputValue - Value to set the input to. * @param {PopupOptions} options - Options for the popup. - * @typedef {{okButton?: string, rows?: number, wide?: boolean, large?: boolean, allowHorizontalScrolling?: boolean, allowVerticalScrolling?: boolean }} PopupOptions - Options for the popup. + * @typedef {{okButton?: string, rows?: number, wide?: boolean, wider?: boolean, large?: boolean, allowHorizontalScrolling?: boolean, allowVerticalScrolling?: boolean }} PopupOptions - Options for the popup. * @returns */ -function callPopup(text, type, inputValue = '', { okButton, rows, wide, large, allowHorizontalScrolling, allowVerticalScrolling } = {}) { +function callPopup(text, type, inputValue = '', { okButton, rows, wide, wider, large, allowHorizontalScrolling, allowVerticalScrolling } = {}) { dialogueCloseStop = true; if (type) { popup_type = type; } $('#dialogue_popup').toggleClass('wide_dialogue_popup', !!wide); + $('#dialogue_popup').toggleClass('wider_dialogue_popup', !!wider); $('#dialogue_popup').toggleClass('large_dialogue_popup', !!large); $('#dialogue_popup').toggleClass('horizontal_scrolling_dialogue_popup', !!allowHorizontalScrolling); $('#dialogue_popup').toggleClass('vertical_scrolling_dialogue_popup', !!allowVerticalScrolling); diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 87cbbff2c..1f1ec38e1 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -86,10 +86,10 @@ observer.observe(document.documentElement, observerConfig); * particular measure (like days or hours), that measure will not be included in the output. * * @param {number} total_gen_time - The total generation time in milliseconds. + * @param {boolean} [short=false] - Optional flag indicating whether short form should be used. ('2h' instead of '2 Hours') * @returns {string} - A human-readable string that represents the time spent generating characters. */ -export function humanizeGenTime(total_gen_time) { - +export function humanizeGenTime(total_gen_time, short = false) { //convert time_spent to humanized format of "_ Hours, _ Minutes, _ Seconds" from milliseconds let time_spent = total_gen_time || 0; time_spent = Math.floor(time_spent / 1000); @@ -100,12 +100,13 @@ export function humanizeGenTime(total_gen_time) { let hours = time_spent % 24; time_spent = Math.floor(time_spent / 24); let days = time_spent; - time_spent = ''; - if (days > 0) { time_spent += `${days} Days, `; } - if (hours > 0) { time_spent += `${hours} Hours, `; } - if (minutes > 0) { time_spent += `${minutes} Minutes, `; } - time_spent += `${seconds} Seconds`; - return time_spent; + let parts = []; + if (days > 0) { parts.push(short ? `${days}d` : `${days} Days`); } + if (hours > 0) { parts.push(short ? `${hours}h` : `${hours} Hours`); } + if (minutes > 0) { parts.push(short ? `${minutes}m` : `${minutes} Minutes`); } + if (seconds > 0) { parts.push(short ? `${seconds}s` : `${seconds} Seconds`); } + if (!parts.length) { parts.push(short ? '<1s' : 'Instant') } + return parts.join(short ? ' ' : ', '); } let parsedUA = null; diff --git a/public/scripts/stats.js b/public/scripts/stats.js index 48983766b..092722e63 100644 --- a/public/scripts/stats.js +++ b/public/scripts/stats.js @@ -1,22 +1,68 @@ // statsHelper.js -import { getRequestHeaders, callPopup, characters, this_chid } from '../script.js'; +import { getRequestHeaders, callPopup, characters, this_chid, buildAvatarList, characterToEntity } from '../script.js'; import { humanizeGenTime } from './RossAscends-mods.js'; import { registerDebugFunction } from './power-user.js'; +import { timestampToMoment } from './utils.js'; -let charStats = {}; +const MIN_TIMESTAMP = 0; +const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime(); +const CURRENT_STATS_VERSION = '1.0'; + +/** @typedef {import('../script.js').Character} Character */ /** - * Creates an HTML stat block. - * - * @param {string} statName - The name of the stat to be displayed. - * @param {number|string} statValue - The value of the stat to be displayed. - * @returns {string} - An HTML string representing the stat block. + * @typedef {object} AggregateStat + * @property {number[]} list - The values stored, so they can be recalculated + * @property {number} total - Total / Sum + * @property {number} min - Minimum value + * @property {number} max - Maximum value + * @property {number} avg - Average value */ -function createStatBlock(statName, statValue) { - return `
-
${statName}:
-
${statValue}
-
`; + +/** + * @typedef {object} Stats - Stats for a chat + * @property {number} chat_size - The size of the actual chat file + * @property {number} date_first_chat - Timestamp of the first chat message (made by user) + * @property {number} date_last_chat - Timestamp of the last chat message in this chat (made by anyone) + * @property {number} total_gen_time - Total generation time in milliseconds + * @property {number} total_msg_count - The total messages of user and non-user, not including swipes + * @property {number} total_swipe_count - The number of swipes in the whole chat + * @property {number} avg_gen_time - Average generation tme in milliseconds + * @property {number} avg_swipe_count - Average swipes per non-user message + * + * @property {number} avg_chat_msg_count - Average messages per chat + * @property {number} avg_chat_duration - Average duration of a chat (from first till last message) + * + * @property {AggregateStat} msg - + * @property {AggregateStat} user_msg - + * @property {AggregateStat} non_user_msg - + * @property {AggregateStat} words - + * @property {AggregateStat} user_words - + * @property {AggregateStat} non_user_words - + * + */ + + + +/** @type {StatsCollection} The collection of all stats, accessable via their key */ +let charStats = { timestamp: 0, version: CURRENT_STATS_VERSION, stats: {} }; + +/** + * Creates an empty new stats object with default values + * @returns {Stats} The stats + */ +function createEmptyStats() { + return { + total_gen_time: 0, + user_word_count: 0, + non_user_word_count: 0, + user_msg_count: 0, + non_user_msg_count: 0, + total_swipe_count: 0, + chat_size: 0, + date_first_chat: MAX_TIMESTAMP, + date_last_chat: MIN_TIMESTAMP, + }; } /** @@ -32,51 +78,91 @@ function verifyStatValue(stat) { /** * Calculates total stats from character statistics. * - * @returns {Object} - Object containing total statistics. + * @returns {Stats} - Object containing total statistics. */ function calculateTotalStats() { - let totalStats = { - total_gen_time: 0, - user_msg_count: 0, - non_user_msg_count: 0, - user_word_count: 0, - non_user_word_count: 0, - total_swipe_count: 0, - date_last_chat: 0, - date_first_chat: new Date('9999-12-31T23:59:59.999Z').getTime(), - }; + let totalStats = createEmptyStats(); - for (let stats of Object.values(charStats)) { + for (let stats of Object.values(charStats.stats)) { totalStats.total_gen_time += verifyStatValue(stats.total_gen_time); totalStats.user_msg_count += verifyStatValue(stats.user_msg_count); - totalStats.non_user_msg_count += verifyStatValue( - stats.non_user_msg_count, - ); + totalStats.non_user_msg_count += verifyStatValue(stats.non_user_msg_count); totalStats.user_word_count += verifyStatValue(stats.user_word_count); - totalStats.non_user_word_count += verifyStatValue( - stats.non_user_word_count, - ); - totalStats.total_swipe_count += verifyStatValue( - stats.total_swipe_count, - ); - - if (verifyStatValue(stats.date_last_chat) != 0) { - totalStats.date_last_chat = Math.max( - totalStats.date_last_chat, - stats.date_last_chat, - ); - } - if (verifyStatValue(stats.date_first_chat) != 0) { - totalStats.date_first_chat = Math.min( - totalStats.date_first_chat, - stats.date_first_chat, - ); - } + totalStats.non_user_word_count += verifyStatValue(stats.non_user_word_count); + totalStats.total_swipe_count += verifyStatValue(stats.total_swipe_count); + totalStats.date_last_chat = Math.max(totalStats.date_last_chat, verifyStatValue(stats.date_last_chat) || MIN_TIMESTAMP); + totalStats.date_first_chat = Math.min(totalStats.date_first_chat, verifyStatValue(stats.date_first_chat) || MAX_TIMESTAMP); } return totalStats; } +/** + * Build a humanized string for a duration + * @param {number} start - Start time (in milliseconds) + * @param {number|null} end - End time (in milliseconds), if null will be replaced with Date.now() + * @param {object} param2 - Optional parameters + * @param {string} [param2.fallback='Never'] - Fallback value no duration can be calculated + * @param {function(string): string} [param2.wrapper=null] - Optional function to wrap/format the resulting humanized duration + * @returns {string} Humanized duration string + */ +function humanizedDuration(start, end = null, { fallback = 'Never', wrapper = null } = {}) { + end = end ?? Date.now(); + if (!start || start > end) { + return fallback; + } + // @ts-ignore + const humanized = moment.duration(end - start).humanize(); + return wrapper ? wrapper(humanized) : humanized; +} + +/** + * @typedef {object} StatField A stat block value to print + * @property {any} value - The value to print + * @property {boolean} [isHeader=false] - Flag indicating whether this is a header + * @property {string|null} [info=null] - Optional text that will be shown as an info icon + * @property {string|null} [title=null] - Optional title for the value - if null and info is set, info will be used as title too + * @property {string[]|null} [classes=null] - Optional list of classes for the stat field + */ + + +/** @param {StatField|any} x @returns {StatField} gets the stat field object for any value */ +function field(x) { return (typeof x === 'object' && x !== null && Object.hasOwn(x, 'value')) ? x : { value: x }; } + +/** + * Creates an HTML stat block + * + * @param {StatField|any} name - The name content of the stat to be displayed + * @param {StatField[]|any[]} values - Value or values to be listed for the stat block + * @returns {string} - An HTML string representing the stat block + */ +function createStatBlock(name, ...values) { + /** @param {StatField} stat @returns {string} */ + function buildField(stat) { + const classes = ['rm_stat_field', stat.isHeader ? 'rm_stat_header' : '', ...(stat.classes ?? [])].filter(x => x?.length); + return `
+ ${stat.value === null || stat.value === '' ? '‌' : stat.value} + ${stat.info ? `
` : ''} +
`; + } + + const statName = field(name); + const statValues = values.flat(Infinity).map(field); + + const isDataRow = !statName.isHeader && !statValues.some(x => x.isHeader); + const isRightSpacing = statValues.slice(-1)[0]?.classes?.includes('rm_stat_right_spacing'); + // Hack right spacing, which is added via a value just having the class + if (isRightSpacing) { + statValues.pop(); + } + + const classes = ['rm_stat_block', isDataRow ? 'rm_stat_block_data_row' : null, isRightSpacing ? 'rm_stat_right_spacing' : null].filter(x => x?.length); + return `
+
${buildField(statName)}
+
${statValues.map(x => buildField(x)).join('')}
+
`; +} + /** * Generates an HTML report of stats. * @@ -84,45 +170,124 @@ function calculateTotalStats() { * chat time, number of user messages and character messages, word count, and swipe count. * The stat blocks are tailored depending on the stats type ("User" or "Character"). * - * @param {string} statsType - The type of stats (e.g., "User", "Character"). - * @param {Object} stats - The stats data. Expected keys in this object include: - * total_gen_time - total generation time - * date_first_chat - timestamp of the first chat - * date_last_chat - timestamp of the most recent chat - * user_msg_count - count of user messages - * non_user_msg_count - count of non-user messages - * user_word_count - count of words used by the user - * non_user_word_count - count of words used by the non-user - * total_swipe_count - total swipe count + * @param {'User'|'Character'} statsType - The type of stats (e.g., "User", "Character") + * @param {number|null} characterId - Character id for these stats, null if global + * @param {Stats} stats - The stats data */ -function createHtml(statsType, stats) { - // Get time string - let timeStirng = humanizeGenTime(stats.total_gen_time); - let chatAge = 'Never'; - if (stats.date_first_chat < Date.now()) { - chatAge = moment - .duration(stats.date_last_chat - stats.date_first_chat) - .humanize(); +function createHtml(statsType, characterId, stats) { + const NOW = Date.now(); + const name = characters[characterId]?.name || 'User'; + const HMTL_STAT_SPACER = '
'; + + /** @param {number} charVal @param {number} userVal @returns {string} */ + function buildBar(userVal, charVal) { + const percentUser = (userVal / (userVal + charVal)) * 100; + const percentChar = 100 - percentUser; + return `
+
+
+
`; + } + /** @param {any[]} values @returns {StatField[]} */ + function buildBarDesc(...values) { + return values.flat(Infinity).map(field).map((x, i) => i % 2 == 0 ? { classes: [...(x.classes ?? []), 'rm_stat_field_lefty'], ...x } : x); } // Create popup HTML with stats - let html = `

${statsType} Stats

`; - if (statsType === 'User') { - html += createStatBlock('Chatting Since', `${chatAge} ago`); - } else { - html += createStatBlock('First Interaction', `${chatAge} ago`); - } - html += createStatBlock('Chat Time', timeStirng); - html += createStatBlock('User Messages', stats.user_msg_count); - html += createStatBlock( - 'Character Messages', - stats.non_user_msg_count - stats.total_swipe_count, + let html = `

${statsType} Stats - ${name}

`; + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Character Overview', isHeader: true }); + html += createStatBlock('Chats', { value: 34 }, { value: null, classes: ['rm_stat_right_spacing'] }); + html += createStatBlock({ value: 'Chats Size', info: 'The chat file sizes calculated and summed.\nThis value is only estimated, and will be refreshed sporadically.' }, { value: `~${'3.54 mb'}` }, { value: null, classes: ['rm_stat_right_spacing'] }); + html += createStatBlock('Most Used Model', { value: 'Noromaid' }, { value: null, classes: ['rm_stat_right_spacing'] }); + html += HMTL_STAT_SPACER; + html += createStatBlock('', + { value: 'First', isHeader: true, info: `The data corresponding to the first chat with ${name}` }, + { value: 'Last', isHeader: true, info: `The data corresponding to the last chat with ${name}` }, + { value: null, classes: ['rm_stat_right_spacing'] }, + ); + html += createStatBlock({ value: 'New Chat', info: 'The first/last time when a new chat was started' }, + { value: humanizedDuration(stats.date_first_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_first_chat).format('LL LT') }, + { value: humanizedDuration(stats.date_last_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_last_chat).format('LL LT') }, + { value: null, classes: ['rm_stat_right_spacing'] }, + ); + html += createStatBlock({ value: 'Chat Ended', info: 'The first/last time when the last message was send to a chat' }, + { value: humanizedDuration(stats.date_first_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_first_chat).format('LL LT') }, + { value: humanizedDuration(stats.date_last_chat, NOW, { wrapper: x => `${x} ago` }), title: timestampToMoment(stats.date_last_chat).format('LL LT') }, + { value: null, classes: ['rm_stat_right_spacing'] }, ); - html += createStatBlock('User Words', stats.user_word_count); - html += createStatBlock('Character Words', stats.non_user_word_count); - html += createStatBlock('Swipes', stats.total_swipe_count); - callPopup(html, 'text'); + html += HMTL_STAT_SPACER; + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Aggregated Stats', isHeader: true, info: 'Values per chat, aggregated over all chats' }); + html += createStatBlock(null, + { value: 'Total', isHeader: true, info: 'Total summed value over all chats' }, + { value: 'Min', isHeader: true, info: 'Minium value for any chat' }, + { value: 'Avg', isHeader: true, info: 'Average value over all chats' }, + { value: 'Max', isHeader: true, info: 'Maximum value for any chat' } + ); + html += createStatBlock({ value: 'Chatting Time', info: 'Total chatting time over all chats, and min/avg/max chatting time per chat' }, + { value: humanizeGenTime(114387009, true) }, { value: humanizeGenTime(7203, true) }, { value: humanizeGenTime(159017, true) }, { value: humanizeGenTime(7884930, true) }); + html += createStatBlock({ value: 'Generation Time', info: 'Total generation time over all chats, and min/avg/max generation time per chat' }, + humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true)); + html += createStatBlock('Generated Tokens', 2355, 43, 180, 2400); + html += HMTL_STAT_SPACER; + html += createStatBlock('Swiping Time', humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true)); + html += createStatBlock({ value: 'Swipes', info: 'Total swipes over all chats, and min/avg/max swipes per chat' }, + { value: 256 }, { value: 1 }, { value: 4 }, { value: 25 }); + html += HMTL_STAT_SPACER; + html += createStatBlock('User Response Time', humanizeGenTime(34680309, true), humanizeGenTime(4566, true), humanizeGenTime(23523, true), humanizeGenTime(286230, true)); + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Messages', info: 'Total messages over all chats (excluding swipes), and min/avg/max messages per chat' }, + 512, 2, 12, 100); + html += createStatBlock('System Messages', 47, 0, 4, 85); + html += createStatBlock({ value: 'Messages (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(145, 359, 2, 27, 8, 54, 66, 100)); + html += createStatBlock({ value: '', info: '' }, + buildBar(145, 359), buildBar(2, 27), buildBar(8, 54), buildBar(66, 100)); + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Words', info: 'Total words over all chats, and min/avg/max words per chat' }, + { value: 5124 }, { value: 26 }, { value: 122 }, { value: 1008 }); + html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(1451, 3594, 22, 279, 84, 625, 762, 2505)); + html += createStatBlock({ value: '', info: '' }, + buildBar(1451, 3594), buildBar(22, 279), buildBar(84, 625), buildBar(762, 2505)); + + html += HMTL_STAT_SPACER; + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Per Message Stats', isHeader: true, info: 'Values per message, aggregated over all chats' }); + html += createStatBlock('', + null, + { value: 'Min', isHeader: true, info: 'Minium ' }, + { value: 'Avg', isHeader: true }, + { value: 'Max', isHeader: true } + ); + html += createStatBlock({ value: 'Generation Time', info: 'min/avg/max generation time per message' }, + null, { value: humanizeGenTime(4566, true) }, { value: humanizeGenTime(23523, true) }, { value: humanizeGenTime(286230, true) }); + html += createStatBlock('Generated Tokens', null, 43, 180, 2400); + html += HMTL_STAT_SPACER; + html += createStatBlock('Swiping Time', null, humanizeGenTime(1456, true), humanizeGenTime(2523, true), humanizeGenTime(28230, true)); + html += createStatBlock({ value: 'Swipes', info: 'min/avg/max swipes per non-user message' }, + null, { value: 1 }, { value: 4 }, { value: 25 }); + html += HMTL_STAT_SPACER; + html += createStatBlock('User Response Time', null, humanizeGenTime(0, true), humanizeGenTime(233, true), humanizeGenTime(13630, true)); + html += HMTL_STAT_SPACER; + html += createStatBlock({ value: 'Words', info: 'min/avg/max words per message' }, + null, { value: 4 }, { value: 145 }, { value: 431 }); + html += createStatBlock({ value: 'Words (User / Char)', classes: ['rm_stat_field_smaller'] }, buildBarDesc(null, null, 22, 279, 84, 625, 762, 2505)); + html += createStatBlock({ value: '', info: '' }, + null, buildBar(22, 279), buildBar(84, 625), buildBar(762, 2505)); + + html += HMTL_STAT_SPACER; + html += HMTL_STAT_SPACER; + + // Hijack avatar list function to draw the user avatar + if (characters[characterId]) { + const placeHolder = $('
'); + const entity = characterToEntity(characters[characterId], characterId); + buildAvatarList(placeHolder, [entity]); + html = placeHolder.prop('outerHTML') + html; + } + + callPopup(html, 'text', '', { wider: true, allowVerticalScrolling: true }); } /** @@ -136,36 +301,28 @@ async function userStatsHandler() { let totalStats = calculateTotalStats(); // Create HTML with stats - createHtml('User', totalStats); + createHtml('User', null, totalStats); } /** * Handles the character stats by getting them from the server and generating the HTML report. * - * @param {Object} characters - Object containing character data. + * @param {{[characterKey: string]: Character}} characters - Object containing character data. * @param {string} this_chid - The character id. */ async function characterStatsHandler(characters, this_chid) { // Get stats from server await getStats(); // Get character stats - let myStats = charStats[characters[this_chid].avatar]; + let myStats = charStats.stats[characters[this_chid].avatar]; if (myStats === undefined) { - myStats = { - total_gen_time: 0, - user_msg_count: 0, - non_user_msg_count: 0, - user_word_count: 0, - non_user_word_count: countWords(characters[this_chid].first_mes), - total_swipe_count: 0, - date_last_chat: 0, - date_first_chat: new Date('9999-12-31T23:59:59.999Z').getTime(), - }; - charStats[characters[this_chid].avatar] = myStats; + myStats = createEmptyStats(); + myStats.non_user_word_count = countWords(characters[this_chid].first_mes); + charStats.stats[characters[this_chid].avatar] = myStats; updateStats(); } // Create HTML with stats - createHtml('Character', myStats); + createHtml('Character', this_chid, myStats); } /** @@ -261,7 +418,7 @@ function countWords(str) { * * @param {Object} line - Object containing message data. * @param {string} type - The type of the message processing (e.g., 'append', 'continue', 'appendFinal', 'swipe'). - * @param {Object} characters - Object containing character data. + * @param {Character[]} characters - Object containing character data. * @param {string} this_chid - The character id. * @param {string} oldMesssage - The old message that's being processed. */ @@ -271,52 +428,43 @@ async function statMesProcess(line, type, characters, this_chid, oldMesssage) { } await getStats(); - let stat = charStats[characters[this_chid].avatar]; + let stats = charStats.stats[characters[this_chid].avatar] ?? createEmptyStats(); - if (!stat) { - stat = { - total_gen_time: 0, - user_word_count: 0, - non_user_msg_count: 0, - user_msg_count: 0, - total_swipe_count: 0, - date_first_chat: Date.now(), - date_last_chat: Date.now(), - }; - } - - stat.total_gen_time += calculateGenTime( - line.gen_started, - line.gen_finished, - ); + stats.total_gen_time += calculateGenTime(line.gen_started, line.gen_finished); if (line.is_user) { if (type != 'append' && type != 'continue' && type != 'appendFinal') { - stat.user_msg_count++; - stat.user_word_count += countWords(line.mes); + stats.user_msg_count++; + stats.user_word_count += countWords(line.mes); } else { let oldLen = oldMesssage.split(' ').length; - stat.user_word_count += countWords(line.mes) - oldLen; + stats.user_word_count += countWords(line.mes) - oldLen; } } else { // if continue, don't add a message, get the last message and subtract it from the word count of // the new message if (type != 'append' && type != 'continue' && type != 'appendFinal') { - stat.non_user_msg_count++; - stat.non_user_word_count += countWords(line.mes); + stats.non_user_msg_count++; + stats.non_user_word_count += countWords(line.mes); } else { let oldLen = oldMesssage.split(' ').length; - stat.non_user_word_count += countWords(line.mes) - oldLen; + stats.non_user_word_count += countWords(line.mes) - oldLen; } } if (type === 'swipe') { - stat.total_swipe_count++; + stats.total_swipe_count++; + } + + // If this is the first user message, set the first chat time + if (line.is_user) { + //get min between firstChatTime and timestampToMoment(json.send_date) + stats.date_first_chat = Math.min(timestampToMoment(line.send_date) ?? MAX_TIMESTAMP, stats.date_first_chat); + } + + // For last chat time, we skip the original first message and then take all user and AI messages + if ((stats.user_msg_count + stats.non_user_msg_count) > 1) { + stats.date_last_chat = Math.max(timestampToMoment(line.send_date) ?? MIN_TIMESTAMP, stats.date_last_chat); } - stat.date_last_chat = Date.now(); - stat.date_first_chat = Math.min( - stat.date_first_chat ?? new Date('9999-12-31T23:59:59.999Z').getTime(), - Date.now(), - ); updateStats(); } diff --git a/public/scripts/tags.js b/public/scripts/tags.js index b1c40c020..2c54f6592 100644 --- a/public/scripts/tags.js +++ b/public/scripts/tags.js @@ -1035,7 +1035,7 @@ function onViewTagsListClick() { makeTagListDraggable(tagContainer); - callPopup(list, 'text'); + callPopup(list, 'text', null, { allowVerticalScrolling: true }); } function makeTagListDraggable(tagContainer) { diff --git a/public/style.css b/public/style.css index f6516beed..8d381d88d 100644 --- a/public/style.css +++ b/public/style.css @@ -2052,10 +2052,6 @@ grammarly-extension { text-align: right; } -.rm_stats_button { - cursor: pointer; -} - /* Focus */ #bulk_tag_popup, @@ -2085,11 +2081,6 @@ grammarly-extension { overflow-x: hidden; } -.rm_stat_block { - display: flex; - justify-content: space-between; -} - .large_dialogue_popup { height: 90vh !important; height: 90svh !important; @@ -2103,12 +2094,8 @@ grammarly-extension { min-width: var(--sheldWidth); } -.horizontal_scrolling_dialogue_popup { - overflow-x: unset !important; -} - -.vertical_scrolling_dialogue_popup { - overflow-y: unset !important; +.wider_dialogue_popup { + min-width: 750px; } #bulk_tag_popup_holder, @@ -2116,20 +2103,32 @@ grammarly-extension { display: flex; flex-direction: column; height: 100%; - overflow-y: hidden; padding: 0 10px; } #dialogue_popup_text { flex-grow: 1; - overflow-y: auto; height: 100%; + position: relative; + overflow-x: hidden; + overflow-y: hidden; + max-height: calc(90vh - 50px); + max-height: calc(90svh - 50px); +} + +.horizontal_scrolling_dialogue_popup #dialogue_popup_text { + overflow-x: unset; +} + +.vertical_scrolling_dialogue_popup #dialogue_popup_text { + overflow-y: unset; } #dialogue_popup_controls { display: flex; align-self: center; gap: 20px; + height: 40px; } #bulk_tag_popup_reset, diff --git a/src/endpoints/stats.js b/src/endpoints/stats.js index b4ff37ab2..2bfa6d50d 100644 --- a/src/endpoints/stats.js +++ b/src/endpoints/stats.js @@ -3,142 +3,28 @@ const path = require('path'); const express = require('express'); const writeFileAtomic = require('write-file-atomic'); const crypto = require('crypto'); +const sanitize = require('sanitize-filename'); const readFile = fs.promises.readFile; const readdir = fs.promises.readdir; const { jsonParser } = require('../express-common'); const { DIRECTORIES } = require('../constants'); +const { readAndParseJsonlFile, timestampToMoment, humanizedToDate, calculateDuration, minDate, maxDate, now } = require('../util'); -let charStats = {}; -let lastSaveTimestamp = 0; const statsFilePath = 'public/stats.json'; -/** - * Convert a timestamp to an integer timestamp. - * (sorry, it's momentless for now, didn't want to add a package just for this) - * This function can handle several different timestamp formats: - * 1. Unix timestamps (the number of seconds since the Unix Epoch) - * 2. ST "humanized" timestamps, formatted like "YYYY-MM-DD @HHh MMm SSs ms" - * 3. Date strings in the format "Month DD, YYYY H:MMam/pm" - * - * The function returns the timestamp as the number of milliseconds since - * the Unix Epoch, which can be converted to a JavaScript Date object with new Date(). - * - * @param {string|number} timestamp - The timestamp to convert. - * @returns {number|null} The timestamp in milliseconds since the Unix Epoch, or null if the input cannot be parsed. - * - * @example - * // Unix timestamp - * timestampToMoment(1609459200); - * // ST humanized timestamp - * timestampToMoment("2021-01-01 @00h 00m 00s 000ms"); - * // Date string - * timestampToMoment("January 1, 2021 12:00am"); - */ -function timestampToMoment(timestamp) { - if (!timestamp) { - return null; - } +const MIN_TIMESTAMP = 0; +const MAX_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z').getTime(); +const MIN_DATE = new Date(MIN_TIMESTAMP); +const MAX_DATE = new Date(MAX_TIMESTAMP); +const CURRENT_STATS_VERSION = '1.1'; +const EMPTY_GLOBAL_STATS = { _calculated: MIN_DATE, version: CURRENT_STATS_VERSION, stats: {} }; - if (typeof timestamp === 'number') { - return timestamp; - } - const pattern1 = - /(\d{4})-(\d{1,2})-(\d{1,2}) @(\d{1,2})h (\d{1,2})m (\d{1,2})s (\d{1,3})ms/; - const replacement1 = ( - match, - year, - month, - day, - hour, - minute, - second, - millisecond, - ) => { - return `${year}-${month.padStart(2, '0')}-${day.padStart( - 2, - '0', - )}T${hour.padStart(2, '0')}:${minute.padStart( - 2, - '0', - )}:${second.padStart(2, '0')}.${millisecond.padStart(3, '0')}Z`; - }; - const isoTimestamp1 = timestamp.replace(pattern1, replacement1); - if (!isNaN(new Date(isoTimestamp1))) { - return new Date(isoTimestamp1).getTime(); - } - - const pattern2 = /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i; - const replacement2 = (match, month, day, year, hour, minute, meridiem) => { - const monthNames = [ - 'January', - 'February', - 'March', - 'April', - 'May', - 'June', - 'July', - 'August', - 'September', - 'October', - 'November', - 'December', - ]; - const monthNum = monthNames.indexOf(month) + 1; - const hour24 = - meridiem.toLowerCase() === 'pm' - ? (parseInt(hour, 10) % 12) + 12 - : parseInt(hour, 10) % 12; - return `${year}-${monthNum.toString().padStart(2, '0')}-${day.padStart( - 2, - '0', - )}T${hour24.toString().padStart(2, '0')}:${minute.padStart( - 2, - '0', - )}:00Z`; - }; - const isoTimestamp2 = timestamp.replace(pattern2, replacement2); - if (!isNaN(new Date(isoTimestamp2))) { - return new Date(isoTimestamp2).getTime(); - } - - return null; -} - -/** - * Collects and aggregates stats for all characters. - * - * @param {string} chatsPath - The path to the directory containing the chat files. - * @param {string} charactersPath - The path to the directory containing the character files. - * @returns {Object} The aggregated stats object. - */ -async function collectAndCreateStats(chatsPath, charactersPath) { - console.log('Collecting and creating stats...'); - const files = await readdir(charactersPath); - - const pngFiles = files.filter((file) => file.endsWith('.png')); - - let processingPromises = pngFiles.map((file, index) => - calculateStats(chatsPath, file, index), - ); - const statsArr = await Promise.all(processingPromises); - - let finalStats = {}; - for (let stat of statsArr) { - finalStats = { ...finalStats, ...stat }; - } - // tag with timestamp on when stats were generated - finalStats.timestamp = Date.now(); - return finalStats; -} - -async function recreateStats(chatsPath, charactersPath) { - charStats = await collectAndCreateStats(chatsPath, charactersPath); - await saveStatsToFile(); - console.debug('Stats (re)created and saved to file.'); -} +/** @type {StatsCollection} The collection of all stats, accessable via their key */ +let globalStats = { ...EMPTY_GLOBAL_STATS }; +let lastSaveDate = MIN_DATE; /** * Loads the stats file into memory. If the file doesn't exist or is invalid, @@ -147,11 +33,17 @@ async function recreateStats(chatsPath, charactersPath) { async function init() { try { const statsFileContent = await readFile(statsFilePath, 'utf-8'); - charStats = JSON.parse(statsFileContent); + const obj = JSON.parse(statsFileContent); + // Migrate/recreate stats if the version has changed + if (obj.version !== CURRENT_STATS_VERSION) { + console.info(`Found outdated stats of version '${obj.version}'. Recreating stats for current version '${CURRENT_STATS_VERSION}'...`); + await recreateStats(); + } + globalStats = obj; } catch (err) { // If the file doesn't exist or is invalid, initialize stats if (err.code === 'ENOENT' || err instanceof SyntaxError) { - recreateStats(DIRECTORIES.chats, DIRECTORIES.characters); + recreateStats(); } else { throw err; // Rethrow the error if it's something we didn't expect } @@ -159,22 +51,6 @@ async function init() { // Save stats every 5 minutes setInterval(saveStatsToFile, 5 * 60 * 1000); } -/** - * Saves the current state of charStats to a file, only if the data has changed since the last save. - */ -async function saveStatsToFile() { - if (charStats.timestamp > lastSaveTimestamp) { - //console.debug("Saving stats to file..."); - try { - await writeFileAtomic(statsFilePath, JSON.stringify(charStats)); - lastSaveTimestamp = Date.now(); - } catch (error) { - console.log('Failed to save stats to file.', error); - } - } else { - //console.debug('Stats have not changed since last save. Skipping file write.'); - } -} /** * Attempts to save charStats to a file and then terminates the process. @@ -189,34 +65,501 @@ async function onExit() { } /** - * Reads the contents of a file and returns the lines in the file as an array. - * - * @param {string} filepath - The path of the file to be read. - * @returns {Array} - The lines in the file. - * @throws Will throw an error if the file cannot be read. + * @typedef {object} MessageLine - The chat message object to process. */ -function readAndParseFile(filepath) { - try { - let file = fs.readFileSync(filepath, 'utf8'); - let lines = file.split('\n'); - return lines; - } catch (error) { - console.error(`Error reading file at ${filepath}: ${error}`); - return []; + +/** + * @typedef {object} StatsCollection - An object holding all character stats, and some additional main stats + * @property {string} version - Version number indication the version of this stats data - so it can be automatically migrated/recalculated if any of the calculation logic changes + * @property {{[characterKey: string]: CharacterStats}} stats - All the dynamically saved stats objecs + * @property {Date} _calculated - + */ + +/** + * @typedef {object} CharacterStats + * @property {string} name - + * @property {string} characterKey - + * @property {number} chats - The creation date of the chat. + * @property {number} chatSize - The size of all chats + * + * @property {Date} firstCreateDate - + * @property {Date} lastCreateDate - + * @property {Date} firstlastInteractionDate - + * @property {Date} lastLastInteractionDate - + * + * @property {AggregateStat} chattingTime - + * @property {AggregateStat} messages - + * @property {AggregateStat} systemMessages - + * @property {AggregateStat} userMessages - + * @property {AggregateStat} charMessages - + * + * @property {AggregateStat} genTime - + * @property {AggregateStat} genTokenCount - + * @property {AggregateStat} swipeGenTime - + * @property {AggregateStat} swipes - + * @property {AggregateStat} userResponseTime - + * @property {AggregateStat} words - + * @property {AggregateStat} userWords - + * @property {AggregateStat} charWords - + * + * @property {AggregateStat} perMessageGenTime - + * @property {AggregateStat} perMessageGenTokenCount - + * @property {AggregateStat} perMessageSwipeGenTime - + * @property {AggregateStat} perMessageSwipeCount - + * @property {AggregateStat} perMessageUserResponseTime - + * @property {AggregateStat} perMessageWords - + * @property {AggregateStat} perMessageUserWords - + * @property {AggregateStat} perMessageCharWords - + * + * @property {{[model: string]: { count: number, tokens: number}}} genModels - model usages + * @property {ChatStats[]} chatsStats - + * @property {Date} _calculated - + */ + +/** + * @typedef {object} ChatStats + * @property {string} chatName - The unique identifier for the chat. + * @property {number} chatId - hash + * @property {number} chatSize - + * @property {Date} createDate - The creation date of the chat. (time in ISO 8601 format) + * @property {Date} lastInteractionDate - (time in ISO 8601 format) + * + * @property {number} chattingTime - + * @property {number} messages - + * @property {number} systemMessages - + * @property {number} userMessages - + * @property {number} charMessages - + * + * @property {AggregateStat} genTime - + * @property {AggregateStat} genTokenCount - + * @property {AggregateStat} swipeGenTime - + * @property {AggregateStat} swipes - + * @property {AggregateStat} userResponseTime - + * @property {AggregateStat} words - + * @property {AggregateStat} userWords - + * @property {AggregateStat} charWords - + * + * @property {{[model: string]: { count: number, tokens: number}}} genModels - model usages + * @property {MessageStats[]} messagesStats - An array of MessageStats objects for individual message analysis. + * @property {Date} _calculated - + */ + +/** + * @typedef {object} MessageStats + * @property {boolean} isUser - + * @property {boolean} isChar - + * @property {string} hash - + * @property {Date} sendDate - The time when the message was sent. + * @property {number?} genTime - The total time taken to generate this message and all swipes. + * @property {number?} genTokenCount - + * @property {number?} swipeGenTime - The total generation time for all swipes excluding the first generation. + * @property {number?} swipes - The count of additional swipes minus the first generated message. + * @property {number} words - The number of words in the message. + * @property {Date[]} genEndDates - + * @property {{[model: string]: { count: number, tokens: number}}} genModels - model usages + * @property {Date} _calculated - + */ + +/** + * An object that aggregates stats for a specific value + * + * By adding values to it, it'll automatically recalculate min, max and average + */ +class AggregateStat { + /** @type {number} The number of stats used for this aggregation - used for recalculating avg */ + count = 0; + /** @type {number} Total / Sum */ + total = 0; + /** @type {number} Minimum value */ + min = Number.NaN; + /** @type {number} Maximum value */ + max = 0; + /** @type {number} Average value */ + avg = 0; + /** @type {number[]} All values listed and saved, so the aggregate stats can be updated if needed when elements get removed */ + values = []; + constructor() { } + + reset() { + this.count, this.total, this.min, this.max, this.avg = 0; + } + + /** + * Adds a given value to this aggregation + * If you want to add all values of an `AggregateStat`, use `addAggregated` + * @param {number?} value - The value to add + */ + add(value) { + if (value === null || isNaN(value)) return; + this.count++; + this.total += value; + this.avg = this.total / this.count; + + this.values.push(value); + this.min = Math.min(isNaN(this.min) ? Number.MAX_SAFE_INTEGER : this.min, value); + this.max = Math.max(this.max, value); + } + + /** + * Adds all values of a given aggregation as single values + * @param {AggregateStat} aggregatedValue - The aggregate stat + */ + addAggregated(aggregatedValue) { + aggregatedValue.values.forEach(x => this.add(x)); + } + + /** + * Removes a given value from this aggregation + * If you want to remove all values of an `AggregateStat`, use `removeAggregated` + * @param {number?} value - The value to remove + */ + remove(value) { + if (value === null || isNaN(value)) return; + + this.count--; + this.total -= value; + this.avg = this.count === 0 ? 0 : this.total / this.count; + + const index = this.values.indexOf(value); + if (index === -1) { + console.warn(`Tried to remove aggregation value ${value} that does not exist. This should not happen...`); + return; + } + this.values.splice(index, 1); + + if (value === this.min) { + this.min = this.values.length > 0 ? Math.min(...this.values) : Number.NaN; + } + if (value === this.max) { + this.max = this.values.length > 0 ? Math.max(...this.values) : 0; + } + } + + /** + * Removes all values of a given aggregation as their respective values + * @param {AggregateStat} aggregatedValue - The aggregate stat + */ + removeAggregated(aggregatedValue) { + aggregatedValue.values.forEach(x => this.add(x)); } } /** - * Calculates the time difference between two dates. * - * @param {string} gen_started - The start time in ISO 8601 format. - * @param {string} gen_finished - The finish time in ISO 8601 format. - * @returns {number} - The difference in time in milliseconds. + * + * @returns {Promise} The aggregated stats object. */ -function calculateGenTime(gen_started, gen_finished) { - let startDate = new Date(gen_started); - let endDate = new Date(gen_finished); - return endDate - startDate; +async function recreateStats() { + console.log('Collecting and creating stats...'); + + // Resetting global stats first + globalStats = { ...EMPTY_GLOBAL_STATS }; + + // Load all char files to process their chat folders + const files = await readdir(DIRECTORIES.characters); + const charFiles = files.filter((file) => file.endsWith('.png')); + let processingPromises = charFiles.map((charFileName, _) => + recreateCharacterStats(charFileName), + ); + await Promise.all(processingPromises); + + await saveStatsToFile(); + console.debug('Stats (re)created and saved to file.'); + + return globalStats; + + /** @param {string} characterKey @return {CharacterStats?} */ + function recreateCharacterStats(characterKey) { + const charName = characterKey.replace('.png', ''); + const charChatsDir = path.join(DIRECTORIES.chats, charName); + if (!fs.existsSync(charChatsDir)) { + return null; + } + + const chatFiles = fs.readdirSync(charChatsDir); + chatFiles.forEach(chatName => { + triggerChatUpdate(characterKey, chatName); + }); + + return globalStats[characterKey]; + }; +} + + +/** + * + * @param {string} charChatsDir - The directoy path + * @param {string} chatName + * @returns {{chatName: string, lines: object[]}} + */ +function loadChatFile(charChatsDir, chatName) { + const fullFilePath = path.join(charChatsDir, sanitize(chatName)); + const lines = readAndParseJsonlFile(fullFilePath); + return { chatName, lines }; +} + +/** + * + * + * @param {string} characterKey - The character key + * @param {string} chatName - The name of the chat + * @returns {ChatStats?} + */ +function triggerChatUpdate(characterKey, chatName) { + const charName = characterKey.replace('.png', ''); + const charChatsDir = path.join(DIRECTORIES.chats, charName); + + // Load and process chats to get its stats + const loadedChat = loadChatFile(charChatsDir, chatName); + const fsStats = fs.statSync(path.join(charChatsDir, chatName)); + + const chatStats = processChat(chatName, loadedChat.lines, { chatSize: fsStats.size }); + if (chatStats === null) { + return null; + } + + // Create empty stats if character stats don't exist yet + globalStats.stats[characterKey] ??= newCharacterStats(characterKey, charName); + + // Update char stats with the processed chat stats + updateCharStatsWithChat(globalStats.stats[characterKey], chatStats); + + return chatStats; +} + +/** + * Recalculates character stats based on the current chat. + * Works with both updating/replacing an existing chat and also adding a new one. + * + * @param {CharacterStats} stats - The stats of the character + * @param {ChatStats} chatStats - The chat stats to add/update + * @returns {boolean} + */ +function updateCharStatsWithChat(stats, chatStats) { + // Check if we need to remove this chat's previous data first + removeChatFromCharStats(stats, chatStats); + + stats.chats++; + stats.chatSize += chatStats.chatSize; + stats.firstCreateDate = minDate(chatStats.createDate, stats.firstCreateDate) ?? stats.firstCreateDate; + stats.lastCreateDate = maxDate(chatStats.createDate, stats.lastCreateDate) ?? stats.lastCreateDate; + stats.firstlastInteractionDate = minDate(chatStats.lastInteractionDate, stats.firstlastInteractionDate) ?? stats.firstlastInteractionDate; + stats.lastLastInteractionDate = maxDate(chatStats.lastInteractionDate, stats.lastLastInteractionDate) ?? stats.lastLastInteractionDate; + + stats.chattingTime.add(chatStats.chattingTime); + stats.messages.add(chatStats.messages); + stats.systemMessages.add(chatStats.systemMessages); + stats.userMessages.add(chatStats.userMessages); + stats.charMessages.add(chatStats.charMessages); + + stats.genTime.add(chatStats.genTime.total); + stats.genTokenCount.add(chatStats.genTokenCount.total); + stats.swipeGenTime.add(chatStats.swipeGenTime.total); + stats.swipes.add(chatStats.swipes.total); + stats.userResponseTime.add(chatStats.userResponseTime.total); + stats.words.add(chatStats.words.total); + stats.userWords.add(chatStats.userWords.total); + stats.charWords.add(chatStats.charWords.total); + + stats.perMessageGenTime.addAggregated(chatStats.genTime); + stats.perMessageGenTokenCount.addAggregated(chatStats.genTokenCount); + stats.perMessageSwipeGenTime.addAggregated(chatStats.swipeGenTime); + stats.perMessageSwipeCount.addAggregated(chatStats.swipes); + stats.perMessageUserResponseTime.addAggregated(chatStats.userResponseTime); + stats.perMessageWords.addAggregated(chatStats.words); + stats.perMessageUserWords.addAggregated(chatStats.userWords); + stats.perMessageCharWords.addAggregated(chatStats.charWords); + + Object.entries(chatStats.genModels).forEach(([model, data]) => addModelUsage(stats.genModels, model, data.tokens, data.count)); + + console.debug(`Successfully updated ${stats.name}'s stats with chat ${chatStats.chatName}`); + return true; +} + +/** + * Removes the given chat stats from the character stats + * Both removing the saved stats object and also "calculating it out" of all existing values + * @param {CharacterStats} stats - The stats of the character + * @param {ChatStats} chatStats - The chat stats to remove + * @returns {boolean} Whether existed and was removed + */ +function removeChatFromCharStats(stats, chatStats) { + const index = stats.chatsStats.findIndex(x => x.chatName == chatStats.chatName); + if (index === -1) { + return false; + } + this.values.splice(index, 1); + + stats.chats--; + stats.chatSize -= chatStats.chatSize; + stats.firstCreateDate = minDate(chatStats.createDate, stats.firstCreateDate) ?? stats.firstCreateDate; + stats.lastCreateDate = maxDate(chatStats.createDate, stats.lastCreateDate) ?? stats.lastCreateDate; + stats.firstlastInteractionDate = minDate(chatStats.lastInteractionDate, stats.firstlastInteractionDate) ?? stats.firstlastInteractionDate; + stats.lastLastInteractionDate = maxDate(chatStats.lastInteractionDate, stats.lastLastInteractionDate) ?? stats.lastLastInteractionDate; + + stats.chattingTime.remove(chatStats.chattingTime); + stats.messages.remove(chatStats.messages); + stats.systemMessages.remove(chatStats.systemMessages); + stats.userMessages.remove(chatStats.userMessages); + stats.charMessages.remove(chatStats.charMessages); + + stats.genTime.remove(chatStats.genTime.total); + stats.genTokenCount.remove(chatStats.genTokenCount.total); + stats.swipeGenTime.remove(chatStats.swipeGenTime.total); + stats.swipes.remove(chatStats.swipes.total); + stats.userResponseTime.remove(chatStats.userResponseTime.total); + stats.words.remove(chatStats.words.total); + stats.userWords.remove(chatStats.userWords.total); + stats.charWords.remove(chatStats.charWords.total); + + stats.perMessageGenTime.removeAggregated(chatStats.genTime); + stats.perMessageGenTokenCount.removeAggregated(chatStats.genTokenCount); + stats.perMessageSwipeGenTime.removeAggregated(chatStats.swipeGenTime); + stats.perMessageSwipeCount.removeAggregated(chatStats.swipes); + stats.perMessageUserResponseTime.removeAggregated(chatStats.userResponseTime); + stats.perMessageWords.removeAggregated(chatStats.words); + stats.perMessageUserWords.removeAggregated(chatStats.userWords); + stats.perMessageCharWords.removeAggregated(chatStats.charWords); + + Object.entries(chatStats.genModels).forEach(([model, data]) => removeModelUsage(stats.genModels, model, data.tokens, data.count)); + + console.debug(`Successfully removed old chat stats for chat ${chatStats.chatName}`); + return true; +} + +/** + * + * @param {string} chatName + * @param {object[]} lines + * @param {{chatSize?: number}} [param0={}] - optional parameter that can be set when processing the chat + * @return {ChatStats?} + */ +function processChat(chatName, lines, { chatSize = 0 } = {}) { + if (!lines.length) { + console.warn('Processing chat file failed.'); + return null; + } + + /** @type {ChatStats} build the stats object first, then fill */ + const stats = newChatStats(chatName); + + // Fill stats that we already can + stats.chatSize = chatSize; + + /** @type {MessageStats?} Always remember the message before, for calculations */ + let lastMessage = null; + + // Process each message + for (const message of lines) { + // Check if this is the first message, the "data storage" + if (message.chat_metadata && message.create_date) { + stats.createDate = humanizedToDate(message.create_date) ?? stats.createDate; + stats.chatId = message.chat_metadata['chat_id_hash']; + continue; + } + + const messageStats = processMessage(message); + stats.messagesStats.push(messageStats); + + stats.lastInteractionDate = maxDate(stats.lastInteractionDate, messageStats.sendDate, ...messageStats.genEndDates) ?? stats.lastInteractionDate; + + // Aggregate chat stats for each message + // stats.chattingTime - is calculated at the end of message progressing + stats.messages += 1; + stats.systemMessages += message.is_system ? 1 : 0; + stats.userMessages += messageStats.isUser ? 1 : 0; + stats.charMessages += messageStats.isChar ? 1 : 0; + + stats.genTime.add(messageStats.genTime); + stats.genTokenCount.add(messageStats.genTokenCount) + stats.swipeGenTime.add(messageStats.swipeGenTime); + stats.swipes.add(messageStats.swipes); + + // If this is a user message, we calculate the response time from the last interaction of the message before + if (messageStats.isUser && lastMessage !== null) { + const lastInteractionBefore = lastMessage.genEndDates.sort().findLast(x => x < messageStats.sendDate) ?? lastMessage.sendDate; + const responseTime = calculateDuration(lastInteractionBefore, messageStats.sendDate); + stats.userResponseTime.add(responseTime); + } + + stats.words.add(messageStats.words); + stats.userWords.add(messageStats.isUser ? messageStats.words : null); + stats.charWords.add(messageStats.isChar ? messageStats.words : null); + + Object.entries(messageStats.genModels).forEach(([model, data]) => addModelUsage(stats.genModels, model, data.tokens, data.count)); + + // Remember this as the last message, for time calculations + lastMessage = messageStats; + } + + // Set up the final values for chat + stats.chattingTime = calculateDuration(stats.createDate, stats.lastInteractionDate); + + stats._calculated = now(); + return stats; +} + +/** + * Process a chat message and calculate relevant stats + * @param {MessageLine} message - The parsed json message line + * @returns {MessageStats} + */ +function processMessage(message, name = null) { + /** @type {MessageStats} build the stats object first, then fill */ + const stats = newMessageStats(); + + stats.isUser = message.is_user; + stats.isChar = !message.is_user && !message.is_system && (!name || message.name == name); + stats.hash = crypto.createHash('sha256').update(message.mes).digest('hex'); + + // Count all additional swipes (this array stores the original message too) + stats.swipes = message.swipe_info?.length ? message.swipe_info.length - 1 : null; + + // Use utility functions to process each message + stats.words = countWordsInString(message.mes); + stats.sendDate = new Date(timestampToMoment(message.send_date) ?? MIN_TIMESTAMP); + + // Only calculate generation time and token count for model messages + if (!message.is_user) { + if (message.gen_started && message.gen_finished) { + stats.genTokenCount = message.extra?.token_count || 0; + stats.genTime = calculateDuration(message.gen_started, message.gen_finished); + stats.genEndDates.push((new Date(message.gen_finished))); + addModelUsage(stats.genModels, message.extra?.model, message.extra?.token_count); + } + + // Sum up swipes. As swiping time counts everything that was not the last, final chosen message + // We also remember the highest timestamp for this message as the "last action" + message.swipe_info?.filter(x => x.gen_started !== message.gen_started && x.gen_started && x.gen_finished) + .forEach(swipeInfo => { + stats.genTokenCount = (stats.genTokenCount ?? 0) + message.extra?.token_count || 0; + const swipeGenTime = calculateDuration(swipeInfo.gen_started, swipeInfo.gen_finished); + stats.genTime = (stats.genTime ?? 0) + swipeGenTime; + stats.swipeGenTime = (stats.swipeGenTime ?? 0) + swipeGenTime; + stats.genEndDates.push((new Date(swipeInfo.gen_finished))); + addModelUsage(stats.genModels, swipeInfo.extra?.model, swipeInfo.extra?.token_count); + }); + } + + stats._calculated = now(); + return stats; +} + +/** @param {{[model: string]: { count: number, tokens: number}}} obj, @param {string} model, @param {number} tokens @param {number} count */ +function addModelUsage(obj, model, tokens, count = 1) { + if (!model) return; + obj[model] ??= { count: 0, tokens: 0 }; + obj[model].count += (count ?? 1); + obj[model].tokens += (tokens ?? 0); +} + +/** @param {{[model: string]: { count: number, tokens: number}}} obj, @param {string} model, @param {number} tokens @param {number} count */ +function removeModelUsage(obj, model, tokens, count = 1) { + if (!model || !obj[model]) return; + obj[model].count -= (count ?? 1); + obj[model].tokens -= (tokens ?? 0); + if (obj[model].count <= 0) + delete obj[model]; } /** @@ -226,192 +569,137 @@ function calculateGenTime(gen_started, gen_finished) { * @returns {number} - The number of words in the string. */ function countWordsInString(str) { - const match = str.match(/\b\w+\b/g); - return match ? match.length : 0; + return str.match(/\b\w+\b/g)?.length ?? 0; } /** - * calculateStats - Calculate statistics for a given character chat directory. - * - * @param {string} char_dir The directory containing the chat files. - * @param {string} item The name of the character. - * @return {object} An object containing the calculated statistics. + * Creates a new, empty character stats object + * @param {string} characterKey - The character key + * @param {string} charName - The characters' name + * @returns {CharacterStats} */ -const calculateStats = (chatsPath, item, index) => { - const char_dir = path.join(chatsPath, item.replace('.png', '')); - const stats = { - total_gen_time: 0, - user_word_count: 0, - non_user_word_count: 0, - user_msg_count: 0, - non_user_msg_count: 0, - total_swipe_count: 0, - chat_size: 0, - date_last_chat: 0, - date_first_chat: new Date('9999-12-31T23:59:59.999Z').getTime(), - }; - let uniqueGenStartTimes = new Set(); - - if (fs.existsSync(char_dir)) { - const chats = fs.readdirSync(char_dir); - if (Array.isArray(chats) && chats.length) { - for (const chat of chats) { - const result = calculateTotalGenTimeAndWordCount( - char_dir, - chat, - uniqueGenStartTimes, - ); - stats.total_gen_time += result.totalGenTime || 0; - stats.user_word_count += result.userWordCount || 0; - stats.non_user_word_count += result.nonUserWordCount || 0; - stats.user_msg_count += result.userMsgCount || 0; - stats.non_user_msg_count += result.nonUserMsgCount || 0; - stats.total_swipe_count += result.totalSwipeCount || 0; - - const chatStat = fs.statSync(path.join(char_dir, chat)); - stats.chat_size += chatStat.size; - stats.date_last_chat = Math.max( - stats.date_last_chat, - Math.floor(chatStat.mtimeMs), - ); - stats.date_first_chat = Math.min( - stats.date_first_chat, - result.firstChatTime, - ); - } - } - } - - return { [item]: stats }; -}; - -/** - * Returns the current charStats object. - * @returns {Object} The current charStats object. - **/ -function getCharStats() { - return charStats; -} - -/** - * Sets the current charStats object. - * @param {Object} stats - The new charStats object. - **/ -function setCharStats(stats) { - charStats = stats; - charStats.timestamp = Date.now(); -} - -/** - * Calculates the total generation time and word count for a chat with a character. - * - * @param {string} char_dir - The directory path where character chat files are stored. - * @param {string} chat - The name of the chat file. - * @returns {Object} - An object containing the total generation time, user word count, and non-user word count. - * @throws Will throw an error if the file cannot be read or parsed. - */ -function calculateTotalGenTimeAndWordCount( - char_dir, - chat, - uniqueGenStartTimes, -) { - let filepath = path.join(char_dir, chat); - let lines = readAndParseFile(filepath); - - let totalGenTime = 0; - let userWordCount = 0; - let nonUserWordCount = 0; - let nonUserMsgCount = 0; - let userMsgCount = 0; - let totalSwipeCount = 0; - let firstChatTime = new Date('9999-12-31T23:59:59.999Z').getTime(); - - for (let line of lines) { - if (line.length) { - try { - let json = JSON.parse(line); - if (json.mes) { - let hash = crypto - .createHash('sha256') - .update(json.mes) - .digest('hex'); - if (uniqueGenStartTimes.has(hash)) { - continue; - } - if (hash) { - uniqueGenStartTimes.add(hash); - } - } - - if (json.gen_started && json.gen_finished) { - let genTime = calculateGenTime( - json.gen_started, - json.gen_finished, - ); - totalGenTime += genTime; - - if (json.swipes && !json.swipe_info) { - // If there are swipes but no swipe_info, estimate the genTime - totalGenTime += genTime * json.swipes.length; - } - } - - if (json.mes) { - let wordCount = countWordsInString(json.mes); - json.is_user - ? (userWordCount += wordCount) - : (nonUserWordCount += wordCount); - json.is_user ? userMsgCount++ : nonUserMsgCount++; - } - - if (json.swipes && json.swipes.length > 1) { - totalSwipeCount += json.swipes.length - 1; // Subtract 1 to not count the first swipe - for (let i = 1; i < json.swipes.length; i++) { - // Start from the second swipe - let swipeText = json.swipes[i]; - - let wordCount = countWordsInString(swipeText); - json.is_user - ? (userWordCount += wordCount) - : (nonUserWordCount += wordCount); - json.is_user ? userMsgCount++ : nonUserMsgCount++; - } - } - - if (json.swipe_info && json.swipe_info.length > 1) { - for (let i = 1; i < json.swipe_info.length; i++) { - // Start from the second swipe - let swipe = json.swipe_info[i]; - if (swipe.gen_started && swipe.gen_finished) { - totalGenTime += calculateGenTime( - swipe.gen_started, - swipe.gen_finished, - ); - } - } - } - - // If this is the first user message, set the first chat time - if (json.is_user) { - //get min between firstChatTime and timestampToMoment(json.send_date) - firstChatTime = Math.min(timestampToMoment(json.send_date), firstChatTime); - } - } catch (error) { - console.error(`Error parsing line ${line}: ${error}`); - } - } - } +function newCharacterStats(characterKey = '', charName = '') { return { - totalGenTime, - userWordCount, - nonUserWordCount, - userMsgCount, - nonUserMsgCount, - totalSwipeCount, - firstChatTime, + name: charName, + characterKey: characterKey, + chats: 0, + chatSize: 0, + + firstCreateDate: MIN_DATE, + lastCreateDate: MIN_DATE, + firstlastInteractionDate: MIN_DATE, + lastLastInteractionDate: MIN_DATE, + + chattingTime: new AggregateStat(), + messages: new AggregateStat(), + systemMessages: new AggregateStat(), + userMessages: new AggregateStat(), + charMessages: new AggregateStat(), + + genTime: new AggregateStat(), + genTokenCount: new AggregateStat(), + swipeGenTime: new AggregateStat(), + swipes: new AggregateStat(), + userResponseTime: new AggregateStat(), + words: new AggregateStat(), + userWords: new AggregateStat(), + charWords: new AggregateStat(), + + perMessageGenTime: new AggregateStat(), + perMessageGenTokenCount: new AggregateStat(), + perMessageSwipeGenTime: new AggregateStat(), + perMessageSwipeCount: new AggregateStat(), + perMessageUserResponseTime: new AggregateStat(), + perMessageWords: new AggregateStat(), + perMessageUserWords: new AggregateStat(), + perMessageCharWords: new AggregateStat(), + + genModels: {}, + chatsStats: [], + _calculated: now(), }; } +/** + * Creates a new, empty chat stats object + * @param {string} chatName - The chats' name + * @returns {ChatStats} + */ +function newChatStats(chatName) { + return { + chatName: chatName, + chatId: 0, + chatSize: 0, + createDate: MIN_DATE, + lastInteractionDate: MIN_DATE, + + chattingTime: 0, + messages: 0, + systemMessages: 0, + userMessages: 0, + charMessages: 0, + + genTime: new AggregateStat(), + genTokenCount: new AggregateStat(), + swipeGenTime: new AggregateStat(), + swipes: new AggregateStat(), + userResponseTime: new AggregateStat(), + words: new AggregateStat(), + userWords: new AggregateStat(), + charWords: new AggregateStat(), + + genModels: {}, + messagesStats: [], + _calculated: now(), + }; +} + +/** + * Creates a new, empty message stats object + * @returns {MessageStats} + */ +function newMessageStats() { + return { + isUser: false, + isChar: false, + hash: '', + sendDate: MIN_DATE, + genTime: null, + genTokenCount: null, + swipeGenTime: null, + swipes: null, + words: 0, + genEndDates: [], + genModels: {}, + _calculated: now(), + }; +} + +/** + * Saves the current state of charStats to a file, only if the data has changed since the last save. + */ +async function saveStatsToFile() { + if (globalStats._calculated > lastSaveDate) { + //console.debug("Saving stats to file..."); + try { + await writeFileAtomic(statsFilePath, JSON.stringify(globalStats)); + lastSaveDate = now(); + } catch (error) { + console.log('Failed to save stats to file.', error); + } + } else { + //console.debug('Stats have not changed since last save. Skipping file write.'); + } +} + +/** + * Returns the current global stats object + * @returns {StatsCollection} + **/ +function getGlobalStats() { + return globalStats; +} + const router = express.Router(); /** @@ -425,7 +713,7 @@ const router = express.Router(); * @returns {void} */ router.post('/get', jsonParser, function (request, response) { - response.send(JSON.stringify(getCharStats())); + response.send(JSON.stringify(getGlobalStats())); }); /** @@ -438,7 +726,7 @@ router.post('/get', jsonParser, function (request, response) { */ router.post('/recreate', jsonParser, async function (request, response) { try { - await recreateStats(DIRECTORIES.chats, DIRECTORIES.characters); + await recreateStats(); return response.sendStatus(200); } catch (error) { console.error(error); @@ -446,7 +734,6 @@ router.post('/recreate', jsonParser, async function (request, response) { } }); - /** * Handle a POST request to update the stats object * diff --git a/src/util.js b/src/util.js index e23acb689..e97cf05f0 100644 --- a/src/util.js +++ b/src/util.js @@ -227,6 +227,28 @@ async function getImageBuffers(zipFilePath) { }); } +/** + * Reads the contents of a .jsonl file and returns the lines parsed as json as an array + * + * @param {string} filepath - The path of the file to be read + * @returns {object[]} - The lines in the file + * @throws Will throw an error if the file cannot be read + */ +function readAndParseJsonlFile(filepath) { + try { + // Copied from /chat/get endpoint + const data = fs.readFileSync(filepath, 'utf8'); + const lines = data.split('\n'); + + // Iterate through the array of strings and parse each line as JSON + const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x); + return jsonData; + } catch (error) { + console.error(`Error reading file at ${filepath}: ${error}`); + return []; + } +} + /** * Gets all chunks of data from the given readable stream. * @param {any} readableStream Readable stream to read from @@ -537,6 +559,162 @@ function trimV1(str) { return String(str ?? '').replace(/\/$/, '').replace(/\/v1$/, ''); } +/** + * Convert a timestamp to an integer timestamp. + * (sorry, it's momentless for now, didn't want to add a package just for this) + * This function can handle several different timestamp formats: + * 1. Unix timestamps (the number of seconds since the Unix Epoch) + * 2. ST "humanized" timestamps, formatted like "YYYY-MM-DD @HHh MMm SSs ms" + * 3. Date strings in the format "Month DD, YYYY H:MMam/pm" + * + * The function returns the timestamp as the number of milliseconds since + * the Unix Epoch, which can be converted to a JavaScript Date object with new Date(). + * + * @param {string|number} timestamp - The timestamp to convert. + * @returns {number|null} The timestamp in milliseconds since the Unix Epoch, or null if the input cannot be parsed. + * + * @example + * // Unix timestamp + * timestampToMoment(1609459200); + * // ST humanized timestamp + * timestampToMoment("2021-01-01 @00h 00m 00s 000ms"); + * // Date string + * timestampToMoment("January 1, 2021 12:00am"); + */ +function timestampToMoment(timestamp) { + if (!timestamp) { + return null; + } + + if (typeof timestamp === 'number') { + return timestamp; + } + + const pattern1 = + /(\d{4})-(\d{1,2})-(\d{1,2}) ?@(\d{1,2})h ?(\d{1,2})m ?(\d{1,2})s ?(?:(\d{1,3})ms)?/; + const replacement1 = ( + match, + year, + month, + day, + hour, + minute, + second, + millisecond, + ) => { + return `${year}-${month.padStart(2, '0')}-${day.padStart( + 2, + '0', + )}T${hour.padStart(2, '0')}:${minute.padStart( + 2, + '0', + )}:${second.padStart(2, '0')}.${(millisecond ?? '0').padStart(3, '0')}Z`; + }; + const isoTimestamp1 = timestamp.replace(pattern1, replacement1); + if (!isNaN(Date.parse(isoTimestamp1))) { + return new Date(isoTimestamp1).getTime(); + } + + const pattern2 = /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i; + const replacement2 = (match, month, day, year, hour, minute, meridiem) => { + const monthNames = [ + 'January', + 'February', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December', + ]; + const monthNum = monthNames.indexOf(month) + 1; + const hour24 = + meridiem.toLowerCase() === 'pm' + ? (parseInt(hour, 10) % 12) + 12 + : parseInt(hour, 10) % 12; + return `${year}-${monthNum.toString().padStart(2, '0')}-${day.padStart( + 2, + '0', + )}T${hour24.toString().padStart(2, '0')}:${minute.padStart( + 2, + '0', + )}:00Z`; + }; + const isoTimestamp2 = timestamp.replace(pattern2, replacement2); + if (!isNaN(Date.parse(isoTimestamp2))) { + return new Date(isoTimestamp2).getTime(); + } + + return null; +} + +/** + * Calculates the time difference between two dates. + * + * @param {Date|string} startTime - The start time in ISO 8601 format, or as a Date object + * @param {Date|string} endTime - The finish time in ISO 8601 format, or as a Date object + * @returns {number} - The difference in time in milliseconds, or 0 if invalid dates are provided or if start date is after end date. + */ +function calculateDuration(startTime, endTime) { + const startDate = startTime instanceof Date ? startTime : new Date(startTime); + const endDate = endTime instanceof Date ? endTime : new Date(endTime); + + return startDate > endDate ? 0 : Math.max(endDate.getDate() - startDate.getDate(), 0); +} + +/** @param {string} timestamp @returns {Date|null} */ +function humanizedToDate(timestamp) { + const moment = timestampToMoment(timestamp); + return moment ? new Date(moment) : null; +} + +/** + * Returns the maximum from all supplied dates + * @param {...Date} dates + * @returns {Date?} + */ +function maxDate(...dates) { + dates = dates.flat(Infinity); + if (dates.length == 0) return null; + /** @type {Date?} */ + let max = null; + for (const date of dates) { + if (max === null || date > max) { + max = date; + } + } + return max; +} + +/** + * Returns the minimum from all supplied dates + * @param {...Date} dates + * @returns {Date?} + */ +function minDate(...dates) { + dates = dates.flat(Infinity); + if (dates.length == 0) return null; + /** @type {Date?} */ + let min = null; + for (const date of dates) { + if (min === null || date < min) { + min = date; + } + } + return min; +} + +/** + * Returns the current date and time + * (Why the fuck is `Date.now()` returning a timestamp and no date... I don't get it) + * @returns {Date} + */ +function now() { return new Date(Date.now()); } + /** * Simple TTL memory cache. */ @@ -599,6 +777,7 @@ module.exports = { getBasicAuthHeader, extractFileFromZipBuffer, getImageBuffers, + readAndParseJsonlFile, readAllChunks, delay, deepMerge, @@ -616,6 +795,12 @@ module.exports = { mergeObjectWithYaml, excludeKeysByYaml, trimV1, + timestampToMoment, + calculateDuration, + humanizedToDate, + maxDate, + minDate, + now, Cache, makeHttp2Request, };