rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #615 from BlipRanger/feature/stats

Browse Source 
Feature/stats
This commit is contained in:
Cohee
2023-07-19 17:40:03 +03:00
committed by GitHub
parent f63a69dd41 284fb68478
commit 7cf02beaa6
9 changed files with 937 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -10,6 +10,7 @@ public/css/bg_load.css
public/themes/
public/OpenAI Settings/
public/scripts/extensions/third-party/
public/stats.json
/uploads/
*.jsonl
config.conf

4
public/index.html
View File

@@ -2634,6 +2634,9 @@
<input type="file" id="avatar_upload_file" accept="image/*" name="avatar">
<input type="hidden" id="avatar_upload_overwrite" name="overwrite_name" value="">
</form>
<button class="menu_button user_stats_button" title="Click for stats!">
<i class="fa-solid fa-circle-info"></i>Usage Stats
</button>
</div>
</div>
</div>
@@ -2672,7 +2675,6 @@
<div id="rm_button_selected_ch">
<h2></h2>
</div>
</div>
</div>
<!-- end group peeking cope structure-->

31
public/script.js
View File

@@ -1,4 +1,5 @@
import { humanizedDateTime, favsToHotswap, getMessageTimeStamp, dragElement, isMobile } from "./scripts/RossAscends-mods.js";
import { humanizedDateTime, favsToHotswap, getMessageTimeStamp, dragElement, isMobile, } from "./scripts/RossAscends-mods.js";
import { userStatsHandler, statMesProcess } from './scripts/stats.js';
import { encode } from "../scripts/gpt-2-3-tokenizer/mod.js";
import { GPT3BrowserTokenizer } from "../scripts/gpt-3-tokenizer/gpt3-tokenizer.js";
import {
@@ -326,6 +327,7 @@ let is_delete_mode = false;
let fav_ch_checked = false;
let scrollLock = false;
//initialize global var for future cropped blobs
let currentCroppedAvatar = '';
@@ -949,8 +951,11 @@ async function getCharacters() {
if (this_chid != undefined && this_chid != "invalid-safety-id") {
$("#avatar_url_pole").val(characters[this_chid].avatar);
}
await getGroups();
await printCharacters();
updateCharacterCount('#rm_print_characters_block > div');
}
}
@@ -1310,6 +1315,8 @@ function addOneMessage(mes, { type = "normal", insertAfter = null, scroll = true
const timestamp = momentDate.isValid() ? momentDate.format('LL LT') : '';
if (mes?.extra?.display_text) {
messageText = mes.extra.display_text;
}
@@ -3045,7 +3052,7 @@ export async function sendMessageAsUser(textareaText, messageBias) {
console.debug('checking bias');
chat[chat.length - 1]['extra']['bias'] = messageBias;
}
statMesProcess(chat[chat.length - 1], 'user', characters, this_chid, '');
addOneMessage(chat[chat.length - 1]);
// Wait for all handlers to finish before continuing with the prompt
await eventSource.emit(event_types.MESSAGE_SENT, (chat.length - 1));
@@ -3730,16 +3737,21 @@ function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete
return getMessage;
}
function saveReply(type, getMessage, this_mes_is_name, title) {
if (type != 'append' && type != 'continue' && type != 'appendFinal' && chat.length && (chat[chat.length - 1]['swipe_id'] === undefined ||
chat[chat.length - 1]['is_user'])) {
type = 'normal';
}
let oldMessage = ''
const generationFinished = new Date();
const img = extractImageFromMessage(getMessage);
getMessage = img.getMessage;
if (type === 'swipe') {
oldMessage = chat[chat.length - 1]['mes'];
chat[chat.length - 1]['swipes'].length++;
if (chat[chat.length - 1]['swipe_id'] === chat[chat.length - 1]['swipes'].length - 1) {
chat[chat.length - 1]['title'] = title;
@@ -3755,6 +3767,7 @@ function saveReply(type, getMessage, this_mes_is_name, title) {
}
} else if (type === 'append' || type === 'continue') {
console.debug("Trying to append.")
oldMessage = chat[chat.length - 1]['mes'];
chat[chat.length - 1]['title'] = title;
chat[chat.length - 1]['mes'] += getMessage;
chat[chat.length - 1]['gen_started'] = generation_started;
@@ -3764,6 +3777,7 @@ function saveReply(type, getMessage, this_mes_is_name, title) {
chat[chat.length - 1]["extra"]["model"] = getGeneratingModel();
addOneMessage(chat[chat.length - 1], { type: 'swipe' });
} else if (type === 'appendFinal') {
oldMessage = chat[chat.length - 1]['mes'];
console.debug("Trying to appendFinal.")
chat[chat.length - 1]['title'] = title;
chat[chat.length - 1]['mes'] = getMessage;
@@ -3831,6 +3845,7 @@ function saveReply(type, getMessage, this_mes_is_name, title) {
extra: JSON.parse(JSON.stringify(chat[chat.length - 1]["extra"])),
};
}
statMesProcess(chat[chat.length - 1], type, characters, this_chid, oldMessage);
return { type, getMessage };
}
@@ -4205,12 +4220,15 @@ async function getChat() {
chat.push(...response);
chat_create_date = chat[0]['create_date'];
chat_metadata = chat[0]['chat_metadata'] ?? {};
chat.shift();
} else {
chat_create_date = humanizedDateTime();
}
await getChatResult();
await saveChat();
setTimeout(function () {
$('#send_textarea').click();
$('#send_textarea').focus();
@@ -4866,7 +4884,6 @@ async function getSettings(type) {
}
const data = await response.json();
if (data.result != "file not find" && data.settings) {
settings = JSON.parse(data.settings);
if (settings.username !== undefined) {
@@ -5092,6 +5109,8 @@ async function saveSettings(type) {
});
}
function setCharacterBlockHeight() {
const $children = $("#rm_print_characters_block").children();
const originalHeight = $children.length * $children.find(':visible').first().outerHeight();
@@ -8582,6 +8601,10 @@ $(document).ready(function () {
restoreCaretPosition($(this).get(0), caretPosition);
});
$(".user_stats_button").on('click', function () {
userStatsHandler();
});
$('#external_import_button').on('click', async () => {
const html = `<h3>Enter the URL of the content to import</h3>
Supported sources:<br>
@@ -8720,4 +8743,4 @@ $(document).ready(function () {
$("#charListGridToggle").on('click', async () => {
doCharListDisplaySwitch();
});
})
});

47
public/scripts/RossAscends-mods.js
View File

@@ -15,6 +15,10 @@ import {
saveSettingsDebounced,
} from "../script.js";
import {
characterStatsHandler,
} from "./stats.js";
import {
power_user,
@@ -104,6 +108,37 @@ function waitForElement(querySelector, timeout) {
});
}
/**
* Converts generation time from milliseconds to a human-readable format.
*
* The function takes total generation time as an input, then converts it to a format
* of "_ Days, _ Hours, _ Minutes, _ Seconds". If the generation time does not exceed a
* particular measure (like days or hours), that measure will not be included in the output.
*
* @param {number} total_gen_time - The total generation time in milliseconds.
* @returns {string} - A human-readable string that represents the time spent generating characters.
*/
export function humanizeGenTime(total_gen_time) {
//convert time_spent to humanized format of "_ Hours, _ Minutes, _ Seconds" from milliseconds
let time_spent = total_gen_time || 0;
time_spent = Math.floor(time_spent / 1000);
let seconds = time_spent % 60;
time_spent = Math.floor(time_spent / 60);
let minutes = time_spent % 60;
time_spent = Math.floor(time_spent / 60);
let hours = time_spent % 24;
time_spent = Math.floor(time_spent / 24);
let days = time_spent;
time_spent = "";
if (days > 0) { time_spent += `${days} Days, `; }
if (hours > 0) { time_spent += `${hours} Hours, `; }
if (minutes > 0) { time_spent += `${minutes} Minutes, `; }
time_spent += `${seconds} Seconds`;
return time_spent;
}
// Device detection
export const deviceInfo = await getDeviceInfo();
@@ -270,10 +305,14 @@ export function RA_CountCharTokens() {
// if neither, probably safety char or some error in loading
} else { console.debug("RA_TC -- no valid char found, closing."); }
}
//label rm_stats_button with a tooltip indicating stats
$("#result_info").html(`<small>${count_tokens} Tokens (${perm_tokens} Permanent)</small>
<i title='Click for stats!' class="fa-solid fa-circle-info rm_stats_button"></i>`);
// display the counted tokens
const tokenLimit = Math.max(((main_api !== 'openai' ? max_context : oai_settings.openai_max_context) / 2), 1024);
if (count_tokens < tokenLimit && perm_tokens < tokenLimit) {
$("#result_info").html(`<small>${count_tokens} Tokens (${perm_tokens} Permanent)</small>`);
} else {
$("#result_info").html(`
<div class="flex-container alignitemscenter">
@@ -281,10 +320,16 @@ export function RA_CountCharTokens() {
<small class="flex-container flexnowrap flexNoGap">
<div class="neutral_warning">${count_tokens}</div>&nbsp;Tokens (<div class="neutral_warning">${perm_tokens}</div><div>&nbsp;Permanent)</div>
</small>
<i title='Click for stats!' class="fa-solid fa-circle-info rm_stats_button"></i>
</div>
<div id="chartokenwarning" class="menu_button margin0 whitespacenowrap"><a href="https://docs.sillytavern.app/usage/core-concepts/characterdesign/#character-tokens" target="_blank">About Token 'Limits'</a></div>
</div>`);
} //warn if either are over 1024
$(".rm_stats_button").on('click', function () {
characterStatsHandler(characters, this_chid);
});
}
//Auto Load Last Charcter -- (fires when active_character is defined and auto_load_chat is true)
async function RA_autoloadchat() {

1
public/scripts/power-user.js
View File

@@ -763,7 +763,6 @@ function loadPowerUserSettings(settings, data) {
loadMovingUIState();
loadCharListState();
//console.log(power_user)
}
async function loadCharListState() {

294
public/scripts/stats.js Normal file
View File

@@ -0,0 +1,294 @@
// statsHelper.js
import { getRequestHeaders, callPopup, token, chat } from "../script.js";
import { humanizeGenTime } from "./RossAscends-mods.js";
let charStats = {};
/**
* Creates an HTML stat block.
*
* @param {string} statName - The name of the stat to be displayed.
* @param {number|string} statValue - The value of the stat to be displayed.
* @returns {string} - An HTML string representing the stat block.
*/
function createStatBlock(statName, statValue) {
return `<div class="rm_stat_block">
<div class="rm_stat_name">${statName}:</div>
<div class="rm_stat_value">${statValue}</div>
</div>`;
}
/**
* Verifies and returns a numerical stat value. If the provided stat is not a number, returns 0.
*
* @param {number|string} stat - The stat value to be checked and returned.
* @returns {number} - The stat value if it is a number, otherwise 0.
*/
function verifyStatValue(stat) {
return isNaN(stat) ? 0 : stat;
}
/**
* Calculates total stats from character statistics.
*
* @param {Object} charStats - Object containing character statistics.
* @returns {Object} - Object containing total statistics.
*/
function calculateTotalStats() {
let totalStats = {
total_gen_time: 0,
user_msg_count: 0,
non_user_msg_count: 0,
user_word_count: 0,
non_user_word_count: 0,
total_swipe_count: 0,
date_last_chat: 0,
date_first_chat: new Date("9999-12-31T23:59:59.999Z").getTime(),
};
for (let stats of Object.values(charStats)) {
totalStats.total_gen_time += verifyStatValue(stats.total_gen_time);
totalStats.user_msg_count += verifyStatValue(stats.user_msg_count);
totalStats.non_user_msg_count += verifyStatValue(
stats.non_user_msg_count
);
totalStats.user_word_count += verifyStatValue(stats.user_word_count);
totalStats.non_user_word_count += verifyStatValue(
stats.non_user_word_count
);
totalStats.total_swipe_count += verifyStatValue(
stats.total_swipe_count
);
if (verifyStatValue(stats.date_last_chat) != 0) {
totalStats.date_last_chat = Math.max(
totalStats.date_last_chat,
stats.date_last_chat
);
}
if (verifyStatValue(stats.date_first_chat) != 0) {
totalStats.date_first_chat = Math.min(
totalStats.date_first_chat,
stats.date_first_chat
);
}
}
return totalStats;
}
/**
* Generates an HTML report of stats.
*
* This function creates an HTML report from the provided stats, including chat age,
* chat time, number of user messages and character messages, word count, and swipe count.
* The stat blocks are tailored depending on the stats type ("User" or "Character").
*
* @param {string} statsType - The type of stats (e.g., "User", "Character").
* @param {Object} stats - The stats data. Expected keys in this object include:
* total_gen_time - total generation time
* date_first_chat - timestamp of the first chat
* date_last_chat - timestamp of the most recent chat
* user_msg_count - count of user messages
* non_user_msg_count - count of non-user messages
* user_word_count - count of words used by the user
* non_user_word_count - count of words used by the non-user
* total_swipe_count - total swipe count
*/
function createHtml(statsType, stats) {
// Get time string
let timeStirng = humanizeGenTime(stats.total_gen_time);
let chatAge = "Never";
if (stats.date_first_chat < Date.now()) {
chatAge = moment
.duration(stats.date_last_chat - stats.date_first_chat)
.humanize();
}
// Create popup HTML with stats
let html = `<h3>${statsType} Stats</h3>`;
if (statsType === "User") {
html += createStatBlock("Chatting Since", `${chatAge} ago`);
} else {
html += createStatBlock("First Interaction", `${chatAge} ago`);
}
html += createStatBlock("Chat Time", timeStirng);
html += createStatBlock("User Messages", stats.user_msg_count);
html += createStatBlock(
"Character Messages",
stats.non_user_msg_count - stats.total_swipe_count
);
html += createStatBlock("User Words", stats.user_word_count);
html += createStatBlock("Character Words", stats.non_user_word_count);
html += createStatBlock("Swipes", stats.total_swipe_count);
callPopup(html, "text");
}
/**
* Handles the user stats by getting them from the server, calculating the total and generating the HTML report.
*
* @param {Object} charStats - Object containing character statistics.
*/
async function userStatsHandler() {
// Get stats from server
await getStats();
// Calculate total stats
let totalStats = calculateTotalStats(charStats);
// Create HTML with stats
createHtml("User", totalStats);
}
/**
* Handles the character stats by getting them from the server and generating the HTML report.
*
* @param {Object} charStats - Object containing character statistics.
* @param {Object} characters - Object containing character data.
* @param {string} this_chid - The character id.
*/
async function characterStatsHandler(characters, this_chid) {
// Get stats from server
await getStats();
// Get character stats
let myStats = charStats[characters[this_chid].avatar];
if (myStats === undefined) {
myStats = {
total_gen_time: 0,
user_msg_count: 0,
non_user_msg_count: 0,
user_word_count: 0,
non_user_word_count: countWords(characters[this_chid].first_mes),
total_swipe_count: 0,
date_last_chat: 0,
date_first_chat: new Date("9999-12-31T23:59:59.999Z").getTime(),
};
charStats[characters[this_chid].avatar] = myStats;
updateStats();
}
// Create HTML with stats
createHtml("Character", myStats);
}
/**
* Fetches the character stats from the server.
*
* @param {Object} charStats - Object containing character statistics.
* @returns {Object} - Object containing fetched character statistics.
*/
async function getStats() {
const response = await fetch("/getstats", {
method: "POST",
headers: getRequestHeaders(),
body: JSON.stringify({}),
cache: "no-cache",
});
if (!response.ok) {
toastr.error("Stats could not be loaded. Try reloading the page.");
throw new Error("Error getting stats");
}
charStats = await response.json();
}
/**
* Calculates the generation time based on start and finish times.
*
* @param {string} gen_started - The start time in ISO 8601 format.
* @param {string} gen_finished - The finish time in ISO 8601 format.
* @returns {number} - The difference in time in milliseconds.
*/
function calculateGenTime(gen_started, gen_finished) {
if (gen_started === undefined || gen_finished === undefined) {
return 0;
}
let startDate = new Date(gen_started);
let endDate = new Date(gen_finished);
return endDate - startDate;
}
/**
* Sends a POST request to the server to update the statistics.
*
* @param {Object} stats - The stats data to update.
*/
async function updateStats() {
const response = await fetch("/updatestats", {
method: "POST",
headers: getRequestHeaders(),
body: JSON.stringify(charStats),
});
if (response.status !== 200) {
console.error("Failed to update stats");
console.log(response).status;
}
}
/**
* Returns the count of words in the given string.
* A word is a sequence of alphanumeric characters (including underscore).
*
* @param {string} str - The string to count words in.
* @returns {number} - Number of words.
*/
function countWords(str) {
const match = str.match(/\b\w+\b/g);
return match ? match.length : 0;
}
/**
* Handles stat processing for messages.
*
* @param {Object} line - Object containing message data.
* @param {string} type - The type of the message processing (e.g., 'append', 'continue', 'appendFinal', 'swipe').
* @param {Object} characters - Object containing character data.
* @param {string} this_chid - The character id.
* @param {Object} charStats - Object containing character statistics.
* @param {string} oldMesssage - The old message that's being processed.
*/
async function statMesProcess(line, type, characters, this_chid, oldMesssage) {
if (this_chid === undefined) {
return;
}
await getStats();
let stat = charStats[characters[this_chid].avatar];
stat.total_gen_time += calculateGenTime(
line.gen_started,
line.gen_finished
);
if (line.is_user) {
if (type != "append" && type != "continue" && type != "appendFinal") {
stat.user_msg_count++;
stat.user_word_count += countWords(line.mes);
} else {
let oldLen = oldMesssage.split(" ").length;
stat.user_word_count += countWords(line.mes) - oldLen;
}
} else {
// if continue, don't add a message, get the last message and subtract it from the word count of
// the new message
if (type != "append" && type != "continue" && type != "appendFinal") {
stat.non_user_msg_count++;
stat.non_user_word_count += countWords(line.mes);
} else {
let oldLen = oldMesssage.split(" ").length;
stat.non_user_word_count += countWords(line.mes) - oldLen;
}
}
if (type === "swipe") {
stat.total_swipe_count++;
}
stat.date_last_chat = Date.now();
stat.date_first_chat = Math.min(
stat.date_first_chat ?? new Date("9999-12-31T23:59:59.999Z").getTime(),
Date.now()
);
updateStats();
}
export { userStatsHandler, characterStatsHandler, getStats, statMesProcess, charStats };

11
public/style.css
View File

@@ -1952,6 +1952,17 @@ grammarly-extension {
overflow-y: hidden;
}
.rm_stat_block {
display: flex;
justify-content: space-between;
}
.user_stats_button {
position: absolute;
right: 0;
bottom: 0;
}
.large_dialogue_popup {
height: 90vh !important;
height: 90svh !important;

183
server.js
View File

@@ -67,6 +67,9 @@ const { TextEncoder, TextDecoder } = require('util');
const utf8Encode = new TextEncoder();
const commandExistsSync = require('command-exists').sync;
// impoort from statsHelpers.js
const statsHelpers = require('./statsHelpers.js');
const characterCardParser = require('./src/character-card-parser.js');
const config = require(path.join(process.cwd(), './config.conf'));
@@ -117,6 +120,7 @@ let response_generate_novel;
let characters = {};
let response_dw_bg;
let response_getstatus;
let first_run = true;
//RossAscends: Added function to format dates used in files and chat timestamps to a humanized format.
@@ -767,6 +771,7 @@ function convertToV2(char) {
return result;
}
function unsetFavFlag(char) {
const _ = require('lodash');
_.set(char, 'fav', false);
@@ -1172,6 +1177,84 @@ async function charaRead(img_url, input_format) {
return characterCardParser.parse(img_url, input_format);
}
/**
* calculateChatSize - Calculates the total chat size for a given character.
*
* @param {string} charDir The directory where the chats are stored.
* @return {number} The total chat size.
*/
const calculateChatSize = (charDir) => {
let chatSize = 0;
let dateLastChat = 0;
if (fs.existsSync(charDir)) {
const chats = fs.readdirSync(charDir);
if (Array.isArray(chats) && chats.length) {
for (const chat of chats) {
const chatStat = fs.statSync(path.join(charDir, chat));
chatSize += chatStat.size;
dateLastChat = Math.max(dateLastChat, chatStat.mtimeMs);
}
}
}
return { chatSize, dateLastChat };
}
/**
* processCharacter - Process a given character, read its data and calculate its statistics.
*
* @param {string} item The name of the character.
* @param {number} i The index of the character in the characters list.
* @return {Promise} A Promise that resolves when the character processing is done.
*/
const processCharacter = async (item, i) => {
try {
const img_data = await charaRead(charactersPath + item);
let jsonObject = getCharaCardV2(json5.parse(img_data));
jsonObject.avatar = item;
characters[i] = jsonObject;
characters[i]['json_data'] = img_data;
const charStat = fs.statSync(path.join(charactersPath, item));
characters[i]['date_added'] = charStat.birthtimeMs;
const char_dir = path.join(chatsPath, item.replace('.png', ''));
const { chatSize, dateLastChat } = calculateChatSize(char_dir);
characters[i]['chat_size'] = chatSize;
characters[i]['date_last_chat'] = dateLastChat;
}
catch (err) {
characters[i] = {
date_added: 0,
date_last_chat: 0,
chat_size: 0
};
console.log(`Could not process character: ${item}`);
if (err instanceof SyntaxError) {
console.log("String [" + i + "] is not valid JSON!");
} else {
console.log("An unexpected error occurred: ", err);
}
}
}
/**
* HTTP POST endpoint for the "/getcharacters" route.
*
* This endpoint is responsible for reading character files from the `charactersPath` directory,
* parsing character data, calculating stats for each character and responding with the data.
* Stats are calculated only on the first run, on subsequent runs the stats are fetched from
* the `charStats` variable.
* The stats are calculated by the `calculateStats` function.
* The characters are processed by the `processCharacter` function.
*
* @param {object} request The HTTP request object.
* @param {object} response The HTTP response object.
* @return {undefined} Does not return a value.
*/
app.post("/getcharacters", jsonParser, function (request, response) {
fs.readdir(charactersPath, async (err, files) => {
if (err) {
@@ -1180,63 +1263,49 @@ app.post("/getcharacters", jsonParser, function (request, response) {
}
const pngFiles = files.filter(file => file.endsWith('.png'));
//console.log(pngFiles);
characters = {};
var i = 0;
for (const item of pngFiles) {
try {
var img_data = await charaRead(charactersPath + item);
let jsonObject = getCharaCardV2(json5.parse(img_data));
jsonObject.avatar = item;
characters[i] = {};
characters[i] = jsonObject;
characters[i]['json_data'] = img_data;
try {
const charStat = fs.statSync(path.join(charactersPath, item));
characters[i]['date_added'] = charStat.birthtimeMs;
const char_dir = path.join(chatsPath, item.replace('.png', ''));
let processingPromises = pngFiles.map((file, index) => processCharacter(file, index));
await Promise.all(processingPromises); performance.mark('B');
let chat_size = 0;
let date_last_chat = 0;
if (fs.existsSync(char_dir)) {
const chats = fs.readdirSync(char_dir);
if (Array.isArray(chats) && chats.length) {
for (const chat of chats) {
const chatStat = fs.statSync(path.join(char_dir, chat));
chat_size += chatStat.size;
date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
}
}
}
characters[i]['date_last_chat'] = date_last_chat;
characters[i]['chat_size'] = chat_size;
}
catch {
characters[i]['date_added'] = 0;
characters[i]['date_last_chat'] = 0;
characters[i]['chat_size'] = 0;
}
i++;
} catch (error) {
console.log(`Could not read character: ${item}`);
if (error instanceof SyntaxError) {
console.log("String [" + (i) + "] is not valid JSON!");
} else {
console.log("An unexpected error occurred: ", error);
}
}
};
//console.log(characters);
response.send(JSON.stringify(characters));
});
});
/**
* Handle a POST request to get the stats object
*
* This function returns the stats object that was calculated by the `calculateStats` function.
*
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*/
app.post("/getstats", jsonParser, function (request, response) {
response.send(JSON.stringify(statsHelpers.getCharStats()));
});
/**
* Handle a POST request to update the stats object
*
* This function updates the stats object with the data from the request body.
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*
*/
app.post("/updatestats", jsonParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
statsHelpers.setCharStats(request.body);
return response.sendStatus(200);
});
app.post("/getbackgrounds", jsonParser, function (request, response) {
var images = getImages("public/backgrounds");
response.send(JSON.stringify(images));
@@ -3607,6 +3676,18 @@ const setupTasks = async function () {
loadClaudeTokenizer('src/claude.json'),
]);
await statsHelpers.loadStatsFile(directories.chats, directories.characters);
// Set up event listeners for a graceful shutdown
process.on('SIGINT', statsHelpers.writeStatsToFileAndExit);
process.on('SIGTERM', statsHelpers.writeStatsToFileAndExit);
process.on('uncaughtException', (err) => {
console.error('Uncaught exception:', err);
statsHelpers.writeStatsToFileAndExit();
});
setInterval(statsHelpers.saveStatsToFile, 5 * 60 * 1000);
console.log('Launching...');
if (autorun) open(autorunUrl.toString());

422
statsHelpers.js Normal file
View File

@@ -0,0 +1,422 @@
/**
* @fileoverview This file contains various utility functions related to
* character and user statistics, such as creating an HTML stat block,
* calculating total stats, and creating an HTML report from the provided stats.
* It also provides methods for handling user stats and character stats,
* as well as a utility for humanizing generation time from milliseconds.
*/
const fs = require("fs");
const path = require("path");
const util = require("util");
const writeFile = util.promisify(fs.writeFile);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const crypto = require("crypto");
let charStats = {};
let lastSaveTimestamp = 0;
const statsFilePath = "public/stats.json";
/**
* Convert a timestamp to an integer timestamp.
* (sorry, it's momentless for now, didn't want to add a package just for this)
* This function can handle several different timestamp formats:
* 1. Unix timestamps (the number of seconds since the Unix Epoch)
* 2. ST "humanized" timestamps, formatted like "YYYY-MM-DD @HHh MMm SSs ms"
* 3. Date strings in the format "Month DD, YYYY H:MMam/pm"
*
* The function returns the timestamp as the number of milliseconds since
* the Unix Epoch, which can be converted to a JavaScript Date object with new Date().
*
* @param {string|number} timestamp - The timestamp to convert.
* @returns {number|null} The timestamp in milliseconds since the Unix Epoch, or null if the input cannot be parsed.
*
* @example
* // Unix timestamp
* timestampToMoment(1609459200);
* // ST humanized timestamp
* timestampToMoment("2021-01-01 @00h 00m 00s 000ms");
* // Date string
* timestampToMoment("January 1, 2021 12:00am");
*/
function timestampToMoment(timestamp) {
if (!timestamp) {
return null;
}
if (typeof timestamp === "number") {
return timestamp;
}
const pattern1 =
/(\d{4})-(\d{1,2})-(\d{1,2}) @(\d{1,2})h (\d{1,2})m (\d{1,2})s (\d{1,3})ms/;
const replacement1 = (
match,
year,
month,
day,
hour,
minute,
second,
millisecond
) => {
return `${year}-${month.padStart(2, "0")}-${day.padStart(
2,
"0"
)}T${hour.padStart(2, "0")}:${minute.padStart(
2,
"0"
)}:${second.padStart(2, "0")}.${millisecond.padStart(3, "0")}Z`;
};
const isoTimestamp1 = timestamp.replace(pattern1, replacement1);
if (!isNaN(new Date(isoTimestamp1))) {
return new Date(isoTimestamp1).getTime();
}
const pattern2 = /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i;
const replacement2 = (match, month, day, year, hour, minute, meridiem) => {
const monthNames = [
"January",
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
];
const monthNum = monthNames.indexOf(month) + 1;
const hour24 =
meridiem.toLowerCase() === "pm"
? (parseInt(hour, 10) % 12) + 12
: parseInt(hour, 10) % 12;
return `${year}-${monthNum.toString().padStart(2, "0")}-${day.padStart(
2,
"0"
)}T${hour24.toString().padStart(2, "0")}:${minute.padStart(
2,
"0"
)}:00Z`;
};
const isoTimestamp2 = timestamp.replace(pattern2, replacement2);
if (!isNaN(new Date(isoTimestamp2))) {
return new Date(isoTimestamp2).getTime();
}
return null;
}
/**
* Collects and aggregates stats for all characters.
*
* @param {string} chatsPath - The path to the directory containing the chat files.
* @param {string} charactersPath - The path to the directory containing the character files.
* @returns {Object} The aggregated stats object.
*/
async function collectAndCreateStats(chatsPath, charactersPath) {
console.log("Collecting and creating stats...");
const files = await readdir(charactersPath);
const pngFiles = files.filter((file) => file.endsWith(".png"));
let processingPromises = pngFiles.map((file, index) =>
calculateStats(chatsPath, file, index)
);
const statsArr = await Promise.all(processingPromises);
let finalStats = {};
for (let stat of statsArr) {
finalStats = { ...finalStats, ...stat };
}
// tag with timestamp on when stats were generated
finalStats.timestamp = Date.now();
return finalStats;
}
/**
* Loads the stats file into memory. If the file doesn't exist or is invalid,
* initializes stats by collecting and creating them for each character.
*
* @param {string} chatsPath - The path to the directory containing the chat files.
* @param {string} charactersPath - The path to the directory containing the character files.
*/
async function loadStatsFile(chatsPath, charactersPath) {
try {
const statsFileContent = await readFile(statsFilePath, "utf-8");
charStats = JSON.parse(statsFileContent);
} catch (err) {
// If the file doesn't exist or is invalid, initialize stats
if (err.code === "ENOENT" || err instanceof SyntaxError) {
charStats = await collectAndCreateStats(chatsPath, charactersPath); // Call your function to collect and create stats
await saveStatsToFile();
} else {
throw err; // Rethrow the error if it's something we didn't expect
}
}
console.debug("Stats loaded from files.");
}
/**
* Saves the current state of charStats to a file, only if the data has changed since the last save.
*/
async function saveStatsToFile() {
if (charStats.timestamp > lastSaveTimestamp) {
console.debug("Saving stats to file...");
await writeFile(statsFilePath, JSON.stringify(charStats));
lastSaveTimestamp = Date.now();
} else {
//console.debug('Stats have not changed since last save. Skipping file write.');
}
}
/**
* Attempts to save charStats to a file and then terminates the process.
* If an error occurs during the file write, it logs the error before exiting.
*/
async function writeStatsToFileAndExit(charStats) {
try {
await saveStatsToFile(charStats);
} catch (err) {
console.error("Failed to write stats to file:", err);
} finally {
process.exit();
}
}
/**
* Reads the contents of a file and returns the lines in the file as an array.
*
* @param {string} filepath - The path of the file to be read.
* @returns {Array<string>} - The lines in the file.
* @throws Will throw an error if the file cannot be read.
*/
function readAndParseFile(filepath) {
try {
let file = fs.readFileSync(filepath, "utf8");
let lines = file.split("\n");
return lines;
} catch (error) {
console.error(`Error reading file at ${filepath}: ${error}`);
return [];
}
}
/**
* Calculates the time difference between two dates.
*
* @param {string} gen_started - The start time in ISO 8601 format.
* @param {string} gen_finished - The finish time in ISO 8601 format.
* @returns {number} - The difference in time in milliseconds.
*/
function calculateGenTime(gen_started, gen_finished) {
let startDate = new Date(gen_started);
let endDate = new Date(gen_finished);
return endDate - startDate;
}
/**
* Counts the number of words in a string.
*
* @param {string} str - The string to count words in.
* @returns {number} - The number of words in the string.
*/
function countWordsInString(str) {
const match = str.match(/\b\w+\b/g);
return match ? match.length : 0;
}
/**
* calculateStats - Calculate statistics for a given character chat directory.
*
* @param {string} char_dir The directory containing the chat files.
* @param {string} item The name of the character.
* @return {object} An object containing the calculated statistics.
*/
const calculateStats = (chatsPath, item, index) => {
const char_dir = path.join(chatsPath, item.replace(".png", ""));
let chat_size = 0;
let date_last_chat = 0;
const stats = {
total_gen_time: 0,
user_word_count: 0,
non_user_word_count: 0,
user_msg_count: 0,
non_user_msg_count: 0,
total_swipe_count: 0,
chat_size: 0,
date_last_chat: 0,
date_first_chat: new Date("9999-12-31T23:59:59.999Z").getTime(),
};
let uniqueGenStartTimes = new Set();
if (fs.existsSync(char_dir)) {
const chats = fs.readdirSync(char_dir);
if (Array.isArray(chats) && chats.length) {
for (const chat of chats) {
const result = calculateTotalGenTimeAndWordCount(
char_dir,
chat,
uniqueGenStartTimes
);
stats.total_gen_time += result.totalGenTime || 0;
stats.user_word_count += result.userWordCount || 0;
stats.non_user_word_count += result.nonUserWordCount || 0;
stats.user_msg_count += result.userMsgCount || 0;
stats.non_user_msg_count += result.nonUserMsgCount || 0;
stats.total_swipe_count += result.totalSwipeCount || 0;
const chatStat = fs.statSync(path.join(char_dir, chat));
stats.chat_size += chatStat.size;
stats.date_last_chat = Math.max(
stats.date_last_chat,
Math.floor(chatStat.mtimeMs)
);
stats.date_first_chat = Math.min(
stats.date_first_chat,
result.firstChatTime
);
}
}
}
return { [item]: stats };
};
/**
* Returns the current charStats object.
* @returns {Object} The current charStats object.
**/
function getCharStats() {
return charStats;
}
/**
* Sets the current charStats object.
* @param {Object} stats - The new charStats object.
**/
function setCharStats(stats) {
charStats = stats;
charStats.timestamp = Date.now();
}
/**
* Calculates the total generation time and word count for a chat with a character.
*
* @param {string} char_dir - The directory path where character chat files are stored.
* @param {string} chat - The name of the chat file.
* @returns {Object} - An object containing the total generation time, user word count, and non-user word count.
* @throws Will throw an error if the file cannot be read or parsed.
*/
function calculateTotalGenTimeAndWordCount(
char_dir,
chat,
uniqueGenStartTimes
) {
let filepath = path.join(char_dir, chat);
let lines = readAndParseFile(filepath);
let totalGenTime = 0;
let userWordCount = 0;
let nonUserWordCount = 0;
let nonUserMsgCount = 0;
let userMsgCount = 0;
let totalSwipeCount = 0;
let firstChatTime = new Date("9999-12-31T23:59:59.999Z").getTime();
for (let [index, line] of lines.entries()) {
if (line.length) {
try {
let json = JSON.parse(line);
if (json.mes) {
let hash = crypto
.createHash("sha256")
.update(json.mes)
.digest("hex");
if (uniqueGenStartTimes.has(hash)) {
continue;
}
if (hash) {
uniqueGenStartTimes.add(hash);
}
}
if (json.gen_started && json.gen_finished) {
let genTime = calculateGenTime(
json.gen_started,
json.gen_finished
);
totalGenTime += genTime;
if (json.swipes && !json.swipe_info) {
// If there are swipes but no swipe_info, estimate the genTime
totalGenTime += genTime * json.swipes.length;
}
}
if (json.mes) {
let wordCount = countWordsInString(json.mes);
json.is_user
? (userWordCount += wordCount)
: (nonUserWordCount += wordCount);
json.is_user ? userMsgCount++ : nonUserMsgCount++;
}
if (json.swipes && json.swipes.length > 1) {
totalSwipeCount += json.swipes.length - 1; // Subtract 1 to not count the first swipe
for (let i = 1; i < json.swipes.length; i++) {
// Start from the second swipe
let swipeText = json.swipes[i];
let wordCount = countWordsInString(swipeText);
json.is_user
? (userWordCount += wordCount)
: (nonUserWordCount += wordCount);
json.is_user ? userMsgCount++ : nonUserMsgCount++;
}
}
if (json.swipe_info && json.swipe_info.length > 1) {
for (let i = 1; i < json.swipe_info.length; i++) {
// Start from the second swipe
let swipe = json.swipe_info[i];
if (swipe.gen_started && swipe.gen_finished) {
totalGenTime += calculateGenTime(
swipe.gen_started,
swipe.gen_finished
);
}
}
}
// If this is the first user message, set the first chat time
if (json.is_user) {
//get min between firstChatTime and timestampToMoment(json.send_date)
firstChatTime = Math.min(timestampToMoment(json.send_date), firstChatTime);
}
} catch (error) {
console.error(`Error parsing line ${line}: ${error}`);
}
}
}
return {
totalGenTime,
userWordCount,
nonUserWordCount,
userMsgCount,
nonUserMsgCount,
totalSwipeCount,
firstChatTime,
};
}
module.exports = {
saveStatsToFile,
loadStatsFile,
writeStatsToFileAndExit,
getCharStats,
setCharStats,
};