From 30e9e90b3850ee8eaeeab11ba04e84414c0b46f9 Mon Sep 17 00:00:00 2001 From: p3il4 <42489293+P3il4@users.noreply.github.com> Date: Wed, 30 Oct 2024 23:35:44 +0300 Subject: [PATCH] optimize chat manager logic --- public/script.js | 135 ++++++++++++++----------------------- src/endpoints/chats.js | 147 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 86 deletions(-) diff --git a/public/script.js b/public/script.js index 55ee1dfbe..63d4bd8eb 100644 --- a/public/script.js +++ b/public/script.js @@ -7025,99 +7025,15 @@ export async function displayPastChats() { $('#select_chat_div').empty(); $('#select_chat_search').val('').off('input'); - const data = await (selected_group ? getGroupPastChats(selected_group) : getPastCharacterChats()); - - if (!data) { - toastr.error(t`Could not load chat data. Try reloading the page.`); - return; - } - const chatDetails = getCurrentChatDetails(); - const group = chatDetails.group; const currentChat = chatDetails.sessionName; const displayName = chatDetails.characterName; const avatarImg = chatDetails.avatarImgURL; - const rawChats = await getChatsFromFiles(data, selected_group); - - // Sort by last message date descending - data.sort((a, b) => sortMoments(timestampToMoment(a.last_mes), timestampToMoment(b.last_mes))); - console.log(data); - $('#load_select_chat_div').css('display', 'none'); - $('#ChatHistoryCharName').text(`${displayName}'s `); - - const displayChats = (searchQuery) => { - $('#select_chat_div').empty(); // Clear the current chats before appending filtered chats - - const filteredData = data.filter(chat => { - const fileName = chat['file_name']; - const chatContent = rawChats[fileName]; - - // Make sure empty chats are displayed when there is no search query - if (Array.isArray(chatContent) && !chatContent.length && !searchQuery) { - return true; - } - - // // Uncomment this to return to old behavior (classical full-substring search). - // return chatContent && Object.values(chatContent).some(message => message?.mes?.toLowerCase()?.includes(searchQuery.toLowerCase())); - - // Fragment search a.k.a. swoop (as in `helm-swoop` in the Helm package of Emacs). - // Split a `query` {string} into its fragments {string[]}. - function makeQueryFragments(query) { - let fragments = query.trim().split(/\s+/).map(str => str.trim().toLowerCase()).filter(onlyUnique); - // fragments = fragments.filter( function(str) { return str.length >= 3; } ); // Helm does this, but perhaps better if we don't. - return fragments; - } - // Check whether `text` {string} includes all of the `fragments` {string[]}. - function matchFragments(fragments, text) { - if (!text || !text.toLowerCase) return false; - return fragments.every(item => text.toLowerCase().includes(item)); - } - const fragments = makeQueryFragments(searchQuery); - // At least one chat message must match *all* the fragments. - // Currently, this doesn't match if the fragment matches are distributed across several chat messages. - return chatContent && Object.values(chatContent).some(message => matchFragments(fragments, message?.mes)); - }); - - console.debug(filteredData); - for (const value of filteredData.values()) { - let strlen = 300; - let mes = value['mes']; - - if (mes !== undefined) { - if (mes.length > strlen) { - mes = '...' + mes.substring(mes.length - strlen); - } - const fileSize = value['file_size']; - const fileName = value['file_name']; - const chatItems = rawChats[fileName].length; - const timestamp = timestampToMoment(value['last_mes']).format('lll'); - const template = $('#past_chat_template .select_chat_block_wrapper').clone(); - template.find('.select_chat_block').attr('file_name', fileName); - template.find('.avatar img').attr('src', avatarImg); - template.find('.select_chat_block_filename').text(fileName); - template.find('.chat_file_size').text(`(${fileSize},`); - template.find('.chat_messages_num').text(`${chatItems}💬)`); - template.find('.select_chat_block_mes').text(mes); - template.find('.PastChat_cross').attr('file_name', fileName); - template.find('.chat_messages_date').text(timestamp); - - if (selected_group) { - template.find('.avatar img').replaceWith(getGroupAvatar(group)); - } - - $('#select_chat_div').append(template); - - if (currentChat === fileName.toString().replace('.jsonl', '')) { - $('#select_chat_div').find('.select_chat_block:last').attr('highlight', String(true)); - } - } - } - }; - displayChats(''); // Display all by default + await displayChats('', currentChat, displayName, avatarImg, selected_group); const debouncedDisplay = debounce((searchQuery) => { - displayChats(searchQuery); + displayChats(searchQuery, currentChat, displayName, avatarImg, selected_group); }); // Define the search input listener @@ -7135,6 +7051,48 @@ export async function displayPastChats() { }, 200); } +async function displayChats(searchQuery, currentChat, displayName, avatarImg, selected_group) { + try { + const response = await fetch('/api/chats/search', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + query: searchQuery, + avatar_url: selected_group ? null : characters[this_chid].avatar, + group_id: selected_group || null + }), + }); + + if (!response.ok) { + throw new Error('Search failed'); + } + + const filteredData = await response.json(); + $('#select_chat_div').empty(); + + for (const chat of filteredData) { + const template = $('#past_chat_template .select_chat_block_wrapper').clone(); + template.find('.select_chat_block').attr('file_name', chat.file_name); + template.find('.avatar img').attr('src', avatarImg); + template.find('.select_chat_block_filename').text(chat.file_name); + template.find('.chat_file_size').text(`(${chat.file_size},`); + template.find('.chat_messages_num').text(`${chat.message_count}💬)`); + template.find('.select_chat_block_mes').text(chat.preview_message); + template.find('.PastChat_cross').attr('file_name', chat.file_name); + template.find('.chat_messages_date').text(timestampToMoment(chat.last_mes).format('lll')); + + $('#select_chat_div').append(template); + + if (currentChat === chat.file_name) { + $('#select_chat_div').find('.select_chat_block:last').attr('highlight', String(true)); + } + } + } catch (error) { + console.error('Error loading chats:', error); + toastr.error('Could not load chat data. Try reloading the page.'); + } +} + export function selectRightMenuWithAnimation(selectedMenuId) { const displayModes = { 'rm_group_chats_block': 'flex', @@ -10421,6 +10379,11 @@ jQuery(async function () { $('#load_select_chat_div').css('display', 'block'); }); + // not sure what that hourglass was for + $('#option_select_chat').click(function () { + $('#load_select_chat_div').css('display', 'none'); + }); + if (navigator.clipboard === undefined) { // No clipboard support $('.mes_copy').remove(); diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js index f7b9ba767..a458f6674 100644 --- a/src/endpoints/chats.js +++ b/src/endpoints/chats.js @@ -52,6 +52,37 @@ function getBackupFunction(handle) { return backupFunctions.get(handle); } +/** + * Formats a byte size into a human-readable string with units + * @param {number} bytes - The size in bytes to format + * @returns {string} The formatted string (e.g., "1.5 MB") + */ +function formatBytes(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + +/** + * Gets a preview message from an array of chat messages + * @param {Array} messages - Array of chat messages, each with a 'mes' property + * @returns {string} A truncated preview of the last message or empty string if no messages + */ +function getPreviewMessage(messages) { + const strlen = 300; + const lastMessage = messages[messages.length - 1]?.mes; + + if (!lastMessage) { + return ''; + } + + return lastMessage.length > strlen + ? '...' + lastMessage.substring(lastMessage.length - strlen) + : lastMessage; +} + process.on('exit', () => { for (const func of backupFunctions.values()) { func.flush(); @@ -516,3 +547,119 @@ router.post('/group/save', jsonParser, (request, response) => { getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData); return response.send({ ok: true }); }); + +router.post('/search', jsonParser, function (request, response) { + try { + const { query, avatar_url, group_id } = request.body; + let chatFiles = []; + + if (group_id) { + // Find group's chat IDs first + const groupDir = path.join(request.user.directories.groups); + const groupFiles = fs.readdirSync(groupDir) + .filter(file => file.endsWith('.json')); + + let targetGroup; + for (const groupFile of groupFiles) { + const groupData = JSON.parse(fs.readFileSync(path.join(groupDir, groupFile), 'utf8')); + if (groupData.id === group_id) { + targetGroup = groupData; + break; + } + } + + if (!targetGroup?.chats) { + return response.send([]); + } + + // Find group chat files for given group ID + const groupChatsDir = path.join(request.user.directories.groupChats); + chatFiles = targetGroup.chats + .map(chatId => { + const filePath = path.join(groupChatsDir, `${chatId}.jsonl`); + if (!fs.existsSync(filePath)) return null; + const stats = fs.statSync(filePath); + return { + file_name: chatId, + file_size: formatBytes(stats.size), + path: filePath, + }; + }) + .filter(x => x); + } else { + // Regular character chat directory + const character_name = avatar_url.replace('.png', ''); + const directoryPath = path.join(request.user.directories.chats, character_name); + + if (!fs.existsSync(directoryPath)) { + return response.send([]); + } + + chatFiles = fs.readdirSync(directoryPath) + .filter(file => file.endsWith('.jsonl')) + .map(fileName => { + const filePath = path.join(directoryPath, fileName); + const stats = fs.statSync(filePath); + return { + file_name: fileName, + file_size: formatBytes(stats.size), + path: filePath, + }; + }); + } + + const results = []; + + // Search logic + for (const chatFile of chatFiles) { + const data = fs.readFileSync(chatFile.path, 'utf8'); + const messages = data.split('\n') + .map(line => { try { return JSON.parse(line); } catch (_) { return null; } }) + .filter(x => x); + + if (messages.length === 0) { + continue; + } + + const lastMessage = messages[messages.length - 1]; + const lastMesDate = lastMessage?.send_date || new Date().toISOString(); + + // If no search query, just return metadata + if (!query) { + results.push({ + file_name: chatFile.file_name, + file_size: chatFile.file_size, + message_count: messages.length, + last_mes: lastMesDate, + preview_message: getPreviewMessage(messages), + }); + continue; + } + + // Search through messages + const fragments = query.trim().toLowerCase().split(/\s+/).filter(x => x); + const hasMatch = messages.some(message => { + const text = message?.mes?.toLowerCase(); + return text && fragments.every(fragment => text.includes(fragment)); + }); + + if (hasMatch) { + results.push({ + file_name: chatFile.file_name, + file_size: chatFile.file_size, + message_count: messages.length, + last_mes: lastMesDate, + preview_message: getPreviewMessage(messages), + }); + } + } + + // Sort by last message date descending + results.sort((a, b) => new Date(b.last_mes) - new Date(a.last_mes)); + return response.send(results); + + } catch (error) { + console.error('Chat search error:', error); + return response.status(500).json({ error: 'Search failed' }); + } +}); \ No newline at end of file