From 6f0f420063e9dddcd1bfea6828c04f741d0e004c Mon Sep 17 00:00:00 2001 From: Xrystal Date: Sat, 21 Oct 2023 18:55:52 +0800 Subject: [PATCH 1/3] Promisify getallchatsofcharacter --- server.js | 69 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/server.js b/server.js index 03cd45014..542359c37 100644 --- a/server.js +++ b/server.js @@ -1829,7 +1829,7 @@ app.post("/getallchatsofcharacter", jsonParser, function (request, response) { if (!request.body) return response.sendStatus(400); var char_dir = (request.body.avatar_url).replace('.png', '') - fs.readdir(chatsPath + char_dir, (err, files) => { + fs.readdir(chatsPath + char_dir, async (err, files) => { if (err) { console.log('found error in history loading'); console.error(err); @@ -1843,20 +1843,20 @@ app.post("/getallchatsofcharacter", jsonParser, function (request, response) { // sort the files by name //jsonFiles.sort().reverse(); // print the sorted file names - var chatData = {}; - let ii = jsonFiles.length; //this is the number of files belonging to the character - if (ii !== 0) { - //console.log('found '+ii+' chat logs to load'); - for (let i = jsonFiles.length - 1; i >= 0; i--) { - const file = jsonFiles[i]; + let ii = jsonFiles.length; //this is the number of files belonging to the character + if (ii === 0) { + response.send({ error: true }); + return; + } + + const jsonFilesPromise = jsonFiles.map((file) => { + return new Promise(async (res) => { const fileStream = fs.createReadStream(chatsPath + char_dir + '/' + file); const fullPathAndFile = chatsPath + char_dir + '/' + file const stats = fs.statSync(fullPathAndFile); const fileSizeInKB = (stats.size / 1024).toFixed(2) + "kb"; - //console.log(fileSizeInKB); - const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity @@ -1869,33 +1869,42 @@ app.post("/getallchatsofcharacter", jsonParser, function (request, response) { lastLine = line; }); rl.on('close', () => { - ii--; - if (lastLine) { + rl.close(); + if (lastLine) { let jsonData = tryParse(lastLine); - if (jsonData && (jsonData.name !== undefined || jsonData.character_name !== undefined)) { - chatData[i] = {}; - chatData[i]['file_name'] = file; - chatData[i]['file_size'] = fileSizeInKB; - chatData[i]['chat_items'] = itemCounter - 1; - chatData[i]['mes'] = jsonData['mes'] || '[The chat is empty]'; - chatData[i]['last_mes'] = jsonData['send_date'] || Date.now(); + if ( + jsonData && + (jsonData.name !== undefined || + jsonData.character_name !== undefined) + ) { + const chatData = {}; + + chatData['file_name'] = file; + chatData['file_size'] = fileSizeInKB; + chatData['chat_items'] = itemCounter - 1; + chatData['mes'] = + jsonData['mes'] || '[The chat is empty]'; + chatData['last_mes'] = + jsonData['send_date'] || Date.now(); + + res(chatData); } else { - console.log('Found an invalid or corrupted chat file: ' + fullPathAndFile); + console.log( + 'Found an invalid or corrupted chat file: ' + + fullPathAndFile + ); + + res({}); } } - if (ii === 0) { - //console.log('ii count went to zero, responding with chatData'); - response.send(chatData); - } - //console.log('successfully closing getallchatsofcharacter'); - rl.close(); }); - }; - } else { - //console.log('Found No Chats. Exiting Load Routine.'); - response.send({ error: true }); - }; + }); + }); + + const chatData = await Promise.all(jsonFilesPromise); + + response.send(chatData); }) }); From 1d38109dcf7788fdc70c681754e464281ca766cf Mon Sep 17 00:00:00 2001 From: Xrystal Date: Sat, 21 Oct 2023 18:56:51 +0800 Subject: [PATCH 2/3] Use JSON instead of json5 --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index 542359c37..06e5e59c6 100644 --- a/server.js +++ b/server.js @@ -713,7 +713,7 @@ app.post("/getchat", jsonParser, function (request, response) { const lines = data.split('\n'); // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map(tryParse).filter(x => x); + const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { }}).filter(x => x); return response.send(jsonData); } catch (error) { console.error(error); From 008fcece04f1705b27af14f0aee301e6af8b33b4 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 21 Oct 2023 15:04:36 +0300 Subject: [PATCH 3/3] Rewrite to sync readdir, add try-catch, filter out invalid files --- server.js | 60 ++++++++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/server.js b/server.js index 06e5e59c6..da56ec549 100644 --- a/server.js +++ b/server.js @@ -713,7 +713,7 @@ app.post("/getchat", jsonParser, function (request, response) { const lines = data.split('\n'); // Iterate through the array of strings and parse each line as JSON - const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { }}).filter(x => x); + const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { } }).filter(x => x); return response.send(jsonData); } catch (error) { console.error(error); @@ -1825,37 +1825,27 @@ function getImages(path) { .sort(Intl.Collator().compare); } -app.post("/getallchatsofcharacter", jsonParser, function (request, response) { +app.post("/getallchatsofcharacter", jsonParser, async function (request, response) { if (!request.body) return response.sendStatus(400); - var char_dir = (request.body.avatar_url).replace('.png', '') - fs.readdir(chatsPath + char_dir, async (err, files) => { - if (err) { - console.log('found error in history loading'); - console.error(err); - response.send({ error: true }); - return; - } + const characterDirectory = (request.body.avatar_url).replace('.png', ''); - // filter for JSON files + try { + const chatsDirectory = path.join(chatsPath, characterDirectory); + const files = fs.readdirSync(chatsDirectory); const jsonFiles = files.filter(file => path.extname(file) === '.jsonl'); - // sort the files by name - //jsonFiles.sort().reverse(); - // print the sorted file names - let ii = jsonFiles.length; //this is the number of files belonging to the character - if (ii === 0) { + if (jsonFiles.length === 0) { response.send({ error: true }); return; } const jsonFilesPromise = jsonFiles.map((file) => { return new Promise(async (res) => { - const fileStream = fs.createReadStream(chatsPath + char_dir + '/' + file); - - const fullPathAndFile = chatsPath + char_dir + '/' + file - const stats = fs.statSync(fullPathAndFile); - const fileSizeInKB = (stats.size / 1024).toFixed(2) + "kb"; + const pathToFile = path.join(chatsPath, characterDirectory, file); + const fileStream = fs.createReadStream(pathToFile); + const stats = fs.statSync(pathToFile); + const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`; const rl = readline.createInterface({ input: fileStream, @@ -1872,29 +1862,19 @@ app.post("/getallchatsofcharacter", jsonParser, function (request, response) { rl.close(); if (lastLine) { - let jsonData = tryParse(lastLine); - if ( - jsonData && - (jsonData.name !== undefined || - jsonData.character_name !== undefined) - ) { + const jsonData = tryParse(lastLine); + if (jsonData && (jsonData.name || jsonData.character_name)) { const chatData = {}; chatData['file_name'] = file; chatData['file_size'] = fileSizeInKB; chatData['chat_items'] = itemCounter - 1; - chatData['mes'] = - jsonData['mes'] || '[The chat is empty]'; - chatData['last_mes'] = - jsonData['send_date'] || Date.now(); + chatData['mes'] = jsonData['mes'] || '[The chat is empty]'; + chatData['last_mes'] = jsonData['send_date'] || Date.now(); res(chatData); } else { - console.log( - 'Found an invalid or corrupted chat file: ' + - fullPathAndFile - ); - + console.log('Found an invalid or corrupted chat file:', pathToFile); res({}); } } @@ -1903,9 +1883,13 @@ app.post("/getallchatsofcharacter", jsonParser, function (request, response) { }); const chatData = await Promise.all(jsonFilesPromise); + const validFiles = chatData.filter(i => i.file_name); - response.send(chatData); - }) + return response.send(validFiles); + } catch (error) { + console.log(error); + return response.send({ error: true }); + } }); function getPngName(file) {