Poe fix + API logging

.gitignore

@@ -21,3 +21,5 @@ secrets.json
 /dist
 poe_device.json
 /backups/
+poe-error.log
+poe-success.log

src/poe-client.js

@@ -266,25 +266,82 @@ function generate_payload(query, variables) {
 }
 
 async function request_with_retries(method, attempts = 10) {
-    const url = '';
     for (let i = 0; i < attempts; i++) {
-        //console.log(method)
         try {
             const response = await method();
             if (response.status === 200) {
+
+                const circularReference = new Set();
+                const responseString = JSON.stringify(response, function (key, value) {
+                    if (typeof value === 'object' && value !== null) {
+                        if (circularReference.has(value)) {
+                            return;
+                        }
+                        circularReference.add(value);
+                    }
+                    if (key === 'data' && typeof value === 'object' && value !== null) {
+                        return '[removed data spam]';
+                    }
+                    if (typeof value === 'object' && value !== null) {
+                        return Array.isArray(value) ? value : { ...value };
+                    }
+                    if (key === "Cookie" || key === "set-cookie" || key === "Set-Cookie") {
+                        return "[PB COOKIE DATA REDACTED BY ST CODE]"
+                    }
+                    if (typeof value === 'string' && value.includes('p-b=')) {
+                        const startIndex = value.indexOf('p-b=');
+                        const endIndex = value.indexOf(';', startIndex);
+                        if (endIndex === -1) {
+                            return value.substring(0, startIndex) + '[P-B COOKIE REDACTED BY ST]';
+                        }
+                        return value.substring(0, startIndex) + '[P-B COOKIE REDACTED BY ST]' + value.substring(endIndex);
+                    }
+                    if (typeof value === 'string' && value.includes('__cf_bm=')) {
+                        const startIndex = value.indexOf('__cf_bm=');
+                        const endIndex = value.indexOf(';', startIndex);
+                        if (endIndex === -1) {
+                            return value.substring(0, startIndex) + '[Cloudflare COOKIE REDACTED BY ST]';
+                        }
+                        return value.substring(0, startIndex) + '[CloudFlare COOKIE REDACTED BY ST]' + value.substring(endIndex);
+                    }
+
+
+                    return value;
+                }, 4);
+                fs.writeFile('poe-success.log', responseString, 'utf-8', () => {
+                    //console.log('Successful query logged to poe-success.log');
+                });
+
+
                 return response;
             }
-            logger.warn(`Server returned a status code of ${response.status} while downloading ${url}. Retrying (${i + 1}/${attempts})...`);
+
+            //this never actually gets seen as any non-200 response jumps to the catch code
+            logger.warn(`Server returned a status code of ${response.status} while downloading. Retrying (${i + 1}/${attempts})...`);
+        } catch (err) {
+            const circularReference = new Set();
+            const errString = JSON.stringify(err, function (key, value) {
+                if (key === 'data' && Array.isArray(value)) {
+                    return '[removed data spam]';
+                } else if (typeof value === 'object' && value !== null) {
+                    if (circularReference.has(value)) {
+                        return '[Circular]';
+                    }
+                    circularReference.add(value);
+                }
+                if (key === "Cookie") {
+                    return "[COOKIE REDACTED BY ST CODE]"
+                }
+                return value;
+            }, 4);
+            fs.writeFile('poe-error.log', errString, 'utf-8', (err) => {
+                if (err) throw err;
+                console.log('Error saved to poe-error.log');
+            });
+            await delay(100)
         }
-        catch (err) {
-            //console.log(`-------------------ERROR-------------------`)
-            //console.log(logObjectStructure(err, 0, 2));
-            console.log(`Error on request attempt ${i + 1}`)
-            //console.log(`-------------------------------------------`)
-        }
-        await delay(100)
     }
-    throw new Error(`Failed to download ${url} too many times.`);
+    throw new Error(`Failed to download too many times.`);
 }
 
 function findKey(obj, key, path = []) {
@@ -303,17 +360,16 @@ function findKey(obj, key, path = []) {
 }
 
 function logObjectStructure(obj, indent = 0, depth = Infinity) {
+    let result = "";
     const keys = Object.keys(obj);
     keys.forEach((key) => {
-        console.log(`${'  '.repeat(indent)}${key}`);
+        result += `${'  '.repeat(indent)}${key}\n`;
         if (typeof obj[key] === 'object' && obj[key] !== null && indent < depth) {
-            logObjectStructure(obj[key], indent + 1, depth);
+            result += logObjectStructure(obj[key], indent + 1, depth);
         }
     });
+    return result;
 }
-
-
-
 class Client {
     gql_url = "https://poe.com/api/gql_POST";
     gql_recv_url = "https://poe.com/api/receive_POST";
@@ -467,6 +523,7 @@ class Client {
         const bots = {};
         const promises = [];
         for (const bot of botList.filter(x => x.deletionState == 'not_deleted')) {
+            await delay(300)
             const promise = new Promise(async (resolve, reject) => {
                 try {
                     const url = `https://poe.com/_next/data/${this.next_data.buildId}/${bot.displayName}.json`;
@@ -537,7 +594,6 @@ class Client {
             //console.log(`------GQL HEADERS-----`)
             //console.log(this.gql_headers)
             //console.log(`----------------------`)
-            //console.log('sending query..')
             const r = await request_with_retries(() => this.session.post(this.gql_url, payload, { headers: this.gql_headers }));
             if (!(r?.data?.data)) {
                 logger.warn(`${queryName} returned an error | Retrying (${i + 1}/20)`);