Merge branch 'dev' of https://github.com/SillyLossy/TavernAI into dev

package-lock.json

@@ -42,6 +42,7 @@
                 "sentencepiece-js": "^1.1.0",
                 "simple-git": "^3.19.1",
                 "uniqolor": "^1.1.0",
+                "user-agents": "^1.0.1444",
                 "webp-converter": "2.3.2",
                 "ws": "^8.13.0",
                 "yargs": "^17.7.1",
@@ -1222,6 +1223,14 @@
                 "npm": "1.2.8000 || >= 1.4.16"
             }
         },
+        "node_modules/detect-indent": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz",
+            "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==",
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/detect-libc": {
             "version": "2.0.1",
             "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz",
@@ -1251,11 +1260,32 @@
                 "node": ">=8"
             }
         },
+        "node_modules/docopt": {
+            "version": "0.6.2",
+            "resolved": "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz",
+            "integrity": "sha512-NqTbaYeE4gA/wU1hdKFdU+AFahpDOpgGLzHP42k6H6DKExJd0A55KEVWYhL9FEmHmgeLvEU2vuKXDuU+4yToOw==",
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
         "node_modules/dom-walk": {
             "version": "0.1.2",
             "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz",
             "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w=="
         },
+        "node_modules/dot-json": {
+            "version": "1.3.0",
+            "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.3.0.tgz",
+            "integrity": "sha512-Pu11Prog/Yjf2lBICow82/DSV46n3a2XT1Rqt/CeuhkO1fuacF7xydYhI0SwQx2Ue0jCyLtQzgKPFEO6ewv+bQ==",
+            "dependencies": {
+                "detect-indent": "~6.0.0",
+                "docopt": "~0.6.2",
+                "underscore-keypath": "~0.0.22"
+            },
+            "bin": {
+                "dot-json": "bin/dot-json.js"
+            }
+        },
         "node_modules/ee-first": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -2007,6 +2037,11 @@
             "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
             "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
         },
+        "node_modules/lodash.clonedeep": {
+            "version": "4.5.0",
+            "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+            "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="
+        },
         "node_modules/lru-cache": {
             "version": "6.0.0",
             "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -3421,6 +3456,19 @@
             "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
             "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
         },
+        "node_modules/underscore": {
+            "version": "1.13.6",
+            "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
+            "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
+        },
+        "node_modules/underscore-keypath": {
+            "version": "0.0.22",
+            "resolved": "https://registry.npmjs.org/underscore-keypath/-/underscore-keypath-0.0.22.tgz",
+            "integrity": "sha512-fU7aYj1J2LQd+jqdQ67AlCOZKK3Pl+VErS8fGYcgZG75XB9/bY+RLM+F2xEcKHhHNtLvqqFyXAoZQlLYfec3Xg==",
+            "dependencies": {
+                "underscore": "*"
+            }
+        },
         "node_modules/uniqolor": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/uniqolor/-/uniqolor-1.1.0.tgz",
@@ -3443,6 +3491,15 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/user-agents": {
+            "version": "1.0.1444",
+            "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.1444.tgz",
+            "integrity": "sha512-6WXJ0RZuUKgif1rW5FN02HnpoJ8EzH6COQoXCiVStZEVPz+YnAx3iA48etY3ZD4UwueYN9ALC7j4ayHvYEh7tA==",
+            "dependencies": {
+                "dot-json": "^1.3.0",
+                "lodash.clonedeep": "^4.5.0"
+            }
+        },
         "node_modules/utf8-byte-length": {
             "version": "1.0.4",
             "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz",

package.json

@@ -33,6 +33,7 @@
         "sentencepiece-js": "^1.1.0",
         "simple-git": "^3.19.1",
         "uniqolor": "^1.1.0",
+        "user-agents": "^1.0.1444",
         "webp-converter": "2.3.2",
         "ws": "^8.13.0",
         "yargs": "^17.7.1",

public/scripts/user-agent/user-agent.js

@@ -0,0 +1,154 @@
+import cloneDeep from 'lodash.clonedeep';
+
+import userAgents from './user-agents.json';
+
+
+// Normalizes the total weight to 1 and constructs a cumulative distribution.
+const makeCumulativeWeightIndexPairs = (weightIndexPairs) => {
+  const totalWeight = weightIndexPairs.reduce((sum, [weight]) => sum + weight, 0);
+  let sum = 0;
+  return weightIndexPairs.map(([weight, index]) => {
+    sum += weight / totalWeight;
+    return [sum, index];
+  });
+};
+
+// Precompute these so that we can quickly generate unfiltered user agents.
+const defaultWeightIndexPairs = userAgents.map(({ weight }, index) => [weight, index]);
+const defaultCumulativeWeightIndexPairs = makeCumulativeWeightIndexPairs(defaultWeightIndexPairs);
+
+
+// Turn the various filter formats into a single filter function that acts on raw user agents.
+const constructFilter = (filters, accessor = parentObject => parentObject) => {
+  let childFilters;
+  if (typeof filters === 'function') {
+    childFilters = [filters];
+  } else if (filters instanceof RegExp) {
+    childFilters = [
+      value => (
+        typeof value === 'object' && value && value.userAgent
+          ? filters.test(value.userAgent)
+          : filters.test(value)
+      ),
+    ];
+  } else if (filters instanceof Array) {
+    childFilters = filters.map(childFilter => constructFilter(childFilter));
+  } else if (typeof filters === 'object') {
+    childFilters = Object.entries(filters).map(([key, valueFilter]) => (
+      constructFilter(valueFilter, parentObject => parentObject[key])
+    ));
+  } else {
+    childFilters = [
+      value => (
+        typeof value === 'object' && value && value.userAgent
+          ? filters === value.userAgent
+          : filters === value
+      ),
+    ];
+  }
+
+  return (parentObject) => {
+    try {
+      const value = accessor(parentObject);
+      return childFilters.every(childFilter => childFilter(value));
+    } catch (error) {
+      // This happens when a user-agent lacks a nested property.
+      return false;
+    }
+  };
+};
+
+
+// Construct normalized cumulative weight index pairs given the filters.
+const constructCumulativeWeightIndexPairsFromFilters = (filters) => {
+  if (!filters) {
+    return defaultCumulativeWeightIndexPairs;
+  }
+
+  const filter = constructFilter(filters);
+
+  const weightIndexPairs = [];
+  userAgents.forEach((rawUserAgent, index) => {
+    if (filter(rawUserAgent)) {
+      weightIndexPairs.push([rawUserAgent.weight, index]);
+    }
+  });
+  return makeCumulativeWeightIndexPairs(weightIndexPairs);
+};
+
+
+const setCumulativeWeightIndexPairs = (userAgent, cumulativeWeightIndexPairs) => {
+  Object.defineProperty(userAgent, 'cumulativeWeightIndexPairs', {
+    configurable: true,
+    enumerable: false,
+    writable: false,
+    value: cumulativeWeightIndexPairs,
+  });
+};
+
+
+export default class UserAgent extends Function {
+  constructor(filters) {
+    super();
+    setCumulativeWeightIndexPairs(this, constructCumulativeWeightIndexPairsFromFilters(filters));
+    if (this.cumulativeWeightIndexPairs.length === 0) {
+      throw new Error('No user agents matched your filters.');
+    }
+
+    this.randomize();
+
+    return new Proxy(this, {
+      apply: () => this.random(),
+      get: (target, property, receiver) => {
+        const dataCandidate = target.data && typeof property === 'string'
+          && Object.prototype.hasOwnProperty.call(target.data, property)
+          && Object.prototype.propertyIsEnumerable.call(target.data, property);
+        if (dataCandidate) {
+          const value = target.data[property];
+          if (value !== undefined) {
+            return value;
+          }
+        }
+
+        return Reflect.get(target, property, receiver);
+      },
+    });
+  }
+
+  static random = (filters) => {
+    try {
+      return new UserAgent(filters);
+    } catch (error) {
+      return null;
+    }
+  };
+
+  //
+  // Standard Object Methods
+  //
+
+  [Symbol.toPrimitive] = () => (
+    this.data.userAgent
+  );
+
+  toString = () => (
+    this.data.userAgent
+  );
+
+  random = () => {
+    const userAgent = new UserAgent();
+    setCumulativeWeightIndexPairs(userAgent, this.cumulativeWeightIndexPairs);
+    userAgent.randomize();
+    return userAgent;
+  };
+
+  randomize = () => {
+    // Find a random raw random user agent.
+    const randomNumber = Math.random();
+    const [, index] = this.cumulativeWeightIndexPairs
+      .find(([cumulativeWeight]) => cumulativeWeight > randomNumber);
+    const rawUserAgent = userAgents[index];
+
+    this.data = cloneDeep(rawUserAgent);
+  }
+}

src/poe-client.js

@@ -26,6 +26,8 @@ const http = require('http');
 const https = require('https');
 const _ = require('lodash');
 
+const UserAgent = require('user-agents');
+
 const directory = __dirname;
 
 function uuidv4() {
@@ -64,7 +66,10 @@ const cached_bots = {};
 const logger = console;
 const delay = ms => new Promise(res => setTimeout(res, ms));
 
-const user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36";
+const userAgent = new UserAgent();
+const user_agent = userAgent.toString();
+
+//const user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36";
 
 function extractFormKey(html) {
     const scriptRegex = /<script>if\(.+\)throw new Error;(.+)<\/script>/;
@@ -267,11 +272,19 @@ function generate_payload(query, variables) {
 
 async function request_with_retries(method, attempts = 10) {
     for (let i = 0; i < attempts; i++) {
-
+        //console.log(user_agent)
         try {
             var ResponseHasFreeSocket = false;
             const response = await method();
             if (response.status === 200) {
+                const responseStringRaw = JSON.stringify(response, null, 4)
+                fs.writeFile('poe-success-raw.log', responseStringRaw, 'utf-8', (err) => {
+                    if (err) {
+                        console.error(err);
+                    } else {
+                        console.log('Successful query logged to poe-success-raw.log');
+                    }
+                });
 
                 const circularReference = new Set();
                 const responseString = JSON.stringify(response, function (key, value) {
@@ -327,7 +340,8 @@ async function request_with_retries(method, attempts = 10) {
         } catch (err) {
             var ErrorHasFreeSocket = false;
             const circularReference = new Set();
-            //const errStringRaw = JSON.stringify(err, null, 4)
+            const errStringRaw = JSON.stringify(err, null, 4)
+
             const errString = JSON.stringify(err, function (key, value) {
                 if (key === 'data' && Array.isArray(value)) {
                     return '[removed data spam]';
@@ -366,8 +380,13 @@ async function request_with_retries(method, attempts = 10) {
             //});
             fs.writeFile('poe-error.log', errString, 'utf-8', (err) => {
                 if (err) throw err;
-                console.log(`Error saved to poe-error.log Free socket? ${ErrorHasFreeSocket}`);
+                //console.log(`Error saved to poe-error.log Free socket? ${ErrorHasFreeSocket}`);
             });
+            fs.writeFile('poe-error-raw.log', errStringRaw, 'utf-8', (err) => {
+                if (err) throw err;
+                //console.log(`Error saved to poe-error-raw.log Free socket? ${ErrorHasFreeSocket}`);
+            });
+            console.log(`Error log files saved`)
             await delay(3000)
         }
     }