From 8c00f38a1fc5582b17154d414a0c8b3c666677c7 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 7 Sep 2023 21:53:47 +0300 Subject: [PATCH 01/34] Add local vectors storage --- .gitignore | 1 + package-lock.json | 683 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 3 + server.js | 3 + src/vectors.js | 192 +++++++++++++ 5 files changed, 879 insertions(+), 3 deletions(-) create mode 100644 src/vectors.js diff --git a/.gitignore b/.gitignore index a3e39cb09..582031246 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ content.log cloudflared.exe public/assets/ access.log +/vectors/ diff --git a/package-lock.json b/package-lock.json index 9bde269ce..ee683edaa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,8 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", + "@tensorflow-models/universal-sentence-encoder": "^1.3.3", + "@tensorflow/tfjs": "^4.10.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -40,6 +42,7 @@ "sanitize-filename": "^1.6.3", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", + "vectra": "^0.2.2", "webp-converter": "2.3.2", "write-file-atomic": "^5.0.1", "ws": "^8.13.0", @@ -642,16 +645,274 @@ "node": ">= 8" } }, + "node_modules/@tensorflow-models/universal-sentence-encoder": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@tensorflow-models/universal-sentence-encoder/-/universal-sentence-encoder-1.3.3.tgz", + "integrity": "sha512-mipL7ad0CW6uQ68FUkNgkNj/zgA4qgBnNcnMMkNTdL9MUMnzCxu3AE8pWnx2ReKHwdqEG4e8IpaYKfH4B8bojg==", + "peerDependencies": { + "@tensorflow/tfjs-converter": "^3.6.0", + "@tensorflow/tfjs-core": "^3.6.0" + } + }, + "node_modules/@tensorflow/tfjs": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.10.0.tgz", + "integrity": "sha512-16q6CcGEoxfg3kimBzNBrZg069GRd4Du49uyQHUMsEvpHbmU0ZpWa2zhjjZ71GlURpbD1LZIZNp6cd2Q1Eqjow==", + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "4.10.0", + "@tensorflow/tfjs-backend-webgl": "4.10.0", + "@tensorflow/tfjs-converter": "4.10.0", + "@tensorflow/tfjs-core": "4.10.0", + "@tensorflow/tfjs-data": "4.10.0", + "@tensorflow/tfjs-layers": "4.10.0", + "argparse": "^1.0.10", + "chalk": "^4.1.0", + "core-js": "3.29.1", + "regenerator-runtime": "^0.13.5", + "yargs": "^16.0.3" + }, + "bin": { + "tfjs-custom-module": "dist/tools/custom_module/cli.js" + } + }, + "node_modules/@tensorflow/tfjs-converter": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz", + "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==", + "peer": true, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs-core": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz", + "integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==", + "peer": true, + "dependencies": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@webgpu/types": "0.1.16", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.10.0.tgz", + "integrity": "sha512-w3f0ORR1smSpkW7om2yQVunRYMjyqWcEbWCPirR1DQ6ImgW+VWqmM2oVPQXRsFYpwg1g6bk2Jp5COafpPA+krw==", + "dependencies": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.10.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-webgl": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.10.0.tgz", + "integrity": "sha512-Vzl/pyXHa9TgFaRJGspExjZVDKgkKvLxOkPaH+psE2LPnQkiH/IOPO7HKO0U3+hZql977BIiZdHc6HNprFS3/A==", + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "4.10.0", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.10.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-converter": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.10.0.tgz", + "integrity": "sha512-ffLpK+ismdmiDcoTID2aidP3/uJYyQPjmKdRZ3hBUkrczy7pQIcCW8blIR9Gk20htB4OLQMf74ZxbpfdQ9nYeQ==", + "peerDependencies": { + "@tensorflow/tfjs-core": "4.10.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.10.0.tgz", + "integrity": "sha512-klc6lUTYRbHQLEFnVKtTICNK+WUlduUcrlXsDs6ixKAOJzLAlIR2JnmJICt2AT2Rxwu0Zj2bAYojSxgcIcUUxA==", + "dependencies": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.7.0", + "@types/seedrandom": "^2.4.28", + "@webgpu/types": "0.1.30", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": { + "version": "2019.7.1", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.1.tgz", + "integrity": "sha512-+HSrJgjBW77ALieQdMJvXhRZUIRN1597L+BKvsyeiIlHHERnqjcuOLyodK3auJ3Y3zRezNKtKAhuQWYJfEgFHQ==" + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-data": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.10.0.tgz", + "integrity": "sha512-71rQ6xSipXdClKja705jrWZkH9ostAYuVZlf7nW2AJXUCzhrGsJAkcHag4m568mDFoAqfQQTBy4Gk26h0/Y+Pg==", + "dependencies": { + "@types/node-fetch": "^2.1.2", + "node-fetch": "~2.6.1", + "string_decoder": "^1.3.0" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.10.0", + "seedrandom": "^3.0.5" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-layers": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.10.0.tgz", + "integrity": "sha512-SLZWnuDF98WmmJQ5NhsXJFlJPwoKxfLowvAHTlLz+Q1Po4juZVZ+BkatRsqrI2sA2B0IIu2TJp4VEAFWMqzTTg==", + "peerDependencies": { + "@tensorflow/tfjs-core": "4.10.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/@webgpu/types": { + "version": "0.1.30", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.30.tgz", + "integrity": "sha512-9AXJSmL3MzY8ZL//JjudA//q+2kBRGhLBFpkdGksWIuxrMy81nFrCzj2Am+mbh8WoU6rXmv7cY5E3rdlyru2Qg==" + }, + "node_modules/@tensorflow/tfjs/node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/@tensorflow/tfjs/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "engines": { + "node": ">=10" + } + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "node_modules/@types/node": { "version": "16.9.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" + }, + "node_modules/@types/seedrandom": { + "version": "2.4.30", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz", + "integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==" + }, + "node_modules/@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==", + "peer": true + }, + "node_modules/@webgpu/types": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz", + "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==", + "peer": true + }, "node_modules/@xmldom/xmldom": { "version": "0.8.9", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.9.tgz", @@ -748,6 +1009,14 @@ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==" }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -767,6 +1036,11 @@ "node": ">=8" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -776,6 +1050,16 @@ "node": ">= 4.0.0" } }, + "node_modules/axios": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", + "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -856,6 +1140,11 @@ "node": ">= 0.8" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", @@ -947,7 +1236,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -959,6 +1247,42 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -994,6 +1318,17 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/command-exists": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", @@ -1104,6 +1439,16 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, + "node_modules/core-js": { + "version": "3.29.1", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz", + "integrity": "sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw==", + "hasInstallScript": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -1137,6 +1482,32 @@ "http-errors": "^2.0.0" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -1177,6 +1548,14 @@ "node": ">=8" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1223,11 +1602,70 @@ "node": ">=8" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, "node_modules/dom-walk": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==" }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dotenv": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", + "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==", + "engines": { + "node": ">=10" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1255,6 +1693,17 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -1268,6 +1717,14 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -1445,6 +1902,38 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1580,6 +2069,11 @@ "resolved": "https://registry.npmjs.org/google-translate-api-browser/-/google-translate-api-browser-3.0.1.tgz", "integrity": "sha512-KTLodkyGBWMK9IW6QIeJ2zCuju4Z0CLpbkADKo+yLhbSTD4l+CXXpQ/xaynGVAzeBezzJG6qn8MLeqOq3SmW0A==" }, + "node_modules/gpt-3-encoder": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/gpt-3-encoder/-/gpt-3-encoder-1.1.4.tgz", + "integrity": "sha512-fSQRePV+HUAhCn7+7HL7lNIXNm6eaFWFbNLOOGtmSJ0qJycyQvj60OvRlH7mee8xAMjBDNRdMXlMwjAbMTDjkg==" + }, "node_modules/gpt3-tokenizer": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/gpt3-tokenizer/-/gpt3-tokenizer-1.1.5.tgz", @@ -1612,7 +2106,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -1639,6 +2132,24 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -1912,6 +2423,71 @@ "node": ">=4" } }, + "node_modules/json-colorizer": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/json-colorizer/-/json-colorizer-2.2.2.tgz", + "integrity": "sha512-56oZtwV1piXrQnRNTtJeqRv+B9Y/dXAYLqBBaYl/COcUdoZxgLBLAO88+CnkbT6MxNs0c5E9mPBIb2sFcNz3vw==", + "dependencies": { + "chalk": "^2.4.1", + "lodash.get": "^4.4.2" + } + }, + "node_modules/json-colorizer/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-colorizer/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-colorizer/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/json-colorizer/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/json-colorizer/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/json-colorizer/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -1955,6 +2531,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" + }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -2204,6 +2790,17 @@ } } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2269,6 +2866,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", + "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", + "dependencies": { + "axios": "^0.26.0", + "form-data": "^4.0.0" + } + }, + "node_modules/openai/node_modules/axios": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "dependencies": { + "follow-redirects": "^1.14.8" + } + }, "node_modules/p-is-promise": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz", @@ -2319,6 +2933,29 @@ "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.5.tgz", "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==" }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2650,6 +3287,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -2899,6 +3541,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -3079,6 +3726,11 @@ "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -3165,7 +3817,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -3372,6 +4023,14 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -3380,6 +4039,24 @@ "node": ">= 0.8" } }, + "node_modules/vectra": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/vectra/-/vectra-0.2.2.tgz", + "integrity": "sha512-Z1d29Zil+dlA9/Qz4VJB3zcWjIARY8QH5xQEnHmGgmTUP58cPRV8NK7P0QH91IRs0RvAWrYiQf9Y5JqQo0vJlQ==", + "dependencies": { + "axios": "^1.3.4", + "cheerio": "^1.0.0-rc.12", + "dotenv": "^8.2.0", + "gpt-3-encoder": "1.1.4", + "json-colorizer": "^2.2.2", + "openai": "^3.2.1", + "uuid": "^9.0.0", + "yargs": "^17.7.2" + }, + "bin": { + "vectra": "bin/vectra.js" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 0dc6193d3..aa74e21a5 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,8 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", + "@tensorflow-models/universal-sentence-encoder": "^1.3.3", + "@tensorflow/tfjs": "^4.10.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -31,6 +33,7 @@ "sanitize-filename": "^1.6.3", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", + "vectra": "^0.2.2", "webp-converter": "2.3.2", "write-file-atomic": "^5.0.1", "ws": "^8.13.0", diff --git a/server.js b/server.js index 4c0497669..18134a04b 100644 --- a/server.js +++ b/server.js @@ -5825,3 +5825,6 @@ app.post('/get_character_assets_list', jsonParser, async (request, response) => return response.sendStatus(500); } }); + +// Vector storage DB +require('./src/vectors').registerEndpoints(app, jsonParser); diff --git a/src/vectors.js b/src/vectors.js new file mode 100644 index 000000000..f09444ec9 --- /dev/null +++ b/src/vectors.js @@ -0,0 +1,192 @@ +const express = require('express'); +const vectra = require('vectra'); +const path = require('path'); +const sanitize = require('sanitize-filename'); +require('@tensorflow/tfjs'); +const encoder = require('@tensorflow-models/universal-sentence-encoder'); + +/** + * Lazy loading class for the embedding model. + */ +class EmbeddingModel { + /** + * @type {encoder.UniversalSentenceEncoder} - The embedding model + */ + #model; + + async get() { + if (!this.#model) { + this.model = await encoder.load(); + } + + return this.#model; + } +} + +/** + * Hard limit on the number of results to return from the vector search. + */ +const TOP_K = 100; +const model = new EmbeddingModel(); + +/** + * Gets the index for the vector collection + * @param {string} collectionId - The collection ID + * @returns {Promise} - The index for the collection + */ +async function getIndex(collectionId) { + const index = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(collectionId))); + + if (!await index.isIndexCreated()) { + await index.createIndex(); + } + + return index; +} + +/** + * Inserts items into the vector collection + * @param {string} collectionId - The collection ID + * @param {{ hash: number; text: string; }[]} items - The items to insert + */ +async function insertVectorItems(collectionId, items) { + const index = await getIndex(collectionId); + const use = await model.get(); + + await index.beginUpdate(); + + for (const item of items) { + const text = item.text; + const hash = item.hash; + const tensor = await use.embed(text); + const vector = Array.from(await tensor.data()); + await index.upsertItem({ vector: vector, metadata: { hash, text } }); + } + + await index.endUpdate(); +} + +/** + * Gets the hashes of the items in the vector collection + * @param {string} collectionId - The collection ID + * @returns {Promise} - The hashes of the items in the collection + */ +async function getSavedHashes(collectionId) { + const index = await getIndex(collectionId); + + const items = await index.listItems(); + const hashes = items.map(x => Number(x.metadata.hash)); + + return hashes; +} + +/** + * Deletes items from the vector collection by hash + * @param {string} collectionId - The collection ID + * @param {number[]} hashes - The hashes of the items to delete + */ +async function deleteVectorItems(collectionId, hashes) { + const index = await getIndex(collectionId); + const items = await index.listItemsByMetadata({ hash: { '$in': hashes } }); + + await index.beginUpdate(); + + for (const item of items) { + await index.deleteItem(item.id); + } + + await index.endUpdate(); +} + +/** + * Gets the hashes of the items in the vector collection that match the search text + * @param {string} collectionId + * @param {string} searchText + * @returns {Promise} - The hashes of the items that match the search text + */ +async function queryCollection(collectionId, searchText) { + const index = await getIndex(collectionId); + const use = await model.get(); + const tensor = await use.embed(searchText); + const vector = Array.from(await tensor.data()); + + const result = await index.queryItems(vector, TOP_K); + const hashes = result.map(x => Number(x.item.metadata.hash)); + return hashes; +} + +/** + * Registers the endpoints for the vector API + * @param {express.Express} app - Express app + * @param {any} jsonParser - Express JSON parser + */ +async function registerEndpoints(app, jsonParser) { + app.post('/api/vector/query', jsonParser, async (req, res) => { + try { + if (!req.body.collectionId || !req.body.searchText) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + const searchText = String(req.body.searchText); + + const results = await queryCollection(collectionId, searchText); + return res.json(results); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); + + app.post('/api/vector/insert', jsonParser, async (req, res) => { + try { + if (!Array.isArray(req.body.items) || !req.body.collectionId) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + const items = req.body.items.map(x => ({ hash: x.hash, text: x.text })); + + await insertVectorItems(collectionId, items); + return res.sendStatus(200); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); + + app.post('/api/vector/list', jsonParser, async (req, res) => { + try { + if (!req.body.collectionId) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + + const hashes = await getSavedHashes(collectionId); + return res.json(hashes); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); + + app.post('/api/vector/delete', jsonParser, async (req, res) => { + try { + if (!Array.isArray(req.body.hashes) || !req.body.collectionId) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + const hashes = req.body.hashes.map(x => Number(x)); + + await deleteVectorItems(collectionId, hashes); + return res.sendStatus(200); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); +} + +module.exports = { registerEndpoints }; From 92ab17b58b6f75fd3cdc5f71d4f36abd9c923ab9 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:28:53 +0300 Subject: [PATCH 02/34] Fix HypeBot plugin settings not saving --- public/scripts/extensions/hypebot/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/scripts/extensions/hypebot/index.js b/public/scripts/extensions/hypebot/index.js index 2fbd66764..a64fe5d9f 100644 --- a/public/scripts/extensions/hypebot/index.js +++ b/public/scripts/extensions/hypebot/index.js @@ -191,11 +191,13 @@ jQuery(() => { settings.enabled = $('#hypebot_enabled').prop('checked'); hypeBotBar.toggle(settings.enabled); abortController?.abort(); + Object.assign(extension_settings.hypebot, settings); saveSettingsDebounced(); }); $('#hypebot_name').val(settings.name).on('input', () => { settings.name = String($('#hypebot_name').val()); + Object.assign(extension_settings.hypebot, settings); saveSettingsDebounced(); }); From 9d45c0a01813abda665a3a166a289aac2199d38a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 8 Sep 2023 00:28:06 +0300 Subject: [PATCH 03/34] Add UI plugin for vectors --- public/scripts/extensions.js | 2 + public/scripts/extensions/vectors/index.js | 267 ++++++++++++++++++ .../scripts/extensions/vectors/manifest.json | 12 + .../scripts/extensions/vectors/settings.html | 14 + src/vectors.js | 6 +- 5 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 public/scripts/extensions/vectors/index.js create mode 100644 public/scripts/extensions/vectors/manifest.json create mode 100644 public/scripts/extensions/vectors/settings.html diff --git a/public/scripts/extensions.js b/public/scripts/extensions.js index 0d0018d17..5e633f17b 100644 --- a/public/scripts/extensions.js +++ b/public/scripts/extensions.js @@ -153,6 +153,8 @@ const extension_settings = { }, speech_recognition: {}, rvc: {}, + hypebot: {}, + vectors: {}, }; let modules = []; diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js new file mode 100644 index 000000000..3b63bf395 --- /dev/null +++ b/public/scripts/extensions/vectors/index.js @@ -0,0 +1,267 @@ +import { eventSource, event_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced } from "../../../script.js"; +import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js"; +import { collapseNewlines } from "../../power-user.js"; +import { debounce, getStringHash as calculateHash } from "../../utils.js"; + +const MODULE_NAME = 'vectors'; +const MIN_TO_LEAVE = 5; +const QUERY_AMOUNT = 2; +const LEAVE_RATIO = 0.5; + +const settings = { + enabled: false, +}; + +const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); + +async function synchronizeChat() { + try { + if (!settings.enabled) { + return; + } + + const context = getContext(); + const chatId = getCurrentChatId(); + + if (!chatId || !Array.isArray(context.chat)) { + console.debug('Vectors: No chat selected'); + return; + } + + const hashedMessages = context.chat.filter(x => !x.is_system).map(x => ({ text: String(x.mes), hash: getStringHash(x.mes) })); + const hashesInCollection = await getSavedHashes(chatId); + + const newVectorItems = hashedMessages.filter(x => !hashesInCollection.includes(x.hash)); + const deletedHashes = hashesInCollection.filter(x => !hashedMessages.some(y => y.hash === x)); + + if (newVectorItems.length > 0) { + await insertVectorItems(chatId, newVectorItems); + console.log(`Vectors: Inserted ${newVectorItems.length} new items`); + } + + if (deletedHashes.length > 0) { + await deleteVectorItems(chatId, deletedHashes); + console.log(`Vectors: Deleted ${deletedHashes.length} old hashes`); + } + } catch (error) { + console.error('Vectors: Failed to synchronize chat', error); + } +} + +// Cache object for storing hash values +const hashCache = {}; + +/** + * Gets the hash value for a given string + * @param {string} str Input string + * @returns {number} Hash value + */ +function getStringHash(str) { + // Check if the hash is already in the cache + if (hashCache.hasOwnProperty(str)) { + return hashCache[str]; + } + + // Calculate the hash value + const hash = calculateHash(str); + + // Store the hash in the cache + hashCache[str] = hash; + + return hash; +} + +/** + * Rearranges the chat based on the relevance of recent messages + * @param {object[]} chat Array of chat messages + */ +async function rearrangeChat(chat) { + try { + if (!settings.enabled) { + return; + } + + const chatId = getCurrentChatId(); + + if (!chatId || !Array.isArray(chat)) { + console.debug('Vectors: No chat selected'); + return; + } + + if (chat.length < MIN_TO_LEAVE) { + console.debug(`Vectors: Not enough messages to rearrange (less than ${MIN_TO_LEAVE})`); + return; + } + + const queryText = getQueryText(chat); + + if (queryText.length === 0) { + console.debug('Vectors: No text to query'); + return; + } + + const queryHashes = await queryCollection(chatId, queryText); + + // Sorting logic + // 1. 50% of messages at the end stay in the same place (minimum 5) + // 2. Messages that are in the query are rearranged to match the query order + // 3. Messages that are not in the query and are not in the top 50% stay in the same place + const queriedMessages = []; + const remainingMessages = []; + + // Leave the last N messages intact + const retainMessagesCount = Math.max(Math.floor(chat.length * LEAVE_RATIO), MIN_TO_LEAVE); + const lastNMessages = chat.slice(-retainMessagesCount); + + // Splitting messages into queried and remaining messages + for (const message of chat) { + if (lastNMessages.includes(message)) { + continue; + } + + if (message.mes && queryHashes.includes(getStringHash(message.mes))) { + queriedMessages.push(message); + } else { + remainingMessages.push(message); + } + } + + // Rearrange queried messages to match query order + // Order is reversed because more relevant are at the lower indices + queriedMessages.sort((a, b) => { + return queryHashes.indexOf(getStringHash(b.mes)) - queryHashes.indexOf(getStringHash(a.mes)); + }); + + // Construct the final rearranged chat + const rearrangedChat = [...remainingMessages, ...queriedMessages, ...lastNMessages]; + + if (rearrangedChat.length !== chat.length) { + console.error('Vectors: Rearranged chat length does not match original chat length! This should not happen.'); + return; + } + + // Update the original chat array in-place + chat.splice(0, chat.length, ...rearrangedChat); + } catch (error) { + console.error('Vectors: Failed to rearrange chat', error); + } +} + +window['vectors_rearrangeChat'] = rearrangeChat; + +const onChatEvent = debounce(async () => await moduleWorker.update(), 500); + +function getQueryText(chat) { + let queryText = ''; + let i = 0; + + for (const message of chat.slice().reverse()) { + if (message.mes) { + queryText += message.mes + '\n'; + i++; + } + + if (i === QUERY_AMOUNT) { + break; + } + } + + return collapseNewlines(queryText).trim(); +} + +/** + * Gets the saved hashes for a collection +* @param {string} collectionId +* @returns {Promise} Saved hashes +*/ +async function getSavedHashes(collectionId) { + const response = await fetch('/api/vector/list', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ collectionId }), + }); + + if (!response.ok) { + throw new Error(`Failed to get saved hashes for collection ${collectionId}`); + } + + const hashes = await response.json(); + return hashes; +} + +/** + * Inserts vector items into a collection + * @param {string} collectionId - The collection to insert into + * @param {{ hash: number, text: string }[]} items - The items to insert + * @returns {Promise} + */ +async function insertVectorItems(collectionId, items) { + const response = await fetch('/api/vector/insert', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ collectionId, items }), + }); + + if (!response.ok) { + throw new Error(`Failed to insert vector items for collection ${collectionId}`); + } +} + +/** + * Deletes vector items from a collection + * @param {string} collectionId - The collection to delete from + * @param {number[]} hashes - The hashes of the items to delete + * @returns {Promise} + */ +async function deleteVectorItems(collectionId, hashes) { + const response = await fetch('/api/vector/delete', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ collectionId, hashes }), + }); + + if (!response.ok) { + throw new Error(`Failed to delete vector items for collection ${collectionId}`); + } +} + +/** + * @param {string} collectionId - The collection to query + * @param {string} searchText - The text to query + * @returns {Promise} - Hashes of the results + */ +async function queryCollection(collectionId, searchText) { + const response = await fetch('/api/vector/query', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ collectionId, searchText }), + }); + + if (!response.ok) { + throw new Error(`Failed to query collection ${collectionId}`); + } + + const results = await response.json(); + return results; +} + +jQuery(async () => { + if (!extension_settings.vectors) { + extension_settings.vectors = settings; + } + + Object.assign(settings, extension_settings.vectors); + $('#extensions_settings2').append(renderExtensionTemplate(MODULE_NAME, 'settings')); + $('#vectors_enabled').prop('checked', settings.enabled).on('input', () => { + settings.enabled = $('#vectors_enabled').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + + eventSource.on(event_types.CHAT_CHANGED, onChatEvent); + eventSource.on(event_types.MESSAGE_DELETED, onChatEvent); + eventSource.on(event_types.MESSAGE_EDITED, onChatEvent); + eventSource.on(event_types.MESSAGE_SENT, onChatEvent); + eventSource.on(event_types.MESSAGE_RECEIVED, onChatEvent); + eventSource.on(event_types.MESSAGE_SWIPED, onChatEvent); +}); diff --git a/public/scripts/extensions/vectors/manifest.json b/public/scripts/extensions/vectors/manifest.json new file mode 100644 index 000000000..7f84c2147 --- /dev/null +++ b/public/scripts/extensions/vectors/manifest.json @@ -0,0 +1,12 @@ +{ + "display_name": "Vector Storage", + "loading_order": 100, + "requires": [], + "optional": [], + "generate_interceptor": "vectors_rearrangeChat", + "js": "index.js", + "css": "", + "author": "Cohee#1207", + "version": "1.0.0", + "homePage": "https://github.com/SillyTavern/SillyTavern" +} diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html new file mode 100644 index 000000000..d0e0b6294 --- /dev/null +++ b/public/scripts/extensions/vectors/settings.html @@ -0,0 +1,14 @@ +
+
+
+ Vector Storage +
+
+
+ +
+
+
diff --git a/src/vectors.js b/src/vectors.js index f09444ec9..81c2a4161 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -12,14 +12,14 @@ class EmbeddingModel { /** * @type {encoder.UniversalSentenceEncoder} - The embedding model */ - #model; + model; async get() { - if (!this.#model) { + if (!this.model) { this.model = await encoder.load(); } - return this.#model; + return this.model; } } From 96df705409249d31c04eb3a87a8688d02c913221 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 8 Sep 2023 01:26:26 +0300 Subject: [PATCH 04/34] Change insertion strategy to an extension block --- public/script.js | 9 +-- .../extensions/infinity-context/index.js | 2 +- public/scripts/extensions/memory/index.js | 2 +- public/scripts/extensions/vectors/index.js | 68 +++++++++---------- public/scripts/openai.js | 8 +++ src/vectors.js | 16 ++--- 6 files changed, 51 insertions(+), 54 deletions(-) diff --git a/public/script.js b/public/script.js index b8292dc73..e9945ee9a 100644 --- a/public/script.js +++ b/public/script.js @@ -382,10 +382,7 @@ const system_message_types = { }; const extension_prompt_types = { - /** - * @deprecated Outdated term. In reality it's "after main prompt or story string" - */ - AFTER_SCENARIO: 0, + IN_PROMPT: 0, IN_CHAT: 1 }; @@ -2533,7 +2530,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject, addPersonaDescriptionExtensionPrompt(); // Call combined AN into Generate let allAnchors = getAllExtensionPrompts(); - const afterScenarioAnchor = getExtensionPrompt(extension_prompt_types.AFTER_SCENARIO); + const afterScenarioAnchor = getExtensionPrompt(extension_prompt_types.IN_PROMPT); let zeroDepthAnchor = getExtensionPrompt(extension_prompt_types.IN_CHAT, 0, ' '); const storyStringParams = { @@ -5591,7 +5588,7 @@ function select_rm_characters() { * @param {number} position Insertion position. 0 is after story string, 1 is in-chat with custom depth. * @param {number} depth Insertion depth. 0 represets the last message in context. Expected values up to 100. */ -function setExtensionPrompt(key, value, position, depth) { +export function setExtensionPrompt(key, value, position, depth) { extension_prompts[key] = { value: String(value), position: Number(position), depth: Number(depth) }; } diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index b06309e5b..303b2eee0 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -739,7 +739,7 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => { // No memories? No prompt. const promptBlob = (tokenApprox == 0) ? "" : wrapperMsg.replace('{{memories}}', allMemoryBlob); console.debug("CHROMADB: prompt blob: %o", promptBlob); - context.setExtensionPrompt(MODULE_NAME, promptBlob, extension_prompt_types.AFTER_SCENARIO); + context.setExtensionPrompt(MODULE_NAME, promptBlob, extension_prompt_types.IN_PROMPT); } if (selectedStrategy === 'custom') { const context = getContext(); diff --git a/public/scripts/extensions/memory/index.js b/public/scripts/extensions/memory/index.js index 8dcf841d5..d2eaebf4f 100644 --- a/public/scripts/extensions/memory/index.js +++ b/public/scripts/extensions/memory/index.js @@ -63,7 +63,7 @@ const defaultSettings = { source: summary_sources.extras, prompt: defaultPrompt, template: defaultTemplate, - position: extension_prompt_types.AFTER_SCENARIO, + position: extension_prompt_types.IN_PROMPT, depth: 2, promptWords: 200, promptMinWords: 25, diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 3b63bf395..cf78cf7ee 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -1,12 +1,14 @@ -import { eventSource, event_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced } from "../../../script.js"; +import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced, setExtensionPrompt } from "../../../script.js"; import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js"; import { collapseNewlines } from "../../power-user.js"; import { debounce, getStringHash as calculateHash } from "../../utils.js"; const MODULE_NAME = 'vectors'; -const MIN_TO_LEAVE = 5; -const QUERY_AMOUNT = 2; -const LEAVE_RATIO = 0.5; +const AMOUNT_TO_LEAVE = 5; +const INSERT_AMOUNT = 3; +const QUERY_TEXT_AMOUNT = 3; + +export const EXTENSION_PROMPT_TAG = '3_vectors'; const settings = { enabled: false, @@ -72,7 +74,7 @@ function getStringHash(str) { } /** - * Rearranges the chat based on the relevance of recent messages + * Removes the most relevant messages from the chat and displays them in the extension prompt * @param {object[]} chat Array of chat messages */ async function rearrangeChat(chat) { @@ -88,8 +90,8 @@ async function rearrangeChat(chat) { return; } - if (chat.length < MIN_TO_LEAVE) { - console.debug(`Vectors: Not enough messages to rearrange (less than ${MIN_TO_LEAVE})`); + if (chat.length < AMOUNT_TO_LEAVE) { + console.debug(`Vectors: Not enough messages to rearrange (less than ${AMOUNT_TO_LEAVE})`); return; } @@ -100,48 +102,34 @@ async function rearrangeChat(chat) { return; } - const queryHashes = await queryCollection(chatId, queryText); - - // Sorting logic - // 1. 50% of messages at the end stay in the same place (minimum 5) - // 2. Messages that are in the query are rearranged to match the query order - // 3. Messages that are not in the query and are not in the top 50% stay in the same place + // Get the most relevant messages, excluding the last few + const queryHashes = await queryCollection(chatId, queryText, INSERT_AMOUNT); const queriedMessages = []; - const remainingMessages = []; + const retainMessages = chat.slice(-AMOUNT_TO_LEAVE); - // Leave the last N messages intact - const retainMessagesCount = Math.max(Math.floor(chat.length * LEAVE_RATIO), MIN_TO_LEAVE); - const lastNMessages = chat.slice(-retainMessagesCount); - - // Splitting messages into queried and remaining messages for (const message of chat) { - if (lastNMessages.includes(message)) { + if (retainMessages.includes(message)) { continue; } - if (message.mes && queryHashes.includes(getStringHash(message.mes))) { queriedMessages.push(message); - } else { - remainingMessages.push(message); } } // Rearrange queried messages to match query order // Order is reversed because more relevant are at the lower indices - queriedMessages.sort((a, b) => { - return queryHashes.indexOf(getStringHash(b.mes)) - queryHashes.indexOf(getStringHash(a.mes)); - }); + queriedMessages.sort((a, b) => queryHashes.indexOf(getStringHash(b.mes)) - queryHashes.indexOf(getStringHash(a.mes))); - // Construct the final rearranged chat - const rearrangedChat = [...remainingMessages, ...queriedMessages, ...lastNMessages]; - - if (rearrangedChat.length !== chat.length) { - console.error('Vectors: Rearranged chat length does not match original chat length! This should not happen.'); - return; + // Remove queried messages from the original chat array + for (const message of chat) { + if (queriedMessages.includes(message)) { + chat.splice(chat.indexOf(message), 1); + } } - // Update the original chat array in-place - chat.splice(0, chat.length, ...rearrangedChat); + // Format queried messages into a single string + const queriedText = 'Past events: ' + queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); + setExtensionPrompt(EXTENSION_PROMPT_TAG, queriedText, extension_prompt_types.IN_PROMPT, 0); } catch (error) { console.error('Vectors: Failed to rearrange chat', error); } @@ -151,6 +139,11 @@ window['vectors_rearrangeChat'] = rearrangeChat; const onChatEvent = debounce(async () => await moduleWorker.update(), 500); +/** + * Gets the text to query from the chat + * @param {object[]} chat Chat messages + * @returns {string} Text to query + */ function getQueryText(chat) { let queryText = ''; let i = 0; @@ -161,7 +154,7 @@ function getQueryText(chat) { i++; } - if (i === QUERY_AMOUNT) { + if (i === QUERY_TEXT_AMOUNT) { break; } } @@ -228,13 +221,14 @@ async function deleteVectorItems(collectionId, hashes) { /** * @param {string} collectionId - The collection to query * @param {string} searchText - The text to query + * @param {number} topK - The number of results to return * @returns {Promise} - Hashes of the results */ -async function queryCollection(collectionId, searchText) { +async function queryCollection(collectionId, searchText, topK) { const response = await fetch('/api/vector/query', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ collectionId, searchText }), + body: JSON.stringify({ collectionId, searchText, topK }), }); if (!response.ok) { diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 3ac66f0df..ce265fd41 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -696,6 +696,14 @@ function preparePromptsForChatCompletion({Scenario, charPersonality, name2, worl identifier: 'authorsNote' }); + // Vectors Memory + const vectorsMemory = extensionPrompts['3_vectors']; + if (vectorsMemory && vectorsMemory.value) systemPrompts.push({ + role: 'system', + content: vectorsMemory.value, + identifier: 'vectorsMemory', + }); + // Persona Description if (power_user.persona_description && power_user.persona_description_position === persona_description_positions.IN_PROMPT) { systemPrompts.push({ role: 'system', content: power_user.persona_description, identifier: 'personaDescription' }); diff --git a/src/vectors.js b/src/vectors.js index 81c2a4161..26265bcb0 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -23,10 +23,6 @@ class EmbeddingModel { } } -/** - * Hard limit on the number of results to return from the vector search. - */ -const TOP_K = 100; const model = new EmbeddingModel(); /** @@ -100,17 +96,18 @@ async function deleteVectorItems(collectionId, hashes) { /** * Gets the hashes of the items in the vector collection that match the search text - * @param {string} collectionId - * @param {string} searchText + * @param {string} collectionId - The collection ID + * @param {string} searchText - The text to search for + * @param {number} topK - The number of results to return * @returns {Promise} - The hashes of the items that match the search text */ -async function queryCollection(collectionId, searchText) { +async function queryCollection(collectionId, searchText, topK) { const index = await getIndex(collectionId); const use = await model.get(); const tensor = await use.embed(searchText); const vector = Array.from(await tensor.data()); - const result = await index.queryItems(vector, TOP_K); + const result = await index.queryItems(vector, topK); const hashes = result.map(x => Number(x.item.metadata.hash)); return hashes; } @@ -129,8 +126,9 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const searchText = String(req.body.searchText); + const topK = Number(req.body.topK) || 10; - const results = await queryCollection(collectionId, searchText); + const results = await queryCollection(collectionId, searchText, topK); return res.json(results); } catch (error) { console.error(error); From 02bdd56e20040cfddbae218d83205ea8939f9da8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 8 Sep 2023 12:10:41 +0300 Subject: [PATCH 05/34] Make printMessages async --- public/script.js | 21 +++++++++++---------- public/scripts/group-chats.js | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/public/script.js b/public/script.js index e9945ee9a..0bd08b2a3 100644 --- a/public/script.js +++ b/public/script.js @@ -1144,10 +1144,11 @@ async function replaceCurrentChat() { } } -function printMessages() { - chat.forEach(function (item, i, arr) { - addOneMessage(item, { scroll: i === arr.length - 1 }); - }); +async function printMessages() { + for (let i = 0; i < chat.length; i++) { + const item = chat[i]; + addOneMessage(item, { scroll: i === chat.length - 1 }); + } if (power_user.lazy_load > 0) { const height = $('#chat').height(); @@ -1190,7 +1191,7 @@ export async function reloadCurrentChat() { } else { resetChatState(); - printMessages(); + await printMessages(); } await eventSource.emit(event_types.CHAT_CHANGED, getCurrentChatId()); @@ -4448,7 +4449,7 @@ async function getChatResult() { chat.push(message); await saveChatConditional(); } - printMessages(); + await printMessages(); select_selected_character(this_chid); await eventSource.emit(event_types.CHAT_CHANGED, (getCurrentChatId())); @@ -5010,12 +5011,12 @@ async function saveSettings(type) { dataType: "json", contentType: "application/json", //processData: false, - success: function (data) { + success: async function (data) { //online_status = data.result; eventSource.emit(event_types.SETTINGS_UPDATED); if (type == "change_name") { clearChat(); - printMessages(); + await printMessages(); } }, error: function (jqXHR, exception) { @@ -6289,7 +6290,7 @@ async function createOrEditCharacter(e) { await eventSource.emit(event_types.MESSAGE_RECEIVED, (chat.length - 1)); clearChat(); - printMessages(); + await printMessages(); await eventSource.emit(event_types.CHARACTER_MESSAGE_RENDERED, (chat.length - 1)); await saveChatConditional(); } @@ -6918,7 +6919,7 @@ export async function deleteCharacter(name, avatar) { delete tag_map[avatar]; await getCharacters(); select_rm_info("char_delete", name); - printMessages(); + await printMessages(); saveSettingsDebounced(); } diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index ee75a3571..eff5eaffb 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -166,7 +166,7 @@ export async function getGroupChat(groupId) { for (let key of data) { chat.push(key); } - printMessages(); + await printMessages(); } else { sendSystemMessage(system_message_types.GROUP, '', { isSmallSys: true }); if (group && Array.isArray(group.members)) { @@ -832,7 +832,7 @@ async function deleteGroup(id) { delete tag_map[id]; resetChatState(); clearChat(); - printMessages(); + await printMessages(); await getCharacters(); select_rm_info("group_delete", id); From a5acc7872d2821cb6415dc4e076068efc0c76260 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 8 Sep 2023 13:57:27 +0300 Subject: [PATCH 06/34] Add OpenAI vector source. --- public/scripts/extensions/vectors/index.js | 66 ++++++-- .../scripts/extensions/vectors/settings.html | 7 + public/scripts/openai.js | 6 + server.js | 120 ++------------ src/local-vectors.js | 38 +++++ src/openai-vectors.js | 47 ++++++ src/secrets.js | 146 ++++++++++++++++++ src/vectors.js | 74 ++++----- 8 files changed, 345 insertions(+), 159 deletions(-) create mode 100644 src/local-vectors.js create mode 100644 src/openai-vectors.js create mode 100644 src/secrets.js diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index cf78cf7ee..c04a6780e 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -6,20 +6,21 @@ import { debounce, getStringHash as calculateHash } from "../../utils.js"; const MODULE_NAME = 'vectors'; const AMOUNT_TO_LEAVE = 5; const INSERT_AMOUNT = 3; -const QUERY_TEXT_AMOUNT = 3; +const QUERY_TEXT_AMOUNT = 2; export const EXTENSION_PROMPT_TAG = '3_vectors'; const settings = { enabled: false, + source: 'local', }; const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); -async function synchronizeChat() { +async function synchronizeChat(batchSize = 10) { try { if (!settings.enabled) { - return; + return -1; } const context = getContext(); @@ -37,7 +38,7 @@ async function synchronizeChat() { const deletedHashes = hashesInCollection.filter(x => !hashedMessages.some(y => y.hash === x)); if (newVectorItems.length > 0) { - await insertVectorItems(chatId, newVectorItems); + await insertVectorItems(chatId, newVectorItems.slice(0, batchSize)); console.log(`Vectors: Inserted ${newVectorItems.length} new items`); } @@ -45,6 +46,8 @@ async function synchronizeChat() { await deleteVectorItems(chatId, deletedHashes); console.log(`Vectors: Deleted ${deletedHashes.length} old hashes`); } + + return newVectorItems.length - batchSize; } catch (error) { console.error('Vectors: Failed to synchronize chat', error); } @@ -59,18 +62,18 @@ const hashCache = {}; * @returns {number} Hash value */ function getStringHash(str) { - // Check if the hash is already in the cache - if (hashCache.hasOwnProperty(str)) { - return hashCache[str]; - } + // Check if the hash is already in the cache + if (hashCache.hasOwnProperty(str)) { + return hashCache[str]; + } - // Calculate the hash value - const hash = calculateHash(str); + // Calculate the hash value + const hash = calculateHash(str); - // Store the hash in the cache - hashCache[str] = hash; + // Store the hash in the cache + hashCache[str] = hash; - return hash; + return hash; } /** @@ -79,6 +82,9 @@ function getStringHash(str) { */ async function rearrangeChat(chat) { try { + // Clear the extension prompt + setExtensionPrompt(EXTENSION_PROMPT_TAG, '', extension_prompt_types.IN_PROMPT, 0); + if (!settings.enabled) { return; } @@ -127,6 +133,11 @@ async function rearrangeChat(chat) { } } + if (queriedMessages.length === 0) { + console.debug('Vectors: No relevant messages found'); + return; + } + // Format queried messages into a single string const queriedText = 'Past events: ' + queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); setExtensionPrompt(EXTENSION_PROMPT_TAG, queriedText, extension_prompt_types.IN_PROMPT, 0); @@ -171,7 +182,10 @@ async function getSavedHashes(collectionId) { const response = await fetch('/api/vector/list', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ collectionId }), + body: JSON.stringify({ + collectionId: collectionId, + source: settings.source, + }), }); if (!response.ok) { @@ -192,7 +206,11 @@ async function insertVectorItems(collectionId, items) { const response = await fetch('/api/vector/insert', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ collectionId, items }), + body: JSON.stringify({ + collectionId: collectionId, + items: items, + source: settings.source, + }), }); if (!response.ok) { @@ -210,7 +228,11 @@ async function deleteVectorItems(collectionId, hashes) { const response = await fetch('/api/vector/delete', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ collectionId, hashes }), + body: JSON.stringify({ + collectionId: collectionId, + hashes: hashes, + source: settings.source, + }), }); if (!response.ok) { @@ -228,7 +250,12 @@ async function queryCollection(collectionId, searchText, topK) { const response = await fetch('/api/vector/query', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ collectionId, searchText, topK }), + body: JSON.stringify({ + collectionId: collectionId, + searchText: searchText, + topK: topK, + source: settings.source, + }), }); if (!response.ok) { @@ -251,6 +278,11 @@ jQuery(async () => { Object.assign(extension_settings.vectors, settings); saveSettingsDebounced(); }); + $('#vectors_source').val(settings.source).on('change', () => { + settings.source = String($('#vectors_source').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); eventSource.on(event_types.CHAT_CHANGED, onChatEvent); eventSource.on(event_types.MESSAGE_DELETED, onChatEvent); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index d0e0b6294..a76db1aa8 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -9,6 +9,13 @@ Enabled + + diff --git a/public/scripts/openai.js b/public/scripts/openai.js index ce265fd41..ba1dafc62 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -629,6 +629,12 @@ function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, ty if (true === afterScenario) chatCompletion.insert(authorsNote, 'scenario'); } + // Vectors Memory + if (prompts.has('vectorsMemory')) { + const vectorsMemory = Message.fromPrompt(prompts.get('vectorsMemory')); + chatCompletion.insert(vectorsMemory, 'main'); + } + // Decide whether dialogue examples should always be added if (power_user.pin_examples) { populateDialogueExamples(prompts, chatCompletion); diff --git a/server.js b/server.js index 18134a04b..c342c3e81 100644 --- a/server.js +++ b/server.js @@ -68,6 +68,7 @@ const characterCardParser = require('./src/character-card-parser.js'); const contentManager = require('./src/content-manager'); const novelai = require('./src/novelai'); const statsHelpers = require('./statsHelpers.js'); +const { writeSecret, readSecret, readSecretState, migrateSecrets, SECRET_KEYS, getAllSecrets } = require('./src/secrets'); function createDefaultFiles() { const files = { @@ -325,6 +326,7 @@ function humanizedISO8601DateTime(date) { var charactersPath = 'public/characters/'; var chatsPath = 'public/chats/'; const UPLOADS_PATH = './uploads'; +const SETTINGS_FILE = './public/settings.json'; const AVATAR_WIDTH = 400; const AVATAR_HEIGHT = 600; const jsonParser = express.json({ limit: '100mb' }); @@ -4070,7 +4072,7 @@ const setupTasks = async function () { console.log(`SillyTavern ${version.pkgVersion}` + (version.gitBranch ? ` '${version.gitBranch}' (${version.gitRevision})` : '')); backupSettings(); - migrateSecrets(); + migrateSecrets(SETTINGS_FILE); ensurePublicDirectoriesExist(); await ensureThumbnailCache(); contentManager.checkForNewContent(); @@ -4221,69 +4223,6 @@ function ensurePublicDirectoriesExist() { } } -const SECRETS_FILE = './secrets.json'; -const SETTINGS_FILE = './public/settings.json'; -const SECRET_KEYS = { - HORDE: 'api_key_horde', - MANCER: 'api_key_mancer', - OPENAI: 'api_key_openai', - NOVEL: 'api_key_novel', - CLAUDE: 'api_key_claude', - DEEPL: 'deepl', - LIBRE: 'libre', - LIBRE_URL: 'libre_url', - OPENROUTER: 'api_key_openrouter', - SCALE: 'api_key_scale', - AI21: 'api_key_ai21', - SCALE_COOKIE: 'scale_cookie', -} - -function migrateSecrets() { - if (!fs.existsSync(SETTINGS_FILE)) { - console.log('Settings file does not exist'); - return; - } - - try { - let modified = false; - const fileContents = fs.readFileSync(SETTINGS_FILE, 'utf8'); - const settings = JSON.parse(fileContents); - const oaiKey = settings?.api_key_openai; - const hordeKey = settings?.horde_settings?.api_key; - const novelKey = settings?.api_key_novel; - - if (typeof oaiKey === 'string') { - console.log('Migrating OpenAI key...'); - writeSecret(SECRET_KEYS.OPENAI, oaiKey); - delete settings.api_key_openai; - modified = true; - } - - if (typeof hordeKey === 'string') { - console.log('Migrating Horde key...'); - writeSecret(SECRET_KEYS.HORDE, hordeKey); - delete settings.horde_settings.api_key; - modified = true; - } - - if (typeof novelKey === 'string') { - console.log('Migrating Novel key...'); - writeSecret(SECRET_KEYS.NOVEL, novelKey); - delete settings.api_key_novel; - modified = true; - } - - if (modified) { - console.log('Writing updated settings.json...'); - const settingsContent = JSON.stringify(settings); - writeFileAtomicSync(SETTINGS_FILE, settingsContent, "utf-8"); - } - } - catch (error) { - console.error('Could not migrate secrets file. Proceed with caution.'); - } -} - app.post('/writesecret', jsonParser, (request, response) => { const key = request.body.key; const value = request.body.value; @@ -4293,19 +4232,9 @@ app.post('/writesecret', jsonParser, (request, response) => { }); app.post('/readsecretstate', jsonParser, (_, response) => { - if (!fs.existsSync(SECRETS_FILE)) { - return response.send({}); - } try { - const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8'); - const secrets = JSON.parse(fileContents); - const state = {}; - - for (const key of Object.values(SECRET_KEYS)) { - state[key] = !!secrets[key]; // convert to boolean - } - + const state = readSecretState(); return response.send(state); } catch (error) { console.error(error); @@ -4351,14 +4280,13 @@ app.post('/viewsecrets', jsonParser, async (_, response) => { return response.sendStatus(403); } - if (!fs.existsSync(SECRETS_FILE)) { - console.error('secrets.json does not exist'); - return response.sendStatus(404); - } - try { - const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); - const secrets = JSON.parse(fileContents); + const secrets = getAllSecrets(); + + if (!secrets) { + return response.sendStatus(404); + } + return response.send(secrets); } catch (error) { console.error(error); @@ -4797,6 +4725,11 @@ app.post('/libre_translate', jsonParser, async (request, response) => { const key = readSecret(SECRET_KEYS.LIBRE); const url = readSecret(SECRET_KEYS.LIBRE_URL); + if (!url) { + console.log('LibreTranslate URL is not configured.'); + return response.sendStatus(401); + } + const text = request.body.text; const lang = request.body.lang; @@ -5292,27 +5225,6 @@ function importRisuSprites(data) { } } -function writeSecret(key, value) { - if (!fs.existsSync(SECRETS_FILE)) { - const emptyFile = JSON.stringify({}); - writeFileAtomicSync(SECRETS_FILE, emptyFile, "utf-8"); - } - - const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); - const secrets = JSON.parse(fileContents); - secrets[key] = value; - writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8"); -} - -function readSecret(key) { - if (!fs.existsSync(SECRETS_FILE)) { - return undefined; - } - - const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); - const secrets = JSON.parse(fileContents); - return secrets[key]; -} async function readAllChunks(readableStream) { return new Promise((resolve, reject) => { @@ -5385,8 +5297,6 @@ async function getImageBuffers(zipFilePath) { }); } - - /** * This function extracts the extension information from the manifest file. * @param {string} extensionPath - The path of the extension folder diff --git a/src/local-vectors.js b/src/local-vectors.js new file mode 100644 index 000000000..73d514840 --- /dev/null +++ b/src/local-vectors.js @@ -0,0 +1,38 @@ + +require('@tensorflow/tfjs'); +const encoder = require('@tensorflow-models/universal-sentence-encoder'); + +/** + * Lazy loading class for the embedding model. + */ +class EmbeddingModel { + /** + * @type {encoder.UniversalSentenceEncoder} - The embedding model + */ + model; + + async get() { + if (!this.model) { + this.model = await encoder.load(); + } + + return this.model; + } +} + +const model = new EmbeddingModel(); + +/** + * @param {string} text + */ +async function getLocalVector(text) { + const use = await model.get(); + const tensor = await use.embed(text); + const vector = Array.from(await tensor.data()); + + return vector; +} + +module.exports = { + getLocalVector, +}; diff --git a/src/openai-vectors.js b/src/openai-vectors.js new file mode 100644 index 000000000..f81590f88 --- /dev/null +++ b/src/openai-vectors.js @@ -0,0 +1,47 @@ +const fetch = require('node-fetch').default; +const { SECRET_KEYS, readSecret } = require('./secrets'); + +/** + * Gets the vector for the given text from OpenAI ada model + * @param {string} text - The text to get the vector for + * @returns {Promise} - The vector for the text + */ +async function getOpenAIVector(text) { + const key = readSecret(SECRET_KEYS.OPENAI); + + if (!key) { + console.log('No OpenAI key found'); + throw new Error('No OpenAI key found'); + } + + const response = await fetch('https://api.openai.com/v1/embeddings', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${key}`, + }, + body: JSON.stringify({ + input: text, + model: 'text-embedding-ada-002', + }) + }); + + if (!response.ok) { + console.log('OpenAI request failed'); + throw new Error('OpenAI request failed'); + } + + const data = await response.json(); + const vector = data?.data[0]?.embedding; + + if (!Array.isArray(vector)) { + console.log('OpenAI response was not an array'); + throw new Error('OpenAI response was not an array'); + } + + return vector; +} + +module.exports = { + getOpenAIVector, +}; diff --git a/src/secrets.js b/src/secrets.js new file mode 100644 index 000000000..eee18dcbc --- /dev/null +++ b/src/secrets.js @@ -0,0 +1,146 @@ +const fs = require('fs'); +const path = require('path'); +const writeFileAtomicSync = require('write-file-atomic').sync; + +const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); +const SECRET_KEYS = { + HORDE: 'api_key_horde', + MANCER: 'api_key_mancer', + OPENAI: 'api_key_openai', + NOVEL: 'api_key_novel', + CLAUDE: 'api_key_claude', + DEEPL: 'deepl', + LIBRE: 'libre', + LIBRE_URL: 'libre_url', + OPENROUTER: 'api_key_openrouter', + SCALE: 'api_key_scale', + AI21: 'api_key_ai21', + SCALE_COOKIE: 'scale_cookie', +} + +/** + * Writes a secret to the secrets file + * @param {string} key Secret key + * @param {string} value Secret value + */ +function writeSecret(key, value) { + if (!fs.existsSync(SECRETS_FILE)) { + const emptyFile = JSON.stringify({}); + writeFileAtomicSync(SECRETS_FILE, emptyFile, "utf-8"); + } + + const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); + const secrets = JSON.parse(fileContents); + secrets[key] = value; + writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8"); +} + +/** + * Reads a secret from the secrets file + * @param {string} key Secret key + * @returns {string} Secret value + */ +function readSecret(key) { + if (!fs.existsSync(SECRETS_FILE)) { + return ''; + } + + const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); + const secrets = JSON.parse(fileContents); + return secrets[key]; +} + +/** + * Reads the secret state from the secrets file + * @returns {object} Secret state + */ +function readSecretState() { + if (!fs.existsSync(SECRETS_FILE)) { + return {}; + } + + const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8'); + const secrets = JSON.parse(fileContents); + const state = {}; + + for (const key of Object.values(SECRET_KEYS)) { + state[key] = !!secrets[key]; // convert to boolean + } + + return state; +} + +/** + * Migrates secrets from settings.json to secrets.json + * @param {string} settingsFile Path to settings.json + * @returns {void} + */ +function migrateSecrets(settingsFile) { + if (!fs.existsSync(settingsFile)) { + console.log('Settings file does not exist'); + return; + } + + try { + let modified = false; + const fileContents = fs.readFileSync(settingsFile, 'utf8'); + const settings = JSON.parse(fileContents); + const oaiKey = settings?.api_key_openai; + const hordeKey = settings?.horde_settings?.api_key; + const novelKey = settings?.api_key_novel; + + if (typeof oaiKey === 'string') { + console.log('Migrating OpenAI key...'); + writeSecret(SECRET_KEYS.OPENAI, oaiKey); + delete settings.api_key_openai; + modified = true; + } + + if (typeof hordeKey === 'string') { + console.log('Migrating Horde key...'); + writeSecret(SECRET_KEYS.HORDE, hordeKey); + delete settings.horde_settings.api_key; + modified = true; + } + + if (typeof novelKey === 'string') { + console.log('Migrating Novel key...'); + writeSecret(SECRET_KEYS.NOVEL, novelKey); + delete settings.api_key_novel; + modified = true; + } + + if (modified) { + console.log('Writing updated settings.json...'); + const settingsContent = JSON.stringify(settings); + writeFileAtomicSync(settingsFile, settingsContent, "utf-8"); + } + } + catch (error) { + console.error('Could not migrate secrets file. Proceed with caution.'); + } +} + +/** + * Reads all secrets from the secrets file + * @returns {Record | undefined} Secrets + */ +function getAllSecrets() { + if (!fs.existsSync(SECRETS_FILE)) { + console.log('Secrets file does not exist'); + return undefined; + } + + const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8'); + const secrets = JSON.parse(fileContents); + return secrets; +} + +module.exports = { + writeSecret, + readSecret, + readSecretState, + migrateSecrets, + getAllSecrets, + SECRET_KEYS, +}; diff --git a/src/vectors.js b/src/vectors.js index 26265bcb0..15a7c3871 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -2,36 +2,32 @@ const express = require('express'); const vectra = require('vectra'); const path = require('path'); const sanitize = require('sanitize-filename'); -require('@tensorflow/tfjs'); -const encoder = require('@tensorflow-models/universal-sentence-encoder'); /** - * Lazy loading class for the embedding model. + * Gets the vector for the given text from the given source. + * @param {string} source - The source of the vector + * @param {string} text - The text to get the vector for + * @returns {Promise} - The vector for the text */ -class EmbeddingModel { - /** - * @type {encoder.UniversalSentenceEncoder} - The embedding model - */ - model; - - async get() { - if (!this.model) { - this.model = await encoder.load(); - } - - return this.model; +async function getVector(source, text) { + switch (source) { + case 'local': + return require('./local-vectors').getLocalVector(text); + case 'openai': + return require('./openai-vectors').getOpenAIVector(text); } -} -const model = new EmbeddingModel(); + throw new Error(`Unknown vector source ${source}`); +} /** * Gets the index for the vector collection * @param {string} collectionId - The collection ID + * @param {string} source - The source of the vector * @returns {Promise} - The index for the collection */ -async function getIndex(collectionId) { - const index = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(collectionId))); +async function getIndex(collectionId, source) { + const index = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(source), sanitize(collectionId))); if (!await index.isIndexCreated()) { await index.createIndex(); @@ -43,19 +39,18 @@ async function getIndex(collectionId) { /** * Inserts items into the vector collection * @param {string} collectionId - The collection ID + * @param {string} source - The source of the vector * @param {{ hash: number; text: string; }[]} items - The items to insert */ -async function insertVectorItems(collectionId, items) { - const index = await getIndex(collectionId); - const use = await model.get(); +async function insertVectorItems(collectionId, source, items) { + const index = await getIndex(collectionId, source); await index.beginUpdate(); for (const item of items) { const text = item.text; const hash = item.hash; - const tensor = await use.embed(text); - const vector = Array.from(await tensor.data()); + const vector = await getVector(source, text); await index.upsertItem({ vector: vector, metadata: { hash, text } }); } @@ -65,10 +60,11 @@ async function insertVectorItems(collectionId, items) { /** * Gets the hashes of the items in the vector collection * @param {string} collectionId - The collection ID + * @param {string} source - The source of the vector * @returns {Promise} - The hashes of the items in the collection */ -async function getSavedHashes(collectionId) { - const index = await getIndex(collectionId); +async function getSavedHashes(collectionId, source) { + const index = await getIndex(collectionId, source); const items = await index.listItems(); const hashes = items.map(x => Number(x.metadata.hash)); @@ -79,10 +75,11 @@ async function getSavedHashes(collectionId) { /** * Deletes items from the vector collection by hash * @param {string} collectionId - The collection ID + * @param {string} source - The source of the vector * @param {number[]} hashes - The hashes of the items to delete */ -async function deleteVectorItems(collectionId, hashes) { - const index = await getIndex(collectionId); +async function deleteVectorItems(collectionId, source, hashes) { + const index = await getIndex(collectionId, source); const items = await index.listItemsByMetadata({ hash: { '$in': hashes } }); await index.beginUpdate(); @@ -97,15 +94,14 @@ async function deleteVectorItems(collectionId, hashes) { /** * Gets the hashes of the items in the vector collection that match the search text * @param {string} collectionId - The collection ID + * @param {string} source - The source of the vector * @param {string} searchText - The text to search for * @param {number} topK - The number of results to return * @returns {Promise} - The hashes of the items that match the search text */ -async function queryCollection(collectionId, searchText, topK) { - const index = await getIndex(collectionId); - const use = await model.get(); - const tensor = await use.embed(searchText); - const vector = Array.from(await tensor.data()); +async function queryCollection(collectionId, source, searchText, topK) { + const index = await getIndex(collectionId, source); + const vector = await getVector(source, searchText); const result = await index.queryItems(vector, topK); const hashes = result.map(x => Number(x.item.metadata.hash)); @@ -127,8 +123,9 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const searchText = String(req.body.searchText); const topK = Number(req.body.topK) || 10; + const source = String(req.body.source) || 'local'; - const results = await queryCollection(collectionId, searchText, topK); + const results = await queryCollection(collectionId, source, searchText, topK); return res.json(results); } catch (error) { console.error(error); @@ -144,8 +141,9 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const items = req.body.items.map(x => ({ hash: x.hash, text: x.text })); + const source = String(req.body.source) || 'local'; - await insertVectorItems(collectionId, items); + await insertVectorItems(collectionId, source, items); return res.sendStatus(200); } catch (error) { console.error(error); @@ -160,8 +158,9 @@ async function registerEndpoints(app, jsonParser) { } const collectionId = String(req.body.collectionId); + const source = String(req.body.source) || 'local'; - const hashes = await getSavedHashes(collectionId); + const hashes = await getSavedHashes(collectionId, source); return res.json(hashes); } catch (error) { console.error(error); @@ -177,8 +176,9 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const hashes = req.body.hashes.map(x => Number(x)); + const source = String(req.body.source) || 'local'; - await deleteVectorItems(collectionId, hashes); + await deleteVectorItems(collectionId, source, hashes); return res.sendStatus(200); } catch (error) { console.error(error); From 3a3ff89047db3d308c364be091eaa2cf81595c50 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:25:10 +0300 Subject: [PATCH 07/34] Add button to vectorize all chat --- public/scripts/extensions/vectors/index.js | 43 +++++++++++++++++++ .../scripts/extensions/vectors/settings.html | 13 ++++++ 2 files changed, 56 insertions(+) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index c04a6780e..7ad0c6fb3 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -17,6 +17,48 @@ const settings = { const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); +async function onVectorizeAllClick() { + try { + if (!settings.enabled) { + return; + } + + const chatId = getCurrentChatId(); + const batchSize = 5; + const elapsedLog = []; + let finished = false; + $('#vectorize_progress').show(); + $('#vectorize_progress_percent').text('0'); + $('#vectorize_progress_eta').text('...'); + + while (!finished) { + const startTime = Date.now(); + const remaining = await synchronizeChat(batchSize); + const elapsed = Date.now() - startTime; + elapsedLog.push(elapsed); + finished = remaining <= 0; + + const total = getContext().chat.length; + const processed = total - remaining; + const processedPercent = Math.round((processed / total) * 100); // percentage of the work done + const averageElapsed = elapsedLog.slice(-5).reduce((a, b) => a + b, 0) / elapsedLog.length; // average time needed to process one item + const pace = averageElapsed / batchSize; // time needed to process one item + const remainingTime = Math.round(pace * remaining / 1000); + + $('#vectorize_progress_percent').text(processedPercent); + $('#vectorize_progress_eta').text(remainingTime); + + if (chatId !== getCurrentChatId()) { + throw new Error('Chat changed'); + } + } + } catch (error) { + console.error('Vectors: Failed to vectorize all', error); + } finally { + $('#vectorize_progress').hide(); + } +} + async function synchronizeChat(batchSize = 10) { try { if (!settings.enabled) { @@ -283,6 +325,7 @@ jQuery(async () => { Object.assign(extension_settings.vectors, settings); saveSettingsDebounced(); }); + $('#vectors_vectorize_all').on('click', onVectorizeAllClick); eventSource.on(event_types.CHAT_CHANGED, onChatEvent); eventSource.on(event_types.MESSAGE_DELETED, onChatEvent); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index a76db1aa8..0d1d7309a 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -16,6 +16,19 @@ +
+ Old messages are vectorized gradually as you chat. + To process all previous messages, click the button below. +
+ + From 4d08e3e9be33a5f51fed03712a3adc76a415c848 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:12:54 +0300 Subject: [PATCH 08/34] Decrease batch size. Add browser console log --- public/scripts/extensions/vectors/index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 7ad0c6fb3..a364ea8a4 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -59,7 +59,7 @@ async function onVectorizeAllClick() { } } -async function synchronizeChat(batchSize = 10) { +async function synchronizeChat(batchSize = 5) { try { if (!settings.enabled) { return -1; @@ -181,8 +181,10 @@ async function rearrangeChat(chat) { } // Format queried messages into a single string - const queriedText = 'Past events: ' + queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); - setExtensionPrompt(EXTENSION_PROMPT_TAG, queriedText, extension_prompt_types.IN_PROMPT, 0); + const queriedText = queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); + console.log('Vectors: relevant past messages found.\n', queriedText); + const insertedText = `Past events: ${queriedText}`; + setExtensionPrompt(EXTENSION_PROMPT_TAG, insertedText, extension_prompt_types.IN_PROMPT, 0); } catch (error) { console.error('Vectors: Failed to rearrange chat', error); } From 967a084aad8fa79a7820a008c54da9246b4a4d00 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:14:16 +0300 Subject: [PATCH 09/34] (WIP) Local emotion classification pipeline --- .gitignore | 1 + package-lock.json | 304 +++++++++++++++--- package.json | 1 + .../scripts/extensions/expressions/index.js | 90 ++++-- .../extensions/expressions/settings.html | 16 +- server.js | 4 + src/classify.mjs | 48 +++ 7 files changed, 389 insertions(+), 75 deletions(-) create mode 100644 src/classify.mjs diff --git a/.gitignore b/.gitignore index 582031246..ab258371e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ cloudflared.exe public/assets/ access.log /vectors/ +/cache/ diff --git a/package-lock.json b/package-lock.json index ee683edaa..f8a23d2ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@dqbd/tiktoken": "^1.0.2", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", "@tensorflow/tfjs": "^4.10.0", + "@xenova/transformers": "^2.6.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -645,6 +646,60 @@ "node": ">= 8" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@tensorflow-models/universal-sentence-encoder": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@tensorflow-models/universal-sentence-encoder/-/universal-sentence-encoder-1.3.3.tgz", @@ -913,6 +968,18 @@ "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==", "peer": true }, + "node_modules/@xenova/transformers": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.6.0.tgz", + "integrity": "sha512-k9bs+reiwhn+kx0d4FYnlBTWtl8D5Q4fIzoKYxKbTTSVyS33KXbQESRpdIxiU9gtlMKML2Sw0Oep4FYK9dQCsQ==", + "dependencies": { + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, "node_modules/@xmldom/xmldom": { "version": "0.8.9", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.9.tgz", @@ -1060,6 +1127,11 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1083,7 +1155,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -1094,7 +1165,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1286,8 +1356,7 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/cliui": { "version": "8.0.1", @@ -1302,6 +1371,18 @@ "node": ">=12" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1318,6 +1399,15 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1520,7 +1610,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -1535,7 +1624,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true, "engines": { "node": ">=4.0.0" } @@ -1574,10 +1662,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", - "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "dev": true, + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", "engines": { "node": ">=8" } @@ -1688,7 +1775,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "dependencies": { "once": "^1.4.0" } @@ -1751,7 +1837,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "dev": true, "engines": { "node": ">=6" } @@ -1824,6 +1909,11 @@ } ] }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "node_modules/fast-glob": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.0.tgz", @@ -1902,6 +1992,11 @@ "node": ">= 0.8" } }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + }, "node_modules/follow-redirects": { "version": "1.15.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", @@ -1963,8 +2058,7 @@ "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { "version": "9.1.0", @@ -2020,8 +2114,7 @@ "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" }, "node_modules/glob-parent": { "version": "5.1.2", @@ -2091,6 +2184,11 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -2264,8 +2362,7 @@ "node_modules/ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "node_modules/into-stream": { "version": "6.0.0", @@ -2296,6 +2393,11 @@ "node": ">= 10" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/is-core-module": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", @@ -2545,7 +2647,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -2630,7 +2731,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, "engines": { "node": ">=10" }, @@ -2671,8 +2771,7 @@ "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/ms": { "version": "2.0.0", @@ -2748,8 +2847,7 @@ "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "dev": true + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" }, "node_modules/negotiator": { "version": "0.6.3", @@ -2763,7 +2861,6 @@ "version": "3.45.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.45.0.tgz", "integrity": "sha512-iwXuFrMAcFVi/ZoZiqq8BzAdsLw9kxDfTC0HMyjXfSL/6CSDAGD5UmR7azrAgWV1zKYq7dUUMj4owusBWKLsiQ==", - "dev": true, "dependencies": { "semver": "^7.3.5" }, @@ -2771,6 +2868,11 @@ "node": ">=10" } }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + }, "node_modules/node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", @@ -2845,11 +2947,50 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } }, + "node_modules/onnx-proto": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz", + "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==", + "dependencies": { + "protobufjs": "^6.8.8" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz", + "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==" + }, + "node_modules/onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.14.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, "node_modules/open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -3189,6 +3330,11 @@ "node": ">=10" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + }, "node_modules/png-chunk-text": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/png-chunk-text/-/png-chunk-text-1.0.0.tgz", @@ -3223,7 +3369,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", - "dev": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -3267,6 +3412,31 @@ "node": ">=0.4.0" } }, + "node_modules/protobufjs": { + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", + "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3296,7 +3466,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -3336,6 +3505,11 @@ } ] }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3370,7 +3544,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3550,7 +3723,6 @@ "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dev": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -3608,6 +3780,48 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/sharp": { + "version": "0.32.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.5.tgz", + "integrity": "sha512-0dap3iysgDkNaPOaOL4X/0akdu0ma62GcdC2NBQ+93eqpePdDdr2/LM0sFdDSMmN7yS+odyZtPsb7tx/cYBKnQ==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/sharp/node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -3636,7 +3850,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true, "funding": [ { "type": "github", @@ -3656,7 +3869,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "dev": true, "funding": [ { "type": "github", @@ -3712,6 +3924,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -3756,6 +3976,15 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", + "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==", + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -3792,7 +4021,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -3840,7 +4068,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", - "dev": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -3852,7 +4079,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -3868,7 +4094,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -3950,7 +4175,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "dev": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -4100,8 +4324,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "5.0.1", @@ -4178,8 +4401,7 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { "version": "17.7.2", diff --git a/package.json b/package.json index aa74e21a5..5d210a53c 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "@dqbd/tiktoken": "^1.0.2", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", "@tensorflow/tfjs": "^4.10.0", + "@xenova/transformers": "^2.6.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index 6026ebf9d..b8b7930df 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -530,7 +530,7 @@ async function moduleWorker() { } const offlineMode = $('.expression_settings .offline_mode'); - if (!modules.includes('classify')) { + if (!modules.includes('classify') && !extension_settings.expressions.local) { $('.expression_settings').show(); offlineMode.css('display', 'block'); lastCharacter = context.groupId || context.characterId; @@ -711,25 +711,45 @@ async function setSpriteSlashCommand(_, spriteId) { async function getExpressionLabel(text) { // Return if text is undefined, saving a costly fetch request - if (!modules.includes('classify') || !text) { + if ((!modules.includes('classify') && !extension_settings.expressions.local) || !text) { return FALLBACK_EXPRESSION; } - const url = new URL(getApiUrl()); - url.pathname = '/api/classify'; + try { + if (extension_settings.expressions.local) { + // Local transformers pipeline + const apiResult = await fetch('/api/extra/classify', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ text: text }), + }); - const apiResult = await doExtrasFetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Bypass-Tunnel-Reminder': 'bypass', - }, - body: JSON.stringify({ text: text }), - }); + if (apiResult.ok) { + const data = await apiResult.json(); + return data.classification[0].label; + } + } else { + // Extras + const url = new URL(getApiUrl()); + url.pathname = '/api/classify'; - if (apiResult.ok) { - const data = await apiResult.json(); - return data.classification[0].label; + const apiResult = await doExtrasFetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Bypass-Tunnel-Reminder': 'bypass', + }, + body: JSON.stringify({ text: text }), + }); + + if (apiResult.ok) { + const data = await apiResult.json(); + return data.classification[0].label; + } + } + } catch (error) { + console.log(error); + return FALLBACK_EXPRESSION; } } @@ -821,7 +841,7 @@ async function getSpritesList(name) { async function getExpressionsList() { // get something for offline mode (default images) - if (!modules.includes('classify')) { + if (!modules.includes('classify') && !extension_settings.expressions.local) { return DEFAULT_EXPRESSIONS; } @@ -829,20 +849,34 @@ async function getExpressionsList() { return expressionsList; } - const url = new URL(getApiUrl()); - url.pathname = '/api/classify/labels'; try { - const apiResult = await doExtrasFetch(url, { - method: 'GET', - headers: { 'Bypass-Tunnel-Reminder': 'bypass' }, - }); + if (extension_settings.expressions.local) { + const apiResult = await fetch('/api/extra/classify/labels', { + method: 'POST', + headers: getRequestHeaders(), + }); - if (apiResult.ok) { + if (apiResult.ok) { + const data = await apiResult.json(); + expressionsList = data.labels; + return expressionsList; + } + } else { + const url = new URL(getApiUrl()); + url.pathname = '/api/classify/labels'; - const data = await apiResult.json(); - expressionsList = data.labels; - return expressionsList; + const apiResult = await doExtrasFetch(url, { + method: 'GET', + headers: { 'Bypass-Tunnel-Reminder': 'bypass' }, + }); + + if (apiResult.ok) { + + const data = await apiResult.json(); + expressionsList = data.labels; + return expressionsList; + } } } catch (error) { @@ -1226,6 +1260,10 @@ function setExpressionOverrideHtml(forceClear = false) { $('#expressions_show_default').on('input', onExpressionsShowDefaultInput); $('#expression_upload_pack_button').on('click', onClickExpressionUploadPackButton); $('#expressions_show_default').prop('checked', extension_settings.expressions.showDefault).trigger('input'); + $('#expression_local').prop('checked', extension_settings.expressions.local).on('input', function () { + extension_settings.expressions.local = !!$(this).prop('checked'); + saveSettingsDebounced(); + }); $('#expression_override_cleanup_button').on('click', onClickExpressionOverrideRemoveAllButton); $(document).on('dragstart', '.expression', (e) => { e.preventDefault() diff --git a/public/scripts/extensions/expressions/settings.html b/public/scripts/extensions/expressions/settings.html index 27779cce2..20c2942c8 100644 --- a/public/scripts/extensions/expressions/settings.html +++ b/public/scripts/extensions/expressions/settings.html @@ -6,14 +6,14 @@
- -
- -
+ +
You are in offline mode. Click on the image below to set the expression.
diff --git a/server.js b/server.js index b4c172360..fe4a546bf 100644 --- a/server.js +++ b/server.js @@ -5741,3 +5741,7 @@ app.post('/get_character_assets_list', jsonParser, async (request, response) => // Vector storage DB require('./src/vectors').registerEndpoints(app, jsonParser); +// Emotion classification +import('./src/classify.mjs').then(module => { + module.default.registerEndpoints(app, jsonParser); +}); diff --git a/src/classify.mjs b/src/classify.mjs new file mode 100644 index 000000000..7b4c07c42 --- /dev/null +++ b/src/classify.mjs @@ -0,0 +1,48 @@ +import { pipeline, TextClassificationPipeline } from '@xenova/transformers'; +import path from 'path'; + +class PipelineAccessor { + /** + * @type {TextClassificationPipeline} + */ + pipe; + + async get() { + if (!this.pipe) { + const cache_dir = path.join(process.cwd(), 'cache'); + this.pipe = await pipeline('text-classification', 'Cohee/distilbert-base-uncased-go-emotions-onnx', { cache_dir, quantized: true }); + } + + return this.pipe; + } +} + +/** + * @param {import("express").Express} app + * @param {any} jsonParser + */ +function registerEndpoints(app, jsonParser) { + const pipelineAccessor = new PipelineAccessor(); + + app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { + const pipe = await pipelineAccessor.get(); + const result = Object.keys(pipe.model.config.label2id); + return res.json({ labels: result }); + }); + + app.post('/api/extra/classify', jsonParser, async (req, res) => { + const { text } = req.body; + + const pipe = await pipelineAccessor.get(); + const result = await pipe(text); + + console.log('Classify input:', text); + console.log('Classify output:', result); + + return res.json({ classification: result }); + }); +} + +export default { + registerEndpoints, +}; From b605b940ebaf0d7f1f5bcd72ce31a95b1d9a815a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 16:29:11 +0300 Subject: [PATCH 10/34] Replace transformers.js with patched version --- package-lock.json | 176 +++++++++++++--------------------------------- package.json | 2 +- server.js | 2 + src/classify.mjs | 2 +- 4 files changed, 54 insertions(+), 128 deletions(-) diff --git a/package-lock.json b/package-lock.json index f8a23d2ac..ee8f375af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@dqbd/tiktoken": "^1.0.2", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", "@tensorflow/tfjs": "^4.10.0", - "@xenova/transformers": "^2.6.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -41,6 +40,7 @@ "png-chunks-extract": "^1.0.0", "response-time": "^2.3.2", "sanitize-filename": "^1.6.3", + "sillytavern-transformers": "^2.6.1", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "vectra": "^0.2.2", @@ -968,18 +968,6 @@ "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==", "peer": true }, - "node_modules/@xenova/transformers": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.6.0.tgz", - "integrity": "sha512-k9bs+reiwhn+kx0d4FYnlBTWtl8D5Q4fIzoKYxKbTTSVyS33KXbQESRpdIxiU9gtlMKML2Sw0Oep4FYK9dQCsQ==", - "dependencies": { - "onnxruntime-web": "1.14.0", - "sharp": "^0.32.0" - }, - "optionalDependencies": { - "onnxruntime-node": "1.14.0" - } - }, "node_modules/@xmldom/xmldom": { "version": "0.8.9", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.9.tgz", @@ -1127,11 +1115,6 @@ "proxy-from-env": "^1.1.0" } }, - "node_modules/b4a": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", - "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1155,6 +1138,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -1165,6 +1149,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1356,7 +1341,8 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true }, "node_modules/cliui": { "version": "8.0.1", @@ -1371,18 +1357,6 @@ "node": ">=12" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1399,15 +1373,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1610,6 +1575,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -1624,6 +1590,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, "engines": { "node": ">=4.0.0" } @@ -1665,6 +1632,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "dev": true, "engines": { "node": ">=8" } @@ -1775,6 +1743,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, "dependencies": { "once": "^1.4.0" } @@ -1837,6 +1806,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, "engines": { "node": ">=6" } @@ -1909,11 +1879,6 @@ } ] }, - "node_modules/fast-fifo": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", - "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" - }, "node_modules/fast-glob": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.0.tgz", @@ -2058,7 +2023,8 @@ "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true }, "node_modules/fs-extra": { "version": "9.1.0", @@ -2114,7 +2080,8 @@ "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true }, "node_modules/glob-parent": { "version": "5.1.2", @@ -2362,7 +2329,8 @@ "node_modules/ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true }, "node_modules/into-stream": { "version": "6.0.0", @@ -2393,11 +2361,6 @@ "node": ">= 10" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-core-module": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", @@ -2647,6 +2610,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -2731,6 +2695,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, "engines": { "node": ">=10" }, @@ -2771,7 +2736,8 @@ "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true }, "node_modules/ms": { "version": "2.0.0", @@ -2847,7 +2813,8 @@ "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "dev": true }, "node_modules/negotiator": { "version": "0.6.3", @@ -2861,6 +2828,7 @@ "version": "3.45.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.45.0.tgz", "integrity": "sha512-iwXuFrMAcFVi/ZoZiqq8BzAdsLw9kxDfTC0HMyjXfSL/6CSDAGD5UmR7azrAgWV1zKYq7dUUMj4owusBWKLsiQ==", + "dev": true, "dependencies": { "semver": "^7.3.5" }, @@ -2868,11 +2836,6 @@ "node": ">=10" } }, - "node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" - }, "node_modules/node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", @@ -2947,6 +2910,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, "dependencies": { "wrappy": "1" } @@ -3369,6 +3333,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dev": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -3466,6 +3431,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -3505,11 +3471,6 @@ } ] }, - "node_modules/queue-tick": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", - "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3544,6 +3505,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3723,6 +3685,7 @@ "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -3780,48 +3743,6 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, - "node_modules/sharp": { - "version": "0.32.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.5.tgz", - "integrity": "sha512-0dap3iysgDkNaPOaOL4X/0akdu0ma62GcdC2NBQ+93eqpePdDdr2/LM0sFdDSMmN7yS+odyZtPsb7tx/cYBKnQ==", - "hasInstallScript": true, - "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.2", - "node-addon-api": "^6.1.0", - "prebuild-install": "^7.1.1", - "semver": "^7.5.4", - "simple-get": "^4.0.1", - "tar-fs": "^3.0.4", - "tunnel-agent": "^0.6.0" - }, - "engines": { - "node": ">=14.15.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/sharp/node_modules/tar-fs": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", - "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", - "dependencies": { - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^3.1.5" - } - }, - "node_modules/sharp/node_modules/tar-stream": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", - "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", - "dependencies": { - "b4a": "^1.6.4", - "fast-fifo": "^1.2.0", - "streamx": "^2.15.0" - } - }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -3846,10 +3767,22 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/sillytavern-transformers": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/sillytavern-transformers/-/sillytavern-transformers-2.6.1.tgz", + "integrity": "sha512-mosQ21oyKDQr8d+qmnaIzUlkDDUhhYrb0kccPBefjedLfSahbrUcakpI68JiBf6CdB65KHUp9P1IVrLjiVICTA==", + "dependencies": { + "onnxruntime-web": "1.14.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true, "funding": [ { "type": "github", @@ -3869,6 +3802,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, "funding": [ { "type": "github", @@ -3924,14 +3858,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -3976,15 +3902,6 @@ "node": ">=10.0.0" } }, - "node_modules/streamx": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", - "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==", - "dependencies": { - "fast-fifo": "^1.1.0", - "queue-tick": "^1.0.1" - } - }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -4021,6 +3938,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -4068,6 +3986,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dev": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -4079,6 +3998,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -4094,6 +4014,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -4175,6 +4096,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -4324,7 +4246,8 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true }, "node_modules/write-file-atomic": { "version": "5.0.1", @@ -4401,7 +4324,8 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true }, "node_modules/yargs": { "version": "17.7.2", diff --git a/package.json b/package.json index 5d210a53c..56f2aeb88 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "@dqbd/tiktoken": "^1.0.2", "@tensorflow-models/universal-sentence-encoder": "^1.3.3", "@tensorflow/tfjs": "^4.10.0", - "@xenova/transformers": "^2.6.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -32,6 +31,7 @@ "png-chunks-extract": "^1.0.0", "response-time": "^2.3.2", "sanitize-filename": "^1.6.3", + "sillytavern-transformers": "^2.6.1", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "vectra": "^0.2.2", diff --git a/server.js b/server.js index fe4a546bf..f7badb75a 100644 --- a/server.js +++ b/server.js @@ -5744,4 +5744,6 @@ require('./src/vectors').registerEndpoints(app, jsonParser); // Emotion classification import('./src/classify.mjs').then(module => { module.default.registerEndpoints(app, jsonParser); +}).catch(err => { + console.error(err); }); diff --git a/src/classify.mjs b/src/classify.mjs index 7b4c07c42..635b2642e 100644 --- a/src/classify.mjs +++ b/src/classify.mjs @@ -1,4 +1,4 @@ -import { pipeline, TextClassificationPipeline } from '@xenova/transformers'; +import { pipeline, TextClassificationPipeline } from 'sillytavern-transformers'; import path from 'path'; class PipelineAccessor { From 307e666c27f3e1632ff6090469e691a7f59f168f Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 16:42:16 +0300 Subject: [PATCH 11/34] onnx runtime to web --- src/classify.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/classify.mjs b/src/classify.mjs index 635b2642e..c4be593b4 100644 --- a/src/classify.mjs +++ b/src/classify.mjs @@ -1,6 +1,8 @@ -import { pipeline, TextClassificationPipeline } from 'sillytavern-transformers'; +import { pipeline, TextClassificationPipeline, env } from 'sillytavern-transformers'; import path from 'path'; +env.backends.onnx = 'onnxruntime-web'; + class PipelineAccessor { /** * @type {TextClassificationPipeline} From 180dcefe40130beca5492e423120f7db92ae79c5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 16:55:54 +0300 Subject: [PATCH 12/34] Patch onnx to always use wasm --- package-lock.json | 8 ++++---- package.json | 2 +- src/classify.mjs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index ee8f375af..5061c1e25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,7 +40,7 @@ "png-chunks-extract": "^1.0.0", "response-time": "^2.3.2", "sanitize-filename": "^1.6.3", - "sillytavern-transformers": "^2.6.1", + "sillytavern-transformers": "^2.6.2", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "vectra": "^0.2.2", @@ -3768,9 +3768,9 @@ } }, "node_modules/sillytavern-transformers": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/sillytavern-transformers/-/sillytavern-transformers-2.6.1.tgz", - "integrity": "sha512-mosQ21oyKDQr8d+qmnaIzUlkDDUhhYrb0kccPBefjedLfSahbrUcakpI68JiBf6CdB65KHUp9P1IVrLjiVICTA==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/sillytavern-transformers/-/sillytavern-transformers-2.6.2.tgz", + "integrity": "sha512-C9l7mEjSHpOMuAAXrCY0r4Mv8FKlrgTr3XV51evybtXc1bk38KzWSyMukFOC+dVCsuTdqBCyjhdBeF4+6mrHzQ==", "dependencies": { "onnxruntime-web": "1.14.0" }, diff --git a/package.json b/package.json index 56f2aeb88..07fa4b297 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "png-chunks-extract": "^1.0.0", "response-time": "^2.3.2", "sanitize-filename": "^1.6.3", - "sillytavern-transformers": "^2.6.1", + "sillytavern-transformers": "^2.6.2", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "vectra": "^0.2.2", diff --git a/src/classify.mjs b/src/classify.mjs index c4be593b4..46d959126 100644 --- a/src/classify.mjs +++ b/src/classify.mjs @@ -1,7 +1,7 @@ import { pipeline, TextClassificationPipeline, env } from 'sillytavern-transformers'; import path from 'path'; -env.backends.onnx = 'onnxruntime-web'; +env.backends.onnx.wasm.numThreads = 1; class PipelineAccessor { /** From 4cf6a1f7daf21ef88a7bce107844f22a7d8b11f4 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 17:31:27 +0300 Subject: [PATCH 13/34] Cache and sample classification results --- public/script.js | 4 +-- .../scripts/extensions/expressions/index.js | 32 ++++++++++++++++++- public/scripts/utils.js | 24 ++++++++++++-- src/classify.mjs | 14 ++++++-- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/public/script.js b/public/script.js index 5433ce081..91d84ca44 100644 --- a/public/script.js +++ b/public/script.js @@ -121,7 +121,7 @@ import { delay, restoreCaretPosition, saveCaretPosition, - end_trim_to_sentence, + trimToEndSentence, countOccurrences, isOdd, sortMoments, @@ -3781,7 +3781,7 @@ function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete getMessage = getRegexedString(getMessage, isImpersonate ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT); if (!displayIncompleteSentences && power_user.trim_sentences) { - getMessage = end_trim_to_sentence(getMessage, power_user.include_newline); + getMessage = trimToEndSentence(getMessage, power_user.include_newline); } if (power_user.collapse_newlines) { diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js index b8b7930df..7437d305f 100644 --- a/public/scripts/extensions/expressions/index.js +++ b/public/scripts/extensions/expressions/index.js @@ -3,7 +3,7 @@ import { dragElement, isMobile } from "../../RossAscends-mods.js"; import { getContext, getApiUrl, modules, extension_settings, ModuleWorkerWrapper, doExtrasFetch, renderExtensionTemplate } from "../../extensions.js"; import { loadMovingUIState, power_user } from "../../power-user.js"; import { registerSlashCommand } from "../../slash-commands.js"; -import { onlyUnique, debounce, getCharaFilename } from "../../utils.js"; +import { onlyUnique, debounce, getCharaFilename, trimToEndSentence, trimToStartSentence } from "../../utils.js"; export { MODULE_NAME }; const MODULE_NAME = 'expressions'; @@ -709,12 +709,42 @@ async function setSpriteSlashCommand(_, spriteId) { await sendExpressionCall(spriteFolderName, spriteItem.label, true, vnMode); } +/** + * Processes the classification text to reduce the amount of text sent to the API. + * Quotes and asterisks are to be removed. If the text is less than 300 characters, it is returned as is. + * If the text is more than 300 characters, the first and last 150 characters are returned. + * The result is trimmed to the end of sentence. + * @param {string} text The text to process. + * @returns {string} + */ +function sampleClassifyText(text) { + if (!text) { + return text; + } + + // Remove asterisks and quotes + let result = text.replace(/[\*\"]/g, ''); + + const SAMPLE_THRESHOLD = 300; + const HALF_SAMPLE_THRESHOLD = SAMPLE_THRESHOLD / 2; + + if (text.length < SAMPLE_THRESHOLD) { + result = trimToEndSentence(result); + } else { + result = trimToEndSentence(result.slice(0, HALF_SAMPLE_THRESHOLD)) + ' ' + trimToStartSentence(result.slice(-HALF_SAMPLE_THRESHOLD)); + } + + return result.trim(); +} + async function getExpressionLabel(text) { // Return if text is undefined, saving a costly fetch request if ((!modules.includes('classify') && !extension_settings.expressions.local) || !text) { return FALLBACK_EXPRESSION; } + text = sampleClassifyText(text); + try { if (extension_settings.expressions.local) { // Local transformers pipeline diff --git a/public/scripts/utils.js b/public/scripts/utils.js index 121c97334..6ea5fc667 100644 --- a/public/scripts/utils.js +++ b/public/scripts/utils.js @@ -438,9 +438,9 @@ export function sortByCssOrder(a, b) { * @param {boolean} include_newline Whether to include a newline character in the trimmed string. * @returns {string} The trimmed string. * @example - * end_trim_to_sentence('Hello, world! I am from'); // 'Hello, world!' + * trimToEndSentence('Hello, world! I am from'); // 'Hello, world!' */ -export function end_trim_to_sentence(input, include_newline = false) { +export function trimToEndSentence(input, include_newline = false) { const punctuation = new Set(['.', '!', '?', '*', '"', ')', '}', '`', ']', '$', '。', '！', '？', '”', '）', '】', '】', '’', '」', '】']); // extend this as you see fit let last = -1; @@ -465,6 +465,26 @@ export function end_trim_to_sentence(input, include_newline = false) { return input.substring(0, last + 1).trimEnd(); } +export function trimToStartSentence(input) { + let p1 = input.indexOf("."); + let p2 = input.indexOf("!"); + let p3 = input.indexOf("?"); + let p4 = input.indexOf("\n"); + let first = p1; + let skip1 = false; + if (p2 > 0 && p2 < first) { first = p2; } + if (p3 > 0 && p3 < first) { first = p3; } + if (p4 > 0 && p4 < first) { first = p4; skip1 = true; } + if (first > 0) { + if (skip1) { + return input.substring(first + 1); + } else { + return input.substring(first + 2); + } + } + return input; +} + /** * Counts the number of occurrences of a character in a string. * @param {string} string The string to count occurrences in. diff --git a/src/classify.mjs b/src/classify.mjs index 46d959126..97062d84c 100644 --- a/src/classify.mjs +++ b/src/classify.mjs @@ -24,6 +24,7 @@ class PipelineAccessor { * @param {any} jsonParser */ function registerEndpoints(app, jsonParser) { + const cacheObject = {}; const pipelineAccessor = new PipelineAccessor(); app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { @@ -35,10 +36,19 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify', jsonParser, async (req, res) => { const { text } = req.body; - const pipe = await pipelineAccessor.get(); - const result = await pipe(text); + async function getResult(text) { + if (cacheObject.hasOwnProperty(text)) { + return cacheObject[text]; + } else { + const pipe = await pipelineAccessor.get(); + const result = await pipe(text); + cacheObject[text] = result; + return result; + } + } console.log('Classify input:', text); + const result = await getResult(text); console.log('Classify output:', result); return res.json({ classification: result }); From 2fa038f91d02586744b6dd1c7ec66db55a5c63f7 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 21:26:04 +0300 Subject: [PATCH 14/34] Add advanced vector controls --- public/scripts/extensions/vectors/index.js | 72 +++++++++++++++---- .../scripts/extensions/vectors/settings.html | 41 ++++++++++- public/style.css | 1 + 3 files changed, 98 insertions(+), 16 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index a364ea8a4..c4ed01796 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -1,18 +1,21 @@ -import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced, setExtensionPrompt } from "../../../script.js"; +import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced, setExtensionPrompt, substituteParams } from "../../../script.js"; import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js"; -import { collapseNewlines } from "../../power-user.js"; +import { collapseNewlines, power_user, ui_mode } from "../../power-user.js"; import { debounce, getStringHash as calculateHash } from "../../utils.js"; const MODULE_NAME = 'vectors'; -const AMOUNT_TO_LEAVE = 5; -const INSERT_AMOUNT = 3; -const QUERY_TEXT_AMOUNT = 2; export const EXTENSION_PROMPT_TAG = '3_vectors'; const settings = { enabled: false, source: 'local', + template: `Past events: {{text}}`, + depth: 2, + position: extension_prompt_types.IN_PROMPT, + protect: 5, + insert: 3, + query: 2, }; const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); @@ -138,8 +141,8 @@ async function rearrangeChat(chat) { return; } - if (chat.length < AMOUNT_TO_LEAVE) { - console.debug(`Vectors: Not enough messages to rearrange (less than ${AMOUNT_TO_LEAVE})`); + if (chat.length < settings.protect) { + console.debug(`Vectors: Not enough messages to rearrange (less than ${settings.protect})`); return; } @@ -151,9 +154,9 @@ async function rearrangeChat(chat) { } // Get the most relevant messages, excluding the last few - const queryHashes = await queryCollection(chatId, queryText, INSERT_AMOUNT); + const queryHashes = await queryCollection(chatId, queryText, settings.insert); const queriedMessages = []; - const retainMessages = chat.slice(-AMOUNT_TO_LEAVE); + const retainMessages = chat.slice(-settings.protect); for (const message of chat) { if (retainMessages.includes(message)) { @@ -181,15 +184,23 @@ async function rearrangeChat(chat) { } // Format queried messages into a single string - const queriedText = queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); - console.log('Vectors: relevant past messages found.\n', queriedText); - const insertedText = `Past events: ${queriedText}`; - setExtensionPrompt(EXTENSION_PROMPT_TAG, insertedText, extension_prompt_types.IN_PROMPT, 0); + const insertedText = getPromptText(queriedMessages); + setExtensionPrompt(EXTENSION_PROMPT_TAG, insertedText, settings.position, settings.depth); } catch (error) { console.error('Vectors: Failed to rearrange chat', error); } } +/** + * @param {any[]} queriedMessages + * @returns {string} + */ +function getPromptText(queriedMessages) { + const queriedText = queriedMessages.map(x => collapseNewlines(`${x.name}: ${x.mes}`).trim()).join('\n\n'); + console.log('Vectors: relevant past messages found.\n', queriedText); + return substituteParams(settings.template.replace(/{{text}}/i, queriedText)); +} + window['vectors_rearrangeChat'] = rearrangeChat; const onChatEvent = debounce(async () => await moduleWorker.update(), 500); @@ -209,7 +220,7 @@ function getQueryText(chat) { i++; } - if (i === QUERY_TEXT_AMOUNT) { + if (i === settings.query) { break; } } @@ -327,6 +338,39 @@ jQuery(async () => { Object.assign(extension_settings.vectors, settings); saveSettingsDebounced(); }); + $('#vectors_template').val(settings.template).on('input', () => { + settings.template = String($('#vectors_template').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $('#vectors_depth').val(settings.depth).on('input', () => { + settings.depth = Number($('#vectors_depth').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $('#vectors_protect').val(settings.protect).on('input', () => { + settings.protect = Number($('#vectors_protect').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $('#vectors_insert').val(settings.insert).on('input', () => { + settings.insert = Number($('#vectors_insert').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $('#vectors_query').val(settings.query).on('input', () => { + settings.query = Number($('#vectors_query').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $(`input[name="vectors_position"][value="${settings.position}"]`).prop('checked', true); + $('input[name="vectors_position"]').on('change', () => { + settings.position = Number($('input[name="vectors_position"]:checked').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + $('#vectors_advanced_settings').toggleClass('displayNone', power_user.ui_mode === ui_mode.SIMPLE); + $('#vectors_vectorize_all').on('click', onVectorizeAllClick); eventSource.on(event_types.CHAT_CHANGED, onChatEvent); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index 0d1d7309a..30c0e0c38 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -16,10 +16,47 @@ -
+
+ + + +
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ Old messages are vectorized gradually as you chat. To process all previous messages, click the button below. -
+ diff --git a/public/style.css b/public/style.css index ef21e17c7..c9d75bd0f 100644 --- a/public/style.css +++ b/public/style.css @@ -1209,6 +1209,7 @@ input[type="file"] { .radio_group { display: flex; flex-direction: column; + margin-top: 5px; } #extension_floating_counter { From ed6417ebcd0fc87e2ee506912ecf307fdd440b86 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 21:36:04 +0300 Subject: [PATCH 15/34] Display vectorization error in toast --- public/scripts/extensions/vectors/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index c4ed01796..a1addc4f4 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -94,7 +94,9 @@ async function synchronizeChat(batchSize = 5) { return newVectorItems.length - batchSize; } catch (error) { + toastr.error('Check server console for more details', 'Vectorization failed'); console.error('Vectors: Failed to synchronize chat', error); + return -1; } } From af38971a0149667fc1f73da3dc1cd77817aa6558 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 22:15:47 +0300 Subject: [PATCH 16/34] Delete vectors on deleting chats --- public/script.js | 16 ++++++++-- public/scripts/extensions/vectors/index.js | 27 +++++++++++++++++ public/scripts/group-chats.js | 10 +++++++ src/openai-vectors.js | 3 +- src/vectors.js | 35 ++++++++++++++++++++-- 5 files changed, 86 insertions(+), 5 deletions(-) diff --git a/public/script.js b/public/script.js index 91d84ca44..2c1b09b0c 100644 --- a/public/script.js +++ b/public/script.js @@ -285,7 +285,9 @@ export const event_types = { CHARACTER_EDITED: 'character_edited', USER_MESSAGE_RENDERED: 'user_message_rendered', CHARACTER_MESSAGE_RENDERED: 'character_message_rendered', - FORCE_SET_BACKGROUND: 'force_set_background,' + FORCE_SET_BACKGROUND: 'force_set_background', + CHAT_DELETED : 'chat_deleted', + GROUP_CHAT_DELETED: 'group_chat_deleted', } export const eventSource = new EventEmitter(); @@ -1106,10 +1108,12 @@ async function delChat(chatfile) { }); if (response.ok === true) { // choose another chat if current was deleted - if (chatfile.replace('.jsonl', '') === characters[this_chid].chat) { + const name = chatfile.replace('.jsonl', ''); + if (name === characters[this_chid].chat) { chat_metadata = {}; await replaceCurrentChat(); } + await eventSource.emit(event_types.CHAT_DELETED, name); } } @@ -6891,6 +6895,7 @@ export async function handleDeleteCharacter(popup_type, this_chid, delete_chats) const avatar = characters[this_chid].avatar; const name = characters[this_chid].name; + const pastChats = await getPastCharacterChats(); const msg = { avatar_url: avatar, delete_chats: delete_chats }; @@ -6903,6 +6908,13 @@ export async function handleDeleteCharacter(popup_type, this_chid, delete_chats) if (response.ok) { await deleteCharacter(name, avatar); + + if (delete_chats) { + for (const chat of pastChats) { + const name = chat.file_name.replace('.jsonl', ''); + await eventSource.emit(event_types.CHAT_DELETED, name); + } + } } else { console.error('Failed to delete character: ', response.status, response.statusText); } diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index a1addc4f4..1383ab46a 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -323,6 +323,31 @@ async function queryCollection(collectionId, searchText, topK) { return results; } +async function purgeVectorIndex(collectionId) { + try { + if (!settings.enabled) { + return; + } + + const response = await fetch('/api/vector/purge', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + collectionId: collectionId, + }), + }); + + if (!response.ok) { + throw new Error(`Could not delete vector index for collection ${collectionId}`); + } + + console.log(`Vectors: Purged vector index for collection ${collectionId}`); + + } catch (error) { + console.error('Vectors: Failed to purge', error); + } +} + jQuery(async () => { if (!extension_settings.vectors) { extension_settings.vectors = settings; @@ -381,4 +406,6 @@ jQuery(async () => { eventSource.on(event_types.MESSAGE_SENT, onChatEvent); eventSource.on(event_types.MESSAGE_RECEIVED, onChatEvent); eventSource.on(event_types.MESSAGE_SWIPED, onChatEvent); + eventSource.on(event_types.CHAT_DELETED, purgeVectorIndex); + eventSource.on(event_types.GROUP_CHAT_DELETED, purgeVectorIndex); }); diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index d204579cf..ddac1ef48 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -816,12 +816,20 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i } async function deleteGroup(id) { + const group = groups.find((x) => x.id === id); + const response = await fetch("/deletegroup", { method: "POST", headers: getRequestHeaders(), body: JSON.stringify({ id: id }), }); + if (group && Array.isArray(group.chats)) { + for (const chatId of group.chats) { + await eventSource.emit(event_types.GROUP_CHAT_DELETED, chatId); + } + } + if (response.ok) { selected_group = null; delete tag_map[id]; @@ -1493,6 +1501,8 @@ export async function deleteGroupChat(groupId, chatId) { } else { await createNewGroupChat(groupId); } + + await eventSource.emit(event_types.GROUP_CHAT_DELETED, chatId); } } diff --git a/src/openai-vectors.js b/src/openai-vectors.js index f81590f88..b7eb89162 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -27,7 +27,8 @@ async function getOpenAIVector(text) { }); if (!response.ok) { - console.log('OpenAI request failed'); + const text = await response.text(); + console.log('OpenAI request failed', response.statusText, text); throw new Error('OpenAI request failed'); } diff --git a/src/vectors.js b/src/vectors.js index 15a7c3871..d42a812f9 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -24,12 +24,13 @@ async function getVector(source, text) { * Gets the index for the vector collection * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector + * @param {boolean} create - Whether to create the index if it doesn't exist * @returns {Promise} - The index for the collection */ -async function getIndex(collectionId, source) { +async function getIndex(collectionId, source, create = true) { const index = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(source), sanitize(collectionId))); - if (!await index.isIndexCreated()) { + if (create && !await index.isIndexCreated()) { await index.createIndex(); } @@ -185,6 +186,36 @@ async function registerEndpoints(app, jsonParser) { return res.sendStatus(500); } }); + + app.post('/api/vector/purge', jsonParser, async (req, res) => { + try { + if (!req.body.collectionId) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + + const sources = ['local', 'openai']; + for (const source of sources) { + const index = await getIndex(collectionId, source, false); + + const exists = await index.isIndexCreated(); + + if (!exists) { + continue; + } + + const path = index.folderPath; + await index.deleteIndex(); + console.log(`Deleted vector index at ${path}`); + } + + return res.sendStatus(200); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); } module.exports = { registerEndpoints }; From 23951b8c8a0040a3d4df878642e8ae8c017ab7ee Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 10 Sep 2023 00:15:02 +0300 Subject: [PATCH 17/34] Prevent sync and generation at the same time --- public/scripts/extensions/vectors/index.js | 26 ++++++++++++++++--- .../scripts/extensions/vectors/settings.html | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 1383ab46a..6232e4519 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -1,7 +1,7 @@ -import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, saveSettingsDebounced, setExtensionPrompt, substituteParams } from "../../../script.js"; +import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, is_send_press, saveSettingsDebounced, setExtensionPrompt, substituteParams } from "../../../script.js"; import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js"; import { collapseNewlines, power_user, ui_mode } from "../../power-user.js"; -import { debounce, getStringHash as calculateHash } from "../../utils.js"; +import { debounce, getStringHash as calculateHash, waitUntilCondition } from "../../utils.js"; const MODULE_NAME = 'vectors'; @@ -35,6 +35,11 @@ async function onVectorizeAllClick() { $('#vectorize_progress_eta').text('...'); while (!finished) { + if (is_send_press) { + toastr.info('Message generation is in progress.', 'Vectorization aborted'); + throw new Error('Message generation is in progress.'); + } + const startTime = Date.now(); const remaining = await synchronizeChat(batchSize); const elapsed = Date.now() - startTime; @@ -44,7 +49,8 @@ async function onVectorizeAllClick() { const total = getContext().chat.length; const processed = total - remaining; const processedPercent = Math.round((processed / total) * 100); // percentage of the work done - const averageElapsed = elapsedLog.slice(-5).reduce((a, b) => a + b, 0) / elapsedLog.length; // average time needed to process one item + const lastElapsed = elapsedLog.slice(-5); // last 5 elapsed times + const averageElapsed = lastElapsed.reduce((a, b) => a + b, 0) / lastElapsed.length; // average time needed to process one item const pace = averageElapsed / batchSize; // time needed to process one item const remainingTime = Math.round(pace * remaining / 1000); @@ -62,12 +68,22 @@ async function onVectorizeAllClick() { } } +let syncBlocked = false; + async function synchronizeChat(batchSize = 5) { + try { + await waitUntilCondition(() => !syncBlocked, 500); + } catch { + console.log('Vectors: Synchronization blocked by another process'); + return -1; + } + try { if (!settings.enabled) { return -1; } + syncBlocked = true; const context = getContext(); const chatId = getCurrentChatId(); @@ -83,8 +99,8 @@ async function synchronizeChat(batchSize = 5) { const deletedHashes = hashesInCollection.filter(x => !hashedMessages.some(y => y.hash === x)); if (newVectorItems.length > 0) { + console.log(`Vectors: Found ${newVectorItems.length} new items. Processing ${batchSize}...`); await insertVectorItems(chatId, newVectorItems.slice(0, batchSize)); - console.log(`Vectors: Inserted ${newVectorItems.length} new items`); } if (deletedHashes.length > 0) { @@ -97,6 +113,8 @@ async function synchronizeChat(batchSize = 5) { toastr.error('Check server console for more details', 'Vectorization failed'); console.error('Vectors: Failed to synchronize chat', error); return -1; + } finally { + syncBlocked = false; } } diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index 30c0e0c38..44f0c501c 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -20,7 +20,7 @@ - +
@@ -448,6 +495,7 @@ jQuery(() => { const optionText = $('#translation_provider option:selected').text(); const exampleURLs = {}; exampleURLs['libre'] = 'http://127.0.0.1:5000/translate'; + exampleURLs['oneringtranslator'] = 'http://127.0.0.1:4990/translate'; const url = await callPopup(`

${optionText} API URL

Example: ` + exampleURLs[extension_settings.translate.provider] + ``, 'input'); if (url == false) { diff --git a/src/secrets.js b/src/secrets.js index eee18dcbc..49c84ba20 100644 --- a/src/secrets.js +++ b/src/secrets.js @@ -16,6 +16,7 @@ const SECRET_KEYS = { SCALE: 'api_key_scale', AI21: 'api_key_ai21', SCALE_COOKIE: 'scale_cookie', + ONERING_URL: 'oneringtranslator_url', } /** diff --git a/src/translate.js b/src/translate.js index 9e11bd433..f70a13f2b 100644 --- a/src/translate.js +++ b/src/translate.js @@ -132,6 +132,54 @@ function registerEndpoints(app, jsonParser) { return response.sendStatus(500); } }); + + app.post('/api/translate/onering', jsonParser, async (request, response) => { + const url = readSecret(SECRET_KEYS.ONERING_URL); + + if (!url) { + console.log('OneRing URL is not configured.'); + return response.sendStatus(401); + } + + const text = request.body.text; + const from_lang = request.body.from_lang; + const to_lang = request.body.to_lang; + + if (!text || !from_lang || !to_lang) { + return response.sendStatus(400); + } + + const params = new URLSearchParams(); + params.append('text', text); + params.append('from_lang', from_lang); + params.append('to_lang', to_lang); + + console.log('Input text: ' + text); + + try { + const fetchUrl = new URL(url); + fetchUrl.search = params.toString(); + + const result = await fetch(fetchUrl, { + method: 'GET', + timeout: 0, + }); + + if (!result.ok) { + const error = await result.text(); + console.log('OneRing error: ', result.statusText, error); + return response.sendStatus(result.status); + } + + const data = await result.json(); + console.log('Translated text: ' + data.result); + + return response.send(data.result); + } catch (error) { + console.log("Translation error: " + error.message); + return response.sendStatus(500); + } + }); } module.exports = { From d17ac770e6169a4159b6c8de7a8a63d6e53935d4 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 10 Sep 2023 18:22:39 +0300 Subject: [PATCH 21/34] Change net workaround for node 20 --- server.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index a2bbd5ccb..840e887e5 100644 --- a/server.js +++ b/server.js @@ -94,8 +94,13 @@ function createDefaultFiles() { } const net = require("net"); -// @ts-ignore work around a node v20 bug: https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 -if (net.setDefaultAutoSelectFamily) net.setDefaultAutoSelectFamily(false); +// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. +// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 +// Safe to remove once support for Node v20 is dropped. +if (process.versions && process.versions.node && process.versions.node.match(/20\.[0-2]\.0/)) { + // @ts-ignore + if (net.setDefaultAutoSelectFamily) net.setDefaultAutoSelectFamily(false); +} const cliArguments = yargs(hideBin(process.argv)) .option('disableCsrf', { From d19c151669ad82390485146d24228b514b979b39 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 10 Sep 2023 18:22:39 +0300 Subject: [PATCH 22/34] Add DeepLX translation provider #1111 --- public/scripts/extensions/translate/index.js | 41 +++++++++++-- src/secrets.js | 1 + src/translate.js | 63 +++++++++++++++++++- 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/translate/index.js b/public/scripts/extensions/translate/index.js index 37bf9e761..dd20a7b29 100644 --- a/public/scripts/extensions/translate/index.js +++ b/public/scripts/extensions/translate/index.js @@ -135,8 +135,8 @@ const languageCodes = { 'Zulu': 'zu', }; -const KEY_REQUIRED = ['deepl','libre']; -const LOCAL_URL = ['libre', 'oneringtranslator']; +const KEY_REQUIRED = ['deepl', 'libre']; +const LOCAL_URL = ['libre', 'oneringtranslator', 'deeplx']; function showKeysButton() { const providerRequiresKey = KEY_REQUIRED.includes(extension_settings.translate.provider); @@ -271,6 +271,27 @@ async function translateProviderDeepl(text, lang) { throw new Error(response.statusText); } +/** + * Translates text using the DeepLX API + * @param {string} text Text to translate + * @param {string} lang Target language code + * @returns {Promise} Translated text + */ +async function translateProviderDeepLX(text, lang) { + const response = await fetch('/api/translate/deeplx', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ text: text, lang: lang }), + }); + + if (response.ok) { + const result = await response.text(); + return result; + } + + throw new Error(response.statusText); +} + /** * Translates text using the selected translation provider * @param {string} text Text to translate @@ -290,6 +311,8 @@ async function translate(text, lang) { return await translateProviderGoogle(text, lang); case 'deepl': return await translateProviderDeepl(text, lang); + case 'deeplx': + return await translateProviderDeepLX(text, lang); case 'oneringtranslator': return await translateProviderOneRing(text, lang); default: @@ -437,6 +460,7 @@ jQuery(() => { + Local server classification -