From d52b4fbbde00cf8ca9168ebc249f883e522aba1b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:37:22 +0300 Subject: [PATCH 01/21] Node: Migrate to ES Modules --- .eslintrc.js | 3 + jsconfig.json | 2 +- package-lock.json | 438 +++++++++++++++++++-- package.json | 25 +- plugins.js | 9 +- post-install.js | 14 +- recover.js | 8 +- server.js | 286 ++++++-------- src/additional-headers.js | 18 +- src/character-card-parser.js | 23 +- src/constants.js | 67 +--- src/endpoints/anthropic.js | 13 +- src/endpoints/assets.js | 28 +- src/endpoints/avatars.js | 24 +- src/endpoints/azure.js | 15 +- src/endpoints/backends/chat-completions.js | 85 ++-- src/endpoints/backends/kobold.js | 18 +- src/endpoints/backends/scale-alt.js | 13 +- src/endpoints/backends/text-completions.js | 29 +- src/endpoints/backgrounds.js | 19 +- src/endpoints/caption.js | 14 +- src/endpoints/characters.js | 42 +- src/endpoints/chats.js | 23 +- src/endpoints/classify.js | 16 +- src/endpoints/content-manager.js | 41 +- src/endpoints/extensions.js | 20 +- src/endpoints/files.js | 23 +- src/endpoints/google.js | 15 +- src/endpoints/groups.js | 19 +- src/endpoints/horde.js | 16 +- src/endpoints/images.js | 17 +- src/endpoints/moving-ui.js | 14 +- src/endpoints/novelai.js | 18 +- src/endpoints/openai.js | 24 +- src/endpoints/openrouter.js | 8 +- src/endpoints/presets.js | 20 +- src/endpoints/quick-replies.js | 17 +- src/endpoints/search.js | 17 +- src/endpoints/secrets.js | 39 +- src/endpoints/settings.js | 26 +- src/endpoints/speech.js | 18 +- src/endpoints/sprites.js | 26 +- src/endpoints/stable-diffusion.js | 26 +- src/endpoints/stats.js | 30 +- src/endpoints/themes.js | 18 +- src/endpoints/thumbnails.js | 36 +- src/endpoints/tokenizers.js | 51 ++- src/endpoints/translate.js | 26 +- src/endpoints/users-admin.js | 25 +- src/endpoints/users-private.js | 30 +- src/endpoints/users-public.js | 21 +- src/endpoints/vectors.js | 66 ++-- src/endpoints/worldinfo.js | 19 +- src/express-common.js | 23 +- src/middleware/basicAuth.js | 8 +- src/middleware/multerMonkeyPatch.js | 4 +- src/middleware/whitelist.js | 14 +- src/plugin-loader.js | 17 +- src/polyfill.js | 2 +- src/prompt-converters.js | 35 +- src/request-proxy.js | 13 +- src/transformers.mjs | 11 +- src/users.js | 104 ++--- src/util.js | 118 +++--- src/validator/TavernCardValidator.js | 4 +- src/vectors/cohere-vectors.js | 12 +- src/vectors/embedding.js | 13 +- src/vectors/extras-vectors.js | 11 +- src/vectors/llamacpp-vectors.js | 15 +- src/vectors/makersuite-vectors.js | 13 +- src/vectors/nomicai-vectors.js | 13 +- src/vectors/ollama-vectors.js | 15 +- src/vectors/openai-vectors.js | 13 +- src/vectors/vllm-vectors.js | 15 +- 74 files changed, 1291 insertions(+), 1140 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 8d8386505..be84dc2de 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -16,6 +16,9 @@ module.exports = { env: { node: true, }, + parserOptions: { + sourceType: 'module', + }, }, { files: ['src/**/*.mjs'], diff --git a/jsconfig.json b/jsconfig.json index 0e554015b..8cc31d841 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "module": "ESNext", "target": "ESNext", - "moduleResolution": "node", + "moduleResolution": "Node", "strictNullChecks": true, "strictFunctionTypes": true, "checkJs": true, diff --git a/package-lock.json b/package-lock.json index e25409398..ca8352b6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,8 +25,8 @@ "express": "^4.21.0", "form-data": "^4.0.0", "google-translate-api-browser": "^3.0.1", - "he": "^1.2.0", "helmet": "^7.1.0", + "html-entities": "^2.5.2", "iconv-lite": "^0.6.3", "ip-matching": "^2.1.2", "ipaddr.js": "^2.0.1", @@ -34,7 +34,7 @@ "lodash": "^4.17.21", "mime-types": "^2.1.35", "multer": "^1.4.5-lts.1", - "node-fetch": "^2.6.11", + "node-fetch": "^3.3.2", "node-persist": "^4.0.1", "open": "^8.4.2", "png-chunk-text": "^1.0.0", @@ -59,9 +59,28 @@ "sillytavern": "server.js" }, "devDependencies": { + "@types/archiver": "^6.0.2", + "@types/command-exists": "^1.2.3", + "@types/compression": "^1.7.5", + "@types/cookie-parser": "^1.4.7", + "@types/cookie-session": "^2.0.49", + "@types/cors": "^2.8.17", "@types/dompurify": "^3.0.5", + "@types/express": "^5.0.0", "@types/jquery": "^3.5.29", + "@types/lodash": "^4.17.10", + "@types/mime-types": "^2.1.4", + "@types/multer": "^1.4.12", + "@types/node-fetch": "^2.6.11", + "@types/node-persist": "^3.1.8", + "@types/png-chunk-text": "^1.0.3", + "@types/png-chunks-encode": "^1.0.2", + "@types/png-chunks-extract": "^1.0.2", + "@types/response-time": "^2.3.8", "@types/toastr": "^2.1.43", + "@types/write-file-atomic": "^4.0.3", + "@types/yargs": "^17.0.33", + "@types/yauzl": "^2.10.3", "eslint": "^8.57.0" }, "engines": { @@ -949,6 +968,27 @@ "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", "license": "MIT" }, + "node_modules/@types/archiver": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz", + "integrity": "sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/readdir-glob": "*" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -961,6 +1001,64 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/command-exists": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@types/command-exists/-/command-exists-1.2.3.tgz", + "integrity": "sha512-PpbaE2XWLaWYboXD6k70TcXO/OdOyyRFq5TVpmlUELNxdkkmXU9fkImNosmXU1DtsNrqdUgWd/nJQYXgwmtdXQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/compression": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cookie-parser": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.7.tgz", + "integrity": "sha512-Fvuyi354Z+uayxzIGCwYTayFKocfV7TuDYZClCdIP9ckhvAu/ixDtCB6qx2TT0FKjPLf1f3P/J1rgf6lPs64mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/cookie-session": { + "version": "2.0.49", + "resolved": "https://registry.npmjs.org/@types/cookie-session/-/cookie-session-2.0.49.tgz", + "integrity": "sha512-4E/bBjlqLhU5l4iGPR+NkVJH593hpNsT4dC3DJDr+ODm6Qpe13kZQVkezRIb+TYDXaBMemS3yLQ+0leba3jlkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/keygrip": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.17", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz", + "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/dompurify": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz", @@ -971,12 +1069,45 @@ "@types/trusted-types": "*" } }, + "node_modules/@types/express": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", + "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz", "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==", "license": "MIT" }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/jquery": { "version": "3.5.31", "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.31.tgz", @@ -987,6 +1118,13 @@ "@types/sizzle": "*" } }, + "node_modules/@types/keygrip": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz", + "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/keyv": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", @@ -996,12 +1134,43 @@ "@types/node": "*" } }, + "node_modules/@types/lodash": { + "version": "4.17.10", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.10.tgz", + "integrity": "sha512-YpS0zzoduEhuOWjAotS6A5AVCva7X4lVlYLF0FYHAY9sdraBfnatttHItlWeZdGhuEkf+OzMNg2ZYAx8t+52uQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", "license": "MIT" }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/multer": { + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz", + "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { "version": "16.9.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", @@ -1009,15 +1178,81 @@ "license": "MIT" }, "node_modules/@types/node-fetch": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz", - "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==", + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", "license": "MIT", "dependencies": { "@types/node": "*", "form-data": "^4.0.0" } }, + "node_modules/@types/node-persist": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/node-persist/-/node-persist-3.1.8.tgz", + "integrity": "sha512-QLidg6/SadZYPrTKxtxL1A85XBoQlG40bhoMdhu6DH6+eNCMr2j+RGfFZ9I9+IY8W/PDwQonJ+iBWD62jZjMfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/png-chunk-text": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@types/png-chunk-text/-/png-chunk-text-1.0.3.tgz", + "integrity": "sha512-7keEFz73uNJ9Ar1XMCNnHEXT9pICJnouMQCCYgBEmHMgdkXaQzSTmSvr6tUDSqgdEgmlRAxZd97wprgliyZoCg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/png-chunks-encode": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/png-chunks-encode/-/png-chunks-encode-1.0.2.tgz", + "integrity": "sha512-Dxn0aXEcSg1wVeHjvNlygm/+fKBDzWMCdxJYhjGUTeefFW/jYxWcrg+W7ppLBfH44iJMqeVBHtHBwtYQUeYvgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/png-chunks-extract": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/png-chunks-extract/-/png-chunks-extract-1.0.2.tgz", + "integrity": "sha512-z6djfFIbrrddtunoMJBOPlyZrnmeuG1kkvHUNi2QfpOb+JMMLuLliHHTmMyRi7k7LiTAut0HbdGCF6ibDtQAHQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/qs": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/readdir-glob": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", + "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/response-time": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/@types/response-time/-/response-time-2.3.8.tgz", + "integrity": "sha512-7qGaNYvdxc0zRab8oHpYx7AW17qj+G0xuag1eCrw3M2VWPJQ/HyKaaghWygiaOUl0y9x7QGQwppDpqLJ5V9pzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/node": "*" + } + }, "node_modules/@types/responselike": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.1.tgz", @@ -1027,6 +1262,29 @@ "@types/node": "*" } }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/sizzle": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.8.tgz", @@ -1050,6 +1308,43 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/write-file-atomic": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/write-file-atomic/-/write-file-atomic-4.0.3.tgz", + "integrity": "sha512-qdo+vZRchyJIHNeuI1nrpsLw+hnkgqP/8mlaN6Wle/NKhydHmUN9l4p3ZE8yP90AJNJW4uB8HQhedb4f1vNayQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", @@ -3009,6 +3304,29 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -3178,6 +3496,18 @@ "node": ">= 14" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3489,14 +3819,6 @@ "node": ">= 0.4" } }, - "node_modules/he": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "bin": { - "he": "bin/he" - } - }, "node_modules/helmet": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz", @@ -3505,6 +3827,22 @@ "node": ">=16.0.0" } }, + "node_modules/html-entities": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", + "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -3876,6 +4214,26 @@ "whatwg-fetch": "^3.4.1" } }, + "node_modules/isomorphic-fetch/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/jackspeak": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", @@ -4355,23 +4713,30 @@ } }, "node_modules/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" }, "engines": { - "node": "4.x || >=6.0.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/node-fetch/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" } }, "node_modules/node-persist": { @@ -4546,6 +4911,26 @@ "undici-types": "~5.26.4" } }, + "node_modules/openai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -5808,6 +6193,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", + "license": "MIT", "dependencies": { "punycode": "^2.3.1" }, @@ -6012,6 +6398,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", "engines": { "node": ">=12" } @@ -6026,6 +6413,7 @@ "version": "14.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", + "license": "MIT", "dependencies": { "tr46": "^5.0.0", "webidl-conversions": "^7.0.0" diff --git a/package.json b/package.json index 97ec01a9c..14b032049 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,8 @@ "express": "^4.21.0", "form-data": "^4.0.0", "google-translate-api-browser": "^3.0.1", - "he": "^1.2.0", "helmet": "^7.1.0", + "html-entities": "^2.5.2", "iconv-lite": "^0.6.3", "ip-matching": "^2.1.2", "ipaddr.js": "^2.0.1", @@ -24,7 +24,7 @@ "lodash": "^4.17.21", "mime-types": "^2.1.35", "multer": "^1.4.5-lts.1", - "node-fetch": "^2.6.11", + "node-fetch": "^3.3.2", "node-persist": "^4.0.1", "open": "^8.4.2", "png-chunk-text": "^1.0.0", @@ -60,7 +60,7 @@ } }, "name": "sillytavern", - "type": "commonjs", + "type": "module", "license": "AGPL-3.0", "repository": { "type": "git", @@ -85,9 +85,28 @@ }, "main": "server.js", "devDependencies": { + "@types/archiver": "^6.0.2", + "@types/command-exists": "^1.2.3", + "@types/compression": "^1.7.5", + "@types/cookie-parser": "^1.4.7", + "@types/cookie-session": "^2.0.49", + "@types/cors": "^2.8.17", "@types/dompurify": "^3.0.5", + "@types/express": "^5.0.0", "@types/jquery": "^3.5.29", + "@types/lodash": "^4.17.10", + "@types/mime-types": "^2.1.4", + "@types/multer": "^1.4.12", + "@types/node-fetch": "^2.6.11", + "@types/node-persist": "^3.1.8", + "@types/png-chunk-text": "^1.0.3", + "@types/png-chunks-encode": "^1.0.2", + "@types/png-chunks-extract": "^1.0.2", + "@types/response-time": "^2.3.8", "@types/toastr": "^2.1.43", + "@types/write-file-atomic": "^4.0.3", + "@types/yargs": "^17.0.33", + "@types/yauzl": "^2.10.3", "eslint": "^8.57.0" } } diff --git a/plugins.js b/plugins.js index e25977de5..3ac5fd63b 100644 --- a/plugins.js +++ b/plugins.js @@ -3,11 +3,12 @@ // 1. node plugins.js update // 2. node plugins.js install // More operations coming soon. -const { default: git } = require('simple-git'); -const fs = require('fs'); -const path = require('path'); -const { color } = require('./src/util'); +import { default as git } from 'simple-git'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { color } from './src/util.js'; +const __dirname = import.meta.dirname; process.chdir(__dirname); const pluginsPath = './plugins'; diff --git a/post-install.js b/post-install.js index f787eeba3..08e8a0573 100644 --- a/post-install.js +++ b/post-install.js @@ -1,11 +1,12 @@ /** * Scripts to be done before starting the server for the first time. */ -const fs = require('fs'); -const path = require('path'); -const crypto = require('crypto'); -const yaml = require('yaml'); -const _ = require('lodash'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as crypto from 'node:crypto'; +import * as yaml from 'yaml'; +import _ from 'lodash'; +import { createRequire } from 'node:module'; /** * Colorizes console output. @@ -59,6 +60,7 @@ function convertConfig() { try { console.log(color.blue('Converting config.conf to config.yaml. Your old config.conf will be renamed to config.conf.bak')); + const require = createRequire(import.meta.url); const config = require(path.join(process.cwd(), './config.conf')); fs.copyFileSync('./config.conf', './config.conf.bak'); fs.rmSync('./config.conf'); @@ -75,7 +77,7 @@ function convertConfig() { * Compares the current config.yaml with the default config.yaml and adds any missing values. */ function addMissingConfigValues() { - try { + try { const defaultConfig = yaml.parse(fs.readFileSync(path.join(process.cwd(), './default/config.yaml'), 'utf8')); let config = yaml.parse(fs.readFileSync(path.join(process.cwd(), './config.yaml'), 'utf8')); diff --git a/recover.js b/recover.js index 938b44153..14379017a 100644 --- a/recover.js +++ b/recover.js @@ -1,7 +1,7 @@ -const yaml = require('yaml'); -const fs = require('fs'); -const storage = require('node-persist'); -const users = require('./src/users'); +import * as fs from 'node:fs'; +import yaml from 'yaml'; +import storage from 'node-persist'; +import * as users from './src/users'; const userAccount = process.argv[2]; const userPassword = process.argv[3]; diff --git a/server.js b/server.js index 99c8c44d3..ed1ce94b7 100644 --- a/server.js +++ b/server.js @@ -1,32 +1,33 @@ #!/usr/bin/env node // native node modules -const fs = require('fs'); -const http = require('http'); -const https = require('https'); -const path = require('path'); -const util = require('util'); +import * as fs from 'node:fs'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import * as path from 'node:path'; +import * as util from 'node:util'; +import * as net from 'node:net'; +import * as dns from 'node:dns'; // cli/fs related library imports -const open = require('open'); -const yargs = require('yargs/yargs'); -const { hideBin } = require('yargs/helpers'); +import * as open from 'open'; +import yargs from 'yargs/yargs'; +import { hideBin } from 'yargs/helpers'; // express/server related library imports -const cors = require('cors'); -const doubleCsrf = require('csrf-csrf').doubleCsrf; -const express = require('express'); -const compression = require('compression'); -const cookieParser = require('cookie-parser'); -const cookieSession = require('cookie-session'); -const multer = require('multer'); -const responseTime = require('response-time'); -const helmet = require('helmet').default; +import cors from 'cors'; +import { doubleCsrf } from 'csrf-csrf'; +import express from 'express'; +import compression from 'compression'; +import cookieParser from 'cookie-parser'; +import cookieSession from 'cookie-session'; +import multer from 'multer'; +import responseTime from 'response-time'; +import helmet from 'helmet'; +import bodyParser from 'body-parser'; // net related library imports -const net = require('net'); -const dns = require('dns'); -const fetch = require('node-fetch').default; +import fetch from 'node-fetch'; // Unrestrict console logs display limit util.inspect.defaultOptions.maxArrayLength = null; @@ -34,20 +35,67 @@ util.inspect.defaultOptions.maxStringLength = null; util.inspect.defaultOptions.depth = 4; // local library imports -const userModule = require('./src/users'); -const basicAuthMiddleware = require('./src/middleware/basicAuth'); -const whitelistMiddleware = require('./src/middleware/whitelist'); -const initRequestProxy = require('./src/request-proxy'); -const contentManager = require('./src/endpoints/content-manager'); -const { +import * as loader from './src/plugin-loader.js'; +import * as userModule from './src/users.js'; +import basicAuthMiddleware from './src/middleware/basicAuth.js'; +import whitelistMiddleware from './src/middleware/whitelist.js'; +import multerMonkeyPatch from './src/middleware/multerMonkeyPatch.js'; +import initRequestProxy from './src/request-proxy.js'; +import * as contentManager from './src/endpoints/content-manager.js'; +import { getVersion, getConfigValue, color, forwardFetchResponse, removeColorFormatting, getSeparator, -} = require('./src/util'); -const { ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +} from './src/util.js'; +import { UPLOADS_DIRECTORY } from './src/constants.js'; +import { ensureThumbnailCache } from './src/endpoints/thumbnails.js'; + +// Routers +import { router as usersPublicRouter } from './src/endpoints/users-public.js'; +import { router as usersPrivateRouter } from './src/endpoints/users-private.js'; +import { router as usersAdminRouter } from './src/endpoints/users-admin.js'; +import { router as movingUIRouter } from './src/endpoints/moving-ui.js'; +import { router as imagesRouter } from './src/endpoints/images.js'; +import { router as quickRepliesRouter } from './src/endpoints/quick-replies.js'; +import { router as avatarsRouter } from './src/endpoints/avatars.js'; +import { router as themesRouter } from './src/endpoints/themes.js'; +import { router as openAiRouter } from './src/endpoints/openai.js'; +import { router as googleRouter } from './src/endpoints/google.js'; +import { router as anthropicRouter } from './src/endpoints/anthropic.js'; +import { router as tokenizersRouter } from './src/endpoints/tokenizers.js'; +import { router as presetsRouter } from './src/endpoints/presets.js'; +import { router as secretsRouter } from './src/endpoints/secrets.js'; +import { router as thumbnailRouter } from './src/endpoints/thumbnails.js'; +import { router as novelAiRouter } from './src/endpoints/novelai.js'; +import { router as extensionsRouter } from './src/endpoints/extensions.js'; +import { router as assetsRouter } from './src/endpoints/assets.js'; +import { router as filesRouter } from './src/endpoints/files.js'; +import { router as charactersRouter } from './src/endpoints/characters.js'; +import { router as chatsRouter } from './src/endpoints/chats.js'; +import { router as groupsRouter } from './src/endpoints/groups.js'; +import { router as worldInfoRouter } from './src/endpoints/worldinfo.js'; +import { router as statsRouter, init as statsInit, onExit as statsOnExit } from './src/endpoints/stats.js'; +import { router as backgroundsRouter } from './src/endpoints/backgrounds.js'; +import { router as spritesRouter } from './src/endpoints/sprites.js'; +import { router as contentManagerRouter } from './src/endpoints/content-manager.js'; +import { router as settingsRouter, init as settingsInit } from './src/endpoints/settings.js'; +import { router as stableDiffusionRouter } from './src/endpoints/stable-diffusion.js'; +import { router as hordeRouter } from './src/endpoints/horde.js'; +import { router as vectorsRouter } from './src/endpoints/vectors.js'; +import { router as translateRouter } from './src/endpoints/translate.js'; +import { router as classifyRouter } from './src/endpoints/classify.js'; +import { router as captionRouter } from './src/endpoints/caption.js'; +import { router as searchRouter } from './src/endpoints/search.js'; +import { router as openRouterRouter } from './src/endpoints/openrouter.js'; +import { router as chatCompletionsRouter } from './src/endpoints/backends/chat-completions.js'; +import { router as koboldRouter } from './src/endpoints/backends/kobold.js'; +import { router as textCompletionsRouter } from './src/endpoints/backends/text-completions.js'; +import { router as scaleAltRouter } from './src/endpoints/backends/scale-alt.js'; +import { router as speechRouter } from './src/endpoints/speech.js'; +import { router as azureRouter } from './src/endpoints/azure.js'; // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 @@ -166,8 +214,8 @@ const cliArguments = yargs(hideBin(process.argv)) }).parseSync(); // change all relative paths -console.log(`Node version: ${process.version}. Running in ${process.env.NODE_ENV} environment.`); -const serverDirectory = __dirname; +const serverDirectory = import.meta.dirname; +console.log(`Node version: ${process.version}. Running in ${process.env.NODE_ENV} environment. Server directory: ${serverDirectory}`); process.chdir(serverDirectory); const app = express(); @@ -188,7 +236,7 @@ const basicAuthMode = cliArguments.basicAuthMode ?? getConfigValue('basicAuthMod const perUserBasicAuth = getConfigValue('perUserBasicAuth', DEFAULT_PER_USER_BASIC_AUTH); const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS); -const uploadsPath = path.join(dataRoot, require('./src/constants').UPLOADS_DIRECTORY); +const uploadsPath = path.join(dataRoot, UPLOADS_DIRECTORY); const enableIPv6 = cliArguments.enableIPv6 ?? getConfigValue('protocol.ipv6', DEFAULT_ENABLE_IPV6); const enableIPv4 = cliArguments.enableIPv4 ?? getConfigValue('protocol.ipv4', DEFAULT_ENABLE_IPV4); @@ -232,7 +280,6 @@ if (listen && basicAuthMode) app.use(basicAuthMiddleware); app.use(whitelistMiddleware(enableWhitelist, listen)); if (enableCorsProxy) { - const bodyParser = require('body-parser'); app.use(bodyParser.json({ limit: '200mb', })); @@ -379,7 +426,7 @@ app.get('/login', async (request, response) => { app.use(express.static(process.cwd() + '/public', {})); // Public API -app.use('/api/users', require('./src/endpoints/users-public').router); +app.use('/api/users', usersPublicRouter); // Everything below this line requires authentication app.use(userModule.requireLoginMiddleware); @@ -387,14 +434,15 @@ app.get('/api/ping', (_, response) => response.sendStatus(204)); // File uploads app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar')); -app.use(require('./src/middleware/multerMonkeyPatch')); +app.use(multerMonkeyPatch); // User data mount app.use('/', userModule.router); // Private endpoints -app.use('/api/users', require('./src/endpoints/users-private').router); + +app.use('/api/users', usersPrivateRouter); // Admin endpoints -app.use('/api/users', require('./src/endpoints/users-admin').router); +app.use('/api/users', usersAdminRouter); app.get('/version', async function (_, response) { const data = await getVersion(); @@ -509,126 +557,45 @@ redirect('/api/serpapi/search', '/api/search/serpapi'); redirect('/api/serpapi/visit', '/api/search/visit'); redirect('/api/serpapi/transcript', '/api/search/transcript'); -// Moving UI -app.use('/api/moving-ui', require('./src/endpoints/moving-ui').router); - -// Image management -app.use('/api/images', require('./src/endpoints/images').router); - -// Quick reply management -app.use('/api/quick-replies', require('./src/endpoints/quick-replies').router); - -// Avatar management -app.use('/api/avatars', require('./src/endpoints/avatars').router); - -// Theme management -app.use('/api/themes', require('./src/endpoints/themes').router); - -// OpenAI API -app.use('/api/openai', require('./src/endpoints/openai').router); - -//Google API -app.use('/api/google', require('./src/endpoints/google').router); - -//Anthropic API -app.use('/api/anthropic', require('./src/endpoints/anthropic').router); - -// Tokenizers -app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router); - -// Preset management -app.use('/api/presets', require('./src/endpoints/presets').router); - -// Secrets managemenet -app.use('/api/secrets', require('./src/endpoints/secrets').router); - -// Thumbnail generation. These URLs are saved in chat, so this route cannot be renamed! -app.use('/thumbnail', require('./src/endpoints/thumbnails').router); - -// NovelAI generation -app.use('/api/novelai', require('./src/endpoints/novelai').router); - -// Third-party extensions -app.use('/api/extensions', require('./src/endpoints/extensions').router); - -// Asset management -app.use('/api/assets', require('./src/endpoints/assets').router); - -// File management -app.use('/api/files', require('./src/endpoints/files').router); - -// Character management -app.use('/api/characters', require('./src/endpoints/characters').router); - -// Chat management -app.use('/api/chats', require('./src/endpoints/chats').router); - -// Group management -app.use('/api/groups', require('./src/endpoints/groups').router); - -// World info management -app.use('/api/worldinfo', require('./src/endpoints/worldinfo').router); - -// Stats calculation -const statsEndpoint = require('./src/endpoints/stats'); -app.use('/api/stats', statsEndpoint.router); - -// Background management -app.use('/api/backgrounds', require('./src/endpoints/backgrounds').router); - -// Character sprite management -app.use('/api/sprites', require('./src/endpoints/sprites').router); - -// Custom content management -app.use('/api/content', require('./src/endpoints/content-manager').router); - -// Settings load/store -const settingsEndpoint = require('./src/endpoints/settings'); -app.use('/api/settings', settingsEndpoint.router); - -// Stable Diffusion generation -app.use('/api/sd', require('./src/endpoints/stable-diffusion').router); - -// LLM and SD Horde generation -app.use('/api/horde', require('./src/endpoints/horde').router); - -// Vector storage DB -app.use('/api/vector', require('./src/endpoints/vectors').router); - -// Chat translation -app.use('/api/translate', require('./src/endpoints/translate').router); - -// Emotion classification -app.use('/api/extra/classify', require('./src/endpoints/classify').router); - -// Image captioning -app.use('/api/extra/caption', require('./src/endpoints/caption').router); - -// Web search and scraping -app.use('/api/search', require('./src/endpoints/search').router); - -// The different text generation APIs - -// Ooba/OpenAI text completions -app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router); - -// OpenRouter -app.use('/api/openrouter', require('./src/endpoints/openrouter').router); - -// KoboldAI -app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router); - -// OpenAI chat completions -app.use('/api/backends/chat-completions', require('./src/endpoints/backends/chat-completions').router); - -// Scale (alt method) -app.use('/api/backends/scale-alt', require('./src/endpoints/backends/scale-alt').router); - -// Speech (text-to-speech and speech-to-text) -app.use('/api/speech', require('./src/endpoints/speech').router); - -// Azure TTS -app.use('/api/azure', require('./src/endpoints/azure').router); +app.use('/api/moving-ui', movingUIRouter); +app.use('/api/images', imagesRouter); +app.use('/api/quick-replies', quickRepliesRouter); +app.use('/api/avatars', avatarsRouter); +app.use('/api/themes', themesRouter); +app.use('/api/openai', openAiRouter); +app.use('/api/google', googleRouter); +app.use('/api/anthropic', anthropicRouter); +app.use('/api/tokenizers', tokenizersRouter); +app.use('/api/presets', presetsRouter); +app.use('/api/secrets', secretsRouter); +app.use('/thumbnail', thumbnailRouter); +app.use('/api/novelai', novelAiRouter); +app.use('/api/extensions', extensionsRouter); +app.use('/api/assets', assetsRouter); +app.use('/api/files', filesRouter); +app.use('/api/characters', charactersRouter); +app.use('/api/chats', chatsRouter); +app.use('/api/groups', groupsRouter); +app.use('/api/worldinfo', worldInfoRouter); +app.use('/api/stats', statsRouter); +app.use('/api/backgrounds', backgroundsRouter); +app.use('/api/sprites', spritesRouter); +app.use('/api/content', contentManagerRouter); +app.use('/api/settings', settingsRouter); +app.use('/api/sd', stableDiffusionRouter); +app.use('/api/horde', hordeRouter); +app.use('/api/vector', vectorsRouter); +app.use('/api/translate', translateRouter); +app.use('/api/extra/classify', classifyRouter); +app.use('/api/extra/caption', captionRouter); +app.use('/api/search', searchRouter); +app.use('/api/backends/text-completions', textCompletionsRouter); +app.use('/api/openrouter', openRouterRouter); +app.use('/api/backends/kobold', koboldRouter); +app.use('/api/backends/chat-completions', chatCompletionsRouter); +app.use('/api/backends/scale-alt', scaleAltRouter); +app.use('/api/speech', speechRouter); +app.use('/api/azure', azureRouter); const tavernUrlV6 = new URL( (cliArguments.ssl ? 'https://' : 'http://') + @@ -663,8 +630,8 @@ const preSetupTasks = async function () { await ensureThumbnailCache(); cleanUploads(); - await settingsEndpoint.init(); - await statsEndpoint.init(); + await settingsInit(); + await statsInit(); const cleanupPlugins = await loadPlugins(); const consoleTitle = process.title; @@ -673,7 +640,7 @@ const preSetupTasks = async function () { const exitProcess = async () => { if (isExiting) return; isExiting = true; - statsEndpoint.onExit(); + await statsOnExit(); if (typeof cleanupPlugins === 'function') { await cleanupPlugins(); } @@ -776,7 +743,6 @@ const postSetupTasks = async function (v6Failed, v4Failed) { async function loadPlugins() { try { const pluginDirectory = path.join(serverDirectory, 'plugins'); - const loader = require('./src/plugin-loader'); const cleanupPlugins = await loader.loadPlugins(app, pluginDirectory); return cleanupPlugins; } catch { diff --git a/src/additional-headers.js b/src/additional-headers.js index 5ac879411..d24afb7d7 100644 --- a/src/additional-headers.js +++ b/src/additional-headers.js @@ -1,6 +1,6 @@ -const { TEXTGEN_TYPES, OPENROUTER_HEADERS, FEATHERLESS_HEADERS } = require('./constants'); -const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); -const { getConfigValue } = require('./util'); +import { TEXTGEN_TYPES, OPENROUTER_HEADERS, FEATHERLESS_HEADERS } from './constants.js'; +import { SECRET_KEYS, readSecret } from './endpoints/secrets.js'; +import { getConfigValue } from './util.js'; /** * Gets the headers for the Mancer API. @@ -172,7 +172,7 @@ function getHuggingFaceHeaders(directories) { }) : {}; } -function getOverrideHeaders(urlHost) { +export function getOverrideHeaders(urlHost) { const requestOverrides = getConfigValue('requestOverrides', []); const overrideHeaders = requestOverrides?.find((e) => e.hosts?.includes(urlHost))?.headers; if (overrideHeaders && urlHost) { @@ -188,7 +188,7 @@ function getOverrideHeaders(urlHost) { * @param {object} args New request arguments * @param {string|null} server API server for new request */ -function setAdditionalHeaders(request, args, server) { +export function setAdditionalHeaders(request, args, server) { setAdditionalHeadersByType(args.headers, request.body.api_type, server, request.user.directories); } @@ -199,7 +199,7 @@ function setAdditionalHeaders(request, args, server) { * @param {string|null} server API server for new request * @param {import('./users').UserDirectoryList} directories User directories */ -function setAdditionalHeadersByType(requestHeaders, type, server, directories) { +export function setAdditionalHeadersByType(requestHeaders, type, server, directories) { const headerGetters = { [TEXTGEN_TYPES.MANCER]: getMancerHeaders, [TEXTGEN_TYPES.VLLM]: getVllmHeaders, @@ -234,9 +234,3 @@ function setAdditionalHeadersByType(requestHeaders, type, server, directories) { Object.assign(requestHeaders, headers); } - -module.exports = { - getOverrideHeaders, - setAdditionalHeaders, - setAdditionalHeadersByType, -}; diff --git a/src/character-card-parser.js b/src/character-card-parser.js index 79b9e5b44..3b7980399 100644 --- a/src/character-card-parser.js +++ b/src/character-card-parser.js @@ -1,8 +1,8 @@ -const fs = require('fs'); +import fs from 'node:fs'; -const encode = require('png-chunks-encode'); -const extract = require('png-chunks-extract'); -const PNGtext = require('png-chunk-text'); +import encode from 'png-chunks-encode'; +import extract from 'png-chunks-extract'; +import PNGtext from 'png-chunk-text'; /** * Writes Character metadata to a PNG image buffer. @@ -11,7 +11,7 @@ const PNGtext = require('png-chunk-text'); * @param {string} data Character data to write * @returns {Buffer} PNG image buffer with metadata */ -const write = (image, data) => { +export const write = (image, data) => { const chunks = extract(image); const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt'); @@ -36,7 +36,9 @@ const write = (image, data) => { const base64EncodedData = Buffer.from(JSON.stringify(v3Data), 'utf8').toString('base64'); chunks.splice(-1, 0, PNGtext.encode('ccv3', base64EncodedData)); - } catch (error) { } + } catch (error) { + // Ignore errors when adding v3 chunk + } const newBuffer = Buffer.from(encode(chunks)); return newBuffer; @@ -48,7 +50,7 @@ const write = (image, data) => { * @param {Buffer} image PNG image buffer * @returns {string} Character data */ -const read = (image) => { +export const read = (image) => { const chunks = extract(image); const textChunks = chunks.filter((chunk) => chunk.name === 'tEXt').map((chunk) => PNGtext.decode(chunk.data)); @@ -80,7 +82,7 @@ const read = (image) => { * @param {string} format File format * @returns {string} Character data */ -const parse = (cardUrl, format) => { +export const parse = (cardUrl, format) => { let fileFormat = format === undefined ? 'png' : format; switch (fileFormat) { @@ -93,8 +95,3 @@ const parse = (cardUrl, format) => { throw new Error('Unsupported format'); }; -module.exports = { - parse, - write, - read, -}; diff --git a/src/constants.js b/src/constants.js index 19197cd05..73896a56d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,18 +1,18 @@ -const PUBLIC_DIRECTORIES = { +export const PUBLIC_DIRECTORIES = { images: 'public/img/', backups: 'backups/', sounds: 'public/sounds', extensions: 'public/scripts/extensions', }; -const SETTINGS_FILE = 'settings.json'; +export const SETTINGS_FILE = 'settings.json'; /** * @type {import('./users').UserDirectoryList} * @readonly * @enum {string} */ -const USER_DIRECTORY_TEMPLATE = Object.freeze({ +export const USER_DIRECTORY_TEMPLATE = Object.freeze({ root: '', thumbnails: 'thumbnails', thumbnailsBg: 'thumbnails/bg', @@ -48,7 +48,7 @@ const USER_DIRECTORY_TEMPLATE = Object.freeze({ * @type {import('./users').User} * @readonly */ -const DEFAULT_USER = Object.freeze({ +export const DEFAULT_USER = Object.freeze({ handle: 'default-user', name: 'User', created: Date.now(), @@ -58,7 +58,7 @@ const DEFAULT_USER = Object.freeze({ salt: '', }); -const UNSAFE_EXTENSIONS = [ +export const UNSAFE_EXTENSIONS = [ '.php', '.exe', '.com', @@ -135,7 +135,7 @@ const UNSAFE_EXTENSIONS = [ '.ws', ]; -const GEMINI_SAFETY = [ +export const GEMINI_SAFETY = [ { category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE', @@ -158,7 +158,7 @@ const GEMINI_SAFETY = [ }, ]; -const BISON_SAFETY = [ +export const BISON_SAFETY = [ { category: 'HARM_CATEGORY_DEROGATORY', threshold: 'BLOCK_NONE', @@ -185,7 +185,7 @@ const BISON_SAFETY = [ }, ]; -const CHAT_COMPLETION_SOURCES = { +export const CHAT_COMPLETION_SOURCES = { OPENAI: 'openai', WINDOWAI: 'windowai', CLAUDE: 'claude', @@ -205,10 +205,10 @@ const CHAT_COMPLETION_SOURCES = { /** * Path to multer file uploads under the data root. */ -const UPLOADS_DIRECTORY = '_uploads'; +export const UPLOADS_DIRECTORY = '_uploads'; // TODO: this is copied from the client code; there should be a way to de-duplicate it eventually -const TEXTGEN_TYPES = { +export const TEXTGEN_TYPES = { OOBA: 'ooba', MANCER: 'mancer', VLLM: 'vllm', @@ -225,7 +225,7 @@ const TEXTGEN_TYPES = { HUGGINGFACE: 'huggingface', }; -const INFERMATICAI_KEYS = [ +export const INFERMATICAI_KEYS = [ 'model', 'prompt', 'max_tokens', @@ -248,7 +248,7 @@ const INFERMATICAI_KEYS = [ 'logprobs', ]; -const FEATHERLESS_KEYS = [ +export const FEATHERLESS_KEYS = [ 'model', 'prompt', 'best_of', @@ -290,9 +290,8 @@ const FEATHERLESS_KEYS = [ 'guided_whitespace_pattern', ]; - // https://dreamgen.com/docs/api#openai-text -const DREAMGEN_KEYS = [ +export const DREAMGEN_KEYS = [ 'model', 'prompt', 'max_tokens', @@ -309,7 +308,7 @@ const DREAMGEN_KEYS = [ ]; // https://docs.together.ai/reference/completions -const TOGETHERAI_KEYS = [ +export const TOGETHERAI_KEYS = [ 'model', 'prompt', 'max_tokens', @@ -325,7 +324,7 @@ const TOGETHERAI_KEYS = [ ]; // https://github.com/jmorganca/ollama/blob/main/docs/api.md#request-with-options -const OLLAMA_KEYS = [ +export const OLLAMA_KEYS = [ 'num_predict', 'num_ctx', 'stop', @@ -345,20 +344,20 @@ const OLLAMA_KEYS = [ 'min_p', ]; -const AVATAR_WIDTH = 512; -const AVATAR_HEIGHT = 768; +export const AVATAR_WIDTH = 512; +export const AVATAR_HEIGHT = 768; -const OPENROUTER_HEADERS = { +export const OPENROUTER_HEADERS = { 'HTTP-Referer': 'https://sillytavern.app', 'X-Title': 'SillyTavern', }; -const FEATHERLESS_HEADERS = { +export const FEATHERLESS_HEADERS = { 'HTTP-Referer': 'https://sillytavern.app', 'X-Title': 'SillyTavern', }; -const OPENROUTER_KEYS = [ +export const OPENROUTER_KEYS = [ 'max_tokens', 'temperature', 'top_k', @@ -378,7 +377,7 @@ const OPENROUTER_KEYS = [ ]; // https://github.com/vllm-project/vllm/blob/0f8a91401c89ac0a8018def3756829611b57727f/vllm/entrypoints/openai/protocol.py#L220 -const VLLM_KEYS = [ +export const VLLM_KEYS = [ 'model', 'prompt', 'best_of', @@ -419,27 +418,3 @@ const VLLM_KEYS = [ 'guided_decoding_backend', 'guided_whitespace_pattern', ]; - -module.exports = { - DEFAULT_USER, - SETTINGS_FILE, - PUBLIC_DIRECTORIES, - USER_DIRECTORY_TEMPLATE, - UNSAFE_EXTENSIONS, - UPLOADS_DIRECTORY, - GEMINI_SAFETY, - BISON_SAFETY, - TEXTGEN_TYPES, - CHAT_COMPLETION_SOURCES, - AVATAR_WIDTH, - AVATAR_HEIGHT, - TOGETHERAI_KEYS, - OLLAMA_KEYS, - INFERMATICAI_KEYS, - DREAMGEN_KEYS, - OPENROUTER_HEADERS, - OPENROUTER_KEYS, - VLLM_KEYS, - FEATHERLESS_KEYS, - FEATHERLESS_HEADERS, -}; diff --git a/src/endpoints/anthropic.js b/src/endpoints/anthropic.js index c30b67f99..ff0b7b6ad 100644 --- a/src/endpoints/anthropic.js +++ b/src/endpoints/anthropic.js @@ -1,9 +1,10 @@ -const { readSecret, SECRET_KEYS } = require('./secrets'); -const fetch = require('node-fetch').default; -const express = require('express'); -const { jsonParser } = require('../express-common'); +import fetch from 'node-fetch'; +import express from 'express'; -const router = express.Router(); +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { jsonParser } from '../express-common.js'; + +export const router = express.Router(); router.post('/caption-image', jsonParser, async (request, response) => { try { @@ -64,5 +65,3 @@ router.post('/caption-image', jsonParser, async (request, response) => { response.status(500).send('Internal server error'); } }); - -module.exports = { router }; diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 508897577..7f282b3f1 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -1,13 +1,15 @@ -const path = require('path'); -const fs = require('fs'); -const mime = require('mime-types'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const fetch = require('node-fetch').default; -const { finished } = require('stream/promises'); -const { UNSAFE_EXTENSIONS } = require('../constants'); -const { jsonParser } = require('../express-common'); -const { clientRelativePath } = require('../util'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { finished } from 'node:stream/promises'; + +import mime from 'mime-types'; +import express from 'express'; +import sanitize from 'sanitize-filename'; +import fetch from 'node-fetch'; + +import { UNSAFE_EXTENSIONS } from '../constants.js'; +import { jsonParser } from '../express-common.js'; +import { clientRelativePath } from '../util.js'; const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character', 'temp']; @@ -16,7 +18,7 @@ const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character' * @param {string} inputFilename Input filename * @returns {{error: boolean, message?: string}} Whether validation failed, and why if so */ -function validateAssetFileName(inputFilename) { +export function validateAssetFileName(inputFilename) { if (!/^[a-zA-Z0-9_\-.]+$/.test(inputFilename)) { return { error: true, @@ -93,7 +95,7 @@ function ensureFoldersExist(directories) { } } -const router = express.Router(); +export const router = express.Router(); /** * HTTP POST handler function to retrieve name of all files of a given folder path. @@ -366,5 +368,3 @@ router.post('/character', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { router, validateAssetFileName }; diff --git a/src/endpoints/avatars.js b/src/endpoints/avatars.js index 5f8509f0f..d16eb2969 100644 --- a/src/endpoints/avatars.js +++ b/src/endpoints/avatars.js @@ -1,16 +1,16 @@ -const express = require('express'); -const path = require('path'); -const fs = require('fs'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { jsonParser, urlencodedParser } = require('../express-common'); -const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); -const { getImages, tryParse } = require('../util'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; -// image processing related library imports -const jimp = require('jimp'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import jimp from 'jimp'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; -const router = express.Router(); +import { jsonParser, urlencodedParser } from '../express-common.js'; +import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js'; +import { getImages, tryParse } from '../util.js'; + +export const router = express.Router(); router.post('/get', jsonParser, function (request, response) { var images = getImages(request.user.directories.avatars); @@ -58,5 +58,3 @@ router.post('/upload', urlencodedParser, async (request, response) => { return response.status(400).send('Is not a valid image'); } }); - -module.exports = { router }; diff --git a/src/endpoints/azure.js b/src/endpoints/azure.js index 4c3b34d5b..c097eb3ca 100644 --- a/src/endpoints/azure.js +++ b/src/endpoints/azure.js @@ -1,9 +1,10 @@ -const { readSecret, SECRET_KEYS } = require('./secrets'); -const fetch = require('node-fetch').default; -const express = require('express'); -const { jsonParser } = require('../express-common'); +import fetch from 'node-fetch'; +import { Router } from 'express'; -const router = express.Router(); +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { jsonParser } from '../express-common.js'; + +export const router = Router(); router.post('/list', jsonParser, async (req, res) => { try { @@ -86,7 +87,3 @@ router.post('/generate', jsonParser, async (req, res) => { return res.sendStatus(500); } }); - -module.exports = { - router, -}; diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index c190e6f71..7e1c5e6fe 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -1,13 +1,40 @@ -const express = require('express'); -const fetch = require('node-fetch').default; +import express from 'express'; +import fetch from 'node-fetch'; -const { jsonParser } = require('../../express-common'); -const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants'); -const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util'); -const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages, mergeMessages } = require('../../prompt-converters'); +import { jsonParser } from '../../express-common.js'; +import { + CHAT_COMPLETION_SOURCES, + GEMINI_SAFETY, + BISON_SAFETY, + OPENROUTER_HEADERS, +} from '../../constants.js'; +import { + forwardFetchResponse, + getConfigValue, + tryParse, + uuidv4, + mergeObjectWithYaml, + excludeKeysByYaml, + color, +} from '../../util.js'; +import { + convertClaudeMessages, + convertGooglePrompt, + convertTextCompletionPrompt, + convertCohereMessages, + convertMistralMessages, + convertAI21Messages, + mergeMessages, +} from '../../prompt-converters.js'; -const { readSecret, SECRET_KEYS } = require('../secrets'); -const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers'); +import { readSecret, SECRET_KEYS } from '../secrets.js'; +import { + getTokenizerModel, + getSentencepiceTokenizer, + getTiktokenTokenizer, + sentencepieceTokenizers, + TEXT_COMPLETION_MODELS, +} from '../tokenizers.js'; const API_OPENAI = 'https://api.openai.com/v1'; const API_CLAUDE = 'https://api.anthropic.com/v1'; @@ -41,43 +68,6 @@ function postProcessPrompt(messages, type, charName, userName) { } } -/** - * Ollama strikes back. Special boy #2's steaming routine. - * Wrap this abomination into proper SSE stream, again. - * @param {Response} jsonStream JSON stream - * @param {import('express').Request} request Express request - * @param {import('express').Response} response Express response - * @returns {Promise} Nothing valuable - */ -async function parseCohereStream(jsonStream, request, response) { - try { - const stream = new CohereStream({ stream: jsonStream.body, eventShape: { type: 'json', messageTerminator: '\n' } }); - - for await (const json of stream.iterMessages()) { - if (json.message) { - const message = json.message || 'Unknown error'; - const chunk = { error: { message: message } }; - response.write(`data: ${JSON.stringify(chunk)}\n\n`); - } else if (json.event_type === 'text-generation') { - const text = json.text || ''; - const chunk = { choices: [{ text }] }; - response.write(`data: ${JSON.stringify(chunk)}\n\n`); - } - } - - console.log('Streaming request finished'); - response.write('data: [DONE]\n\n'); - response.end(); - } catch (error) { - console.log('Error forwarding streaming response:', error); - if (!response.headersSent) { - return response.status(500).send({ error: true }); - } else { - return response.end(); - } - } -} - /** * Sends a request to Claude API. * @param {express.Request} request Express request @@ -626,7 +616,7 @@ async function sendCohereRequest(request, response) { } } -const router = express.Router(); +export const router = express.Router(); router.post('/status', jsonParser, async function (request, response_getstatus_openai) { if (!request.body) return response_getstatus_openai.sendStatus(400); @@ -1069,6 +1059,3 @@ router.post('/generate', jsonParser, function (request, response) { } }); -module.exports = { - router, -}; diff --git a/src/endpoints/backends/kobold.js b/src/endpoints/backends/kobold.js index 07aec7dcb..990e0085d 100644 --- a/src/endpoints/backends/kobold.js +++ b/src/endpoints/backends/kobold.js @@ -1,13 +1,13 @@ -const express = require('express'); -const fetch = require('node-fetch').default; -const fs = require('fs'); +import * as fs from 'node:fs'; +import express from 'express'; +import fetch from 'node-fetch'; -const { jsonParser, urlencodedParser } = require('../../express-common'); -const { forwardFetchResponse, delay } = require('../../util'); -const { getOverrideHeaders, setAdditionalHeaders, setAdditionalHeadersByType } = require('../../additional-headers'); -const { TEXTGEN_TYPES } = require('../../constants'); +import { jsonParser, urlencodedParser } from '../../express-common.js'; +import { forwardFetchResponse, delay } from '../../util.js'; +import { getOverrideHeaders, setAdditionalHeaders, setAdditionalHeadersByType } from '../../additional-headers.js'; +import { TEXTGEN_TYPES } from '../../constants.js'; -const router = express.Router(); +export const router = express.Router(); router.post('/generate', jsonParser, async function (request, response_generate) { if (!request.body) return response_generate.sendStatus(400); @@ -237,5 +237,3 @@ router.post('/transcribe-audio', urlencodedParser, async function (request, resp response.status(500).send('Internal server error'); } }); - -module.exports = { router }; diff --git a/src/endpoints/backends/scale-alt.js b/src/endpoints/backends/scale-alt.js index 7dc2c981a..0bdbcd336 100644 --- a/src/endpoints/backends/scale-alt.js +++ b/src/endpoints/backends/scale-alt.js @@ -1,11 +1,10 @@ -const express = require('express'); -const fetch = require('node-fetch').default; +import express from 'express'; +import fetch from 'node-fetch'; -const { jsonParser } = require('../../express-common'); +import { jsonParser } from '../../express-common.js'; +import { readSecret, SECRET_KEYS } from '../secrets.js'; -const { readSecret, SECRET_KEYS } = require('../secrets'); - -const router = express.Router(); +export const router = express.Router(); router.post('/generate', jsonParser, async function (request, response) { if (!request.body) return response.sendStatus(400); @@ -97,5 +96,3 @@ router.post('/generate', jsonParser, async function (request, response) { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index e42ef96c8..6c6942621 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -1,14 +1,23 @@ -const express = require('express'); -const fetch = require('node-fetch').default; -const _ = require('lodash'); -const Readable = require('stream').Readable; +import { Readable } from 'node:stream'; +import fetch from 'node-fetch'; +import express from 'express'; +import _ from 'lodash'; -const { jsonParser } = require('../../express-common'); -const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS, VLLM_KEYS, DREAMGEN_KEYS, FEATHERLESS_KEYS } = require('../../constants'); -const { forwardFetchResponse, trimV1, getConfigValue } = require('../../util'); -const { setAdditionalHeaders } = require('../../additional-headers'); +import { jsonParser } from '../../express-common.js'; +import { + TEXTGEN_TYPES, + TOGETHERAI_KEYS, + OLLAMA_KEYS, + INFERMATICAI_KEYS, + OPENROUTER_KEYS, + VLLM_KEYS, + DREAMGEN_KEYS, + FEATHERLESS_KEYS, +} from '../../constants.js'; +import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js'; +import { setAdditionalHeaders } from '../../additional-headers.js'; -const router = express.Router(); +export const router = express.Router(); /** * Special boy's steaming routine. Wrap this abomination into proper SSE stream. @@ -641,5 +650,3 @@ tabby.post('/download', jsonParser, async function (request, response) { router.use('/ollama', ollama); router.use('/llamacpp', llamacpp); router.use('/tabby', tabby); - -module.exports = { router }; diff --git a/src/endpoints/backgrounds.js b/src/endpoints/backgrounds.js index 6ef59f923..4ee1f7c1c 100644 --- a/src/endpoints/backgrounds.js +++ b/src/endpoints/backgrounds.js @@ -1,13 +1,14 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const { jsonParser, urlencodedParser } = require('../express-common'); -const { invalidateThumbnail } = require('./thumbnails'); -const { getImages } = require('../util'); +import express from 'express'; +import sanitize from 'sanitize-filename'; -const router = express.Router(); +import { jsonParser, urlencodedParser } from '../express-common.js'; +import { invalidateThumbnail } from './thumbnails.js'; +import { getImages } from '../util.js'; + +export const router = express.Router(); router.post('/all', jsonParser, function (request, response) { var images = getImages(request.user.directories.backgrounds); @@ -72,5 +73,3 @@ router.post('/upload', urlencodedParser, function (request, response) { response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/caption.js b/src/endpoints/caption.js index ac6e2b896..01d931c61 100644 --- a/src/endpoints/caption.js +++ b/src/endpoints/caption.js @@ -1,23 +1,23 @@ -const express = require('express'); -const { jsonParser } = require('../express-common'); +import express from 'express'; +import { jsonParser } from '../express-common.js'; +import { getPipeline, getRawImage } from '../transformers.mjs'; const TASK = 'image-to-text'; -const router = express.Router(); +export const router = express.Router(); router.post('/', jsonParser, async (req, res) => { try { const { image } = req.body; - const module = await import('../transformers.mjs'); - const rawImage = await module.default.getRawImage(image); + const rawImage = await getRawImage(image); if (!rawImage) { console.log('Failed to parse captioned image'); return res.sendStatus(400); } - const pipe = await module.default.getPipeline(TASK); + const pipe = await getPipeline(TASK); const result = await pipe(rawImage); const text = result[0].generated_text; console.log('Image caption:', text); @@ -28,5 +28,3 @@ router.post('/', jsonParser, async (req, res) => { return res.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 3082e0f54..380d8a72f 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -1,24 +1,24 @@ -const path = require('path'); -const fs = require('fs'); -const fsPromises = require('fs').promises; -const readline = require('readline'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const yaml = require('yaml'); -const _ = require('lodash'); -const mime = require('mime-types'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { promises as fsPromises } from 'node:fs'; +import * as readline from 'node:readline'; -const jimp = require('jimp'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import yaml from 'yaml'; +import _ from 'lodash'; +import mime from 'mime-types'; +import jimp from 'jimp'; -const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); -const { jsonParser, urlencodedParser } = require('../express-common'); -const { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } = require('../util'); -const { TavernCardValidator } = require('../validator/TavernCardValidator'); -const characterCardParser = require('../character-card-parser.js'); -const { readWorldInfoFile } = require('./worldinfo'); -const { invalidateThumbnail } = require('./thumbnails'); -const { importRisuSprites } = require('./sprites'); +import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js'; +import { jsonParser, urlencodedParser } from '../express-common.js'; +import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } from '../util.js'; +import { TavernCardValidator } from '../validator/TavernCardValidator.js'; +import * as characterCardParser from '../character-card-parser.js'; +import { readWorldInfoFile } from './worldinfo.js'; +import { invalidateThumbnail } from './thumbnails.js'; +import { importRisuSprites } from './sprites.js'; const defaultAvatarPath = './public/img/ai4.png'; // KV-store for parsed character data @@ -715,7 +715,7 @@ async function importFromPng(uploadPath, { request }, preservedFileName) { return ''; } -const router = express.Router(); +export const router = express.Router(); router.post('/create', urlencodedParser, async function (request, response) { try { @@ -1231,5 +1231,3 @@ router.post('/export', jsonParser, async function (request, response) { response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js index 0621e585b..cc23f9d3e 100644 --- a/src/endpoints/chats.js +++ b/src/endpoints/chats.js @@ -1,13 +1,14 @@ -const fs = require('fs'); -const path = require('path'); -const readline = require('readline'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const _ = require('lodash'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as readline from 'node:readline'; -const { jsonParser, urlencodedParser } = require('../express-common'); -const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import _ from 'lodash'; + +import { jsonParser, urlencodedParser } from '../express-common.js'; +import { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } from '../util.js'; /** * Saves a chat to the backups directory. @@ -188,7 +189,7 @@ function flattenChubChat(userName, characterName, lines) { return (lines ?? []).map(convert).join('\n'); } -const router = express.Router(); +export const router = express.Router(); router.post('/save', jsonParser, function (request, response) { try { @@ -514,5 +515,3 @@ router.post('/group/save', jsonParser, (request, response) => { getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData); return response.send({ ok: true }); }); - -module.exports = { router }; diff --git a/src/endpoints/classify.js b/src/endpoints/classify.js index 758b95247..7e0fd2393 100644 --- a/src/endpoints/classify.js +++ b/src/endpoints/classify.js @@ -1,9 +1,11 @@ -const express = require('express'); -const { jsonParser } = require('../express-common'); +import express from 'express'; + +import { getPipeline } from '../transformers.mjs'; +import { jsonParser } from '../express-common.js'; const TASK = 'text-classification'; -const router = express.Router(); +export const router = express.Router(); /** * @type {Map} Cache for classification results @@ -12,8 +14,7 @@ const cacheObject = new Map(); router.post('/labels', jsonParser, async (req, res) => { try { - const module = await import('../transformers.mjs'); - const pipe = await module.default.getPipeline(TASK); + const pipe = await getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); } catch (error) { @@ -35,8 +36,7 @@ router.post('/', jsonParser, async (req, res) => { if (cacheObject.has(text)) { return cacheObject.get(text); } else { - const module = await import('../transformers.mjs'); - const pipe = await module.default.getPipeline(TASK); + const pipe = await getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); cacheObject.set(text, result); @@ -54,5 +54,3 @@ router.post('/', jsonParser, async (req, res) => { return res.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 9938ab567..7523844db 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -1,16 +1,18 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const fetch = require('node-fetch').default; -const sanitize = require('sanitize-filename'); -const { getConfigValue, color } = require('../util'); -const { jsonParser } = require('../express-common'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import express from 'express'; +import fetch from 'node-fetch'; +import sanitize from 'sanitize-filename'; + +import { getConfigValue, color } from '../util.js'; +import { jsonParser } from '../express-common.js'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; const contentDirectory = path.join(process.cwd(), 'default/content'); const scaffoldDirectory = path.join(process.cwd(), 'default/scaffold'); const contentIndexPath = path.join(contentDirectory, 'index.json'); const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json'); -const characterCardParser = require('../character-card-parser.js'); +import * as characterCardParser from '../character-card-parser.js'; const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []); @@ -26,7 +28,7 @@ const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDo * @typedef {string} ContentType * @enum {string} */ -const CONTENT_TYPES = { +export const CONTENT_TYPES = { SETTINGS: 'settings', CHARACTER: 'character', SPRITES: 'sprites', @@ -51,7 +53,7 @@ const CONTENT_TYPES = { * @param {import('../users').UserDirectoryList} directories User directories * @returns {object[]} Array of default presets */ -function getDefaultPresets(directories) { +export function getDefaultPresets(directories) { try { const contentIndex = getContentIndex(); const presets = []; @@ -76,7 +78,7 @@ function getDefaultPresets(directories) { * @param {string} filename Name of the file to get * @returns {object | null} JSON object or null if the file doesn't exist */ -function getDefaultPresetFile(filename) { +export function getDefaultPresetFile(filename) { try { const contentPath = path.join(contentDirectory, filename); @@ -158,7 +160,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) { * @param {string[]} forceCategories List of categories to force check (even if content check is skipped) * @returns {Promise} */ -async function checkForNewContent(directoriesList, forceCategories = []) { +export async function checkForNewContent(directoriesList, forceCategories = []) { try { const contentCheckSkip = getConfigValue('skipContentCheck', false); if (contentCheckSkip && forceCategories?.length === 0) { @@ -224,7 +226,7 @@ function getContentIndex() { * @param {'json'|'string'|'raw'} format Format of content * @returns {string[]|Buffer[]} Array of content */ -function getContentOfType(type, format) { +export function getContentOfType(type, format) { const contentIndex = getContentIndex(); const indexItems = contentIndex.filter((item) => item.type === type && item.folder); const files = []; @@ -618,7 +620,7 @@ function isHostWhitelisted(host) { return WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES.includes(host); } -const router = express.Router(); +export const router = express.Router(); router.post('/importURL', jsonParser, async (request, response) => { if (!request.body.url) { @@ -753,12 +755,3 @@ router.post('/importUUID', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { - CONTENT_TYPES, - checkForNewContent, - getDefaultPresets, - getDefaultPresetFile, - getContentOfType, - router, -}; diff --git a/src/endpoints/extensions.js b/src/endpoints/extensions.js index 87c06da9c..1c58b4e8c 100644 --- a/src/endpoints/extensions.js +++ b/src/endpoints/extensions.js @@ -1,10 +1,12 @@ -const path = require('path'); -const fs = require('fs'); -const express = require('express'); -const { default: simpleGit } = require('simple-git'); -const sanitize = require('sanitize-filename'); -const { PUBLIC_DIRECTORIES } = require('../constants'); -const { jsonParser } = require('../express-common'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { default as simpleGit } from 'simple-git'; + +import { PUBLIC_DIRECTORIES } from '../constants.js'; +import { jsonParser } from '../express-common.js'; /** * This function extracts the extension information from the manifest file. @@ -47,7 +49,7 @@ async function checkIfRepoIsUpToDate(extensionPath) { }; } -const router = express.Router(); +export const router = express.Router(); /** * HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest, @@ -240,5 +242,3 @@ router.get('/discover', jsonParser, function (request, response) { return response.send(extensions); }); - -module.exports = { router }; diff --git a/src/endpoints/files.js b/src/endpoints/files.js index 000110758..e92a892d5 100644 --- a/src/endpoints/files.js +++ b/src/endpoints/files.js @@ -1,12 +1,15 @@ -const path = require('path'); -const fs = require('fs'); -const writeFileSyncAtomic = require('write-file-atomic').sync; -const express = require('express'); -const sanitize = require('sanitize-filename'); -const router = express.Router(); -const { validateAssetFileName } = require('./assets'); -const { jsonParser } = require('../express-common'); -const { clientRelativePath } = require('../util'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileSyncAtomic } from 'write-file-atomic'; + +import { validateAssetFileName } from './assets.js'; +import { jsonParser } from '../express-common.js'; +import { clientRelativePath } from '../util.js'; + +export const router = express.Router(); router.post('/sanitize-filename', jsonParser, async (request, response) => { try { @@ -97,5 +100,3 @@ router.post('/verify', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/google.js b/src/endpoints/google.js index 21aa6d82a..09f66f599 100644 --- a/src/endpoints/google.js +++ b/src/endpoints/google.js @@ -1,12 +1,13 @@ -const { readSecret, SECRET_KEYS } = require('./secrets'); -const fetch = require('node-fetch').default; -const express = require('express'); -const { jsonParser } = require('../express-common'); -const { GEMINI_SAFETY } = require('../constants'); +import fetch from 'node-fetch'; +import express from 'express'; + +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { jsonParser } from '../express-common.js'; +import { GEMINI_SAFETY } from '../constants.js'; const API_MAKERSUITE = 'https://generativelanguage.googleapis.com'; -const router = express.Router(); +export const router = express.Router(); router.post('/caption-image', jsonParser, async (request, response) => { try { @@ -67,5 +68,3 @@ router.post('/caption-image', jsonParser, async (request, response) => { response.status(500).send('Internal server error'); } }); - -module.exports = { router }; diff --git a/src/endpoints/groups.js b/src/endpoints/groups.js index 0a4aca561..0c1c55131 100644 --- a/src/endpoints/groups.js +++ b/src/endpoints/groups.js @@ -1,13 +1,14 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const { jsonParser } = require('../express-common'); -const { humanizedISO8601DateTime } = require('../util'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; -const router = express.Router(); +import { jsonParser } from '../express-common.js'; +import { humanizedISO8601DateTime } from '../util.js'; + +export const router = express.Router(); router.post('/all', jsonParser, (request, response) => { const groups = []; @@ -131,5 +132,3 @@ router.post('/delete', jsonParser, async (request, response) => { return response.send({ ok: true }); }); - -module.exports = { router }; diff --git a/src/endpoints/horde.js b/src/endpoints/horde.js index 4ca11a192..cd42bdb96 100644 --- a/src/endpoints/horde.js +++ b/src/endpoints/horde.js @@ -1,14 +1,14 @@ -const fetch = require('node-fetch').default; -const express = require('express'); -const { AIHorde, ModelGenerationInputStableSamplers, ModelInterrogationFormTypes, HordeAsyncRequestStates } = require('@zeldafan0225/ai_horde'); -const { getVersion, delay, Cache } = require('../util'); -const { readSecret, SECRET_KEYS } = require('./secrets'); -const { jsonParser } = require('../express-common'); +import fetch from 'node-fetch'; +import express from 'express'; +import { AIHorde, ModelGenerationInputStableSamplers, ModelInterrogationFormTypes, HordeAsyncRequestStates } from '@zeldafan0225/ai_horde'; +import { getVersion, delay, Cache } from '../util.js'; +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { jsonParser } from '../express-common.js'; const ANONYMOUS_KEY = '0000000000'; const HORDE_TEXT_MODEL_METADATA_URL = 'https://raw.githubusercontent.com/db0/AI-Horde-text-model-reference/main/db.json'; const cache = new Cache(60 * 1000); -const router = express.Router(); +export const router = express.Router(); /** * Returns the AIHorde client agent. @@ -403,5 +403,3 @@ router.post('/generate-image', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/images.js b/src/endpoints/images.js index 7290307ca..dd3be719e 100644 --- a/src/endpoints/images.js +++ b/src/endpoints/images.js @@ -1,10 +1,11 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const { jsonParser } = require('../express-common'); -const { clientRelativePath, removeFileExtension, getImages } = require('../util'); +import express from 'express'; +import sanitize from 'sanitize-filename'; + +import { jsonParser } from '../express-common.js'; +import { clientRelativePath, removeFileExtension, getImages } from '../util.js'; /** * Ensure the directory for the provided file path exists. @@ -21,7 +22,7 @@ function ensureDirectoryExistence(filePath) { fs.mkdirSync(dirname); } -const router = express.Router(); +export const router = express.Router(); /** * Endpoint to handle image uploads. @@ -89,5 +90,3 @@ router.post('/list/:folder', (request, response) => { return response.status(500).send({ error: 'Unable to retrieve files' }); } }); - -module.exports = { router }; diff --git a/src/endpoints/moving-ui.js b/src/endpoints/moving-ui.js index e3bba3e6c..0b82ea919 100644 --- a/src/endpoints/moving-ui.js +++ b/src/endpoints/moving-ui.js @@ -1,11 +1,11 @@ -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as path from 'node:path'; +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; -const { jsonParser } = require('../express-common'); +import { jsonParser } from '../express-common.js'; -const router = express.Router(); +export const router = express.Router(); router.post('/save', jsonParser, (request, response) => { if (!request.body || !request.body.name) { @@ -17,5 +17,3 @@ router.post('/save', jsonParser, (request, response) => { return response.sendStatus(200); }); - -module.exports = { router }; diff --git a/src/endpoints/novelai.js b/src/endpoints/novelai.js index e2c6fef19..b42b1de31 100644 --- a/src/endpoints/novelai.js +++ b/src/endpoints/novelai.js @@ -1,9 +1,11 @@ -const fetch = require('node-fetch').default; -const express = require('express'); -const util = require('util'); -const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer, forwardFetchResponse } = require('../util'); -const { jsonParser } = require('../express-common'); +import * as util from 'node:util'; + +import fetch from 'node-fetch'; +import express from 'express'; + +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { readAllChunks, extractFileFromZipBuffer, forwardFetchResponse } from '../util.js'; +import { jsonParser } from '../express-common.js'; const API_NOVELAI = 'https://api.novelai.net'; const TEXT_NOVELAI = 'https://text.novelai.net'; @@ -110,7 +112,7 @@ function getRepPenaltyWhitelist(model) { return null; } -const router = express.Router(); +export const router = express.Router(); router.post('/status', jsonParser, async function (req, res) { if (!req.body) return res.sendStatus(400); @@ -432,5 +434,3 @@ router.post('/generate-voice', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/openai.js b/src/endpoints/openai.js index e7760a8fb..d6e393681 100644 --- a/src/endpoints/openai.js +++ b/src/endpoints/openai.js @@ -1,14 +1,16 @@ -const { readSecret, SECRET_KEYS } = require('./secrets'); -const fetch = require('node-fetch').default; -const express = require('express'); -const FormData = require('form-data'); -const fs = require('fs'); -const { jsonParser, urlencodedParser } = require('../express-common'); -const { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } = require('../util'); -const { setAdditionalHeaders } = require('../additional-headers'); -const { OPENROUTER_HEADERS } = require('../constants'); +import * as fs from 'node:fs'; -const router = express.Router(); +import fetch from 'node-fetch'; +import FormData from 'form-data'; +import express from 'express'; + +import { jsonParser, urlencodedParser } from '../express-common.js'; +import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } from '../util.js'; +import { setAdditionalHeaders } from '../additional-headers.js'; +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { OPENROUTER_HEADERS } from '../constants.js'; + +export const router = express.Router(); router.post('/caption-image', jsonParser, async (request, response) => { try { @@ -341,5 +343,3 @@ custom.post('/generate-voice', jsonParser, async (request, response) => { }); router.use('/custom', custom); - -module.exports = { router }; diff --git a/src/endpoints/openrouter.js b/src/endpoints/openrouter.js index 6ac69a720..484c326de 100644 --- a/src/endpoints/openrouter.js +++ b/src/endpoints/openrouter.js @@ -1,7 +1,7 @@ -const express = require('express'); -const { jsonParser } = require('../express-common'); +import express from 'express'; +import { jsonParser } from '../express-common.js'; -const router = express.Router(); +export const router = express.Router(); const API_OPENROUTER = 'https://openrouter.ai/api/v1'; router.post('/models/multimodal', jsonParser, async (_req, res) => { @@ -28,5 +28,3 @@ router.post('/models/multimodal', jsonParser, async (_req, res) => { return res.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/presets.js b/src/endpoints/presets.js index bc3ed59b3..173b53cce 100644 --- a/src/endpoints/presets.js +++ b/src/endpoints/presets.js @@ -1,10 +1,12 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); -const { jsonParser } = require('../express-common'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { getDefaultPresetFile, getDefaultPresets } from './content-manager.js'; +import { jsonParser } from '../express-common.js'; /** * Gets the folder and extension for the preset settings based on the API source ID. @@ -34,7 +36,7 @@ function getPresetSettingsByAPI(apiId, directories) { } } -const router = express.Router(); +export const router = express.Router(); router.post('/save', jsonParser, function (request, response) { const name = sanitize(request.body.name); @@ -127,5 +129,3 @@ router.post('/delete-openai', jsonParser, function (request, response) { return response.send({ error: true }); }); - -module.exports = { router }; diff --git a/src/endpoints/quick-replies.js b/src/endpoints/quick-replies.js index ed53985f2..45df14ce7 100644 --- a/src/endpoints/quick-replies.js +++ b/src/endpoints/quick-replies.js @@ -1,12 +1,13 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const { jsonParser } = require('../express-common'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; -const router = express.Router(); +import { jsonParser } from '../express-common.js'; + +export const router = express.Router(); router.post('/save', jsonParser, (request, response) => { if (!request.body || !request.body.name) { @@ -31,5 +32,3 @@ router.post('/delete', jsonParser, (request, response) => { return response.sendStatus(200); }); - -module.exports = { router }; diff --git a/src/endpoints/search.js b/src/endpoints/search.js index 64993589e..e15c2b7a6 100644 --- a/src/endpoints/search.js +++ b/src/endpoints/search.js @@ -1,9 +1,11 @@ -const fetch = require('node-fetch').default; -const express = require('express'); -const { readSecret, SECRET_KEYS } = require('./secrets'); -const { jsonParser } = require('../express-common'); +import fetch from 'node-fetch'; +import express from 'express'; -const router = express.Router(); +import { decode } from 'html-entities'; +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { jsonParser } from '../express-common.js'; + +export const router = express.Router(); // Cosplay as Chrome const visitHeaders = { @@ -29,7 +31,6 @@ const visitHeaders = { * @returns {Promise} Transcript text */ async function extractTranscript(videoPageBody, lang) { - const he = require('he'); const RE_XML_TRANSCRIPT = /([^<]*)<\/text>/g; const splittedHTML = videoPageBody.split('"captions":'); @@ -84,7 +85,7 @@ async function extractTranscript(videoPageBody, lang) { lang: lang ?? captions.captionTracks[0].languageCode, })); // The text is double-encoded - const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' '); + const transcriptText = transcript.map((line) => decode(decode(line.text))).join(' '); return transcriptText; } @@ -263,5 +264,3 @@ router.post('/visit', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index ea946a5f9..99db701f6 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -1,12 +1,13 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const { getConfigValue } = require('../util'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { jsonParser } = require('../express-common'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const SECRETS_FILE = 'secrets.json'; -const SECRET_KEYS = { +import express from 'express'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import { getConfigValue } from '../util.js'; +import { jsonParser } from '../express-common.js'; + +export const SECRETS_FILE = 'secrets.json'; +export const SECRET_KEYS = { HORDE: 'api_key_horde', MANCER: 'api_key_mancer', VLLM: 'api_key_vllm', @@ -62,7 +63,7 @@ const EXPORTABLE_KEYS = [ * @param {string} key Secret key * @param {string} value Secret value */ -function writeSecret(directories, key, value) { +export function writeSecret(directories, key, value) { const filePath = path.join(directories.root, SECRETS_FILE); if (!fs.existsSync(filePath)) { @@ -82,7 +83,7 @@ function writeSecret(directories, key, value) { * @param {string} key Secret key * @returns */ -function deleteSecret(directories, key) { +export function deleteSecret(directories, key) { const filePath = path.join(directories.root, SECRETS_FILE); if (!fs.existsSync(filePath)) { @@ -101,7 +102,7 @@ function deleteSecret(directories, key) { * @param {string} key Secret key * @returns {string} Secret value */ -function readSecret(directories, key) { +export function readSecret(directories, key) { const filePath = path.join(directories.root, SECRETS_FILE); if (!fs.existsSync(filePath)) { @@ -118,7 +119,7 @@ function readSecret(directories, key) { * @param {import('../users').UserDirectoryList} directories User directories * @returns {object} Secret state */ -function readSecretState(directories) { +export function readSecretState(directories) { const filePath = path.join(directories.root, SECRETS_FILE); if (!fs.existsSync(filePath)) { @@ -141,7 +142,7 @@ function readSecretState(directories) { * @param {import('../users').UserDirectoryList} directories User directories * @returns {Record | undefined} Secrets */ -function getAllSecrets(directories) { +export function getAllSecrets(directories) { const filePath = path.join(directories.root, SECRETS_FILE); if (!fs.existsSync(filePath)) { @@ -154,7 +155,7 @@ function getAllSecrets(directories) { return secrets; } -const router = express.Router(); +export const router = express.Router(); router.post('/write', jsonParser, (request, response) => { const key = request.body.key; @@ -218,13 +219,3 @@ router.post('/find', jsonParser, (request, response) => { return response.sendStatus(500); } }); - -module.exports = { - writeSecret, - readSecret, - deleteSecret, - readSecretState, - getAllSecrets, - SECRET_KEYS, - router, -}; diff --git a/src/endpoints/settings.js b/src/endpoints/settings.js index 95d871d1c..8e9e052db 100644 --- a/src/endpoints/settings.js +++ b/src/endpoints/settings.js @@ -1,12 +1,14 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const _ = require('lodash'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { SETTINGS_FILE } = require('../constants'); -const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util'); -const { jsonParser } = require('../express-common'); -const { getAllUserHandles, getUserDirectories } = require('../users'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import express from 'express'; +import _ from 'lodash'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { SETTINGS_FILE } from '../constants.js'; +import { getConfigValue, generateTimestamp, removeOldBackups } from '../util.js'; +import { jsonParser } from '../express-common.js'; +import { getAllUserHandles, getUserDirectories } from '../users.js'; const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true); const ENABLE_EXTENSIONS_AUTO_UPDATE = getConfigValue('enableExtensionsAutoUpdate', true); @@ -190,7 +192,7 @@ function getLatestBackup(handle) { return path.join(userDirectories.backups, latestBackup); } -const router = express.Router(); +export const router = express.Router(); router.post('/save', jsonParser, function (request, response) { try { @@ -357,8 +359,6 @@ router.post('/restore-snapshot', jsonParser, async (request, response) => { /** * Initializes the settings endpoint */ -async function init() { +export async function init() { await backupSettings(); } - -module.exports = { router, init }; diff --git a/src/endpoints/speech.js b/src/endpoints/speech.js index 27769f72f..9540b0411 100644 --- a/src/endpoints/speech.js +++ b/src/endpoints/speech.js @@ -1,7 +1,9 @@ -const express = require('express'); -const { jsonParser } = require('../express-common'); +import express from 'express'; +import wavefile from 'wavefile'; +import { jsonParser } from '../express-common.js'; +import { getPipeline } from '../transformers.mjs'; -const router = express.Router(); +export const router = express.Router(); /** * Gets the audio data from a base64-encoded audio file. @@ -9,7 +11,6 @@ const router = express.Router(); * @returns {Float64Array} Audio data */ function getWaveFile(audio) { - const wavefile = require('wavefile'); const wav = new wavefile.WaveFile(); wav.fromDataURI(audio); wav.toBitDepth('32f'); @@ -36,8 +37,7 @@ router.post('/recognize', jsonParser, async (req, res) => { try { const TASK = 'automatic-speech-recognition'; const { model, audio, lang } = req.body; - const module = await import('../transformers.mjs'); - const pipe = await module.default.getPipeline(TASK, model); + const pipe = await getPipeline(TASK, model); const wav = getWaveFile(audio); const start = performance.now(); const result = await pipe(wav, { language: lang || null, task: 'transcribe' }); @@ -54,11 +54,9 @@ router.post('/recognize', jsonParser, async (req, res) => { router.post('/synthesize', jsonParser, async (req, res) => { try { - const wavefile = require('wavefile'); const TASK = 'text-to-speech'; const { text, model, speaker } = req.body; - const module = await import('../transformers.mjs'); - const pipe = await module.default.getPipeline(TASK, model); + const pipe = await getPipeline(TASK, model); const speaker_embeddings = speaker ? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer) : null; @@ -78,5 +76,3 @@ router.post('/synthesize', jsonParser, async (req, res) => { return res.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/sprites.js b/src/endpoints/sprites.js index 43bfdb145..8d2e34020 100644 --- a/src/endpoints/sprites.js +++ b/src/endpoints/sprites.js @@ -1,12 +1,13 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const mime = require('mime-types'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { getImageBuffers } = require('../util'); -const { jsonParser, urlencodedParser } = require('../express-common'); +import express from 'express'; +import mime from 'mime-types'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { getImageBuffers } from '../util.js'; +import { jsonParser, urlencodedParser } from '../express-common.js'; /** * Gets the path to the sprites folder for the provided character name @@ -45,7 +46,7 @@ function getSpritesPath(directories, name, isSubfolder) { * @param {object} data RisuAI character data * @returns {void} */ -function importRisuSprites(directories, data) { +export function importRisuSprites(directories, data) { try { const name = data?.data?.name; const risuData = data?.data?.extensions?.risuai; @@ -106,7 +107,7 @@ function importRisuSprites(directories, data) { } } -const router = express.Router(); +export const router = express.Router(); router.get('/get', jsonParser, function (request, response) { const name = String(request.query.name); @@ -259,8 +260,3 @@ router.post('/upload', urlencodedParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { - router, - importRisuSprites, -}; diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index b6348b8b2..4be405c58 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,13 +1,15 @@ -const express = require('express'); -const fetch = require('node-fetch').default; -const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay, getHexString } = require('../util.js'); -const fs = require('fs'); -const path = require('path'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { jsonParser } = require('../express-common'); -const { readSecret, SECRET_KEYS } = require('./secrets.js'); -const FormData = require('form-data'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import express from 'express'; +import fetch from 'node-fetch'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import FormData from 'form-data'; + +import { getBasicAuthHeader, delay, getHexString } from '../util.js'; +import { jsonParser } from '../express-common.js'; +import { readSecret, SECRET_KEYS } from './secrets.js'; /** * Gets the comfy workflows. @@ -21,7 +23,7 @@ function getComfyWorkflows(directories) { .sort(Intl.Collator().compare); } -const router = express.Router(); +export const router = express.Router(); router.post('/ping', jsonParser, async (request, response) => { try { @@ -993,5 +995,3 @@ router.use('/pollinations', pollinations); router.use('/stability', stability); router.use('/blockentropy', blockentropy); router.use('/huggingface', huggingface); - -module.exports = { router }; diff --git a/src/endpoints/stats.js b/src/endpoints/stats.js index bd9b6d4bf..a7478f7c2 100644 --- a/src/endpoints/stats.js +++ b/src/endpoints/stats.js @@ -1,14 +1,15 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const writeFileAtomic = require('write-file-atomic'); -const crypto = require('crypto'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as crypto from 'node:crypto'; + +import express from 'express'; +import writeFileAtomic from 'write-file-atomic'; const readFile = fs.promises.readFile; const readdir = fs.promises.readdir; -const { jsonParser } = require('../express-common'); -const { getAllUserHandles, getUserDirectories } = require('../users'); +import { jsonParser } from '../express-common.js'; +import { getAllUserHandles, getUserDirectories } from '../users.js'; const STATS_FILE = 'stats.json'; @@ -146,7 +147,7 @@ async function collectAndCreateStats(chatsPath, charactersPath) { * @param {string} chatsPath Path to the directory containing the chat files. * @param {string} charactersPath Path to the directory containing the character files. */ -async function recreateStats(handle, chatsPath, charactersPath) { +export async function recreateStats(handle, chatsPath, charactersPath) { console.log('Collecting and creating stats for user:', handle); const stats = await collectAndCreateStats(chatsPath, charactersPath); STATS.set(handle, stats); @@ -157,7 +158,7 @@ async function recreateStats(handle, chatsPath, charactersPath) { * Loads the stats file into memory. If the file doesn't exist or is invalid, * initializes stats by collecting and creating them for each character. */ -async function init() { +export async function init() { try { const userHandles = await getAllUserHandles(); for (const handle of userHandles) { @@ -209,7 +210,7 @@ async function saveStatsToFile() { * Attempts to save charStats to a file and then terminates the process. * If an error occurs during the file write, it logs the error before exiting. */ -async function onExit() { +export async function onExit() { try { await saveStatsToFile(); } catch (err) { @@ -434,7 +435,7 @@ function calculateTotalGenTimeAndWordCount( }; } -const router = express.Router(); +export const router = express.Router(); /** * Handle a POST request to get the stats object @@ -465,10 +466,3 @@ router.post('/update', jsonParser, function (request, response) { setCharStats(request.user.profile.handle, request.body); return response.sendStatus(200); }); - -module.exports = { - router, - recreateStats, - init, - onExit, -}; diff --git a/src/endpoints/themes.js b/src/endpoints/themes.js index 72f874b80..9edca31be 100644 --- a/src/endpoints/themes.js +++ b/src/endpoints/themes.js @@ -1,11 +1,13 @@ -const express = require('express'); -const path = require('path'); -const fs = require('fs'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { jsonParser } = require('../express-common'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; -const router = express.Router(); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { jsonParser } from '../express-common.js'; + +export const router = express.Router(); router.post('/save', jsonParser, (request, response) => { if (!request.body || !request.body.name) { @@ -36,5 +38,3 @@ router.post('/delete', jsonParser, function (request, response) { return response.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/thumbnails.js b/src/endpoints/thumbnails.js index 6bc402357..d095c135c 100644 --- a/src/endpoints/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -1,14 +1,16 @@ -const fs = require('fs'); -const fsPromises = require('fs').promises; -const path = require('path'); -const mime = require('mime-types'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const jimp = require('jimp'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const { getAllUserHandles, getUserDirectories } = require('../users'); -const { getConfigValue } = require('../util'); -const { jsonParser } = require('../express-common'); +import * as fs from 'node:fs'; +import * as fsPromises from 'node:fs/promises'; +import * as path from 'node:path'; + +import mime from 'mime-types'; +import express from 'express'; +import sanitize from 'sanitize-filename'; +import jimp from 'jimp'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { getAllUserHandles, getUserDirectories } from '../users.js'; +import { getConfigValue } from '../util.js'; +import { jsonParser } from '../express-common.js'; const thumbnailsDisabled = getConfigValue('disableThumbnails', false); const quality = getConfigValue('thumbnailsQuality', 95); @@ -62,7 +64,7 @@ function getOriginalFolder(directories, type) { * @param {'bg' | 'avatar'} type Type of the thumbnail * @param {string} file Name of the file */ -function invalidateThumbnail(directories, type, file) { +export function invalidateThumbnail(directories, type, file) { const folder = getThumbnailFolder(directories, type); if (folder === undefined) throw new Error('Invalid thumbnail type'); @@ -141,7 +143,7 @@ async function generateThumbnail(directories, type, file) { * Ensures that the thumbnail cache for backgrounds is valid. * @returns {Promise} Promise that resolves when the cache is validated */ -async function ensureThumbnailCache() { +export async function ensureThumbnailCache() { const userHandles = await getAllUserHandles(); for (const handle of userHandles) { const directories = getUserDirectories(handle); @@ -166,7 +168,7 @@ async function ensureThumbnailCache() { } } -const router = express.Router(); +export const router = express.Router(); // Important: This route must be mounted as '/thumbnail'. It is used in the client code and saved to chat files. router.get('/', jsonParser, async function (request, response) { @@ -227,9 +229,3 @@ router.get('/', jsonParser, async function (request, response) { return response.sendStatus(500); } }); - -module.exports = { - invalidateThumbnail, - ensureThumbnailCache, - router, -}; diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index 0be60d3a4..57ce530ec 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -1,15 +1,18 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); -const tiktoken = require('tiktoken'); -const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('../prompt-converters'); -const { TEXTGEN_TYPES } = require('../constants'); -const { jsonParser } = require('../express-common'); -const { setAdditionalHeaders } = require('../additional-headers'); -const { getConfigValue, isValidUrl } = require('../util'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import express from 'express'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { Tokenizer } from '@agnai/web-tokenizers'; +import { SentencePieceProcessor } from '@agnai/sentencepiece-js'; +import tiktoken from 'tiktoken'; + +import { convertClaudePrompt } from '../prompt-converters.js'; +import { TEXTGEN_TYPES } from '../constants.js'; +import { jsonParser } from '../express-common.js'; +import { setAdditionalHeaders } from '../additional-headers.js'; +import { getConfigValue, isValidUrl } from '../util.js'; /** * @typedef { (req: import('express').Request, res: import('express').Response) => Promise } TokenizationHandler @@ -23,7 +26,7 @@ const tokenizersCache = {}; /** * @type {string[]} */ -const TEXT_COMPLETION_MODELS = [ +export const TEXT_COMPLETION_MODELS = [ 'gpt-3.5-turbo-instruct', 'gpt-3.5-turbo-instruct-0914', 'text-davinci-003', @@ -223,7 +226,7 @@ const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyT const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json'); const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json'); -const sentencepieceTokenizers = [ +export const sentencepieceTokenizers = [ 'llama', 'nerdstash', 'nerdstash_v2', @@ -238,7 +241,7 @@ const sentencepieceTokenizers = [ * @param {string} model Sentencepiece model name * @returns {SentencePieceTokenizer|null} Sentencepiece tokenizer */ -function getSentencepiceTokenizer(model) { +export function getSentencepiceTokenizer(model) { if (model.includes('llama')) { return spp_llama; } @@ -350,7 +353,7 @@ function getWebTokenizersChunks(tokenizer, ids) { * @param {string} requestModel Models to use for tokenization * @returns {string} Tokenizer model to use */ -function getTokenizerModel(requestModel) { +export function getTokenizerModel(requestModel) { if (requestModel.includes('o1-preview') || requestModel.includes('o1-mini')) { return 'gpt-4o'; } @@ -427,7 +430,7 @@ function getTokenizerModel(requestModel) { return 'gpt-3.5-turbo'; } -function getTiktokenTokenizer(model) { +export function getTiktokenTokenizer(model) { if (tokenizersCache[model]) { return tokenizersCache[model]; } @@ -444,7 +447,7 @@ function getTiktokenTokenizer(model) { * @param {object[]} messages Array of messages * @returns {number} Number of tokens */ -function countWebTokenizerTokens(tokenizer, messages) { +export function countWebTokenizerTokens(tokenizer, messages) { // Should be fine if we use the old conversion method instead of the messages API one i think? const convertedPrompt = convertClaudePrompt(messages, false, '', false, false, '', false); @@ -636,7 +639,7 @@ function createWebTokenizerDecodingHandler(tokenizer) { }; } -const router = express.Router(); +export const router = express.Router(); router.post('/llama/encode', jsonParser, createSentencepieceEncodingHandler(spp_llama)); router.post('/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd)); @@ -1002,13 +1005,3 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re return response.send({ error: true }); } }); - -module.exports = { - TEXT_COMPLETION_MODELS, - getTokenizerModel, - getTiktokenTokenizer, - countWebTokenizerTokens, - getSentencepiceTokenizer, - sentencepieceTokenizers, - router, -}; diff --git a/src/endpoints/translate.js b/src/endpoints/translate.js index 20759a271..0ab66527b 100644 --- a/src/endpoints/translate.js +++ b/src/endpoints/translate.js @@ -1,10 +1,16 @@ -const fetch = require('node-fetch').default; -const https = require('https'); -const express = require('express'); -const iconv = require('iconv-lite'); -const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue, uuidv4 } = require('../util'); -const { jsonParser } = require('../express-common'); +import * as https from 'node:https'; +import { createRequire } from 'node:module'; +import fetch from 'node-fetch'; +import express from 'express'; +import iconv from 'iconv-lite'; +import bingTranslateApi from 'bing-translate-api'; + +const require = createRequire(import.meta.url); +const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser'); + +import { readSecret, SECRET_KEYS } from './secrets.js'; +import { getConfigValue, uuidv4 } from '../util.js'; +import { jsonParser } from '../express-common.js'; const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; @@ -23,7 +29,7 @@ function decodeBuffer(buffer) { } } -const router = express.Router(); +export const router = express.Router(); router.post('/libre', jsonParser, async (request, response) => { const key = readSecret(request.user.directories, SECRET_KEYS.LIBRE); @@ -82,7 +88,6 @@ router.post('/libre', jsonParser, async (request, response) => { router.post('/google', jsonParser, async (request, response) => { try { - const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser'); const text = request.body.text; const lang = request.body.lang; @@ -386,7 +391,6 @@ router.post('/deeplx', jsonParser, async (request, response) => { }); router.post('/bing', jsonParser, async (request, response) => { - const bingTranslateApi = require('bing-translate-api'); const text = request.body.text; let lang = request.body.lang; @@ -408,5 +412,3 @@ router.post('/bing', jsonParser, async (request, response) => { return response.sendStatus(500); }); }); - -module.exports = { router }; diff --git a/src/endpoints/users-admin.js b/src/endpoints/users-admin.js index a67233c46..1e75336ba 100644 --- a/src/endpoints/users-admin.js +++ b/src/endpoints/users-admin.js @@ -1,10 +1,11 @@ -const fsPromises = require('fs').promises; -const storage = require('node-persist'); -const express = require('express'); -const lodash = require('lodash'); -const { jsonParser } = require('../express-common'); -const { checkForNewContent } = require('./content-manager'); -const { +import { promises as fsPromises } from 'node:fs'; + +import storage from 'node-persist'; +import express from 'express'; +import lodash from 'lodash'; +import { jsonParser } from '../express-common.js'; +import { checkForNewContent } from './content-manager.js'; +import { KEY_PREFIX, toKey, requireAdminMiddleware, @@ -14,10 +15,10 @@ const { getPasswordHash, getUserDirectories, ensurePublicDirectoriesExist, -} = require('../users'); -const { DEFAULT_USER } = require('../constants'); +} from '../users.js'; +import { DEFAULT_USER } from '../constants.js'; -const router = express.Router(); +export const router = express.Router(); router.post('/get', requireAdminMiddleware, jsonParser, async (_request, response) => { try { @@ -249,7 +250,3 @@ router.post('/slugify', requireAdminMiddleware, jsonParser, async (request, resp return response.sendStatus(500); } }); - -module.exports = { - router, -}; diff --git a/src/endpoints/users-private.js b/src/endpoints/users-private.js index 40b0cbe03..c87a1a2b2 100644 --- a/src/endpoints/users-private.js +++ b/src/endpoints/users-private.js @@ -1,18 +1,20 @@ -const path = require('path'); -const fsPromises = require('fs').promises; -const storage = require('node-persist'); -const express = require('express'); -const crypto = require('crypto'); -const { jsonParser } = require('../express-common'); -const { getUserAvatar, toKey, getPasswordHash, getPasswordSalt, createBackupArchive, ensurePublicDirectoriesExist, toAvatarKey } = require('../users'); -const { SETTINGS_FILE } = require('../constants'); -const contentManager = require('./content-manager'); -const { color, Cache } = require('../util'); -const { checkForNewContent } = require('./content-manager'); +import * as path from 'node:path'; +import { promises as fsPromises } from 'node:fs'; +import * as crypto from 'node:crypto'; + +import storage from 'node-persist'; +import express from 'express'; + +import { jsonParser } from '../express-common.js'; +import { getUserAvatar, toKey, getPasswordHash, getPasswordSalt, createBackupArchive, ensurePublicDirectoriesExist, toAvatarKey } from '../users.js'; +import { SETTINGS_FILE } from '../constants.js'; +import * as contentManager from './content-manager.js'; +import { color, Cache } from '../util.js'; +import { checkForNewContent } from './content-manager.js'; const RESET_CACHE = new Cache(5 * 60 * 1000); -const router = express.Router(); +export const router = express.Router(); router.post('/logout', async (request, response) => { try { @@ -251,7 +253,3 @@ router.post('/reset-step2', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { - router, -}; diff --git a/src/endpoints/users-public.js b/src/endpoints/users-public.js index 9eb765ca3..40dc479ad 100644 --- a/src/endpoints/users-public.js +++ b/src/endpoints/users-public.js @@ -1,15 +1,16 @@ -const crypto = require('crypto'); -const storage = require('node-persist'); -const express = require('express'); -const { RateLimiterMemory, RateLimiterRes } = require('rate-limiter-flexible'); -const { jsonParser, getIpFromRequest } = require('../express-common'); -const { color, Cache, getConfigValue } = require('../util'); -const { KEY_PREFIX, getUserAvatar, toKey, getPasswordHash, getPasswordSalt } = require('../users'); +import * as crypto from 'node:crypto'; + +import storage from 'node-persist'; +import express from 'express'; +import { RateLimiterMemory, RateLimiterRes } from 'rate-limiter-flexible'; +import { jsonParser, getIpFromRequest } from '../express-common.js'; +import { color, Cache, getConfigValue } from '../util.js'; +import { KEY_PREFIX, getUserAvatar, toKey, getPasswordHash, getPasswordSalt } from '../users.js'; const DISCREET_LOGIN = getConfigValue('enableDiscreetLogin', false); const MFA_CACHE = new Cache(5 * 60 * 1000); -const router = express.Router(); +export const router = express.Router(); const loginLimiter = new RateLimiterMemory({ points: 5, duration: 60, @@ -193,7 +194,3 @@ router.post('/recover-step2', jsonParser, async (request, response) => { return response.sendStatus(500); } }); - -module.exports = { - router, -}; diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index 32e25ccf5..619244256 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -1,10 +1,22 @@ -const vectra = require('vectra'); -const path = require('path'); -const fs = require('fs'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const { jsonParser } = require('../express-common'); -const { getConfigValue } = require('../util'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; + +import vectra from 'vectra'; +import express from 'express'; +import sanitize from 'sanitize-filename'; + +import { jsonParser } from '../express-common.js'; +import { getConfigValue } from '../util.js'; + +import { getNomicAIBatchVector, getNomicAIVector } from '../vectors/nomicai-vectors.js'; +import { getOpenAIVector, getOpenAIBatchVector } from '../vectors/openai-vectors.js'; +import { getTransformersVector, getTransformersBatchVector } from '../vectors/embedding.js'; +import { getExtrasVector, getExtrasBatchVector } from '../vectors/extras-vectors.js'; +import { getMakerSuiteVector, getMakerSuiteBatchVector } from '../vectors/makersuite-vectors.js'; +import { getCohereVector, getCohereBatchVector } from '../vectors/cohere-vectors.js'; +import { getLlamaCppVector, getLlamaCppBatchVector } from '../vectors/llamacpp-vectors.js'; +import { getVllmVector, getVllmBatchVector } from '../vectors/vllm-vectors.js'; +import { getOllamaVector, getOllamaBatchVector } from '../vectors/ollama-vectors.js'; // Don't forget to add new sources to the SOURCES array const SOURCES = [ @@ -33,25 +45,25 @@ const SOURCES = [ async function getVector(source, sourceSettings, text, isQuery, directories) { switch (source) { case 'nomicai': - return require('../vectors/nomicai-vectors').getNomicAIVector(text, source, directories); + return getNomicAIVector(text, source, directories); case 'togetherai': case 'mistral': case 'openai': - return require('../vectors/openai-vectors').getOpenAIVector(text, source, directories, sourceSettings.model); + return getOpenAIVector(text, source, directories, sourceSettings.model); case 'transformers': - return require('../vectors/embedding').getTransformersVector(text); + return getTransformersVector(text); case 'extras': - return require('../vectors/extras-vectors').getExtrasVector(text, sourceSettings.extrasUrl, sourceSettings.extrasKey); + return getExtrasVector(text, sourceSettings.extrasUrl, sourceSettings.extrasKey); case 'palm': - return require('../vectors/makersuite-vectors').getMakerSuiteVector(text, directories); + return getMakerSuiteVector(text, directories); case 'cohere': - return require('../vectors/cohere-vectors').getCohereVector(text, isQuery, directories, sourceSettings.model); + return getCohereVector(text, isQuery, directories, sourceSettings.model); case 'llamacpp': - return require('../vectors/llamacpp-vectors').getLlamaCppVector(text, sourceSettings.apiUrl, directories); + return getLlamaCppVector(text, sourceSettings.apiUrl, directories); case 'vllm': - return require('../vectors/vllm-vectors').getVllmVector(text, sourceSettings.apiUrl, sourceSettings.model, directories); + return getVllmVector(text, sourceSettings.apiUrl, sourceSettings.model, directories); case 'ollama': - return require('../vectors/ollama-vectors').getOllamaVector(text, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories); + return getOllamaVector(text, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories); } throw new Error(`Unknown vector source ${source}`); @@ -74,33 +86,33 @@ async function getBatchVector(source, sourceSettings, texts, isQuery, directorie for (let batch of batches) { switch (source) { case 'nomicai': - results.push(...await require('../vectors/nomicai-vectors').getNomicAIBatchVector(batch, source, directories)); + results.push(...await getNomicAIBatchVector(batch, source, directories)); break; case 'togetherai': case 'mistral': case 'openai': - results.push(...await require('../vectors/openai-vectors').getOpenAIBatchVector(batch, source, directories, sourceSettings.model)); + results.push(...await getOpenAIBatchVector(batch, source, directories, sourceSettings.model)); break; case 'transformers': - results.push(...await require('../vectors/embedding').getTransformersBatchVector(batch)); + results.push(...await getTransformersBatchVector(batch)); break; case 'extras': - results.push(...await require('../vectors/extras-vectors').getExtrasBatchVector(batch, sourceSettings.extrasUrl, sourceSettings.extrasKey)); + results.push(...await getExtrasBatchVector(batch, sourceSettings.extrasUrl, sourceSettings.extrasKey)); break; case 'palm': - results.push(...await require('../vectors/makersuite-vectors').getMakerSuiteBatchVector(batch, directories)); + results.push(...await getMakerSuiteBatchVector(batch, directories)); break; case 'cohere': - results.push(...await require('../vectors/cohere-vectors').getCohereBatchVector(batch, isQuery, directories, sourceSettings.model)); + results.push(...await getCohereBatchVector(batch, isQuery, directories, sourceSettings.model)); break; case 'llamacpp': - results.push(...await require('../vectors/llamacpp-vectors').getLlamaCppBatchVector(batch, sourceSettings.apiUrl, directories)); + results.push(...await getLlamaCppBatchVector(batch, sourceSettings.apiUrl, directories)); break; case 'vllm': - results.push(...await require('../vectors/vllm-vectors').getVllmBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, directories)); + results.push(...await getVllmBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, directories)); break; case 'ollama': - results.push(...await require('../vectors/ollama-vectors').getOllamaBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories)); + results.push(...await getOllamaBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories)); break; default: throw new Error(`Unknown vector source ${source}`); @@ -359,7 +371,7 @@ async function regenerateCorruptedIndexErrorHandler(req, res, error) { return res.sendStatus(500); } -const router = express.Router(); +export const router = express.Router(); router.post('/query', jsonParser, async (req, res) => { try { @@ -495,5 +507,3 @@ router.post('/purge', jsonParser, async (req, res) => { return res.sendStatus(500); } }); - -module.exports = { router }; diff --git a/src/endpoints/worldinfo.js b/src/endpoints/worldinfo.js index 4f125bcc4..37c05d3d8 100644 --- a/src/endpoints/worldinfo.js +++ b/src/endpoints/worldinfo.js @@ -1,10 +1,11 @@ -const fs = require('fs'); -const path = require('path'); -const express = require('express'); -const sanitize = require('sanitize-filename'); -const writeFileAtomicSync = require('write-file-atomic').sync; +import * as fs from 'node:fs'; +import * as path from 'node:path'; -const { jsonParser, urlencodedParser } = require('../express-common'); +import express from 'express'; +import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; + +import { jsonParser, urlencodedParser } from '../express-common.js'; /** * Reads a World Info file and returns its contents @@ -13,7 +14,7 @@ const { jsonParser, urlencodedParser } = require('../express-common'); * @param {boolean} allowDummy If true, returns an empty object if the file doesn't exist * @returns {object} World Info file contents */ -function readWorldInfoFile(directories, worldInfoName, allowDummy) { +export function readWorldInfoFile(directories, worldInfoName, allowDummy) { const dummyObject = allowDummy ? { entries: {} } : null; if (!worldInfoName) { @@ -33,7 +34,7 @@ function readWorldInfoFile(directories, worldInfoName, allowDummy) { return worldInfo; } -const router = express.Router(); +export const router = express.Router(); router.post('/get', jsonParser, (request, response) => { if (!request.body?.name) { @@ -122,5 +123,3 @@ router.post('/edit', jsonParser, (request, response) => { return response.send({ ok: true }); }); - -module.exports = { router, readWorldInfoFile }; diff --git a/src/express-common.js b/src/express-common.js index e1c862d53..630d62c59 100644 --- a/src/express-common.js +++ b/src/express-common.js @@ -1,28 +1,27 @@ -const express = require('express'); -const ipaddr = require('ipaddr.js'); +import express from 'express'; +import ipaddr from 'ipaddr.js'; // Instantiate parser middleware here with application-level size limits -const jsonParser = express.json({ limit: '200mb' }); -const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' }); +export const jsonParser = express.json({ limit: '200mb' }); +export const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' }); /** * Gets the IP address of the client from the request object. - * @param {import('express'.Request)} req Request object + * @param {import('express').Request} req Request object * @returns {string} IP address of the client */ -function getIpFromRequest(req) { - let clientIp = req.connection.remoteAddress; +export function getIpFromRequest(req) { + let clientIp = req.socket.remoteAddress; + if (!clientIp) { + return 'unknown'; + } let ip = ipaddr.parse(clientIp); // Check if the IP address is IPv4-mapped IPv6 address if (ip.kind() === 'ipv6' && ip instanceof ipaddr.IPv6 && ip.isIPv4MappedAddress()) { const ipv4 = ip.toIPv4Address().toString(); clientIp = ipv4; } else { - clientIp = ip; - clientIp = clientIp.toString(); + clientIp = ip.toString(); } return clientIp; } - - -module.exports = { jsonParser, urlencodedParser, getIpFromRequest }; diff --git a/src/middleware/basicAuth.js b/src/middleware/basicAuth.js index 07408e7bd..0ab7af343 100644 --- a/src/middleware/basicAuth.js +++ b/src/middleware/basicAuth.js @@ -2,9 +2,9 @@ * When applied, this middleware will ensure the request contains the required header for basic authentication and only * allow access to the endpoint after successful authentication. */ -const { getAllUserHandles, toKey, getPasswordHash } = require('../users.js'); -const { getConfig, getConfigValue } = require('../util.js'); -const storage = require('node-persist'); +import storage from 'node-persist'; +import { getAllUserHandles, toKey, getPasswordHash } from '../users.js'; +import { getConfig, getConfigValue } from '../util.js'; const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false); const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false); @@ -49,4 +49,4 @@ const basicAuthMiddleware = async function (request, response, callback) { return unauthorizedResponse(response); }; -module.exports = basicAuthMiddleware; +export default basicAuthMiddleware; diff --git a/src/middleware/multerMonkeyPatch.js b/src/middleware/multerMonkeyPatch.js index afe5d5966..ed359a704 100644 --- a/src/middleware/multerMonkeyPatch.js +++ b/src/middleware/multerMonkeyPatch.js @@ -14,7 +14,7 @@ function decodeFileName(str) { * @param {import('express').Response} _res Response * @param {import('express').NextFunction} next Next middleware */ -function multerMonkeyPatch(req, _res, next) { +export default function multerMonkeyPatch(req, _res, next) { try { if (req.file) { req.file.originalname = decodeFileName(req.file.originalname); @@ -26,5 +26,3 @@ function multerMonkeyPatch(req, _res, next) { next(); } } - -module.exports = multerMonkeyPatch; diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index 24c1af8e5..eda161bc9 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -1,9 +1,9 @@ -const path = require('path'); -const fs = require('fs'); -const ipMatching = require('ip-matching'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import ipMatching from 'ip-matching'; -const { getIpFromRequest } = require('../express-common'); -const { color, getConfigValue } = require('../util'); +import { getIpFromRequest } from '../express-common.js'; +import { color, getConfigValue } from '../util.js'; const whitelistPath = path.join(process.cwd(), './whitelist.txt'); const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false); @@ -50,7 +50,7 @@ function getForwardedIp(req) { * @param {boolean} listen If listen mode is enabled via config or command line * @returns {import('express').RequestHandler} The middleware function */ -function whitelistMiddleware(whitelistMode, listen) { +export default function whitelistMiddleware(whitelistMode, listen) { return function (req, res, next) { const clientIp = getIpFromRequest(req); const forwardedIp = getForwardedIp(req); @@ -82,5 +82,3 @@ function whitelistMiddleware(whitelistMode, listen) { next(); }; } - -module.exports = whitelistMiddleware; diff --git a/src/plugin-loader.js b/src/plugin-loader.js index cf9267e25..c5c7e2995 100644 --- a/src/plugin-loader.js +++ b/src/plugin-loader.js @@ -1,8 +1,9 @@ -const fs = require('fs'); -const path = require('path'); -const url = require('url'); -const express = require('express'); -const { getConfigValue } = require('./util'); +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as url from 'node:url'; + +import express from 'express'; +import { getConfigValue } from './util.js'; const enableServerPlugins = getConfigValue('enableServerPlugins', false); /** @@ -32,7 +33,7 @@ const isESModule = (file) => path.extname(file) === '.mjs'; * @returns {Promise} Promise that resolves when all plugins are loaded. Resolves to a "cleanup" function to * be called before the server shuts down. */ -async function loadPlugins(app, pluginsPath) { +export async function loadPlugins(app, pluginsPath) { const exitHooks = []; const emptyFn = () => {}; @@ -217,7 +218,3 @@ async function initPlugin(app, plugin, exitHooks) { return true; } - -module.exports = { - loadPlugins, -}; diff --git a/src/polyfill.js b/src/polyfill.js index 2cc9d64e3..390b4b45a 100644 --- a/src/polyfill.js +++ b/src/polyfill.js @@ -7,4 +7,4 @@ if (!Array.prototype.findLastIndex) { }; } -module.exports = {}; +export default () => {}; diff --git a/src/prompt-converters.js b/src/prompt-converters.js index 47cd9e368..bfc08ad68 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -1,6 +1,8 @@ -require('./polyfill.js'); -const { getConfigValue } = require('./util.js'); -const crypto = require('crypto'); +import * as crypto from 'node:crypto'; +import polyfill from './polyfill.js'; +import { getConfigValue } from './util.js'; + +polyfill(); const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.'); @@ -17,7 +19,7 @@ const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get start * @returns {string} Prompt for Claude * @copyright Prompt Conversion script taken from RisuAI by kwaroran (GPLv3). */ -function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSysPromptSupport, useSystemPrompt, addSysHumanMsg, excludePrefixes) { +export function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSysPromptSupport, useSystemPrompt, addSysHumanMsg, excludePrefixes) { //Prepare messages for claude. //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception). @@ -96,7 +98,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, * @param {string} charName Character name * @param {string} userName User name */ -function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') { +export function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') { let systemPrompt = []; if (useSysPrompt) { // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array. @@ -279,7 +281,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, * @param {string} userName User name * @returns {{chatHistory: object[]}} Prompt for Cohere */ -function convertCohereMessages(messages, charName = '', userName = '') { +export function convertCohereMessages(messages, charName = '', userName = '') { if (messages.length === 0) { messages.unshift({ role: 'user', @@ -333,7 +335,7 @@ function convertCohereMessages(messages, charName = '', userName = '') { * @param {string} userName User name * @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models */ -function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '', userName = '') { +export function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '', userName = '') { // This is a 1x1 transparent PNG const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='; @@ -456,7 +458,7 @@ function convertGooglePrompt(messages, model, useSysPrompt = false, charName = ' * @param {string} charName Character name * @param {string} userName User name */ -function convertAI21Messages(messages, charName = '', userName = '') { +export function convertAI21Messages(messages, charName = '', userName = '') { if (!Array.isArray(messages)) { return []; } @@ -528,7 +530,7 @@ function convertAI21Messages(messages, charName = '', userName = '') { * @param {string} charName Character name * @param {string} userName User name */ -function convertMistralMessages(messages, charName = '', userName = '') { +export function convertMistralMessages(messages, charName = '', userName = '') { if (!Array.isArray(messages)) { return []; } @@ -612,7 +614,7 @@ function convertMistralMessages(messages, charName = '', userName = '') { * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message * @returns {any[]} Merged messages */ -function mergeMessages(messages, charName, userName, strict) { +export function mergeMessages(messages, charName, userName, strict) { let mergedMessages = []; // Remove names from the messages @@ -686,7 +688,7 @@ function mergeMessages(messages, charName, userName, strict) { * @param {object[]} messages Array of messages * @returns {string} Prompt for Text Completion API */ -function convertTextCompletionPrompt(messages) { +export function convertTextCompletionPrompt(messages) { if (typeof messages === 'string') { return messages; } @@ -705,14 +707,3 @@ function convertTextCompletionPrompt(messages) { }); return messageStrings.join('\n') + '\nassistant:'; } - -module.exports = { - convertClaudePrompt, - convertClaudeMessages, - convertGooglePrompt, - convertTextCompletionPrompt, - convertCohereMessages, - convertMistralMessages, - convertAI21Messages, - mergeMessages, -}; diff --git a/src/request-proxy.js b/src/request-proxy.js index 815fa76a3..4d490ce22 100644 --- a/src/request-proxy.js +++ b/src/request-proxy.js @@ -1,7 +1,8 @@ -const http = require('node:http'); -const https = require('node:https'); +import * as http from 'node:http'; +import * as https from 'node:https'; -const { isValidUrl, color } = require('./util.js'); +import { ProxyAgent } from 'proxy-agent'; +import { isValidUrl, color } from './util.js'; const LOG_HEADER = '[Request Proxy]'; @@ -13,10 +14,8 @@ const LOG_HEADER = '[Request Proxy]'; * @property {string} url Proxy URL. * @property {string[]} bypass List of URLs to bypass proxy. */ -function initRequestProxy({ enabled, url, bypass }) { +export default function initRequestProxy({ enabled, url, bypass }) { try { - const { ProxyAgent } = require('proxy-agent'); - // No proxy is enabled, so return if (!enabled) { return; @@ -52,5 +51,3 @@ function initRequestProxy({ enabled, url, bypass }) { console.error(color.red(LOG_HEADER), 'Failed to initialize request proxy:', error); } } - -module.exports = initRequestProxy; diff --git a/src/transformers.mjs b/src/transformers.mjs index 09714e3bb..6669bf00f 100644 --- a/src/transformers.mjs +++ b/src/transformers.mjs @@ -1,7 +1,8 @@ +import path from 'node:path'; +import fs from 'node:fs'; + import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers'; import { getConfigValue } from './util.js'; -import path from 'path'; -import fs from 'fs'; configureTransformers(); @@ -50,7 +51,7 @@ const tasks = { * @param {string} image Base64-encoded image * @returns {Promise} Object representing the image */ -async function getRawImage(image) { +export async function getRawImage(image) { try { const buffer = Buffer.from(image, 'base64'); const byteArray = new Uint8Array(buffer); @@ -116,7 +117,7 @@ async function migrateCacheToDataDir() { * @param {string} forceModel The model to use for the pipeline, if any * @returns {Promise} Pipeline for the task */ -async function getPipeline(task, forceModel = '') { +export async function getPipeline(task, forceModel = '') { await migrateCacheToDataDir(); if (tasks[task].pipeline) { @@ -138,6 +139,6 @@ async function getPipeline(task, forceModel = '') { } export default { - getPipeline, getRawImage, + getPipeline, }; diff --git a/src/users.js b/src/users.js index 4fb85ab0d..f5c8099c7 100644 --- a/src/users.js +++ b/src/users.js @@ -1,22 +1,23 @@ // Native Node Modules -const path = require('path'); -const fs = require('fs'); -const crypto = require('crypto'); -const os = require('os'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import * as crypto from 'node:crypto'; +import * as os from 'node:os'; // Express and other dependencies -const storage = require('node-persist'); -const express = require('express'); -const mime = require('mime-types'); -const archiver = require('archiver'); -const writeFileAtomicSync = require('write-file-atomic').sync; -const _ = require('lodash'); +import storage from 'node-persist'; +import express from 'express'; +import mime from 'mime-types'; +import archiver from 'archiver'; +import _ from 'lodash'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; -const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } = require('./constants'); -const { getConfigValue, color, delay, setConfigValue, generateTimestamp } = require('./util'); -const { readSecret, writeSecret } = require('./endpoints/secrets'); +import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } from './constants.js'; +import { getConfigValue, color, delay, setConfigValue, generateTimestamp } from './util.js'; +import { readSecret, writeSecret } from './endpoints/secrets.js'; +import { getContentOfType } from './endpoints/content-manager.js'; -const KEY_PREFIX = 'user:'; +export const KEY_PREFIX = 'user:'; const AVATAR_PREFIX = 'avatar:'; const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false); const AUTHELIA_AUTH = getConfigValue('autheliaAuth', false); @@ -94,7 +95,7 @@ const STORAGE_KEYS = { * Ensures that the content directories exist. * @returns {Promise} - The list of user directories */ -async function ensurePublicDirectoriesExist() { +export async function ensurePublicDirectoriesExist() { for (const dir of Object.values(PUBLIC_DIRECTORIES)) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); @@ -117,7 +118,7 @@ async function ensurePublicDirectoriesExist() { * Gets a list of all user directories. * @returns {Promise} - The list of user directories */ -async function getUserDirectoriesList() { +export async function getUserDirectoriesList() { const userHandles = await getAllUserHandles(); const directoriesList = userHandles.map(handle => getUserDirectories(handle)); return directoriesList; @@ -126,7 +127,7 @@ async function getUserDirectoriesList() { /** * Perform migration from the old user data format to the new one. */ -async function migrateUserData() { +export async function migrateUserData() { const publicDirectory = path.join(process.cwd(), 'public'); // No need to migrate if the characters directory doesn't exists @@ -328,14 +329,13 @@ async function migrateUserData() { console.log(color.green('Migration completed!')); } -async function migrateSystemPrompts() { +export async function migrateSystemPrompts() { /** * Gets the default system prompts. * @returns {Promise} - The list of default system prompts */ async function getDefaultSystemPrompts() { try { - const { getContentOfType } = await import('./endpoints/content-manager.js'); return getContentOfType('sysprompt', 'json'); } catch { return []; @@ -391,7 +391,7 @@ async function migrateSystemPrompts() { * @param {string} handle User handle * @returns {string} The key for the user storage */ -function toKey(handle) { +export function toKey(handle) { return `${KEY_PREFIX}${handle}`; } @@ -400,7 +400,7 @@ function toKey(handle) { * @param {string} handle User handle * @returns {string} The key for the avatar storage */ -function toAvatarKey(handle) { +export function toAvatarKey(handle) { return `${AVATAR_PREFIX}${handle}`; } @@ -409,7 +409,7 @@ function toAvatarKey(handle) { * @param {string} dataRoot The root directory for user data * @returns {Promise} */ -async function initUserStorage(dataRoot) { +export async function initUserStorage(dataRoot) { global.DATA_ROOT = dataRoot; console.log('Using data root:', color.green(global.DATA_ROOT)); console.log(); @@ -430,7 +430,7 @@ async function initUserStorage(dataRoot) { * Get the cookie secret from the config. If it doesn't exist, generate a new one. * @returns {string} The cookie secret */ -function getCookieSecret() { +export function getCookieSecret() { let secret = getConfigValue(STORAGE_KEYS.cookieSecret); if (!secret) { @@ -446,7 +446,7 @@ function getCookieSecret() { * Generates a random password salt. * @returns {string} The password salt */ -function getPasswordSalt() { +export function getPasswordSalt() { return crypto.randomBytes(16).toString('base64'); } @@ -454,7 +454,7 @@ function getPasswordSalt() { * Get the session name for the current server. * @returns {string} The session name */ -function getCookieSessionName() { +export function getCookieSessionName() { // Get server hostname and hash it to generate a session suffix const suffix = crypto.createHash('sha256').update(os.hostname()).digest('hex').slice(0, 8); return `session-${suffix}`; @@ -466,7 +466,7 @@ function getCookieSessionName() { * @param {string} salt Salt to use for hashing * @returns {string} Hashed password */ -function getPasswordHash(password, salt) { +export function getPasswordHash(password, salt) { return crypto.scryptSync(password.normalize(), salt, 64).toString('base64'); } @@ -475,7 +475,7 @@ function getPasswordHash(password, salt) { * @param {import('express').Request} [request] HTTP request object * @returns {string} The CSRF secret */ -function getCsrfSecret(request) { +export function getCsrfSecret(request) { if (!request || !request.user) { return ANON_CSRF_SECRET; } @@ -494,7 +494,7 @@ function getCsrfSecret(request) { * Gets a list of all user handles. * @returns {Promise} - The list of user handles */ -async function getAllUserHandles() { +export async function getAllUserHandles() { const keys = await storage.keys(x => x.key.startsWith(KEY_PREFIX)); const handles = keys.map(x => x.replace(KEY_PREFIX, '')); return handles; @@ -505,7 +505,7 @@ async function getAllUserHandles() { * @param {string} handle User handle * @returns {UserDirectoryList} User directories */ -function getUserDirectories(handle) { +export function getUserDirectories(handle) { if (DIRECTORIES_CACHE.has(handle)) { const cache = DIRECTORIES_CACHE.get(handle); if (cache) { @@ -526,7 +526,7 @@ function getUserDirectories(handle) { * @param {string} handle User handle * @returns {Promise} User avatar URL */ -async function getUserAvatar(handle) { +export async function getUserAvatar(handle) { try { // Check if the user has a custom avatar const avatarKey = toAvatarKey(handle); @@ -563,7 +563,7 @@ async function getUserAvatar(handle) { * @param {import('express').Request} request Request object * @returns {boolean} Whether the user should be redirected to the login page */ -function shouldRedirectToLogin(request) { +export function shouldRedirectToLogin(request) { return ENABLE_ACCOUNTS && !request.user; } @@ -574,7 +574,7 @@ function shouldRedirectToLogin(request) { * @param {boolean} basicAuthMode If Basic auth mode is enabled * @returns {Promise} Whether auto-login was performed */ -async function tryAutoLogin(request, basicAuthMode) { +export async function tryAutoLogin(request, basicAuthMode) { if (!ENABLE_ACCOUNTS || request.user || !request.session) { return false; } @@ -693,7 +693,7 @@ async function basicUserLogin(request) { * @param {import('express').Response} response Response object * @param {import('express').NextFunction} next Next function */ -async function setUserDataMiddleware(request, response, next) { +export async function setUserDataMiddleware(request, response, next) { // If user accounts are disabled, use the default user if (!ENABLE_ACCOUNTS) { const handle = DEFAULT_USER.handle; @@ -751,7 +751,7 @@ async function setUserDataMiddleware(request, response, next) { * @param {import('express').Response} response Response object * @param {import('express').NextFunction} next Next function */ -function requireLoginMiddleware(request, response, next) { +export function requireLoginMiddleware(request, response, next) { if (!request.user) { return response.sendStatus(403); } @@ -787,7 +787,7 @@ function createRouteHandler(directoryFn) { * @param {import('express').NextFunction} next Next function * @returns {any} */ -function requireAdminMiddleware(request, response, next) { +export function requireAdminMiddleware(request, response, next) { if (!request.user) { return response.sendStatus(403); } @@ -806,7 +806,7 @@ function requireAdminMiddleware(request, response, next) { * @param {import('express').Response} response Express response object to write to * @returns {Promise} Promise that resolves when the archive is created */ -async function createBackupArchive(handle, response) { +export async function createBackupArchive(handle, response) { const directories = getUserDirectories(handle); console.log('Backup requested for', handle); @@ -855,7 +855,7 @@ async function getAllUsers() { * Gets all of the enabled users. * @returns {Promise} */ -async function getAllEnabledUsers() { +export async function getAllEnabledUsers() { const users = await getAllUsers(); return users.filter(x => x.enabled); } @@ -863,7 +863,7 @@ async function getAllEnabledUsers() { /** * Express router for serving files from the user's directories. */ -const router = express.Router(); +export const router = express.Router(); router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds)); router.use('/characters/*', createRouteHandler(req => req.user.directories.characters)); router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars)); @@ -871,31 +871,3 @@ router.use('/assets/*', createRouteHandler(req => req.user.directories.assets)); router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages)); router.use('/user/files/*', createRouteHandler(req => req.user.directories.files)); router.use('/scripts/extensions/third-party/*', createRouteHandler(req => req.user.directories.extensions)); - -module.exports = { - KEY_PREFIX, - toKey, - toAvatarKey, - initUserStorage, - ensurePublicDirectoriesExist, - getUserDirectoriesList, - getAllUserHandles, - getUserDirectories, - setUserDataMiddleware, - requireLoginMiddleware, - requireAdminMiddleware, - migrateUserData, - migrateSystemPrompts, - getPasswordSalt, - getPasswordHash, - getCsrfSecret, - getCookieSecret, - getCookieSessionName, - getUserAvatar, - shouldRedirectToLogin, - createBackupArchive, - tryAutoLogin, - getAllUsers, - getAllEnabledUsers, - router, -}; diff --git a/src/util.js b/src/util.js index b513cba3f..3601a3f0e 100644 --- a/src/util.js +++ b/src/util.js @@ -1,13 +1,16 @@ -const path = require('path'); -const fs = require('fs'); -const commandExistsSync = require('command-exists').sync; -const writeFileAtomicSync = require('write-file-atomic').sync; -const _ = require('lodash'); -const yauzl = require('yauzl'); -const mime = require('mime-types'); -const yaml = require('yaml'); -const { default: simpleGit } = require('simple-git'); -const { Readable } = require('stream'); +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import * as http2 from 'node:http2'; +import { Readable } from 'node:stream'; +import { createRequire } from 'node:module'; + +import yaml from 'yaml'; +import { sync as commandExistsSync } from 'command-exists'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import _ from 'lodash'; +import yauzl from 'yauzl'; +import mime from 'mime-types'; +import { default as simpleGit } from 'simple-git'; /** * Parsed config object. @@ -18,7 +21,7 @@ let CACHED_CONFIG = null; * Returns the config object from the config.yaml file. * @returns {object} Config object */ -function getConfig() { +export function getConfig() { if (CACHED_CONFIG) { return CACHED_CONFIG; } @@ -46,7 +49,7 @@ function getConfig() { * @param {any} defaultValue - Default value to return if the key is not found * @returns {any} Value for the given key */ -function getConfigValue(key, defaultValue = null) { +export function getConfigValue(key, defaultValue = null) { const config = getConfig(); return _.get(config, key, defaultValue); } @@ -56,7 +59,7 @@ function getConfigValue(key, defaultValue = null) { * @param {string} key Key to set * @param {any} value Value to set */ -function setConfigValue(key, value) { +export function setConfigValue(key, value) { // Reset cache so that the next getConfig call will read the updated config file CACHED_CONFIG = null; const config = getConfig(); @@ -69,7 +72,7 @@ function setConfigValue(key, value) { * @param {string} auth username:password * @returns {string} Basic Auth header value */ -function getBasicAuthHeader(auth) { +export function getBasicAuthHeader(auth) { const encoded = Buffer.from(`${auth}`).toString('base64'); return `Basic ${encoded}`; } @@ -79,7 +82,7 @@ function getBasicAuthHeader(auth) { * Also returns the agent string for the Horde API. * @returns {Promise<{agent: string, pkgVersion: string, gitRevision: string | null, gitBranch: string | null, commitDate: string | null, isLatest: boolean}>} Version info object */ -async function getVersion() { +export async function getVersion() { let pkgVersion = 'UNKNOWN'; let gitRevision = null; let gitBranch = null; @@ -87,9 +90,10 @@ async function getVersion() { let isLatest = true; try { + const require = createRequire(import.meta.url); const pkgJson = require(path.join(process.cwd(), './package.json')); pkgVersion = pkgJson.version; - if (!process['pkg'] && commandExistsSync('git')) { + if (commandExistsSync('git')) { const git = simpleGit(); const cwd = process.cwd(); gitRevision = await git.cwd(cwd).revparse(['--short', 'HEAD']); @@ -117,7 +121,7 @@ async function getVersion() { * @param {number} ms Milliseconds to wait * @returns {Promise} Promise that resolves after the given amount of milliseconds */ -function delay(ms) { +export function delay(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -127,7 +131,7 @@ function delay(ms) { * @returns {string} Random hex string * @example getHexString(8) // 'a1b2c3d4' */ -function getHexString(length) { +export function getHexString(length) { const chars = '0123456789abcdef'; let result = ''; for (let i = 0; i < length; i++) { @@ -142,7 +146,7 @@ function getHexString(length) { * @param {string} fileExtension File extension to look for * @returns {Promise} Buffer containing the extracted file. Null if the file was not found. */ -async function extractFileFromZipBuffer(archiveBuffer, fileExtension) { +export async function extractFileFromZipBuffer(archiveBuffer, fileExtension) { return await new Promise((resolve, reject) => yauzl.fromBuffer(Buffer.from(archiveBuffer), { lazyEntries: true }, (err, zipfile) => { if (err) { reject(err); @@ -181,7 +185,7 @@ async function extractFileFromZipBuffer(archiveBuffer, fileExtension) { * @param {string} zipFilePath Path to the ZIP archive * @returns {Promise<[string, Buffer][]>} Array of image buffers */ -async function getImageBuffers(zipFilePath) { +export async function getImageBuffers(zipFilePath) { return new Promise((resolve, reject) => { // Check if the zip file exists if (!fs.existsSync(zipFilePath)) { @@ -237,7 +241,7 @@ async function getImageBuffers(zipFilePath) { * @param {any} readableStream Readable stream to read from * @returns {Promise} Array of chunks */ -async function readAllChunks(readableStream) { +export async function readAllChunks(readableStream) { return new Promise((resolve, reject) => { // Consume the readable stream const chunks = []; @@ -261,7 +265,7 @@ function isObject(item) { return (item && typeof item === 'object' && !Array.isArray(item)); } -function deepMerge(target, source) { +export function deepMerge(target, source) { let output = Object.assign({}, target); if (isObject(target) && isObject(source)) { Object.keys(source).forEach(key => { @@ -278,7 +282,7 @@ function deepMerge(target, source) { return output; } -const color = { +export const color = { byNum: (mess, fgNum) => { mess = mess || ''; fgNum = fgNum === undefined ? 31 : fgNum; @@ -298,7 +302,7 @@ const color = { * Gets a random UUIDv4 string. * @returns {string} A UUIDv4 string */ -function uuidv4() { +export function uuidv4() { if ('crypto' in global && 'randomUUID' in global.crypto) { return global.crypto.randomUUID(); } @@ -309,7 +313,7 @@ function uuidv4() { }); } -function humanizedISO8601DateTime(date) { +export function humanizedISO8601DateTime(date) { let baseDate = typeof date === 'number' ? new Date(date) : new Date(); let humanYear = baseDate.getFullYear(); let humanMonth = (baseDate.getMonth() + 1); @@ -322,7 +326,7 @@ function humanizedISO8601DateTime(date) { return HumanizedDateTime; } -function tryParse(str) { +export function tryParse(str) { try { return JSON.parse(str); } catch { @@ -337,7 +341,7 @@ function tryParse(str) { * @param {string} inputPath The path to be converted. * @returns The relative URL path from which the client can access the file. */ -function clientRelativePath(root, inputPath) { +export function clientRelativePath(root, inputPath) { if (!inputPath.startsWith(root)) { throw new Error('Input path does not start with the root directory'); } @@ -350,11 +354,11 @@ function clientRelativePath(root, inputPath) { * @param {string} filename The file name to remove the extension from. * @returns The file name, sans extension */ -function removeFileExtension(filename) { +export function removeFileExtension(filename) { return filename.replace(/\.[^.]+$/, ''); } -function generateTimestamp() { +export function generateTimestamp() { const now = new Date(); const year = now.getFullYear(); const month = String(now.getMonth() + 1).padStart(2, '0'); @@ -371,7 +375,7 @@ function generateTimestamp() { * @param {string} directory The root directory to remove backups from. * @param {string} prefix File prefix to filter backups by. */ -function removeOldBackups(directory, prefix) { +export function removeOldBackups(directory, prefix) { const MAX_BACKUPS = Number(getConfigValue('numberOfBackups', 50)); let files = fs.readdirSync(directory).filter(f => f.startsWith(prefix)); @@ -389,7 +393,7 @@ function removeOldBackups(directory, prefix) { * @param {'name' | 'date'} sortBy Sort images by name or date * @returns {string[]} List of image file names */ -function getImages(directoryPath, sortBy = 'name') { +export function getImages(directoryPath, sortBy = 'name') { function getSortFunction() { switch (sortBy) { case 'name': @@ -415,7 +419,7 @@ function getImages(directoryPath, sortBy = 'name') { * @param {import('node-fetch').Response} from The Fetch API response to pipe from. * @param {import('express').Response} to The Express response to pipe to. */ -function forwardFetchResponse(from, to) { +export function forwardFetchResponse(from, to) { let statusCode = from.status; let statusText = from.statusText; @@ -434,6 +438,7 @@ function forwardFetchResponse(from, to) { to.statusCode = statusCode; to.statusMessage = statusText; + from.body.pipe(to); to.socket.on('close', function () { @@ -457,10 +462,9 @@ function forwardFetchResponse(from, to) { * @param {object} headers Request headers * @returns {Promise} Response body */ -function makeHttp2Request(endpoint, method, body, headers) { +export function makeHttp2Request(endpoint, method, body, headers) { return new Promise((resolve, reject) => { try { - const http2 = require('http2'); const url = new URL(endpoint); const client = http2.connect(url.origin); @@ -511,7 +515,7 @@ function makeHttp2Request(endpoint, method, body, headers) { * @param {string} yamlString YAML-serialized object * @returns */ -function mergeObjectWithYaml(obj, yamlString) { +export function mergeObjectWithYaml(obj, yamlString) { if (!yamlString) { return; } @@ -540,7 +544,7 @@ function mergeObjectWithYaml(obj, yamlString) { * @param {string} yamlString YAML-serialized array * @returns {void} Nothing */ -function excludeKeysByYaml(obj, yamlString) { +export function excludeKeysByYaml(obj, yamlString) { if (!yamlString) { return; } @@ -569,14 +573,14 @@ function excludeKeysByYaml(obj, yamlString) { * @param {string} str Input string * @returns {string} Trimmed string */ -function trimV1(str) { +export function trimV1(str) { return String(str ?? '').replace(/\/$/, '').replace(/\/v1$/, ''); } /** * Simple TTL memory cache. */ -class Cache { +export class Cache { /** * @param {number} ttl Time to live in milliseconds */ @@ -633,7 +637,7 @@ class Cache { * @param {string} text Text with color formatting * @returns {string} Text without color formatting */ -function removeColorFormatting(text) { +export function removeColorFormatting(text) { // ANSI escape codes for colors are usually in the format \x1b[m return text.replace(/\x1b\[\d{1,2}(;\d{1,2})*m/g, ''); } @@ -643,7 +647,7 @@ function removeColorFormatting(text) { * @param {number} n Number of times to repeat the separator * @returns {string} Separator string */ -function getSeparator(n) { +export function getSeparator(n) { return '='.repeat(n); } @@ -652,7 +656,7 @@ function getSeparator(n) { * @param {string} url String to check * @returns {boolean} If the URL is valid */ -function isValidUrl(url) { +export function isValidUrl(url) { try { new URL(url); return true; @@ -660,35 +664,3 @@ function isValidUrl(url) { return false; } } - -module.exports = { - getConfig, - getConfigValue, - setConfigValue, - getVersion, - getBasicAuthHeader, - extractFileFromZipBuffer, - getImageBuffers, - readAllChunks, - delay, - deepMerge, - color, - uuidv4, - humanizedISO8601DateTime, - tryParse, - clientRelativePath, - removeFileExtension, - generateTimestamp, - removeOldBackups, - getImages, - forwardFetchResponse, - getHexString, - mergeObjectWithYaml, - excludeKeysByYaml, - trimV1, - Cache, - makeHttp2Request, - removeColorFormatting, - getSeparator, - isValidUrl, -}; diff --git a/src/validator/TavernCardValidator.js b/src/validator/TavernCardValidator.js index dec2fbb89..82593d0aa 100644 --- a/src/validator/TavernCardValidator.js +++ b/src/validator/TavernCardValidator.js @@ -5,7 +5,7 @@ * * @link https://github.com/malfoyslastname/character-card-spec-v2 */ -class TavernCardValidator { +export class TavernCardValidator { /** * @type {string|null} */ @@ -167,5 +167,3 @@ class TavernCardValidator { return true; } } - -module.exports = { TavernCardValidator }; diff --git a/src/vectors/cohere-vectors.js b/src/vectors/cohere-vectors.js index 96ce5bff5..1eea1658d 100644 --- a/src/vectors/cohere-vectors.js +++ b/src/vectors/cohere-vectors.js @@ -1,5 +1,5 @@ -const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('../endpoints/secrets'); +import fetch from 'node-fetch'; +import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js'; /** * Gets the vector for the given text batch from an OpenAI compatible endpoint. @@ -9,7 +9,7 @@ const { SECRET_KEYS, readSecret } = require('../endpoints/secrets'); * @param {string} model - The model to use for the embedding * @returns {Promise} - The array of vectors for the texts */ -async function getCohereBatchVector(texts, isQuery, directories, model) { +export async function getCohereBatchVector(texts, isQuery, directories, model) { const key = readSecret(directories, SECRET_KEYS.COHERE); if (!key) { @@ -55,12 +55,8 @@ async function getCohereBatchVector(texts, isQuery, directories, model) { * @param {string} model - The model to use for the embedding * @returns {Promise} - The vector for the text */ -async function getCohereVector(text, isQuery, directories, model) { +export async function getCohereVector(text, isQuery, directories, model) { const vectors = await getCohereBatchVector([text], isQuery, directories, model); return vectors[0]; } -module.exports = { - getCohereBatchVector, - getCohereVector, -}; diff --git a/src/vectors/embedding.js b/src/vectors/embedding.js index 3f02e07db..a3d499014 100644 --- a/src/vectors/embedding.js +++ b/src/vectors/embedding.js @@ -1,3 +1,4 @@ +import { getPipeline } from '../transformers.mjs'; const TASK = 'feature-extraction'; /** @@ -5,9 +6,8 @@ const TASK = 'feature-extraction'; * @param {string} text - The text to vectorize * @returns {Promise} - The vectorized text in form of an array of numbers */ -async function getTransformersVector(text) { - const module = await import('../transformers.mjs'); - const pipe = await module.default.getPipeline(TASK); +export async function getTransformersVector(text) { + const pipe = await getPipeline(TASK); const result = await pipe(text, { pooling: 'mean', normalize: true }); const vector = Array.from(result.data); return vector; @@ -18,15 +18,10 @@ async function getTransformersVector(text) { * @param {string[]} texts - The texts to vectorize * @returns {Promise} - The vectorized texts in form of an array of arrays of numbers */ -async function getTransformersBatchVector(texts) { +export async function getTransformersBatchVector(texts) { const result = []; for (const text of texts) { result.push(await getTransformersVector(text)); } return result; } - -module.exports = { - getTransformersVector, - getTransformersBatchVector, -}; diff --git a/src/vectors/extras-vectors.js b/src/vectors/extras-vectors.js index 56da20633..5a0857ef6 100644 --- a/src/vectors/extras-vectors.js +++ b/src/vectors/extras-vectors.js @@ -1,4 +1,4 @@ -const fetch = require('node-fetch').default; +import fetch from 'node-fetch'; /** * Gets the vector for the given text from SillyTavern-extras @@ -7,7 +7,7 @@ const fetch = require('node-fetch').default; * @param {string} apiKey - The Extras API key, or empty string if API key not enabled * @returns {Promise} - The array of vectors for the texts */ -async function getExtrasBatchVector(texts, apiUrl, apiKey) { +export async function getExtrasBatchVector(texts, apiUrl, apiKey) { return getExtrasVectorImpl(texts, apiUrl, apiKey); } @@ -18,7 +18,7 @@ async function getExtrasBatchVector(texts, apiUrl, apiKey) { * @param {string} apiKey - The Extras API key, or empty string if API key not enabled * @returns {Promise} - The vector for the text */ -async function getExtrasVector(text, apiUrl, apiKey) { +export async function getExtrasVector(text, apiUrl, apiKey) { return getExtrasVectorImpl(text, apiUrl, apiKey); } @@ -71,8 +71,3 @@ async function getExtrasVectorImpl(text, apiUrl, apiKey) { return vector; } - -module.exports = { - getExtrasVector, - getExtrasBatchVector, -}; diff --git a/src/vectors/llamacpp-vectors.js b/src/vectors/llamacpp-vectors.js index 489861590..8efa21d03 100644 --- a/src/vectors/llamacpp-vectors.js +++ b/src/vectors/llamacpp-vectors.js @@ -1,6 +1,6 @@ -const fetch = require('node-fetch').default; -const { setAdditionalHeadersByType } = require('../additional-headers'); -const { TEXTGEN_TYPES } = require('../constants'); +import fetch from 'node-fetch'; +import { setAdditionalHeadersByType } from '../additional-headers.js'; +import { TEXTGEN_TYPES } from '../constants.js'; /** * Gets the vector for the given text from LlamaCpp @@ -9,7 +9,7 @@ const { TEXTGEN_TYPES } = require('../constants'); * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ -async function getLlamaCppBatchVector(texts, apiUrl, directories) { +export async function getLlamaCppBatchVector(texts, apiUrl, directories) { const url = new URL(apiUrl); url.pathname = '/v1/embeddings'; @@ -50,12 +50,7 @@ async function getLlamaCppBatchVector(texts, apiUrl, directories) { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ -async function getLlamaCppVector(text, apiUrl, directories) { +export async function getLlamaCppVector(text, apiUrl, directories) { const vectors = await getLlamaCppBatchVector([text], apiUrl, directories); return vectors[0]; } - -module.exports = { - getLlamaCppBatchVector, - getLlamaCppVector, -}; diff --git a/src/vectors/makersuite-vectors.js b/src/vectors/makersuite-vectors.js index 145b9203e..1fbffb3a6 100644 --- a/src/vectors/makersuite-vectors.js +++ b/src/vectors/makersuite-vectors.js @@ -1,5 +1,5 @@ -const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('../endpoints/secrets'); +import fetch from 'node-fetch'; +import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js'; const API_MAKERSUITE = 'https://generativelanguage.googleapis.com'; /** @@ -8,7 +8,7 @@ const API_MAKERSUITE = 'https://generativelanguage.googleapis.com'; * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ -async function getMakerSuiteBatchVector(texts, directories) { +export async function getMakerSuiteBatchVector(texts, directories) { const promises = texts.map(text => getMakerSuiteVector(text, directories)); return await Promise.all(promises); } @@ -19,7 +19,7 @@ async function getMakerSuiteBatchVector(texts, directories) { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ -async function getMakerSuiteVector(text, directories) { +export async function getMakerSuiteVector(text, directories) { const key = readSecret(directories, SECRET_KEYS.MAKERSUITE); if (!key) { @@ -56,8 +56,3 @@ async function getMakerSuiteVector(text, directories) { // noinspection JSValidateTypes return data['embedding']['values']; } - -module.exports = { - getMakerSuiteVector, - getMakerSuiteBatchVector, -}; diff --git a/src/vectors/nomicai-vectors.js b/src/vectors/nomicai-vectors.js index 29b322926..ef565108e 100644 --- a/src/vectors/nomicai-vectors.js +++ b/src/vectors/nomicai-vectors.js @@ -1,5 +1,5 @@ -const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('../endpoints/secrets'); +import fetch from 'node-fetch'; +import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js'; const SOURCES = { 'nomicai': { @@ -16,7 +16,7 @@ const SOURCES = { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ -async function getNomicAIBatchVector(texts, source, directories) { +export async function getNomicAIBatchVector(texts, source, directories) { const config = SOURCES[source]; if (!config) { @@ -67,12 +67,7 @@ async function getNomicAIBatchVector(texts, source, directories) { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ -async function getNomicAIVector(text, source, directories) { +export async function getNomicAIVector(text, source, directories) { const vectors = await getNomicAIBatchVector([text], source, directories); return vectors[0]; } - -module.exports = { - getNomicAIVector, - getNomicAIBatchVector, -}; diff --git a/src/vectors/ollama-vectors.js b/src/vectors/ollama-vectors.js index 4ada1e7d0..5c654ba6f 100644 --- a/src/vectors/ollama-vectors.js +++ b/src/vectors/ollama-vectors.js @@ -1,6 +1,6 @@ -const fetch = require('node-fetch').default; -const { setAdditionalHeadersByType } = require('../additional-headers'); -const { TEXTGEN_TYPES } = require('../constants'); +import fetch from 'node-fetch'; +import { setAdditionalHeadersByType } from '../additional-headers.js'; +import { TEXTGEN_TYPES } from '../constants.js'; /** * Gets the vector for the given text from Ollama @@ -11,7 +11,7 @@ const { TEXTGEN_TYPES } = require('../constants'); * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ -async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) { +export async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) { const result = []; for (const text of texts) { const vector = await getOllamaVector(text, apiUrl, model, keep, directories); @@ -29,7 +29,7 @@ async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ -async function getOllamaVector(text, apiUrl, model, keep, directories) { +export async function getOllamaVector(text, apiUrl, model, keep, directories) { const url = new URL(apiUrl); url.pathname = '/api/embeddings'; @@ -62,8 +62,3 @@ async function getOllamaVector(text, apiUrl, model, keep, directories) { return data.embedding; } - -module.exports = { - getOllamaBatchVector, - getOllamaVector, -}; diff --git a/src/vectors/openai-vectors.js b/src/vectors/openai-vectors.js index 6a30d3f2b..2d5ff6938 100644 --- a/src/vectors/openai-vectors.js +++ b/src/vectors/openai-vectors.js @@ -1,5 +1,5 @@ -const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('../endpoints/secrets'); +import fetch from 'node-fetch'; +import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js'; const SOURCES = { 'togetherai': { @@ -27,7 +27,7 @@ const SOURCES = { * @param {string} model - The model to use for the embedding * @returns {Promise} - The array of vectors for the texts */ -async function getOpenAIBatchVector(texts, source, directories, model = '') { +export async function getOpenAIBatchVector(texts, source, directories, model = '') { const config = SOURCES[source]; if (!config) { @@ -83,12 +83,7 @@ async function getOpenAIBatchVector(texts, source, directories, model = '') { * @param {string} model - The model to use for the embedding * @returns {Promise} - The vector for the text */ -async function getOpenAIVector(text, source, directories, model = '') { +export async function getOpenAIVector(text, source, directories, model = '') { const vectors = await getOpenAIBatchVector([text], source, directories, model); return vectors[0]; } - -module.exports = { - getOpenAIVector, - getOpenAIBatchVector, -}; diff --git a/src/vectors/vllm-vectors.js b/src/vectors/vllm-vectors.js index 64879edaa..cc5f181bf 100644 --- a/src/vectors/vllm-vectors.js +++ b/src/vectors/vllm-vectors.js @@ -1,6 +1,6 @@ -const fetch = require('node-fetch').default; -const { setAdditionalHeadersByType } = require('../additional-headers'); -const { TEXTGEN_TYPES } = require('../constants'); +import fetch from 'node-fetch'; +import { setAdditionalHeadersByType } from '../additional-headers.js'; +import { TEXTGEN_TYPES } from '../constants.js'; /** * Gets the vector for the given text from VLLM @@ -10,7 +10,7 @@ const { TEXTGEN_TYPES } = require('../constants'); * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ -async function getVllmBatchVector(texts, apiUrl, model, directories) { +export async function getVllmBatchVector(texts, apiUrl, model, directories) { const url = new URL(apiUrl); url.pathname = '/v1/embeddings'; @@ -52,12 +52,7 @@ async function getVllmBatchVector(texts, apiUrl, model, directories) { * @param {import('../users').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ -async function getVllmVector(text, apiUrl, model, directories) { +export async function getVllmVector(text, apiUrl, model, directories) { const vectors = await getVllmBatchVector([text], apiUrl, model, directories); return vectors[0]; } - -module.exports = { - getVllmBatchVector, - getVllmVector, -}; From 1616e7e06760cdb593d269b56e37c7f6184099b8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:36:22 +0300 Subject: [PATCH 02/21] Explicitly import Buffer. Specify jsdoc local extensions --- src/additional-headers.js | 28 ++++++++++++++-------------- src/character-card-parser.js | 1 + src/constants.js | 4 ++-- src/endpoints/assets.js | 2 +- src/endpoints/characters.js | 11 ++++++----- src/endpoints/content-manager.js | 9 +++++---- src/endpoints/images.js | 1 + src/endpoints/novelai.js | 1 + src/endpoints/openai.js | 1 + src/endpoints/presets.js | 2 +- src/endpoints/secrets.js | 10 +++++----- src/endpoints/speech.js | 1 + src/endpoints/sprites.js | 4 ++-- src/endpoints/thumbnails.js | 8 ++++---- src/endpoints/tokenizers.js | 1 + src/endpoints/translate.js | 2 ++ src/endpoints/users-admin.js | 12 ++++++------ src/endpoints/users-private.js | 6 +++--- src/endpoints/users-public.js | 10 +++++----- src/endpoints/vectors.js | 16 ++++++++-------- src/endpoints/worldinfo.js | 2 +- src/middleware/basicAuth.js | 1 + src/middleware/multerMonkeyPatch.js | 2 ++ src/transformers.mjs | 1 + src/users.js | 5 +++-- src/util.js | 1 + src/vectors/cohere-vectors.js | 4 ++-- src/vectors/llamacpp-vectors.js | 4 ++-- src/vectors/makersuite-vectors.js | 4 ++-- src/vectors/nomicai-vectors.js | 4 ++-- src/vectors/ollama-vectors.js | 4 ++-- src/vectors/openai-vectors.js | 4 ++-- src/vectors/vllm-vectors.js | 4 ++-- 33 files changed, 93 insertions(+), 77 deletions(-) diff --git a/src/additional-headers.js b/src/additional-headers.js index d24afb7d7..e35456ae4 100644 --- a/src/additional-headers.js +++ b/src/additional-headers.js @@ -4,7 +4,7 @@ import { getConfigValue } from './util.js'; /** * Gets the headers for the Mancer API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getMancerHeaders(directories) { @@ -18,7 +18,7 @@ function getMancerHeaders(directories) { /** * Gets the headers for the TogetherAI API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getTogetherAIHeaders(directories) { @@ -31,7 +31,7 @@ function getTogetherAIHeaders(directories) { /** * Gets the headers for the InfermaticAI API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getInfermaticAIHeaders(directories) { @@ -44,7 +44,7 @@ function getInfermaticAIHeaders(directories) { /** * Gets the headers for the DreamGen API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getDreamGenHeaders(directories) { @@ -57,7 +57,7 @@ function getDreamGenHeaders(directories) { /** * Gets the headers for the OpenRouter API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getOpenRouterHeaders(directories) { @@ -69,7 +69,7 @@ function getOpenRouterHeaders(directories) { /** * Gets the headers for the vLLM API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getVllmHeaders(directories) { @@ -82,7 +82,7 @@ function getVllmHeaders(directories) { /** * Gets the headers for the Aphrodite API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getAphroditeHeaders(directories) { @@ -96,7 +96,7 @@ function getAphroditeHeaders(directories) { /** * Gets the headers for the Tabby API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getTabbyHeaders(directories) { @@ -110,7 +110,7 @@ function getTabbyHeaders(directories) { /** * Gets the headers for the LlamaCPP API. - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories * @returns {object} Headers for the request */ function getLlamaCppHeaders(directories) { @@ -123,7 +123,7 @@ function getLlamaCppHeaders(directories) { /** * Gets the headers for the Ooba API. - * @param {import('./users').UserDirectoryList} directories + * @param {import('./users.js').UserDirectoryList} directories * @returns {object} Headers for the request */ function getOobaHeaders(directories) { @@ -136,7 +136,7 @@ function getOobaHeaders(directories) { /** * Gets the headers for the KoboldCpp API. - * @param {import('./users').UserDirectoryList} directories + * @param {import('./users.js').UserDirectoryList} directories * @returns {object} Headers for the request */ function getKoboldCppHeaders(directories) { @@ -149,7 +149,7 @@ function getKoboldCppHeaders(directories) { /** * Gets the headers for the Featherless API. - * @param {import('./users').UserDirectoryList} directories + * @param {import('./users.js').UserDirectoryList} directories * @returns {object} Headers for the request */ function getFeatherlessHeaders(directories) { @@ -161,7 +161,7 @@ function getFeatherlessHeaders(directories) { /** * Gets the headers for the HuggingFace API. - * @param {import('./users').UserDirectoryList} directories + * @param {import('./users.js').UserDirectoryList} directories * @returns {object} Headers for the request */ function getHuggingFaceHeaders(directories) { @@ -197,7 +197,7 @@ export function setAdditionalHeaders(request, args, server) { * @param {object} requestHeaders Request headers * @param {string} type API type * @param {string|null} server API server for new request - * @param {import('./users').UserDirectoryList} directories User directories + * @param {import('./users.js').UserDirectoryList} directories User directories */ export function setAdditionalHeadersByType(requestHeaders, type, server, directories) { const headerGetters = { diff --git a/src/character-card-parser.js b/src/character-card-parser.js index 3b7980399..a8433f3b7 100644 --- a/src/character-card-parser.js +++ b/src/character-card-parser.js @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { Buffer } from 'node:buffer'; import encode from 'png-chunks-encode'; import extract from 'png-chunks-extract'; diff --git a/src/constants.js b/src/constants.js index 73896a56d..0befdaf65 100644 --- a/src/constants.js +++ b/src/constants.js @@ -8,7 +8,7 @@ export const PUBLIC_DIRECTORIES = { export const SETTINGS_FILE = 'settings.json'; /** - * @type {import('./users').UserDirectoryList} + * @type {import('./users.js').UserDirectoryList} * @readonly * @enum {string} */ @@ -45,7 +45,7 @@ export const USER_DIRECTORY_TEMPLATE = Object.freeze({ }); /** - * @type {import('./users').User} + * @type {import('./users.js').User} * @readonly */ export const DEFAULT_USER = Object.freeze({ diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 7f282b3f1..7379aceba 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -79,7 +79,7 @@ function getFiles(dir, files = []) { /** * Ensure that the asset folders exist. - * @param {import('../users').UserDirectoryList} directories - The user's directories + * @param {import('../users.js').UserDirectoryList} directories - The user's directories */ function ensureFoldersExist(directories) { const folderPath = path.join(directories.assets); diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 380d8a72f..20a947ef8 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -2,6 +2,7 @@ import * as path from 'node:path'; import * as fs from 'node:fs'; import { promises as fsPromises } from 'node:fs'; import * as readline from 'node:readline'; +import { Buffer } from 'node:buffer'; import express from 'express'; import sanitize from 'sanitize-filename'; @@ -188,7 +189,7 @@ const calculateDataSize = (data) => { * processCharacter - Process a given character, read its data and calculate its statistics. * * @param {string} item The name of the character. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @return {Promise} A Promise that resolves when the character processing is done. */ const processCharacter = async (item, directories) => { @@ -232,7 +233,7 @@ const processCharacter = async (item, directories) => { /** * Convert a character object to Spec V2 format. * @param {object} jsonObject Character object - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {boolean} hoistDate Will set the chat and create_date fields to the current date if they are missing * @returns {object} Character object in Spec V2 format */ @@ -252,7 +253,7 @@ function getCharaCardV2(jsonObject, directories, hoistDate = true) { /** * Convert a character object to Spec V2 format. * @param {object} char Character object - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {object} Character object in Spec V2 format */ function convertToV2(char, directories) { @@ -342,7 +343,7 @@ function readFromV2(char) { /** * Format character data to Spec V2 format. * @param {object} data Character data - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns */ function charaFormatData(data, directories) { @@ -1081,7 +1082,7 @@ router.post('/chats', jsonParser, async function (request, response) { /** * Gets the name for the uploaded PNG file. * @param {string} file File name - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {string} - The name for the uploaded PNG file */ function getPngName(file, directories) { diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 7523844db..d587c2a34 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -1,5 +1,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { Buffer } from 'node:buffer'; import express from 'express'; import fetch from 'node-fetch'; @@ -50,7 +51,7 @@ export const CONTENT_TYPES = { /** * Gets the default presets from the content directory. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {object[]} Array of default presets */ export function getDefaultPresets(directories) { @@ -97,7 +98,7 @@ export function getDefaultPresetFile(filename) { /** * Seeds content for a user. * @param {ContentItem[]} contentIndex Content index - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {string[]} forceCategories List of categories to force check (even if content check is skipped) * @returns {Promise} Whether any content was added */ @@ -156,7 +157,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) { /** * Checks for new content and seeds it for all users. - * @param {import('../users').UserDirectoryList[]} directoriesList List of user directories + * @param {import('../users.js').UserDirectoryList[]} directoriesList List of user directories * @param {string[]} forceCategories List of categories to force check (even if content check is skipped) * @returns {Promise} */ @@ -258,7 +259,7 @@ export function getContentOfType(type, format) { /** * Gets the target directory for the specified asset type. * @param {ContentType} type Asset type - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {string | null} Target directory */ function getTargetByType(type, directories) { diff --git a/src/endpoints/images.js b/src/endpoints/images.js index dd3be719e..cf52c6a76 100644 --- a/src/endpoints/images.js +++ b/src/endpoints/images.js @@ -1,5 +1,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { Buffer } from 'node:buffer'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/novelai.js b/src/endpoints/novelai.js index b42b1de31..19046a2a5 100644 --- a/src/endpoints/novelai.js +++ b/src/endpoints/novelai.js @@ -1,4 +1,5 @@ import * as util from 'node:util'; +import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; import express from 'express'; diff --git a/src/endpoints/openai.js b/src/endpoints/openai.js index d6e393681..55b2d7c11 100644 --- a/src/endpoints/openai.js +++ b/src/endpoints/openai.js @@ -1,4 +1,5 @@ import * as fs from 'node:fs'; +import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; import FormData from 'form-data'; diff --git a/src/endpoints/presets.js b/src/endpoints/presets.js index 173b53cce..36eb53b38 100644 --- a/src/endpoints/presets.js +++ b/src/endpoints/presets.js @@ -11,7 +11,7 @@ import { jsonParser } from '../express-common.js'; /** * Gets the folder and extension for the preset settings based on the API source ID. * @param {string} apiId API source ID - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {object} Object containing the folder and extension for the preset settings */ function getPresetSettingsByAPI(apiId, directories) { diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 99db701f6..51cbcf2fe 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -59,7 +59,7 @@ const EXPORTABLE_KEYS = [ /** * Writes a secret to the secrets file - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {string} key Secret key * @param {string} value Secret value */ @@ -79,7 +79,7 @@ export function writeSecret(directories, key, value) { /** * Deletes a secret from the secrets file - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {string} key Secret key * @returns */ @@ -98,7 +98,7 @@ export function deleteSecret(directories, key) { /** * Reads a secret from the secrets file - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {string} key Secret key * @returns {string} Secret value */ @@ -116,7 +116,7 @@ export function readSecret(directories, key) { /** * Reads the secret state from the secrets file - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {object} Secret state */ export function readSecretState(directories) { @@ -139,7 +139,7 @@ export function readSecretState(directories) { /** * Reads all secrets from the secrets file - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @returns {Record | undefined} Secrets */ export function getAllSecrets(directories) { diff --git a/src/endpoints/speech.js b/src/endpoints/speech.js index 9540b0411..eee6a7679 100644 --- a/src/endpoints/speech.js +++ b/src/endpoints/speech.js @@ -1,3 +1,4 @@ +import { Buffer } from 'node:buffer'; import express from 'express'; import wavefile from 'wavefile'; import { jsonParser } from '../express-common.js'; diff --git a/src/endpoints/sprites.js b/src/endpoints/sprites.js index 8d2e34020..e1ce6024d 100644 --- a/src/endpoints/sprites.js +++ b/src/endpoints/sprites.js @@ -11,7 +11,7 @@ import { jsonParser, urlencodedParser } from '../express-common.js'; /** * Gets the path to the sprites folder for the provided character name - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} name - The name of the character * @param {boolean} isSubfolder - Whether the name contains a subfolder * @returns {string | null} The path to the sprites folder. Null if the name is invalid. @@ -42,7 +42,7 @@ function getSpritesPath(directories, name, isSubfolder) { * Imports base64 encoded sprites from RisuAI character data. * The sprites are saved in the character's sprites folder. * The additionalAssets and emotions are removed from the data. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {object} data RisuAI character data * @returns {void} */ diff --git a/src/endpoints/thumbnails.js b/src/endpoints/thumbnails.js index d095c135c..9f1812b72 100644 --- a/src/endpoints/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -18,7 +18,7 @@ const pngFormat = getConfigValue('avatarThumbnailsPng', false); /** * Gets a path to thumbnail folder based on the type. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Thumbnail type * @returns {string} Path to the thumbnails folder */ @@ -39,7 +39,7 @@ function getThumbnailFolder(directories, type) { /** * Gets a path to the original images folder based on the type. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Thumbnail type * @returns {string} Path to the original images folder */ @@ -60,7 +60,7 @@ function getOriginalFolder(directories, type) { /** * Removes the generated thumbnail from the disk. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Type of the thumbnail * @param {string} file Name of the file */ @@ -77,7 +77,7 @@ export function invalidateThumbnail(directories, type, file) { /** * Generates a thumbnail for the given file. - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {'bg' | 'avatar'} type Type of the thumbnail * @param {string} file Name of the file * @returns diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index 57ce530ec..719d2219c 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -1,5 +1,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { Buffer } from 'node:buffer'; import express from 'express'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; diff --git a/src/endpoints/translate.js b/src/endpoints/translate.js index 0ab66527b..e254506cb 100644 --- a/src/endpoints/translate.js +++ b/src/endpoints/translate.js @@ -1,5 +1,7 @@ import * as https from 'node:https'; import { createRequire } from 'node:module'; +import { Buffer } from 'node:buffer'; + import fetch from 'node-fetch'; import express from 'express'; import iconv from 'iconv-lite'; diff --git a/src/endpoints/users-admin.js b/src/endpoints/users-admin.js index 1e75336ba..6b3e251aa 100644 --- a/src/endpoints/users-admin.js +++ b/src/endpoints/users-admin.js @@ -22,10 +22,10 @@ export const router = express.Router(); router.post('/get', requireAdminMiddleware, jsonParser, async (_request, response) => { try { - /** @type {import('../users').User[]} */ + /** @type {import('../users.js').User[]} */ const users = await storage.values(x => x.key.startsWith(KEY_PREFIX)); - /** @type {Promise[]} */ + /** @type {Promise[]} */ const viewModelPromises = users .map(user => new Promise(resolve => { getUserAvatar(user.handle).then(avatar => @@ -62,7 +62,7 @@ router.post('/disable', requireAdminMiddleware, jsonParser, async (request, resp return response.status(400).json({ error: 'Cannot disable yourself' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -86,7 +86,7 @@ router.post('/enable', requireAdminMiddleware, jsonParser, async (request, respo return response.status(400).json({ error: 'Missing required fields' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -110,7 +110,7 @@ router.post('/promote', requireAdminMiddleware, jsonParser, async (request, resp return response.status(400).json({ error: 'Missing required fields' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -139,7 +139,7 @@ router.post('/demote', requireAdminMiddleware, jsonParser, async (request, respo return response.status(400).json({ error: 'Cannot demote yourself' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { diff --git a/src/endpoints/users-private.js b/src/endpoints/users-private.js index c87a1a2b2..65e9a22f3 100644 --- a/src/endpoints/users-private.js +++ b/src/endpoints/users-private.js @@ -72,7 +72,7 @@ router.post('/change-avatar', jsonParser, async (request, response) => { return response.status(400).json({ error: 'Invalid data URL' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -101,7 +101,7 @@ router.post('/change-password', jsonParser, async (request, response) => { return response.status(403).json({ error: 'Unauthorized' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -189,7 +189,7 @@ router.post('/change-name', jsonParser, async (request, response) => { return response.status(403).json({ error: 'Unauthorized' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { diff --git a/src/endpoints/users-public.js b/src/endpoints/users-public.js index 40dc479ad..618b222b2 100644 --- a/src/endpoints/users-public.js +++ b/src/endpoints/users-public.js @@ -26,10 +26,10 @@ router.post('/list', async (_request, response) => { return response.sendStatus(204); } - /** @type {import('../users').User[]} */ + /** @type {import('../users.js').User[]} */ const users = await storage.values(x => x.key.startsWith(KEY_PREFIX)); - /** @type {Promise[]} */ + /** @type {Promise[]} */ const viewModelPromises = users .filter(x => x.enabled) .map(user => new Promise(async (resolve) => { @@ -63,7 +63,7 @@ router.post('/login', jsonParser, async (request, response) => { const ip = getIpFromRequest(request); await loginLimiter.consume(ip); - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -111,7 +111,7 @@ router.post('/recover-step1', jsonParser, async (request, response) => { const ip = getIpFromRequest(request); await recoverLimiter.consume(ip); - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); if (!user) { @@ -148,7 +148,7 @@ router.post('/recover-step2', jsonParser, async (request, response) => { return response.status(400).json({ error: 'Missing required fields' }); } - /** @type {import('../users').User} */ + /** @type {import('../users.js').User} */ const user = await storage.getItem(toKey(request.body.handle)); const ip = getIpFromRequest(request); diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index 619244256..386b804f6 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -39,7 +39,7 @@ const SOURCES = [ * @param {Object} sourceSettings - Settings for the source, if it needs any * @param {string} text - The text to get the vector for * @param {boolean} isQuery - If the text is a query for embedding search - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ async function getVector(source, sourceSettings, text, isQuery, directories) { @@ -75,7 +75,7 @@ async function getVector(source, sourceSettings, text, isQuery, directories) { * @param {Object} sourceSettings - Settings for the source, if it needs any * @param {string[]} texts - The array of texts to get the vector for * @param {boolean} isQuery - If the text is a query for embedding search - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ async function getBatchVector(source, sourceSettings, texts, isQuery, directories) { @@ -195,7 +195,7 @@ function getModelScope(sourceSettings) { /** * Gets the index for the vector collection - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector * @param {object} sourceSettings - The model for the source @@ -215,7 +215,7 @@ async function getIndex(directories, collectionId, source, sourceSettings) { /** * Inserts items into the vector collection - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector * @param {Object} sourceSettings - Settings for the source, if it needs any @@ -239,7 +239,7 @@ async function insertVectorItems(directories, collectionId, source, sourceSettin /** * Gets the hashes of the items in the vector collection - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector * @param {Object} sourceSettings - Settings for the source, if it needs any @@ -256,7 +256,7 @@ async function getSavedHashes(directories, collectionId, source, sourceSettings) /** * Deletes items from the vector collection by hash - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector * @param {Object} sourceSettings - Settings for the source, if it needs any @@ -277,7 +277,7 @@ async function deleteVectorItems(directories, collectionId, source, sourceSettin /** * Gets the hashes of the items in the vector collection that match the search text - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector * @param {Object} sourceSettings - Settings for the source, if it needs any @@ -298,7 +298,7 @@ async function queryCollection(directories, collectionId, source, sourceSettings /** * Queries multiple collections for the given search queries. Returns the overall top K results. - * @param {import('../users').UserDirectoryList} directories - User directories + * @param {import('../users.js').UserDirectoryList} directories - User directories * @param {string[]} collectionIds - The collection IDs to query * @param {string} source - The source of the vector * @param {Object} sourceSettings - Settings for the source, if it needs any diff --git a/src/endpoints/worldinfo.js b/src/endpoints/worldinfo.js index 37c05d3d8..3fe2593c6 100644 --- a/src/endpoints/worldinfo.js +++ b/src/endpoints/worldinfo.js @@ -9,7 +9,7 @@ import { jsonParser, urlencodedParser } from '../express-common.js'; /** * Reads a World Info file and returns its contents - * @param {import('../users').UserDirectoryList} directories User directories + * @param {import('../users.js').UserDirectoryList} directories User directories * @param {string} worldInfoName Name of the World Info file * @param {boolean} allowDummy If true, returns an empty object if the file doesn't exist * @returns {object} World Info file contents diff --git a/src/middleware/basicAuth.js b/src/middleware/basicAuth.js index 0ab7af343..87b7fbcf8 100644 --- a/src/middleware/basicAuth.js +++ b/src/middleware/basicAuth.js @@ -2,6 +2,7 @@ * When applied, this middleware will ensure the request contains the required header for basic authentication and only * allow access to the endpoint after successful authentication. */ +import { Buffer } from 'node:buffer'; import storage from 'node-persist'; import { getAllUserHandles, toKey, getPasswordHash } from '../users.js'; import { getConfig, getConfigValue } from '../util.js'; diff --git a/src/middleware/multerMonkeyPatch.js b/src/middleware/multerMonkeyPatch.js index ed359a704..0ea456ec8 100644 --- a/src/middleware/multerMonkeyPatch.js +++ b/src/middleware/multerMonkeyPatch.js @@ -1,3 +1,5 @@ +import { Buffer } from 'node:buffer'; + /** * Decodes a file name from Latin1 to UTF-8. * @param {string} str Input string diff --git a/src/transformers.mjs b/src/transformers.mjs index 6669bf00f..b23ed984b 100644 --- a/src/transformers.mjs +++ b/src/transformers.mjs @@ -1,5 +1,6 @@ import path from 'node:path'; import fs from 'node:fs'; +import { Buffer } from 'node:buffer'; import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers'; import { getConfigValue } from './util.js'; diff --git a/src/users.js b/src/users.js index f5c8099c7..41632931b 100644 --- a/src/users.js +++ b/src/users.js @@ -3,6 +3,7 @@ import * as path from 'node:path'; import * as fs from 'node:fs'; import * as crypto from 'node:crypto'; import * as os from 'node:os'; +import { Buffer } from 'node:buffer'; // Express and other dependencies import storage from 'node-persist'; @@ -93,7 +94,7 @@ const STORAGE_KEYS = { /** * Ensures that the content directories exist. - * @returns {Promise} - The list of user directories + * @returns {Promise} - The list of user directories */ export async function ensurePublicDirectoriesExist() { for (const dir of Object.values(PUBLIC_DIRECTORIES)) { @@ -116,7 +117,7 @@ export async function ensurePublicDirectoriesExist() { /** * Gets a list of all user directories. - * @returns {Promise} - The list of user directories + * @returns {Promise} - The list of user directories */ export async function getUserDirectoriesList() { const userHandles = await getAllUserHandles(); diff --git a/src/util.js b/src/util.js index 3601a3f0e..210056b66 100644 --- a/src/util.js +++ b/src/util.js @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as http2 from 'node:http2'; import { Readable } from 'node:stream'; import { createRequire } from 'node:module'; +import { Buffer } from 'node:buffer'; import yaml from 'yaml'; import { sync as commandExistsSync } from 'command-exists'; diff --git a/src/vectors/cohere-vectors.js b/src/vectors/cohere-vectors.js index 1eea1658d..7a940ac23 100644 --- a/src/vectors/cohere-vectors.js +++ b/src/vectors/cohere-vectors.js @@ -5,7 +5,7 @@ import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js'; * Gets the vector for the given text batch from an OpenAI compatible endpoint. * @param {string[]} texts - The array of texts to get the vector for * @param {boolean} isQuery - If the text is a query for embedding search - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @param {string} model - The model to use for the embedding * @returns {Promise} - The array of vectors for the texts */ @@ -51,7 +51,7 @@ export async function getCohereBatchVector(texts, isQuery, directories, model) { * Gets the vector for the given text from an OpenAI compatible endpoint. * @param {string} text - The text to get the vector for * @param {boolean} isQuery - If the text is a query for embedding search - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @param {string} model - The model to use for the embedding * @returns {Promise} - The vector for the text */ diff --git a/src/vectors/llamacpp-vectors.js b/src/vectors/llamacpp-vectors.js index 8efa21d03..0af91336e 100644 --- a/src/vectors/llamacpp-vectors.js +++ b/src/vectors/llamacpp-vectors.js @@ -6,7 +6,7 @@ import { TEXTGEN_TYPES } from '../constants.js'; * Gets the vector for the given text from LlamaCpp * @param {string[]} texts - The array of texts to get the vectors for * @param {string} apiUrl - The API URL - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ export async function getLlamaCppBatchVector(texts, apiUrl, directories) { @@ -47,7 +47,7 @@ export async function getLlamaCppBatchVector(texts, apiUrl, directories) { * Gets the vector for the given text from LlamaCpp * @param {string} text - The text to get the vector for * @param {string} apiUrl - The API URL - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ export async function getLlamaCppVector(text, apiUrl, directories) { diff --git a/src/vectors/makersuite-vectors.js b/src/vectors/makersuite-vectors.js index 1fbffb3a6..48c8fd056 100644 --- a/src/vectors/makersuite-vectors.js +++ b/src/vectors/makersuite-vectors.js @@ -5,7 +5,7 @@ const API_MAKERSUITE = 'https://generativelanguage.googleapis.com'; /** * Gets the vector for the given text from gecko model * @param {string[]} texts - The array of texts to get the vector for - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ export async function getMakerSuiteBatchVector(texts, directories) { @@ -16,7 +16,7 @@ export async function getMakerSuiteBatchVector(texts, directories) { /** * Gets the vector for the given text from Gemini API text-embedding-004 model * @param {string} text - The text to get the vector for - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ export async function getMakerSuiteVector(text, directories) { diff --git a/src/vectors/nomicai-vectors.js b/src/vectors/nomicai-vectors.js index ef565108e..2f832956a 100644 --- a/src/vectors/nomicai-vectors.js +++ b/src/vectors/nomicai-vectors.js @@ -13,7 +13,7 @@ const SOURCES = { * Gets the vector for the given text batch from an OpenAI compatible endpoint. * @param {string[]} texts - The array of texts to get the vector for * @param {string} source - The source of the vector - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ export async function getNomicAIBatchVector(texts, source, directories) { @@ -64,7 +64,7 @@ export async function getNomicAIBatchVector(texts, source, directories) { * Gets the vector for the given text from an OpenAI compatible endpoint. * @param {string} text - The text to get the vector for * @param {string} source - The source of the vector - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ export async function getNomicAIVector(text, source, directories) { diff --git a/src/vectors/ollama-vectors.js b/src/vectors/ollama-vectors.js index 5c654ba6f..237d6979d 100644 --- a/src/vectors/ollama-vectors.js +++ b/src/vectors/ollama-vectors.js @@ -8,7 +8,7 @@ import { TEXTGEN_TYPES } from '../constants.js'; * @param {string} apiUrl - The API URL * @param {string} model - The model to use * @param {boolean} keep - Keep the model loaded in memory - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ export async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) { @@ -26,7 +26,7 @@ export async function getOllamaBatchVector(texts, apiUrl, model, keep, directori * @param {string} apiUrl - The API URL * @param {string} model - The model to use * @param {boolean} keep - Keep the model loaded in memory - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ export async function getOllamaVector(text, apiUrl, model, keep, directories) { diff --git a/src/vectors/openai-vectors.js b/src/vectors/openai-vectors.js index 2d5ff6938..f408e505e 100644 --- a/src/vectors/openai-vectors.js +++ b/src/vectors/openai-vectors.js @@ -23,7 +23,7 @@ const SOURCES = { * Gets the vector for the given text batch from an OpenAI compatible endpoint. * @param {string[]} texts - The array of texts to get the vector for * @param {string} source - The source of the vector - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @param {string} model - The model to use for the embedding * @returns {Promise} - The array of vectors for the texts */ @@ -79,7 +79,7 @@ export async function getOpenAIBatchVector(texts, source, directories, model = ' * Gets the vector for the given text from an OpenAI compatible endpoint. * @param {string} text - The text to get the vector for * @param {string} source - The source of the vector - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @param {string} model - The model to use for the embedding * @returns {Promise} - The vector for the text */ diff --git a/src/vectors/vllm-vectors.js b/src/vectors/vllm-vectors.js index cc5f181bf..e20c85a72 100644 --- a/src/vectors/vllm-vectors.js +++ b/src/vectors/vllm-vectors.js @@ -7,7 +7,7 @@ import { TEXTGEN_TYPES } from '../constants.js'; * @param {string[]} texts - The array of texts to get the vectors for * @param {string} apiUrl - The API URL * @param {string} model - The model to use - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The array of vectors for the texts */ export async function getVllmBatchVector(texts, apiUrl, model, directories) { @@ -49,7 +49,7 @@ export async function getVllmBatchVector(texts, apiUrl, model, directories) { * @param {string} text - The text to get the vector for * @param {string} apiUrl - The API URL * @param {string} model - The model to use - * @param {import('../users').UserDirectoryList} directories - The directories object for the user + * @param {import('../users.js').UserDirectoryList} directories - The directories object for the user * @returns {Promise} - The vector for the text */ export async function getVllmVector(text, apiUrl, model, directories) { From 0355bc7a85584cbd043ef9bf578ac459041408df Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:49:33 +0300 Subject: [PATCH 03/21] Fix globalAgent assignment --- src/request-proxy.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/request-proxy.js b/src/request-proxy.js index 4d490ce22..cae810bec 100644 --- a/src/request-proxy.js +++ b/src/request-proxy.js @@ -1,9 +1,10 @@ -import * as http from 'node:http'; -import * as https from 'node:https'; - +import { createRequire } from 'node:module'; import { ProxyAgent } from 'proxy-agent'; import { isValidUrl, color } from './util.js'; +const require = createRequire(import.meta.url); +const http = require('http'); +const https = require('https'); const LOG_HEADER = '[Request Proxy]'; /** From 478976f6479c733d5f43d636901129948981aa90 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:50:51 +0300 Subject: [PATCH 04/21] Fix open autorun import --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index ed1ce94b7..3569f59fe 100644 --- a/server.js +++ b/server.js @@ -10,7 +10,7 @@ import * as net from 'node:net'; import * as dns from 'node:dns'; // cli/fs related library imports -import * as open from 'open'; +import open from 'open'; import yargs from 'yargs/yargs'; import { hideBin } from 'yargs/helpers'; From c67c0cbf7e8acf06af1b668b7080d3a2e44920ae Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:06:02 +0300 Subject: [PATCH 05/21] Fix dirname for Node 18 --- plugins.js | 6 ++++-- server.js | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins.js b/plugins.js index 3ac5fd63b..107ea4baf 100644 --- a/plugins.js +++ b/plugins.js @@ -3,12 +3,14 @@ // 1. node plugins.js update // 2. node plugins.js install // More operations coming soon. -import { default as git } from 'simple-git'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { default as git } from 'simple-git'; import { color } from './src/util.js'; -const __dirname = import.meta.dirname; +const __dirname = import.meta.dirname ?? path.dirname(fileURLToPath(import.meta.url)); process.chdir(__dirname); const pluginsPath = './plugins'; diff --git a/server.js b/server.js index 3569f59fe..3c1994a7b 100644 --- a/server.js +++ b/server.js @@ -8,6 +8,7 @@ import * as path from 'node:path'; import * as util from 'node:util'; import * as net from 'node:net'; import * as dns from 'node:dns'; +import { fileURLToPath } from 'node:url'; // cli/fs related library imports import open from 'open'; @@ -214,7 +215,7 @@ const cliArguments = yargs(hideBin(process.argv)) }).parseSync(); // change all relative paths -const serverDirectory = import.meta.dirname; +const serverDirectory = import.meta.dirname ?? path.dirname(fileURLToPath(import.meta.url)); console.log(`Node version: ${process.version}. Running in ${process.env.NODE_ENV} environment. Server directory: ${serverDirectory}`); process.chdir(serverDirectory); From 8afa887d07a221e19b807b015bbb1c81a93a4182 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:07:00 +0300 Subject: [PATCH 06/21] Fix relative import in recover.js --- recover.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recover.js b/recover.js index 14379017a..531c77bb4 100644 --- a/recover.js +++ b/recover.js @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import yaml from 'yaml'; import storage from 'node-persist'; -import * as users from './src/users'; +import * as users from './src/users.js'; const userAccount = process.argv[2]; const userPassword = process.argv[3]; From 4bafacaec461df5ac9c67aa42a9af259f82f2437 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:08:53 +0300 Subject: [PATCH 07/21] Add usage for plugins.js --- plugins.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins.js b/plugins.js index 107ea4baf..b881df336 100644 --- a/plugins.js +++ b/plugins.js @@ -16,6 +16,14 @@ const pluginsPath = './plugins'; const command = process.argv[2]; +if (!command) { + console.log('Usage: node plugins.js '); + console.log('Commands:'); + console.log(' update - Update all installed plugins'); + console.log(' install - Install plugin from a Git URL'); + process.exit(1); +} + if (command === 'update') { console.log(color.magenta('Updating all plugins')); updatePlugins(); From a0e6030836d18bd17c66d302cbb0af43c453988c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:28:17 +0300 Subject: [PATCH 08/21] Simplify node imports --- plugins.js | 4 ++-- post-install.js | 8 ++++---- recover.js | 2 +- server.js | 15 +++++++-------- src/endpoints/assets.js | 4 ++-- src/endpoints/avatars.js | 4 ++-- src/endpoints/backends/kobold.js | 2 +- src/endpoints/backgrounds.js | 4 ++-- src/endpoints/characters.js | 6 +++--- src/endpoints/chats.js | 6 +++--- src/endpoints/content-manager.js | 4 ++-- src/endpoints/extensions.js | 4 ++-- src/endpoints/files.js | 4 ++-- src/endpoints/groups.js | 4 ++-- src/endpoints/images.js | 4 ++-- src/endpoints/moving-ui.js | 2 +- src/endpoints/novelai.js | 2 +- src/endpoints/openai.js | 2 +- src/endpoints/presets.js | 4 ++-- src/endpoints/quick-replies.js | 4 ++-- src/endpoints/secrets.js | 4 ++-- src/endpoints/settings.js | 4 ++-- src/endpoints/sprites.js | 4 ++-- src/endpoints/stable-diffusion.js | 4 ++-- src/endpoints/stats.js | 6 +++--- src/endpoints/themes.js | 4 ++-- src/endpoints/thumbnails.js | 6 +++--- src/endpoints/tokenizers.js | 4 ++-- src/endpoints/translate.js | 2 +- src/endpoints/users-private.js | 4 ++-- src/endpoints/users-public.js | 2 +- src/endpoints/vectors.js | 4 ++-- src/endpoints/worldinfo.js | 4 ++-- src/middleware/whitelist.js | 4 ++-- src/plugin-loader.js | 6 +++--- src/prompt-converters.js | 2 +- src/users.js | 8 ++++---- src/util.js | 6 +++--- 38 files changed, 83 insertions(+), 84 deletions(-) diff --git a/plugins.js b/plugins.js index b881df336..f55de2ccb 100644 --- a/plugins.js +++ b/plugins.js @@ -3,8 +3,8 @@ // 1. node plugins.js update // 2. node plugins.js install // More operations coming soon. -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { default as git } from 'simple-git'; diff --git a/post-install.js b/post-install.js index 08e8a0573..962927c22 100644 --- a/post-install.js +++ b/post-install.js @@ -1,10 +1,10 @@ /** * Scripts to be done before starting the server for the first time. */ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import * as crypto from 'node:crypto'; -import * as yaml from 'yaml'; +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import yaml from 'yaml'; import _ from 'lodash'; import { createRequire } from 'node:module'; diff --git a/recover.js b/recover.js index 531c77bb4..176983718 100644 --- a/recover.js +++ b/recover.js @@ -1,4 +1,4 @@ -import * as fs from 'node:fs'; +import fs from 'node:fs'; import yaml from 'yaml'; import storage from 'node-persist'; import * as users from './src/users.js'; diff --git a/server.js b/server.js index 3c1994a7b..7f53fe534 100644 --- a/server.js +++ b/server.js @@ -1,13 +1,13 @@ #!/usr/bin/env node // native node modules -import * as fs from 'node:fs'; -import * as http from 'node:http'; -import * as https from 'node:https'; -import * as path from 'node:path'; -import * as util from 'node:util'; -import * as net from 'node:net'; -import * as dns from 'node:dns'; +import fs from 'node:fs'; +import http from 'node:http'; +import https from 'node:https'; +import path from 'node:path'; +import util from 'node:util'; +import net from 'node:net'; +import dns from 'node:dns'; import { fileURLToPath } from 'node:url'; // cli/fs related library imports @@ -440,7 +440,6 @@ app.use(multerMonkeyPatch); // User data mount app.use('/', userModule.router); // Private endpoints - app.use('/api/users', usersPrivateRouter); // Admin endpoints app.use('/api/users', usersAdminRouter); diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 7379aceba..a9d7fd201 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import { finished } from 'node:stream/promises'; import mime from 'mime-types'; diff --git a/src/endpoints/avatars.js b/src/endpoints/avatars.js index d16eb2969..73b995ffb 100644 --- a/src/endpoints/avatars.js +++ b/src/endpoints/avatars.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/backends/kobold.js b/src/endpoints/backends/kobold.js index 990e0085d..e2fa630ff 100644 --- a/src/endpoints/backends/kobold.js +++ b/src/endpoints/backends/kobold.js @@ -1,4 +1,4 @@ -import * as fs from 'node:fs'; +import fs from 'node:fs'; import express from 'express'; import fetch from 'node-fetch'; diff --git a/src/endpoints/backgrounds.js b/src/endpoints/backgrounds.js index 4ee1f7c1c..0638415e6 100644 --- a/src/endpoints/backgrounds.js +++ b/src/endpoints/backgrounds.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 20a947ef8..8504a3462 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -1,7 +1,7 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import { promises as fsPromises } from 'node:fs'; -import * as readline from 'node:readline'; +import readline from 'node:readline'; import { Buffer } from 'node:buffer'; import express from 'express'; diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js index cc23f9d3e..e1b6fc097 100644 --- a/src/endpoints/chats.js +++ b/src/endpoints/chats.js @@ -1,6 +1,6 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import * as readline from 'node:readline'; +import fs from 'node:fs'; +import path from 'node:path'; +import readline from 'node:readline'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index d587c2a34..542611202 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import { Buffer } from 'node:buffer'; import express from 'express'; diff --git a/src/endpoints/extensions.js b/src/endpoints/extensions.js index 1c58b4e8c..ecc2f1d23 100644 --- a/src/endpoints/extensions.js +++ b/src/endpoints/extensions.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/files.js b/src/endpoints/files.js index e92a892d5..dd34a4ab6 100644 --- a/src/endpoints/files.js +++ b/src/endpoints/files.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/groups.js b/src/endpoints/groups.js index 0c1c55131..fb2e51d75 100644 --- a/src/endpoints/groups.js +++ b/src/endpoints/groups.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/images.js b/src/endpoints/images.js index cf52c6a76..eeacb90fc 100644 --- a/src/endpoints/images.js +++ b/src/endpoints/images.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import { Buffer } from 'node:buffer'; import express from 'express'; diff --git a/src/endpoints/moving-ui.js b/src/endpoints/moving-ui.js index 0b82ea919..484dfbba8 100644 --- a/src/endpoints/moving-ui.js +++ b/src/endpoints/moving-ui.js @@ -1,4 +1,4 @@ -import * as path from 'node:path'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; diff --git a/src/endpoints/novelai.js b/src/endpoints/novelai.js index 19046a2a5..a4b770308 100644 --- a/src/endpoints/novelai.js +++ b/src/endpoints/novelai.js @@ -1,4 +1,4 @@ -import * as util from 'node:util'; +import util from 'node:util'; import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; diff --git a/src/endpoints/openai.js b/src/endpoints/openai.js index 55b2d7c11..08d138e52 100644 --- a/src/endpoints/openai.js +++ b/src/endpoints/openai.js @@ -1,4 +1,4 @@ -import * as fs from 'node:fs'; +import fs from 'node:fs'; import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; diff --git a/src/endpoints/presets.js b/src/endpoints/presets.js index 36eb53b38..fc7f72cb3 100644 --- a/src/endpoints/presets.js +++ b/src/endpoints/presets.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/quick-replies.js b/src/endpoints/quick-replies.js index 45df14ce7..f2f05852c 100644 --- a/src/endpoints/quick-replies.js +++ b/src/endpoints/quick-replies.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 51cbcf2fe..078792cb6 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; diff --git a/src/endpoints/settings.js b/src/endpoints/settings.js index 8e9e052db..0c4a7cf28 100644 --- a/src/endpoints/settings.js +++ b/src/endpoints/settings.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import _ from 'lodash'; diff --git a/src/endpoints/sprites.js b/src/endpoints/sprites.js index e1ce6024d..733b12478 100644 --- a/src/endpoints/sprites.js +++ b/src/endpoints/sprites.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import mime from 'mime-types'; diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 4be405c58..75cec8c91 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import fetch from 'node-fetch'; diff --git a/src/endpoints/stats.js b/src/endpoints/stats.js index a7478f7c2..dbde6cb17 100644 --- a/src/endpoints/stats.js +++ b/src/endpoints/stats.js @@ -1,6 +1,6 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import * as crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; import express from 'express'; import writeFileAtomic from 'write-file-atomic'; diff --git a/src/endpoints/themes.js b/src/endpoints/themes.js index 9edca31be..7d8145162 100644 --- a/src/endpoints/themes.js +++ b/src/endpoints/themes.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/thumbnails.js b/src/endpoints/thumbnails.js index 9f1812b72..275c64f07 100644 --- a/src/endpoints/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -1,6 +1,6 @@ -import * as fs from 'node:fs'; -import * as fsPromises from 'node:fs/promises'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import { promises as fsPromises } from 'node:fs'; +import path from 'node:path'; import mime from 'mime-types'; import express from 'express'; diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index 719d2219c..c3766f8e8 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import { Buffer } from 'node:buffer'; import express from 'express'; diff --git a/src/endpoints/translate.js b/src/endpoints/translate.js index e254506cb..67f98ddc8 100644 --- a/src/endpoints/translate.js +++ b/src/endpoints/translate.js @@ -1,4 +1,4 @@ -import * as https from 'node:https'; +import https from 'node:https'; import { createRequire } from 'node:module'; import { Buffer } from 'node:buffer'; diff --git a/src/endpoints/users-private.js b/src/endpoints/users-private.js index 65e9a22f3..10051c5f5 100644 --- a/src/endpoints/users-private.js +++ b/src/endpoints/users-private.js @@ -1,6 +1,6 @@ -import * as path from 'node:path'; +import path from 'node:path'; import { promises as fsPromises } from 'node:fs'; -import * as crypto from 'node:crypto'; +import crypto from 'node:crypto'; import storage from 'node-persist'; import express from 'express'; diff --git a/src/endpoints/users-public.js b/src/endpoints/users-public.js index 618b222b2..5e6b60941 100644 --- a/src/endpoints/users-public.js +++ b/src/endpoints/users-public.js @@ -1,4 +1,4 @@ -import * as crypto from 'node:crypto'; +import crypto from 'node:crypto'; import storage from 'node-persist'; import express from 'express'; diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index 386b804f6..325684601 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import vectra from 'vectra'; import express from 'express'; diff --git a/src/endpoints/worldinfo.js b/src/endpoints/worldinfo.js index 3fe2593c6..e0b3bd269 100644 --- a/src/endpoints/worldinfo.js +++ b/src/endpoints/worldinfo.js @@ -1,5 +1,5 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; +import fs from 'node:fs'; +import path from 'node:path'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index eda161bc9..d3a2c117b 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -1,5 +1,5 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; +import path from 'node:path'; +import fs from 'node:fs'; import ipMatching from 'ip-matching'; import { getIpFromRequest } from '../express-common.js'; diff --git a/src/plugin-loader.js b/src/plugin-loader.js index c5c7e2995..433d9f411 100644 --- a/src/plugin-loader.js +++ b/src/plugin-loader.js @@ -1,6 +1,6 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import * as url from 'node:url'; +import fs from 'node:fs'; +import path from 'node:path'; +import url from 'node:url'; import express from 'express'; import { getConfigValue } from './util.js'; diff --git a/src/prompt-converters.js b/src/prompt-converters.js index bfc08ad68..bdb4c8233 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -1,4 +1,4 @@ -import * as crypto from 'node:crypto'; +import crypto from 'node:crypto'; import polyfill from './polyfill.js'; import { getConfigValue } from './util.js'; diff --git a/src/users.js b/src/users.js index 41632931b..e43025816 100644 --- a/src/users.js +++ b/src/users.js @@ -1,8 +1,8 @@ // Native Node Modules -import * as path from 'node:path'; -import * as fs from 'node:fs'; -import * as crypto from 'node:crypto'; -import * as os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import crypto from 'node:crypto'; +import os from 'node:os'; import { Buffer } from 'node:buffer'; // Express and other dependencies diff --git a/src/util.js b/src/util.js index 210056b66..33bbed9bc 100644 --- a/src/util.js +++ b/src/util.js @@ -1,6 +1,6 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs'; -import * as http2 from 'node:http2'; +import path from 'node:path'; +import fs from 'node:fs'; +import http2 from 'node:http2'; import { Readable } from 'node:stream'; import { createRequire } from 'node:module'; import { Buffer } from 'node:buffer'; From 0c8ccf176511a33fb84e1528af57e17027fa733c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:41:08 +0300 Subject: [PATCH 09/21] Update module imports --- recover.js | 19 ++++++---- server.js | 60 ++++++++++++++++++++------------ src/endpoints/characters.js | 6 ++-- src/endpoints/content-manager.js | 7 ++-- src/endpoints/users-private.js | 5 ++- 5 files changed, 58 insertions(+), 39 deletions(-) diff --git a/recover.js b/recover.js index 176983718..7db25fc70 100644 --- a/recover.js +++ b/recover.js @@ -1,7 +1,12 @@ import fs from 'node:fs'; import yaml from 'yaml'; import storage from 'node-persist'; -import * as users from './src/users.js'; +import { + initUserStorage, + getPasswordSalt, + getPasswordHash, + toKey, +} from './src/users.js'; const userAccount = process.argv[2]; const userPassword = process.argv[3]; @@ -22,7 +27,7 @@ async function initStorage() { process.exit(1); } - await users.initUserStorage(dataRoot); + await initUserStorage(dataRoot); } async function main() { @@ -31,22 +36,22 @@ async function main() { /** * @type {import('./src/users').User} */ - const user = await storage.get(users.toKey(userAccount)); + const user = await storage.get(toKey(userAccount)); if (!user) { console.error(`User "${userAccount}" not found.`); process.exit(1); } - if (!user.enabled) { + if (!user.enabled) { console.log('User is disabled. Enabling...'); user.enabled = true; } if (userPassword) { console.log('Setting new password...'); - const salt = users.getPasswordSalt(); - const passwordHash = users.getPasswordHash(userPassword, salt); + const salt = getPasswordSalt(); + const passwordHash = getPasswordHash(userPassword, salt); user.password = passwordHash; user.salt = salt; } else { @@ -55,7 +60,7 @@ async function main() { user.salt = ''; } - await storage.setItem(users.toKey(userAccount), user); + await storage.setItem(toKey(userAccount), user); console.log('User recovered. A program will exit now.'); } diff --git a/server.js b/server.js index 7f53fe534..7f4e8ca8a 100644 --- a/server.js +++ b/server.js @@ -36,13 +36,27 @@ util.inspect.defaultOptions.maxStringLength = null; util.inspect.defaultOptions.depth = 4; // local library imports -import * as loader from './src/plugin-loader.js'; -import * as userModule from './src/users.js'; +import{ loadPlugins } from './src/plugin-loader.js'; +import { + initUserStorage, + getCsrfSecret, + getCookieSecret, + getCookieSessionName, + getAllEnabledUsers, + ensurePublicDirectoriesExist, + getUserDirectoriesList, + migrateSystemPrompts, + migrateUserData, + requireLoginMiddleware, + setUserDataMiddleware, + shouldRedirectToLogin, + tryAutoLogin, + router as userDataRouter, +} from './src/users.js'; import basicAuthMiddleware from './src/middleware/basicAuth.js'; import whitelistMiddleware from './src/middleware/whitelist.js'; import multerMonkeyPatch from './src/middleware/multerMonkeyPatch.js'; import initRequestProxy from './src/request-proxy.js'; -import * as contentManager from './src/endpoints/content-manager.js'; import { getVersion, getConfigValue, @@ -81,7 +95,7 @@ import { router as worldInfoRouter } from './src/endpoints/worldinfo.js'; import { router as statsRouter, init as statsInit, onExit as statsOnExit } from './src/endpoints/stats.js'; import { router as backgroundsRouter } from './src/endpoints/backgrounds.js'; import { router as spritesRouter } from './src/endpoints/sprites.js'; -import { router as contentManagerRouter } from './src/endpoints/content-manager.js'; +import { router as contentManagerRouter, checkForNewContent } from './src/endpoints/content-manager.js'; import { router as settingsRouter, init as settingsInit } from './src/endpoints/settings.js'; import { router as stableDiffusionRouter } from './src/endpoints/stable-diffusion.js'; import { router as hordeRouter } from './src/endpoints/horde.js'; @@ -349,21 +363,21 @@ function getSessionCookieAge() { } app.use(cookieSession({ - name: userModule.getCookieSessionName(), + name: getCookieSessionName(), sameSite: 'strict', httpOnly: true, maxAge: getSessionCookieAge(), - secret: userModule.getCookieSecret(), + secret: getCookieSecret(), })); -app.use(userModule.setUserDataMiddleware); +app.use(setUserDataMiddleware); // CSRF Protection // if (!disableCsrf) { - const COOKIES_SECRET = userModule.getCookieSecret(); + const COOKIES_SECRET = getCookieSecret(); const { generateToken, doubleCsrfProtection } = doubleCsrf({ - getSecret: userModule.getCsrfSecret, + getSecret: getCsrfSecret, cookieName: 'X-CSRF-Token', cookieOptions: { httpOnly: true, @@ -394,7 +408,7 @@ if (!disableCsrf) { // Static files // Host index page app.get('/', (request, response) => { - if (userModule.shouldRedirectToLogin(request)) { + if (shouldRedirectToLogin(request)) { const query = request.url.split('?')[1]; const redirectUrl = query ? `/login?${query}` : '/login'; return response.redirect(redirectUrl); @@ -411,7 +425,7 @@ app.get('/login', async (request, response) => { } try { - const autoLogin = await userModule.tryAutoLogin(request, basicAuthMode); + const autoLogin = await tryAutoLogin(request, basicAuthMode); if (autoLogin) { return response.redirect('/'); @@ -430,7 +444,7 @@ app.use(express.static(process.cwd() + '/public', {})); app.use('/api/users', usersPublicRouter); // Everything below this line requires authentication -app.use(userModule.requireLoginMiddleware); +app.use(requireLoginMiddleware); app.get('/api/ping', (_, response) => response.sendStatus(204)); // File uploads @@ -438,7 +452,7 @@ app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).s app.use(multerMonkeyPatch); // User data mount -app.use('/', userModule.router); +app.use('/', userDataRouter); // Private endpoints app.use('/api/users', usersPrivateRouter); // Admin endpoints @@ -625,15 +639,15 @@ const preSetupTasks = async function () { } console.log(); - const directories = await userModule.getUserDirectoriesList(); - await contentManager.checkForNewContent(directories); + const directories = await getUserDirectoriesList(); + await checkForNewContent(directories); await ensureThumbnailCache(); cleanUploads(); await settingsInit(); await statsInit(); - const cleanupPlugins = await loadPlugins(); + const cleanupPlugins = await initializePlugins(); const consoleTitle = process.title; let isExiting = false; @@ -740,10 +754,10 @@ const postSetupTasks = async function (v6Failed, v4Failed) { * Loads server plugins from a directory. * @returns {Promise} Function to be run on server exit */ -async function loadPlugins() { +async function initializePlugins() { try { const pluginDirectory = path.join(serverDirectory, 'plugins'); - const cleanupPlugins = await loader.loadPlugins(app, pluginDirectory); + const cleanupPlugins = await loadPlugins(app, pluginDirectory); return cleanupPlugins; } catch { console.log('Plugin loading failed.'); @@ -883,7 +897,7 @@ async function verifySecuritySettings() { logSecurityAlert('Your SillyTavern is currently insecurely open to the public. Enable whitelisting, basic authentication or user accounts.'); } - const users = await userModule.getAllEnabledUsers(); + const users = await getAllEnabledUsers(); const unprotectedUsers = users.filter(x => !x.password); const unprotectedAdminUsers = unprotectedUsers.filter(x => x.admin); @@ -901,10 +915,10 @@ async function verifySecuritySettings() { } // User storage module needs to be initialized before starting the server -userModule.initUserStorage(dataRoot) - .then(userModule.ensurePublicDirectoriesExist) - .then(userModule.migrateUserData) - .then(userModule.migrateSystemPrompts) +initUserStorage(dataRoot) + .then(ensurePublicDirectoriesExist) + .then(migrateUserData) + .then(migrateSystemPrompts) .then(verifySecuritySettings) .then(preSetupTasks) .finally(startServer); diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 8504a3462..7a75d4884 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -16,7 +16,7 @@ import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js'; import { jsonParser, urlencodedParser } from '../express-common.js'; import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } from '../util.js'; import { TavernCardValidator } from '../validator/TavernCardValidator.js'; -import * as characterCardParser from '../character-card-parser.js'; +import { parse, write } from '../character-card-parser.js'; import { readWorldInfoFile } from './worldinfo.js'; import { invalidateThumbnail } from './thumbnails.js'; import { importRisuSprites } from './sprites.js'; @@ -38,7 +38,7 @@ async function readCharacterData(inputFile, inputFormat = 'png') { return characterDataCache.get(cacheKey); } - const result = characterCardParser.parse(inputFile, inputFormat); + const result = parse(inputFile, inputFormat); characterDataCache.set(cacheKey, result); return result; } @@ -77,7 +77,7 @@ async function writeCharacterData(inputFile, data, outputFile, request, crop = u const inputImage = await getInputImage(); // Get the chunks - const outputImage = characterCardParser.write(inputImage, data); + const outputImage = write(inputImage, data); const outputImagePath = path.join(request.user.directories.characters, `${outputFile}.png`); writeFileAtomicSync(outputImagePath, outputImage); diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 542611202..2d54f25a4 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -5,15 +5,16 @@ import { Buffer } from 'node:buffer'; import express from 'express'; import fetch from 'node-fetch'; import sanitize from 'sanitize-filename'; +import { sync as writeFileAtomicSync } from 'write-file-atomic'; import { getConfigValue, color } from '../util.js'; import { jsonParser } from '../express-common.js'; -import { sync as writeFileAtomicSync } from 'write-file-atomic'; +import { write } from '../character-card-parser.js'; + const contentDirectory = path.join(process.cwd(), 'default/content'); const scaffoldDirectory = path.join(process.cwd(), 'default/scaffold'); const contentIndexPath = path.join(contentDirectory, 'index.json'); const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json'); -import * as characterCardParser from '../character-card-parser.js'; const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []); @@ -397,7 +398,7 @@ async function downloadPygmalionCharacter(id) { const avatarResult = await fetch(avatarUrl); const avatarBuffer = await avatarResult.buffer(); - const cardBuffer = characterCardParser.write(avatarBuffer, JSON.stringify(characterData)); + const cardBuffer = write(avatarBuffer, JSON.stringify(characterData)); return { buffer: cardBuffer, diff --git a/src/endpoints/users-private.js b/src/endpoints/users-private.js index 10051c5f5..440079092 100644 --- a/src/endpoints/users-private.js +++ b/src/endpoints/users-private.js @@ -8,9 +8,8 @@ import express from 'express'; import { jsonParser } from '../express-common.js'; import { getUserAvatar, toKey, getPasswordHash, getPasswordSalt, createBackupArchive, ensurePublicDirectoriesExist, toAvatarKey } from '../users.js'; import { SETTINGS_FILE } from '../constants.js'; -import * as contentManager from './content-manager.js'; +import { checkForNewContent, CONTENT_TYPES } from './content-manager.js'; import { color, Cache } from '../util.js'; -import { checkForNewContent } from './content-manager.js'; const RESET_CACHE = new Cache(5 * 60 * 1000); @@ -168,7 +167,7 @@ router.post('/reset-settings', jsonParser, async (request, response) => { const pathToFile = path.join(request.user.directories.root, SETTINGS_FILE); await fsPromises.rm(pathToFile, { force: true }); - await contentManager.checkForNewContent([request.user.directories], [contentManager.CONTENT_TYPES.SETTINGS]); + await checkForNewContent([request.user.directories], [CONTENT_TYPES.SETTINGS]); return response.sendStatus(204); } catch (error) { From 7ca1b2e5326869ede048de34721fb0e331097ac1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:33:09 +0300 Subject: [PATCH 10/21] Remove node polyfill --- src/polyfill.js | 10 ---------- src/prompt-converters.js | 3 --- 2 files changed, 13 deletions(-) delete mode 100644 src/polyfill.js diff --git a/src/polyfill.js b/src/polyfill.js deleted file mode 100644 index 390b4b45a..000000000 --- a/src/polyfill.js +++ /dev/null @@ -1,10 +0,0 @@ -if (!Array.prototype.findLastIndex) { - Array.prototype.findLastIndex = function (callback, thisArg) { - for (let i = this.length - 1; i >= 0; i--) { - if (callback.call(thisArg, this[i], i, this)) return i; - } - return -1; - }; -} - -export default () => {}; diff --git a/src/prompt-converters.js b/src/prompt-converters.js index bdb4c8233..0fd257877 100644 --- a/src/prompt-converters.js +++ b/src/prompt-converters.js @@ -1,9 +1,6 @@ import crypto from 'node:crypto'; -import polyfill from './polyfill.js'; import { getConfigValue } from './util.js'; -polyfill(); - const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.'); /** From a0889a15fd9f24fcea1ea989234412e1292e4cb1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:43:29 +0300 Subject: [PATCH 11/21] Explicitly import node process --- plugins.js | 1 + post-install.js | 1 + recover.js | 1 + server.js | 1 + src/endpoints/backends/chat-completions.js | 1 + src/endpoints/chats.js | 1 + src/endpoints/content-manager.js | 1 + src/middleware/whitelist.js | 1 + src/request-proxy.js | 1 + src/transformers.mjs | 1 + src/users.js | 1 + src/util.js | 1 + 12 files changed, 12 insertions(+) diff --git a/plugins.js b/plugins.js index f55de2ccb..2ca5d9957 100644 --- a/plugins.js +++ b/plugins.js @@ -5,6 +5,7 @@ // More operations coming soon. import fs from 'node:fs'; import path from 'node:path'; +import process from 'node:process'; import { fileURLToPath } from 'node:url'; import { default as git } from 'simple-git'; diff --git a/post-install.js b/post-install.js index 962927c22..fef8cb3c6 100644 --- a/post-install.js +++ b/post-install.js @@ -4,6 +4,7 @@ import fs from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; +import process from 'node:process'; import yaml from 'yaml'; import _ from 'lodash'; import { createRequire } from 'node:module'; diff --git a/recover.js b/recover.js index 7db25fc70..a017bf9a3 100644 --- a/recover.js +++ b/recover.js @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import process from 'node:process'; import yaml from 'yaml'; import storage from 'node-persist'; import { diff --git a/server.js b/server.js index 7f4e8ca8a..42d6c73d2 100644 --- a/server.js +++ b/server.js @@ -8,6 +8,7 @@ import path from 'node:path'; import util from 'node:util'; import net from 'node:net'; import dns from 'node:dns'; +import process from 'node:process'; import { fileURLToPath } from 'node:url'; // cli/fs related library imports diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 7e1c5e6fe..6bf6d9e1b 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -1,3 +1,4 @@ +import process from 'node:process'; import express from 'express'; import fetch from 'node-fetch'; diff --git a/src/endpoints/chats.js b/src/endpoints/chats.js index e1b6fc097..f7b9ba767 100644 --- a/src/endpoints/chats.js +++ b/src/endpoints/chats.js @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import readline from 'node:readline'; +import process from 'node:process'; import express from 'express'; import sanitize from 'sanitize-filename'; diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 2d54f25a4..88c2c6cd2 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -1,5 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; +import process from 'node:process'; import { Buffer } from 'node:buffer'; import express from 'express'; diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index d3a2c117b..9864e19ae 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -1,5 +1,6 @@ import path from 'node:path'; import fs from 'node:fs'; +import process from 'node:process'; import ipMatching from 'ip-matching'; import { getIpFromRequest } from '../express-common.js'; diff --git a/src/request-proxy.js b/src/request-proxy.js index cae810bec..312d9376e 100644 --- a/src/request-proxy.js +++ b/src/request-proxy.js @@ -1,3 +1,4 @@ +import process from 'node:process'; import { createRequire } from 'node:module'; import { ProxyAgent } from 'proxy-agent'; import { isValidUrl, color } from './util.js'; diff --git a/src/transformers.mjs b/src/transformers.mjs index b23ed984b..416f2031f 100644 --- a/src/transformers.mjs +++ b/src/transformers.mjs @@ -1,5 +1,6 @@ import path from 'node:path'; import fs from 'node:fs'; +import process from 'node:process'; import { Buffer } from 'node:buffer'; import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers'; diff --git a/src/users.js b/src/users.js index e43025816..cf221c811 100644 --- a/src/users.js +++ b/src/users.js @@ -3,6 +3,7 @@ import path from 'node:path'; import fs from 'node:fs'; import crypto from 'node:crypto'; import os from 'node:os'; +import process from 'node:process'; import { Buffer } from 'node:buffer'; // Express and other dependencies diff --git a/src/util.js b/src/util.js index 33bbed9bc..f966739d9 100644 --- a/src/util.js +++ b/src/util.js @@ -1,6 +1,7 @@ import path from 'node:path'; import fs from 'node:fs'; import http2 from 'node:http2'; +import process from 'node:process'; import { Readable } from 'node:stream'; import { createRequire } from 'node:module'; import { Buffer } from 'node:buffer'; From 5ff58e20c178746bfb79cd858cff2d9ff483b7a8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 08:43:01 +0000 Subject: [PATCH 12/21] Plugins: Fix classic commonjs single file modules --- plugins/package.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 plugins/package.json diff --git a/plugins/package.json b/plugins/package.json new file mode 100644 index 000000000..ba698398c --- /dev/null +++ b/plugins/package.json @@ -0,0 +1,4 @@ +{ + "name": "sillytavern-plugins", + "type": "commonjs" +} From afc3bfb740bb406f1220fa8866aee8fb6b3149be Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:33:55 +0300 Subject: [PATCH 13/21] Install correct version of express types --- package-lock.json | 24 ++++++++++-------------- package.json | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca8352b6b..bc729cf22 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,7 +66,7 @@ "@types/cookie-session": "^2.0.49", "@types/cors": "^2.8.17", "@types/dompurify": "^3.0.5", - "@types/express": "^5.0.0", + "@types/express": "^4.17.21", "@types/jquery": "^3.5.29", "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", @@ -1070,24 +1070,22 @@ } }, "node_modules/@types/express": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", - "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^5.0.0", + "@types/express-serve-static-core": "^4.17.33", "@types/qs": "*", "@types/serve-static": "*" } }, "node_modules/@types/express-serve-static-core": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", - "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "version": "4.19.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", "dev": true, - "license": "MIT", "dependencies": { "@types/node": "*", "@types/qs": "*", @@ -1222,15 +1220,13 @@ "version": "6.9.16", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", - "dev": true, - "license": "MIT" + "dev": true }, "node_modules/@types/range-parser": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "dev": true, - "license": "MIT" + "dev": true }, "node_modules/@types/readdir-glob": { "version": "1.1.5", diff --git a/package.json b/package.json index 14b032049..22a6f3526 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "@types/cookie-session": "^2.0.49", "@types/cors": "^2.8.17", "@types/dompurify": "^3.0.5", - "@types/express": "^5.0.0", + "@types/express": "^4.17.21", "@types/jquery": "^3.5.29", "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", From ab2427142c4f05deff61bbc5cca0fc699a099681 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:34:13 +0300 Subject: [PATCH 14/21] [chore] Fix type errors in server.js --- server.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index 42d6c73d2..9e97297a0 100644 --- a/server.js +++ b/server.js @@ -381,7 +381,6 @@ if (!disableCsrf) { getSecret: getCsrfSecret, cookieName: 'X-CSRF-Token', cookieOptions: { - httpOnly: true, sameSite: 'strict', secure: false, }, @@ -831,7 +830,7 @@ function createHttpsServer(url) { }, app); server.on('error', reject); server.on('listening', resolve); - server.listen(url.port || 443, url.hostname); + server.listen(Number(url.port || 443), url.hostname); }); } @@ -846,7 +845,7 @@ function createHttpServer(url) { const server = http.createServer(app); server.on('error', reject); server.on('listening', resolve); - server.listen(url.port || 80, url.hostname); + server.listen(Number(url.port || 80), url.hostname); }); } From bee19908e7e298e94d4cc7f8ac630498ed188bf1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:38:19 +0300 Subject: [PATCH 15/21] [chore] Fix type error in post-install --- post-install.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-install.js b/post-install.js index fef8cb3c6..d2c1a8687 100644 --- a/post-install.js +++ b/post-install.js @@ -135,7 +135,7 @@ function createDefaultFiles() { function getMd5Hash(data) { return crypto .createHash('md5') - .update(data) + .update(new Uint8Array(data)) .digest('hex'); } From 72138c632d736a911be5ce700a9d1e1c8ba9b9f1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:58:06 +0300 Subject: [PATCH 16/21] [chore] Remove unused import --- src/endpoints/stable-diffusion.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 457fffcc3..811020f9a 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -7,7 +7,7 @@ import sanitize from 'sanitize-filename'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; import FormData from 'form-data'; -import { getBasicAuthHeader, delay, getHexString } from '../util.js'; +import { getBasicAuthHeader, delay } from '../util.js'; import { jsonParser } from '../express-common.js'; import { readSecret, SECRET_KEYS } from './secrets.js'; From 4fcad0752f6e03d4796cda9838f96604298e02e9 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:33:36 +0300 Subject: [PATCH 17/21] [chore] Fix type errors --- package-lock.json | 1 - package.json | 1 - src/character-card-parser.js | 4 ++-- src/endpoints/anthropic.js | 2 +- src/endpoints/backends/chat-completions.js | 9 ++++---- src/endpoints/backends/kobold.js | 3 ++- src/endpoints/backends/scale-alt.js | 2 +- src/endpoints/backends/text-completions.js | 17 +++++++++----- src/endpoints/content-manager.js | 2 ++ src/endpoints/google.js | 2 +- src/endpoints/images.js | 2 +- src/endpoints/novelai.js | 6 ++--- src/endpoints/openai.js | 3 +-- src/endpoints/stable-diffusion.js | 26 +++++++++++++++++----- src/endpoints/translate.js | 9 ++++---- src/transformers.mjs | 5 +++-- src/util.js | 20 ++++++++++------- src/vectors/cohere-vectors.js | 1 + src/vectors/extras-vectors.js | 1 + src/vectors/llamacpp-vectors.js | 1 + src/vectors/makersuite-vectors.js | 1 + src/vectors/nomicai-vectors.js | 1 + src/vectors/ollama-vectors.js | 1 + src/vectors/openai-vectors.js | 1 + src/vectors/vllm-vectors.js | 1 + 25 files changed, 78 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index bc729cf22..cdbaa662a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,7 +71,6 @@ "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", "@types/multer": "^1.4.12", - "@types/node-fetch": "^2.6.11", "@types/node-persist": "^3.1.8", "@types/png-chunk-text": "^1.0.3", "@types/png-chunks-encode": "^1.0.2", diff --git a/package.json b/package.json index 22a6f3526..2e702c499 100644 --- a/package.json +++ b/package.json @@ -97,7 +97,6 @@ "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", "@types/multer": "^1.4.12", - "@types/node-fetch": "^2.6.11", "@types/node-persist": "^3.1.8", "@types/png-chunk-text": "^1.0.3", "@types/png-chunks-encode": "^1.0.2", diff --git a/src/character-card-parser.js b/src/character-card-parser.js index a8433f3b7..a9a414c37 100644 --- a/src/character-card-parser.js +++ b/src/character-card-parser.js @@ -13,7 +13,7 @@ import PNGtext from 'png-chunk-text'; * @returns {Buffer} PNG image buffer with metadata */ export const write = (image, data) => { - const chunks = extract(image); + const chunks = extract(new Uint8Array(image)); const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt'); // Remove existing tEXt chunks @@ -52,7 +52,7 @@ export const write = (image, data) => { * @returns {string} Character data */ export const read = (image) => { - const chunks = extract(image); + const chunks = extract(new Uint8Array(image)); const textChunks = chunks.filter((chunk) => chunk.name === 'tEXt').map((chunk) => PNGtext.decode(chunk.data)); diff --git a/src/endpoints/anthropic.js b/src/endpoints/anthropic.js index ff0b7b6ad..79f3fc5e9 100644 --- a/src/endpoints/anthropic.js +++ b/src/endpoints/anthropic.js @@ -42,7 +42,6 @@ router.post('/caption-image', jsonParser, async (request, response) => { 'anthropic-version': '2023-06-01', 'x-api-key': request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE), }, - timeout: 0, }); if (!result.ok) { @@ -51,6 +50,7 @@ router.post('/caption-image', jsonParser, async (request, response) => { return response.status(result.status).send({ error: true }); } + /** @type {any} */ const generateResponseJson = await result.json(); const caption = generateResponseJson.content[0].text; console.log('Claude response:', generateResponseJson); diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 7bfff462b..0d80ed453 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -152,7 +152,6 @@ async function sendClaudeRequest(request, response) { 'x-api-key': apiKey, ...additionalHeaders, }, - timeout: 0, }); if (request.body.stream) { @@ -165,6 +164,7 @@ async function sendClaudeRequest(request, response) { return response.status(generateResponse.status).send({ error: true }); } + /** @type {any} */ const generateResponseJson = await generateResponse.json(); const responseText = generateResponseJson?.content?.[0]?.text || ''; console.log('Claude response:', generateResponseJson); @@ -212,7 +212,6 @@ async function sendScaleRequest(request, response) { 'Content-Type': 'application/json', 'Authorization': `Basic ${apiKey}`, }, - timeout: 0, }); if (!generateResponse.ok) { @@ -220,6 +219,7 @@ async function sendScaleRequest(request, response) { return response.status(500).send({ error: true }); } + /** @type {any} */ const generateResponseJson = await generateResponse.json(); console.log('Scale response:', generateResponseJson); @@ -335,7 +335,6 @@ async function sendMakerSuiteRequest(request, response) { 'Content-Type': 'application/json', }, signal: controller.signal, - timeout: 0, }); // have to do this because of their busted ass streaming endpoint if (stream) { @@ -354,6 +353,7 @@ async function sendMakerSuiteRequest(request, response) { return response.status(generateResponse.status).send({ error: true }); } + /** @type {any} */ const generateResponseJson = await generateResponse.json(); const candidates = generateResponseJson?.candidates; @@ -676,6 +676,7 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o }); if (response.ok) { + /** @type {any} */ const data = await response.json(); response_getstatus_openai.send(data); @@ -979,7 +980,6 @@ router.post('/generate', jsonParser, function (request, response) { }, body: JSON.stringify(requestBody), signal: controller.signal, - timeout: 0, }; console.log(requestBody); @@ -1005,6 +1005,7 @@ router.post('/generate', jsonParser, function (request, response) { } if (fetchResponse.ok) { + /** @type {any} */ let json = await fetchResponse.json(); response.send(json); console.log(json); diff --git a/src/endpoints/backends/kobold.js b/src/endpoints/backends/kobold.js index e2fa630ff..a6e7fc89e 100644 --- a/src/endpoints/backends/kobold.js +++ b/src/endpoints/backends/kobold.js @@ -96,7 +96,7 @@ router.post('/generate', jsonParser, async function (request, response_generate) for (let i = 0; i < MAX_RETRIES; i++) { try { const url = request.body.streaming ? `${request.body.api_server}/extra/generate/stream` : `${request.body.api_server}/v1/generate`; - const response = await fetch(url, { method: 'POST', timeout: 0, ...args }); + const response = await fetch(url, { method: 'POST', ...args }); if (request.body.streaming) { // Pipe remote SSE stream to Express response @@ -156,6 +156,7 @@ router.post('/status', jsonParser, async function (request, response) { const result = {}; + /** @type {any} */ const [koboldUnitedResponse, koboldExtraResponse, koboldModelResponse] = await Promise.all([ // We catch errors both from the response not having a successful HTTP status and from JSON parsing failing diff --git a/src/endpoints/backends/scale-alt.js b/src/endpoints/backends/scale-alt.js index 0bdbcd336..5e621e17c 100644 --- a/src/endpoints/backends/scale-alt.js +++ b/src/endpoints/backends/scale-alt.js @@ -70,7 +70,6 @@ router.post('/generate', jsonParser, async function (request, response) { 'Content-Type': 'application/json', 'cookie': `_jwt=${cookie}`, }, - timeout: 0, body: JSON.stringify(body), }); @@ -80,6 +79,7 @@ router.post('/generate', jsonParser, async function (request, response) { return response.status(500).send({ error: { message: result.statusText } }); } + /** @type {any} */ const data = await result.json(); const output = data?.result?.data?.json?.outputs?.[0] || ''; diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 6c6942621..7dcea23f1 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -28,6 +28,10 @@ export const router = express.Router(); */ async function parseOllamaStream(jsonStream, request, response) { try { + if (!jsonStream.body) { + throw new Error('No body in the response'); + } + let partialData = ''; jsonStream.body.on('data', (data) => { const chunk = data.toString(); @@ -153,6 +157,7 @@ router.post('/status', jsonParser, async function (request, response) { return response.status(400); } + /** @type {any} */ let data = await modelsReply.json(); if (request.body.legacy_api) { @@ -190,6 +195,7 @@ router.post('/status', jsonParser, async function (request, response) { const modelInfoReply = await fetch(modelInfoUrl, args); if (modelInfoReply.ok) { + /** @type {any} */ const modelInfo = await modelInfoReply.json(); console.log('Ooba model info:', modelInfo); @@ -206,6 +212,7 @@ router.post('/status', jsonParser, async function (request, response) { const modelInfoReply = await fetch(modelInfoUrl, args); if (modelInfoReply.ok) { + /** @type {any} */ const modelInfo = await modelInfoReply.json(); console.log('Tabby model info:', modelInfo); @@ -359,6 +366,7 @@ router.post('/generate', jsonParser, async function (request, response) { const completionsReply = await fetch(url, args); if (completionsReply.ok) { + /** @type {any} */ const data = await completionsReply.json(); console.log('Endpoint response:', data); @@ -415,7 +423,6 @@ ollama.post('/download', jsonParser, async function (request, response) { name: name, stream: false, }), - timeout: 0, }); if (!fetchResponse.ok) { @@ -448,7 +455,6 @@ ollama.post('/caption-image', jsonParser, async function (request, response) { images: [request.body.image], stream: false, }), - timeout: 0, }); if (!fetchResponse.ok) { @@ -456,6 +462,7 @@ ollama.post('/caption-image', jsonParser, async function (request, response) { return response.status(500).send({ error: true }); } + /** @type {any} */ const data = await fetchResponse.json(); console.log('Ollama caption response:', data); @@ -487,7 +494,6 @@ llamacpp.post('/caption-image', jsonParser, async function (request, response) { const fetchResponse = await fetch(`${baseUrl}/completion`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - timeout: 0, body: JSON.stringify({ prompt: `USER:[img-1]${String(request.body.prompt).trim()}\nASSISTANT:`, image_data: [{ data: request.body.image, id: 1 }], @@ -502,6 +508,7 @@ llamacpp.post('/caption-image', jsonParser, async function (request, response) { return response.status(500).send({ error: true }); } + /** @type {any} */ const data = await fetchResponse.json(); console.log('LlamaCpp caption response:', data); @@ -531,7 +538,6 @@ llamacpp.post('/props', jsonParser, async function (request, response) { const fetchResponse = await fetch(`${baseUrl}/props`, { method: 'GET', - timeout: 0, }); if (!fetchResponse.ok) { @@ -566,7 +572,6 @@ llamacpp.post('/slots', jsonParser, async function (request, response) { if (request.body.action === 'info') { fetchResponse = await fetch(`${baseUrl}/slots`, { method: 'GET', - timeout: 0, }); } else { if (!/^\d+$/.test(request.body.id_slot)) { @@ -579,7 +584,6 @@ llamacpp.post('/slots', jsonParser, async function (request, response) { fetchResponse = await fetch(`${baseUrl}/slots/${request.body.id_slot}?action=${request.body.action}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - timeout: 0, body: JSON.stringify({ filename: request.body.action !== 'erase' ? `${request.body.filename}` : undefined, }), @@ -623,6 +627,7 @@ tabby.post('/download', jsonParser, async function (request, response) { }); if (permissionResponse.ok) { + /** @type {any} */ const permissionJson = await permissionResponse.json(); if (permissionJson['permission'] !== 'admin') { diff --git a/src/endpoints/content-manager.js b/src/endpoints/content-manager.js index 88c2c6cd2..90aaee002 100644 --- a/src/endpoints/content-manager.js +++ b/src/endpoints/content-manager.js @@ -380,6 +380,7 @@ async function downloadPygmalionCharacter(id) { throw new Error('Failed to download character'); } + /** @type {any} */ const jsonData = await result.json(); const characterData = jsonData?.character; @@ -472,6 +473,7 @@ async function downloadJannyCharacter(uuid) { }); if (result.ok) { + /** @type {any} */ const downloadResult = await result.json(); if (downloadResult.status === 'ok') { const imageResult = await fetch(downloadResult.downloadUrl); diff --git a/src/endpoints/google.js b/src/endpoints/google.js index 09f66f599..b0ba87260 100644 --- a/src/endpoints/google.js +++ b/src/endpoints/google.js @@ -40,7 +40,6 @@ router.post('/caption-image', jsonParser, async (request, response) => { headers: { 'Content-Type': 'application/json', }, - timeout: 0, }); if (!result.ok) { @@ -49,6 +48,7 @@ router.post('/caption-image', jsonParser, async (request, response) => { return response.status(result.status).send({ error: true }); } + /** @type {any} */ const data = await result.json(); console.log('Multimodal captioning response', data); diff --git a/src/endpoints/images.js b/src/endpoints/images.js index eeacb90fc..22c1d5b17 100644 --- a/src/endpoints/images.js +++ b/src/endpoints/images.js @@ -68,7 +68,7 @@ router.post('/upload', jsonParser, async (request, response) => { ensureDirectoryExistence(pathToNewFile); const imageBuffer = Buffer.from(base64Data, 'base64'); - await fs.promises.writeFile(pathToNewFile, imageBuffer); + await fs.promises.writeFile(pathToNewFile, new Uint8Array(imageBuffer)); response.send({ path: clientRelativePath(request.user.directories.root, pathToNewFile) }); } catch (error) { console.log(error); diff --git a/src/endpoints/novelai.js b/src/endpoints/novelai.js index a4b770308..53ed8258e 100644 --- a/src/endpoints/novelai.js +++ b/src/endpoints/novelai.js @@ -252,7 +252,7 @@ router.post('/generate', jsonParser, async function (req, res) { try { const baseURL = (req.body.model.includes('kayra') || req.body.model.includes('erato')) ? TEXT_NOVELAI : API_NOVELAI; const url = req.body.streaming ? `${baseURL}/ai/generate-stream` : `${baseURL}/ai/generate`; - const response = await fetch(url, { method: 'POST', timeout: 0, ...args }); + const response = await fetch(url, { method: 'POST', ...args }); if (req.body.streaming) { // Pipe remote SSE stream to Express response @@ -274,6 +274,7 @@ router.post('/generate', jsonParser, async function (req, res) { return res.status(response.status).send({ error: { message } }); } + /** @type {any} */ const data = await response.json(); console.log('NovelAI Output', data?.output); return res.send(data); @@ -416,7 +417,6 @@ router.post('/generate-voice', jsonParser, async (request, response) => { 'Authorization': `Bearer ${token}`, 'Accept': 'audio/mpeg', }, - timeout: 0, }); if (!result.ok) { @@ -426,7 +426,7 @@ router.post('/generate-voice', jsonParser, async (request, response) => { } const chunks = await readAllChunks(result.body); - const buffer = Buffer.concat(chunks); + const buffer = Buffer.concat(chunks.map(chunk => new Uint8Array(chunk))); response.setHeader('Content-Type', 'audio/mpeg'); return response.send(buffer); } diff --git a/src/endpoints/openai.js b/src/endpoints/openai.js index 08d138e52..8c820dc55 100644 --- a/src/endpoints/openai.js +++ b/src/endpoints/openai.js @@ -154,7 +154,6 @@ router.post('/caption-image', jsonParser, async (request, response) => { ...headers, }, body: JSON.stringify(body), - timeout: 0, }); if (!result.ok) { @@ -163,6 +162,7 @@ router.post('/caption-image', jsonParser, async (request, response) => { return response.status(500).send(text); } + /** @type {any} */ const data = await result.json(); console.log('Multimodal captioning response', data); const caption = data?.choices[0]?.message?.content; @@ -284,7 +284,6 @@ router.post('/generate-image', jsonParser, async (request, response) => { Authorization: `Bearer ${key}`, }, body: JSON.stringify(request.body), - timeout: 0, }); if (!result.ok) { diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 811020f9a..dbd9a22fd 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -65,6 +65,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.name); return names; @@ -85,6 +86,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.name); return names; @@ -118,6 +120,7 @@ router.post('/vaes', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.model_name); return response.send(names); @@ -143,6 +146,7 @@ router.post('/samplers', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.name); return response.send(names); @@ -169,6 +173,7 @@ router.post('/schedulers', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.name); return response.send(names); @@ -194,6 +199,7 @@ router.post('/models', jsonParser, async (request, response) => { throw new Error('SD WebUI returned an error.'); } + /** @type {any} */ const data = await result.json(); const models = data.map(x => ({ value: x.title, text: x.title })); return response.send(models); @@ -214,6 +220,7 @@ router.post('/get-model', jsonParser, async (request, response) => { 'Authorization': getBasicAuthHeader(request.body.auth), }, }); + /** @type {any} */ const data = await result.json(); return response.send(data['sd_model_checkpoint']); } catch (error) { @@ -233,7 +240,6 @@ router.post('/set-model', jsonParser, async (request, response) => { headers: { 'Authorization': getBasicAuthHeader(request.body.auth), }, - timeout: 0, }); const data = await result.json(); return data; @@ -253,7 +259,6 @@ router.post('/set-model', jsonParser, async (request, response) => { 'Content-Type': 'application/json', 'Authorization': getBasicAuthHeader(request.body.auth), }, - timeout: 0, }); if (!result.ok) { @@ -264,6 +269,7 @@ router.post('/set-model', jsonParser, async (request, response) => { const CHECK_INTERVAL = 2000; for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + /** @type {any} */ const progressState = await getProgress(); const progress = progressState['progress']; @@ -308,8 +314,6 @@ router.post('/generate', jsonParser, async (request, response) => { 'Content-Type': 'application/json', 'Authorization': getBasicAuthHeader(request.body.auth), }, - timeout: 0, - // @ts-ignore signal: controller.signal, }); @@ -345,6 +349,7 @@ router.post('/sd-next/upscalers', jsonParser, async (request, response) => { // Vlad doesn't provide Latent Upscalers in the API, so we have to hardcode them here const latentUpscalers = ['Latent', 'Latent (antialiased)', 'Latent (bicubic)', 'Latent (bicubic antialiased)', 'Latent (nearest)', 'Latent (nearest-exact)']; + /** @type {any} */ const data = await result.json(); const names = data.map(x => x.name); @@ -387,6 +392,7 @@ comfy.post('/samplers', jsonParser, async (request, response) => { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const data = await result.json(); return response.send(data.KSampler.input.required.sampler_name[0]); } catch (error) { @@ -404,6 +410,7 @@ comfy.post('/models', jsonParser, async (request, response) => { if (!result.ok) { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const data = await result.json(); return response.send(data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it }))); } catch (error) { @@ -422,6 +429,7 @@ comfy.post('/schedulers', jsonParser, async (request, response) => { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const data = await result.json(); return response.send(data.KSampler.input.required.scheduler[0]); } catch (error) { @@ -440,6 +448,7 @@ comfy.post('/vaes', jsonParser, async (request, response) => { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const data = await result.json(); return response.send(data.VAELoader.input.required.vae_name[0]); } catch (error) { @@ -521,6 +530,7 @@ comfy.post('/generate', jsonParser, async (request, response) => { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const data = await promptResult.json(); const id = data.prompt_id; let item; @@ -531,6 +541,7 @@ comfy.post('/generate', jsonParser, async (request, response) => { if (!result.ok) { throw new Error('ComfyUI returned an error.'); } + /** @type {any} */ const history = await result.json(); item = history[id]; if (item) { @@ -633,6 +644,7 @@ together.post('/generate', jsonParser, async (request, response) => { return response.sendStatus(500); } + /** @type {any} */ const data = await result.json(); console.log('TogetherAI response:', data); @@ -681,6 +693,8 @@ drawthings.post('/get-model', jsonParser, async (request, response) => { const result = await fetch(url, { method: 'GET', }); + + /** @type {any} */ const data = await result.json(); return response.send(data['model']); @@ -698,6 +712,8 @@ drawthings.post('/get-upscaler', jsonParser, async (request, response) => { const result = await fetch(url, { method: 'GET', }); + + /** @type {any} */ const data = await result.json(); return response.send(data['upscaler']); @@ -726,7 +742,6 @@ drawthings.post('/generate', jsonParser, async (request, response) => { 'Content-Type': 'application/json', 'Authorization': auth, }, - timeout: 0, }); if (!result.ok) { @@ -848,7 +863,6 @@ stability.post('/generate', jsonParser, async (request, response) => { 'Accept': 'image/*', }, body: formData, - timeout: 0, }); if (!result.ok) { diff --git a/src/endpoints/translate.js b/src/endpoints/translate.js index 67f98ddc8..57b7a9a5a 100644 --- a/src/endpoints/translate.js +++ b/src/endpoints/translate.js @@ -78,6 +78,7 @@ router.post('/libre', jsonParser, async (request, response) => { return response.sendStatus(result.status); } + /** @type {any} */ const json = await result.json(); console.log('Translated text: ' + json.translatedText); @@ -158,7 +159,6 @@ router.post('/yandex', jsonParser, async (request, response) => { headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, - timeout: 0, }); if (!result.ok) { @@ -167,6 +167,7 @@ router.post('/yandex', jsonParser, async (request, response) => { return response.sendStatus(500); } + /** @type {any} */ const json = await result.json(); const translated = json.text.join(); console.log('Translated text: ' + translated); @@ -264,7 +265,6 @@ router.post('/deepl', jsonParser, async (request, response) => { 'Authorization': `DeepL-Auth-Key ${key}`, 'Content-Type': 'application/x-www-form-urlencoded', }, - timeout: 0, }); if (!result.ok) { @@ -273,6 +273,7 @@ router.post('/deepl', jsonParser, async (request, response) => { return response.sendStatus(result.status); } + /** @type {any} */ const json = await result.json(); console.log('Translated text: ' + json.translations[0].text); @@ -317,7 +318,6 @@ router.post('/onering', jsonParser, async (request, response) => { const result = await fetch(fetchUrl, { method: 'GET', - timeout: 0, }); if (!result.ok) { @@ -326,6 +326,7 @@ router.post('/onering', jsonParser, async (request, response) => { return response.sendStatus(result.status); } + /** @type {any} */ const data = await result.json(); console.log('Translated text: ' + data.result); @@ -373,7 +374,6 @@ router.post('/deeplx', jsonParser, async (request, response) => { 'Accept': 'application/json', 'Content-Type': 'application/json', }, - timeout: 0, }); if (!result.ok) { @@ -382,6 +382,7 @@ router.post('/deeplx', jsonParser, async (request, response) => { return response.sendStatus(result.status); } + /** @type {any} */ const json = await result.json(); console.log('Translated text: ' + json.data); diff --git a/src/transformers.mjs b/src/transformers.mjs index 416f2031f..a4981a191 100644 --- a/src/transformers.mjs +++ b/src/transformers.mjs @@ -3,7 +3,7 @@ import fs from 'node:fs'; import process from 'node:process'; import { Buffer } from 'node:buffer'; -import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers'; +import { pipeline, env, RawImage } from 'sillytavern-transformers'; import { getConfigValue } from './util.js'; configureTransformers(); @@ -117,7 +117,7 @@ async function migrateCacheToDataDir() { * Gets the transformers.js pipeline for a given task. * @param {import('sillytavern-transformers').PipelineType} task The task to get the pipeline for * @param {string} forceModel The model to use for the pipeline, if any - * @returns {Promise} Pipeline for the task + * @returns {Promise} The transformers.js pipeline */ export async function getPipeline(task, forceModel = '') { await migrateCacheToDataDir(); @@ -137,6 +137,7 @@ export async function getPipeline(task, forceModel = '') { const instance = await pipeline(task, model, { cache_dir: cacheDir, quantized: tasks[task].quantized ?? true, local_files_only: localOnly }); tasks[task].pipeline = instance; tasks[task].currentModel = model; + // @ts-ignore return instance; } diff --git a/src/util.js b/src/util.js index f966739d9..b3a6cca75 100644 --- a/src/util.js +++ b/src/util.js @@ -441,17 +441,21 @@ export function forwardFetchResponse(from, to) { to.statusCode = statusCode; to.statusMessage = statusText; - from.body.pipe(to); + if (from.body && to.socket) { + from.body.pipe(to); - to.socket.on('close', function () { - if (from.body instanceof Readable) from.body.destroy(); // Close the remote stream - to.end(); // End the Express response - }); + to.socket.on('close', function () { + if (from.body instanceof Readable) from.body.destroy(); // Close the remote stream + to.end(); // End the Express response + }); - from.body.on('end', function () { - console.log('Streaming request finished'); + from.body.on('end', function () { + console.log('Streaming request finished'); + to.end(); + }); + } else { to.end(); - }); + } } /** diff --git a/src/vectors/cohere-vectors.js b/src/vectors/cohere-vectors.js index 7a940ac23..01bc3214d 100644 --- a/src/vectors/cohere-vectors.js +++ b/src/vectors/cohere-vectors.js @@ -38,6 +38,7 @@ export async function getCohereBatchVector(texts, isQuery, directories, model) { throw new Error('API request failed'); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.embeddings?.float)) { console.log('API response was not an array'); diff --git a/src/vectors/extras-vectors.js b/src/vectors/extras-vectors.js index 5a0857ef6..8a0229b5e 100644 --- a/src/vectors/extras-vectors.js +++ b/src/vectors/extras-vectors.js @@ -66,6 +66,7 @@ async function getExtrasVectorImpl(text, apiUrl, apiKey) { throw new Error('Extras request failed'); } + /** @type {any} */ const data = await response.json(); const vector = data.embedding; // `embedding`: number[] (one text item), or number[][] (multiple text items). diff --git a/src/vectors/llamacpp-vectors.js b/src/vectors/llamacpp-vectors.js index 0af91336e..83b823a17 100644 --- a/src/vectors/llamacpp-vectors.js +++ b/src/vectors/llamacpp-vectors.js @@ -30,6 +30,7 @@ export async function getLlamaCppBatchVector(texts, apiUrl, directories) { throw new Error(`LlamaCpp: Failed to get vector for text: ${response.statusText} ${responseText}`); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.data)) { diff --git a/src/vectors/makersuite-vectors.js b/src/vectors/makersuite-vectors.js index 48c8fd056..1ff4a0d50 100644 --- a/src/vectors/makersuite-vectors.js +++ b/src/vectors/makersuite-vectors.js @@ -52,6 +52,7 @@ export async function getMakerSuiteVector(text, directories) { throw new Error('Google AI Studio request failed'); } + /** @type {any} */ const data = await response.json(); // noinspection JSValidateTypes return data['embedding']['values']; diff --git a/src/vectors/nomicai-vectors.js b/src/vectors/nomicai-vectors.js index 2f832956a..5acedfe8f 100644 --- a/src/vectors/nomicai-vectors.js +++ b/src/vectors/nomicai-vectors.js @@ -51,6 +51,7 @@ export async function getNomicAIBatchVector(texts, source, directories) { throw new Error('API request failed'); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.embeddings)) { console.log('API response was not an array'); diff --git a/src/vectors/ollama-vectors.js b/src/vectors/ollama-vectors.js index 237d6979d..415fde604 100644 --- a/src/vectors/ollama-vectors.js +++ b/src/vectors/ollama-vectors.js @@ -54,6 +54,7 @@ export async function getOllamaVector(text, apiUrl, model, keep, directories) { throw new Error(`Ollama: Failed to get vector for text: ${response.statusText} ${responseText}`); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.embedding)) { diff --git a/src/vectors/openai-vectors.js b/src/vectors/openai-vectors.js index f408e505e..76ce7e55b 100644 --- a/src/vectors/openai-vectors.js +++ b/src/vectors/openai-vectors.js @@ -61,6 +61,7 @@ export async function getOpenAIBatchVector(texts, source, directories, model = ' throw new Error('API request failed'); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.data)) { diff --git a/src/vectors/vllm-vectors.js b/src/vectors/vllm-vectors.js index e20c85a72..436bc57c4 100644 --- a/src/vectors/vllm-vectors.js +++ b/src/vectors/vllm-vectors.js @@ -31,6 +31,7 @@ export async function getVllmBatchVector(texts, apiUrl, model, directories) { throw new Error(`VLLM: Failed to get vector for text: ${response.statusText} ${responseText}`); } + /** @type {any} */ const data = await response.json(); if (!Array.isArray(data?.data)) { From fa84b3ede4ef915d5fa505d82712edee23d60067 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 22:32:30 +0300 Subject: [PATCH 18/21] Fix legacy config conversion --- post-install.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/post-install.js b/post-install.js index d2c1a8687..f477df0e0 100644 --- a/post-install.js +++ b/post-install.js @@ -61,14 +61,15 @@ function convertConfig() { try { console.log(color.blue('Converting config.conf to config.yaml. Your old config.conf will be renamed to config.conf.bak')); + fs.renameSync('./config.conf', './config.conf.cjs'); // Force loading as CommonJS const require = createRequire(import.meta.url); - const config = require(path.join(process.cwd(), './config.conf')); - fs.copyFileSync('./config.conf', './config.conf.bak'); - fs.rmSync('./config.conf'); + const config = require(path.join(process.cwd(), './config.conf.cjs')); + fs.copyFileSync('./config.conf.cjs', './config.conf.bak'); + fs.rmSync('./config.conf.cjs'); fs.writeFileSync('./config.yaml', yaml.stringify(config)); console.log(color.green('Conversion successful. Please check your config.yaml and fix it if necessary.')); } catch (error) { - console.error(color.red('FATAL: Config conversion failed. Please check your config.conf file and try again.')); + console.error(color.red('FATAL: Config conversion failed. Please check your config.conf file and try again.'), error); return; } } From ffa8716d070689fd84c3a3bfd07e6fed36e261b1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 23:00:37 +0300 Subject: [PATCH 19/21] Plugins: support .cjs module extension --- src/plugin-loader.js | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/plugin-loader.js b/src/plugin-loader.js index 433d9f411..dd97a80df 100644 --- a/src/plugin-loader.js +++ b/src/plugin-loader.js @@ -17,7 +17,7 @@ const loadedPlugins = new Map(); * @param {string} file Path to file * @returns {boolean} True if file is a CommonJS module */ -const isCommonJS = (file) => path.extname(file) === '.js'; +const isCommonJS = (file) => path.extname(file) === '.js' || path.extname(file) === '.cjs'; /** * Determine if a file is an ECMAScript module. @@ -35,7 +35,7 @@ const isESModule = (file) => path.extname(file) === '.mjs'; */ export async function loadPlugins(app, pluginsPath) { const exitHooks = []; - const emptyFn = () => {}; + const emptyFn = () => { }; // Server plugins are disabled. if (!enableServerPlugins) { @@ -90,19 +90,15 @@ async function loadFromDirectory(app, pluginDirectoryPath, exitHooks) { } } - // Plugin is a CommonJS module. - const cjsFilePath = path.join(pluginDirectoryPath, 'index.js'); - if (fs.existsSync(cjsFilePath)) { - if (await loadFromFile(app, cjsFilePath, exitHooks)) { - return; - } - } + // Plugin is a module file. + const fileTypes = ['index.js', 'index.cjs', 'index.mjs']; - // Plugin is an ECMAScript module. - const esmFilePath = path.join(pluginDirectoryPath, 'index.mjs'); - if (fs.existsSync(esmFilePath)) { - if (await loadFromFile(app, esmFilePath, exitHooks)) { - return; + for (const fileType of fileTypes) { + const filePath = path.join(pluginDirectoryPath, fileType); + if (fs.existsSync(filePath)) { + if (await loadFromFile(app, filePath, exitHooks)) { + return; + } } } } From 89c0b0e1cda75035853ab73ff0b902d3662bd370 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 12 Oct 2024 10:47:05 +0300 Subject: [PATCH 20/21] Replace Google Translate API package --- package-lock.json | 19 ++++++++++----- package.json | 2 +- src/endpoints/translate.js | 50 +++++--------------------------------- 3 files changed, 20 insertions(+), 51 deletions(-) diff --git a/package-lock.json b/package-lock.json index cdbaa662a..9109342b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "csrf-csrf": "^2.2.3", "express": "^4.21.0", "form-data": "^4.0.0", - "google-translate-api-browser": "^3.0.1", + "google-translate-api-x": "^10.7.1", "helmet": "^7.1.0", "html-entities": "^2.5.2", "iconv-lite": "^0.6.3", @@ -3689,11 +3689,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/google-translate-api-browser": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/google-translate-api-browser/-/google-translate-api-browser-3.0.1.tgz", - "integrity": "sha512-KTLodkyGBWMK9IW6QIeJ2zCuju4Z0CLpbkADKo+yLhbSTD4l+CXXpQ/xaynGVAzeBezzJG6qn8MLeqOq3SmW0A==", - "license": "MIT" + "node_modules/google-translate-api-x": { + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/google-translate-api-x/-/google-translate-api-x-10.7.1.tgz", + "integrity": "sha512-OdZDS6jRWzn1woOk62aOKQ5OyVaJSA+eyc6CktOWxo36IWfstOjwG/dkvnGl3Z2Sbpmk1A+jc2WwrBiRjqaY2A==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/AidanWelch" + } }, "node_modules/gopd": { "version": "1.0.1", diff --git a/package.json b/package.json index 2e702c499..30aaa9d96 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "csrf-csrf": "^2.2.3", "express": "^4.21.0", "form-data": "^4.0.0", - "google-translate-api-browser": "^3.0.1", + "google-translate-api-x": "^10.7.1", "helmet": "^7.1.0", "html-entities": "^2.5.2", "iconv-lite": "^0.6.3", diff --git a/src/endpoints/translate.js b/src/endpoints/translate.js index 57b7a9a5a..57387b63c 100644 --- a/src/endpoints/translate.js +++ b/src/endpoints/translate.js @@ -1,14 +1,9 @@ import https from 'node:https'; -import { createRequire } from 'node:module'; -import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; import express from 'express'; -import iconv from 'iconv-lite'; import bingTranslateApi from 'bing-translate-api'; - -const require = createRequire(import.meta.url); -const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser'); +import googleTranslateApi from 'google-translate-api-x'; import { readSecret, SECRET_KEYS } from './secrets.js'; import { getConfigValue, uuidv4 } from '../util.js'; @@ -17,20 +12,6 @@ import { jsonParser } from '../express-common.js'; const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; -/** - * Tries to decode a Node.js Buffer to a string using iconv-lite for UTF-8. - * @param {Buffer} buffer Node.js Buffer - * @returns {string} Decoded string - */ -function decodeBuffer(buffer) { - try { - return iconv.decode(buffer, 'utf-8'); - } catch (error) { - console.log('Failed to decode buffer:', error); - return buffer.toString('utf-8'); - } -} - export const router = express.Router(); router.post('/libre', jsonParser, async (request, response) => { @@ -100,31 +81,12 @@ router.post('/google', jsonParser, async (request, response) => { console.log('Input text: ' + text); - const url = generateRequestUrl(text, { to: lang }); + const result = await googleTranslateApi(text, { to: lang, forceBatch: false }); + const translatedText = Array.isArray(result) ? result.map(x => x.text).join('') : result.text; - https.get(url, (resp) => { - const data = []; - - resp.on('data', (chunk) => { - data.push(chunk); - }); - - resp.on('end', () => { - try { - const decodedData = decodeBuffer(Buffer.concat(data)); - const result = normaliseResponse(JSON.parse(decodedData)); - console.log('Translated text: ' + result.text); - response.setHeader('Content-Type', 'text/plain; charset=utf-8'); - return response.send(result.text); - } catch (error) { - console.log('Translation error', error); - return response.sendStatus(500); - } - }); - }).on('error', (err) => { - console.log('Translation error: ' + err.message); - return response.sendStatus(500); - }); + response.setHeader('Content-Type', 'text/plain; charset=utf-8'); + console.log('Translated text: ' + translatedText); + return response.send(translatedText); } catch (error) { console.log('Translation error', error); return response.sendStatus(500); From 131e60ffe639bdf27e13782fe56802b6b6ed7084 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 12 Oct 2024 11:36:26 +0300 Subject: [PATCH 21/21] Update node types --- package-lock.json | 27 ++++++++++++++------------- package.json | 1 + src/request-proxy.js | 7 ++----- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9109342b3..b684fa861 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,6 +71,7 @@ "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", "@types/multer": "^1.4.12", + "@types/node": "^18.19.55", "@types/node-persist": "^3.1.8", "@types/png-chunk-text": "^1.0.3", "@types/png-chunks-encode": "^1.0.2", @@ -1169,10 +1170,13 @@ } }, "node_modules/@types/node": { - "version": "16.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", - "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", - "license": "MIT" + "version": "18.19.55", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.55.tgz", + "integrity": "sha512-zzw5Vw52205Zr/nmErSEkN5FLqXPuKX/k5d1D7RKHATGqU7y6YfX9QxZraUzUrFGqH6XzOzG196BC35ltJC4Cw==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/node-fetch": { "version": "2.6.11", @@ -4031,6 +4035,12 @@ "@types/node": "16.9.1" } }, + "node_modules/image-q/node_modules/@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "license": "MIT" + }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -4904,15 +4914,6 @@ "openai": "bin/cli" } }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.18.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.9.tgz", - "integrity": "sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, "node_modules/openai/node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", diff --git a/package.json b/package.json index 30aaa9d96..7bb9783fa 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "@types/lodash": "^4.17.10", "@types/mime-types": "^2.1.4", "@types/multer": "^1.4.12", + "@types/node": "^18.19.55", "@types/node-persist": "^3.1.8", "@types/png-chunk-text": "^1.0.3", "@types/png-chunks-encode": "^1.0.2", diff --git a/src/request-proxy.js b/src/request-proxy.js index 312d9376e..3ad8a7046 100644 --- a/src/request-proxy.js +++ b/src/request-proxy.js @@ -1,11 +1,9 @@ import process from 'node:process'; -import { createRequire } from 'node:module'; +import http from 'node:http'; +import https from 'node:https'; import { ProxyAgent } from 'proxy-agent'; import { isValidUrl, color } from './util.js'; -const require = createRequire(import.meta.url); -const http = require('http'); -const https = require('https'); const LOG_HEADER = '[Request Proxy]'; /** @@ -37,7 +35,6 @@ export default function initRequestProxy({ enabled, url, bypass }) { // Reference: https://github.com/Rob--W/proxy-from-env process.env.all_proxy = url; - if (Array.isArray(bypass) && bypass.length > 0) { process.env.no_proxy = bypass.join(','); }