diff --git a/package-lock.json b/package-lock.json index 9bb63f3aa..4c137a946 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,8 +13,6 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", - "@tensorflow-models/universal-sentence-encoder": "^1.3.3", - "@tensorflow/tfjs": "^4.10.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", @@ -697,215 +695,6 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "node_modules/@tensorflow-models/universal-sentence-encoder": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@tensorflow-models/universal-sentence-encoder/-/universal-sentence-encoder-1.3.3.tgz", - "integrity": "sha512-mipL7ad0CW6uQ68FUkNgkNj/zgA4qgBnNcnMMkNTdL9MUMnzCxu3AE8pWnx2ReKHwdqEG4e8IpaYKfH4B8bojg==", - "peerDependencies": { - "@tensorflow/tfjs-converter": "^3.6.0", - "@tensorflow/tfjs-core": "^3.6.0" - } - }, - "node_modules/@tensorflow/tfjs": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.10.0.tgz", - "integrity": "sha512-16q6CcGEoxfg3kimBzNBrZg069GRd4Du49uyQHUMsEvpHbmU0ZpWa2zhjjZ71GlURpbD1LZIZNp6cd2Q1Eqjow==", - "dependencies": { - "@tensorflow/tfjs-backend-cpu": "4.10.0", - "@tensorflow/tfjs-backend-webgl": "4.10.0", - "@tensorflow/tfjs-converter": "4.10.0", - "@tensorflow/tfjs-core": "4.10.0", - "@tensorflow/tfjs-data": "4.10.0", - "@tensorflow/tfjs-layers": "4.10.0", - "argparse": "^1.0.10", - "chalk": "^4.1.0", - "core-js": "3.29.1", - "regenerator-runtime": "^0.13.5", - "yargs": "^16.0.3" - }, - "bin": { - "tfjs-custom-module": "dist/tools/custom_module/cli.js" - } - }, - "node_modules/@tensorflow/tfjs-converter": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz", - "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==", - "peer": true, - "peerDependencies": { - "@tensorflow/tfjs-core": "3.21.0" - } - }, - "node_modules/@tensorflow/tfjs-core": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz", - "integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==", - "peer": true, - "dependencies": { - "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "^2.4.28", - "@types/webgl-ext": "0.0.30", - "@webgpu/types": "0.1.16", - "long": "4.0.0", - "node-fetch": "~2.6.1", - "seedrandom": "^3.0.5" - }, - "engines": { - "yarn": ">= 1.3.2" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-cpu": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.10.0.tgz", - "integrity": "sha512-w3f0ORR1smSpkW7om2yQVunRYMjyqWcEbWCPirR1DQ6ImgW+VWqmM2oVPQXRsFYpwg1g6bk2Jp5COafpPA+krw==", - "dependencies": { - "@types/seedrandom": "^2.4.28", - "seedrandom": "^3.0.5" - }, - "engines": { - "yarn": ">= 1.3.2" - }, - "peerDependencies": { - "@tensorflow/tfjs-core": "4.10.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-webgl": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.10.0.tgz", - "integrity": "sha512-Vzl/pyXHa9TgFaRJGspExjZVDKgkKvLxOkPaH+psE2LPnQkiH/IOPO7HKO0U3+hZql977BIiZdHc6HNprFS3/A==", - "dependencies": { - "@tensorflow/tfjs-backend-cpu": "4.10.0", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "^2.4.28", - "seedrandom": "^3.0.5" - }, - "engines": { - "yarn": ">= 1.3.2" - }, - "peerDependencies": { - "@tensorflow/tfjs-core": "4.10.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-converter": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.10.0.tgz", - "integrity": "sha512-ffLpK+ismdmiDcoTID2aidP3/uJYyQPjmKdRZ3hBUkrczy7pQIcCW8blIR9Gk20htB4OLQMf74ZxbpfdQ9nYeQ==", - "peerDependencies": { - "@tensorflow/tfjs-core": "4.10.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.10.0.tgz", - "integrity": "sha512-klc6lUTYRbHQLEFnVKtTICNK+WUlduUcrlXsDs6ixKAOJzLAlIR2JnmJICt2AT2Rxwu0Zj2bAYojSxgcIcUUxA==", - "dependencies": { - "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.7.0", - "@types/seedrandom": "^2.4.28", - "@webgpu/types": "0.1.30", - "long": "4.0.0", - "node-fetch": "~2.6.1", - "seedrandom": "^3.0.5" - }, - "engines": { - "yarn": ">= 1.3.2" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": { - "version": "2019.7.1", - "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.1.tgz", - "integrity": "sha512-+HSrJgjBW77ALieQdMJvXhRZUIRN1597L+BKvsyeiIlHHERnqjcuOLyodK3auJ3Y3zRezNKtKAhuQWYJfEgFHQ==" - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-data": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.10.0.tgz", - "integrity": "sha512-71rQ6xSipXdClKja705jrWZkH9ostAYuVZlf7nW2AJXUCzhrGsJAkcHag4m568mDFoAqfQQTBy4Gk26h0/Y+Pg==", - "dependencies": { - "@types/node-fetch": "^2.1.2", - "node-fetch": "~2.6.1", - "string_decoder": "^1.3.0" - }, - "peerDependencies": { - "@tensorflow/tfjs-core": "4.10.0", - "seedrandom": "^3.0.5" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-layers": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.10.0.tgz", - "integrity": "sha512-SLZWnuDF98WmmJQ5NhsXJFlJPwoKxfLowvAHTlLz+Q1Po4juZVZ+BkatRsqrI2sA2B0IIu2TJp4VEAFWMqzTTg==", - "peerDependencies": { - "@tensorflow/tfjs-core": "4.10.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/@webgpu/types": { - "version": "0.1.30", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.30.tgz", - "integrity": "sha512-9AXJSmL3MzY8ZL//JjudA//q+2kBRGhLBFpkdGksWIuxrMy81nFrCzj2Am+mbh8WoU6rXmv7cY5E3rdlyru2Qg==" - }, - "node_modules/@tensorflow/tfjs/node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/@tensorflow/tfjs/node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@tensorflow/tfjs/node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "engines": { - "node": ">=10" - } - }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -921,50 +710,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==" }, - "node_modules/@types/node-fetch": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", - "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", - "dependencies": { - "@types/node": "*", - "form-data": "^3.0.0" - } - }, - "node_modules/@types/node-fetch/node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@types/offscreencanvas": { - "version": "2019.3.0", - "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", - "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" - }, - "node_modules/@types/seedrandom": { - "version": "2.4.30", - "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz", - "integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==" - }, - "node_modules/@types/webgl-ext": { - "version": "0.0.30", - "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", - "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==", - "peer": true - }, - "node_modules/@webgpu/types": { - "version": "0.1.16", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz", - "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==", - "peer": true - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1052,14 +797,6 @@ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==" }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -1279,6 +1016,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -1482,16 +1220,6 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, - "node_modules/core-js": { - "version": "3.29.1", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz", - "integrity": "sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw==", - "hasInstallScript": true, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/core-js" - } - }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -2150,6 +1878,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "engines": { "node": ">=8" } @@ -3624,11 +3353,6 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, - "node_modules/seedrandom": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", - "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" - }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -3821,11 +3545,6 @@ "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -3912,6 +3631,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "dependencies": { "has-flag": "^4.0.0" }, diff --git a/package.json b/package.json index c3e855035..c434e4e4b 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,6 @@ "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", "@dqbd/tiktoken": "^1.0.2", - "@tensorflow-models/universal-sentence-encoder": "^1.3.3", - "@tensorflow/tfjs": "^4.10.0", "command-exists": "^1.2.9", "compression": "^1", "cookie-parser": "^1.4.6", diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 59c9b237b..1fcea4caf 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -1,7 +1,7 @@ import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, is_send_press, saveSettingsDebounced, setExtensionPrompt, substituteParams } from "../../../script.js"; import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js"; import { collapseNewlines, power_user, ui_mode } from "../../power-user.js"; -import { debounce, getStringHash as calculateHash, waitUntilCondition } from "../../utils.js"; +import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique } from "../../utils.js"; const MODULE_NAME = 'vectors'; @@ -9,7 +9,7 @@ export const EXTENSION_PROMPT_TAG = '3_vectors'; const settings = { enabled: false, - source: 'local', + source: 'transformers', template: `Past events: {{text}}`, depth: 2, position: extension_prompt_types.IN_PROMPT, @@ -180,7 +180,7 @@ async function rearrangeChat(chat) { } // Get the most relevant messages, excluding the last few - const queryHashes = await queryCollection(chatId, queryText, settings.insert); + const queryHashes = (await queryCollection(chatId, queryText, settings.insert)).filter(onlyUnique); const queriedMessages = []; const retainMessages = chat.slice(-settings.protect); @@ -378,6 +378,8 @@ jQuery(async () => { } Object.assign(settings, extension_settings.vectors); + // Migrate from TensorFlow to Transformers + settings.source = settings.source !== 'local' ? settings.source : 'transformers'; $('#extensions_settings2').append(renderExtensionTemplate(MODULE_NAME, 'settings')); $('#vectors_enabled').prop('checked', settings.enabled).on('input', () => { settings.enabled = $('#vectors_enabled').prop('checked'); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index fff8289ca..1606540fd 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -14,7 +14,6 @@
diff --git a/src/local-vectors.js b/src/local-vectors.js deleted file mode 100644 index 73d514840..000000000 --- a/src/local-vectors.js +++ /dev/null @@ -1,38 +0,0 @@ - -require('@tensorflow/tfjs'); -const encoder = require('@tensorflow-models/universal-sentence-encoder'); - -/** - * Lazy loading class for the embedding model. - */ -class EmbeddingModel { - /** - * @type {encoder.UniversalSentenceEncoder} - The embedding model - */ - model; - - async get() { - if (!this.model) { - this.model = await encoder.load(); - } - - return this.model; - } -} - -const model = new EmbeddingModel(); - -/** - * @param {string} text - */ -async function getLocalVector(text) { - const use = await model.get(); - const tensor = await use.embed(text); - const vector = Array.from(await tensor.data()); - - return vector; -} - -module.exports = { - getLocalVector, -}; diff --git a/src/vectors.js b/src/vectors.js index 98bb1cd30..374d6d9d9 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -11,8 +11,6 @@ const sanitize = require('sanitize-filename'); */ async function getVector(source, text) { switch (source) { - case 'local': - return require('./local-vectors').getLocalVector(text); case 'openai': return require('./openai-vectors').getOpenAIVector(text); case 'transformers': @@ -126,7 +124,7 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const searchText = String(req.body.searchText); const topK = Number(req.body.topK) || 10; - const source = String(req.body.source) || 'local'; + const source = String(req.body.source) || 'transformers'; const results = await queryCollection(collectionId, source, searchText, topK); return res.json(results); @@ -144,7 +142,7 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const items = req.body.items.map(x => ({ hash: x.hash, text: x.text })); - const source = String(req.body.source) || 'local'; + const source = String(req.body.source) || 'transformers'; await insertVectorItems(collectionId, source, items); return res.sendStatus(200); @@ -161,7 +159,7 @@ async function registerEndpoints(app, jsonParser) { } const collectionId = String(req.body.collectionId); - const source = String(req.body.source) || 'local'; + const source = String(req.body.source) || 'transformers'; const hashes = await getSavedHashes(collectionId, source); return res.json(hashes); @@ -179,7 +177,7 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); const hashes = req.body.hashes.map(x => Number(x)); - const source = String(req.body.source) || 'local'; + const source = String(req.body.source) || 'transformers'; await deleteVectorItems(collectionId, source, hashes); return res.sendStatus(200); @@ -197,7 +195,7 @@ async function registerEndpoints(app, jsonParser) { const collectionId = String(req.body.collectionId); - const sources = ['local', 'openai']; + const sources = ['transformers', 'openai']; for (const source of sources) { const index = await getIndex(collectionId, source, false);