rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove tensorflow vector source.

Browse Source
This commit is contained in:
Cohee
2023-09-17 14:09:24 +03:00
parent 323493962a
commit dc1121b72a
6 changed files with 13 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
package-lock.json generated
View File

@ -13,8 +13,6 @@
"@agnai/sentencepiece-js": "^1.1.1",
"@agnai/web-tokenizers": "^0.1.3",
"@dqbd/tiktoken": "^1.0.2",
"@tensorflow-models/universal-sentence-encoder": "^1.3.3",
"@tensorflow/tfjs": "^4.10.0",
"command-exists": "^1.2.9",
"compression": "^1",
"cookie-parser": "^1.4.6",
@ -697,215 +695,6 @@
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
},
"node_modules/@tensorflow-models/universal-sentence-encoder": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/@tensorflow-models/universal-sentence-encoder/-/universal-sentence-encoder-1.3.3.tgz",
"integrity": "sha512-mipL7ad0CW6uQ68FUkNgkNj/zgA4qgBnNcnMMkNTdL9MUMnzCxu3AE8pWnx2ReKHwdqEG4e8IpaYKfH4B8bojg==",
"peerDependencies": {
"@tensorflow/tfjs-converter": "^3.6.0",
"@tensorflow/tfjs-core": "^3.6.0"
}
},
"node_modules/@tensorflow/tfjs": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.10.0.tgz",
"integrity": "sha512-16q6CcGEoxfg3kimBzNBrZg069GRd4Du49uyQHUMsEvpHbmU0ZpWa2zhjjZ71GlURpbD1LZIZNp6cd2Q1Eqjow==",
"dependencies": {
"@tensorflow/tfjs-backend-cpu": "4.10.0",
"@tensorflow/tfjs-backend-webgl": "4.10.0",
"@tensorflow/tfjs-converter": "4.10.0",
"@tensorflow/tfjs-core": "4.10.0",
"@tensorflow/tfjs-data": "4.10.0",
"@tensorflow/tfjs-layers": "4.10.0",
"argparse": "^1.0.10",
"chalk": "^4.1.0",
"core-js": "3.29.1",
"regenerator-runtime": "^0.13.5",
"yargs": "^16.0.3"
},
"bin": {
"tfjs-custom-module": "dist/tools/custom_module/cli.js"
}
},
"node_modules/@tensorflow/tfjs-converter": {
"version": "3.21.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz",
"integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==",
"peer": true,
"peerDependencies": {
"@tensorflow/tfjs-core": "3.21.0"
}
},
"node_modules/@tensorflow/tfjs-core": {
"version": "3.21.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz",
"integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==",
"peer": true,
"dependencies": {
"@types/long": "^4.0.1",
"@types/offscreencanvas": "~2019.3.0",
"@types/seedrandom": "^2.4.28",
"@types/webgl-ext": "0.0.30",
"@webgpu/types": "0.1.16",
"long": "4.0.0",
"node-fetch": "~2.6.1",
"seedrandom": "^3.0.5"
},
"engines": {
"yarn": ">= 1.3.2"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-cpu": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.10.0.tgz",
"integrity": "sha512-w3f0ORR1smSpkW7om2yQVunRYMjyqWcEbWCPirR1DQ6ImgW+VWqmM2oVPQXRsFYpwg1g6bk2Jp5COafpPA+krw==",
"dependencies": {
"@types/seedrandom": "^2.4.28",
"seedrandom": "^3.0.5"
},
"engines": {
"yarn": ">= 1.3.2"
},
"peerDependencies": {
"@tensorflow/tfjs-core": "4.10.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-webgl": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.10.0.tgz",
"integrity": "sha512-Vzl/pyXHa9TgFaRJGspExjZVDKgkKvLxOkPaH+psE2LPnQkiH/IOPO7HKO0U3+hZql977BIiZdHc6HNprFS3/A==",
"dependencies": {
"@tensorflow/tfjs-backend-cpu": "4.10.0",
"@types/offscreencanvas": "~2019.3.0",
"@types/seedrandom": "^2.4.28",
"seedrandom": "^3.0.5"
},
"engines": {
"yarn": ">= 1.3.2"
},
"peerDependencies": {
"@tensorflow/tfjs-core": "4.10.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-converter": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.10.0.tgz",
"integrity": "sha512-ffLpK+ismdmiDcoTID2aidP3/uJYyQPjmKdRZ3hBUkrczy7pQIcCW8blIR9Gk20htB4OLQMf74ZxbpfdQ9nYeQ==",
"peerDependencies": {
"@tensorflow/tfjs-core": "4.10.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.10.0.tgz",
"integrity": "sha512-klc6lUTYRbHQLEFnVKtTICNK+WUlduUcrlXsDs6ixKAOJzLAlIR2JnmJICt2AT2Rxwu0Zj2bAYojSxgcIcUUxA==",
"dependencies": {
"@types/long": "^4.0.1",
"@types/offscreencanvas": "~2019.7.0",
"@types/seedrandom": "^2.4.28",
"@webgpu/types": "0.1.30",
"long": "4.0.0",
"node-fetch": "~2.6.1",
"seedrandom": "^3.0.5"
},
"engines": {
"yarn": ">= 1.3.2"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": {
"version": "2019.7.1",
"resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.1.tgz",
"integrity": "sha512-+HSrJgjBW77ALieQdMJvXhRZUIRN1597L+BKvsyeiIlHHERnqjcuOLyodK3auJ3Y3zRezNKtKAhuQWYJfEgFHQ=="
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-data": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.10.0.tgz",
"integrity": "sha512-71rQ6xSipXdClKja705jrWZkH9ostAYuVZlf7nW2AJXUCzhrGsJAkcHag4m568mDFoAqfQQTBy4Gk26h0/Y+Pg==",
"dependencies": {
"@types/node-fetch": "^2.1.2",
"node-fetch": "~2.6.1",
"string_decoder": "^1.3.0"
},
"peerDependencies": {
"@tensorflow/tfjs-core": "4.10.0",
"seedrandom": "^3.0.5"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-layers": {
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.10.0.tgz",
"integrity": "sha512-SLZWnuDF98WmmJQ5NhsXJFlJPwoKxfLowvAHTlLz+Q1Po4juZVZ+BkatRsqrI2sA2B0IIu2TJp4VEAFWMqzTTg==",
"peerDependencies": {
"@tensorflow/tfjs-core": "4.10.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/@webgpu/types": {
"version": "0.1.30",
"resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.30.tgz",
"integrity": "sha512-9AXJSmL3MzY8ZL//JjudA//q+2kBRGhLBFpkdGksWIuxrMy81nFrCzj2Am+mbh8WoU6rXmv7cY5E3rdlyru2Qg=="
},
"node_modules/@tensorflow/tfjs/node_modules/cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"dependencies": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
]
},
"node_modules/@tensorflow/tfjs/node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"dependencies": {
"safe-buffer": "~5.2.0"
}
},
"node_modules/@tensorflow/tfjs/node_modules/yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"dependencies": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@tensorflow/tfjs/node_modules/yargs-parser": {
"version": "20.2.9",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
"integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
"engines": {
"node": ">=10"
}
},
"node_modules/@tokenizer/token": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz",
@ -921,50 +710,6 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
"integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g=="
},
"node_modules/@types/node-fetch": {
"version": "2.6.4",
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz",
"integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==",
"dependencies": {
"@types/node": "*",
"form-data": "^3.0.0"
}
},
"node_modules/@types/node-fetch/node_modules/form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/@types/offscreencanvas": {
"version": "2019.3.0",
"resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz",
"integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q=="
},
"node_modules/@types/seedrandom": {
"version": "2.4.30",
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz",
"integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ=="
},
"node_modules/@types/webgl-ext": {
"version": "0.0.30",
"resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",
"integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==",
"peer": true
},
"node_modules/@webgpu/types": {
"version": "0.1.16",
"resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz",
"integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==",
"peer": true
},
"node_modules/accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@ -1052,14 +797,6 @@
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw=="
},
"node_modules/argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"dependencies": {
"sprintf-js": "~1.0.2"
}
},
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -1279,6 +1016,7 @@
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
@ -1482,16 +1220,6 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
},
"node_modules/core-js": {
"version": "3.29.1",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz",
"integrity": "sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw==",
"hasInstallScript": true,
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/core-js"
}
},
"node_modules/core-util-is": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@ -2150,6 +1878,7 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"engines": {
"node": ">=8"
}
@ -3624,11 +3353,6 @@
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"node_modules/seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
},
"node_modules/semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@ -3821,11 +3545,6 @@
"resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz",
"integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA=="
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@ -3912,6 +3631,7 @@
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"dependencies": {
"has-flag": "^4.0.0"
},

2
package.json
View File

@ -3,8 +3,6 @@
"@agnai/sentencepiece-js": "^1.1.1",
"@agnai/web-tokenizers": "^0.1.3",
"@dqbd/tiktoken": "^1.0.2",
"@tensorflow-models/universal-sentence-encoder": "^1.3.3",
"@tensorflow/tfjs": "^4.10.0",
"command-exists": "^1.2.9",
"compression": "^1",
"cookie-parser": "^1.4.6",

8
public/scripts/extensions/vectors/index.js
View File

@ -1,7 +1,7 @@
import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, is_send_press, saveSettingsDebounced, setExtensionPrompt, substituteParams } from "../../../script.js";
import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from "../../extensions.js";
import { collapseNewlines, power_user, ui_mode } from "../../power-user.js";
import { debounce, getStringHash as calculateHash, waitUntilCondition } from "../../utils.js";
import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique } from "../../utils.js";
const MODULE_NAME = 'vectors';
@ -9,7 +9,7 @@ export const EXTENSION_PROMPT_TAG = '3_vectors';
const settings = {
enabled: false,
source: 'local',
source: 'transformers',
template: `Past events: {{text}}`,
depth: 2,
position: extension_prompt_types.IN_PROMPT,
@ -180,7 +180,7 @@ async function rearrangeChat(chat) {
}
// Get the most relevant messages, excluding the last few
const queryHashes = await queryCollection(chatId, queryText, settings.insert);
const queryHashes = (await queryCollection(chatId, queryText, settings.insert)).filter(onlyUnique);
const queriedMessages = [];
const retainMessages = chat.slice(-settings.protect);
@ -378,6 +378,8 @@ jQuery(async () => {
}
Object.assign(settings, extension_settings.vectors);
// Migrate from TensorFlow to Transformers
settings.source = settings.source !== 'local' ? settings.source : 'transformers';
$('#extensions_settings2').append(renderExtensionTemplate(MODULE_NAME, 'settings'));
$('#vectors_enabled').prop('checked', settings.enabled).on('input', () => {
settings.enabled = $('#vectors_enabled').prop('checked');

1
public/scripts/extensions/vectors/settings.html
View File

@ -14,7 +14,6 @@
</label>
<select id="vectors_source" class="select">
<option value="transformers">Local (Transformers)</option>
<option value="local">Local (Tensorflow)</option>
<option value="openai">OpenAI</option>
</select>
<div id="vectors_advanced_settings" data-newbie-hidden>

38
src/local-vectors.js
View File

@ -1,38 +0,0 @@
require('@tensorflow/tfjs');
const encoder = require('@tensorflow-models/universal-sentence-encoder');
/**
* Lazy loading class for the embedding model.
*/
class EmbeddingModel {
/**
* @type {encoder.UniversalSentenceEncoder} - The embedding model
*/
model;
async get() {
if (!this.model) {
this.model = await encoder.load();
}
return this.model;
}
}
const model = new EmbeddingModel();
/**
* @param {string} text
*/
async function getLocalVector(text) {
const use = await model.get();
const tensor = await use.embed(text);
const vector = Array.from(await tensor.data());
return vector;
}
module.exports = {
getLocalVector,
};

12
src/vectors.js
View File

@ -11,8 +11,6 @@ const sanitize = require('sanitize-filename');
*/
async function getVector(source, text) {
switch (source) {
case 'local':
return require('./local-vectors').getLocalVector(text);
case 'openai':
return require('./openai-vectors').getOpenAIVector(text);
case 'transformers':
@ -126,7 +124,7 @@ async function registerEndpoints(app, jsonParser) {
const collectionId = String(req.body.collectionId);
const searchText = String(req.body.searchText);
const topK = Number(req.body.topK) || 10;
const source = String(req.body.source) || 'local';
const source = String(req.body.source) || 'transformers';
const results = await queryCollection(collectionId, source, searchText, topK);
return res.json(results);
@ -144,7 +142,7 @@ async function registerEndpoints(app, jsonParser) {
const collectionId = String(req.body.collectionId);
const items = req.body.items.map(x => ({ hash: x.hash, text: x.text }));
const source = String(req.body.source) || 'local';
const source = String(req.body.source) || 'transformers';
await insertVectorItems(collectionId, source, items);
return res.sendStatus(200);
@ -161,7 +159,7 @@ async function registerEndpoints(app, jsonParser) {
}
const collectionId = String(req.body.collectionId);
const source = String(req.body.source) || 'local';
const source = String(req.body.source) || 'transformers';
const hashes = await getSavedHashes(collectionId, source);
return res.json(hashes);
@ -179,7 +177,7 @@ async function registerEndpoints(app, jsonParser) {
const collectionId = String(req.body.collectionId);
const hashes = req.body.hashes.map(x => Number(x));
const source = String(req.body.source) || 'local';
const source = String(req.body.source) || 'transformers';
await deleteVectorItems(collectionId, source, hashes);
return res.sendStatus(200);
@ -197,7 +195,7 @@ async function registerEndpoints(app, jsonParser) {
const collectionId = String(req.body.collectionId);
const sources = ['local', 'openai'];
const sources = ['transformers', 'openai'];
for (const source of sources) {
const index = await getIndex(collectionId, source, false);