From df3d7a048edf0d3fb2e45e47d0581ae01726f2ee Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Tue, 24 Sep 2024 21:51:10 +0300
Subject: [PATCH] Deprecate unscoped vectors

---
 default/config.yaml                           |  3 --
 public/scripts/extensions/vectors/index.js    | 26 ----------------
 .../scripts/extensions/vectors/settings.html  |  8 -----
 src/endpoints/vectors.js                      | 31 +++++++------------
 4 files changed, 11 insertions(+), 57 deletions(-)

diff --git a/default/config.yaml b/default/config.yaml
index 8a55f6afb..585988686 100644
--- a/default/config.yaml
+++ b/default/config.yaml
@@ -110,9 +110,6 @@ enableExtensionsAutoUpdate: true
 # Additional model tokenizers can be downloaded on demand.
 # Disabling will fallback to another locally available tokenizer.
 enableDownloadableTokenizers: true
-# Vector storage settings
-vectors:
-  enableModelScopes: false
 # Extension settings
 extras:
   # Disables automatic model download from HuggingFace
diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js
index 408af9cf1..3d1a7a539 100644
--- a/public/scripts/extensions/vectors/index.js
+++ b/public/scripts/extensions/vectors/index.js
@@ -999,25 +999,6 @@ async function purgeAllVectorIndexes() {
     }
 }
 
-async function isModelScopesEnabled() {
-    try {
-        const response = await fetch('/api/vector/scopes-enabled', {
-            method: 'GET',
-            headers: getVectorHeaders(),
-        });
-
-        if (!response.ok) {
-            return false;
-        }
-
-        const data = await response.json();
-        return data?.enabled ?? false;
-    } catch (error) {
-        console.error('Vectors: Failed to check model scopes', error);
-        return false;
-    }
-}
-
 function toggleSettings() {
     $('#vectors_files_settings').toggle(!!settings.enabled_files);
     $('#vectors_chats_settings').toggle(!!settings.enabled_chats);
@@ -1282,7 +1263,6 @@ jQuery(async () => {
     }
 
     Object.assign(settings, extension_settings.vectors);
-    const scopesEnabled = await isModelScopesEnabled();
 
     // Migrate from TensorFlow to Transformers
     settings.source = settings.source !== 'local' ? settings.source : 'transformers';
@@ -1294,7 +1274,6 @@ jQuery(async () => {
         saveSettingsDebounced();
         toggleSettings();
     });
-    $('#vectors_modelWarning').hide();
     $('#vectors_enabled_files').prop('checked', settings.enabled_files).on('input', () => {
         settings.enabled_files = $('#vectors_enabled_files').prop('checked');
         Object.assign(extension_settings.vectors, settings);
@@ -1334,31 +1313,26 @@ jQuery(async () => {
         saveSettingsDebounced();
     });
     $('#vectors_togetherai_model').val(settings.togetherai_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.togetherai_model = String($('#vectors_togetherai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_openai_model').val(settings.openai_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.openai_model = String($('#vectors_openai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_cohere_model').val(settings.cohere_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.cohere_model = String($('#vectors_cohere_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_ollama_model').val(settings.ollama_model).on('input', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.ollama_model = String($('#vectors_ollama_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_vllm_model').val(settings.vllm_model).on('input', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.vllm_model = String($('#vectors_vllm_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html
index f1e73016e..a7686a55d 100644
--- a/public/scripts/extensions/vectors/settings.html
+++ b/public/scripts/extensions/vectors/settings.html
@@ -96,14 +96,6 @@
                 </i>
             </div>
 
-            <small id="vectors_modelWarning">
-                <i class="fa-solid fa-exclamation-triangle"></i>
-                <span>
-                    Set <code>vectors.enableModelScopes</code> to true in config.yaml to switch between vectorization models without needing to purge existing vectors.
-                    This option will soon be enabled by default.
-                </span>
-            </small>
-
             <div class="flex-container alignItemsCenter" id="nomicai_apiKey">
                 <label for="api_key_nomicai" class="flex1">
                     <span data-i18n="NomicAI API Key">NomicAI API Key</span>
diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js
index 790b5693d..32e25ccf5 100644
--- a/src/endpoints/vectors.js
+++ b/src/endpoints/vectors.js
@@ -4,7 +4,7 @@ const fs = require('fs');
 const express = require('express');
 const sanitize = require('sanitize-filename');
 const { jsonParser } = require('../express-common');
-const { getConfigValue, color } = require('../util');
+const { getConfigValue } = require('../util');
 
 // Don't forget to add new sources to the SOURCES array
 const SOURCES = [
@@ -150,7 +150,7 @@ function getSourceSettings(source, request) {
                 extrasUrl: String(request.headers['x-extras-url']),
                 extrasKey: String(request.headers['x-extras-key']),
             };
-        case 'local':
+        case 'transformers':
             return {
                 model: getConfigValue('extras.embeddingModel', ''),
             };
@@ -159,6 +159,14 @@ function getSourceSettings(source, request) {
                 // TODO: Add support for multiple models
                 model: 'text-embedding-004',
             };
+        case 'mistral':
+            return {
+                model: 'mistral-embed',
+            };
+        case 'nomicai':
+            return {
+                model: 'nomic-embed-text-v1.5',
+            };
         default:
             return {};
     }
@@ -170,19 +178,7 @@ function getSourceSettings(source, request) {
  * @returns {string} The model scope for the source
  */
 function getModelScope(sourceSettings) {
-    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
-    const warningShown = global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN === 'true';
-
-    if (!scopesEnabled && !warningShown) {
-        console.log();
-        console.warn(color.red('[DEPRECATION NOTICE]'), 'Model scopes for Vectore Storage are disabled, but will soon be required.');
-        console.log(`To enable model scopes, set the ${color.cyan('vectors.enableModelScopes')} in config.yaml to ${color.green(true)}.`);
-        console.log('This message won\'t be shown again in the current session.');
-        console.log();
-        global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN = 'true';
-    }
-
-    return scopesEnabled ? (sourceSettings?.model || '') : '';
+    return (sourceSettings?.model || '');
 }
 
 /**
@@ -365,11 +361,6 @@ async function regenerateCorruptedIndexErrorHandler(req, res, error) {
 
 const router = express.Router();
 
-router.get('/scopes-enabled', (_req, res) => {
-    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
-    return res.json({ enabled: scopesEnabled });
-});
-
 router.post('/query', jsonParser, async (req, res) => {
     try {
         if (!req.body.collectionId || !req.body.searchText) {