From ca89be8930564e89b5c45f7258a63a0c1be2f7ae Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 21 Apr 2024 03:24:01 +0300
Subject: [PATCH] Add experimental setting for file translation

---
 public/scripts/extensions/vectors/index.js      | 13 +++++++++++++
 public/scripts/extensions/vectors/settings.html |  7 +++++++
 2 files changed, 20 insertions(+)

diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js
index d228326f1..6ff5c6af5 100644
--- a/public/scripts/extensions/vectors/index.js
+++ b/public/scripts/extensions/vectors/index.js
@@ -53,6 +53,7 @@ const settings = {
 
     // For files
     enabled_files: false,
+    translate_files: false,
     size_threshold: 10,
     chunk_size: 5000,
     chunk_count: 2,
@@ -437,6 +438,12 @@ async function retrieveFileChunks(queryText, collectionId) {
  */
 async function vectorizeFile(fileText, fileName, collectionId, chunkSize) {
     try {
+        if (settings.translate_files && typeof window['translate'] === 'function') {
+            console.log(`Vectors: Translating file ${fileName} to English...`);
+            const translatedText = await window['translate'](fileText, 'en');
+            fileText = translatedText;
+        }
+
         const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
         const chunks = splitRecursive(fileText, chunkSize);
         console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks`, chunks);
@@ -1121,6 +1128,12 @@ jQuery(async () => {
         saveSettingsDebounced();
     });
 
+    $('#vectors_translate_files').prop('checked', settings.translate_files).on('input', () => {
+        settings.translate_files = !!$('#vectors_translate_files').prop('checked');
+        Object.assign(extension_settings.vectors, settings);
+        saveSettingsDebounced();
+    });
+
     const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
     const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
     $('#api_key_nomicai').attr('placeholder', placeholder);
diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html
index cdb91981b..02499d120 100644
--- a/public/scripts/extensions/vectors/settings.html
+++ b/public/scripts/extensions/vectors/settings.html
@@ -107,6 +107,13 @@
             </label>
 
             <div id="vectors_files_settings" class="marginTopBot5">
+                <label class="checkbox_label" for="vectors_translate_files" title="This can help with retrieval accuracy if using embedding models that are trained on English data. Uses the selected API from Chat Translation extension settings.">
+                    <input id="vectors_translate_files" type="checkbox" class="checkbox">
+                    <span data-i18n="Translate files into English before processing">
+                        Translate files into English before processing
+                    </span>
+                    <i class="fa-solid fa-flask" title="Experimental feature"></i>
+                </label>
                 <div class="flex justifyCenter" title="These settings apply to files attached directly to messages.">
                     <span>Message attachments</span>
                 </div>