rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add experimental setting for file translation

Browse Source
This commit is contained in:
Cohee
2024-04-21 03:24:01 +03:00
parent c2256c2ac7
commit ca89be8930
2 changed files with 20 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
public/scripts/extensions/vectors/index.js
View File

@ -53,6 +53,7 @@ const settings = {
// For files // For files
enabled_files: false, enabled_files: false,
translate_files: false,
size_threshold: 10, size_threshold: 10,
chunk_size: 5000, chunk_size: 5000,
chunk_count: 2, chunk_count: 2,
@ -437,6 +438,12 @@ async function retrieveFileChunks(queryText, collectionId) {
*/ */
async function vectorizeFile(fileText, fileName, collectionId, chunkSize) { async function vectorizeFile(fileText, fileName, collectionId, chunkSize) {
try { try {
if (settings.translate_files && typeof window['translate'] === 'function') {
console.log(`Vectors: Translating file ${fileName} to English...`);
const translatedText = await window['translate'](fileText, 'en');
fileText = translatedText;
}
const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`); const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
const chunks = splitRecursive(fileText, chunkSize); const chunks = splitRecursive(fileText, chunkSize);
console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks`, chunks); console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks`, chunks);
@ -1121,6 +1128,12 @@ jQuery(async () => {
saveSettingsDebounced(); saveSettingsDebounced();
}); });
$('#vectors_translate_files').prop('checked', settings.translate_files).on('input', () => {
settings.translate_files = !!$('#vectors_translate_files').prop('checked');
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
const validSecret = !!secret_state[SECRET_KEYS.NOMICAI]; const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key'; const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
$('#api_key_nomicai').attr('placeholder', placeholder); $('#api_key_nomicai').attr('placeholder', placeholder);

7
public/scripts/extensions/vectors/settings.html
View File

@ -107,6 +107,13 @@
</label> </label>
<div id="vectors_files_settings" class="marginTopBot5"> <div id="vectors_files_settings" class="marginTopBot5">
<label class="checkbox_label" for="vectors_translate_files" title="This can help with retrieval accuracy if using embedding models that are trained on English data. Uses the selected API from Chat Translation extension settings.">
<input id="vectors_translate_files" type="checkbox" class="checkbox">
<span data-i18n="Translate files into English before processing">
Translate files into English before processing
</span>
<i class="fa-solid fa-flask" title="Experimental feature"></i>
</label>
<div class="flex justifyCenter" title="These settings apply to files attached directly to messages."> <div class="flex justifyCenter" title="These settings apply to files attached directly to messages.">
<span>Message attachments</span> <span>Message attachments</span>
</div> </div>