rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-03-02 10:57:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix file attachment chunk size inconsistency

Browse Source
This commit is contained in:
Cohee 2024-05-13 00:27:32 +03:00
parent 38585cb6af
commit 5147233391
1 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
public/scripts/extensions/vectors/index.js
View File

@ -916,6 +916,28 @@ async function onVectorizeAllFilesClick() {
const chatAttachments = getContext().chat.filter(x => x.extra?.file).map(x => x.extra.file);
const allFiles = [...dataBank, ...chatAttachments];
/**
* Gets the chunk size for a file attachment.
* @param file {import('../../chats.js').FileAttachment} File attachment
* @returns {number} Chunk size for the file
*/
function getChunkSize(file) {
if (chatAttachments.includes(file)) {
// Convert kilobytes to string length
const thresholdLength = settings.size_threshold * 1024;
return file.size > thresholdLength ? settings.chunk_size : -1;
}
if (dataBank.includes(file)) {
// Convert kilobytes to string length
const thresholdLength = settings.size_threshold_db * 1024;
// Use chunk size from settings if file is larger than threshold
return file.size > thresholdLength ? settings.chunk_size_db : -1;
}
return -1;
}
let allSuccess = true;
for (const file of allFiles) {
@ -928,7 +950,8 @@ async function onVectorizeAllFilesClick() {
continue;
}
const result = await vectorizeFile(text, file.name, collectionId, settings.chunk_size);
const chunkSize = getChunkSize(file);
const result = await vectorizeFile(text, file.name, collectionId, chunkSize);
if (!result) {
allSuccess = false;