rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Custom vector chunk boundary

Browse Source
This commit is contained in:
Cohee
2024-06-16 02:16:27 +03:00
parent 67b7cbe920
commit bba16f5263
2 changed files with 34 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
public/scripts/extensions/vectors/index.js
View File

@ -50,6 +50,7 @@ const settings = {
summarize_sent: false,
summary_source: 'main',
summary_prompt: 'Pause your roleplay. Summarize the most important parts of the message. Limit yourself to 250 words or less. Your response should include nothing but the summary.',
force_chunk_delimiter: '',
// For chats
enabled_chats: false,
@ -153,6 +154,20 @@ async function onVectorizeAllClick() {
let syncBlocked = false;
/**
* Gets the chunk delimiters for splitting text.
* @returns {string[]} Array of chunk delimiters
*/
function getChunkDelimiters() {
const delimiters = ['\n\n', '\n', ' ', ''];
if (settings.force_chunk_delimiter) {
delimiters.unshift(settings.force_chunk_delimiter);
}
return delimiters;
}
/**
* Splits messages into chunks before inserting them into the vector index.
* @param {object[]} items Array of vector items
@ -166,7 +181,7 @@ function splitByChunks(items) {
const chunkedItems = [];
for (const item of items) {
const chunks = splitRecursive(item.text, settings.message_chunk_size);
const chunks = splitRecursive(item.text, settings.message_chunk_size, getChunkDelimiters());
for (const chunk of chunks) {
const chunkedItem = { ...item, text: chunk };
chunkedItems.push(chunkedItem);
@ -484,9 +499,10 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overla
const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
const overlapSize = Math.round(chunkSize * overlapPercent / 100);
const delimiters = getChunkDelimiters();
// Overlap should not be included in chunk size. It will be later compensated by overlapChunks
chunkSize = overlapSize > 0 ? (chunkSize - overlapSize) : chunkSize;
const chunks = splitRecursive(fileText, chunkSize).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);
const chunks = splitRecursive(fileText, chunkSize, delimiters).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);
console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);
const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));
@ -1480,6 +1496,12 @@ jQuery(async () => {
saveSettingsDebounced();
});
$('#vectors_force_chunk_delimiter').prop('checked', settings.force_chunk_delimiter).on('input', () => {
settings.force_chunk_delimiter = String($('#vectors_force_chunk_delimiter').val());
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
$('#api_key_nomicai').attr('placeholder', placeholder);