Merge branch 'dev' of https://github.com/Cohee1207/SillyTavern into dev

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "sillytavern",
-    "version": "1.8.2",
+    "version": "1.8.3",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "sillytavern",
-            "version": "1.8.2",
+            "version": "1.8.3",
             "license": "AGPL-3.0",
             "dependencies": {
                 "@dqbd/tiktoken": "^1.0.2",

package.json

@@ -49,7 +49,7 @@
         "type": "git",
         "url": "https://github.com/SillyTavern/SillyTavern.git"
     },
-    "version": "1.8.2",
+    "version": "1.8.3",
     "scripts": {
         "start": "node server.js",
         "pkg": "pkg --compress Gzip --no-bytecode --public ."

public/script.js

@@ -757,7 +757,6 @@ $.get("/csrf-token").then(async (data) => {
 
 function checkOnlineStatus() {
     ///////// REMOVED LINES THAT DUPLICATE RA_CHeckOnlineStatus FEATURES
-
     if (online_status == "no_connection") {
         $("#online_status_indicator2").css("background-color", "red");  //Kobold
         $("#online_status_text2").html("No connection...");

public/scripts/extensions/infinity-context/index.js

@@ -123,7 +123,7 @@ async function loadSettings() {
 function onStrategyChange() {
     console.debug('changing chromadb strat');
     extension_settings.chromadb.strategy = $('#chromadb_strategy').val();
-    if(extension_settings.chromadb.strategy === "custom"){
+    if (extension_settings.chromadb.strategy === "custom") {
         $('#chromadb_custom_depth').show();
         $('label[for="chromadb_custom_depth"]').show();
         $('#chromadb_custom_msg').show();
@@ -186,6 +186,16 @@ function onFileSplitLengthInput() {
     saveSettingsDebounced();
 }
 
+function onChunkNLInput() {
+    let shouldSplit = $('#onChunkNLInput').is(':checked');
+    if (shouldSplit) {
+        extension_settings.chromadb.file_split_type = "newline";
+    } else {
+        extension_settings.chromadb.file_split_type = "length";
+    }
+    saveSettingsDebounced();
+}
+
 function checkChatId(chat_id) {
     if (!chat_id || chat_id.trim() === '') {
         toastr.error('Please select a character and try again.');
@@ -396,7 +406,7 @@ async function queryMultiMessages(chat_id, query) {
     const context = getContext();
     const response = await fetch("/getallchatsofcharacter", {
         method: 'POST',
-        body: JSON.stringify({ avatar_url: context.characters[context.characterId].avatar}),
+        body: JSON.stringify({ avatar_url: context.characters[context.characterId].avatar }),
         headers: getRequestHeaders(),
     });
     if (!response.ok) {
@@ -438,8 +448,14 @@ async function onSelectInjectFile(e) {
     try {
         toastr.info('This may take some time, depending on the file size', 'Processing...');
         const text = await getFileText(file);
-
-        const split = splitRecursive(text, extension_settings.chromadb.file_split_length).filter(onlyUnique);
+        extension_settings.chromadb.file_split_type = "newline";
+        //allow splitting on newlines or splitrecursively
+        let split = [];
+        if (extension_settings.chromadb.file_split_type == "newline") {
+            split = text.split(/\r?\n/).filter(onlyUnique);
+        } else {
+            split = splitRecursive(text, extension_settings.chromadb.file_split_length).filter(onlyUnique);
+        }
         const baseDate = Date.now();
 
         const messages = split.map((m, i) => ({
@@ -538,10 +554,19 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => {
     let recallMsg = extension_settings.chromadb.recall_msg || defaultSettings.chroma_default_msg;
     const chromaDepth = extension_settings.chromadb.chroma_depth;
     const chromaSortStrategy = extension_settings.chromadb.sort_strategy;
+
+    //log the current settings
+    console.debug("CHROMADB: Current settings: %o", extension_settings.chromadb);
+
+
     if (currentChatId) {
         const messagesToStore = chat.slice(0, -extension_settings.chromadb.keep_context);
+        //log the messages to store
+        console.debug("CHROMADB: Messages to store: %o", messagesToStore);
 
         if (messagesToStore.length > 0 || extension_settings.chromadb.freeze) {
+            //log the messages to store length vs keep context
+            console.debug("CHROMADB: Messages to store length vs keep context: %o vs %o", messagesToStore.length, extension_settings.chromadb.keep_context);
             await addMessages(currentChatId, messagesToStore);
 
             const lastMessage = chat[chat.length - 1];
@@ -549,18 +574,17 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => {
             let queriedMessages;
             console.debug(recallStrategy)
             if (lastMessage) {
-                if (recallStrategy === 'multichat'){
-                    console.log("Utilizing multichat")
+                if (recallStrategy === 'multichat') {
                     queriedMessages = await queryMultiMessages(currentChatId, lastMessage.mes);
                 }
-                else{
+                else {
                     queriedMessages = await queryMessages(currentChatId, lastMessage.mes);
                 }
 
-                if(chromaSortStrategy === "date"){
+                if (chromaSortStrategy === "date") {
                     queriedMessages.sort((a, b) => a.date - b.date);
                 }
-                else{
+                else {
                     queriedMessages.sort((a, b) => b.distance - a.distance);
                 }
 
@@ -625,10 +649,10 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => {
                     let chatset = new Set(chat.map(obj => obj.mes));
                     newChat = newChat.filter(obj => !chatset.has(obj.mes));
 
-                    if(chromaDepth === -1){
+                    if (chromaDepth === -1) {
                         chat.splice(chat.length, 0, ...newChat);
                     }
-                    else{
+                    else {
                         chat.splice(chromaDepth, 0, ...newChat);
                     }
                 }
@@ -639,7 +663,6 @@ window.chromadb_interceptGeneration = async (chat, maxContext) => {
                     chat.splice(0, messagesToStore.length, ...newChat);
 
                 }
-                console.log('ChromaDB chat after injection', chat);
             }
         }
     }
@@ -725,6 +748,10 @@ jQuery(async () => {
                 <input type="checkbox" id="chromadb_auto_adjust" />
                 <span>Use % strategy</span>
             </label>
+            <label class="checkbox_label" for="chromadb_chunk_nl" title="Chunk injected documents on newline instead of at set character size." >
+                <input type="checkbox" id="chromadb_chunk_nl" />
+                <span>Chunk on Newlines</span>
+            </label>
             <div class="flex-container spaceEvenly">
                 <div id="chromadb_inject" title="Upload custom textual data to use in the context of the current chat" class="menu_button">
                     <i class="fa-solid fa-file-arrow-up"></i>
@@ -766,6 +793,7 @@ jQuery(async () => {
     $('#chromadb_purge').on('click', onPurgeClick);
     $('#chromadb_export').on('click', onExportClick);
     $('#chromadb_freeze').on('input', onFreezeInput);
+    $('#chromadb_chunk_nl').on('input', onChunkNLInput);
     $('#chromadb_auto_adjust').on('input', onAutoAdjustInput);
     $('#chromadb_keep_context_proportion').on('input', onKeepContextProportionInput);
     await loadSettings();