Add ability to inject custom files into the prompt (broken ATM)

public/index.html

@@ -43,6 +43,7 @@
     <script src="scripts/cropper.min.js"></script>
     <script src="scripts/jquery-cropper.min.js"></script>
     <script src="scripts/toastr.min.js"></script>
+    <script src="scripts/short-unique-id.min.js"></script>
     <script type="module" src="scripts/eventemitter.js"></script>
     <script type="module" src="scripts/power-user.js"></script>
     <script type="module" src="scripts/swiped-events.js"></script>

public/script.js

@@ -840,7 +840,7 @@ async function getCharacters() {
             $("#avatar_url_pole").val(characters[this_chid].avatar);
         }
         await getGroups();
-        printCharacters();
+        await printCharacters();
     }
 }
 
@@ -1219,7 +1219,7 @@ function addOneMessage(mes, { type = "normal", insertAfter = null, scroll = true
         $(".mes_prompt").hide();
         //console.log(itemizedPrompts);
     } else {
-        //console.log('skipping prompt data for User Message'); 
+        //console.log('skipping prompt data for User Message');
     }
 
     newMessage.find('.avatar img').on('error', function () {

public/scripts/extensions/infinity-context/index.js

@@ -1,10 +1,12 @@
 import { saveSettingsDebounced, getCurrentChatId } from "../../../script.js";
 import { getApiUrl, extension_settings } from "../../extensions.js";
-import { splitRecursive } from "../../utils.js";
+import { getFileText, getStringHash, splitRecursive } from "../../utils.js";
 export { MODULE_NAME };
 
 const MODULE_NAME = 'chromadb';
 
+const fileSplitLength = 2048;
+
 const defaultSettings = {
     keep_context: 10,
     keep_context_min: 1,
@@ -115,6 +117,52 @@ async function queryMessages(chat_id, query) {
     return [];
 }
 
+async function onSelectInjectFile(e) {
+    const file = e.target.files[0];
+
+    if (!file) {
+        return;
+    }
+
+    try {
+        const currentChatId = getCurrentChatId();
+        const text = await getFileText(file);
+
+        const split = splitRecursive(text, fileSplitLength);
+
+        const messages = split.map(m => ({
+            id: `${getStringHash(file.name)}-${getStringHash(m)}`,
+            role: 'assistant', // probably need a system role?
+            content: m,
+            date: Date.now(),
+            meta: file.name,
+        }));
+
+        const url = new URL(getApiUrl());
+        url.pathname = '/api/chromadb';
+
+        const addMessagesResult = await fetch(url, {
+            method: 'POST',
+            headers: postHeaders,
+            body: JSON.stringify({ chat_id: currentChatId, messages: messages }),
+        });
+
+        if (addMessagesResult.ok) {
+            const addMessagesData = await addMessagesResult.json();
+
+            toastr.info(`Number of chunks: ${addMessagesData.count}`, 'Injected successfully!');
+            return addMessagesData;
+        }
+    }
+    catch (error) {
+        console.log(error);
+        toastr.error('Something went wrong while injecting the data');
+    }
+    finally {
+        e.target.form.reset();
+    }
+}
+
 window.chromadb_interceptGeneration = async (chat) => {
     const currentChatId = getCurrentChatId();
 
@@ -132,9 +180,9 @@ window.chromadb_interceptGeneration = async (chat) => {
                 queriedMessages.sort((a, b) => a.date - b.date);
 
                 const newChat = queriedMessages.map(m => JSON.parse(m.meta));
-                
+
                 chat.splice(0, messagesToStore.length, ...newChat);
-                
+
                 console.log('ChromaDB chat after injection', chat);
             }
         }
@@ -156,13 +204,20 @@ jQuery(async () => {
             <input id="chromadb_n_results" type="range" min="${defaultSettings.n_results_min}" max="${defaultSettings.n_results_max}" step="${defaultSettings.n_results_step}" value="${defaultSettings.n_results}" />
             <label for="chromadb_split_length">Max length for message chunks (<span id="chromadb_split_length_value"></span>)</label>
             <input id="chromadb_split_length" type="range" min="${defaultSettings.split_length_min}" max="${defaultSettings.split_length_max}" step="${defaultSettings.split_length_step}" value="${defaultSettings.split_length}" />
+            <div id="chromadb_inject" title="Upload custom textual data to use in the context of the current chat" class="menu_button">
+                <i class="fa-solid fa-file-arrow-up"></i>
+                <span>Inject data to the context (TXT file)</span>
+            </div>
         </div>
+        <form><input id="chromadb_inject_file" type="file" accept="text/plain" hidden></form>
     </div>`;
 
     $('#extensions_settings').append(settingsHtml);
     $('#chromadb_keep_context').on('input', onKeepContextInput);
     $('#chromadb_n_results').on('input', onNResultsInput);
     $('#chromadb_split_length').on('input', onSplitLengthInput);
+    $('#chromadb_inject').on('click', () => $('#chromadb_inject_file').trigger('click'));
+    $('#chromadb_inject_file').on('change', onSelectInjectFile);
 
     await loadSettings();
 });

public/scripts/extensions/infinity-context/style.css

@@ -0,0 +1,7 @@
+#chromadb_inject {
+    width: fit-content;
+    display: flex;
+    gap: 10px;
+    align-items: center;
+    flex-direction: row;
+}

public/scripts/short-unique-id.min.js

@@ -0,0 +1,2 @@
+var ShortUniqueId=(()=>{var h=Object.defineProperty;var _=Object.getOwnPropertySymbols;var m=Object.prototype.hasOwnProperty,f=Object.prototype.propertyIsEnumerable;var g=(r,t,i)=>t in r?h(r,t,{enumerable:!0,configurable:!0,writable:!0,value:i}):r[t]=i,l=(r,t)=>{for(var i in t||(t={}))m.call(t,i)&&g(r,i,t[i]);if(_)for(var i of _(t))f.call(t,i)&&g(r,i,t[i]);return r};var I=r=>h(r,"__esModule",{value:!0});var U=(r,t)=>{I(r);for(var i in t)h(r,i,{get:t[i],enumerable:!0})};var y={};U(y,{DEFAULT_UUID_LENGTH:()=>d,default:()=>n});var b="4.4.4";var d=6,q={dictionary:"alphanum",shuffle:!0,debug:!1,length:d},p=class extends Function{constructor(t={}){super();this.dictIndex=0;this.dictRange=[];this.lowerBound=0;this.upperBound=0;this.dictLength=0;this._digit_first_ascii=48;this._digit_last_ascii=58;this._alpha_lower_first_ascii=97;this._alpha_lower_last_ascii=123;this._hex_last_ascii=103;this._alpha_upper_first_ascii=65;this._alpha_upper_last_ascii=91;this._number_dict_ranges={digits:[this._digit_first_ascii,this._digit_last_ascii]};this._alpha_dict_ranges={lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]};this._alpha_lower_dict_ranges={lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii]};this._alpha_upper_dict_ranges={upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]};this._alphanum_dict_ranges={digits:[this._digit_first_ascii,this._digit_last_ascii],lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]};this._alphanum_lower_dict_ranges={digits:[this._digit_first_ascii,this._digit_last_ascii],lowerCase:[this._alpha_lower_first_ascii,this._alpha_lower_last_ascii]};this._alphanum_upper_dict_ranges={digits:[this._digit_first_ascii,this._digit_last_ascii],upperCase:[this._alpha_upper_first_ascii,this._alpha_upper_last_ascii]};this._hex_dict_ranges={decDigits:[this._digit_first_ascii,this._digit_last_ascii],alphaDigits:[this._alpha_lower_first_ascii,this._hex_last_ascii]};this.log=(...t)=>{let i=[...t];if(i[0]=`[short-unique-id] ${t[0]}`,this.debug===!0&&typeof console!="undefined"&&console!==null)return console.log(...i)};this.setDictionary=(t,i)=>{let e;if(t&&Array.isArray(t)&&t.length>1)e=t;else{e=[];let s;this.dictIndex=s=0;let a=`_${t}_dict_ranges`,o=this[a];Object.keys(o).forEach(c=>{let u=c;for(this.dictRange=o[u],this.lowerBound=this.dictRange[0],this.upperBound=this.dictRange[1],this.dictIndex=s=this.lowerBound;this.lowerBound<=this.upperBound?s<this.upperBound:s>this.upperBound;this.dictIndex=this.lowerBound<=this.upperBound?s+=1:s-=1)e.push(String.fromCharCode(this.dictIndex))})}if(i){let s=.5;e=e.sort(()=>Math.random()-s)}this.dict=e,this.dictLength=this.dict.length,this.counter=0};this.seq=()=>this.sequentialUUID();this.sequentialUUID=()=>{let t,i,e="";t=this.counter;do i=t%this.dictLength,t=Math.trunc(t/this.dictLength),e+=this.dict[i];while(t!==0);return this.counter+=1,e};this.randomUUID=(t=this.uuidLength||d)=>{let i,e,s;if(t===null||typeof t=="undefined"||t<1)throw new Error("Invalid UUID Length Provided");let a=t>=0;for(i="",s=0;s<t;s+=1)e=parseInt((Math.random()*this.dictLength).toFixed(0),10)%this.dictLength,i+=this.dict[e];return i};this.availableUUIDs=(t=this.uuidLength)=>parseFloat(Math.pow([...new Set(this.dict)].length,t).toFixed(0));this.approxMaxBeforeCollision=(t=this.availableUUIDs(this.uuidLength))=>parseFloat(Math.sqrt(Math.PI/2*t).toFixed(20));this.collisionProbability=(t=this.availableUUIDs(this.uuidLength),i=this.uuidLength)=>parseFloat((this.approxMaxBeforeCollision(t)/this.availableUUIDs(i)).toFixed(20));this.uniqueness=(t=this.availableUUIDs(this.uuidLength))=>{let i=parseFloat((1-this.approxMaxBeforeCollision(t)/t).toFixed(20));return i>1?1:i<0?0:i};this.getVersion=()=>this.version;this.stamp=t=>{if(typeof t!="number"||t<10)throw new Error("Param finalLength must be number greater than 10");let i=Math.floor(+new Date/1e3).toString(16),e=t-9,s=Math.round(Math.random()*(e>15?15:e)),a=this.randomUUID(e);return`${a.substr(0,s)}${i}${a.substr(s)}${s.toString(16)}`};this.parseStamp=t=>{if(t.length<10)throw new Error("Stamp length invalid");let i=parseInt(t.substr(t.length-1,1),16);return new Date(parseInt(t.substr(i,8),16)*1e3)};let i=l(l({},q),t);this.counter=0,this.debug=!1,this.dict=[],this.version=b;let{dictionary:e,shuffle:s,length:a}=i;return this.uuidLength=a,this.setDictionary(e,s),this.debug=i.debug,this.log(this.dict),this.log(`Generator instantiated with Dictionary Size ${this.dictLength}`),new Proxy(this,{apply:(o,c,u)=>this.randomUUID(...u)})}},n=p;n.default=p;return y;})();
+'undefined'!=typeof module&&(module.exports=ShortUniqueId.default),'undefined'!=typeof window&&(ShortUniqueId=ShortUniqueId.default);

public/scripts/utils.js

@@ -35,6 +35,19 @@ export async function urlContentToDataUri(url, params) {
     });
 }
 
+export function getFileText(file) {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.readAsText(file);
+        reader.onload = function () {
+            resolve(reader.result);
+        };
+        reader.onerror = function (error) {
+            reject(error);
+        };
+    });
+}
+
 export function getBase64Async(file) {
     return new Promise((resolve, reject) => {
         const reader = new FileReader();
@@ -261,7 +274,7 @@ export function splitRecursive(input, length, delimitiers = ['\n\n', '\n', ' ',
     let currentChunk = '';
     for (let i = 0; i < flatParts.length;) {
         currentChunk = flatParts[i];
-        let j = i + 1; 
+        let j = i + 1;
         while (j < flatParts.length) {
             const nextChunk = flatParts[j];
             if (currentChunk.length + nextChunk.length + delim.length <= length) {