#21 Adds basic support of World Info files loading and synchronization with Kobold API

public/KoboldAI Worlds/Sample.json

@@ -0,0 +1,65 @@
+{
+	"folders": {
+		"New Folder": [
+			0,
+			1
+		]
+	},
+	"entries": {
+		"0": {
+			"uid": 0,
+			"title": "New World Info Entry",
+			"key": [
+				"Test",
+				"test"
+			],
+			"keysecondary": [
+				"ttest    "
+			],
+			"folder": "New Folder",
+			"constant": false,
+			"manual_text": "tststststs",
+			"content": "tststststs",
+			"comment": "",
+			"type": "wi",
+			"token_length": 0,
+			"selective": true,
+			"used_in_game": false,
+			"wpp": {
+				"name": "",
+				"type": "",
+				"format": "W++",
+				"attributes": {}
+			},
+			"use_wpp": false,
+			"object_type": null
+		},
+		"1": {
+			"uid": 1,
+			"title": "New World Info Entry",
+			"key": [
+				"kkkk"
+			],
+			"keysecondary": [
+				"kkkkk"
+			],
+			"folder": "New Folder",
+			"constant": false,
+			"manual_text": "gsfgsafgasgsf",
+			"content": "gsfgsafgasgsf",
+			"comment": "",
+			"type": "wi",
+			"token_length": 0,
+			"selective": true,
+			"used_in_game": false,
+			"wpp": {
+				"name": "",
+				"type": "",
+				"format": "W++",
+				"attributes": {}
+			},
+			"use_wpp": false,
+			"object_type": null
+		}
+	}
+}

public/index.html

@@ -102,6 +102,9 @@
             var user_avatar = 'you.png';
             var temp = 0.5;
             var amount_gen = 80;
+            var kobold_world = null;
+            var koboldai_world_names;
+            var kobold_world_synced = false;
             var max_context = 2048;//2048;
             var rep_pen = 1;
             var rep_pen_size = 100;
@@ -263,6 +266,7 @@
                             resultCheckStatus();
                             if(online_status !== 'no_connection'){
                                 var checkStatusNow = setTimeout(getStatus, 3000);//getStatus();
+                                syncKoboldWorldInfo(false);
                             }
                         },
                         error: function (jqXHR, exception) {
@@ -1927,6 +1931,19 @@
 
             });
 
+            $("#world_info").change(function() {
+                const selectedWorld = $('#world_info').find(":selected").val();
+                kobold_world_synced = false;
+                kobold_world = null;
+
+                if (selectedWorld !== 'None') {
+                    const worldIndex = Number(selectedWorld);
+                    kobold_world = !isNaN(worldIndex) ? koboldai_world_names[worldIndex] : null;
+                }
+
+                syncKoboldWorldInfo(true);
+            });
+
             $( "#settings_perset" ).change(function() {
                 
                 if($('#settings_perset').find(":selected").val() != 'gui'){
@@ -2005,6 +2022,7 @@
                     main_api = 'kobold';
                     $('#max_context_block').css('display', 'block');
                     $('#amount_gen_block').css('display', 'block');
+                    $('#world_info').css('display', 'block');
                 }
                 if($('#main_api').find(":selected").val() == 'novel'){
                     $('#kobold_api').css("display", "none");
@@ -2012,6 +2030,7 @@
                     main_api = 'novel';
                     $('#max_context_block').css('display', 'none');
                     $('#amount_gen_block').css('display', 'none');
+                    $('#world_info').css('display', 'none');
                 }
             }
             async function getUserAvatars(){
@@ -2037,7 +2056,35 @@
 
                 }
             }
+            async function syncKoboldWorldInfo(force) {
+                // Don't sync if no world selected
+                if (!kobold_world || online_status == 'no_connection') {
+                    return;
+                }
 
+                // Don't sync if synced and not forcing
+                if (kobold_world_synced && !force) {
+                    return;
+                }
+
+                const response = await fetch("/synckoboldworld", {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json" },
+                    body: JSON.stringify({ "name": kobold_world })
+                });
+
+                if (response.ok === true) {
+                    const syncData = await response.json();
+
+                    if (syncData.ok) {
+                        kobold_world_synced = true;
+                    }
+
+                    if (syncData.busy) {
+                        console.log('Sync API is busy');
+                    }
+                }
+            }
             
             $(document).on('input', '#temp', function() {
                 temp = $(this).val();
@@ -2181,6 +2228,11 @@
                             koboldai_setting_names = {};
                             koboldai_setting_names = arr_holder;
 
+                            koboldai_world_names = data.koboldai_world_names;
+                            koboldai_world_names.forEach((item, i) => {
+                                $('#world_info').append(`<option value='${i}'>${item}</option>`);
+                            });
+
                             preset_settings = settings.preset_settings;
 
                             temp = settings.temp;
@@ -2322,6 +2374,7 @@
                             location.reload();
                         }
                         
+                        syncKoboldWorldInfo(false);
                     },
                     error: function (jqXHR, exception) {
                         console.log(exception);
@@ -2946,6 +2999,10 @@
                     <h4>Repetition Penalty Range</h4><h5 id="rep_pen_size_counter">select</h5>
                     <input type="range" id="rep_pen_size" name="volume" min="0" max="2048" step="1">
                 </div>
+                <h4>World Info</h4><!-- <h5>More info (<a href="http://example.com" target="_blank">?</a>)</h5> -->
+                <select id="world_info" class="option_select_right_menu">
+                    <option value="None">None</option>
+                </select>
             </div>
             <div id="novel_api" style="display: none;position: relative;">
                 <div style="position: absolute; right:152px; top:-25px; opacity:0.25;"><a href="https://novelai.net/" target="_blank"><img src="img/novelai.png" style="width:auto;height:22px;"></a></div>

server.js

@@ -629,6 +629,14 @@ app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's cod
         new Date(fs.statSync(`public/KoboldAI Settings/${a}`).mtime)
     );
 
+    const worldFiles = fs
+        .readdirSync('public/KoboldAI Worlds')
+        .filter(file => {
+            return path.extname(file).toLowerCase() === '.json';
+        })
+        .sort((a, b) => a < b);
+    const koboldai_world_names = worldFiles.map(item => path.parse(item).name);
+
     files.forEach(item => {
         const file = fs.readFileSync(
         `public/KoboldAI Settings/${item}`,
@@ -671,11 +679,37 @@ app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's cod
         settings,
         koboldai_settings,
         koboldai_setting_names,
+        koboldai_world_names,
         novelai_settings,
         novelai_setting_names
     });
 });
 
+// Work around to disable parallel requests to endpoint
+let kobold_world_sync_busy = false;
+
+app.post('/synckoboldworld', jsonParser, async (request, response) => {
+    if(!request.body) return response.sendStatus(400);
+
+    if (!api_server || kobold_world_sync_busy) {
+        response.send({ busy: true });
+        return;
+    }
+
+    try {
+        kobold_world_sync_busy = true;
+        const worldName = request.body.name;
+        await synchronizeKoboldWorldInfo(worldName);
+        response.send({ ok: true });
+        console.log('World info synchronized with Kobold');
+    } catch (err) {
+        console.error(`Error during world synchronization: ${JSON.stringify(err)}`);
+        response.sendStatus(500);
+    } finally {
+        kobold_world_sync_busy = false;
+    }
+});
+
 
 function getCharaterFile(directories,response,i){ //old need del
     if(directories.length > i){
@@ -1038,56 +1072,203 @@ app.post("/importchat", urlencodedParser, function(request, response){
 
 });
 
+function findTavernWorldEntry(info, key) {
+    for (const entryId in info.entries) {
+        const keyString = info.entries[entryId].key.join(',');
 
-async function generateKoboldWorldInfo(worldInfoName) {
+        if (keyString === key) {
-    let worldInfoUid = null;
+            return info.entries[entryId];
-
-    try {
-        const filename = `${worldInfoName}.json`;
-        const pathToWorldInfo = path.join('public/worldinfos/', filename);
-        const koboldFolderName = getKoboldWorldInfoName(worldInfoName);
-        if (!fs.existsSync(pathToWorldInfo)) {
-            console.log(`World info file ${filename} doesn't exist. Skipping...`);
-            return null;
         }
-
-        const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
-
-        const baseArgs = {
-            headers: { "Content-Type": "application/json" }
-        };
-
-        // Check is folder exists
-        const foldersData = await getToPromise(api_server + "/v1/world_info/folders", baseArgs);
-        if (foldersData?.folders?.length) {
-            const existingFolder = foldersData.folders.find(x => x.name === koboldFolderName);
-            if (existingFolder) {
-                console.log(`World info folder found: ${existingFolder.uid}`);
-                worldInfoUid = existingFolder.uid;
-            }
-        }
-
-        // Create folder if not already exists
-        if (!worldInfoUid) {
-            const createArgs = Object.assign(baseArgs, { data: {} });
-            const createdFolder = await postToPromise(api_server + "/v1/world_info/folders", createArgs);
-            console.log(`World info folder created: ${createdFolder.uid}`);
-            worldInfoUid = createdFolder.uid;
-
-            // Set a name so we could find the folder later
-            const setNameArgs = Object.assign(baseArgs, { data: { value: koboldFolderName } });
-            await putToPromise(api_server + `/v1/world_info/folders/${worldInfoUid}/name`, setNameArgs);
-            console.log(`World info folder name set: ${koboldFolderName}`);
-        }
-    } catch (err) {
-        console.error(err);
-        return null;
     }
 
-    return worldInfoUid;
+    return null;
 }
 
-function putToPromise(url, args) {
+async function synchronizeKoboldWorldInfo(worldInfoName) {
+    const koboldFolderName = getKoboldWorldInfoName(worldInfoName);
+    const filename = `${worldInfoName}.json`;
+    const pathToWorldInfo = path.join('public/KoboldAI Worlds/', filename);
+
+    if (!fs.existsSync(pathToWorldInfo)) {
+        throw new Error(`World info file ${filename} doesn't exist.`);
+    }
+
+    const tavernWorldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
+    const tavernWorldInfo = JSON.parse(tavernWorldInfoText);
+
+    const baseRequestArgs = { headers: { "Content-Type": "application/json" } };
+
+    // Get existing world info
+    const koboldWorldInfo = await getAsync(api_server + "/v1/world_info", baseRequestArgs);
+
+    // Validate kobold world info
+    let {
+        shouldCreateWorld,
+        koboldWorldUid,
+        tavernEntriesToCreate,
+        koboldEntriesToDelete,
+        koboldFoldersToDelete,
+    } = await validateKoboldWorldInfo(koboldFolderName, koboldWorldInfo, tavernWorldInfo);
+
+    // Create folder if not already exists
+    if (shouldCreateWorld) {
+        const createdFolder = await postAsync(`${api_server}/v1/world_info/folders`, { data: {}, ...baseRequestArgs });
+        koboldWorldUid = createdFolder.uid;
+
+        // Set a name so we could find the folder later
+        const setNameArgs = { data: { value: koboldFolderName }, ...baseRequestArgs };
+        await putAsync(`${api_server}/v1/world_info/folders/${koboldWorldUid}/name`, setNameArgs);
+
+        // Create all world info entries
+        tavernEntriesToCreate.push(...Object.keys(tavernWorldInfo.entries));
+    }
+
+    if (koboldFoldersToDelete.length) {
+        await Promise.all(koboldFoldersToDelete.map((uid) => deleteAsync(api_server + `/v1/world_info/folders/${uid}`, baseRequestArgs)));
+    }
+
+    if (koboldEntriesToDelete.length) {
+        await Promise.all(koboldEntriesToDelete.map((uid) => deleteAsync(`${api_server}/v1/world_info/${uid}`)));
+    }
+
+    if (tavernEntriesToCreate.length && koboldWorldUid) {
+        for (const tavernUid in tavernEntriesToCreate) {
+            const tavernEntry = tavernWorldInfo.entries[tavernUid];
+            const koboldEntry = await postAsync(`${api_server}/v1/world_info/folders/${koboldWorldUid}`, { data: {}, ...baseRequestArgs });
+            await setKoboldEntryData(tavernEntry, koboldEntry);
+        }
+    }
+}
+
+async function setKoboldEntryData(tavernEntry, koboldEntry) {
+    const baseRequestArgs = { headers: { "Content-Type": "application/json" } };
+    const setDataRequests = [];
+
+    // 1. Set primary key
+    if (tavernEntry.key.length) {
+        const keyArgs = { data: { value: tavernEntry.key.join(',') }, ...baseRequestArgs };
+        await putAsync(`${api_server}/v1/world_info/${koboldEntry.uid}/key`, keyArgs);
+    }
+
+    // 2. Set secondary key
+    if (tavernEntry.keysecondary) {
+        const keySecondaryArgs = { data: { value: tavernEntry.keysecondary.join(',') }, ...baseRequestArgs };
+        await putAsync(`${api_server}/v1/world_info/${koboldEntry.uid}/keysecondary`, keySecondaryArgs);
+    }
+
+    // 3. Set content
+    if (tavernEntry.content) {
+        const contentArgs = { data: { value: tavernEntry.content }, ...baseRequestArgs };
+        await putAsync(`${api_server}/v1/world_info/${koboldEntry.uid}/content`, contentArgs);
+    }
+
+    // 4. Set comment
+    if (tavernEntry.comment) {
+        const commentArgs = { data: { value: tavernEntry.comment }, ...baseRequestArgs };
+        await putAsync(`${api_server}/v1/world_info/${koboldEntry.uid}/comment`, commentArgs);
+    };
+
+    /* Can't set these via API due to bug in Kobold)
+    // 5. Set constant flag
+    if (tavernEntry.constant) {
+        const constantArgs = { data: { value: tavernEntry.constant.toString() }, ...baseRequestArgs };
+        await putToPromise(`${api_server}/v1/world_info/${koboldEntry.uid}/constant`, constantArgs);
+
+    }
+    // 6. Set selective flag
+    if (tavernEntry.selective) {
+        const selectiveArgs = { data: { value: tavernEntry.selective.toString() }, ...baseRequestArgs };
+        await putToPromise(`${api_server}/v1/world_info/${koboldEntry.uid}/selective`, selectiveArgs);
+    }
+    */
+
+    return setDataRequests;
+}
+
+async function validateKoboldWorldInfo(koboldFolderName, koboldWorldInfo, tavernWorldInfo) {
+    let shouldCreateWorld = true;
+    let koboldWorldUid = null;
+
+    const koboldEntriesToDelete = []; // KoboldUIDs
+    const koboldFoldersToDelete = []; // KoboldUIDs
+    const tavernEntriesToCreate = []; // TavernUIDs
+
+    if (koboldWorldInfo?.folders?.length) {
+        let existingFolderAlreadyFound = false;
+
+        for (const folderUid in koboldWorldInfo.folders) {
+            const folder = koboldWorldInfo.folders[folderUid];
+            // Don't care about non-Tavern folders
+            if (!isTavernKoboldWorldInfo(folder.name)) {
+                continue;
+            }
+
+            // Other Tavern folders should be deleted (including dupes). If folder name selected is null, then delete anyway to clean-up
+            if (!koboldFolderName || folder.name !== koboldFolderName || existingFolderAlreadyFound) {
+                koboldFoldersToDelete.push(folder.uid);
+            }
+
+            // Validate existing entries in Kobold world
+            if (folder.name === koboldFolderName) {
+                existingFolderAlreadyFound = true;
+                shouldCreateWorld = false;
+                koboldWorldUid = folder.uid;
+                if (folder.entries?.length) {
+                    const foundTavernEntries = [];
+                    for (const koboldEntryUid in folder.entries) {
+                        const koboldEntry = folder.entries[koboldEntryUid];
+                        const tavernEntry = findTavernWorldEntry(tavernWorldInfo, koboldEntry.key);
+
+                        if (tavernEntry) {
+                            foundTavernEntries.push(tavernEntry.uid);
+                            if (isEntryOutOfSync(tavernEntry, koboldEntry)) {
+                                // Entry is out of sync. Should be recreated
+                                koboldEntriesToDelete.push(koboldEntry.uid);
+                                tavernEntriesToCreate.push(tavernEntry.uid);
+                            }
+                        }
+                        else {
+                            // We don't have that entry in our world. It should be deleted
+                            koboldEntriesToDelete.push(koboldEntry.uid);
+                        }
+                    }
+
+                    // Check if every tavern entry was found in kobold world
+                    for (const tavernEntryUid in tavernWorldInfo.entries) {
+                        if (!foundTavernEntries.includes(tavernWorldInfo.entries[tavernEntryUid].uid)) {
+                            tavernEntriesToCreate.push(tavernWorldInfo.entries[tavernEntryUid].uid);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return { shouldCreateWorld, koboldWorldUid, tavernEntriesToCreate, koboldEntriesToDelete, koboldFoldersToDelete };
+}
+
+function isEntryOutOfSync(tavernEntry, koboldEntry) {
+    return tavernEntry.content !== koboldEntry.content ||
+        tavernEntry.comment !== koboldEntry.comment ||
+        tavernEntry.selective !== koboldEntry.selective ||
+        tavernEntry.constant !== koboldEntry.constant ||
+        tavernEntry.key.join(',') !== koboldEntry.key ||
+        tavernEntry.keysecondary(',') !== koboldEntry.keysecondary;
+}
+
+// ** REST CLIENT ASYNC WRAPPERS **
+function deleteAsync(url, args) {
+    return new Promise((resolve, reject) => {
+        client.delete(url, args, (data, response) => {
+            if (response.statusCode >= 400) {
+                reject(data);
+            }
+            resolve(data);
+        }).on('error', e => reject(e));
+    })
+}
+
+function putAsync(url, args) {
     return new Promise((resolve, reject) => {
         client.put(url, args, (data, response) => {
             if (response.statusCode >= 400) {
@@ -1098,7 +1279,7 @@ function putToPromise(url, args) {
     })
 }
 
-function postToPromise(url, args) {
+function postAsync(url, args) {
     return new Promise((resolve, reject) => {
         client.post(url, args, (data, response) => {
             if (response.statusCode >= 400) {
@@ -1109,7 +1290,7 @@ function postToPromise(url, args) {
     })
 }
 
-function getToPromise(url, args) {
+function getAsync(url, args) {
     return new Promise((resolve, reject) => {
         client.get(url, args, (data, response) => {
             if (response.statusCode >= 400) {
@@ -1119,11 +1300,15 @@ function getToPromise(url, args) {
         }).on('error', e => reject(e));
     })
 }
+// ** END **
 
 function getKoboldWorldInfoName(worldInfoName) {
-    return `TavernAI_${worldInfoName}_WI`;
+    return worldInfoName ? `TavernAI_${worldInfoName}_WI` : null;
 }
 
+function isTavernKoboldWorldInfo(folderName) {
+    return /^TavernAI_(.*)_WI$/.test(folderName);
+}
 
 
 app.listen(server_port, function() {