Fix LLAMA tokenization. Add case-sensitive WI matching

public/index.html

@@ -1458,7 +1458,7 @@
                             </div>
                         </div>
 
-                        <div class="flex1 range-block">
+                        <div class="flex1 range-block flex-container flexFlowColumn">
                             <label title="Entries can activate other entries by mentioning their keywords" class="checkbox_label">
                                 <input id="world_info_recursive" type="checkbox" />
                                 <span>
@@ -1468,6 +1468,15 @@
                                     </a>
                                 </span>
                             </label>
+                            <label title="Lookup for the entry keys in the context will respect the case" class="checkbox_label">
+                                <input id="world_info_case_sensitive" type="checkbox" />
+                                <span>
+                                    Case-sensitive keys
+                                    <a href="/notes#casesensitivekeys" class="notes-link" target="_blank">
+                                        <span class="note-link-span">?</span>
+                                    </a>
+                                </span>
+                            </label>
                         </div>
                     </div>
 

public/notes/content.md

@@ -163,7 +163,7 @@ _It is important to note that while World Info helps guide the AI towards your d
 
 #### Key
 
-A list of keywords that trigger the activation of a World Info entry.
+A list of keywords that trigger the activation of a World Info entry. Keys are not case-sensitive by default (this is [configurable](#casesensitivekeys)).
 
 #### Secondary Key
 
@@ -234,6 +234,14 @@ Content: Rufus is a dog.
 
 **Both** of them will be pulled into the context if the message text mentions **just Bessie**.
 
+### Case-sensitive keys
+
+**To get pulled into the context, entry keys need to match the case as they are defined in the World Info entry.**
+
+This is useful when your keys are common words or parts of common words.
+
+For example, when this setting is active, keys 'rose' and 'Rose' will be treated differently, depending on the inputs.
+
 ## KoboldAI
 
 ### Basic Settings

public/script.js

@@ -458,7 +458,7 @@ function getTokenCount(str, padding = undefined) {
             jQuery.ajax({
                 async: false,
                 type: 'POST', //
-                url: `/ tokenize_llama`,
+                url: `/tokenize_llama`,
                 data: JSON.stringify({ text: str }),
                 dataType: "json",
                 contentType: "application/json",
@@ -3852,6 +3852,7 @@ async function saveSettings(type) {
             world_info_depth: world_info_depth,
             world_info_budget: world_info_budget,
             world_info_recursive: world_info_recursive,
+            world_info_case_sensitive: world_info_case_sensitive,
             textgenerationwebui_settings: textgenerationwebui_settings,
             swipes: swipes,
             horde_settings: horde_settings,

public/scripts/world-info.js

@@ -7,6 +7,7 @@ export {
     world_info_budget,
     world_info_depth,
     world_info_recursive,
+    world_info_case_sensitive,
     world_names,
     imported_world_name,
     checkWorldInfo,
@@ -23,6 +24,7 @@ let world_info_depth = 2;
 let world_info_budget = 128;
 let is_world_edit_open = false;
 let world_info_recursive = false;
+let world_info_case_sensitive = false;
 let imported_world_name = "";
 const saveWorldDebounced = debounce(async () => await _save(), 500);
 const saveSettingsDebounced = debounce(() => saveSettings(), 500);
@@ -51,6 +53,8 @@ function setWorldInfoSettings(settings, data) {
         world_info_budget = Number(settings.world_info_budget);
     if (settings.world_info_recursive !== undefined)
         world_info_recursive = Boolean(settings.world_info_recursive);
+    if (settings.world_info_case_sensitive !== undefined)
+        world_info_case_sensitive = Boolean(settings.world_info_case_sensitive);
 
     $("#world_info_depth_counter").text(world_info_depth);
     $("#world_info_depth").val(world_info_depth);
@@ -59,6 +63,7 @@ function setWorldInfoSettings(settings, data) {
     $("#world_info_budget").val(world_info_budget);
 
     $("#world_info_recursive").prop('checked', world_info_recursive);
+    $("#world_info_case_sensitive").prop('checked', world_info_case_sensitive);
 
     world_names = data.world_names?.length ? data.world_names : [];
 
@@ -476,13 +481,18 @@ async function createNewWorldInfo() {
     }
 }
 
+// Gets a string that respects the case sensitivity setting
+function transformString(str) {
+    return world_info_case_sensitive ? str : str.toLowerCase();
+}
+
 function checkWorldInfo(chat) {
     if (world_info_data.entries.length == 0) {
         return "";
     }
 
     const messagesToLookBack = world_info_depth * 2;
-    let textToScan = chat.slice(0, messagesToLookBack).join("").toLowerCase();
+    let textToScan = transformString(chat.slice(0, messagesToLookBack).join(""));
     let worldInfoBefore = "";
     let worldInfoAfter = "";
     let needsToScan = true;
@@ -507,7 +517,7 @@ function checkWorldInfo(chat) {
             if (Array.isArray(entry.key) && entry.key.length) {
                 primary: for (let key of entry.key) {
                     const substituted = substituteParams(key);
-                    if (substituted && textToScan.includes(substituted.trim().toLowerCase())) {
+                    if (substituted && textToScan.includes(transformString(substituted.trim()))) {
                         if (
                             entry.selective &&
                             Array.isArray(entry.keysecondary) &&
@@ -517,7 +527,7 @@ function checkWorldInfo(chat) {
                                 const secondarySubstituted = substituteParams(keysecondary);
                                 if (
                                     secondarySubstituted &&
-                                    textToScan.includes(secondarySubstituted.trim().toLowerCase())
+                                    textToScan.includes(transformString(secondarySubstituted.trim()))
                                 ) {
                                     activatedNow.add(entry.uid);
                                     break secondary;
@@ -557,11 +567,7 @@ function checkWorldInfo(chat) {
         }
 
         if (needsToScan) {
-            textToScan =
+            textToScan = (transformString(newEntries.map(x => x.content).join('\n')) + textToScan);
-                newEntries
-                    .map((x) => x.content)
-                    .join("\n")
-                    .toLowerCase() + textToScan;
         }
 
         allActivatedEntries = new Set([...allActivatedEntries, ...activatedNow]);
@@ -583,7 +589,7 @@ function selectImportedWorldInfo() {
     imported_world_name = "";
 }
 
-$(document).ready(() => {
+jQuery(() => {
     $("#world_info").change(async function () {
         const selectedWorld = $("#world_info").find(":selected").val();
         world_info = null;
@@ -688,4 +694,9 @@ $(document).ready(() => {
         world_info_recursive = !!$(this).prop('checked');
         saveSettingsDebounced();
     })
+
+    $('#world_info_case_sensitive').on('input', function () {
+        world_info_case_sensitive = !!$(this).prop('checked');
+        saveSettingsDebounced();
+    })
 });