From e566a498ef5f775a357802ee986ef5a12812d3f6 Mon Sep 17 00:00:00 2001 From: ebolam Date: Fri, 30 Sep 2022 08:56:35 -0400 Subject: [PATCH] World Info Speed Increase --- koboldai_settings.py | 22 +++++++++++----------- static/koboldai.js | 11 +++++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index fff681f8..fc80c9c2 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1055,6 +1055,7 @@ class KoboldStoryRegister(object): old_length = self.actions[i]["Selected Text Length"] if self.actions[i]["Selected Text"] != text: self.actions[i]["Selected Text"] = text + self.actions[i]["WI Search Text"] = re.sub("[^0-9a-z \'\"]", "", text) self.actions[i]["Probabilities"] = [] if "Options" in self.actions[i]: for j in range(len(self.actions[i]["Options"])): @@ -1066,7 +1067,7 @@ class KoboldStoryRegister(object): old_text = None old_length = None old = None - self.actions[i] = {"Selected Text": text, "Probabilities": [], "Options": []} + self.actions[i] = {"Selected Text": text, "WI Search Text": re.sub("[^0-9a-z \'\"]", "", text), "Probabilities": [], "Options": []} if self.tokenizer is not None: self.actions[i]['Selected Text Length'] = len(self.tokenizer.encode(text)) @@ -1105,6 +1106,8 @@ class KoboldStoryRegister(object): temp = {} for item in json_data['actions']: temp[int(item)] = json_data['actions'][item] + if "WI Search Text" not in temp[int(item)]: + temp[int(item)]["WI Search Text"] = re.sub("[^0-9a-z \'\"]", "", temp[int(item)]['Selected Text']) process_variable_changes(self.socketio, "story", 'actions', {"id": item, 'action': temp[int(item)]}, None) self.actions = temp self.set_game_saved() @@ -1117,11 +1120,9 @@ class KoboldStoryRegister(object): if self.action_count in self.actions: if self.actions[self.action_count]["Selected Text"] != text: self.actions[self.action_count]["Selected Text"] = text + self.actions[i]["WI Search Text"] = re.sub("[^0-9a-z \'\"]", "", text) self.actions[self.action_count]["Probabilities"] = [] - if self.tokenizer is not None: - selected_text_length = len(self.tokenizer.encode(text)) - else: - selected_text_length = 0 + selected_text_length = 0 self.actions[self.action_count]["Selected Text Length"] = selected_text_length self.actions[self.action_count]["In AI Input"] = False for item in self.actions[self.action_count]["Options"]: @@ -1130,12 +1131,11 @@ class KoboldStoryRegister(object): del item else: - if self.tokenizer is not None: - selected_text_length = len(self.tokenizer.encode(text)) - else: - selected_text_length = 0 + selected_text_length = 0 - self.actions[self.action_count] = {"Selected Text": text, "Selected Text Length": selected_text_length, "In AI Input": False, "Options": [], "Probabilities": []} + self.actions[self.action_count] = {"Selected Text": text, "Selected Text Length": selected_text_length, + "WI Search Text": re.sub("[^0-9a-z \'\"]", "", text), + "In AI Input": False, "Options": [], "Probabilities": []} process_variable_changes(self.socketio, "story", 'actions', {"id": self.action_count, 'action': self.actions[self.action_count]}, None) self.set_game_saved() @@ -1198,7 +1198,6 @@ class KoboldStoryRegister(object): process_variable_changes(self.socketio, "story", 'actions', {"id": pointer, 'action': self.actions[pointer]}, None) self.set_game_saved() - def set_action_in_ai(self, action_id, used=True): if 'In AI Input' in self.actions[action_id]: old = self.actions[action_id]['In AI Input'] @@ -1243,6 +1242,7 @@ class KoboldStoryRegister(object): old_length = self.actions[action_step]["Selected Text Length"] if option_number < len(self.actions[action_step]['Options']): self.actions[action_step]["Selected Text"] = self.actions[action_step]['Options'][option_number]['text'] + self.actions[action_step]["WI Search Text"] = re.sub("[^0-9a-z \'\"]", "", self.actions[action_step]["Selected Text"]) if 'Probabilities' in self.actions[action_step]['Options'][option_number]: self.actions[action_step]["Probabilities"] = self.actions[action_step]['Options'][option_number]['Probabilities'] if self.tokenizer is not None: diff --git a/static/koboldai.js b/static/koboldai.js index 5d3b95d4..17a4495c 100644 --- a/static/koboldai.js +++ b/static/koboldai.js @@ -304,6 +304,7 @@ function do_story_text_updates(data) { span.textContent = data.value.action['Selected Text']; item.append(span); item.original_text = data.value.action['Selected Text']; + item.setAttribute("WI_Search_Text", data.value.action['WI Search Text']); item.setAttribute("world_info_uids", ""); item.classList.remove("pulse") //item.scrollIntoView(); @@ -318,6 +319,7 @@ function do_story_text_updates(data) { span.original_text = data.value.action['Selected Text']; new_span = document.createElement("span"); new_span.textContent = data.value.action['Selected Text']; + span.setAttribute("WI_Search_Text", data.value.action['WI Search Text']); span.append(new_span); @@ -345,7 +347,8 @@ function do_prompt(data) { span = document.createElement("span"); span.textContent = data.value; item.append(span); - item.setAttribute("old_text", data.value) + item.setAttribute("old_text", data.value); + item.setAttribute("WI_Search_Text", data.value.replace(/[^0-9a-z \'\"]/gi, '')); item.classList.remove("pulse"); assign_world_info_to_action(item, null); } @@ -3159,11 +3162,11 @@ function assign_world_info_to_action(action_item, uid) { tag.parentElement.setAttribute("world_info_uids", current_ids.join(",")); } for (keyword of worldinfo['key']) { - if ((action.textContent.replace(/[^0-9a-z \'\"]/gi, '')).includes(keyword)) { + if ((action.getAttribute('WI_Search_Text')).includes(keyword)) { //Ok we have a key match, but we need to check for secondary keys if applicable if (worldinfo['keysecondary'].length > 0) { for (second_key of worldinfo['keysecondary']) { - if (action.textContent.replace(/[^0-9a-z \'\"]/gi, '').includes(second_key)) { + if (action.getAttribute('WI_Search_Text').includes(second_key)) { highlight_world_info_text_in_chunk(action, worldinfo); break; } @@ -3194,7 +3197,7 @@ function highlight_world_info_text_in_chunk(action, wi) { //First let's find the largest key that matches let largest_key = ""; for (keyword of wi['key']) { - if ((keyword.length > largest_key.length) && (action.textContent.replace(/[^0-9a-z \'\"]/gi, '').includes(keyword))) { + if ((keyword.length > largest_key.length) && (action.getAttribute('WI_Search_Text').includes(keyword))) { largest_key = keyword; } }