From 862af9746a69675dd1467d3749a20ec0aa406670 Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 19 Oct 2022 18:41:47 -0500 Subject: [PATCH 1/2] Change tooltip --- templates/settings flyout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/settings flyout.html b/templates/settings flyout.html index 1830fb8e..88c459c5 100644 --- a/templates/settings flyout.html +++ b/templates/settings flyout.html @@ -24,7 +24,7 @@ open_in_new -search +search
From 911bda76d43ce5ed240a558cf86eaf6fff000eae Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 19 Oct 2022 21:06:01 -0500 Subject: [PATCH 2/2] Expand attention bias --- aiserver.py | 25 ++++++++++++++----------- koboldai_settings.py | 8 +++++--- templates/story flyout.html | 26 ++++++++++++++++++++------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/aiserver.py b/aiserver.py index 14298d68..31f03630 100644 --- a/aiserver.py +++ b/aiserver.py @@ -5005,24 +5005,27 @@ def calcsubmit(txt): #subtxt, min, max = calcsubmitbudget(actionlen, winfo, mem, anotetxt, koboldai_vars.actions, submission=txt) subtxt, min, max, found_entries = koboldai_vars.calc_ai_text(submitted_text=txt) - if koboldai_vars.experimental_features and koboldai_vars.memory_attn_bias > 1: + if koboldai_vars.experimental_features: offset = 0 - bounds = None + applied_biases = [] for c in koboldai_vars.context: - length = len(tokenizer.encode(c["text"])) - if c["type"] == "memory": - bounds = [offset, offset + length] - break + length = len(c["tokens"]) + if c.get("attention_multiplier") and c["attention_multiplier"] != 1: + applied_biases.append({"start": offset, "end": offset + length, "multiplier": c.get("attention_multiplier", 1)}) offset += length - print(f"Memory bounds: {bounds}") - assert bounds + logger.info(f"Applied Biases: {applied_biases}") - bias = [1] * bounds[0] - bias += [koboldai_vars.memory_attn_bias] * bounds[1] + bias = [] + for b in applied_biases: + for i in range(b["start"], b["end"]): + top_index = len(bias) - 1 + if i > top_index: + bias += [1] * (i - top_index) + bias[i] = b["multiplier"] attention_bias.attention_bias = torch.Tensor(bias).to(breakmodel.primary_device) - print(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") + logger.info(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") generate(subtxt, min, max, found_entries) attention_bias.attention_bias = None diff --git a/koboldai_settings.py b/koboldai_settings.py index d85ab2ed..12bb414d 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -241,7 +241,8 @@ class koboldai_vars(object): if len(memory_tokens) != 0: context.append({"type": "memory", "text": "".join([x[1] for x in memory_data]), - "tokens": memory_data}) + "tokens": memory_data, + "attention_multiplier": self.memory_attn_bias}) used_tokens += len(memory_tokens) @@ -350,7 +351,7 @@ class koboldai_vars(object): #Add our author's note if we've hit andepth if not inserted_author_note and len(actions_seen) >= self.andepth and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) inserted_author_note = True action_data = [[x, self.tokenizer.decode(x)] for x in self.tokenizer.encode(action_text_split[i][0])] @@ -426,7 +427,7 @@ class koboldai_vars(object): ######################################### Verify Author's Note Data in AI Text ######################################################## #if we don't have enough actions to get to author's note depth then we just add it right before the game text if not inserted_author_note and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) ######################################### Add our prompt data ######################################################## @@ -783,6 +784,7 @@ class story_settings(settings): # bias experiment self.memory_attn_bias = 1 + self.an_attn_bias = 1 def save_story(self): if not self.no_save: diff --git a/templates/story flyout.html b/templates/story flyout.html index de6cfe38..6130e0d7 100644 --- a/templates/story flyout.html +++ b/templates/story flyout.html @@ -13,18 +13,18 @@
-
-

+
+

Note: This only works on OPT models for now! Patches will be written for other models once it's known this actually has a positive effect. Upon first use of this bias, you should see "Applying attention bias" in the console.
This setting may change how the AI pays attention to memory. Any high number in the ballpark of 15 may cause incoherence. The option to select higher numbers is present for experimentation. - + 1
@@ -49,6 +49,20 @@


+ +
+

+ See disclaimer in memory page. + + 1 + +
+
{% with menu='author_notes' %} {% with sub_path='' %}