diff --git a/aiserver.py b/aiserver.py index 14298d68..31f03630 100644 --- a/aiserver.py +++ b/aiserver.py @@ -5005,24 +5005,27 @@ def calcsubmit(txt): #subtxt, min, max = calcsubmitbudget(actionlen, winfo, mem, anotetxt, koboldai_vars.actions, submission=txt) subtxt, min, max, found_entries = koboldai_vars.calc_ai_text(submitted_text=txt) - if koboldai_vars.experimental_features and koboldai_vars.memory_attn_bias > 1: + if koboldai_vars.experimental_features: offset = 0 - bounds = None + applied_biases = [] for c in koboldai_vars.context: - length = len(tokenizer.encode(c["text"])) - if c["type"] == "memory": - bounds = [offset, offset + length] - break + length = len(c["tokens"]) + if c.get("attention_multiplier") and c["attention_multiplier"] != 1: + applied_biases.append({"start": offset, "end": offset + length, "multiplier": c.get("attention_multiplier", 1)}) offset += length - print(f"Memory bounds: {bounds}") - assert bounds + logger.info(f"Applied Biases: {applied_biases}") - bias = [1] * bounds[0] - bias += [koboldai_vars.memory_attn_bias] * bounds[1] + bias = [] + for b in applied_biases: + for i in range(b["start"], b["end"]): + top_index = len(bias) - 1 + if i > top_index: + bias += [1] * (i - top_index) + bias[i] = b["multiplier"] attention_bias.attention_bias = torch.Tensor(bias).to(breakmodel.primary_device) - print(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") + logger.info(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") generate(subtxt, min, max, found_entries) attention_bias.attention_bias = None diff --git a/koboldai_settings.py b/koboldai_settings.py index 7717a348..d127c078 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -241,7 +241,8 @@ class koboldai_vars(object): if len(memory_tokens) != 0: context.append({"type": "memory", "text": "".join([x[1] for x in memory_data]), - "tokens": memory_data}) + "tokens": memory_data, + "attention_multiplier": self.memory_attn_bias}) used_tokens += len(memory_tokens) @@ -350,7 +351,7 @@ class koboldai_vars(object): #Add our author's note if we've hit andepth if not inserted_author_note and len(actions_seen) >= self.andepth and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) inserted_author_note = True action_data = [[x, self.tokenizer.decode(x)] for x in self.tokenizer.encode(action_text_split[i][0])] @@ -426,7 +427,7 @@ class koboldai_vars(object): ######################################### Verify Author's Note Data in AI Text ######################################################## #if we don't have enough actions to get to author's note depth then we just add it right before the game text if not inserted_author_note and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) ######################################### Add our prompt data ######################################################## @@ -783,6 +784,7 @@ class story_settings(settings): # bias experiment self.memory_attn_bias = 1 + self.an_attn_bias = 1 def save_story(self): if not self.no_save: diff --git a/templates/settings flyout.html b/templates/settings flyout.html index 1830fb8e..88c459c5 100644 --- a/templates/settings flyout.html +++ b/templates/settings flyout.html @@ -24,7 +24,7 @@ open_in_new -search +search
diff --git a/templates/story flyout.html b/templates/story flyout.html index de6cfe38..6130e0d7 100644 --- a/templates/story flyout.html +++ b/templates/story flyout.html @@ -13,18 +13,18 @@
-
-

+
+

Note: This only works on OPT models for now! Patches will be written for other models once it's known this actually has a positive effect. Upon first use of this bias, you should see "Applying attention bias" in the console.
This setting may change how the AI pays attention to memory. Any high number in the ballpark of 15 may cause incoherence. The option to select higher numbers is present for experimentation.
- + 1
@@ -49,6 +49,20 @@


+ +
+

+ See disclaimer in memory page. + + 1 + +
+
{% with menu='author_notes' %} {% with sub_path='' %}