diff --git a/aiserver.py b/aiserver.py index 14298d68..31f03630 100644 --- a/aiserver.py +++ b/aiserver.py @@ -5005,24 +5005,27 @@ def calcsubmit(txt): #subtxt, min, max = calcsubmitbudget(actionlen, winfo, mem, anotetxt, koboldai_vars.actions, submission=txt) subtxt, min, max, found_entries = koboldai_vars.calc_ai_text(submitted_text=txt) - if koboldai_vars.experimental_features and koboldai_vars.memory_attn_bias > 1: + if koboldai_vars.experimental_features: offset = 0 - bounds = None + applied_biases = [] for c in koboldai_vars.context: - length = len(tokenizer.encode(c["text"])) - if c["type"] == "memory": - bounds = [offset, offset + length] - break + length = len(c["tokens"]) + if c.get("attention_multiplier") and c["attention_multiplier"] != 1: + applied_biases.append({"start": offset, "end": offset + length, "multiplier": c.get("attention_multiplier", 1)}) offset += length - print(f"Memory bounds: {bounds}") - assert bounds + logger.info(f"Applied Biases: {applied_biases}") - bias = [1] * bounds[0] - bias += [koboldai_vars.memory_attn_bias] * bounds[1] + bias = [] + for b in applied_biases: + for i in range(b["start"], b["end"]): + top_index = len(bias) - 1 + if i > top_index: + bias += [1] * (i - top_index) + bias[i] = b["multiplier"] attention_bias.attention_bias = torch.Tensor(bias).to(breakmodel.primary_device) - print(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") + logger.info(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}") generate(subtxt, min, max, found_entries) attention_bias.attention_bias = None diff --git a/koboldai_settings.py b/koboldai_settings.py index d85ab2ed..12bb414d 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -241,7 +241,8 @@ class koboldai_vars(object): if len(memory_tokens) != 0: context.append({"type": "memory", "text": "".join([x[1] for x in memory_data]), - "tokens": memory_data}) + "tokens": memory_data, + "attention_multiplier": self.memory_attn_bias}) used_tokens += len(memory_tokens) @@ -350,7 +351,7 @@ class koboldai_vars(object): #Add our author's note if we've hit andepth if not inserted_author_note and len(actions_seen) >= self.andepth and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) inserted_author_note = True action_data = [[x, self.tokenizer.decode(x)] for x in self.tokenizer.encode(action_text_split[i][0])] @@ -426,7 +427,7 @@ class koboldai_vars(object): ######################################### Verify Author's Note Data in AI Text ######################################################## #if we don't have enough actions to get to author's note depth then we just add it right before the game text if not inserted_author_note and self.authornote != "": - game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data}) + game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias}) ######################################### Add our prompt data ######################################################## @@ -783,6 +784,7 @@ class story_settings(settings): # bias experiment self.memory_attn_bias = 1 + self.an_attn_bias = 1 def save_story(self): if not self.no_save: diff --git a/templates/story flyout.html b/templates/story flyout.html index de6cfe38..6130e0d7 100644 --- a/templates/story flyout.html +++ b/templates/story flyout.html @@ -13,18 +13,18 @@ -