rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Expand attention bias

Browse Source
This commit is contained in:
somebody
2022-10-19 21:06:01 -05:00
parent b797126f11
commit 911bda76d4
3 changed files with 39 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
aiserver.py
View File

@@ -5005,24 +5005,27 @@ def calcsubmit(txt):
#subtxt, min, max = calcsubmitbudget(actionlen, winfo, mem, anotetxt, koboldai_vars.actions, submission=txt)
subtxt, min, max, found_entries = koboldai_vars.calc_ai_text(submitted_text=txt)
if koboldai_vars.experimental_features and koboldai_vars.memory_attn_bias > 1:
if koboldai_vars.experimental_features:
offset = 0
bounds = None
applied_biases = []
for c in koboldai_vars.context:
length = len(tokenizer.encode(c["text"]))
if c["type"] == "memory":
bounds = [offset, offset + length]
break
length = len(c["tokens"])
if c.get("attention_multiplier") and c["attention_multiplier"] != 1:
applied_biases.append({"start": offset, "end": offset + length, "multiplier": c.get("attention_multiplier", 1)})
offset += length
print(f"Memory bounds: {bounds}")
assert bounds
logger.info(f"Applied Biases: {applied_biases}")
bias = [1] * bounds[0]
bias += [koboldai_vars.memory_attn_bias] * bounds[1]
bias = []
for b in applied_biases:
for i in range(b["start"], b["end"]):
top_index = len(bias) - 1
if i > top_index:
bias += [1] * (i - top_index)
bias[i] = b["multiplier"]
attention_bias.attention_bias = torch.Tensor(bias).to(breakmodel.primary_device)
print(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}")
logger.info(f"Bias by {koboldai_vars.memory_attn_bias} -- {attention_bias.attention_bias}")
generate(subtxt, min, max, found_entries)
attention_bias.attention_bias = None

8
koboldai_settings.py
View File

@@ -241,7 +241,8 @@ class koboldai_vars(object):
if len(memory_tokens) != 0:
context.append({"type": "memory",
"text": "".join([x[1] for x in memory_data]),
"tokens": memory_data})
"tokens": memory_data,
"attention_multiplier": self.memory_attn_bias})
used_tokens += len(memory_tokens)
@@ -350,7 +351,7 @@ class koboldai_vars(object):
#Add our author's note if we've hit andepth
if not inserted_author_note and len(actions_seen) >= self.andepth and self.authornote != "":
game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data})
game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias})
inserted_author_note = True
action_data = [[x, self.tokenizer.decode(x)] for x in self.tokenizer.encode(action_text_split[i][0])]
@@ -426,7 +427,7 @@ class koboldai_vars(object):
######################################### Verify Author's Note Data in AI Text ########################################################
#if we don't have enough actions to get to author's note depth then we just add it right before the game text
if not inserted_author_note and self.authornote != "":
game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data})
game_context.insert(0, {"type": "authors_note", "text": authors_note_text, "tokens": authors_note_data, "attention_multiplier": self.an_attn_bias})
######################################### Add our prompt data ########################################################
@@ -783,6 +784,7 @@ class story_settings(settings):
# bias experiment
self.memory_attn_bias = 1
self.an_attn_bias = 1
def save_story(self):
if not self.no_save:

26
templates/story flyout.html
View File

@@ -13,18 +13,18 @@
</div>
<textarea rows=20 id="memory" class="var_sync_story_memory var_sync_alt_story_memory_length fullwidth" onchange='sync_to_server(this);' oninput="autoResize(this)" autocomplete="off"></textarea>
<div id="Attention-Bias" class="var_sync_alt_system_experimental_features">
<h4 class="section_header"><label for="Attention-Bias">Attention Bias Test</label></h4>
<div id="Mem-Attention-Bias" class="var_sync_alt_system_experimental_features">
<h4 class="section_header"><label for="Mem-Attention-Bias">Attention Bias Test</label></h4>
<span class="help_text">
<b>Note: This only works on OPT models for now! Patches will be written for other models once it's known this actually has a positive effect. Upon first use of this bias, you should see "Applying attention bias" in the console.</b><br>
This setting <i>may</i> change how the AI pays attention to memory. Any high number in the ballpark of 15 may cause incoherence. The option to select higher numbers is present for experimentation.
</span>
<input type="range" oninput="memAttn(this.value);" onchange="sync_to_server(this);" class="var_sync_story_memory_attn_bias sync_as_float" value="1" min="1" max="50" step="0.1"></input>
<input type="range" oninput="memAttnInput(this.value);" onchange="sync_to_server(this);" class="var_sync_story_memory_attn_bias sync_as_float" value="1" min="1" max="50" step="0.1"></input>
<span id="memattnbiaslabel">1</span>
<script>
const label = document.querySelector("#memattnbiaslabel");
function memAttn(val) {
label.innerText = val;
const memAttn = document.querySelector("#memattnbiaslabel");
function memAttnInput(val) {
memAttn.innerText = val;
}
</script>
</div>
@@ -49,6 +49,20 @@
<input autocomplete="off" id=authors_notes_template type=text class="var_sync_story_authornotetemplate fullwidth" onchange='sync_to_server(this);'><br/>
<label for="authors_notes">Author's Notes:</label><br/>
<textarea autocomplete="off" rows=16 id="authors_notes" class="var_sync_story_authornote var_sync_alt_story_authornote_length fullwidth" oninput="autoResize(this)" onchange='sync_to_server(this);'></textarea><br/>
<div id="An-Attention-Bias" class="var_sync_alt_system_experimental_features">
<h4 class="section_header"><label for="An-Attention-Bias">Attention Bias Test</label></h4>
<span class="help_text">See disclaimer in memory page.</span>
<input type="range" oninput="anAttnInput(this.value);" onchange="sync_to_server(this);" class="var_sync_story_an_attn_bias sync_as_float" value="1" min="1" max="50" step="0.1"></input>
<span id="anattnbiaslabel">1</span>
<script>
const anAttn = document.querySelector("#anattnbiaslabel");
function anAttnInput(val) {
anAttn.innerText = val;
}
</script>
</div>
<div class="setting_tile_area">
{% with menu='author_notes' %}
{% with sub_path='' %}