diff --git a/aiserver.py b/aiserver.py
index 3c54beda..8d481b75 100644
--- a/aiserver.py
+++ b/aiserver.py
@@ -3509,7 +3509,7 @@ def actionsubmit(data, actionmode=0, force_submit=False, force_prompt_gen=False,
botname = (koboldai_vars.botname + ":")
else:
botname = ""
- data = re.sub(r'\n+', ' ', data)
+ data = re.sub(r'\n+\Z', '', data)
if(len(data)):
data = f"\n{koboldai_vars.chatname}: {data}\n{botname}"
@@ -6308,7 +6308,7 @@ def UI_2_download_story():
@logger.catch
def UI_2_Set_Selected_Text(data):
if not koboldai_vars.quiet:
- print("Updating Selected Text: {}".format(data))
+ logger.info("Updating Selected Text: {}".format(data))
action_id = int(data["id"])
if not koboldai_vars.actions.actions[action_id].get("Original Text"):
diff --git a/koboldai_settings.py b/koboldai_settings.py
index b2cc720e..dfccd4ef 100644
--- a/koboldai_settings.py
+++ b/koboldai_settings.py
@@ -397,8 +397,8 @@ class koboldai_vars(object):
######################################### Setup Author's Note Data ########################################################
authors_note_text = self.authornotetemplate.replace("<|>", self.authornote)
- if len(authors_note_text) > 0 and authors_note_text[-1] not in [" ", "\n"]:
- authors_note_text += " "
+ if len(authors_note_text) > 0 and authors_note_text[0] not in [" ", "\n"]:
+ authors_note_text = " " + authors_note_text
authors_note_data = [[x, self.tokenizer.decode(x)] for x in self.tokenizer.encode(authors_note_text)]
if used_tokens + len(authors_note_data) <= token_budget:
used_tokens += len(authors_note_data)
@@ -1384,7 +1384,11 @@ class KoboldStoryRegister(object):
self.action_count = -1
# The id of the last submission action, or 0 if the last append was not a submission
self.submission_id = 0
- self.sentence_re = re.compile(r"[^.!?]*[.!?]+\"?\s*", re.S)
+ # A regular expression used to break the story into sentences so that the author's
+ # note can be inserted with minimal disruption. Avoid ending a sentance with
+ # whitespace because most tokenizers deal better with whitespace at the beginning of text.
+ # Search for sentence end delimeters (i.e. .!?) and also capture closing parenthesis and quotes.
+ self.sentence_re = re.compile(r".*?[.!?]+(?=[.!?\]\)}>'\"›»\s])[.!?\]\)}>'\"›»]*", re.S)
self.story_settings = story_settings
self.tts_model = None
self.tortoise = None
diff --git a/modeling/inference_models/hf.py b/modeling/inference_models/hf.py
index 013590ef..cd609fed 100644
--- a/modeling/inference_models/hf.py
+++ b/modeling/inference_models/hf.py
@@ -20,8 +20,89 @@ class HFInferenceModel(InferenceModel):
def _post_load(self) -> None:
# These are model specific tokenizer overrides if a model has bad defaults
if utils.koboldai_vars.model_type == "llama":
- self.tokenizer.decode_with_prefix_space = True
+ # Note: self.tokenizer is a GenericTokenizer, and self.tokenizer.tokenizer is the actual LlamaTokenizer
self.tokenizer.add_bos_token = False
+
+ # HF transformers no longer supports decode_with_prefix_space
+ # We work around this by wrapping decode, encode, and __call__
+ # with versions that work around the 'prefix space' misfeature
+ # of sentencepiece.
+ vocab = self.tokenizer.convert_ids_to_tokens(range(self.tokenizer.vocab_size))
+ has_prefix_space = {i for i, tok in enumerate(vocab) if tok.startswith("▁")}
+
+ # Wrap 'decode' with a method that always returns text starting with a space
+ # when the head token starts with a space. This is what 'decode_with_prefix_space'
+ # used to do, and we implement it using the same technique (building a cache of
+ # tokens that should have a prefix space, and then prepending a space if the first
+ # token is in this set.) We also work around a bizarre behavior in which decoding
+ # a single token 13 behaves differently than decoding a squence containing only [13].
+ original_decode = type(self.tokenizer.tokenizer).decode
+ def decode_wrapper(self, token_ids, *args, **kwargs):
+ first = None
+ # Note, the code below that wraps single-value token_ids in a list
+ # is to work around this wonky behavior:
+ # >>> t.decode(13)
+ # '<0x0A>'
+ # >>> t.decode([13])
+ # '\n'
+ # Not doing this causes token streaming to receive <0x0A> characters
+ # instead of newlines.
+ if isinstance(token_ids, int):
+ first = token_ids
+ token_ids = [first]
+ elif hasattr(token_ids, 'dim'): # Check for e.g. torch.Tensor
+ # Tensors don't support the Python standard of 'empty is False'
+ # and the special case of dimension 0 tensors also needs to be
+ # handled separately.
+ if token_ids.dim() == 0:
+ first = int(token_ids.item())
+ token_ids = [first]
+ elif len(token_ids) > 0:
+ first = int(token_ids[0])
+ elif token_ids:
+ first = token_ids[0]
+ result = original_decode(self, token_ids, *args, **kwargs)
+ if first is not None and first in has_prefix_space:
+ result = " " + result
+ return result
+ # GenericTokenizer overrides __setattr__ so we need to use object.__setattr__ to bypass it
+ object.__setattr__(self.tokenizer, 'decode', decode_wrapper.__get__(self.tokenizer))
+
+ # Wrap encode and __call__ to work around the 'prefix space' misfeature also.
+ # The problem is that "Bob" at the start of text is encoded as if it is
+ # " Bob". This creates a problem because it means you can't split text, encode
+ # the pieces, concatenate the tokens, decode them, and get the original text back.
+ # The workaround is to prepend a known token that (1) starts with a space; and
+ # (2) is not the prefix of any other token. After searching through the vocab
+ # " ," (space comma) is the only token containing only printable ascii characters
+ # that fits this bill. By prepending ',' to the text, the original encode
+ # method always returns [1919, ...], where the tail of the sequence is the
+ # actual encoded result we want without the prefix space behavior.
+ original_encode = type(self.tokenizer.tokenizer).encode
+ def encode_wrapper(self, text, *args, **kwargs):
+ if type(text) is str:
+ text = ',' + text
+ result = original_encode(self, text, *args, **kwargs)
+ result = result[1:]
+ else:
+ result = original_encode(self, text, *args, **kwargs)
+ return result
+ object.__setattr__(self.tokenizer, 'encode', encode_wrapper.__get__(self.tokenizer))
+
+ # Since 'encode' is documented as being deprecated, also override __call__.
+ # This doesn't appear to currently be used by KoboldAI, but doing so
+ # in case someone uses it in the future.
+ original_call = type(self.tokenizer.tokenizer).__call__
+ def call_wrapper(self, text, *args, **kwargs):
+ if type(text) is str:
+ text = ',' + text
+ result = original_call(self, text, *args, **kwargs)
+ result = result[1:]
+ else:
+ result = original_call(self, text, *args, **kwargs)
+ return result
+ object.__setattr__(self.tokenizer, '__call__', call_wrapper.__get__(self.tokenizer))
+
elif utils.koboldai_vars.model_type == "opt":
self.tokenizer._koboldai_header = self.tokenizer.encode("")
self.tokenizer.add_bos_token = False
diff --git a/static/klite.html b/static/klite.html
index 211b623b..0893ebbe 100644
--- a/static/klite.html
+++ b/static/klite.html
@@ -1 +1,7667 @@
-Coming Soon?
\ No newline at end of file
+
+
+
+
+
+
+
+
+ KoboldAI Lite
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Automatically select AI model ?
+ This option picks a suitable AI model based on the selected scenario. If no text model is currently selected, an appropriate one will be automatically picked for you.
+
+
+
+ Allow NSFW Models ?
+ If disabled, NSFW only models like Erebus will never be selected
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Select A Model To Load
+
+
+
+
+ Your Kobold Horde API Key ?
+ You need an API key to use KoboldAI Horde to generate text. Get one at
+ https://horde.koboldai.net/register or use the anonymous key 0000000000.
+
+
+ Need a Key? (Register New User)
+
+
+
+
+
+
+
+ Select Kobold Horde AI Model ?
+ These are the models currently provided by KoboldAI Horde
+ volunteers.
+
+
+ [See Current Volunteers]
+
+
+
+
+
+ Manually Select Worker ?
+ This option explicitly assigns worker IDs, fixed based on the current workers available at model selection time.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Custom Remote Play Endpoint
+
+
+
+
+
+ You can use this to connect to a KoboldAI instance running via a remote tunnel such as trycloudflare, localtunnel, ngrok.
+ Localhost IPs require host mode enabled. You can use the remote address displayed in the remote-play.bat window or colab window, note that the model must be loaded first.
+ Please input URL of the KoboldAI instance.
+
+
+
+
Emulate Token Streaming
+
+
Show Console Logging
+
+
+
+
+ Entering your OpenAI API key will allow you to use KoboldAI Lite with their API.
+ Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the OpenAI API and is not transmitted to us. Only Temperature, Top-P and Repetition Penalty samplers are used.
+
+ Please input OpenAI API URL and Key.
+
+
+ Model Choice:
+
+
+
Improve Prompt (Context Injection)
+
+
+ Uses Spellbook by Scale. This is an experimental endpoint. It may break at any time.
+ This endpoint does not support custom settings - please configure all properties at the scale webpage. Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Due to CORS limitations, your connection will be proxied.
+ Please input Spellbook by Scale API Key.
+
+
+ Please input the Deployment URL provided from the website.
+
+
+
+
+
+
+
+
+
+
+
Really Start A New Story?
+
+
+ Unsaved data will be lost.
+
+
Keep AI Selected?
+
+
+
+
+
+
+
+
+
+
+
+
+
Settings
+
+
+
+
+
+
+
Quick Presets ?Pick from an easy selection of curated generation presets, or configure your own.
+
+
+
+
+
+
+
Temperature ?Randomness of sampling. High values can increase creativity but
+ may make text less sensible. Lower values will make text more predictable but
+ can become repetitious.
+
+
+
+
+
0.1
+
2
+
+
+
+
+
+
Max Tokens ?Max
+ number of tokens of context to submit to the AI for sampling. Make sure this is
+ higher than Amount to Generate.
+
+
+
+
+
512
+
2048
+
+
+
Auto-Adjust Limits
+
+
+
+
+
+
+
+
Amount to Generate ?Number of tokens the AI should generate. Higher numbers will
+ take longer to generate.
+
+
+
+
+
16
+
512
+
+
+
Auto-Adjust Limits
+
+
+
+
+
+
+
Top p Sampling ?Used
+ to discard unlikely text in the sampling process. Lower values will make text
+ more predictable but can become repetitious.
+
+
+
+
+
0
+
1
+
+
+
+
+
+
+
Advanced Settings ?These settings control alternative samplers configurations. They are inactive by default, you usually do not need to change them.
+
+
+
Top-K
+
Top-A
+
Typ.
+
TFS
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Repetition Penalty ?Used to penalize words that were already generated or belong to
+ the context (Going over 1.2 breaks 6B models).
+
+
+
+
+
1
+
3
+
+
+
+
+
+
+
+
RpRng.
+
RpSlp.
+
Smp.Order ?The order by which all 7 samplers are applied, separated by commas. 0=top_k, 1=top_a, 2=top_p, 3=tfs, 4=typ, 5=temp, 6=rep_pen
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Generate Images ?Use Stable Horde to insert AI generated images into your story.
TTS ?Enable Text-To-Speech to have your story automatically read to you.
+
+
+
+
Beep on Done
+
+
+
+
+
+
+
+
Format ?Story Mode is best for novel style writing. Adventure Mode is best for Interactive Fiction RPGs. Chat Mode is best for chat conversations with the AI.
+
+
+
+
+
Your Name
+
AI Name ?The name of the person you want to chat with. Multiple opponents can be specified, creating a group chat, separate their names with ||$||
+
+
+
+
+
+
+
+
Aesthetic Chat UI
+
+
+
+
Multiline Replies
+
+
+
+
+
Improved Prompt ?Modifies the context, injecting tokens to improve adventure quality for adventure mode.
+
+
+
+
+ Start Sequence:
+
+ End Sequence:
+
+
+
+
+
+
+
+
+
Autoscroll
+
+
+
+
Trim Sentences
+
+
+
+
Trim Whitespace
+
+
+
+
Persist Session
+
+
+
+
Export Settings
+
+
+
+
Filter Comment Tags
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Memory
+
+
Memory
+
+
+
Author's Note
+
+
+
+ Author's Note Template
+
+ A/N Strength
+
+
+
+
+
+
+
Extra Stopping Sequence (Kobold API Only) ?Triggers the text generator to stop generating early if this sequence appears, in addition to default stop sequences. If you want multiple sequences, separate them with the following delimiter: ||$||
+
+
+
+
+
+
+
+
+
+
+
+
+
+
World Info
+
+
+
+
+
+
+
Case Sensitive Keys
+
+
+
+
+
+
+
+
+
+
+
+
Worker List
+
+
+
+
+
Name
Model
Capabilities
Uptime
Kudos
Cluster
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Share Story URL
+
+
+
+
+
+
Warning: This story is very long. It may not load in some browsers. You should save it as a file instead.
+
+
+
+
+
+
+
+
Image Information
+
+
+
+
+
+
+
+ Loading...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/static/koboldai.js b/static/koboldai.js
index 7dfc4a5e..cfc32d21 100644
--- a/static/koboldai.js
+++ b/static/koboldai.js
@@ -3129,7 +3129,6 @@ function gametextwatcher(records) {
}
}
}
- //console.log(dirty_chunks);
}
function fix_dirty_game_text() {
@@ -3154,16 +3153,26 @@ function fix_dirty_game_text() {
//Fixing text outside of chunks
for (node of game_text.childNodes) {
if ((!(node instanceof HTMLElement) || !node.hasAttribute("chunk")) && (node.textContent.trim() != "")) {
- console.log("Found Node that needs to be combined");
- console.log(node);
- //We have a text only node. It should be moved into the previous chunk
+ //We have a text only node. It should be moved into the previous chunk if it is marked as dirty, next node if not and it's dirty, or the previous if neither is dirty
+ var node_text = ""
if (node instanceof HTMLElement) {
- node.previousElementSibling.innerText = node.previousElementSibling.innerText + node.innerText;
+ node_text = node.innerText;
} else {
- node.previousElementSibling.innerText = node.previousElementSibling.innerText + node.data;
+ node_text = node.data;
}
- if (!dirty_chunks.includes(node.previousElementSibling.getAttribute("chunk"))) {
- dirty_chunks.push(node.previousElementSibling.getAttribute("chunk"));
+ if (!(node.nextElementSibling) || !(dirty_chunks.includes(node.nextElementSibling.getAttribute("chunk"))) || dirty_chunks.includes(node.previousElementSibling.getAttribute("chunk"))) {
+ node.previousElementSibling.innerText = node.previousElementSibling.innerText + node_text;
+ if (!dirty_chunks.includes(node.previousElementSibling.getAttribute("chunk"))) {
+ dirty_chunks.push(node.previousElementSibling.getAttribute("chunk"));
+ }
+ } else {
+ node.nextElementSibling.innerText = node.nextElementSibling.innerText + node_text;
+ }
+
+ //Looks like sometimes it splits the parent. Let's look for that and fix it too
+ if (node.nextElementSibling && (node.nextElementSibling.getAttribute("chunk") == node.previousElementSibling.getAttribute("chunk"))) {
+ node.previousElementSibling.innerText = node.previousElementSibling.innerText + node.nextElementSibling.innerText;
+ node.nextElementSibling.remove();
}
node.remove();
}
diff --git a/utils.py b/utils.py
index 3a9a884c..13ebb6a3 100644
--- a/utils.py
+++ b/utils.py
@@ -144,21 +144,17 @@ def singlelineprocessing(txt, koboldai_vars):
return txt
def chatmodeprocessing(txt, koboldai_vars):
- chatregex = re.compile(r'%s:[.|\n|\W|\w]*'%koboldai_vars.chatname)
+ chatregex = re.compile(r'\s+%s:[.|\n|\W|\w]*'%koboldai_vars.chatname)
txt = chatregex.sub('', txt)
if(len(koboldai_vars.actions) > 0):
if(len(koboldai_vars.actions[-1]) > 0):
action = koboldai_vars.actions[-1]
- lastchar = action[-1] if len(action) else ""
else:
# Last action is blank, this should never happen, but
# since it did let's bail out.
return txt
else:
action = koboldai_vars.prompt
- lastchar = action[-1] if len(action) else ""
- if(lastchar != "\n"):
- txt = txt + "\n"
return txt
#==================================================================#
@@ -745,4 +741,4 @@ def applyoutputformatting(txt, no_sentence_trimming=False, no_single_line=False)
txt = txt.replace(sub["trueTarget"], sub["substitution"])
txt = txt.replace(sub["target"], sub["substitution"])
- return txt
\ No newline at end of file
+ return txt