From d5143eeb8030f161c3e0ed9b2a8c4bfe8388cb1f Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 12 Oct 2022 01:23:00 +0200 Subject: [PATCH 1/6] LUA Error as Error --- aiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index c0485cd1..771f1430 100644 --- a/aiserver.py +++ b/aiserver.py @@ -2964,7 +2964,7 @@ def load_lua_scripts(): if(vars.serverstarted): emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True) sendUSStatItems() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") if(vars.serverstarted): set_aibusy(0) From 64715b18d6cc9fef806ead307f7251b611cd8d38 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 12 Oct 2022 14:54:11 +0200 Subject: [PATCH 2/6] Version bump --- aiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index 771f1430..d883640a 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 #==================================================================# # KoboldAI -# Version: 1.19.0 +# Version: 1.19.1 # By: The KoboldAI Community #==================================================================# From bdc73ef39398d962a294cddddc8eeca3fe6dbd74 Mon Sep 17 00:00:00 2001 From: vfbd Date: Wed, 12 Oct 2022 14:31:18 -0400 Subject: [PATCH 3/6] Decrease TPU loading times by eliminating a transpose operation --- tpu_mtj_backend.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index d992ba45..99efa05f 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1148,6 +1148,9 @@ def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpo if param not in params: params[param] = default_params[param] + # Use an optimization that will allow us to avoid one extra transpose operation + params["transposed_linear"] = True + # Load tokenizer if vars.model == "TPUMeshTransformerGPTNeoX": tokenizer = Tokenizer.from_file(os.path.join(path, "20B_tokenizer.json")) @@ -1305,8 +1308,9 @@ def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpo tensor /= params["cores_per_replica"] if "vocab_pad" in transforms: tensor = torch.nn.functional.pad(tensor, (0, 0, 0, params["n_vocab_padding"])) - if "no_transpose" not in transforms and tensor.ndim == 2: - tensor = tensor.T + # We don't need to transpose linear module weights anymore because MTJ will do it for us if `transposed_linear` is set to True in the config + #if "no_transpose" not in transforms and tensor.ndim == 2: + # tensor = tensor.T tensor.unsqueeze_(0) if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: tensor = tensor.bfloat16() From 4a01f345de93474a612fec0907d4071d0a91362e Mon Sep 17 00:00:00 2001 From: Llama <34464159+pi6am@users.noreply.github.com> Date: Wed, 12 Oct 2022 23:18:19 -0700 Subject: [PATCH 4/6] Add include_anote kwarg to lua_compute_context. Add an optional keyword argument to lua_compute_context to control whether the author's note should be included in the context. The default value is true, so if the include_anote kwarg is not specified then the author's note will be included, which was the default behavior prior to this change. Also update the Lua API documentation to describe this kwarg. --- aiserver.py | 2 ++ userscripts/api_documentation.html | 3 +++ userscripts/api_documentation.md | 3 +++ 3 files changed, 8 insertions(+) diff --git a/aiserver.py b/aiserver.py index 771f1430..4091c61a 100644 --- a/aiserver.py +++ b/aiserver.py @@ -3046,6 +3046,8 @@ def lua_compute_context(submission, entries, folders, kwargs): force_use_txt=True, scan_story=kwargs["scan_story"] if kwargs["scan_story"] != None else True, ) + if kwargs["include_anote"] is not None and not kwargs["include_anote"]: + anotetxt = "" txt, _, _ = calcsubmitbudget( len(actions), winfo, diff --git a/userscripts/api_documentation.html b/userscripts/api_documentation.html index b581eb5f..462216c8 100644 --- a/userscripts/api_documentation.html +++ b/userscripts/api_documentation.html @@ -500,6 +500,7 @@
  • kwargs? (table<string, any>): Table of optional keyword arguments from the following list. Defaults to {}.
    • scan_story? (boolean): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to false, only the submission is scanned for world info keys. Defaults to true.
      • +
    • include_anote? (boolean): Whether to include the author's note in the story. Defaults to true, pass false to suppress including the author's note.
    • @@ -574,6 +575,7 @@
  • kwargs? (table<string, any>): Table of optional keyword arguments from the following list. Defaults to {}.
    • scan_story? (boolean): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to false, only the submission is scanned for world info keys. Defaults to true.
      • +
    • include_anote? (boolean): Whether to include the author's note in the story. Defaults to true, pass false to suppress including the author's note.
    • @@ -687,6 +689,7 @@
  • kwargs? (table<string, any>): Table of optional keyword arguments from the following list. Defaults to {}.
    • scan_story? (boolean): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to false, only the submission is scanned for world info keys. Defaults to true.
      • +
    • include_anote? (boolean): Whether to include the author's note in the story. Defaults to true, pass false to suppress including the author's note.
    • diff --git a/userscripts/api_documentation.md b/userscripts/api_documentation.md index 198d272e..a50eb9c9 100644 --- a/userscripts/api_documentation.md +++ b/userscripts/api_documentation.md @@ -538,6 +538,7 @@ Computes the context that would be sent to the generator with the user's current * entries? (`KoboldWorldInfoEntry|table`): A `KoboldWorldInfoEntry` or table thereof that indicates an allowed subset of world info entries to include in the context. Defaults to all world info entries. * kwargs? (`table`): Table of optional keyword arguments from the following list. Defaults to `{}`. * scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`. + * include_anote? (`boolean`): Whether to include the author's note in the story. Defaults to `true`, pass `false` to suppress including the author's note. ### Returns @@ -636,6 +637,7 @@ The same as calling `kobold.worldinfo:compute_context()` with this world info en * submission (`string`): String to use as simulated user's input after being formatted by input formatting. * kwargs? (`table`): Table of optional keyword arguments from the following list. Defaults to `{}`. * scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`. + * include_anote? (`boolean`): Whether to include the author's note in the story. Defaults to `true`, pass `false` to suppress including the author's note. ### Returns @@ -819,6 +821,7 @@ Unlike `kobold.worldinfo:compute_context()`, this function doesn't include world * entries? (`KoboldWorldInfoEntry|table`): A `KoboldWorldInfoEntry` or table thereof that indicates an allowed subset of world info entries to include in the context. Entries that are not inside of the folder are still not included. Defaults to all world info entries in the folder. * kwargs? (`table`): Table of optional keyword arguments from the following list. Defaults to `{}`. * scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`. + * include_anote? (`boolean`): Whether to include the author's note in the story. Defaults to `true`, pass `false` to suppress including the author's note. ### Returns From e5d0cc7b49b5ddb52d74753ca4cde69ba2ace687 Mon Sep 17 00:00:00 2001 From: Llama <34464159+pi6am@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:20:33 -0700 Subject: [PATCH 5/6] Fix exception thrown by kobold.modeltype in Lua Fixes this exception: File "aiserver.py", line 3389, in lua_get_modeltype hidden_size = get_hidden_size_from_model(model) NameError: name 'get_hidden_size_from_model' is not defined The kobold.modeltype method eventually attempts to call get_hidden_size_from_model in Python, but this method was previously defined only within a local scope and so is not visible from within lua_get_modeltype. Since get_hidden_size_from_model only accesses its model argument, there is no reason not to make it a module-level method. Also change the severity of several more Lua error logs to error. --- aiserver.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/aiserver.py b/aiserver.py index 0d3ea356..87a4eca3 100644 --- a/aiserver.py +++ b/aiserver.py @@ -756,6 +756,12 @@ def getmodelname(): modelname = vars.model return modelname +#==================================================================# +# Get hidden size from model +#==================================================================# +def get_hidden_size_from_model(model): + return model.get_input_embeddings().embedding_dim + #==================================================================# # Breakmodel configuration functions #==================================================================# @@ -2415,9 +2421,6 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal return lazy_load_callback - def get_hidden_size_from_model(model): - return model.get_input_embeddings().embedding_dim - def maybe_low_cpu_mem_usage() -> Dict[str, Any]: if(packaging.version.parse(transformers_version) < packaging.version.parse("4.11.0")): logger.warning(f"Please upgrade to transformers 4.11.0 for lower RAM usage. You have transformers {transformers_version}.") @@ -2905,7 +2908,7 @@ def lua_startup(): except lupa.LuaError as e: print(colors.RED + "ERROR!" + colors.END) vars.lua_koboldbridge.obliterate_multiverse() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") exit(1) logger.init_ok("LUA bridge", status="OK") @@ -3463,7 +3466,7 @@ def execute_inmod(): vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True) sendUSStatItems() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") set_aibusy(0) @@ -3480,7 +3483,7 @@ def execute_outmod(): vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True) sendUSStatItems() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") set_aibusy(0) if(vars.lua_koboldbridge.resend_settings_required): @@ -4900,7 +4903,7 @@ def generate(txt, minimum, maximum, found_entries=None): vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True) sendUSStatItems() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") else: emit('from_server', {'cmd': 'errmsg', 'data': 'Error occurred during generator call; please check console.'}, broadcast=True) @@ -5408,7 +5411,7 @@ def tpumtjgenerate(txt, minimum, maximum, found_entries=None): vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True) sendUSStatItems() - logger.debug('LUA ERROR: ' + str(e).replace("\033", "")) + logger.error('LUA ERROR: ' + str(e).replace("\033", "")) logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.") else: emit('from_server', {'cmd': 'errmsg', 'data': 'Error occurred during generator call; please check console.'}, broadcast=True) From 9ff50d81fdb85892be64fd35f1a95c19b28636c4 Mon Sep 17 00:00:00 2001 From: vfbd Date: Mon, 17 Oct 2022 14:36:23 -0400 Subject: [PATCH 6/6] Fix regex for the prompt parameter of the POST /story/end endpoint --- aiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index d883640a..c42cb7df 100644 --- a/aiserver.py +++ b/aiserver.py @@ -631,7 +631,7 @@ tags = [ api_version = None # This gets set automatically so don't change this value api_v1 = KoboldAPISpec( - version="1.2.0", + version="1.2.1", prefixes=["/api/v1", "/api/latest"], tags=tags, ) @@ -7858,7 +7858,7 @@ def prompt_validator(prompt: str): raise ValidationError("String does not match expected pattern.") class SubmissionInputSchema(KoboldSchema): - prompt: str = fields.String(required=True, validate=prompt_validator, metadata={"pattern": r"^.*\S.*$", "description": "This is the submission."}) + prompt: str = fields.String(required=True, validate=prompt_validator, metadata={"pattern": r"^[\S\s]*\S[\S\s]*$", "description": "This is the submission."}) disable_input_formatting: bool = fields.Boolean(load_default=True, metadata={"description": "When enabled, disables all input formatting options, overriding their individual enabled/disabled states."}) frmtadsnsp: Optional[bool] = fields.Boolean(metadata={"description": "Input formatting option. When enabled, adds a leading space to your input if there is no trailing whitespace at the end of the previous action."})