From 7d84466d9d6c3be749b9f394ad776b429c70acb6 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 31 May 2023 22:48:29 +0200 Subject: [PATCH 1/6] Clean-up trust remote code --- aiserver.py | 11 +---------- koboldai_settings.py | 5 ++--- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/aiserver.py b/aiserver.py index 5c93f87f..7cd88393 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1389,9 +1389,7 @@ def general_startup(override_args=None): parser.add_argument("--summarizer_model", action='store', default="philschmid/bart-large-cnn-samsum", help="Huggingface model to use for summarization. Defaults to sshleifer/distilbart-cnn-12-6") parser.add_argument("--max_summary_length", action='store', default=75, help="Maximum size for summary to send to image generation") parser.add_argument("--multi_story", action='store_true', default=False, help="Allow multi-story mode (experimental)") - parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") - parser.add_argument("--trust_remote_code", action='store_true', default=False, help="Allow Huggingface Models to Execute Code (Insecure!)") - + parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") parser.add_argument('-f', action='store', help="option for compatability with colab memory profiles") parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen") parser.add_argument('-q', '--quiesce', action='count', default=0, help="The default logging level is ERROR or higher. This value decreases the amount of logging seen in your screen") @@ -1520,13 +1518,6 @@ def general_startup(override_args=None): allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')]))) print(f"Allowed IPs: {allowed_ips}") - if args.trust_remote_code: - logger.warning("EXECUTION OF UNSAFE REMOTE CODE IS ENABLED!!!") - logger.warning("You are not protected from Model Viruses in this mode!") - logger.warning("Exit the program now to abort execution!") - logger.warning("Only use this mode with models that you trust and verified!") - time.sleep(25) - koboldai_vars.trust_remote_code = True if args.cpu: koboldai_vars.use_colab_tpu = False koboldai_vars.hascuda = False diff --git a/koboldai_settings.py b/koboldai_settings.py index 29a82406..e9cc16d0 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1206,12 +1206,12 @@ class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui', 'trust_remote_code'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'on_colab' 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model', - 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch', 'trust_remote_code'] + 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch'] settings_name = "system" def __init__(self, socketio, koboldai_var): self._socketio = socketio @@ -1297,7 +1297,6 @@ class system_settings(settings): self.seen_messages = [] self.git_repository = "" self.git_branch = "" - self.trust_remote_code = False @dataclass From cdf4098c97953b1d260a1531b957ca550e353865 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:40:42 +0300 Subject: [PATCH 2/6] Reverting the RTE (Custom) VE was right, concedo is right --- presets/Custom.presets | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/presets/Custom.presets b/presets/Custom.presets index eb089cab..3efadd8f 100644 --- a/presets/Custom.presets +++ b/presets/Custom.presets @@ -17,13 +17,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 2, 1, - 0, - 6 + 0 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 1, 2, - 0, - 6 + 0 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 4, 5, 1, 0, 2, - 3, - 6 + 3 ] } ] From 077d64f6df9834f1a8acd6e3df2904b7c57d82e8 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:42:35 +0300 Subject: [PATCH 3/6] Reverting the RTE (13B) VE was right, concedo is right --- presets/Official_13B.presets | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/presets/Official_13B.presets b/presets/Official_13B.presets index 994f7abd..333dd983 100644 --- a/presets/Official_13B.presets +++ b/presets/Official_13B.presets @@ -17,13 +17,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 2, 0, 3, 5, 1, - 4, - 6 + 4 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 404, "rep_pen_slope": 0.8, "sampler_order": [ + 6, 0, 5, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 7, "sampler_order": [ + 6, 3, 2, 0, 5, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 0, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 0, 4, 1, 3, 5, - 2, - 6 + 2 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 400, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -206,13 +206,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -233,13 +233,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -260,13 +260,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 3, 2, 5, 0, 1, - 4, - 6 + 4 ] } ] From e2c422aef8a537cf93afeb1e33366d6d83bea223 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:43:44 +0300 Subject: [PATCH 4/6] Reverting the RTE (6B) VE was right, concedo is right --- presets/Official_6B.presets | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/presets/Official_6B.presets b/presets/Official_6B.presets index 6fcf1a33..1ce6a627 100644 --- a/presets/Official_6B.presets +++ b/presets/Official_6B.presets @@ -8,13 +8,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.72, "tfs": 1.0, @@ -35,13 +35,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.51, "tfs": 0.99, @@ -71,13 +71,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 3.4, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 6.8, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] } ] From acfcde765b6a20e49bfc098a9729afd770e28c39 Mon Sep 17 00:00:00 2001 From: Henk Date: Fri, 2 Jun 2023 01:26:40 +0200 Subject: [PATCH 5/6] Allow breakmodel on Colab --- aiserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index 7cd88393..ef075586 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1471,7 +1471,6 @@ def general_startup(override_args=None): args.remote = True; args.override_rename = True; args.override_delete = True; - args.nobreakmodel = True; args.quiet = True; args.lowmem = True; args.noaimenu = True; From 18dc7069447e7b7284e461f0d689cf0ad03cbd17 Mon Sep 17 00:00:00 2001 From: Bogdan Drema Date: Sat, 3 Jun 2023 12:21:52 +0100 Subject: [PATCH 6/6] fix: tpu support models saved in bfloat16 --- tpu_mtj_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index df37e0be..401d6ccf 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1255,7 +1255,11 @@ def load_model(path: str, model_type: str, badwordsids=koboldai_settings.badword params["cores_per_replica"], network.state["params"][spec["module"]][spec["param"]].shape, ) - tensor = jnp.array(tensor.detach()) + tensor = tensor.detach() + # numpy does not support bfloat16 + if tensor.dtype is torch.bfloat16: + tensor = tensor.to(torch.float32) + tensor = jnp.array(tensor) if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: tensor = tensor.bfloat16() network.state["params"][spec["module"]][spec["param"]] = move_xmap(