From c1abbf1473096b8be6216dbc8d60ce3ee9f4b5c6 Mon Sep 17 00:00:00 2001 From: Henk Date: Tue, 30 May 2023 12:56:22 +0200 Subject: [PATCH 01/10] More aggressive isolation --- commandline.bat | 2 ++ install_requirements.bat | 2 ++ play.bat | 4 +++- update-koboldai.bat | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/commandline.bat b/commandline.bat index 5372a9a3..f7293eda 100644 --- a/commandline.bat +++ b/commandline.bat @@ -2,6 +2,8 @@ cd /D %~dp0 :Isolation +conda deactivate 2>NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 SET PYTHONPATH= diff --git a/install_requirements.bat b/install_requirements.bat index 69e4a47a..73e73594 100644 --- a/install_requirements.bat +++ b/install_requirements.bat @@ -10,6 +10,8 @@ Reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v "LongPathsEnabled" cd /D %~dp0 :Isolation +conda deactivate 2>NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 SET PYTHONPATH= diff --git a/play.bat b/play.bat index 76261a37..91688b54 100644 --- a/play.bat +++ b/play.bat @@ -2,11 +2,13 @@ cd /D %~dp0 :Isolation +conda deactivate 2>NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 SET PYTHONPATH= -rmdir /S /Q flask_session +rmdir /S /Q flask_session 2>NUL TITLE KoboldAI - Server SET /P M=NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 SET PYTHONPATH= From 953f615e3d1135f1016d001a53c7fedc016e0422 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 31 May 2023 02:29:26 +0200 Subject: [PATCH 02/10] Conda deactivation fix --- Jupyter.bat | 7 +++++++ commandline.bat | 2 +- install_git_transformers.bat | 2 ++ install_requirements.bat | 2 +- play.bat | 2 +- update-koboldai.bat | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Jupyter.bat b/Jupyter.bat index df19a07f..c00d250d 100644 --- a/Jupyter.bat +++ b/Jupyter.bat @@ -6,6 +6,13 @@ IF %M%==1 GOTO drivemap IF %M%==2 GOTO subfolder IF %M%==3 GOTO drivemap_B +:Isolation +call conda deactivate 2>NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ +SET CONDA_SHLVL= +SET PYTHONNOUSERSITE=1 +SET PYTHONPATH= + :subfolder umamba.exe install --no-shortcuts -r miniconda3 -n base -c conda-forge jupyterlab jupyterlab-git call miniconda3\condabin\activate diff --git a/commandline.bat b/commandline.bat index f7293eda..94e61608 100644 --- a/commandline.bat +++ b/commandline.bat @@ -2,7 +2,7 @@ cd /D %~dp0 :Isolation -conda deactivate 2>NUL +call conda deactivate 2>NUL set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 diff --git a/install_git_transformers.bat b/install_git_transformers.bat index 8c6257cc..34194459 100644 --- a/install_git_transformers.bat +++ b/install_git_transformers.bat @@ -2,6 +2,8 @@ cd /D %~dp0 :Isolation +call conda deactivate 2>NUL +set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 SET PYTHONPATH= diff --git a/install_requirements.bat b/install_requirements.bat index 73e73594..9756a18f 100644 --- a/install_requirements.bat +++ b/install_requirements.bat @@ -10,7 +10,7 @@ Reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v "LongPathsEnabled" cd /D %~dp0 :Isolation -conda deactivate 2>NUL +call conda deactivate 2>NUL set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 diff --git a/play.bat b/play.bat index 91688b54..c9e82b83 100644 --- a/play.bat +++ b/play.bat @@ -2,7 +2,7 @@ cd /D %~dp0 :Isolation -conda deactivate 2>NUL +call conda deactivate 2>NUL set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 diff --git a/update-koboldai.bat b/update-koboldai.bat index f99e6598..1fd66e4a 100644 --- a/update-koboldai.bat +++ b/update-koboldai.bat @@ -2,7 +2,7 @@ cd /d %~dp0 :Isolation -conda deactivate 2>NUL +call conda deactivate 2>NUL set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\ SET CONDA_SHLVL= SET PYTHONNOUSERSITE=1 From bda8c931f864a312b909ff83053f2412f03db2b0 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 31 May 2023 16:56:08 +0200 Subject: [PATCH 03/10] Horde backend fix --- modeling/inference_models/horde/class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modeling/inference_models/horde/class.py b/modeling/inference_models/horde/class.py index f7da6604..3c19a0c1 100644 --- a/modeling/inference_models/horde/class.py +++ b/modeling/inference_models/horde/class.py @@ -41,7 +41,7 @@ class model_backend(InferenceModel): return model_name == "CLUSTER" or model_name in [x['value'] for x in self.models] def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}): - if os.path.exists("settings/api.model_backend.settings") and 'base_url' not in vars(self): + if os.path.exists("settings/horde.model_backend.settings") and 'base_url' not in vars(self): with open("settings/horde.model_backend.settings", "r") as f: temp = json.load(f) self.base_url = temp['url'] From 7d84466d9d6c3be749b9f394ad776b429c70acb6 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 31 May 2023 22:48:29 +0200 Subject: [PATCH 04/10] Clean-up trust remote code --- aiserver.py | 11 +---------- koboldai_settings.py | 5 ++--- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/aiserver.py b/aiserver.py index 5c93f87f..7cd88393 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1389,9 +1389,7 @@ def general_startup(override_args=None): parser.add_argument("--summarizer_model", action='store', default="philschmid/bart-large-cnn-samsum", help="Huggingface model to use for summarization. Defaults to sshleifer/distilbart-cnn-12-6") parser.add_argument("--max_summary_length", action='store', default=75, help="Maximum size for summary to send to image generation") parser.add_argument("--multi_story", action='store_true', default=False, help="Allow multi-story mode (experimental)") - parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") - parser.add_argument("--trust_remote_code", action='store_true', default=False, help="Allow Huggingface Models to Execute Code (Insecure!)") - + parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") parser.add_argument('-f', action='store', help="option for compatability with colab memory profiles") parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen") parser.add_argument('-q', '--quiesce', action='count', default=0, help="The default logging level is ERROR or higher. This value decreases the amount of logging seen in your screen") @@ -1520,13 +1518,6 @@ def general_startup(override_args=None): allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')]))) print(f"Allowed IPs: {allowed_ips}") - if args.trust_remote_code: - logger.warning("EXECUTION OF UNSAFE REMOTE CODE IS ENABLED!!!") - logger.warning("You are not protected from Model Viruses in this mode!") - logger.warning("Exit the program now to abort execution!") - logger.warning("Only use this mode with models that you trust and verified!") - time.sleep(25) - koboldai_vars.trust_remote_code = True if args.cpu: koboldai_vars.use_colab_tpu = False koboldai_vars.hascuda = False diff --git a/koboldai_settings.py b/koboldai_settings.py index 29a82406..e9cc16d0 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1206,12 +1206,12 @@ class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui', 'trust_remote_code'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'on_colab' 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model', - 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch', 'trust_remote_code'] + 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch'] settings_name = "system" def __init__(self, socketio, koboldai_var): self._socketio = socketio @@ -1297,7 +1297,6 @@ class system_settings(settings): self.seen_messages = [] self.git_repository = "" self.git_branch = "" - self.trust_remote_code = False @dataclass From cdf4098c97953b1d260a1531b957ca550e353865 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:40:42 +0300 Subject: [PATCH 05/10] Reverting the RTE (Custom) VE was right, concedo is right --- presets/Custom.presets | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/presets/Custom.presets b/presets/Custom.presets index eb089cab..3efadd8f 100644 --- a/presets/Custom.presets +++ b/presets/Custom.presets @@ -17,13 +17,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 2, 1, - 0, - 6 + 0 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 1, 2, - 0, - 6 + 0 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 4, 5, 1, 0, 2, - 3, - 6 + 3 ] } ] From 077d64f6df9834f1a8acd6e3df2904b7c57d82e8 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:42:35 +0300 Subject: [PATCH 06/10] Reverting the RTE (13B) VE was right, concedo is right --- presets/Official_13B.presets | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/presets/Official_13B.presets b/presets/Official_13B.presets index 994f7abd..333dd983 100644 --- a/presets/Official_13B.presets +++ b/presets/Official_13B.presets @@ -17,13 +17,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 2, 0, 3, 5, 1, - 4, - 6 + 4 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 404, "rep_pen_slope": 0.8, "sampler_order": [ + 6, 0, 5, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 7, "sampler_order": [ + 6, 3, 2, 0, 5, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 0, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 0, 4, 1, 3, 5, - 2, - 6 + 2 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 400, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -206,13 +206,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -233,13 +233,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -260,13 +260,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 3, 2, 5, 0, 1, - 4, - 6 + 4 ] } ] From e2c422aef8a537cf93afeb1e33366d6d83bea223 Mon Sep 17 00:00:00 2001 From: LightSaveUs <108632621+LightSaveUs@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:43:44 +0300 Subject: [PATCH 07/10] Reverting the RTE (6B) VE was right, concedo is right --- presets/Official_6B.presets | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/presets/Official_6B.presets b/presets/Official_6B.presets index 6fcf1a33..1ce6a627 100644 --- a/presets/Official_6B.presets +++ b/presets/Official_6B.presets @@ -8,13 +8,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.72, "tfs": 1.0, @@ -35,13 +35,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.51, "tfs": 0.99, @@ -71,13 +71,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 3.4, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 6.8, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] } ] From acfcde765b6a20e49bfc098a9729afd770e28c39 Mon Sep 17 00:00:00 2001 From: Henk Date: Fri, 2 Jun 2023 01:26:40 +0200 Subject: [PATCH 08/10] Allow breakmodel on Colab --- aiserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index 7cd88393..ef075586 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1471,7 +1471,6 @@ def general_startup(override_args=None): args.remote = True; args.override_rename = True; args.override_delete = True; - args.nobreakmodel = True; args.quiet = True; args.lowmem = True; args.noaimenu = True; From 5c4d580aace8dd2e0157db3f7b93467e923e8dc4 Mon Sep 17 00:00:00 2001 From: ebolam Date: Fri, 2 Jun 2023 12:58:59 -0400 Subject: [PATCH 09/10] Fix for --nobreakmodel forcing CPU Put importing of colab packages into a if function so it doesn't error out --- aiserver.py | 1 - .../inference_models/generic_hf_torch/class.py | 2 +- modeling/warpers.py | 15 +++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/aiserver.py b/aiserver.py index 5c93f87f..87449abd 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1682,7 +1682,6 @@ class RestrictedUnpickler(pickle.Unpickler): ) def load(self, *args, **kwargs): - logger.info("Using safe unpickle") self.original_persistent_load = getattr( self, "persistent_load", pickle.Unpickler.persistent_load ) diff --git a/modeling/inference_models/generic_hf_torch/class.py b/modeling/inference_models/generic_hf_torch/class.py index fd4c2a1a..57bcd5ad 100644 --- a/modeling/inference_models/generic_hf_torch/class.py +++ b/modeling/inference_models/generic_hf_torch/class.py @@ -250,7 +250,7 @@ class model_backend(HFTorchInferenceModel): if utils.koboldai_vars.hascuda: - if self.usegpu: + if self.usegpu or self.nobreakmodel: # Use just VRAM self.model = self.model.half().to(utils.koboldai_vars.gpu_device) elif self.breakmodel: diff --git a/modeling/warpers.py b/modeling/warpers.py index ca7e7396..b8035023 100644 --- a/modeling/warpers.py +++ b/modeling/warpers.py @@ -42,10 +42,17 @@ import utils import torch import numpy as np -if utils.koboldai_vars.use_colab_tpu: - import jax - import jax.numpy as jnp - import tpu_mtj_backend +try: + ignore = utils.koboldai_vars.use_colab_tpu + ok = True +except: + ok = False + +if ok: + if utils.koboldai_vars.use_colab_tpu: + import jax + import jax.numpy as jnp + import tpu_mtj_backend def update_settings(): From 18dc7069447e7b7284e461f0d689cf0ad03cbd17 Mon Sep 17 00:00:00 2001 From: Bogdan Drema Date: Sat, 3 Jun 2023 12:21:52 +0100 Subject: [PATCH 10/10] fix: tpu support models saved in bfloat16 --- tpu_mtj_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index df37e0be..401d6ccf 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1255,7 +1255,11 @@ def load_model(path: str, model_type: str, badwordsids=koboldai_settings.badword params["cores_per_replica"], network.state["params"][spec["module"]][spec["param"]].shape, ) - tensor = jnp.array(tensor.detach()) + tensor = tensor.detach() + # numpy does not support bfloat16 + if tensor.dtype is torch.bfloat16: + tensor = tensor.to(torch.float32) + tensor = jnp.array(tensor) if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: tensor = tensor.bfloat16() network.state["params"][spec["module"]][spec["param"]] = move_xmap(