diff --git a/aiserver.py b/aiserver.py index fdcfb297..62265e7b 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1395,9 +1395,7 @@ def general_startup(override_args=None): parser.add_argument("--summarizer_model", action='store', default="philschmid/bart-large-cnn-samsum", help="Huggingface model to use for summarization. Defaults to sshleifer/distilbart-cnn-12-6") parser.add_argument("--max_summary_length", action='store', default=75, help="Maximum size for summary to send to image generation") parser.add_argument("--multi_story", action='store_true', default=False, help="Allow multi-story mode (experimental)") - parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") - parser.add_argument("--trust_remote_code", action='store_true', default=False, help="Allow Huggingface Models to Execute Code (Insecure!)") - + parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") parser.add_argument('-f', action='store', help="option for compatability with colab memory profiles") parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen") parser.add_argument('-q', '--quiesce', action='count', default=0, help="The default logging level is ERROR or higher. This value decreases the amount of logging seen in your screen") @@ -1479,7 +1477,6 @@ def general_startup(override_args=None): args.remote = True; args.override_rename = True; args.override_delete = True; - args.nobreakmodel = True; args.quiet = True; args.lowmem = True; args.noaimenu = True; @@ -1526,13 +1523,6 @@ def general_startup(override_args=None): allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')]))) print(f"Allowed IPs: {allowed_ips}") - if args.trust_remote_code: - logger.warning("EXECUTION OF UNSAFE REMOTE CODE IS ENABLED!!!") - logger.warning("You are not protected from Model Viruses in this mode!") - logger.warning("Exit the program now to abort execution!") - logger.warning("Only use this mode with models that you trust and verified!") - time.sleep(25) - koboldai_vars.trust_remote_code = True if args.cpu: koboldai_vars.use_colab_tpu = False koboldai_vars.hascuda = False diff --git a/koboldai_settings.py b/koboldai_settings.py index 29a82406..e9cc16d0 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1206,12 +1206,12 @@ class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui', 'trust_remote_code'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'on_colab' 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model', - 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch', 'trust_remote_code'] + 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch'] settings_name = "system" def __init__(self, socketio, koboldai_var): self._socketio = socketio @@ -1297,7 +1297,6 @@ class system_settings(settings): self.seen_messages = [] self.git_repository = "" self.git_branch = "" - self.trust_remote_code = False @dataclass diff --git a/presets/Custom.presets b/presets/Custom.presets index eb089cab..3efadd8f 100644 --- a/presets/Custom.presets +++ b/presets/Custom.presets @@ -17,13 +17,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 2, 1, - 0, - 6 + 0 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 5, 4, 3, 1, 2, - 0, - 6 + 0 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 0, 1, 2, 3, 4, - 5, - 6 + 5 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 1024, "rep_pen_slope": 0.7, "sampler_order": [ + 6, 4, 5, 1, 0, 2, - 3, - 6 + 3 ] } ] diff --git a/presets/Official_13B.presets b/presets/Official_13B.presets index 994f7abd..333dd983 100644 --- a/presets/Official_13B.presets +++ b/presets/Official_13B.presets @@ -17,13 +17,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 2, 0, 3, 5, 1, - 4, - 6 + 4 ] }, { @@ -44,13 +44,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -71,13 +71,13 @@ "rep_pen_range": 404, "rep_pen_slope": 0.8, "sampler_order": [ + 6, 0, 5, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 7, "sampler_order": [ + 6, 3, 2, 0, 5, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 0, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 0, 4, 1, 3, 5, - 2, - 6 + 2 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 400, "rep_pen_slope": 0.3, "sampler_order": [ + 6, 5, 0, 3, 2, 1, - 4, - 6 + 4 ] }, { @@ -206,13 +206,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -233,13 +233,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -260,13 +260,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.1, "sampler_order": [ + 6, 3, 2, 5, 0, 1, - 4, - 6 + 4 ] } ] diff --git a/presets/Official_6B.presets b/presets/Official_6B.presets index 6fcf1a33..1ce6a627 100644 --- a/presets/Official_6B.presets +++ b/presets/Official_6B.presets @@ -8,13 +8,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.2, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.72, "tfs": 1.0, @@ -35,13 +35,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ], "temp": 0.51, "tfs": 0.99, @@ -71,13 +71,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -98,13 +98,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -125,13 +125,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 0.0, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -152,13 +152,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 3.4, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] }, { @@ -179,13 +179,13 @@ "rep_pen_range": 2048, "rep_pen_slope": 6.8, "sampler_order": [ + 6, 5, 0, 2, 3, 1, - 4, - 6 + 4 ] } ] diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index df37e0be..401d6ccf 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1255,7 +1255,11 @@ def load_model(path: str, model_type: str, badwordsids=koboldai_settings.badword params["cores_per_replica"], network.state["params"][spec["module"]][spec["param"]].shape, ) - tensor = jnp.array(tensor.detach()) + tensor = tensor.detach() + # numpy does not support bfloat16 + if tensor.dtype is torch.bfloat16: + tensor = tensor.to(torch.float32) + tensor = jnp.array(tensor) if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: tensor = tensor.bfloat16() network.state["params"][spec["module"]][spec["param"]] = move_xmap(