rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Pull upstream changes, fix conflict

Browse Source
This commit is contained in:
0cc4m
2023-06-04 09:06:31 +02:00
parent 974328ed22 0ac2800601
commit eace95cc72
15 changed files with 85 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Jupyter.bat
View File

@@ -6,6 +6,13 @@ IF %M%==1 GOTO drivemap
IF %M%==2 GOTO subfolder IF %M%==2 GOTO subfolder
IF %M%==3 GOTO drivemap_B IF %M%==3 GOTO drivemap_B
:Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1
SET PYTHONPATH=
:subfolder :subfolder
umamba.exe install --no-shortcuts -r miniconda3 -n base -c conda-forge jupyterlab jupyterlab-git umamba.exe install --no-shortcuts -r miniconda3 -n base -c conda-forge jupyterlab jupyterlab-git
call miniconda3\condabin\activate call miniconda3\condabin\activate

13
aiserver.py
View File

@@ -1390,9 +1390,7 @@ def general_startup(override_args=None):
parser.add_argument("--summarizer_model", action='store', default="philschmid/bart-large-cnn-samsum", help="Huggingface model to use for summarization. Defaults to sshleifer/distilbart-cnn-12-6") parser.add_argument("--summarizer_model", action='store', default="philschmid/bart-large-cnn-samsum", help="Huggingface model to use for summarization. Defaults to sshleifer/distilbart-cnn-12-6")
parser.add_argument("--max_summary_length", action='store', default=75, help="Maximum size for summary to send to image generation") parser.add_argument("--max_summary_length", action='store', default=75, help="Maximum size for summary to send to image generation")
parser.add_argument("--multi_story", action='store_true', default=False, help="Allow multi-story mode (experimental)") parser.add_argument("--multi_story", action='store_true', default=False, help="Allow multi-story mode (experimental)")
parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)") parser.add_argument("--peft", type=str, help="Specify the path or HuggingFace ID of a Peft to load it. Not supported on TPU. (Experimental)")
parser.add_argument("--trust_remote_code", action='store_true', default=False, help="Allow Huggingface Models to Execute Code (Insecure!)")
parser.add_argument('-f', action='store', help="option for compatability with colab memory profiles") parser.add_argument('-f', action='store', help="option for compatability with colab memory profiles")
parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen") parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen")
parser.add_argument('-q', '--quiesce', action='count', default=0, help="The default logging level is ERROR or higher. This value decreases the amount of logging seen in your screen") parser.add_argument('-q', '--quiesce', action='count', default=0, help="The default logging level is ERROR or higher. This value decreases the amount of logging seen in your screen")
@@ -1474,7 +1472,6 @@ def general_startup(override_args=None):
args.remote = True; args.remote = True;
args.override_rename = True; args.override_rename = True;
args.override_delete = True; args.override_delete = True;
args.nobreakmodel = True;
args.quiet = True; args.quiet = True;
args.lowmem = True; args.lowmem = True;
args.noaimenu = True; args.noaimenu = True;
@@ -1521,13 +1518,6 @@ def general_startup(override_args=None):
allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')]))) allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')])))
print(f"Allowed IPs: {allowed_ips}") print(f"Allowed IPs: {allowed_ips}")
if args.trust_remote_code:
logger.warning("EXECUTION OF UNSAFE REMOTE CODE IS ENABLED!!!")
logger.warning("You are not protected from Model Viruses in this mode!")
logger.warning("Exit the program now to abort execution!")
logger.warning("Only use this mode with models that you trust and verified!")
time.sleep(25)
koboldai_vars.trust_remote_code = True
if args.cpu: if args.cpu:
koboldai_vars.use_colab_tpu = False koboldai_vars.use_colab_tpu = False
koboldai_vars.hascuda = False koboldai_vars.hascuda = False
@@ -1682,7 +1672,6 @@ class RestrictedUnpickler(pickle.Unpickler):
) )
def load(self, *args, **kwargs): def load(self, *args, **kwargs):
logger.info("Using safe unpickle")
self.original_persistent_load = getattr( self.original_persistent_load = getattr(
self, "persistent_load", pickle.Unpickler.persistent_load self, "persistent_load", pickle.Unpickler.persistent_load
) )

2
commandline.bat
View File

@@ -2,6 +2,8 @@
cd /D %~dp0 cd /D %~dp0
:Isolation :Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL= SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1 SET PYTHONNOUSERSITE=1
SET PYTHONPATH= SET PYTHONPATH=

2
install_git_transformers.bat
View File

@@ -2,6 +2,8 @@
cd /D %~dp0 cd /D %~dp0
:Isolation :Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL= SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1 SET PYTHONNOUSERSITE=1
SET PYTHONPATH= SET PYTHONPATH=

2
install_requirements.bat
View File

@@ -10,6 +10,8 @@ Reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v "LongPathsEnabled"
cd /D %~dp0 cd /D %~dp0
:Isolation :Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL= SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1 SET PYTHONNOUSERSITE=1
SET PYTHONPATH= SET PYTHONPATH=

5
koboldai_settings.py
View File

@@ -1206,12 +1206,12 @@ class system_settings(settings):
local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold',
'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui',
'sp', '_horde_pid', 'inference_config', 'image_pipeline', 'sp', '_horde_pid', 'inference_config', 'image_pipeline',
'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui', 'trust_remote_code'] 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui']
no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold',
'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy',
'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'on_colab' 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'on_colab'
'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model', 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model',
'generating_image', 'bit_8_available', 'bit_4_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch', 'trust_remote_code'] 'generating_image', 'bit_8_available', 'bit_4_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui', 'git_repository', 'git_branch']
settings_name = "system" settings_name = "system"
def __init__(self, socketio, koboldai_var): def __init__(self, socketio, koboldai_var):
self._socketio = socketio self._socketio = socketio
@@ -1304,7 +1304,6 @@ class system_settings(settings):
self.seen_messages = [] self.seen_messages = []
self.git_repository = "" self.git_repository = ""
self.git_branch = "" self.git_branch = ""
self.trust_remote_code = False
@dataclass @dataclass

2
modeling/inference_models/generic_hf_torch/class.py
View File

@@ -251,7 +251,7 @@ class model_backend(HFTorchInferenceModel):
if utils.koboldai_vars.hascuda: if utils.koboldai_vars.hascuda:
if self.usegpu: if self.usegpu or self.nobreakmodel:
# Use just VRAM # Use just VRAM
self.model = self.model.half().to(utils.koboldai_vars.gpu_device) self.model = self.model.half().to(utils.koboldai_vars.gpu_device)
elif self.breakmodel: elif self.breakmodel:

2
modeling/inference_models/horde/class.py
View File

@@ -41,7 +41,7 @@ class model_backend(InferenceModel):
return model_name == "CLUSTER" or model_name in [x['value'] for x in self.models] return model_name == "CLUSTER" or model_name in [x['value'] for x in self.models]
def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}): def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}):
if os.path.exists("settings/api.model_backend.settings") and 'base_url' not in vars(self): if os.path.exists("settings/horde.model_backend.settings") and 'base_url' not in vars(self):
with open("settings/horde.model_backend.settings", "r") as f: with open("settings/horde.model_backend.settings", "r") as f:
temp = json.load(f) temp = json.load(f)
self.base_url = temp['url'] self.base_url = temp['url']

15
modeling/warpers.py
View File

@@ -42,10 +42,17 @@ import utils
import torch import torch
import numpy as np import numpy as np
if utils.koboldai_vars.use_colab_tpu: try:
import jax ignore = utils.koboldai_vars.use_colab_tpu
import jax.numpy as jnp ok = True
import tpu_mtj_backend except:
ok = False
if ok:
if utils.koboldai_vars.use_colab_tpu:
import jax
import jax.numpy as jnp
import tpu_mtj_backend
def update_settings(): def update_settings():

4
play.bat
View File

@@ -2,11 +2,13 @@
cd /D %~dp0 cd /D %~dp0
:Isolation :Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL= SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1 SET PYTHONNOUSERSITE=1
SET PYTHONPATH= SET PYTHONPATH=
rmdir /S /Q flask_session rmdir /S /Q flask_session 2>NUL
TITLE KoboldAI - Server TITLE KoboldAI - Server
SET /P M=<loader.settings SET /P M=<loader.settings

24
presets/Custom.presets
View File

@@ -17,13 +17,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
5, 5,
4, 4,
3, 3,
2, 2,
1, 1,
0, 0
6
] ]
}, },
{ {
@@ -44,13 +44,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
5, 5,
4, 4,
3, 3,
1, 1,
2, 2,
0, 0
6
] ]
}, },
{ {
@@ -71,13 +71,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
0, 0,
1, 1,
2, 2,
3, 3,
4, 4,
5, 5
6
] ]
}, },
{ {
@@ -98,13 +98,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
0, 0,
1, 1,
2, 2,
3, 3,
4, 4,
5, 5
6
] ]
}, },
{ {
@@ -125,13 +125,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
0, 0,
1, 1,
2, 2,
3, 3,
4, 4,
5, 5
6
] ]
}, },
{ {
@@ -152,13 +152,13 @@
"rep_pen_range": 1024, "rep_pen_range": 1024,
"rep_pen_slope": 0.7, "rep_pen_slope": 0.7,
"sampler_order": [ "sampler_order": [
6,
4, 4,
5, 5,
1, 1,
0, 0,
2, 2,
3, 3
6
] ]
} }
] ]

40
presets/Official_13B.presets
View File

@@ -17,13 +17,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.1, "rep_pen_slope": 0.1,
"sampler_order": [ "sampler_order": [
6,
2, 2,
0, 0,
3, 3,
5, 5,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -44,13 +44,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.3, "rep_pen_slope": 0.3,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -71,13 +71,13 @@
"rep_pen_range": 404, "rep_pen_range": 404,
"rep_pen_slope": 0.8, "rep_pen_slope": 0.8,
"sampler_order": [ "sampler_order": [
6,
0, 0,
5, 5,
3, 3,
2, 2,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -98,13 +98,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 7, "rep_pen_slope": 7,
"sampler_order": [ "sampler_order": [
6,
3, 3,
2, 2,
0, 0,
5, 5,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -125,13 +125,13 @@
"rep_pen_range": 0, "rep_pen_range": 0,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
0, 0,
4, 4,
1, 1,
3, 3,
5, 5,
2, 2
6
] ]
}, },
{ {
@@ -152,13 +152,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.1, "rep_pen_slope": 0.1,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -179,13 +179,13 @@
"rep_pen_range": 400, "rep_pen_range": 400,
"rep_pen_slope": 0.3, "rep_pen_slope": 0.3,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
3, 3,
2, 2,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -206,13 +206,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.2, "rep_pen_slope": 0.2,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -233,13 +233,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -260,13 +260,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.1, "rep_pen_slope": 0.1,
"sampler_order": [ "sampler_order": [
6,
3, 3,
2, 2,
5, 5,
0, 0,
1, 1,
4, 4
6
] ]
} }
] ]

28
presets/Official_6B.presets
View File

@@ -8,13 +8,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.2, "rep_pen_slope": 0.2,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
], ],
"temp": 0.72, "temp": 0.72,
"tfs": 1.0, "tfs": 1.0,
@@ -35,13 +35,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
], ],
"temp": 0.51, "temp": 0.51,
"tfs": 0.99, "tfs": 0.99,
@@ -71,13 +71,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -98,13 +98,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -125,13 +125,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 0.0, "rep_pen_slope": 0.0,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -152,13 +152,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 3.4, "rep_pen_slope": 3.4,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
}, },
{ {
@@ -179,13 +179,13 @@
"rep_pen_range": 2048, "rep_pen_range": 2048,
"rep_pen_slope": 6.8, "rep_pen_slope": 6.8,
"sampler_order": [ "sampler_order": [
6,
5, 5,
0, 0,
2, 2,
3, 3,
1, 1,
4, 4
6
] ]
} }
] ]

6
tpu_mtj_backend.py
View File

@@ -1255,7 +1255,11 @@ def load_model(path: str, model_type: str, badwordsids=koboldai_settings.badword
params["cores_per_replica"], params["cores_per_replica"],
network.state["params"][spec["module"]][spec["param"]].shape, network.state["params"][spec["module"]][spec["param"]].shape,
) )
tensor = jnp.array(tensor.detach()) tensor = tensor.detach()
# numpy does not support bfloat16
if tensor.dtype is torch.bfloat16:
tensor = tensor.to(torch.float32)
tensor = jnp.array(tensor)
if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: if tensor.dtype is torch.float16 or tensor.dtype is torch.float32:
tensor = tensor.bfloat16() tensor = tensor.bfloat16()
network.state["params"][spec["module"]][spec["param"]] = move_xmap( network.state["params"][spec["module"]][spec["param"]] = move_xmap(

2
update-koboldai.bat
View File

@@ -2,6 +2,8 @@
cd /d %~dp0 cd /d %~dp0
:Isolation :Isolation
call conda deactivate 2>NUL
set Path=%windir%\system32;%windir%;C:\Windows\System32\Wbem;%windir%\System32\WindowsPowerShell\v1.0\;%windir%\System32\OpenSSH\
SET CONDA_SHLVL= SET CONDA_SHLVL=
SET PYTHONNOUSERSITE=1 SET PYTHONNOUSERSITE=1
SET PYTHONPATH= SET PYTHONPATH=