From 2de9672b954c6d34f83c3fdc6f00476a222a6f31 Mon Sep 17 00:00:00 2001 From: Divided by Zer0 Date: Thu, 23 Feb 2023 18:27:11 +0100 Subject: [PATCH 1/5] attempt1 --- aiserver.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/aiserver.py b/aiserver.py index 665b43f6..92769066 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1671,7 +1671,7 @@ def get_cluster_models(msg): # Get list of models from public cluster logger.init("KAI Horde Models", status="Retrieving") try: - req = requests.get("{}/api/v1/models".format(url)) + req = requests.get(f"{url}/api/v2/status/models?type=text") except requests.exceptions.ConnectionError: logger.init_err("KAI Horde Models", status="Failed") logger.error("Provided KoboldAI Horde URL unreachable") @@ -1687,10 +1687,11 @@ def get_cluster_models(msg): engines = req.json() logger.debug(engines) try: - engines = [[en, en] for en in engines] + engines = [[en["name"], en["name"]] for en in engines] except: logger.error(engines) raise + logger.debug(engines) online_model = "" changed=False @@ -5269,15 +5270,21 @@ def sendtocluster(txt, min, max): cluster_metadata = { 'prompt': txt, 'params': reqdata, - 'api_key': vars.apikey, 'models': vars.cluster_requested_models, - } + 'trusted_workers': False, + } + client_agent = "KoboldAI:1.19.3:(discord)Henky!!#2205" + cluster_headers = { + 'apikey': koboldai_vars.apikey, + "Client-Agent": client_agent + } logger.debug(f"Horde Payload: {cluster_metadata}") try: # Create request req = requests.post( - vars.colaburl[:-8] + "/api/v1/generate/sync", + vars.colaburl[:-8] + "/api/v2/generate/text/async", json=cluster_metadata, + headers=cluster_headers, ) except requests.exceptions.ConnectionError: errmsg = f"Horde unavailable. Please try again later" @@ -5305,13 +5312,76 @@ def sendtocluster(txt, min, max): emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) set_aibusy(0) return - gen_servers = [(cgen['server_name'],cgen['server_id']) for cgen in js] - logger.info(f"Generations by: {gen_servers}") + + request_id = js["id"] + logger.debug("Horde Request ID: {}".format(request_id)) + + cluster_agent_headers = { + "Client-Agent": client_agent + } + finished = False + + while not finished: + try: + req = requests.get(koboldai_vars.colaburl[:-8] + "/api/v2/generate/text/status/" + request_id, headers=cluster_agent_headers) + except requests.exceptions.ConnectionError: + errmsg = f"Horde unavailable. Please try again later" + logger.error(errmsg) + emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) + set_aibusy(0) + return + + if not req.ok: + errmsg = f"KoboldAI API Error: Failed to get a standard reply from the Horde. Please check the console." + logger.error(req.text) + emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) + set_aibusy(0) + return + + try: + req_status = req.json() + except requests.exceptions.JSONDecodeError: + errmsg = f"Unexpected message received from the KoboldAI Horde: '{req.text}'" + logger.error(errmsg) + emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) + set_aibusy(0) + return + + if "done" not in req_status: + errmsg = f"Unexpected response received from the KoboldAI Horde: '{js}'" + logger.error(errmsg) + emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) + set_aibusy(0) + return + + finished = req_status["done"] + + if not finished: + logger.debug(req_status) + time.sleep(1) + + logger.debug("Last Horde Status Message: {}".format(js)) + if req_status["faulted"]: + errmsg = "Horde Text generation faulted! Please try again" + logger.error(errmsg) + emit('from_server', {'cmd': 'errmsg', 'data': errmsg}, broadcast=True) + set_aibusy(0) + return + + generations = req_status['generations'] + gen_workers = [(cgen['worker_name'],cgen['worker_id']) for cgen in generations] + logger.info(f"Generations by: {gen_workers}") + + + + + + # Just in case we want to announce it to the user - if len(js) == 1: - warnmsg = f"Text generated by {js[0]['server_name']}" + if len(generations) == 1: + warnmsg = f"Text generated by {[w[0] for w in gen_workers]}" emit('from_server', {'cmd': 'warnmsg', 'data': warnmsg}, broadcast=True) - genout = [cgen['text'] for cgen in js] + genout = [cgen['text'] for cgen in generations] for i in range(vars.numseqs): vars.lua_koboldbridge.outputs[i+1] = genout[i] From 6426e3ca24cdc0d133050a8204fd4741a92febfd Mon Sep 17 00:00:00 2001 From: Divided by Zer0 Date: Thu, 23 Feb 2023 18:34:46 +0100 Subject: [PATCH 2/5] changes --- aiserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiserver.py b/aiserver.py index 92769066..55a550a7 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1511,7 +1511,7 @@ def get_model_info(model, directory=""): models_on_url = True url = True key = True - default_url = 'https://koboldai.net' + default_url = 'https://stablehorde.net' multi_online_models = True if path.exists(get_config_filename(model)): with open(get_config_filename(model), "r") as file: @@ -5275,7 +5275,7 @@ def sendtocluster(txt, min, max): } client_agent = "KoboldAI:1.19.3:(discord)Henky!!#2205" cluster_headers = { - 'apikey': koboldai_vars.apikey, + 'apikey': vars.apikey, "Client-Agent": client_agent } logger.debug(f"Horde Payload: {cluster_metadata}") @@ -5323,7 +5323,7 @@ def sendtocluster(txt, min, max): while not finished: try: - req = requests.get(koboldai_vars.colaburl[:-8] + "/api/v2/generate/text/status/" + request_id, headers=cluster_agent_headers) + req = requests.get(vars.colaburl[:-8] + "/api/v2/generate/text/status/" + request_id, headers=cluster_agent_headers) except requests.exceptions.ConnectionError: errmsg = f"Horde unavailable. Please try again later" logger.error(errmsg) From 93bf023bd7a0c68e4278acc012abb1122f3c9a25 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 1 Mar 2023 17:54:39 +0100 Subject: [PATCH 3/5] Use our own horde URL --- aiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 55a550a7..913a35c9 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1511,7 +1511,7 @@ def get_model_info(model, directory=""): models_on_url = True url = True key = True - default_url = 'https://stablehorde.net' + default_url = 'https://horde.koboldai.net' multi_online_models = True if path.exists(get_config_filename(model)): with open(get_config_filename(model), "r") as file: @@ -5273,7 +5273,7 @@ def sendtocluster(txt, min, max): 'models': vars.cluster_requested_models, 'trusted_workers': False, } - client_agent = "KoboldAI:1.19.3:(discord)Henky!!#2205" + client_agent = "KoboldAI:1.19.3:koboldai.org" cluster_headers = { 'apikey': vars.apikey, "Client-Agent": client_agent From 142cb354f9224e9521248e2c107ae9b433ad53cc Mon Sep 17 00:00:00 2001 From: henk717 Date: Wed, 1 Mar 2023 22:33:11 +0100 Subject: [PATCH 4/5] Nerybus 13B - TPU colab --- colab/TPU.ipynb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/colab/TPU.ipynb b/colab/TPU.ipynb index 4a6efcd9..3976f6dd 100644 --- a/colab/TPU.ipynb +++ b/colab/TPU.ipynb @@ -66,7 +66,7 @@ "#@title <-- Select your model below and then click this to start KoboldAI\n", "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n", "\n", - "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n", + "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n", "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n", "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n", "use_google_drive = True #@param {type:\"boolean\"}\n", @@ -97,6 +97,10 @@ " Model = \"KoboldAI/OPT-13B-Nerys-v2\"\n", " path = \"\"\n", " download = \"\"\n", + "elif Model == \"Nerybus 13B\":\n", + " Model = \"KoboldAI/OPT-13B-Nerybus-Mix\"\n", + " path = \"\"\n", + " download = \"\"\n", "elif Model == \"Erebus 13B\":\n", " Model = \"KoboldAI/OPT-13B-Erebus\"\n", " path = \"\"\n", From 2ed6cdb411a7201024bfdc786e646e53ca8c07e3 Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 8 Mar 2023 18:03:36 +0100 Subject: [PATCH 5/5] Huggingface Hub Pin --- environments/huggingface.yml | 2 +- environments/rocm.yml | 2 +- requirements.txt | 2 +- requirements_mtj.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environments/huggingface.yml b/environments/huggingface.yml index 8b36a529..5942d79a 100644 --- a/environments/huggingface.yml +++ b/environments/huggingface.yml @@ -28,7 +28,7 @@ dependencies: - flask-ngrok - lupa==1.10 - transformers==4.24.0 - - huggingface_hub>=0.10.1 + - huggingface_hub==0.12.1 - safetensors - accelerate - git+https://github.com/VE-FORBRYDERNE/mkultra diff --git a/environments/rocm.yml b/environments/rocm.yml index 6605709f..c9c6b487 100644 --- a/environments/rocm.yml +++ b/environments/rocm.yml @@ -27,7 +27,7 @@ dependencies: - flask-ngrok - lupa==1.10 - transformers==4.24.0 - - huggingface_hub>=0.10.1 + - huggingface_hub==0.12.1 - safetensors - accelerate - git+https://github.com/VE-FORBRYDERNE/mkultra diff --git a/requirements.txt b/requirements.txt index bed8308a..b79ca5cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ transformers==4.24.0 -huggingface_hub>=0.10.1 +huggingface_hub==0.12.1 Flask Flask-SocketIO requests diff --git a/requirements_mtj.txt b/requirements_mtj.txt index c03a4c5d..c99852e7 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -6,7 +6,7 @@ dm-haiku == 0.0.5 jax == 0.2.21 jaxlib >= 0.1.69, <= 0.3.7 transformers == 4.24.0 -huggingface_hub >= 0.10.1 +huggingface_hub==0.12.1 progressbar2 git+https://github.com/VE-FORBRYDERNE/mesh-transformer-jax@ck flask