From 0e7a8dd051047d71890332edb51d30c91a6cda97 Mon Sep 17 00:00:00 2001 From: ebolam Date: Wed, 17 Aug 2022 15:20:11 -0400 Subject: [PATCH 1/4] Colab Debug --- aiserver.py | 13 ++++++++----- koboldai_settings.py | 10 ++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/aiserver.py b/aiserver.py index ec628535..9f272e1f 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1157,6 +1157,7 @@ def general_startup(override_args=None): parser.add_argument("--savemodel", action='store_true', help="Saves the model to the models folder even if --colab is used (Allows you to save models to Google Drive)") parser.add_argument("--customsettings", help="Preloads arguements from json file. You only need to provide the location of the json file. Use customsettings.json template file. It can be renamed if you wish so that you can store multiple configurations. Leave any settings you want as default as null. Any values you wish to set need to be in double quotation marks") parser.add_argument("--no_ui", action='store_true', default=False, help="Disables the GUI and Socket.IO server while leaving the API server running.") + parser.add_argument("--use_relay", action='store_true', default=False, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") #args: argparse.Namespace = None if "pytest" in sys.modules and override_args is None: args = parser.parse_args([]) @@ -1253,10 +1254,13 @@ def general_startup(override_args=None): koboldai_vars.colaburl = args.path + "/request"; # Lets just use the same parameter to keep it simple #setup socketio relay queue - koboldai_settings.queue = multiprocessing.Queue() - #t = threading.Thread(target=socket_io_relay, args=(koboldai_settings.queue, socketio)) - socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) - print("continued") + if args.use_relay: + koboldai_settings.no_relay = False + koboldai_settings.queue = multiprocessing.Queue() + socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) + else: + koboldai_settings.no_relay = True + #==================================================================# # Load Model #==================================================================# @@ -7255,7 +7259,6 @@ def socket_io_relay(queue, socketio): data = queue.get() socketio.emit(data[0], data[1], **data[2]) #socketio.emit(data[0], data[1], broadcast=True, room="UI_2") - print("sent") time.sleep(0.05) diff --git a/koboldai_settings.py b/koboldai_settings.py index 3f19181e..f8822c00 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -9,6 +9,7 @@ rely_clients = {} serverstarted = False port = 5000 queue = None +no_relay=False def clean_var_for_emit(value): if isinstance(value, KoboldStoryRegister) or isinstance(value, KoboldWorldInfo): @@ -46,10 +47,11 @@ def process_variable_changes(socketio, classname, name, value, old_value, debug_ else: #If we got a variable change from a thread other than what the app is run it, eventlet seems to block and no further messages are sent. Instead, we'll rely the message to the app and have the main thread send it if not has_request_context(): - data = ["var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, {"include_self":True, "broadcast":True, "room":"UI_2"}] - print("putting data in queue") - queue.put(data) - print("put data in queue") + if not no_relay: + data = ["var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, {"include_self":True, "broadcast":True, "room":"UI_2"}] + print("putting data in queue") + queue.put(data) + print("put data in queue") else: socketio.emit("var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, include_self=True, broadcast=True, room="UI_2") From a9650034d79e5e33989bdc4a4a2fb3134d4d487a Mon Sep 17 00:00:00 2001 From: ebolam Date: Wed, 17 Aug 2022 16:13:23 -0400 Subject: [PATCH 2/4] Debug --- aiserver.py | 6 ++++-- koboldai_settings.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 9f272e1f..10ba0869 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1157,7 +1157,7 @@ def general_startup(override_args=None): parser.add_argument("--savemodel", action='store_true', help="Saves the model to the models folder even if --colab is used (Allows you to save models to Google Drive)") parser.add_argument("--customsettings", help="Preloads arguements from json file. You only need to provide the location of the json file. Use customsettings.json template file. It can be renamed if you wish so that you can store multiple configurations. Leave any settings you want as default as null. Any values you wish to set need to be in double quotation marks") parser.add_argument("--no_ui", action='store_true', default=False, help="Disables the GUI and Socket.IO server while leaving the API server running.") - parser.add_argument("--use_relay", action='store_true', default=False, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") + parser.add_argument("--use_relay", action='store_false', default=True, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") #args: argparse.Namespace = None if "pytest" in sys.modules and override_args is None: args = parser.parse_args([]) @@ -1256,7 +1256,9 @@ def general_startup(override_args=None): #setup socketio relay queue if args.use_relay: koboldai_settings.no_relay = False - koboldai_settings.queue = multiprocessing.Queue() + #koboldai_settings.queue = multiprocessing.Queue() + import gevent.queue + koboldai_settings.queue = gevent.queue.Queue() socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) else: koboldai_settings.no_relay = True diff --git a/koboldai_settings.py b/koboldai_settings.py index f8822c00..3e3bcd77 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2,6 +2,7 @@ import os, re, time, threading, json, pickle, base64, copy, tqdm, datetime from io import BytesIO from flask import has_request_context import socketio as socketio_client +from flask_socketio import SocketIO from collections import OrderedDict import requests @@ -52,6 +53,7 @@ def process_variable_changes(socketio, classname, name, value, old_value, debug_ print("putting data in queue") queue.put(data) print("put data in queue") + else: socketio.emit("var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, include_self=True, broadcast=True, room="UI_2") From edf433e9494193022d93016b32b857e9b77c6359 Mon Sep 17 00:00:00 2001 From: ebolam Date: Wed, 17 Aug 2022 16:13:23 -0400 Subject: [PATCH 3/4] Debug --- aiserver.py | 6 ++++-- koboldai_settings.py | 2 ++ requirements.txt | 1 + requirements_mtj.txt | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 9f272e1f..10ba0869 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1157,7 +1157,7 @@ def general_startup(override_args=None): parser.add_argument("--savemodel", action='store_true', help="Saves the model to the models folder even if --colab is used (Allows you to save models to Google Drive)") parser.add_argument("--customsettings", help="Preloads arguements from json file. You only need to provide the location of the json file. Use customsettings.json template file. It can be renamed if you wish so that you can store multiple configurations. Leave any settings you want as default as null. Any values you wish to set need to be in double quotation marks") parser.add_argument("--no_ui", action='store_true', default=False, help="Disables the GUI and Socket.IO server while leaving the API server running.") - parser.add_argument("--use_relay", action='store_true', default=False, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") + parser.add_argument("--use_relay", action='store_false', default=True, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") #args: argparse.Namespace = None if "pytest" in sys.modules and override_args is None: args = parser.parse_args([]) @@ -1256,7 +1256,9 @@ def general_startup(override_args=None): #setup socketio relay queue if args.use_relay: koboldai_settings.no_relay = False - koboldai_settings.queue = multiprocessing.Queue() + #koboldai_settings.queue = multiprocessing.Queue() + import gevent.queue + koboldai_settings.queue = gevent.queue.Queue() socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) else: koboldai_settings.no_relay = True diff --git a/koboldai_settings.py b/koboldai_settings.py index f8822c00..3e3bcd77 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -2,6 +2,7 @@ import os, re, time, threading, json, pickle, base64, copy, tqdm, datetime from io import BytesIO from flask import has_request_context import socketio as socketio_client +from flask_socketio import SocketIO from collections import OrderedDict import requests @@ -52,6 +53,7 @@ def process_variable_changes(socketio, classname, name, value, old_value, debug_ print("putting data in queue") queue.put(data) print("put data in queue") + else: socketio.emit("var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, include_self=True, broadcast=True, room="UI_2") diff --git a/requirements.txt b/requirements.txt index f087a16b..0349bdbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ python-socketio[client] flask_session marshmallow>=3.13 apispec-webframeworks +gevent \ No newline at end of file diff --git a/requirements_mtj.txt b/requirements_mtj.txt index e681e2e3..a3683e47 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -21,3 +21,4 @@ python-socketio[client] flask-session marshmallow>=3.13 apispec-webframeworks +gevent \ No newline at end of file From 63cd4560e03c16bd897dbd134fd935a7cac0ac97 Mon Sep 17 00:00:00 2001 From: ebolam Date: Wed, 17 Aug 2022 18:10:31 -0400 Subject: [PATCH 4/4] Colab Debug --- aiserver.py | 22 ++++++++-------------- koboldai_settings.py | 10 ++++------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/aiserver.py b/aiserver.py index 10ba0869..b2df8f36 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1157,7 +1157,6 @@ def general_startup(override_args=None): parser.add_argument("--savemodel", action='store_true', help="Saves the model to the models folder even if --colab is used (Allows you to save models to Google Drive)") parser.add_argument("--customsettings", help="Preloads arguements from json file. You only need to provide the location of the json file. Use customsettings.json template file. It can be renamed if you wish so that you can store multiple configurations. Leave any settings you want as default as null. Any values you wish to set need to be in double quotation marks") parser.add_argument("--no_ui", action='store_true', default=False, help="Disables the GUI and Socket.IO server while leaving the API server running.") - parser.add_argument("--use_relay", action='store_false', default=True, help="Use messaging relay when the thread is busy (can loose data to UI if not used)") #args: argparse.Namespace = None if "pytest" in sys.modules and override_args is None: args = parser.parse_args([]) @@ -1254,14 +1253,8 @@ def general_startup(override_args=None): koboldai_vars.colaburl = args.path + "/request"; # Lets just use the same parameter to keep it simple #setup socketio relay queue - if args.use_relay: - koboldai_settings.no_relay = False - #koboldai_settings.queue = multiprocessing.Queue() - import gevent.queue - koboldai_settings.queue = gevent.queue.Queue() - socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) - else: - koboldai_settings.no_relay = True + koboldai_settings.queue = multiprocessing.Queue() + socketio.start_background_task(socket_io_relay, koboldai_settings.queue, socketio) #==================================================================# # Load Model @@ -1736,7 +1729,7 @@ def patch_transformers(): "score": float(score), }) - koboldai_vars.token_stream_queue.probability_buffer = token_prob_info + #koboldai_vars.token_stream_queue.probability_buffer = token_prob_info return scores def new_get_logits_processor(*args, **kwargs) -> LogitsProcessorList: @@ -7258,10 +7251,11 @@ def socket_io_relay(queue, socketio): while True: if not queue.empty(): print("got relay message") - data = queue.get() - socketio.emit(data[0], data[1], **data[2]) - #socketio.emit(data[0], data[1], broadcast=True, room="UI_2") - time.sleep(0.05) + while not queue.empty(): + data = queue.get() + socketio.emit(data[0], data[1], **data[2]) + #socketio.emit(data[0], data[1], broadcast=True, room="UI_2") + time.sleep(0.2) #==================================================================# diff --git a/koboldai_settings.py b/koboldai_settings.py index 3e3bcd77..af690a3e 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -10,7 +10,6 @@ rely_clients = {} serverstarted = False port = 5000 queue = None -no_relay=False def clean_var_for_emit(value): if isinstance(value, KoboldStoryRegister) or isinstance(value, KoboldWorldInfo): @@ -48,11 +47,10 @@ def process_variable_changes(socketio, classname, name, value, old_value, debug_ else: #If we got a variable change from a thread other than what the app is run it, eventlet seems to block and no further messages are sent. Instead, we'll rely the message to the app and have the main thread send it if not has_request_context(): - if not no_relay: - data = ["var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, {"include_self":True, "broadcast":True, "room":"UI_2"}] - print("putting data in queue") - queue.put(data) - print("put data in queue") + data = ["var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, {"include_self":True, "broadcast":True, "room":"UI_2"}] + print("putting data in queue") + queue.put(data) + print("put data in queue") else: socketio.emit("var_changed", {"classname": classname, "name": name, "old_value": clean_var_for_emit(old_value), "value": clean_var_for_emit(value)}, include_self=True, broadcast=True, room="UI_2")