rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Don't patch lazyload on TPU

Browse Source
This commit is contained in:
somebody
2023-07-03 16:52:18 -05:00
parent 39049e8a46
commit 686c3d1592
2 changed files with 6 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
aiserver.py
View File

@@ -27,9 +27,6 @@ from ansi2html import Ansi2HTMLConverter
logging.getLogger("urllib3").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR)
from modeling import patches
patches.patch_transformers_for_lazyload()
import attention_bias import attention_bias
attention_bias.do_patches() attention_bias.do_patches()
@@ -10809,7 +10806,7 @@ def run():
Session(app) Session(app)
logger.init_ok("Flask", status="OK") logger.init_ok("Flask", status="OK")
logger.init("Webserver", status="Starting") logger.init("Webserver", status="Starting")
patch_transformers() patch_transformers(use_tpu=koboldai_vars.use_colab_tpu)
# Start Flask/SocketIO (Blocking, so this must be last method!) # Start Flask/SocketIO (Blocking, so this must be last method!)
port = args.port if "port" in args and args.port is not None else 5000 port = args.port if "port" in args and args.port is not None else 5000
@@ -10906,7 +10903,7 @@ else:
logger.init("Flask", status="Starting") logger.init("Flask", status="Starting")
Session(app) Session(app)
logger.init_ok("Flask", status="OK") logger.init_ok("Flask", status="OK")
patch_transformers() patch_transformers(use_tpu=koboldai_vars.use_colab_tpu)
startup(command_line_backend) startup(command_line_backend)
koboldai_settings.port = args.port if "port" in args and args.port is not None else 5000 koboldai_settings.port = args.port if "port" in args and args.port is not None else 5000
print("{0}\nServer started in WSGI mode!{1}".format(colors.GREEN, colors.END), flush=True) print("{0}\nServer started in WSGI mode!{1}".format(colors.GREEN, colors.END), flush=True)

8
modeling/patches.py
View File

@@ -164,7 +164,6 @@ def patch_transformers_for_lazyload() -> None:
# both for short term compatibility # both for short term compatibility
load_in_8bit=False, load_in_8bit=False,
is_quantized=False, is_quantized=False,
is_safetensors=False, is_safetensors=False,
keep_in_fp32_modules=None, keep_in_fp32_modules=None,
): ):
@@ -303,9 +302,10 @@ def patch_transformers_for_lazyload() -> None:
) )
def patch_transformers() -> None: def patch_transformers(use_tpu: bool) -> None:
patch_transformers_download() patch_transformers_download()
patch_transformers_loader() patch_transformers_loader()
# Doesn't do anything for TPU if not use_tpu:
patch_transformers_generation() patch_transformers_generation()
patch_transformers_for_lazyload()