From 057ddb4fb2a6989cbca9bd1bda1dc7c32d209bec Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 28 Sep 2022 00:45:17 +0200 Subject: [PATCH 1/2] Better --cpu handling --- aiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 1ff967b9..56e304b0 100644 --- a/aiserver.py +++ b/aiserver.py @@ -2228,8 +2228,8 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal loadmodelsettings() loadsettings() logger.init("GPU support", status="Searching") - vars.hascuda = torch.cuda.is_available() - vars.bmsupported = (utils.HAS_ACCELERATE or vars.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not vars.nobreakmodel + vars.hascuda = torch.cuda.is_available() and not args.cpu + vars.bmsupported = (utils.HAS_ACCELERATE or vars.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not vars.nobreakmodel and not args.cpu if(args.breakmodel is not None and args.breakmodel): logger.warning("--breakmodel is no longer supported. Breakmodel mode is now automatically enabled when --breakmodel_gpulayers is used (see --help for details).") if(args.breakmodel_layers is not None): From 52e120c70640d99c21318a73f528a6e061e62a6e Mon Sep 17 00:00:00 2001 From: Henk Date: Wed, 28 Sep 2022 01:00:06 +0200 Subject: [PATCH 2/2] Disable breakmodel if we error on the check --- aiserver.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aiserver.py b/aiserver.py index e2976a8f..f0ba3b64 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1566,10 +1566,12 @@ def get_layer_count(model, directory=""): model_config = AutoConfig.from_pretrained(directory, revision=vars.revision, cache_dir="cache") else: model_config = AutoConfig.from_pretrained(model, revision=vars.revision, cache_dir="cache") - - if ((utils.HAS_ACCELERATE and model_config.model_type != 'gpt2') or model_config.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not vars.nobreakmodel: - return utils.num_layers(model_config) - else: + try: + if ((utils.HAS_ACCELERATE and model_config.model_type != 'gpt2') or model_config.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not vars.nobreakmodel: + return utils.num_layers(model_config) + else: + return None + except: return None else: return None