From aabc73b3f74467aaf75a71afc7686584a87989cc Mon Sep 17 00:00:00 2001 From: ebolam Date: Tue, 29 Nov 2022 19:29:06 -0500 Subject: [PATCH] Adding debug for 8 bit --- aiserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiserver.py b/aiserver.py index 4a302a13..d99dc16a 100644 --- a/aiserver.py +++ b/aiserver.py @@ -2948,6 +2948,7 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal except Exception as e: tokenizer = GPT2Tokenizer.from_pretrained("gpt2", revision=koboldai_vars.revision, cache_dir="cache") try: + logger.info("Using 8 bit: {}".format(use_8_bit)) model = AutoModelForCausalLM.from_pretrained(koboldai_vars.model, revision=koboldai_vars.revision, cache_dir="cache", load_in_8bit=use_8_bit, device_map="auto", **lowmem) except Exception as e: if("out of memory" in traceback.format_exc().lower()):