From 88c536b194c685cf8d4bb802e0fcb214342c816f Mon Sep 17 00:00:00 2001 From: ebolam Date: Thu, 1 Dec 2022 08:46:35 -0500 Subject: [PATCH] Disable bitsandbytes if there are no GPUs with proper CUDA Compute capabilities (needs 7.2 or greater) --- koboldai_settings.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index a914af33..061de2a0 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1173,7 +1173,17 @@ class system_settings(settings): self.cookies = {} #cookies for colab since colab's URL changes, cookies are lost self.experimental_features = False #check if bitsandbytes is installed - self.bit_8_available = importlib.util.find_spec("bitsandbytes") is not None and sys.platform.startswith('linux') #We can install bitsandbytes, but it doesn't work on windows, so limit it here + self.bit_8_available = False + if importlib.util.find_spec("bitsandbytes") is not None and sys.platform.startswith('linux'): #We can install bitsandbytes, but it doesn't work on windows, so limit it here + if torch.cuda.is_available(): + for device in range(torch.cuda.device_count()): + if torch.cuda.get_device_properties(device).major > 7: + self.bit_8_available = True + break + elif torch.cuda.get_device_properties(device).major = 7 and torch.cuda.get_device_properties(device).minor >= 2: + self.bit_8_available = True + break + @dataclass class _inference_config: