From b628aec7194783da09035a3b8fe01f674df542ea Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Mon, 10 Apr 2023 22:37:16 +0200 Subject: [PATCH] Automatic installation of the quant_cuda module during install_requirements Kepler (K40+) and Maxwell support --- install_requirements.bat | 4 ++++ install_requirements.sh | 3 +++ repos/gptq | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/install_requirements.bat b/install_requirements.bat index 2a4534c1..05264259 100644 --- a/install_requirements.bat +++ b/install_requirements.bat @@ -49,6 +49,8 @@ umamba.exe install --no-shortcuts -r B:\python\ -n base -f "%~dp0\environments\h umamba.exe -r B:\ clean -a -y rd B:\Python\pkgs /S /Q subst B: /d +call B:\python\condabin\activate +cmd /k "pip install https://github.com/0cc4m/GPTQ-for-LLaMa/releases/download/2023-04-10/quant_cuda-0.0.0-cp38-cp38-win_amd64.whl" pause exit @@ -60,5 +62,7 @@ umamba.exe create -r miniconda3\ -n base umamba.exe install --no-shortcuts -r miniconda3 -n base -f environments\huggingface.yml -y --always-copy umamba.exe clean -a -y rd miniconda3\Python\pkgs /S /Q +call miniconda3\condabin\activate +cmd /k "pip install https://github.com/0cc4m/GPTQ-for-LLaMa/releases/download/2023-04-10/quant_cuda-0.0.0-cp38-cp38-win_amd64.whl" pause exit diff --git a/install_requirements.sh b/install_requirements.sh index 6f0e0dfd..7b5a8d5b 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -5,6 +5,9 @@ wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar - bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y # Weird micromamba bug causes it to fail the first time, running it twice just to be safe, the second time is much faster bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y + +# Install quant_cuda module for 4-bit +bin/micromamba run -r runtime -n koboldai pip install https://github.com/0cc4m/GPTQ-for-LLaMa/releases/download/2023-04-10/quant_cuda-0.0.0-cp38-cp38-linux_x86_64.whl exit fi if [[ $1 = "rocm" ]]; then diff --git a/repos/gptq b/repos/gptq index 17c46a59..50b22e2b 160000 --- a/repos/gptq +++ b/repos/gptq @@ -1 +1 @@ -Subproject commit 17c46a59ff20da657e68f3267f853b0243b983bc +Subproject commit 50b22e2ba8ec0f5cf0dca719392a2ec5254e7228