From b141ee015566ad1206e93d995acba9dbc213ad64 Mon Sep 17 00:00:00 2001 From: Henk Date: Thu, 28 Sep 2023 18:01:34 +0200 Subject: [PATCH] Universal downloader for more backends --- modeling/inference_models/basic_hf/class.py | 7 +++++++ modeling/inference_models/exllama/class.py | 6 ++++++ modeling/inference_models/exllamav2/class.py | 7 ++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index afca13ee..5ae2aa0d 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -148,6 +148,13 @@ class model_backend(InferenceModel): self.get_local_model_path(ignore_existance=True), ) + if not self.get_local_model_path(): + print(self.get_local_model_path()) + from huggingface_hub import snapshot_download + target_dir = "models/" + self.model_name.replace("/", "_") + print(self.model_name) + snapshot_download(self.model_name, local_dir=target_dir, local_dir_use_symlinks=False, cache_dir="cache/", revision=utils.koboldai_vars.revision) + self.init_model_config() self.model = AutoModelForCausalLM.from_pretrained( diff --git a/modeling/inference_models/exllama/class.py b/modeling/inference_models/exllama/class.py index 4539b7a3..f688d611 100644 --- a/modeling/inference_models/exllama/class.py +++ b/modeling/inference_models/exllama/class.py @@ -128,6 +128,12 @@ class model_backend(InferenceModel): return config def _load(self, save_model: bool, initial_load: bool) -> None: + if not self.get_local_model_path(): + from huggingface_hub import snapshot_download + target_dir = "models/" + self.model_name.replace("/", "_") + print(self.model_name) + snapshot_download(self.model_name, local_dir=target_dir, local_dir_use_symlinks=False, cache_dir="cache/", revision=utils.koboldai_vars.revision) + self.model = self._get_model(self.get_local_model_path(), {}) self.tokenizer = self._get_tokenizer(self.get_local_model_path()) diff --git a/modeling/inference_models/exllamav2/class.py b/modeling/inference_models/exllamav2/class.py index 95795506..15b91c8d 100644 --- a/modeling/inference_models/exllamav2/class.py +++ b/modeling/inference_models/exllamav2/class.py @@ -36,7 +36,7 @@ try: except: load_failed = True -model_backend_type = "Exl2" +model_backend_type = "GPTQ" model_backend_name = "ExLlama V2" # When set to true, messages will appear in the console if samplers are not @@ -99,6 +99,11 @@ class model_backend(InferenceModel): return config def _load(self, save_model: bool, initial_load: bool) -> None: + if not self.get_local_model_path(): + from huggingface_hub import snapshot_download + target_dir = "models/" + self.model_name.replace("/", "_") + print(self.model_name) + snapshot_download(self.model_name, local_dir=target_dir, local_dir_use_symlinks=False, cache_dir="cache/", revision=utils.koboldai_vars.revision) self.model = self._get_model(self.get_local_model_path(), {}) #TODO support GPU split self.model.load(None)