From 20b4b4bcef597e9c72ca330049d9c9bad1bd0636 Mon Sep 17 00:00:00 2001 From: somebody Date: Sat, 8 Jul 2023 17:12:16 -0500 Subject: [PATCH 1/8] Add basic hf backend --- modeling/inference_models/basic_hf/class.py | 105 ++++++++++++++++++++ modeling/inference_models/hf.py | 1 + 2 files changed, 106 insertions(+) create mode 100644 modeling/inference_models/basic_hf/class.py diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py new file mode 100644 index 00000000..2914c9bc --- /dev/null +++ b/modeling/inference_models/basic_hf/class.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import os +import shutil +import time +from typing import List, Optional, Union + +import torch +from transformers.models.auto.modeling_auto import AutoModelForCausalLM + +import utils +from logger import logger +from modeling.inference_model import GenerationResult, GenerationSettings +from modeling.inference_models.hf import HFInferenceModel + +model_backend_name = "Basic Huggingface" +model_backend_type = "Huggingface" + +class model_backend(HFInferenceModel): + def __init__(self) -> None: + super().__init__() + self.model_name = "Basic Huggingface" + + # TODO: These feel weird to be in HFInferenceModel, maybe we could implement + # them in subclasses? + self.hf_torch = True + self.nobreakmodel = True + + def _load(self, save_model: bool, initial_load: bool) -> None: + utils.koboldai_vars.allowsp = False + + if self.model_name == "NeoCustom": + self.model_name = os.path.basename(os.path.normpath(self.path)) + utils.koboldai_vars.model = self.model_name + + # If we specify a model and it's in the root directory, we need to move + # it to the models directory (legacy folder structure to new) + if self.get_local_model_path(legacy=True): + shutil.move( + self.get_local_model_path(legacy=True, ignore_existance=True), + self.get_local_model_path(ignore_existance=True), + ) + + self.init_model_config() + + self.model = AutoModelForCausalLM.from_pretrained(self.get_local_model_path(), low_cpu_mem_usage=True) + + if self.usegpu: + self.model = self.model.to("cuda") + + self.tokenizer = self._get_tokenizer(self.get_local_model_path()) + + self.model.kai_model = self + utils.koboldai_vars.modeldim = self.model.get_input_embeddings().embedding_dim + + + def _raw_generate( + self, + prompt_tokens: Union[List[int], torch.Tensor], + max_new: int, + gen_settings: GenerationSettings, + single_line: bool = False, + batch_count: int = 1, + seed: Optional[int] = None, + **kwargs, + ) -> GenerationResult: + if not isinstance(prompt_tokens, torch.Tensor): + gen_in = torch.tensor(prompt_tokens, dtype=torch.long)[None] + else: + gen_in = prompt_tokens + if not self.usegpu: + gen_in = gen_in.to("cpu") + else: + device = self.get_auxilary_device() + gen_in = gen_in.to(device) + + additional_bad_words_ids = [self.tokenizer.encode("\n")] if single_line else [] + + if seed is not None: + torch.manual_seed(seed) + + with torch.no_grad(): + start_time = time.time() + genout = self.model.generate( + gen_in, + do_sample=True, + max_length=min( + len(prompt_tokens) + max_new, utils.koboldai_vars.max_length + ), + repetition_penalty=1.0, + bad_words_ids=self.badwordsids + additional_bad_words_ids, + use_cache=True, + num_return_sequences=batch_count, + ) + logger.debug( + "torch_raw_generate: run generator {}s".format(time.time() - start_time) + ) + + return GenerationResult( + self, + out_batches=genout, + prompt=prompt_tokens, + is_whole_generation=False, + output_includes_prompt=True, + ) \ No newline at end of file diff --git a/modeling/inference_models/hf.py b/modeling/inference_models/hf.py index 881ca604..e407f5b4 100644 --- a/modeling/inference_models/hf.py +++ b/modeling/inference_models/hf.py @@ -17,6 +17,7 @@ class HFInferenceModel(InferenceModel): self.model_config = None #self.model_name = model_name + self.hf_torch = False self.model = None self.tokenizer = None self.badwordsids = koboldai_settings.badwordsids_default From 8077d6c3f97002d36b4272791f1ad28e57b28813 Mon Sep 17 00:00:00 2001 From: onesome Date: Wed, 12 Jul 2023 03:22:43 -0500 Subject: [PATCH 2/8] Self-contained sampler patch (Don't merge) Completely untested 3:00 AM code; beware! I will test and add more documentation tomorrow. --- modeling/inference_models/basic_hf/class.py | 77 +++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index 2914c9bc..5cb64c30 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -6,16 +6,24 @@ import time from typing import List, Optional, Union import torch +import transformers +from transformers import LogitsProcessorList from transformers.models.auto.modeling_auto import AutoModelForCausalLM import utils from logger import logger -from modeling.inference_model import GenerationResult, GenerationSettings +from modeling import warpers +from modeling.inference_model import ( + GenerationResult, + GenerationSettings, + use_core_manipulations, +) from modeling.inference_models.hf import HFInferenceModel model_backend_name = "Basic Huggingface" model_backend_type = "Huggingface" + class model_backend(HFInferenceModel): def __init__(self) -> None: super().__init__() @@ -25,7 +33,7 @@ class model_backend(HFInferenceModel): # them in subclasses? self.hf_torch = True self.nobreakmodel = True - + def _load(self, save_model: bool, initial_load: bool) -> None: utils.koboldai_vars.allowsp = False @@ -43,16 +51,77 @@ class model_backend(HFInferenceModel): self.init_model_config() - self.model = AutoModelForCausalLM.from_pretrained(self.get_local_model_path(), low_cpu_mem_usage=True) + self.model = AutoModelForCausalLM.from_pretrained( + self.get_local_model_path(), low_cpu_mem_usage=True + ) if self.usegpu: self.model = self.model.to("cuda") self.tokenizer = self._get_tokenizer(self.get_local_model_path()) + # Patch sampler to use KAI samplers + def _patched_get_logits_processor(*args, **kwargs) -> LogitsProcessorList: + processors = _patched_get_logits_processor.original(*args, **kwargs) + return processors + + use_core_manipulations.get_logits_processor = _patched_get_logits_processor + _patched_get_logits_processor.original = ( + transformers.GenerationMixin._get_logits_processor + ) + + class KoboldLogitsWarperList(LogitsProcessorList): + def __call__( + _self, # Unused + input_ids: torch.LongTensor, + scores: torch.FloatTensor, + *args, + **kwargs, + ): + scores = self._apply_warpers(scores=scores, input_ids=input_ids) + + for processor in self.logits_processors: + scores = processor(self, scores=scores, input_ids=input_ids) + assert ( + scores is not None + ), f"Scores are None; processor '{processor}' is to blame" + return scores + + def new_sample(self, *args, **kwargs): + assert kwargs.pop("logits_warper", None) is not None + kwargs["logits_warper"] = lambda: KoboldLogitsWarperList() + + if utils.koboldai_vars.newlinemode in ["s", "ns"]: + kwargs["eos_token_id"] = -1 + kwargs.setdefault("pad_token_id", 2) + + return new_sample.old_sample(self, *args, **kwargs) + + new_sample.old_sample = transformers.GenerationMixin.sample + use_core_manipulations.sample = new_sample + self.model.kai_model = self utils.koboldai_vars.modeldim = self.model.get_input_embeddings().embedding_dim + def _apply_warpers( + self, scores: torch.Tensor, input_ids: torch.Tensor + ) -> torch.Tensor: + warpers.update_settings() + + for sid in utils.koboldai_vars.sampler_order: + warper = warpers.Warper.from_id(sid) + + if not warper.value_is_valid(): + continue + + if warper == warpers.RepetitionPenalty: + # Rep pen needs more data than other samplers + scores = warper.torch(scores, input_ids=input_ids) + else: + scores = warper.torch(scores) + + assert scores is not None, f"Scores are None; warper '{warper}' is to blame" + return scores def _raw_generate( self, @@ -102,4 +171,4 @@ class model_backend(HFInferenceModel): prompt=prompt_tokens, is_whole_generation=False, output_includes_prompt=True, - ) \ No newline at end of file + ) From 60473d4c23c1cd94978ab17c006950747a2e428b Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 12 Jul 2023 17:16:05 -0500 Subject: [PATCH 3/8] Fix and add some documentation to basic hf backend --- modeling/inference_models/basic_hf/class.py | 35 ++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index 5cb64c30..ecbc55cc 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -25,6 +25,9 @@ model_backend_type = "Huggingface" class model_backend(HFInferenceModel): + # Model backends must inherit from InferenceModel. We inherit from HFInferenceModel here, + # as it provides some helpers for handling Huggingface configs. + def __init__(self) -> None: super().__init__() self.model_name = "Basic Huggingface" @@ -60,16 +63,10 @@ class model_backend(HFInferenceModel): self.tokenizer = self._get_tokenizer(self.get_local_model_path()) - # Patch sampler to use KAI samplers - def _patched_get_logits_processor(*args, **kwargs) -> LogitsProcessorList: - processors = _patched_get_logits_processor.original(*args, **kwargs) - return processors - - use_core_manipulations.get_logits_processor = _patched_get_logits_processor - _patched_get_logits_processor.original = ( - transformers.GenerationMixin._get_logits_processor - ) + self.model.kai_model = self + utils.koboldai_vars.modeldim = self.model.get_input_embeddings().embedding_dim + # Patch Huggingface stuff to use our samplers class KoboldLogitsWarperList(LogitsProcessorList): def __call__( _self, # Unused @@ -78,8 +75,10 @@ class model_backend(HFInferenceModel): *args, **kwargs, ): + # Kobold sampling is done here. scores = self._apply_warpers(scores=scores, input_ids=input_ids) + # Things like Lua integration, phrase bias, and probability visualization are done here. for processor in self.logits_processors: scores = processor(self, scores=scores, input_ids=input_ids) assert ( @@ -89,7 +88,7 @@ class model_backend(HFInferenceModel): def new_sample(self, *args, **kwargs): assert kwargs.pop("logits_warper", None) is not None - kwargs["logits_warper"] = lambda: KoboldLogitsWarperList() + kwargs["logits_warper"] = KoboldLogitsWarperList() if utils.koboldai_vars.newlinemode in ["s", "ns"]: kwargs["eos_token_id"] = -1 @@ -100,12 +99,18 @@ class model_backend(HFInferenceModel): new_sample.old_sample = transformers.GenerationMixin.sample use_core_manipulations.sample = new_sample - self.model.kai_model = self - utils.koboldai_vars.modeldim = self.model.get_input_embeddings().embedding_dim - def _apply_warpers( self, scores: torch.Tensor, input_ids: torch.Tensor ) -> torch.Tensor: + """Applies samplers/warpers to the given scores, returning the altered scores. + + Args: + scores (torch.Tensor): The original scores. + input_ids (torch.Tensor): The input token sequence. + + Returns: + torch.Tensor: The altered scores. + """ warpers.update_settings() for sid in utils.koboldai_vars.sampler_order: @@ -115,7 +120,7 @@ class model_backend(HFInferenceModel): continue if warper == warpers.RepetitionPenalty: - # Rep pen needs more data than other samplers + # Rep pen needs access to input tokens to decide what to penalize scores = warper.torch(scores, input_ids=input_ids) else: scores = warper.torch(scores) @@ -137,6 +142,7 @@ class model_backend(HFInferenceModel): gen_in = torch.tensor(prompt_tokens, dtype=torch.long)[None] else: gen_in = prompt_tokens + if not self.usegpu: gen_in = gen_in.to("cpu") else: @@ -161,6 +167,7 @@ class model_backend(HFInferenceModel): use_cache=True, num_return_sequences=batch_count, ) + logger.debug( "torch_raw_generate: run generator {}s".format(time.time() - start_time) ) From d17ce8461dccc7bc7914a0e882395fa3484040bc Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 12 Jul 2023 17:27:48 -0500 Subject: [PATCH 4/8] Use device_map="auto" --- modeling/inference_models/basic_hf/class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index ecbc55cc..9d4b643b 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -55,7 +55,7 @@ class model_backend(HFInferenceModel): self.init_model_config() self.model = AutoModelForCausalLM.from_pretrained( - self.get_local_model_path(), low_cpu_mem_usage=True + self.get_local_model_path(), low_cpu_mem_usage=True, device_map="auto" ) if self.usegpu: From f67cb7fa055fb9ddc9a6f99385ae380b2ac0c0a4 Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 12 Jul 2023 18:36:30 -0500 Subject: [PATCH 5/8] Make basic hf independant from hf --- modeling/inference_models/basic_hf/class.py | 177 +++++++++++++++++--- 1 file changed, 158 insertions(+), 19 deletions(-) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index 9d4b643b..74bfcd17 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -1,41 +1,119 @@ from __future__ import annotations +import gc import os import shutil import time +import warnings from typing import List, Optional, Union import torch import transformers -from transformers import LogitsProcessorList -from transformers.models.auto.modeling_auto import AutoModelForCausalLM +from transformers import AutoConfig, AutoModelForCausalLM, LogitsProcessorList import utils from logger import logger +import koboldai_settings from modeling import warpers from modeling.inference_model import ( GenerationResult, GenerationSettings, + InferenceModel, use_core_manipulations, ) -from modeling.inference_models.hf import HFInferenceModel model_backend_name = "Basic Huggingface" model_backend_type = "Huggingface" -class model_backend(HFInferenceModel): - # Model backends must inherit from InferenceModel. We inherit from HFInferenceModel here, - # as it provides some helpers for handling Huggingface configs. +class model_backend(InferenceModel): + # Model backends must inherit from InferenceModel. def __init__(self) -> None: super().__init__() self.model_name = "Basic Huggingface" + self.path = None - # TODO: These feel weird to be in HFInferenceModel, maybe we could implement - # them in subclasses? - self.hf_torch = True - self.nobreakmodel = True + def get_requested_parameters( + self, model_name: str, model_path: str, menu_path: str, parameters: dict = {} + ): + requested_parameters = [] + + if model_name == "customhuggingface": + requested_parameters.append( + { + "uitype": "text", + "unit": "text", + "label": "Huggingface Model Name", + "id": "custom_model_name", + "default": parameters.get("custom_model_name", ""), + "check": {"value": "", "check": "!="}, + "tooltip": "Model name from https://huggingface.co/", + "menu_path": "", + "refresh_model_inputs": True, + "extra_classes": "", + } + ) + + if model_name != "customhuggingface" or "custom_model_name" in parameters: + model_name = parameters.get("custom_model_name", None) or model_name + alt_model_path = self.get_local_model_path() + + if model_path and os.path.exists(model_path): + # Use passed model path + self.model_config = AutoConfig.from_pretrained(model_path) + elif alt_model_path: + # Use known model path + self.model_config = AutoConfig.from_pretrained( + alt_model_path, + revision=utils.koboldai_vars.revision, + cache_dir="cache", + ) + else: + # No model path locally, we'll probably have to download + self.model_config = AutoConfig.from_pretrained( + model_name, revision=utils.koboldai_vars.revision, cache_dir="cache" + ) + + return requested_parameters + + def set_input_parameters(self, parameters: dict): + self.model_name = parameters.get("custom_model_name", parameters["id"]) + self.path = parameters.get("path", None) + logger.info(parameters) + + def unload(self): + if hasattr(self, "model"): + self.model = None + + if hasattr(self, "tokenizer"): + self.tokenizer = None + + if hasattr(self, "model_config"): + self.model_config = None + + with torch.no_grad(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="torch.distributed.reduce_op is deprecated" + ) + for tensor in gc.get_objects(): + try: + if torch.is_tensor(tensor): + tensor.set_( + torch.tensor( + (), device=tensor.device, dtype=tensor.dtype + ) + ) + except: + pass + gc.collect() + + try: + with torch.no_grad(): + torch.cuda.empty_cache() + except: + pass def _load(self, save_model: bool, initial_load: bool) -> None: utils.koboldai_vars.allowsp = False @@ -58,12 +136,9 @@ class model_backend(HFInferenceModel): self.get_local_model_path(), low_cpu_mem_usage=True, device_map="auto" ) - if self.usegpu: - self.model = self.model.to("cuda") - self.tokenizer = self._get_tokenizer(self.get_local_model_path()) - self.model.kai_model = self + self.badwordsids = koboldai_settings.badwordsids_default utils.koboldai_vars.modeldim = self.model.get_input_embeddings().embedding_dim # Patch Huggingface stuff to use our samplers @@ -143,11 +218,8 @@ class model_backend(HFInferenceModel): else: gen_in = prompt_tokens - if not self.usegpu: - gen_in = gen_in.to("cpu") - else: - device = self.get_auxilary_device() - gen_in = gen_in.to(device) + device = self.get_auxilary_device() + gen_in = gen_in.to(device) additional_bad_words_ids = [self.tokenizer.encode("\n")] if single_line else [] @@ -179,3 +251,70 @@ class model_backend(HFInferenceModel): is_whole_generation=False, output_includes_prompt=True, ) + + def get_local_model_path( + self, legacy: bool = False, ignore_existance: bool = False + ) -> Optional[str]: + """ + Returns a string of the model's path locally, or None if it is not downloaded. + If ignore_existance is true, it will always return a path. + """ + if self.path is not None: + if os.path.exists(self.path): + return self.path + + if self.model_name in [ + "NeoCustom", + "GPT2Custom", + "TPUMeshTransformerGPTJ", + "TPUMeshTransformerGPTNeoX", + ]: + model_path = self.path + assert model_path + + # Path can be absolute or relative to models directory + if os.path.exists(model_path): + return model_path + + model_path = os.path.join("models", model_path) + + try: + assert os.path.exists(model_path) + except AssertionError: + logger.error( + f"Custom model does not exist at '{utils.koboldai_vars.custmodpth}' or '{model_path}'." + ) + raise + + return model_path + + basename = self.model_name.replace("/", "_") + if legacy: + ret = basename + else: + ret = os.path.join("models", basename) + + if os.path.isdir(ret) or ignore_existance: + return ret + return None + + def init_model_config(self) -> None: + # Get the model_type from the config or assume a model type if it isn't present + try: + self.model_config = AutoConfig.from_pretrained( + self.get_local_model_path() or self.model_name, + revision=utils.koboldai_vars.revision, + cache_dir="cache", + ) + self.model_type = self.model_config.model_type + except ValueError: + self.model_type = { + "NeoCustom": "gpt_neo", + "GPT2Custom": "gpt2", + }.get(self.model) + + if not self.model_type: + logger.warning( + "No model type detected, assuming Neo (If this is a GPT2 model use the other menu option or --model GPT2Custom)" + ) + self.model_type = "gpt_neo" From 8549c7c8967988fe6205d7cfc55284eb4969ec37 Mon Sep 17 00:00:00 2001 From: somebody Date: Wed, 12 Jul 2023 19:03:49 -0500 Subject: [PATCH 6/8] Basic backend module prioritization not secure; we're loading these modules so they can obviously execute code that manipulates the prioritization --- aiserver.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/aiserver.py b/aiserver.py index 74b8bca8..8c6d98d6 100644 --- a/aiserver.py +++ b/aiserver.py @@ -628,18 +628,30 @@ import importlib model_backend_code = {} model_backends = {} model_backend_type_crosswalk = {} + +PRIORITIZED_BACKEND_MODULES = ["generic_hf_torch"] + for module in os.listdir("./modeling/inference_models"): if not os.path.isfile(os.path.join("./modeling/inference_models",module)) and module != '__pycache__': try: - model_backend_code[module] = importlib.import_module('modeling.inference_models.{}.class'.format(module)) - model_backends[model_backend_code[module].model_backend_name] = model_backend_code[module].model_backend() - if 'disable' in vars(model_backends[model_backend_code[module].model_backend_name]) and model_backends[model_backend_code[module].model_backend_name].disable: - del model_backends[model_backend_code[module].model_backend_name] - else: - if model_backend_code[module].model_backend_type in model_backend_type_crosswalk: - model_backend_type_crosswalk[model_backend_code[module].model_backend_type].append(model_backend_code[module].model_backend_name) + backend_code = importlib.import_module('modeling.inference_models.{}.class'.format(module)) + backend_name = backend_code.model_backend_name + backend_type = backend_code.model_backend_type + backend_object = backend_code.model_backend() + + if "disable" in vars(backend_object) and backend_object.disable: + continue + + model_backends[backend_name] = backend_object + model_backend_code[module] = backend_code + + if backend_type in model_backend_type_crosswalk: + if module in PRIORITIZED_BACKEND_MODULES: + model_backend_type_crosswalk[backend_type].insert(0, backend_name) else: - model_backend_type_crosswalk[model_backend_code[module].model_backend_type] = [model_backend_code[module].model_backend_name] + model_backend_type_crosswalk[backend_type].append(backend_name) + else: + model_backend_type_crosswalk[backend_type] = [backend_name] except Exception: logger.error("Model Backend {} failed to load".format(module)) From afa8766ea68d8b08218edfab36526b75562609a6 Mon Sep 17 00:00:00 2001 From: onesome Date: Fri, 14 Jul 2023 18:01:18 -0500 Subject: [PATCH 7/8] Add is_valid --- modeling/inference_models/basic_hf/class.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modeling/inference_models/basic_hf/class.py b/modeling/inference_models/basic_hf/class.py index 74bfcd17..afca13ee 100644 --- a/modeling/inference_models/basic_hf/class.py +++ b/modeling/inference_models/basic_hf/class.py @@ -34,6 +34,24 @@ class model_backend(InferenceModel): self.model_name = "Basic Huggingface" self.path = None + def is_valid(self, model_name, model_path, menu_path): + try: + if model_path is not None and os.path.exists(model_path): + self.model_config = AutoConfig.from_pretrained(model_path) + elif os.path.exists("models/{}".format(model_name.replace("/", "_"))): + self.model_config = AutoConfig.from_pretrained( + "models/{}".format(model_name.replace("/", "_")), + revision=utils.koboldai_vars.revision, + cache_dir="cache", + ) + else: + self.model_config = AutoConfig.from_pretrained( + model_name, revision=utils.koboldai_vars.revision, cache_dir="cache" + ) + return True + except: + return False + def get_requested_parameters( self, model_name: str, model_path: str, menu_path: str, parameters: dict = {} ): From 7e2e75070b3c5e9238dda399d44ada125ac02665 Mon Sep 17 00:00:00 2001 From: onesome Date: Fri, 14 Jul 2023 18:47:52 -0500 Subject: [PATCH 8/8] Fix prioritization in load from dir In the future the module/backend list should probably contain objects that would make this whole deal a lot less hackier. --- aiserver.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index 8c6d98d6..f1dacc40 100644 --- a/aiserver.py +++ b/aiserver.py @@ -627,6 +627,7 @@ from modeling.patches import patch_transformers import importlib model_backend_code = {} model_backends = {} +model_backend_module_names = {} model_backend_type_crosswalk = {} PRIORITIZED_BACKEND_MODULES = ["generic_hf_torch"] @@ -645,6 +646,10 @@ for module in os.listdir("./modeling/inference_models"): model_backends[backend_name] = backend_object model_backend_code[module] = backend_code + if backend_name in model_backend_module_names: + raise RuntimeError(f"{module} cannot make backend '{backend_name}'; it already exists!") + model_backend_module_names[backend_name] = module + if backend_type in model_backend_type_crosswalk: if module in PRIORITIZED_BACKEND_MODULES: model_backend_type_crosswalk[backend_type].insert(0, backend_name) @@ -6252,7 +6257,11 @@ def UI_2_select_model(data): else: #Here we have a model that's not in our menu structure (either a custom model or a custom path #so we'll just go through all the possible loaders - for model_backend in model_backends: + for model_backend in sorted( + model_backends, + key=lambda x: model_backend_module_names[x] in PRIORITIZED_BACKEND_MODULES, + reverse=True, + ): if model_backends[model_backend].is_valid(data["name"], data["path"] if 'path' in data else None, data["menu"]): valid_loaders[model_backend] = model_backends[model_backend].get_requested_parameters(data["name"], data["path"] if 'path' in data else None, data["menu"]) emit("selected_model_info", {"model_backends": valid_loaders})