From 9df1f03b12ffa2513b15472a96338483178fe760 Mon Sep 17 00:00:00 2001 From: ebolam Date: Fri, 19 May 2023 14:28:36 -0400 Subject: [PATCH] Fix for custom huggingface model menu entry --- aiserver.py | 36 ++++---- modeling/inference_models/hf.py | 154 ++++++++++++++++++-------------- static/application.js | 23 ++++- static/koboldai.js | 23 ++++- 4 files changed, 139 insertions(+), 97 deletions(-) diff --git a/aiserver.py b/aiserver.py index b4aad4e7..fe6d7606 100644 --- a/aiserver.py +++ b/aiserver.py @@ -233,7 +233,7 @@ model_menu = { "mainmenu": [ MenuPath("Load a model from its directory", "NeoCustom"), MenuPath("Load an old GPT-2 model (eg CloverEdition)", "GPT2Custom"), - MenuFolder("Load custom model from Hugging Face", "customhuggingface"), + MenuModel("Load custom model from Hugging Face", "customhuggingface", ""), MenuFolder("Adventure Models", "adventurelist"), MenuFolder("Novel Models", "novellist"), MenuFolder("Chat Models", "chatlist"), @@ -6135,7 +6135,7 @@ def UI_2_select_model(data): valid_loaders = {} for model_backend in set([item.model_backend for sublist in model_menu for item in model_menu[sublist] if item.name == data['id']]): valid_loaders[model_backend] = model_backends[model_backend].get_requested_parameters(data["name"], data["path"] if 'path' in data else None, data["menu"]) - emit("selected_model_info", {"model_backends": valid_loaders, "preselected": "Huggingface"}) + emit("selected_model_info", {"model_backends": valid_loaders}) else: #Get directories paths, breadcrumbs = get_folder_path_info(data['path']) @@ -6149,24 +6149,20 @@ def UI_2_select_model(data): output.append({'label': path[1], 'name': path[0], 'size': "", "menu": "Custom", 'path': path[0], 'isMenu': not valid}) emit("open_model_load_menu", {"items": output+[{'label': 'Return to Main Menu', 'name':'mainmenu', 'size': "", "menu": "Custom", 'isMenu': True}], 'breadcrumbs': breadcrumbs}) return - - - #We've selected a menu - if data['model'] in model_menu: - sendModelSelection(menu=data['model']) - #We've selected a custom line - elif data['menu'] in ("NeoCustom", "GPT2Custom"): - get_model_info(data['menu'], directory=data['display_name']) - #We've selected a custom menu folder - elif data['model'] in ("NeoCustom", "GPT2Custom") and 'path' in data: - sendModelSelection(menu=data['model'], folder=data['path']) - #We've selected a custom menu - elif data['model'] in ("NeoCustom", "GPT2Custom", "customhuggingface"): - sendModelSelection(menu=data['model'], folder="./models") - else: - #We now have some model we want to potentially load. - #First we need to send the client the model parameters (layers, etc) - get_model_info(data['model']) + + + + +#==================================================================# +# Event triggered when user changes a model parameter and it's set to resubmit +#==================================================================# +@socketio.on('resubmit_model_info') +@logger.catch +def UI_2_resubmit_model_info(data): + valid_loaders = {} + for model_backend in set([item.model_backend for sublist in model_menu for item in model_menu[sublist] if item.name == data['id']]): + valid_loaders[model_backend] = model_backends[model_backend].get_requested_parameters(data["name"], data["path"] if 'path' in data else None, data["menu"], parameters=data) + emit("selected_model_info", {"model_backends": valid_loaders}) #==================================================================# # Event triggered when user loads a model diff --git a/modeling/inference_models/hf.py b/modeling/inference_models/hf.py index 6f848fa9..eff3d1ce 100644 --- a/modeling/inference_models/hf.py +++ b/modeling/inference_models/hf.py @@ -33,95 +33,111 @@ class HFInferenceModel(InferenceModel): except: return False - def get_requested_parameters(self, model_name, model_path, menu_path): + def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}): requested_parameters = [] if not self.hf_torch: return [] - if model_path is not None and os.path.exists(model_path): - self.model_config = AutoConfig.from_pretrained(model_path) - elif(os.path.exists("models/{}".format(model_name.replace('/', '_')))): - self.model_config = AutoConfig.from_pretrained("models/{}".format(model_name.replace('/', '_')), revision=utils.koboldai_vars.revision, cache_dir="cache") - else: - self.model_config = AutoConfig.from_pretrained(model_name, revision=utils.koboldai_vars.revision, cache_dir="cache") - layer_count = self.model_config["n_layer"] if isinstance(self.model_config, dict) else self.model_config.num_layers if hasattr(self.model_config, "num_layers") else self.model_config.n_layer if hasattr(self.model_config, "n_layer") else self.model_config.num_hidden_layers if hasattr(self.model_config, 'num_hidden_layers') else None - if layer_count is not None and layer_count >= 0 and not self.nobreakmodel: - if os.path.exists("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_"))) and 'base_url' not in vars(self): - with open("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_")), "r") as f: - temp = json.load(f) - break_values = temp['layers'] if 'layers' in temp else [layer_count] - disk_blocks = temp['disk_layers'] if 'disk_layers' in temp else 0 + if model_name == 'customhuggingface': + requested_parameters.append({ + "uitype": "text", + "unit": "text", + "label": "Huggingface Model Name", + "id": "custom_model_name", + "default": parameters["custom_model_name"] if "custom_model_name" in parameters and parameters["custom_model_name"] != "" else "", + "check": {"value": "", 'check': "!="}, + "tooltip": "Model name from https://huggingface.co/", + "menu_path": "", + "refresh_model_inputs": True, + "extra_classes": "" + }) + + if model_name != 'customhuggingface' or "custom_model_name" in parameters: + model_name = parameters["custom_model_name"] if "custom_model_name" in parameters and parameters["custom_model_name"] != "" else model_name + if model_path is not None and os.path.exists(model_path): + self.model_config = AutoConfig.from_pretrained(model_path) + elif(os.path.exists("models/{}".format(model_name.replace('/', '_')))): + self.model_config = AutoConfig.from_pretrained("models/{}".format(model_name.replace('/', '_')), revision=utils.koboldai_vars.revision, cache_dir="cache") else: - break_values = [layer_count] - disk_blocks = 0 - - break_values = [int(x) for x in break_values if x != '' and x is not None] - gpu_count = torch.cuda.device_count() - break_values += [0] * (gpu_count - len(break_values)) - if disk_blocks is not None: - break_values += [int(disk_blocks)] - for i in range(gpu_count): + self.model_config = AutoConfig.from_pretrained(model_name, revision=utils.koboldai_vars.revision, cache_dir="cache") + layer_count = self.model_config["n_layer"] if isinstance(self.model_config, dict) else self.model_config.num_layers if hasattr(self.model_config, "num_layers") else self.model_config.n_layer if hasattr(self.model_config, "n_layer") else self.model_config.num_hidden_layers if hasattr(self.model_config, 'num_hidden_layers') else None + if layer_count is not None and layer_count >= 0 and not self.nobreakmodel: + if os.path.exists("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_"))) and 'base_url' not in vars(self): + with open("settings/{}.generic_hf_torch.model_backend.settings".format(model_name.replace("/", "_")), "r") as f: + temp = json.load(f) + break_values = temp['layers'] if 'layers' in temp else [layer_count] + disk_blocks = temp['disk_layers'] if 'disk_layers' in temp else 0 + else: + break_values = [layer_count] + disk_blocks = 0 + + break_values = [int(x) for x in break_values if x != '' and x is not None] + gpu_count = torch.cuda.device_count() + break_values += [0] * (gpu_count - len(break_values)) + if disk_blocks is not None: + break_values += [int(disk_blocks)] + for i in range(gpu_count): + requested_parameters.append({ + "uitype": "slider", + "unit": "int", + "label": "{} Layers".format(torch.cuda.get_device_name(i)), + "id": "{}_Layers".format(i), + "min": 0, + "max": layer_count, + "step": 1, + "check": {"sum": ["{}_Layers".format(i) for i in range(gpu_count)]+['CPU_Layers']+(['Disk_Layers'] if disk_blocks is not None else []), "value": layer_count, 'check': "="}, + "check_message": "The sum of assigned layers must equal {}".format(layer_count), + "default": break_values[i], + "tooltip": "The number of layers to put on {}.".format(torch.cuda.get_device_name(i)), + "menu_path": "Layers", + "extra_classes": "", + "refresh_model_inputs": False + }) requested_parameters.append({ "uitype": "slider", "unit": "int", - "label": "{} Layers".format(torch.cuda.get_device_name(i)), - "id": "{}_Layers".format(i), + "label": "CPU Layers", + "id": "CPU_Layers", "min": 0, "max": layer_count, "step": 1, "check": {"sum": ["{}_Layers".format(i) for i in range(gpu_count)]+['CPU_Layers']+(['Disk_Layers'] if disk_blocks is not None else []), "value": layer_count, 'check': "="}, "check_message": "The sum of assigned layers must equal {}".format(layer_count), - "default": break_values[i], - "tooltip": "The number of layers to put on {}.".format(torch.cuda.get_device_name(i)), + "default": layer_count - sum(break_values), + "tooltip": "The number of layers to put on the CPU. This will use your system RAM. It will also do inference partially on CPU. Use if you must.", "menu_path": "Layers", "extra_classes": "", "refresh_model_inputs": False }) - requested_parameters.append({ - "uitype": "slider", - "unit": "int", - "label": "CPU Layers", - "id": "CPU_Layers", - "min": 0, - "max": layer_count, - "step": 1, - "check": {"sum": ["{}_Layers".format(i) for i in range(gpu_count)]+['CPU_Layers']+(['Disk_Layers'] if disk_blocks is not None else []), "value": layer_count, 'check': "="}, - "check_message": "The sum of assigned layers must equal {}".format(layer_count), - "default": layer_count - sum(break_values), - "tooltip": "The number of layers to put on the CPU. This will use your system RAM. It will also do inference partially on CPU. Use if you must.", - "menu_path": "Layers", - "extra_classes": "", - "refresh_model_inputs": False - }) - if disk_blocks is not None: + if disk_blocks is not None: + requested_parameters.append({ + "uitype": "slider", + "unit": "int", + "label": "Disk Layers", + "id": "Disk_Layers", + "min": 0, + "max": layer_count, + "step": 1, + "check": {"sum": ["{}_Layers".format(i) for i in range(gpu_count)]+['CPU_Layers']+(['Disk_Layers'] if disk_blocks is not None else []), "value": layer_count, 'check': "="}, + "check_message": "The sum of assigned layers must equal {}".format(layer_count), + "default": disk_blocks, + "tooltip": "The number of layers to put on the disk. This will use your hard drive. The is VERY slow in comparison to GPU or CPU. Use as a last resort.", + "menu_path": "Layers", + "extra_classes": "", + "refresh_model_inputs": False + }) + else: requested_parameters.append({ - "uitype": "slider", - "unit": "int", - "label": "Disk Layers", - "id": "Disk_Layers", - "min": 0, - "max": layer_count, - "step": 1, - "check": {"sum": ["{}_Layers".format(i) for i in range(gpu_count)]+['CPU_Layers']+(['Disk_Layers'] if disk_blocks is not None else []), "value": layer_count, 'check': "="}, - "check_message": "The sum of assigned layers must equal {}".format(layer_count), - "default": disk_blocks, - "tooltip": "The number of layers to put on the disk. This will use your hard drive. The is VERY slow in comparison to GPU or CPU. Use as a last resort.", + "uitype": "toggle", + "unit": "bool", + "label": "Use GPU", + "id": "use_gpu", + "default": False, + "tooltip": "Whether or not to use the GPU", "menu_path": "Layers", "extra_classes": "", "refresh_model_inputs": False }) - else: - requested_parameters.append({ - "uitype": "toggle", - "unit": "bool", - "label": "Use GPU", - "id": "use_gpu", - "default": False, - "tooltip": "Whether or not to use the GPU", - "menu_path": "Layers", - "extra_classes": "", - "refresh_model_inputs": False - }) - + return requested_parameters @@ -153,7 +169,7 @@ class HFInferenceModel(InferenceModel): self.usegpu = parameters['use_gpu'] if 'use_gpu' in parameters else None self.model_type = self.get_model_type() self.breakmodel = ((self.model_type != 'gpt2') or self.model_type in ("gpt_neo", "gptj", "xglm", "opt")) and not self.nobreakmodel - self.model_name = parameters['id'] + self.model_name = parameters['custom_model_name'] if 'custom_model_name' in parameters else parameters['id'] self.path = parameters['path'] if 'path' in parameters else None def unload(self): diff --git a/static/application.js b/static/application.js index 99a65ed7..ca445c5f 100644 --- a/static/application.js +++ b/static/application.js @@ -4009,7 +4009,25 @@ function model_settings_checker() { if (valid || missing_element) { //if we are supposed to refresh when this value changes we'll resubmit if ((this.getAttribute("refresh_model_inputs") == "true") && !missing_element && !this.noresubmit) { - console.log("resubmit"); + //get an object of all the input settings from the user + data = {} + settings_area = document.getElementById(document.getElementById("modelplugin").value + "_settings_area"); + for (const element of settings_area.querySelectorAll(".model_settings_input:not(.hidden)")) { + var element_data = element.value; + if (element.getAttribute("data_type") == "int") { + element_data = parseInt(element_data); + } else if (element.getAttribute("data_type") == "float") { + element_data = parseFloat(element_data); + } else if (element.getAttribute("data_type") == "bool") { + element_data = (element_data == 'on'); + } + data[element.id.split("|")[1].replace("_value", "")] = element_data; + } + data = {...data, ...selected_model_data}; + + data['plugin'] = document.getElementById("modelplugin").value; + + socket.emit("resubmit_model_info", data); } if ('sum' in this.check_data) { for (const temp of this.check_data['sum']) { @@ -4099,9 +4117,6 @@ function selected_model_info(sent_data) { modelpluginoption.innerText = loader; modelpluginoption.value = loader; modelplugin.append(modelpluginoption); - if (loader == sent_data['preselected']) { - modelplugin.value = sent_data['preselected']; - } //create the user input for each requested input for (item of items) { diff --git a/static/koboldai.js b/static/koboldai.js index 99595879..dabbcda9 100644 --- a/static/koboldai.js +++ b/static/koboldai.js @@ -1683,7 +1683,25 @@ function model_settings_checker() { if (valid || missing_element) { //if we are supposed to refresh when this value changes we'll resubmit if ((this.getAttribute("refresh_model_inputs") == "true") && !missing_element && !this.noresubmit) { - console.log("resubmit"); + //get an object of all the input settings from the user + data = {} + settings_area = document.getElementById(document.getElementById("modelplugin").value + "_settings_area"); + for (const element of settings_area.querySelectorAll(".model_settings_input:not(.hidden)")) { + var element_data = element.value; + if (element.getAttribute("data_type") == "int") { + element_data = parseInt(element_data); + } else if (element.getAttribute("data_type") == "float") { + element_data = parseFloat(element_data); + } else if (element.getAttribute("data_type") == "bool") { + element_data = (element_data == 'on'); + } + data[element.id.split("|")[1].replace("_value", "")] = element_data; + } + data = {...data, ...selected_model_data}; + + data['plugin'] = document.getElementById("modelplugin").value; + + socket.emit("resubmit_model_info", data); } if ('sum' in this.check_data) { for (const temp of this.check_data['sum']) { @@ -1773,9 +1791,6 @@ function selected_model_info(sent_data) { modelpluginoption.innerText = loader; modelpluginoption.value = loader; modelplugin.append(modelpluginoption); - if (loader == sent_data['preselected']) { - modelplugin.value = sent_data['preselected']; - } //create the user input for each requested input for (item of items) {