mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-06-05 21:59:24 +02:00
Make modellist easier to work with
This commit is contained in:
somebody
403
aiserver.py
403
aiserver.py
@@ -7,6 +7,7 @@
|
|||||||
|
|
||||||
# External packages
|
# External packages
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum
|
||||||
import random
|
import random
|
||||||
import shutil
|
import shutil
|
||||||
import eventlet
|
import eventlet
|
||||||
@@ -103,6 +104,10 @@ def new_pretrainedtokenizerbase_from_pretrained(cls, *args, **kwargs):
|
|||||||
PreTrainedTokenizerBase.from_pretrained = new_pretrainedtokenizerbase_from_pretrained
|
PreTrainedTokenizerBase.from_pretrained = new_pretrainedtokenizerbase_from_pretrained
|
||||||
|
|
||||||
|
|
||||||
|
def is_model_downloaded(model_name: str) -> bool:
|
||||||
|
model_stub = model_name.replace("/", "_")
|
||||||
|
return os.path.isdir(os.path.join("models", model_stub))
|
||||||
|
|
||||||
#==================================================================#
|
#==================================================================#
|
||||||
# Variables & Storage
|
# Variables & Storage
|
||||||
#==================================================================#
|
#==================================================================#
|
||||||
@@ -118,172 +123,239 @@ class colors:
|
|||||||
END = '\033[0m'
|
END = '\033[0m'
|
||||||
UNDERLINE = '\033[4m'
|
UNDERLINE = '\033[4m'
|
||||||
|
|
||||||
|
class MenuModelType(Enum):
|
||||||
|
HUGGINGFACE = 0
|
||||||
|
ONLINE_API = 1
|
||||||
|
OTHER = 2
|
||||||
|
RWKV = 3
|
||||||
|
|
||||||
|
class MenuItem:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
label: str,
|
||||||
|
name: str,
|
||||||
|
experimental: bool = False
|
||||||
|
) -> None:
|
||||||
|
self.label = label
|
||||||
|
self.name = name
|
||||||
|
self.experimental = experimental
|
||||||
|
|
||||||
|
def should_show(self) -> bool:
|
||||||
|
return koboldai_vars.experimental_features or not self.experimental
|
||||||
|
|
||||||
|
class MenuFolder(MenuItem):
|
||||||
|
def to_ui1(self) -> list:
|
||||||
|
return [
|
||||||
|
self.label,
|
||||||
|
self.name,
|
||||||
|
"",
|
||||||
|
True,
|
||||||
|
]
|
||||||
|
|
||||||
|
def to_json(self) -> dict:
|
||||||
|
return {
|
||||||
|
"label": self.label,
|
||||||
|
"name": self.name,
|
||||||
|
"size": "",
|
||||||
|
"isMenu": True,
|
||||||
|
"isDownloaded": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
class MenuModel(MenuItem):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
label: str,
|
||||||
|
name: str,
|
||||||
|
vram_requirements: str = "",
|
||||||
|
model_type: MenuModelType = MenuModelType.HUGGINGFACE,
|
||||||
|
experimental: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(label, name, experimental)
|
||||||
|
self.model_type = model_type
|
||||||
|
self.vram_requirements = vram_requirements
|
||||||
|
self.is_downloaded = is_model_downloaded(self.name)
|
||||||
|
|
||||||
|
def to_ui1(self) -> list:
|
||||||
|
return [
|
||||||
|
self.label,
|
||||||
|
self.name,
|
||||||
|
self.vram_requirements,
|
||||||
|
False,
|
||||||
|
self.is_downloaded
|
||||||
|
]
|
||||||
|
|
||||||
|
def to_json(self) -> dict:
|
||||||
|
return {
|
||||||
|
"label": self.label,
|
||||||
|
"name": self.name,
|
||||||
|
"size": self.vram_requirements,
|
||||||
|
"isMenu": False,
|
||||||
|
"isDownloaded": self.is_downloaded,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# AI models Menu
|
# AI models Menu
|
||||||
# This is a dict of lists where they key is the menu name, and the list is the menu items.
|
# This is a dict of lists where they key is the menu name, and the list is the menu items.
|
||||||
# Each item takes the 4 elements, 1: Text to display, 2: Model Name (koboldai_vars.model) or menu name (Key name for another menu),
|
# Each item takes the 4 elements, 1: Text to display, 2: Model Name (koboldai_vars.model) or menu name (Key name for another menu),
|
||||||
# 3: the memory requirement for the model, 4: if the item is a menu or not (True/False)
|
# 3: the memory requirement for the model, 4: if the item is a menu or not (True/False)
|
||||||
model_menu = {
|
model_menu = {
|
||||||
'mainmenu': [
|
"mainmenu": [
|
||||||
["Load a model from its directory", "NeoCustom", "", False],
|
MenuModel("Load a model from its directory", "NeoCustom"),
|
||||||
["Load an old GPT-2 model (eg CloverEdition)", "GPT2Custom", "", False],
|
MenuModel("Load an old GPT-2 model (eg CloverEdition)", "GPT2Custom"),
|
||||||
["Load custom model from Hugging Face", "customhuggingface", "", True],
|
MenuFolder("Load custom model from Hugging Face", "customhuggingface"),
|
||||||
["Adventure Models", "adventurelist", "", True],
|
MenuFolder("Adventure Models", "adventurelist"),
|
||||||
["Novel Models", "novellist", "", True],
|
MenuFolder("Novel Models", "novellist"),
|
||||||
["Chat Models", "chatlist", "", True],
|
MenuFolder("Chat Models", "chatlist"),
|
||||||
["NSFW Models", "nsfwlist", "", True],
|
MenuFolder("NSFW Models", "nsfwlist"),
|
||||||
["Untuned OPT", "optlist", "", True],
|
MenuFolder("Untuned OPT", "optlist"),
|
||||||
["Untuned GPT-Neo/J", "gptneolist", "", True],
|
MenuFolder("Untuned GPT-Neo/J", "gptneolist"),
|
||||||
["Untuned Pythia", "pythialist", "", True],
|
MenuFolder("Untuned Pythia", "pythialist"),
|
||||||
["Untuned Fairseq Dense", "fsdlist", "", True],
|
MenuFolder("Untuned Fairseq Dense", "fsdlist"),
|
||||||
["Untuned Bloom", "bloomlist", "", True],
|
MenuFolder("Untuned Bloom", "bloomlist"),
|
||||||
["Untuned XGLM", "xglmlist", "", True],
|
MenuFolder("Untuned XGLM", "xglmlist"),
|
||||||
# ["Untuned RWKV-4 (Experimental)", "rwkvlist", "", True],
|
MenuFolder("Untuned RWKV-4 (Experimental)", "rwkvlist"),
|
||||||
["Untuned GPT2", "gpt2list", "", True],
|
MenuFolder("Untuned GPT2", "gpt2list"),
|
||||||
["Online Services", "apilist", "", True],
|
MenuFolder("Online Services", "apilist"),
|
||||||
["Read Only (No AI)", "ReadOnly", "", False]
|
MenuModel("Read Only (No AI)", "ReadOnly", model_type=MenuModelType.OTHER),
|
||||||
],
|
],
|
||||||
'adventurelist': [
|
'adventurelist': [
|
||||||
["Skein 20B", "KoboldAI/GPT-NeoX-20B-Skein", "64GB", False],
|
MenuModel("Skein 20B", "KoboldAI/GPT-NeoX-20B-Skein", "64GB"),
|
||||||
["Nerys OPT 13B V2 (Hybrid)", "KoboldAI/OPT-13B-Nerys-v2", "32GB", False],
|
MenuModel("Nerys OPT 13B V2 (Hybrid)", "KoboldAI/OPT-13B-Nerys-v2", "32GB"),
|
||||||
["Nerys FSD 13B V2 (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys-v2", "32GB", False],
|
MenuModel("Nerys FSD 13B V2 (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys-v2", "32GB"),
|
||||||
["Nerys FSD 13B (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys", "32GB", False],
|
MenuModel("Nerys FSD 13B (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys", "32GB"),
|
||||||
["Skein 6B", "KoboldAI/GPT-J-6B-Skein", "16GB", False],
|
MenuModel("Skein 6B", "KoboldAI/GPT-J-6B-Skein", "16GB"),
|
||||||
["OPT Nerys 6B V2 (Hybrid)", "KoboldAI/OPT-6B-nerys-v2", "16GB", False],
|
MenuModel("OPT Nerys 6B V2 (Hybrid)", "KoboldAI/OPT-6B-nerys-v2", "16GB"),
|
||||||
["Adventure 6B", "KoboldAI/GPT-J-6B-Adventure", "16GB", False],
|
MenuModel("Adventure 6B", "KoboldAI/GPT-J-6B-Adventure", "16GB"),
|
||||||
["Nerys FSD 2.7B (Hybrid)", "KoboldAI/fairseq-dense-2.7B-Nerys", "8GB", False],
|
MenuModel("Nerys FSD 2.7B (Hybrid)", "KoboldAI/fairseq-dense-2.7B-Nerys", "8GB"),
|
||||||
["Adventure 2.7B", "KoboldAI/GPT-Neo-2.7B-AID", "8GB", False],
|
MenuModel("Adventure 2.7B", "KoboldAI/GPT-Neo-2.7B-AID", "8GB"),
|
||||||
["Adventure 1.3B", "KoboldAI/GPT-Neo-1.3B-Adventure", "6GB", False],
|
MenuModel("Adventure 1.3B", "KoboldAI/GPT-Neo-1.3B-Adventure", "6GB"),
|
||||||
["Adventure 125M (Mia)", "Merry/AID-Neo-125M", "2GB", False],
|
MenuModel("Adventure 125M (Mia)", "Merry/AID-Neo-125M", "2GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'novellist': [
|
'novellist': [
|
||||||
["Nerys OPT 13B V2 (Hybrid)", "KoboldAI/OPT-13B-Nerys-v2", "32GB", False],
|
MenuModel("Nerys OPT 13B V2 (Hybrid)", "KoboldAI/OPT-13B-Nerys-v2", "32GB"),
|
||||||
["Nerys FSD 13B V2 (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys-v2", "32GB", False],
|
MenuModel("Nerys FSD 13B V2 (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys-v2", "32GB"),
|
||||||
["Janeway FSD 13B", "KoboldAI/fairseq-dense-13B-Janeway", "32GB", False],
|
MenuModel("Janeway FSD 13B", "KoboldAI/fairseq-dense-13B-Janeway", "32GB"),
|
||||||
["Nerys FSD 13B (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys", "32GB", False],
|
MenuModel("Nerys FSD 13B (Hybrid)", "KoboldAI/fairseq-dense-13B-Nerys", "32GB"),
|
||||||
["OPT Nerys 6B V2 (Hybrid)", "KoboldAI/OPT-6B-nerys-v2", "16GB", False],
|
MenuModel("OPT Nerys 6B V2 (Hybrid)", "KoboldAI/OPT-6B-nerys-v2", "16GB"),
|
||||||
["Janeway FSD 6.7B", "KoboldAI/fairseq-dense-6.7B-Janeway", "16GB", False],
|
MenuModel("Janeway FSD 6.7B", "KoboldAI/fairseq-dense-6.7B-Janeway", "16GB"),
|
||||||
["Janeway Neo 6B", "KoboldAI/GPT-J-6B-Janeway", "16GB", False],
|
MenuModel("Janeway Neo 6B", "KoboldAI/GPT-J-6B-Janeway", "16GB"),
|
||||||
["Qilin Lit 6B (SFW)", "rexwang8/qilin-lit-6b", "16GB", False],
|
MenuModel("Qilin Lit 6B (SFW)", "rexwang8/qilin-lit-6b", "16GB"),
|
||||||
["Janeway Neo 2.7B", "KoboldAI/GPT-Neo-2.7B-Janeway", "8GB", False],
|
MenuModel("Janeway Neo 2.7B", "KoboldAI/GPT-Neo-2.7B-Janeway", "8GB"),
|
||||||
["Janeway FSD 2.7B", "KoboldAI/fairseq-dense-2.7B-Janeway", "8GB", False],
|
MenuModel("Janeway FSD 2.7B", "KoboldAI/fairseq-dense-2.7B-Janeway", "8GB"),
|
||||||
["Nerys FSD 2.7B (Hybrid)", "KoboldAI/fairseq-dense-2.7B-Nerys", "8GB", False],
|
MenuModel("Nerys FSD 2.7B (Hybrid)", "KoboldAI/fairseq-dense-2.7B-Nerys", "8GB"),
|
||||||
["Horni-LN 2.7B", "KoboldAI/GPT-Neo-2.7B-Horni-LN", "8GB", False],
|
MenuModel("Horni-LN 2.7B", "KoboldAI/GPT-Neo-2.7B-Horni-LN", "8GB"),
|
||||||
["Picard 2.7B (Older Janeway)", "KoboldAI/GPT-Neo-2.7B-Picard", "8GB", False],
|
MenuModel("Picard 2.7B (Older Janeway)", "KoboldAI/GPT-Neo-2.7B-Picard", "8GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'nsfwlist': [
|
'nsfwlist': [
|
||||||
["Erebus 20B (NSFW)", "KoboldAI/GPT-NeoX-20B-Erebus", "64GB", False],
|
MenuModel("Erebus 20B (NSFW)", "KoboldAI/GPT-NeoX-20B-Erebus", "64GB"),
|
||||||
["Erebus 13B (NSFW)", "KoboldAI/OPT-13B-Erebus", "32GB", False],
|
MenuModel("Erebus 13B (NSFW)", "KoboldAI/OPT-13B-Erebus", "32GB"),
|
||||||
["Shinen FSD 13B (NSFW)", "KoboldAI/fairseq-dense-13B-Shinen", "32GB", False],
|
MenuModel("Shinen FSD 13B (NSFW)", "KoboldAI/fairseq-dense-13B-Shinen", "32GB"),
|
||||||
["Erebus 6.7B (NSFW)", "KoboldAI/OPT-6.7B-Erebus", "16GB", False],
|
MenuModel("Erebus 6.7B (NSFW)", "KoboldAI/OPT-6.7B-Erebus", "16GB"),
|
||||||
["Shinen FSD 6.7B (NSFW)", "KoboldAI/fairseq-dense-6.7B-Shinen", "16GB", False],
|
MenuModel("Shinen FSD 6.7B (NSFW)", "KoboldAI/fairseq-dense-6.7B-Shinen", "16GB"),
|
||||||
["Lit V2 6B (NSFW)", "hakurei/litv2-6B-rev3", "16GB", False],
|
MenuModel("Lit V2 6B (NSFW)", "hakurei/litv2-6B-rev3", "16GB"),
|
||||||
["Lit 6B (NSFW)", "hakurei/lit-6B", "16GB", False],
|
MenuModel("Lit 6B (NSFW)", "hakurei/lit-6B", "16GB"),
|
||||||
["Shinen 6B (NSFW)", "KoboldAI/GPT-J-6B-Shinen", "16GB", False],
|
MenuModel("Shinen 6B (NSFW)", "KoboldAI/GPT-J-6B-Shinen", "16GB"),
|
||||||
["Erebus 2.7B (NSFW)", "KoboldAI/OPT-2.7B-Erebus", "8GB", False],
|
MenuModel("Erebus 2.7B (NSFW)", "KoboldAI/OPT-2.7B-Erebus", "8GB"),
|
||||||
["Horni 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Horni", "8GB", False],
|
MenuModel("Horni 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Horni", "8GB"),
|
||||||
["Shinen 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Shinen", "8GB", False],
|
MenuModel("Shinen 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Shinen", "8GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'chatlist': [
|
'chatlist': [
|
||||||
["Pygmalion 6B", "PygmalionAI/pygmalion-6b", "16GB", False],
|
MenuModel("Pygmalion 6B", "PygmalionAI/pygmalion-6b", "16GB"),
|
||||||
["Pygmalion 2.7B", "PygmalionAI/pygmalion-2.7b", "8GB", False],
|
MenuModel("Pygmalion 2.7B", "PygmalionAI/pygmalion-2.7b", "8GB"),
|
||||||
["Pygmalion 1.3B", "PygmalionAI/pygmalion-1.3b", "6GB", False],
|
MenuModel("Pygmalion 1.3B", "PygmalionAI/pygmalion-1.3b", "6GB"),
|
||||||
["Pygmalion 350M", "PygmalionAI/pygmalion-350m", "2GB", False],
|
MenuModel("Pygmalion 350M", "PygmalionAI/pygmalion-350m", "2GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'gptneolist': [
|
'gptneolist': [
|
||||||
["GPT-NeoX 20B", "EleutherAI/gpt-neox-20b", "64GB", False],
|
MenuModel("GPT-NeoX 20B", "EleutherAI/gpt-neox-20b", "64GB"),
|
||||||
["Pythia 13B (NeoX, Same dataset)", "EleutherAI/pythia-13b", "32GB", False],
|
MenuModel("Pythia 13B (NeoX, Same dataset)", "EleutherAI/pythia-13b", "32GB"),
|
||||||
["GPT-J 6B", "EleutherAI/gpt-j-6B", "16GB", False],
|
MenuModel("GPT-J 6B", "EleutherAI/gpt-j-6B", "16GB"),
|
||||||
["GPT-Neo 2.7B", "EleutherAI/gpt-neo-2.7B", "8GB", False],
|
MenuModel("GPT-Neo 2.7B", "EleutherAI/gpt-neo-2.7B", "8GB"),
|
||||||
["GPT-Neo 1.3B", "EleutherAI/gpt-neo-1.3B", "6GB", False],
|
MenuModel("GPT-Neo 1.3B", "EleutherAI/gpt-neo-1.3B", "6GB"),
|
||||||
["Pythia 800M (NeoX, Same dataset)", "EleutherAI/pythia-800m", "4GB", False],
|
MenuModel("Pythia 800M (NeoX, Same dataset)", "EleutherAI/pythia-800m", "4GB"),
|
||||||
["Pythia 350M (NeoX, Same dataset)", "EleutherAI/pythia-350m", "2GB", False],
|
MenuModel("Pythia 350M (NeoX, Same dataset)", "EleutherAI/pythia-350m", "2GB"),
|
||||||
["GPT-Neo 125M", "EleutherAI/gpt-neo-125M", "2GB", False],
|
MenuModel("GPT-Neo 125M", "EleutherAI/gpt-neo-125M", "2GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'pythialist': [
|
'pythialist': [
|
||||||
["Pythia 13B Deduped", "EleutherAI/pythia-13b-deduped", "32GB", False],
|
MenuModel("Pythia 13B Deduped", "EleutherAI/pythia-13b-deduped", "32GB"),
|
||||||
["Pythia 13B", "EleutherAI/pythia-13b", "32GB", False],
|
MenuModel("Pythia 13B", "EleutherAI/pythia-13b", "32GB"),
|
||||||
["Pythia 6.7B Deduped", "EleutherAI/pythia-6.7b-deduped", "16GB", False],
|
MenuModel("Pythia 6.7B Deduped", "EleutherAI/pythia-6.7b-deduped", "16GB"),
|
||||||
["Pythia 6.7B", "EleutherAI/pythia-6.7b", "16GB", False],
|
MenuModel("Pythia 6.7B", "EleutherAI/pythia-6.7b", "16GB"),
|
||||||
["Pythia 1.3B Deduped", "EleutherAI/pythia-1.3b-deduped", "6GB", False],
|
MenuModel("Pythia 1.3B Deduped", "EleutherAI/pythia-1.3b-deduped", "6GB"),
|
||||||
["Pythia 1.3B", "EleutherAI/pythia-1.3b", "6GB", False],
|
MenuModel("Pythia 1.3B", "EleutherAI/pythia-1.3b", "6GB"),
|
||||||
["Pythia 800M", "EleutherAI/pythia-800m", "4GB", False],
|
MenuModel("Pythia 800M", "EleutherAI/pythia-800m", "4GB"),
|
||||||
["Pythia 350M Deduped", "EleutherAI/pythia-350m-deduped", "2GB", False],
|
MenuModel("Pythia 350M Deduped", "EleutherAI/pythia-350m-deduped", "2GB"),
|
||||||
["Pythia 350M", "EleutherAI/pythia-350m", "2GB", False],
|
MenuModel("Pythia 350M", "EleutherAI/pythia-350m", "2GB"),
|
||||||
["Pythia 125M Deduped", "EleutherAI/pythia-125m-deduped", "2GB", False],
|
MenuModel("Pythia 125M Deduped", "EleutherAI/pythia-125m-deduped", "2GB"),
|
||||||
["Pythia 125M", "EleutherAI/pythia-125m", "2GB", False],
|
MenuModel("Pythia 125M", "EleutherAI/pythia-125m", "2GB"),
|
||||||
["Pythia 19M Deduped", "EleutherAI/pythia-19m-deduped", "1GB", False],
|
MenuModel("Pythia 19M Deduped", "EleutherAI/pythia-19m-deduped", "1GB"),
|
||||||
["Pythia 19M", "EleutherAI/pythia-19m", "1GB", False],
|
MenuModel("Pythia 19M", "EleutherAI/pythia-19m", "1GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'gpt2list': [
|
'gpt2list': [
|
||||||
["GPT-2 XL", "gpt2-xl", "6GB", False],
|
MenuModel("GPT-2 XL", "gpt2-xl", "6GB"),
|
||||||
["GPT-2 Large", "gpt2-large", "4GB", False],
|
MenuModel("GPT-2 Large", "gpt2-large", "4GB"),
|
||||||
["GPT-2 Med", "gpt2-medium", "2GB", False],
|
MenuModel("GPT-2 Med", "gpt2-medium", "2GB"),
|
||||||
["GPT-2", "gpt2", "2GB", False],
|
MenuModel("GPT-2", "gpt2", "2GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'bloomlist': [
|
'bloomlist': [
|
||||||
["Bloom 176B", "bigscience/bloom", "", False],
|
MenuModel("Bloom 176B", "bigscience/bloom"),
|
||||||
["Bloom 7.1B", "bigscience/bloom-7b1", "", False],
|
MenuModel("Bloom 7.1B", "bigscience/bloom-7b1"),
|
||||||
["Bloom 3B", "bigscience/bloom-3b", "", False],
|
MenuModel("Bloom 3B", "bigscience/bloom-3b"),
|
||||||
["Bloom 1.7B", "bigscience/bloom-1b7", "", False],
|
MenuModel("Bloom 1.7B", "bigscience/bloom-1b7"),
|
||||||
["Bloom 560M", "bigscience/bloom-560m", "", False],
|
MenuModel("Bloom 560M", "bigscience/bloom-560m"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'optlist': [
|
'optlist': [
|
||||||
["OPT 66B", "facebook/opt-66b", "128GB", False],
|
MenuModel("OPT 66B", "facebook/opt-66b", "128GB"),
|
||||||
["OPT 30B", "facebook/opt-30b", "64GB", False],
|
MenuModel("OPT 30B", "facebook/opt-30b", "64GB"),
|
||||||
["OPT 13B", "facebook/opt-13b", "32GB", False],
|
MenuModel("OPT 13B", "facebook/opt-13b", "32GB"),
|
||||||
["OPT 6.7B", "facebook/opt-6.7b", "16GB", False],
|
MenuModel("OPT 6.7B", "facebook/opt-6.7b", "16GB"),
|
||||||
["OPT 2.7B", "facebook/opt-2.7b", "8GB", False],
|
MenuModel("OPT 2.7B", "facebook/opt-2.7b", "8GB"),
|
||||||
["OPT 1.3B", "facebook/opt-1.3b", "4GB", False],
|
MenuModel("OPT 1.3B", "facebook/opt-1.3b", "4GB"),
|
||||||
["OPT 350M", "facebook/opt-350m", "2GB", False],
|
MenuModel("OPT 350M", "facebook/opt-350m", "2GB"),
|
||||||
["OPT 125M", "facebook/opt-125m", "1GB", False],
|
MenuModel("OPT 125M", "facebook/opt-125m", "1GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'fsdlist': [
|
'fsdlist': [
|
||||||
["Fairseq Dense 13B", "KoboldAI/fairseq-dense-13B", "32GB", False],
|
MenuModel("Fairseq Dense 13B", "KoboldAI/fairseq-dense-13B", "32GB"),
|
||||||
["Fairseq Dense 6.7B", "KoboldAI/fairseq-dense-6.7B", "16GB", False],
|
MenuModel("Fairseq Dense 6.7B", "KoboldAI/fairseq-dense-6.7B", "16GB"),
|
||||||
["Fairseq Dense 2.7B", "KoboldAI/fairseq-dense-2.7B", "8GB", False],
|
MenuModel("Fairseq Dense 2.7B", "KoboldAI/fairseq-dense-2.7B", "8GB"),
|
||||||
["Fairseq Dense 1.3B", "KoboldAI/fairseq-dense-1.3B", "4GB", False],
|
MenuModel("Fairseq Dense 1.3B", "KoboldAI/fairseq-dense-1.3B", "4GB"),
|
||||||
["Fairseq Dense 355M", "KoboldAI/fairseq-dense-355M", "2GB", False],
|
MenuModel("Fairseq Dense 355M", "KoboldAI/fairseq-dense-355M", "2GB"),
|
||||||
["Fairseq Dense 125M", "KoboldAI/fairseq-dense-125M", "1GB", False],
|
MenuModel("Fairseq Dense 125M", "KoboldAI/fairseq-dense-125M", "1GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
'xglmlist': [
|
'xglmlist': [
|
||||||
["XGLM 4.5B (Larger Dataset)", "facebook/xglm-4.5B", "12GB", False],
|
MenuModel("XGLM 4.5B (Larger Dataset)", "facebook/xglm-4.5B", "12GB"),
|
||||||
["XGLM 7.5B", "facebook/xglm-7.5B", "18GB", False],
|
MenuModel("XGLM 7.5B", "facebook/xglm-7.5B", "18GB"),
|
||||||
["XGLM 2.9B", "facebook/xglm-2.9B", "10GB", False],
|
MenuModel("XGLM 2.9B", "facebook/xglm-2.9B", "10GB"),
|
||||||
["XGLM 1.7B", "facebook/xglm-1.7B", "6GB", False],
|
MenuModel("XGLM 1.7B", "facebook/xglm-1.7B", "6GB"),
|
||||||
["XGLM 564M", "facebook/xglm-564M", "4GB", False],
|
MenuModel("XGLM 564M", "facebook/xglm-564M", "4GB"),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
|
],
|
||||||
|
'rwkvlist': [
|
||||||
|
MenuModel("RWKV-4 14B", "rwkv-4-pile-14b", "??GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuModel("RWKV-4 7B", "rwkv-4-pile-7b", "??GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuModel("RWKV-4 3B", "rwkv-4-pile-3b", "?GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuModel("RWKV-4 1.5B", "rwkv-4-pile-1b5", "9GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuModel("RWKV-4 340M", "rwkv-4-pile-430m", "?GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuModel("RWKV-4 169M", "rwkv-4-pile-169m", "?GB", model_type=MenuModelType.RWKV),
|
||||||
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
],
|
],
|
||||||
# 'rwkvlist': [
|
|
||||||
# ["RWKV-4 7B (GPU)", "RWKV-7B-GPU", "??GB", False],
|
|
||||||
# ["RWKV-4 7B (CPU)", "RWKV-7B-CPU", "??GB", False],
|
|
||||||
# ["RWKV-4 3B (GPU)", "RWKV-3B-GPU", "?GB", False],
|
|
||||||
# ["RWKV-4 3B (CPU)", "RWKV-3B-CPU", "?GB", False],
|
|
||||||
# ["RWKV-4 1.5B (GPU)", "RWKV-1B5-GPU", "9GB", False],
|
|
||||||
# ["RWKV-4 1.5B (CPU)", "RWKV-1B5-CPU", "6GB", False],
|
|
||||||
# ["RWKV-4 340M (GPU)", "RWKV-340M-GPU", "?GB", False],
|
|
||||||
# ["RWKV-4 340M (CPU)", "RWKV-340M-CPU", "?GB", False],
|
|
||||||
# ["RWKV-4 169M (GPU)", "RWKV-169M-GPU", "?GB", False],
|
|
||||||
# ["RWKV-4 169M (CPU)", "RWKV-169M-CPU", "?GB", False],
|
|
||||||
# ["Return to Main Menu", "mainmenu", "", True],
|
|
||||||
# ],
|
|
||||||
'apilist': [
|
'apilist': [
|
||||||
["GooseAI API (requires API key)", "GooseAI", "None", False],
|
MenuModel("GooseAI API (requires API key)", "GooseAI", model_type=MenuModelType.ONLINE_API),
|
||||||
["OpenAI API (requires API key)", "OAI", "None", False],
|
MenuModel("OpenAI API (requires API key)", "OAI", model_type=MenuModelType.ONLINE_API),
|
||||||
["InferKit API (requires API key)", "InferKit", "None", False],
|
MenuModel("InferKit API (requires API key)", "InferKit", model_type=MenuModelType.ONLINE_API),
|
||||||
["KoboldAI API", "API", "None", False],
|
MenuModel("KoboldAI API", "API", model_type=MenuModelType.ONLINE_API),
|
||||||
["Basic Model API", "Colab", "", False],
|
MenuModel("Basic Model API", "Colab", model_type=MenuModelType.ONLINE_API),
|
||||||
["KoboldAI Horde", "CLUSTER", "None", False],
|
MenuModel("KoboldAI Horde", "CLUSTER", model_type=MenuModelType.ONLINE_API),
|
||||||
["Return to Main Menu", "mainmenu", "", True],
|
MenuFolder("Return to Main Menu", "mainmenu"),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -753,24 +825,22 @@ def get_config_filename(model_name = None):
|
|||||||
logger.warning(f"Empty configfile name sent back. Defaulting to ReadOnly")
|
logger.warning(f"Empty configfile name sent back. Defaulting to ReadOnly")
|
||||||
return(f"settings/ReadOnly.settings")
|
return(f"settings/ReadOnly.settings")
|
||||||
|
|
||||||
def is_model_downloaded(model_name: str) -> bool:
|
|
||||||
model_stub = model_name.replace("/", "_")
|
|
||||||
return os.path.isdir(os.path.join("models", model_stub))
|
|
||||||
|
|
||||||
#==================================================================#
|
#==================================================================#
|
||||||
# Function to get model selection at startup
|
# Function to get model selection at startup
|
||||||
#==================================================================#
|
#==================================================================#
|
||||||
def sendModelSelection(menu="mainmenu", folder="./models"):
|
def sendModelSelection(menu="mainmenu", folder="./models"):
|
||||||
#If we send one of the manual load options, send back the list of model directories, otherwise send the menu
|
#If we send one of the manual load options, send back the list of model directories, otherwise send the menu
|
||||||
if menu in ('NeoCustom', 'GPT2Custom'):
|
if menu in ('NeoCustom', 'GPT2Custom'):
|
||||||
(paths, breadcrumbs) = get_folder_path_info(folder)
|
paths, breadcrumbs = get_folder_path_info(folder)
|
||||||
paths = [x for x in paths if "rwkv" not in x[1].lower()]
|
# paths = [x for x in paths if "rwkv" not in x[1].lower()]
|
||||||
if koboldai_vars.host:
|
if koboldai_vars.host:
|
||||||
breadcrumbs = []
|
breadcrumbs = []
|
||||||
|
|
||||||
menu_list = [[folder, menu, "", False] for folder in paths]
|
menu_list = [[folder, menu, "", False] for folder in paths]
|
||||||
menu_list_ui_2 = [[folder[0], folder[1], "", False] for folder in paths]
|
menu_list_ui_2 = [[folder[0], folder[1], "", False] for folder in paths]
|
||||||
menu_list.append(["Return to Main Menu", "mainmenu", "", True])
|
menu_list.append(["Return to Main Menu", "mainmenu", "", True])
|
||||||
menu_list_ui_2.append(["Return to Main Menu", "mainmenu", "", True])
|
menu_list_ui_2.append(["Return to Main Menu", "mainmenu", "", True])
|
||||||
|
|
||||||
if os.path.abspath("{}/models".format(os.getcwd())) == os.path.abspath(folder):
|
if os.path.abspath("{}/models".format(os.getcwd())) == os.path.abspath(folder):
|
||||||
showdelete=True
|
showdelete=True
|
||||||
else:
|
else:
|
||||||
@@ -785,7 +855,7 @@ def sendModelSelection(menu="mainmenu", folder="./models"):
|
|||||||
"isDownloaded": True,
|
"isDownloaded": True,
|
||||||
} for m in menu_list_ui_2]
|
} for m in menu_list_ui_2]
|
||||||
emit('show_model_menu', {'data': p_menu, 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=False)
|
emit('show_model_menu', {'data': p_menu, 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=False)
|
||||||
elif menu in ('customhuggingface'):
|
elif menu == "customhuggingface":
|
||||||
p_menu = [{
|
p_menu = [{
|
||||||
"label": "Return to Main Menu",
|
"label": "Return to Main Menu",
|
||||||
"name": "mainmenu",
|
"name": "mainmenu",
|
||||||
@@ -798,18 +868,31 @@ def sendModelSelection(menu="mainmenu", folder="./models"):
|
|||||||
emit('from_server', {'cmd': 'show_model_menu', 'data': [["Return to Main Menu", "mainmenu", "", True]], 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=True, room="UI_1")
|
emit('from_server', {'cmd': 'show_model_menu', 'data': [["Return to Main Menu", "mainmenu", "", True]], 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=True, room="UI_1")
|
||||||
emit('show_model_menu', {'data': p_menu, 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=False)
|
emit('show_model_menu', {'data': p_menu, 'menu': menu, 'breadcrumbs': breadcrumbs, "showdelete": showdelete}, broadcast=False)
|
||||||
else:
|
else:
|
||||||
# Hide experimental models unless experimental mode is enabled
|
filtered_menu = [item for item in model_menu[menu] if item.should_show()]
|
||||||
filtered_menu = [x for x in model_menu[menu] if koboldai_vars.experimental_features or "(experimental)" not in x[0].lower()]
|
|
||||||
emit('from_server', {'cmd': 'show_model_menu', 'data': filtered_menu, 'menu': menu, 'breadcrumbs': [], "showdelete": False}, broadcast=True, room="UI_1")
|
|
||||||
|
|
||||||
p_menu = [{
|
emit(
|
||||||
"label": m[0],
|
"from_server",
|
||||||
"name": m[1],
|
{
|
||||||
"size": m[2],
|
"cmd": "show_model_menu",
|
||||||
"isMenu": m[3],
|
"data": [item.to_ui1() for item in filtered_menu],
|
||||||
"isDownloaded": is_model_downloaded(m[1]) if not m[3] else False,
|
"menu": menu,
|
||||||
} for m in filtered_menu]
|
"breadcrumbs": [],
|
||||||
emit('show_model_menu', {'data': p_menu, 'menu': menu, 'breadcrumbs': [], "showdelete": False}, broadcast=False)
|
"showdelete": False
|
||||||
|
},
|
||||||
|
broadcast=True,
|
||||||
|
room="UI_1"
|
||||||
|
)
|
||||||
|
|
||||||
|
emit(
|
||||||
|
"show_model_menu",
|
||||||
|
{
|
||||||
|
"data": [item.to_json() for item in filtered_menu],
|
||||||
|
"menu": menu,
|
||||||
|
"breadcrumbs": [],
|
||||||
|
"showdelete": False
|
||||||
|
},
|
||||||
|
broadcast=False
|
||||||
|
)
|
||||||
|
|
||||||
def get_folder_path_info(base):
|
def get_folder_path_info(base):
|
||||||
if base == 'This PC':
|
if base == 'This PC':
|
||||||
@@ -1406,7 +1489,7 @@ def get_model_info(model, directory=""):
|
|||||||
url = koboldai_vars.horde_url
|
url = koboldai_vars.horde_url
|
||||||
if key_value:
|
if key_value:
|
||||||
send_horde_models = True
|
send_horde_models = True
|
||||||
elif model in [x[1] for x in model_menu['apilist']]:
|
elif model in [x.name for x in model_menu['apilist']]:
|
||||||
show_online_model_select=True
|
show_online_model_select=True
|
||||||
if path.exists("settings/{}.v2_settings".format(model)):
|
if path.exists("settings/{}.v2_settings".format(model)):
|
||||||
with open("settings/{}.v2_settings".format(model), "r") as file:
|
with open("settings/{}.v2_settings".format(model), "r") as file:
|
||||||
@@ -1425,7 +1508,7 @@ def get_model_info(model, directory=""):
|
|||||||
print(":(")
|
print(":(")
|
||||||
pass
|
pass
|
||||||
key = True
|
key = True
|
||||||
elif model.startswith("RWKV"):
|
elif "rwkv" in model.lower():
|
||||||
pass
|
pass
|
||||||
elif model == 'ReadOnly':
|
elif model == 'ReadOnly':
|
||||||
pass
|
pass
|
||||||
@@ -1470,7 +1553,7 @@ def get_model_info(model, directory=""):
|
|||||||
'show_custom_model_box': show_custom_model_box})
|
'show_custom_model_box': show_custom_model_box})
|
||||||
if send_horde_models:
|
if send_horde_models:
|
||||||
get_cluster_models({'key': key_value, 'url': default_url})
|
get_cluster_models({'key': key_value, 'url': default_url})
|
||||||
elif key_value != "" and model in [x[1] for x in model_menu['apilist']] and model != 'CLUSTER':
|
elif key_value != "" and model in [x.name for x in model_menu['apilist']] and model != 'CLUSTER':
|
||||||
get_oai_models(key_value)
|
get_oai_models(key_value)
|
||||||
|
|
||||||
|
|
||||||
@@ -1802,6 +1885,10 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal
|
|||||||
koboldai_vars.usegpu = False
|
koboldai_vars.usegpu = False
|
||||||
koboldai_vars.breakmodel = False
|
koboldai_vars.breakmodel = False
|
||||||
model.load(initial_load=initial_load)
|
model.load(initial_load=initial_load)
|
||||||
|
elif koboldai_vars.model.startswith("rwkv:"):
|
||||||
|
if koboldai_vars.use_colab_tpu:
|
||||||
|
raise RuntimeError("RWKV is not supported on the TPU.")
|
||||||
|
print("Trying to load", koboldai_vars.model)
|
||||||
elif not koboldai_vars.use_colab_tpu and not koboldai_vars.noai:
|
elif not koboldai_vars.use_colab_tpu and not koboldai_vars.noai:
|
||||||
# HF Torch
|
# HF Torch
|
||||||
logger.init("Transformers", status='Starting')
|
logger.init("Transformers", status='Starting')
|
||||||
|
|||||||
Reference in New Issue
Block a user