rixty
/
KoboldAI-Client
mirror of https://github.com/KoboldAI/KoboldAI-Client.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Automatically use breakmodel instead of GPU-only where supported 

There's really no reason to use GPU-only mode if breakmodel is supported
because breakmodel can run in GPU-only mode too.
This commit is contained in:
Gnome Ann 2021-10-05 10:36:51 -04:00
parent fb90a7ed17
commit a1e4405aa6
1 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
aiserver.py
View File

@ -314,12 +314,12 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly"]):
genselected = True genselected = True
vars.usegpu = True vars.usegpu = True
vars.breakmodel = False vars.breakmodel = False
if(vars.bmsupported):
vars.usegpu = False
vars.breakmodel = True
if(args.cpu): if(args.cpu):
vars.usegpu = False vars.usegpu = False
vars.breakmodel = False vars.breakmodel = False
if(vars.bmsupported and args.breakmodel):
vars.usegpu = False
vars.breakmodel = True
elif(vars.hascuda): elif(vars.hascuda):
if(vars.bmsupported): if(vars.bmsupported):
print(colors.YELLOW + "You're using a model that supports hybrid generation!") print(colors.YELLOW + "You're using a model that supports hybrid generation!")
@ -327,9 +327,8 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly"]):
print("(slower than GPU-only but uses less VRAM) or between multiple GPUs") print("(slower than GPU-only but uses less VRAM) or between multiple GPUs")
print("(allowing you to use the combined VRAM of all your GPUs).") print("(allowing you to use the combined VRAM of all your GPUs).")
print("Currently only GPT-Neo and GPT-J models support this feature.") print("Currently only GPT-Neo and GPT-J models support this feature.")
print("{0}Use GPU or CPU for generation?: (Default GPU){1}".format(colors.CYAN, colors.END)) print("{0}Use hybrid generation or CPU-only generation?: (Default hybrid){1}".format(colors.CYAN, colors.END))
if(vars.bmsupported): print(f" 1 - Hybrid generation\n 2 - CPU\n")
print(f" 1 - GPU\n 2 - CPU\n 3 - Hybrid generation\n")
else: else:
print(" 1 - GPU\n 2 - CPU\n") print(" 1 - GPU\n 2 - CPU\n")
genselected = False genselected = False
@ -342,17 +341,18 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly"]):
vars.usegpu = True vars.usegpu = True
genselected = True genselected = True
elif(genselect.isnumeric() and int(genselect) == 1): elif(genselect.isnumeric() and int(genselect) == 1):
vars.breakmodel = False if(vars.bmsupported):
vars.usegpu = True vars.breakmodel = True
genselected = True vars.usegpu = False
genselected = True
else:
vars.breakmodel = False
vars.usegpu = True
genselected = True
elif(genselect.isnumeric() and int(genselect) == 2): elif(genselect.isnumeric() and int(genselect) == 2):
vars.breakmodel = False vars.breakmodel = False
vars.usegpu = False vars.usegpu = False
genselected = True genselected = True
elif(vars.bmsupported and genselect.isnumeric() and int(genselect) == 3):
vars.breakmodel = True
vars.usegpu = False
genselected = True
else: else:
print("{0}Please enter a valid selection.{1}".format(colors.RED, colors.END)) print("{0}Please enter a valid selection.{1}".format(colors.RED, colors.END))