rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-02-17 12:10:49 +01:00
Code Issues Projects Releases Wiki Activity

Fix for --cpu on command line and MAYBE --nobreakmodel

Browse Source
This commit is contained in:
ebolam 2022-08-11 15:23:35 -04:00
parent 64664dc61e
commit 45495d8792
1 changed files with 18 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
aiserver.py
View File

@ -543,7 +543,9 @@ def device_config(config):
global breakmodel, generator global breakmodel, generator
import breakmodel import breakmodel
n_layers = utils.num_layers(config) n_layers = utils.num_layers(config)
if(args.breakmodel_gpulayers is not None or (utils.HAS_ACCELERATE and args.breakmodel_disklayers is not None)): if args.cpu:
breakmodel.gpu_blocks = [0]*n_layers
elif(args.breakmodel_gpulayers is not None or (utils.HAS_ACCELERATE and args.breakmodel_disklayers is not None)):
try: try:
if(not args.breakmodel_gpulayers): if(not args.breakmodel_gpulayers):
breakmodel.gpu_blocks = [] breakmodel.gpu_blocks = []
@ -1730,8 +1732,12 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal
time.sleep(0.1) time.sleep(0.1)
if gpu_layers is not None: if gpu_layers is not None:
args.breakmodel_gpulayers = gpu_layers args.breakmodel_gpulayers = gpu_layers
elif initial_load:
gpu_layers = args.breakmodel_gpulayers
if disk_layers is not None: if disk_layers is not None:
args.breakmodel_disklayers = int(disk_layers) args.breakmodel_disklayers = int(disk_layers)
elif initial_load:
disk_layers = args.breakmodel_disklayers
#We need to wipe out the existing model and refresh the cuda cache #We need to wipe out the existing model and refresh the cuda cache
model = None model = None
@ -1840,41 +1846,19 @@ def load_model(use_gpu=True, gpu_layers=None, disk_layers=None, initial_load=Fal
else: else:
print("{0}NOT FOUND!{1}".format(colors.YELLOW, colors.END)) print("{0}NOT FOUND!{1}".format(colors.YELLOW, colors.END))
if args.model: if args.cpu:
if(vars.hascuda): vars.usegpu = False
genselected = True gpu_layers = None
disk_layers = None
vars.breakmodel = False
elif vars.hascuda:
if(vars.bmsupported):
vars.usegpu = False
vars.breakmodel = True
else:
vars.breakmodel = False
vars.usegpu = True vars.usegpu = True
vars.breakmodel = utils.HAS_ACCELERATE
if(vars.bmsupported):
vars.usegpu = False
vars.breakmodel = True
if(args.cpu):
vars.usegpu = False
vars.breakmodel = utils.HAS_ACCELERATE
elif(vars.hascuda):
if(vars.bmsupported):
genselected = True
vars.usegpu = False
vars.breakmodel = True
else:
genselected = False
else:
genselected = False
if(vars.hascuda):
if(use_gpu):
if(vars.bmsupported):
vars.breakmodel = True
vars.usegpu = False
genselected = True
else:
vars.breakmodel = False
vars.usegpu = True
genselected = True
else:
vars.breakmodel = utils.HAS_ACCELERATE
vars.usegpu = False
genselected = True
# Ask for API key if InferKit was selected # Ask for API key if InferKit was selected
if(vars.model == "InferKit"): if(vars.model == "InferKit"):