mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-06-05 21:59:24 +02:00
In breakmodel mode, move layers to GPU as soon as model loads
Rather than during the first generation.
This commit is contained in:
@ -303,6 +303,7 @@ def device_config(model):
|
||||
gc.collect()
|
||||
GPTNeoModel.forward = breakmodel.new_forward
|
||||
generator = model.generate
|
||||
breakmodel.move_hidden_layers(model.transformer)
|
||||
|
||||
#==================================================================#
|
||||
# Startup
|
||||
|
Reference in New Issue
Block a user