rixty
/
KoboldAI-Client
mirror of https://github.com/KoboldAI/KoboldAI-Client.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Fix for non-rotary models without "rotary" in config.json

This commit is contained in:
Gnome Ann 2021-08-20 13:00:53 -04:00
parent 56c9dc2c04
commit 8bfcf86a8b
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
aiserver.py
View File

@ -386,7 +386,7 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly"]):
model.lm_head.to(breakmodel.gpu_device)
model.transformer.wte.to(breakmodel.gpu_device)
model.transformer.ln_f.to(breakmodel.gpu_device)
if(not model.config.rotary):
if(not hasattr(model.config, 'rotary') or not model.config.rotary):
model.transformer.wpe.to(breakmodel.gpu_device)
gc.collect()
if(vars.bmsupported and args.breakmodel):
@ -436,7 +436,7 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly"]):
model.lm_head.to(breakmodel.gpu_device)
model.transformer.wte.to(breakmodel.gpu_device)
model.transformer.ln_f.to(breakmodel.gpu_device)
if(not model.config.rotary):
if(not hasattr(model.config, 'rotary') or not model.config.rotary):
model.transformer.wpe.to(breakmodel.gpu_device)
gc.collect()
if(vars.bmsupported and args.breakmodel):

2
breakmodel.py
View File

@ -378,7 +378,7 @@ def new_forward(
inputs_embeds[:, pos:pos+emb.shape[1]] = emb
offset += emb.shape[1]
if self.rotary:
if hasattr(self, 'rotary') and self.rotary:
hidden_states = inputs_embeds
else:
position_embeds = self.wpe(position_ids)