Make logit bias work correctly(?) when prob is -inf

samplers'll do that to you

though now i am curious: what kind of effect would running the bias
before the samplers have? maybe a future option
This commit is contained in:
somebody
2023-07-21 18:33:35 -05:00
parent 418f341560
commit 5f4216730e
2 changed files with 11 additions and 12 deletions

View File

@@ -225,9 +225,6 @@ class HFTorchInferenceModel(HFInferenceModel):
)
class KoboldLogitsWarperList(LogitsProcessorList):
def __init__(self):
pass
def __call__(
lw_self,
input_ids: torch.LongTensor,
@@ -244,16 +241,10 @@ class HFTorchInferenceModel(HFInferenceModel):
), f"Scores are None; processor '{processor}' is to blame"
return scores
def new_get_logits_warper(
beams: int = 1,
) -> LogitsProcessorList:
return KoboldLogitsWarperList()
def new_sample(self, *args, **kwargs):
assert kwargs.pop("logits_warper", None) is not None
kwargs["logits_warper"] = new_get_logits_warper(
beams=1,
)
kwargs["logits_warper"] = KoboldLogitsWarperList()
if utils.koboldai_vars.newlinemode in ["s", "ns"]:
kwargs["eos_token_id"] = -1
kwargs.setdefault("pad_token_id", 2)