diff --git a/modeling/stoppers.py b/modeling/stoppers.py index 94c09e85..14cfe752 100644 --- a/modeling/stoppers.py +++ b/modeling/stoppers.py @@ -122,6 +122,7 @@ class Stoppers: input_ids: torch.LongTensor, ) -> bool: + print(f"[stop_sequence_stopper] Input ids: {input_ids}") data = [model.tokenizer.decode(x) for x in input_ids] # null_character = model.tokenizer.encode(chr(0))[0] if "completed" not in model.gen_state: diff --git a/modeling/tokenizer.py b/modeling/tokenizer.py index 99e0fbe0..cd35758b 100644 --- a/modeling/tokenizer.py +++ b/modeling/tokenizer.py @@ -28,6 +28,7 @@ class GenericTokenizer: return ret.ids def decode(self, tokens: Union[int, List[int], torch.Tensor]) -> str: + print(f"[decode] Tokens: {tokens}") if isinstance(tokens, torch.Tensor): tokens = tokens.cpu().tolist()