From b256a8fbc7f64d1f917c83e2294d67ba321844b5 Mon Sep 17 00:00:00 2001 From: somebody Date: Thu, 27 Apr 2023 19:33:03 -0500 Subject: [PATCH] Debug --- modeling/stoppers.py | 1 + modeling/tokenizer.py | 1 + 2 files changed, 2 insertions(+) diff --git a/modeling/stoppers.py b/modeling/stoppers.py index 94c09e85..14cfe752 100644 --- a/modeling/stoppers.py +++ b/modeling/stoppers.py @@ -122,6 +122,7 @@ class Stoppers: input_ids: torch.LongTensor, ) -> bool: + print(f"[stop_sequence_stopper] Input ids: {input_ids}") data = [model.tokenizer.decode(x) for x in input_ids] # null_character = model.tokenizer.encode(chr(0))[0] if "completed" not in model.gen_state: diff --git a/modeling/tokenizer.py b/modeling/tokenizer.py index 99e0fbe0..cd35758b 100644 --- a/modeling/tokenizer.py +++ b/modeling/tokenizer.py @@ -28,6 +28,7 @@ class GenericTokenizer: return ret.ids def decode(self, tokens: Union[int, List[int], torch.Tensor]) -> str: + print(f"[decode] Tokens: {tokens}") if isinstance(tokens, torch.Tensor): tokens = tokens.cpu().tolist()