diff --git a/model/tokenizer.py b/model/tokenizer.py index 449e8fc..d4051aa 100644 --- a/model/tokenizer.py +++ b/model/tokenizer.py @@ -355,7 +355,11 @@ def merge_vocab(pair: Tuple[str, str], v_in: Dict[str, int]) -> Dict[str, int]: p = re.compile(r'(?