🚨🚨🚨 [SPM] Finish fix spm models 🚨🚨🚨

src/transformers/models/llama/tokenization_llama.py

@@ -24,7 +24,9 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 import sentencepiece as spm
+from sentencepiece import SentencePieceProcessor
 
+from ...convert_slow_tokenizer import import_protobuf
 from ...tokenization_utils import AddedToken, PreTrainedTokenizer
 from ...utils import logging
 
@@ -70,9 +72,10 @@ class LlamaTokenizer(PreTrainedTokenizer):
     Args:
         vocab_file (`str`):
             Path to the vocabulary file.
-        legacy (`bool`, *optional*, defaults to `True`):
+        legacy (`bool`, *optional*):
             Whether or not the `legacy` behaviour of the tokenizer should be used. Legacy is before the merge of #24622
-            which includes fixes to properly handle tokens that appear after special tokens. A simple example:
+            and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple
+            example:
 
             - `legacy=True`:
             ```python
@@ -90,7 +93,7 @@ class LlamaTokenizer(PreTrainedTokenizer):
             >>> tokenizer.encode("Hello <extra_id_0>.")  # the extra space `[3]` is no longer here
             [8774, 32099, 5, 1]
             ```
-            Checkout the pull request and the issue [here](https://github.com/huggingface/transformers/pull/24565) for
+            Checkout the pull request and the issue [here](https://github.com/huggingface/transformers/pull/25224) for
             more details.
 
     """
@@ -111,6 +114,7 @@ def __init__(
         add_bos_token=True,
         add_eos_token=False,
         clean_up_tokenization_spaces=False,
+        spaces_between_special_tokens=False,
         legacy=None,
         **kwargs,
     ):
@@ -128,6 +132,7 @@ def __init__(
             add_eos_token=add_eos_token,
             sp_model_kwargs=self.sp_model_kwargs,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
             legacy=legacy,
             **kwargs,
         )
@@ -142,8 +147,24 @@ def __init__(
         self.vocab_file = vocab_file
         self.add_bos_token = add_bos_token
         self.add_eos_token = add_eos_token
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
+        self.sp_model = self.get_spm_processor()
+
+        self.unk_token_length = len(self.sp_model.encode(str(self.unk_token)))
+
+    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor
+    def get_spm_processor(self):
+        tokenizer = SentencePieceProcessor(**self.sp_model_kwargs)
+        with open(self.vocab_file, "rb") as f:
+            sp_model = f.read()
+            model_pb2 = import_protobuf()
+            model = model_pb2.ModelProto.FromString(sp_model)
+            if not self.legacy:
+                normalizer_spec = model_pb2.NormalizerSpec()
+                normalizer_spec.add_dummy_prefix = False
+                model.normalizer_spec.MergeFrom(normalizer_spec)
+            sp_model = model.SerializeToString()
+            tokenizer.LoadFromSerializedProto(sp_model)
+        return tokenizer
 
     def __getstate__(self):
         state = self.__dict__.copy()
@@ -172,6 +193,7 @@ def tokenize(self, text, **kwargs) -> List[str]:
         # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
         # the beginning of the text
         if not self.legacy:
+            # replacing " " by SPIECE_UNDERLINE prevents any form of stripping...
             text = SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " ")
         return super().tokenize(text, **kwargs)
 
@@ -187,15 +209,10 @@ def _tokenize(self, text):
         the extra `SPIECE_UNDERLINE` prepended.
         """
         if not self.legacy:
-            is_first = text.startswith(SPIECE_UNDERLINE)
-            if is_first:
-                text = text[1:]
-
-        tokens = self.sp_model.encode(text, out_type=str)
-
-        if not self.legacy and not is_first and not text.startswith(" ") and tokens[0].startswith(SPIECE_UNDERLINE):
-            tokens = ([tokens[0][1:]] if len(tokens[0]) > 1 else []) + tokens[1:]
-        return tokens
+            text = self.unk_token + text
+            tokens = self.sp_model.encode(text, out_type=str)
+            return tokens[self.unk_token_length :]
+        return self.sp_model.encode(text, out_type=str)
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
@@ -209,6 +226,8 @@ def _convert_id_to_token(self, index):
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
         current_sub_tokens = []
+        # since we manually add the prefix space, we have to remove it
+        tokens[0] = tokens[0].strip(SPIECE_UNDERLINE)
         out_string = ""
         prev_is_special = False
         for i, token in enumerate(tokens):

src/transformers/models/t5/tokenization_t5.py

@@ -22,7 +22,9 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 import sentencepiece as spm
+from sentencepiece import SentencePieceProcessor
 
+from ...convert_slow_tokenizer import import_protobuf
 from ...tokenization_utils import PreTrainedTokenizer
 
 
@@ -106,9 +108,10 @@ class T5Tokenizer(PreTrainedTokenizer):
 
             - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
               BPE-dropout.
-        legacy (`bool`, *optional*, defaults to `True`):
+        legacy (`bool`, *optional*):
             Whether or not the `legacy` behaviour of the tokenizer should be used. Legacy is before the merge of #24622
-            which includes fixes to properly handle tokens that appear after special tokens. A simple example:
+            and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple
+            example:
 
             - `legacy=True`:
             ```python
@@ -126,7 +129,7 @@ class T5Tokenizer(PreTrainedTokenizer):
             >>> tokenizer.encode("Hello <extra_id_0>.")  # the extra space `[3]` is no longer here
             [8774, 32099, 5, 1]
             ```
-            Checkout the pull request and the issue [here](https://github.com/huggingface/transformers/pull/24565) for
+            Checkout the pull request and the issue [here](https://github.com/huggingface/transformers/pull/25224) for
             more details.
 
     Attributes:
@@ -187,8 +190,23 @@ def __init__(
         self.vocab_file = vocab_file
         self._extra_ids = extra_ids
 
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
+        self.sp_model = self.get_spm_processor()
+
+        self.unk_token_length = len(self.sp_model.encode(str(self.unk_token)))
+
+    def get_spm_processor(self):
+        tokenizer = SentencePieceProcessor(**self.sp_model_kwargs)
+        with open(self.vocab_file, "rb") as f:
+            sp_model = f.read()
+            model_pb2 = import_protobuf()
+            model = model_pb2.ModelProto.FromString(sp_model)
+            if not self.legacy:
+                normalizer_spec = model_pb2.NormalizerSpec()
+                normalizer_spec.add_dummy_prefix = False
+                model.normalizer_spec.MergeFrom(normalizer_spec)
+            sp_model = model.SerializeToString()
+            tokenizer.LoadFromSerializedProto(sp_model)
+        return tokenizer
 
     @staticmethod
     def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
@@ -335,6 +353,7 @@ def tokenize(self, text: "TextInput", **kwargs) -> List[str]:
         # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
         # the beginning of the text
         if not self.legacy:
+            # replacing " " by SPIECE_UNDERLINE prevents any form of stripping...
             text = SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " ")
         return super().tokenize(text, **kwargs)
 
@@ -349,15 +368,10 @@ def _tokenize(self, text, **kwargs):
         the extra `SPIECE_UNDERLINE` prepended.
         """
         if not self.legacy:
-            is_first = text.startswith(SPIECE_UNDERLINE)
-            if is_first:
-                text = text[1:]
-
-        tokens = self.sp_model.encode(text, out_type=str)
-
-        if not self.legacy and not is_first and not text.startswith(" ") and tokens[0].startswith(SPIECE_UNDERLINE):
-            tokens = ([tokens[0][1:]] if len(tokens[0]) > 1 else []) + tokens[1:]
-        return tokens
+            text = self.unk_token + text
+            tokens = self.sp_model.encode(text, out_type=str)
+            return tokens[self.unk_token_length :]
+        return self.sp_model.encode(text, out_type=str)
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
@@ -378,6 +392,8 @@ def _convert_id_to_token(self, index):
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
         current_sub_tokens = []
+        # since we manually add the prefix space, we have to remove it
+        tokens[0] = tokens[0].strip(SPIECE_UNDERLINE)
         out_string = ""
         prev_is_special = False
         for token in tokens:

tests/models/llama/test_tokenization_llama.py

@@ -499,6 +499,27 @@ def test_integration_test_xnli(self):
 
                 self.assertEqual(decoded1, decoded2)
 
+    def test_special_token_special_word(self):
+        # the word inform should be split as ['in', 'form']
+        tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", legacy=False)
+        tokenizer.add_tokens(["<REPR_END>"], special_tokens=True)
+        out1 = tokenizer.decode(
+            tokenizer.encode("<REPR_END>inform", add_special_tokens=False), spaces_between_special_tokens=False
+        )
+        self.assertEquals(out1, "<REPR_END>inform")
+        out2 = tokenizer.decode(
+            tokenizer.encode("<REPR_END>inform", add_special_tokens=False), spaces_between_special_tokens=True
+        )
+        self.assertEquals(out2, " <REPR_END> inform")
+        input_ids = tokenizer.encode("<REPR_END>inform", add_special_tokens=False)
+        self.assertEquals(input_ids, [29871, 32000, 262, 689])  # 29871 is the spiece underline, '▁'
+
+        out2 = tokenizer.decode(
+            tokenizer.encode(" <REPR_END> inform", add_special_tokens=False), spaces_between_special_tokens=False
+        )
+        # TODO ArthurZ currently we strip left and right, so this will not keep the spaces
+        self.assertEquals(out2, "<REPR_END>inform")
+
 
 @require_sentencepiece
 @require_tokenizers
@@ -523,7 +544,7 @@ def test_add_dummy_prefix(self):
         input_ids = self.tokenizer.encode(". Hello")
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode(". Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(". Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
@@ -534,15 +555,19 @@ def test_remove_extra_whitespaces(self):
         input_ids = self.tokenizer.encode("       . Hello")
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode("       . Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(" . Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
         # `'▁'` is also a whitespace
         input_ids = self.tokenizer.encode("▁He is not")
         self.assertEqual(input_ids, [156, 46, 44])
         tokens = self.tokenizer.tokenize("▁He is not")
-        sp_encode = self.tokenizer.sp_model.encode("▁He is not")
+        sp_encode = [
+            self.tokenizer.sp_model.piece_to_id("▁He"),
+            self.tokenizer.sp_model.piece_to_id("▁is"),
+            self.tokenizer.sp_model.piece_to_id("▁not"),
+        ]
         self.assertEqual(input_ids, sp_encode)
         self.assertEqual(tokens, ["▁He", "▁is", "▁not"])  # no extra space added
 

tests/models/t5/test_tokenization_t5.py

@@ -410,9 +410,11 @@ class CommonSpmIntegrationTests(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=0, legacy=False)
-        tokenizer.add_special_tokens({"additional_special_tokens": ["<extra_id_0>"]})
+        tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=1, legacy=False)
+        # tokenizer.add_tokens("<extra_id_0>", special_tokens = True)
+        # tokenizer._additional_special_tokens = ["<extra_id_0>"]
         tokenizer._create_trie(tokenizer.all_special_tokens)
+        tokenizer.unique_no_split_tokens = ["<extra_id_0>"]
         # TODO ArthurZ the above is necessary as addedTokens / intialization sucks. Trie is not correctly created
         # So the extra ids are split....
         cls.tokenizer = tokenizer
@@ -423,7 +425,7 @@ def test_add_dummy_prefix(self):
         input_ids = self.tokenizer.encode(". Hello", add_special_tokens=False)
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode(". Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(". Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
@@ -433,7 +435,7 @@ def test_remove_extra_whitespaces(self):
         input_ids = self.tokenizer.encode("       . Hello", add_special_tokens=False)
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode("       . Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(" . Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
@@ -444,12 +446,13 @@ def test_remove_extra_whitespaces(self):
         self.assertEqual(tokens, ["▁He", "▁is", "▁not"])  # no extra space added
 
         input_ids = self.tokenizer.encode("▁He is not<extra_id_0>             ▁He")
-        # here t5x does not eat with lstrip, so there is and extra ▁He in the original one
-        # TODO @arthurzucker we should probably not srip right since it is done by default
-        # for certain models...
-        self.assertEqual(input_ids, [156, 46, 44, 999, 0, 2])
+        # TODO another example of lstrip
+        self.assertEqual(input_ids, [156, 46, 44, 1000, 262, 15, 2])
+
         tokens = self.tokenizer.tokenize("▁He is not<extra_id_0>              ▁He")
-        self.assertEqual(tokens, ["▁He", "▁is", "▁not", "<extra_id_0>", "He"])  # spaces are eaten by spm + our strip
+        self.assertEqual(
+            tokens, ["▁He", "▁is", "▁not", "<extra_id_0>", "H", "e"]
+        )  # spaces are eaten by spm + our strip
         # make sure that the output after the extra id is the same as if
         # extra_id was not there
         input_ids = self.tokenizer.encode("▁He is not             ▁He")
@@ -461,28 +464,28 @@ def test_character_after_special_token(self):
         # Make sure that `tokenizer.tokenize` is similar to
         # adding the equivalent special token to the vocab
         input_ids = self.tokenizer.encode("Hey <extra_id_0>I")
-        self.assertEqual(input_ids, [156, 30, 999, 100, 2])
+        self.assertEqual(input_ids, [156, 30, 1000, 100, 2])
         tokens = self.tokenizer.tokenize("Hey <extra_id_0>I")
         self.assertEqual(tokens, ["▁He", "y", "<extra_id_0>", "I"])
 
         input_ids = self.tokenizer.encode("Hello, <extra_id_0>,")
-        self.assertEqual(input_ids, [156, 86, 20, 3, 999, 3, 2])
+        self.assertEqual(input_ids, [156, 86, 20, 3, 1000, 3, 2])
         tokens = self.tokenizer.tokenize("Hello, <extra_id_0>,")
         self.assertEqual(tokens, ["▁He", "ll", "o", ",", "<extra_id_0>", ","])
 
     def test_special_tokens_strip(self):
         input_ids = self.tokenizer.encode(" <extra_id_0> ,")
-        self.assertEqual(input_ids, [999, 3, 2])
+        self.assertEqual(input_ids, [1000, 3, 2])
         tokens = self.tokenizer.tokenize(" <extra_id_0> ,")
         # spaces are eaten by rstrip / lstrip
         self.assertEqual(tokens, ["<extra_id_0>", ","])
 
         # test with a begin of word like `▁He`
         input_ids = self.tokenizer.encode("No <extra_id_0> He")
-        self.assertEqual(input_ids, [284, 999, 0, 2])
+        self.assertEqual(input_ids, [284, 1000, 262, 15, 2])
         # spaces are eaten by rstrip / lstrip, so this is expected. Don't strip otherwise you break
         tokens = self.tokenizer.tokenize("No <extra_id_0> He")
-        self.assertEqual(tokens, ["▁No", "<extra_id_0>", "He"])
+        self.assertEqual(tokens, ["▁No", "<extra_id_0>", "H", "e"])
 
         # Make sure this does not happen if we don't strip
         tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=0)