⚗️⚗️ Tokenizer magic ⚗️⚗️

src/transformers/models/llama/tokenization_llama.py

@@ -147,7 +147,9 @@ def __init__(
         self.add_bos_token = add_bos_token
         self.add_eos_token = add_eos_token
         self.sp_model = self.get_spm_processor()
-
+
+        self.unk_token_length = len(self.sp_model.encode(str(self.unk_token)))
+
     def get_spm_processor(self):
         tokenizer = SentencePieceProcessor(**self.sp_model_kwargs)
         with open(self.vocab_file, "rb") as f:
@@ -203,8 +205,11 @@ def _tokenize(self, text):
         passed to `_tokenize`. Thus if a subsequence did not start with a `" "` or SPIECE_UNDERLINE, we have to remove
         the extra `SPIECE_UNDERLINE` prepended.
         """
-        tokens = self.sp_model.encode(text, out_type=str)
-        return tokens
+        if not self.legacy:
+            text = self.unk_token + text
+            tokens = self.sp_model.encode(text, out_type=str)
+            return tokens[self.unk_token_length:]
+        return self.sp_model.encode(text, out_type=str)
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""

tests/models/llama/test_tokenization_llama.py

@@ -505,10 +505,14 @@ def test_special_token_special_word(self):
         tokenizer.add_tokens(['<REPR_END>'], special_tokens=True)
         out1 = tokenizer.decode(tokenizer.encode("<REPR_END>inform", add_special_tokens = False), spaces_between_special_tokens = False)
         self.assertEquals(out1, "<REPR_END>inform")
-        tokenizer.decode(tokenizer.encode("<REPR_END>inform", add_special_tokens = False), spaces_between_special_tokens = True)
-        self.assertEquals(out1, " <REPR_END> inform")
-        input_ids = tokenizer("<REPR_END>inform", add_special_tokens = False)
-        self.assertEquals(input_ids,[29871, 32003, 262, 689] ) # 29871 is the spiece underline, '▁'
+        out2 = tokenizer.decode(tokenizer.encode("<REPR_END>inform", add_special_tokens = False), spaces_between_special_tokens = True)
+        self.assertEquals(out2, " <REPR_END> inform")
+        input_ids = tokenizer.encode("<REPR_END>inform", add_special_tokens = False)
+        self.assertEquals(input_ids,[29871, 32000, 262, 689] ) # 29871 is the spiece underline, '▁'
+
+        out2 = tokenizer.decode(tokenizer.encode(" <REPR_END> inform", add_special_tokens = False), spaces_between_special_tokens = False)
+        # TODO ArthurZ currently we strip left and right, so this will not keep the spaces
+        self.assertEquals(out2, " <REPR_END> inform")
 
 
 @require_sentencepiece
@@ -534,7 +538,7 @@ def test_add_dummy_prefix(self):
         input_ids = self.tokenizer.encode(". Hello")
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode(". Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(". Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
@@ -545,15 +549,15 @@ def test_remove_extra_whitespaces(self):
         input_ids = self.tokenizer.encode("       . Hello")
         self.assertEqual(input_ids, [7, 4, 156, 86, 20])
         sp_encode = self.tokenizer.sp_model.encode("       . Hello")
-        self.assertEqual(input_ids, sp_encode)
+        self.assertEqual(input_ids, [7] + sp_encode)
         tokens = self.tokenizer.tokenize(" . Hello")
         self.assertEqual(tokens, ["▁", ".", "▁He", "ll", "o"])
 
         # `'▁'` is also a whitespace
         input_ids = self.tokenizer.encode("▁He is not")
         self.assertEqual(input_ids, [156, 46, 44])
         tokens = self.tokenizer.tokenize("▁He is not")
-        sp_encode = self.tokenizer.sp_model.encode("▁He is not")
+        sp_encode = [self.tokenizer.sp_model.piece_to_id("▁He"), self.tokenizer.sp_model.piece_to_id("▁is"), self.tokenizer.sp_model.piece_to_id("▁not")]
         self.assertEqual(input_ids, sp_encode)
         self.assertEqual(tokens, ["▁He", "▁is", "▁not"])  # no extra space added