Update src/transformers/models/llama/tokenization_llama.py

Co-authored-by: amyeroberts <[email protected]>

src/transformers/models/llama/tokenization_llama.py

@@ -209,10 +209,11 @@ def _tokenize(self, text):
         the extra `SPIECE_UNDERLINE` prepended.
         """
         if not self.legacy:
-            text = self.unk_token + text
-            tokens = self.sp_model.encode(text, out_type=str)
-            return tokens[self.unk_token_length :]
-        return self.sp_model.encode(text, out_type=str)
+        if self.legacy:
+            return self.sp_model.encode(text, out_type=str) 
+        text = self.unk_token + text
+        tokens = self.sp_model.encode(text, out_type=str)
+        return tokens[self.unk_token_length :]
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""