Fix mistral and electra tokenizer to match kaggle changes (#1387)

We are changing all tokenizer to store vocabularies via assets (and
not in the config). This requires some changes to tokenizer so files
state can be set after object creation.

Author: mattdangerw

keras_nlp/models/electra/electra_tokenizer.py

@@ -58,22 +58,31 @@ class ElectraTokenizer(WordPieceTokenizer):
     """
 
     def __init__(self, vocabulary, lowercase=False, **kwargs):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "[PAD]"
+        self.mask_token = "[MASK]"
         super().__init__(vocabulary=vocabulary, lowercase=lowercase, **kwargs)
 
-        # Check for special tokens
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "[PAD]"
-        mask_token = "[MASK]"
-
-        for token in [cls_token, pad_token, sep_token, mask_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_vocabulary(self, vocabulary):
+        super().set_vocabulary(vocabulary)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [self.cls_token, self.pad_token, self.sep_token]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None

keras_nlp/models/mistral/mistral_tokenizer.py

@@ -58,18 +58,22 @@ class MistralTokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
+        self.start_token = "<s>"
+        self.end_token = "</s>"
         super().__init__(proto=proto, **kwargs)
 
-        # Check for necessary special tokens.
-        start_token = "<s>"
-        end_token = "</s>"
-        for token in [start_token, end_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.start_token_id = self.token_to_id(start_token)
-        self.end_token_id = self.token_to_id(end_token)
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            for token in [self.start_token, self.end_token]:
+                if token not in self.get_vocabulary():
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+            self.start_token_id = self.token_to_id(self.start_token)
+            self.end_token_id = self.token_to_id(self.end_token)
+        else:
+            self.start_token_id = None
+            self.end_token_id = None