Add UNKNOWN data type

Author: kevinhu

gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json

@@ -842,7 +842,7 @@
     "name" : "MLFeatureDataType",
     "namespace" : "com.linkedin.common",
     "doc" : "MLFeature Data Type",
-    "symbols" : [ "USELESS", "NOMINAL", "ORDINAL", "BINARY", "COUNT", "TIME", "INTERVAL", "IMAGE", "VIDEO", "AUDIO", "TEXT", "MAP", "SEQUENCE", "SET", "CONTINUOUS", "BYTE" ],
+    "symbols" : [ "USELESS", "NOMINAL", "ORDINAL", "BINARY", "COUNT", "TIME", "INTERVAL", "IMAGE", "VIDEO", "AUDIO", "TEXT", "MAP", "SEQUENCE", "SET", "CONTINUOUS", "BYTE", "UNKNOWN" ],
     "symbolDocs" : {
       "AUDIO" : "Audio Data",
       "BINARY" : "Binary data is discrete data that can be in only one of two categories — either yes or no, 1 or 0, off or on, etc",
@@ -858,6 +858,7 @@
       "SET" : "Set Data Type ex: set, frozenset",
       "TEXT" : "Text Data",
       "TIME" : "Time data is a cyclical, repeating continuous form of data.\nThe relevant time features can be any period— daily, weekly, monthly, annual, etc.",
+      "UNKNOWN" : "Unknown data are data that we don't know the type for.",
       "USELESS" : "Useless data is unique, discrete data with no potential relationship with the outcome variable.\nA useless feature has high cardinality. An example would be bank account numbers that were generated randomly.",
       "VIDEO" : "Video Data"
     }

li-utils/src/main/pegasus/com/linkedin/common/MLFeatureDataType.pdl

@@ -90,4 +90,9 @@ enum MLFeatureDataType {
    * Bytes data are binary-encoded values that can represent complex objects.
    */
   BYTE
+
+  /**
+   * Unknown data are data that we don't know the type for.
+   */
+  UNKNOWN
 }

metadata-ingestion/src/datahub/ingestion/source/feast.py

@@ -99,7 +99,7 @@ def get_field_type(self, field_type: str, parent_name: str) -> str:
             self.report.report_warning(
                 parent_name, f"unable to map type {field_type} to metadata schema"
             )
-            enum_type = MLFeatureDataType.USELESS
+            enum_type = MLFeatureDataType.UNKNOWN
 
         return enum_type
 

metadata-ingestion/src/datahub/metadata/schema.avsc

@@ -3664,7 +3664,8 @@
                               "SEQUENCE",
                               "SET",
                               "CONTINUOUS",
-                              "BYTE"
+                              "BYTE",
+                              "UNKNOWN"
                             ],
                             "symbolDocs": {
                               "AUDIO": "Audio Data",
@@ -3681,6 +3682,7 @@
                               "SET": "Set Data Type ex: set, frozenset",
                               "TEXT": "Text Data",
                               "TIME": "Time data is a cyclical, repeating continuous form of data.\nThe relevant time features can be any period\u2014 daily, weekly, monthly, annual, etc.",
+                              "UNKNOWN": "Unknown data are data that we don't know the type for.",
                               "USELESS": "Useless data is unique, discrete data with no potential relationship with the outcome variable.\nA useless feature has high cardinality. An example would be bank account numbers that were generated randomly.",
                               "VIDEO": "Video Data"
                             }

metadata-ingestion/src/datahub/metadata/schema_classes.py

@@ -1109,6 +1109,9 @@ class MLFeatureDataTypeClass(object):
     """Bytes data are binary-encoded values that can represent complex objects."""
     BYTE = "BYTE"
 
+    """Unknown data are data that we don't know the type for."""
+    UNKNOWN = "UNKNOWN"
+    
 
 class OwnerClass(DictWrapper):
     """Ownership information"""