mimecast: resolve field data type conflicts between data streams

packages/mimecast/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.0.0"
+  changes:
+    - description: Change field type from keyword to boolean for taggedExternal and taggedMalicious fields to resolve conflicts with other data streams.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/13825
 - version: "2.7.3"
   changes:
     - description: Fix default request trace enabled behavior.

packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log

@@ -59,3 +59,4 @@
 {"_offset":72869,"_partition":60,"accountId":"CUSB4A274","aggregateId":"5I6WzskaO0anMqefgzUxMQ_1731617561","messageId":"<[email protected]>","processingId":"44x8Ss3UswVtAAZEys6AvV2YDHgvX68PHyHjte7bw_w_1731617561","recipients":"[email protected]","route":"Inbound","senderDomain":"pilot-meadow.b41.one","senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","subject":null,"subtype":null,"timestamp":1731617562340,"type":"spam"}
 {"_offset":72407,"_partition":60,"accountId":"CUSB4A274","aggregateId":"nNsYsZo5PsiS7tsPPMhScw_1731597441","messageId":"<[email protected]>","processingId":"Ju7DcVaik3AmIzS7u9rY8klrnT3r33IyQUYVFC__byk_1731597441","recipients":"[email protected]","route":"Inbound","senderDomain":"pilot-meadow.b41.one","senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","subject":"Stranger","subtype":null,"timestamp":1731597442889,"type":"spam"}
 {"_offset":74368,"_partition":60,"accountId":"CUSB4A274","aggregateId":"Zyc2YQp5NiebreJzicSY0g_1731827848","messageId":"<[email protected]>","processingId":"rdbUbf7dO9buFz-hm-jV4eMdfFO2Qnu6iZy__ONwPng_1731827848","recipients":"[email protected]","route":"Inbound","senderDomain":"arcadia-group.b41.one","senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","subject":"Weekly Report","subtype":null,"timestamp":1731827851036,"type":"spam"}
+{"_offset":74369,"_partition":60,"taggedExternal":"true","taggedMalicious":"false","accountId":"CUSB4A274","aggregateId":"Zyc2YQp5NiebreJzicSY0g_1731827123","messageId":"<[email protected]>","processingId":"rdbUbf7dO9buFz-hm-jV4eMdfFO2Qnu6iZy__ONwPng_1731827848","recipients":"[email protected]","route":"Inbound","senderDomain":"arcadia-group.b41.one","senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","subject":"Weekly Report","subtype":null,"timestamp":1731827851036,"type":"spam"}

packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-v2-logs.log-expected.json

@@ -3491,6 +3491,69 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2024-11-17T07:17:31.036Z",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "email": {
+                "direction": "inbound",
+                "from": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                },
+                "message_id": "<[email protected]>",
+                "subject": "Weekly Report",
+                "to": {
+                    "address": [
+                        "[email protected]"
+                    ]
+                }
+            },
+            "event": {
+                "category": [
+                    "email"
+                ],
+                "created": "2024-11-17T07:17:31.036Z",
+                "original": "{\"_offset\":74369,\"_partition\":60,\"taggedExternal\":\"true\",\"taggedMalicious\":\"false\",\"accountId\":\"CUSB4A274\",\"aggregateId\":\"Zyc2YQp5NiebreJzicSY0g_1731827123\",\"messageId\":\"<[email protected]>\",\"processingId\":\"rdbUbf7dO9buFz-hm-jV4eMdfFO2Qnu6iZy__ONwPng_1731827848\",\"recipients\":\"[email protected]\",\"route\":\"Inbound\",\"senderDomain\":\"arcadia-group.b41.one\",\"senderEnvelope\":\"[email protected]\",\"senderHeader\":\"[email protected]\",\"senderIp\":\"81.2.69.144\",\"subject\":\"Weekly Report\",\"subtype\":null,\"timestamp\":1731827851036,\"type\":\"spam\"}",
+                "outcome": "unknown"
+            },
+            "mimecast": {
+                "accountId": "CUSB4A274",
+                "aggregateId": "Zyc2YQp5NiebreJzicSY0g_1731827123",
+                "log_type": "spam",
+                "processingId": "rdbUbf7dO9buFz-hm-jV4eMdfFO2Qnu6iZy__ONwPng_1731827848",
+                "senderHeader": "[email protected]",
+                "taggedExternal": true,
+                "taggedMalicious": false
+            },
+            "related": {
+                "user": [
+                    "[email protected]",
+                    "[email protected]"
+                ]
+            },
+            "source": {
+                "domain": "arcadia-group.b41.one",
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.144"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }

packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml

@@ -242,6 +242,28 @@ processors:
       field: mimecast.newDomain
       target_field: source.domain
       ignore_missing: true
+  - convert:
+      field: mimecast.taggedExternal
+      tag: convert_tagged_external_to_boolean
+      type: boolean
+      ignore_missing: true
+      on_failure:
+        - remove:
+            field: mimecast.taggedExternal
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - convert:
+      field: mimecast.taggedMalicious
+      tag: convert_tagged_malicious_to_boolean
+      type: boolean
+      ignore_missing: true
+      on_failure:
+        - remove:
+            field: mimecast.taggedMalicious
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
 
   ### SIEM TTP Url Logs
   - rename:

packages/mimecast/data_stream/siem_logs/fields/field.yml

@@ -432,11 +432,11 @@
       description: >-
         The content of the email was detected to contain words in the client custom threat dictionary. (True; False)
     - name: taggedMalicious
-      type: keyword
+      type: boolean
       description: >-
         The message has been tagged as malicious. (True; False)
     - name: taggedExternal
-      type: keyword
+      type: boolean
       description: >-
         The message has been tagged as originating from a external source. (True; False)
 

packages/mimecast/docs/README.md

@@ -887,8 +887,8 @@ An example event for `siem` looks as following:
 | mimecast.spamScore | The metric that measures the likelihood of the event being considered spam. | keyword |
 | mimecast.subject | The subject of the email. | keyword |
 | mimecast.subtype |  | keyword |
-| mimecast.taggedExternal | The message has been tagged as originating from a external source. (True; False) | keyword |
-| mimecast.taggedMalicious | The message has been tagged as malicious. (True; False) | keyword |
+| mimecast.taggedExternal | The message has been tagged as originating from a external source. (True; False) | boolean |
+| mimecast.taggedMalicious | The message has been tagged as malicious. (True; False) | boolean |
 | mimecast.timestamp | The date and time of event. | keyword |
 | mimecast.tlsCipher | The TLS Cipher used if the email was delivered or received using TLS. | keyword |
 | mimecast.tlsUsed | If the message was delivered using TLS or not. (Yes; No) | keyword |

packages/mimecast/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: mimecast
 title: "Mimecast"
-version: "2.7.3"
+version: "3.0.0"
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration
 categories: ["security", "email_security"]

packages/ti_misp/data_stream/threat/_dev/test/pipeline/test-misp-attributes-ndjson.log-expected.json

@@ -2604,4 +2604,4 @@
             }
         }
     ]
-}
+}

packages/ti_misp/data_stream/threat/_dev/test/pipeline/test-misp-long-ndjson.log-expected.json

@@ -220,4 +220,4 @@
             }
         }
     ]
-}
+}

packages/ti_misp/data_stream/threat/_dev/test/pipeline/test-misp-sample-ndjson.log-expected.json

@@ -1221,4 +1221,4 @@
         },
         null
     ]
-}
+}