Update test files

internal/fields/testdata/mongodb-multi-fields.json

@@ -0,0 +1,13 @@
+{
+    "@timestamp": "2023-06-27T15:08:06.769Z",
+    "data_stream": {
+        "dataset": "mongodb.status",
+        "namespace": "ep",
+        "type": "metrics"
+    },
+    "process": {
+       "name": {
+         "text": "mongodb"
+       }
+    }
+}

internal/fields/validate.go

@@ -8,6 +8,7 @@ import (
 	"bufio"
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -166,15 +167,12 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa
 		if err != nil {
 			return nil, fmt.Errorf("can't find package root: %w", err)
 		}
-		// As every command starts with approximating where is the package root, it isn't required to return an error in case the root is missing.
-		// This is also useful for testing purposes, where we don't have a real package, but just "fields" directory. The package root is always absent.
 		if !found {
-			logger.Debug("Package root not found, dependency management will be disabled.")
-		} else {
-			fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema)
-			if err != nil {
-				return nil, fmt.Errorf("failed to initialize dependency management: %w", err)
-			}
+			return nil, errors.New("package root not found and dependency management is enabled")
+		}
+		fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize dependency management: %w", err)
 		}
 	}
 

internal/fields/validate_test.go

@@ -30,7 +30,7 @@ func (p packageRootTestFinder) FindPackageRoot() (string, bool, error) {
 }
 
 func TestValidate_NoWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/elb_logs")
+	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/elb_logs", WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -42,7 +42,7 @@ func TestValidate_NoWildcardFields(t *testing.T) {
 }
 
 func TestValidate_WithWildcardFields(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/sns")
+	validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/sns", WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -112,7 +112,7 @@ func TestValidate_WithDisabledImportAllECSSchema(t *testing.T) {
 }
 
 func TestValidate_constantKeyword(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("testdata")
+	validator, err := CreateValidatorForDirectory("testdata", WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -126,7 +126,7 @@ func TestValidate_constantKeyword(t *testing.T) {
 }
 
 func TestValidate_ipAddress(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("testdata", WithEnabledAllowedIPCheck())
+	validator, err := CreateValidatorForDirectory("testdata", WithEnabledAllowedIPCheck(), WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -141,7 +141,7 @@ func TestValidate_ipAddress(t *testing.T) {
 }
 
 func TestValidate_WithSpecVersion(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"))
+	validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"), WithDisabledDependencyManagement())
 	require.NoError(t, err)
 
 	e := readSampleEvent(t, "testdata/invalid-array-normalization.json")
@@ -154,7 +154,7 @@ func TestValidate_WithSpecVersion(t *testing.T) {
 	require.Empty(t, errs)
 
 	// Check now that this validation was only enabled for 2.0.0.
-	validator, err = CreateValidatorForDirectory("testdata", WithSpecVersion("1.99.99"))
+	validator, err = CreateValidatorForDirectory("testdata", WithSpecVersion("1.99.99"), WithDisabledDependencyManagement())
 	require.NoError(t, err)
 
 	e = readSampleEvent(t, "testdata/invalid-array-normalization.json")
@@ -163,7 +163,7 @@ func TestValidate_WithSpecVersion(t *testing.T) {
 }
 
 func TestValidate_ExpectedEventType(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"))
+	validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"), WithDisabledDependencyManagement())
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
@@ -255,6 +255,7 @@ func TestValidate_ExpectedDataset(t *testing.T) {
 	validator, err := CreateValidatorForDirectory("testdata",
 		WithSpecVersion("2.0.0"),
 		WithExpectedDataset("apache.status"),
+		WithDisabledDependencyManagement(),
 	)
 	require.NoError(t, err)
 	require.NotNil(t, validator)
@@ -787,7 +788,7 @@ func TestCompareKeys(t *testing.T) {
 }
 
 func TestValidateGeoPoint(t *testing.T) {
-	validator, err := CreateValidatorForDirectory("../../test/packages/other/fields_tests/data_stream/first")
+	validator, err := CreateValidatorForDirectory("../../test/packages/other/fields_tests/data_stream/first", WithDisabledDependencyManagement())
 
 	require.NoError(t, err)
 	require.NotNil(t, validator)
@@ -806,7 +807,18 @@ func TestValidateExternalMultiField(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, validator)
 
-	e := readSampleEvent(t, filepath.Join(dataStreamRoot, "sample_event.json"))
+	def := FindElementDefinition("process.name", validator.Schema)
+	require.NotEmpty(t, def.MultiFields, "expected to test with a data stream with a field with multifields")
+
+	e := readSampleEvent(t, "testdata/mongodb-multi-fields.json")
+	var event common.MapStr
+	err = json.Unmarshal(e, &event)
+	require.NoError(t, err)
+
+	v, err := event.GetValue("process.name.text")
+	require.NotNil(t, v, "expected document with multi-field")
+	require.NoError(t, err, "expected document with multi-field")
+
 	errs := validator.ValidateDocumentBody(e)
 	require.Empty(t, errs)
 }

test/packages/parallel/mongodb/_dev/deploy/docker/docker-compose.yml

@@ -13,3 +13,4 @@ services:
       - ${SERVICE_LOGS_DIR}:/var/log/mongodb
     entrypoint: >
       bash -c "chmod a+wx /var/log/mongodb && chmod a+r -R /var/log/mongodb && touch /var/log/mongodb/mongod.log && chmod 644 /var/log/mongodb/mongod.log && mongod --replSet beats --logpath /var/log/mongodb/mongod.log --logappend"
+

test/packages/parallel/mongodb/data_stream/collstats/fields/agent.yml

@@ -210,4 +210,4 @@
     - name: id
       type: keyword
       ignore_above: 1024
-      dimension: true
+      dimension: true

test/packages/parallel/mongodb/data_stream/collstats/manifest.yml

@@ -13,4 +13,4 @@ streams:
     title: MongoDB collstats metrics
     description: Collect MongoDB collstats metrics
 elasticsearch:
-  index_mode: "time_series"
+  index_mode: "time_series"

test/packages/parallel/mongodb/data_stream/collstats/sample_event.json

@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2023-06-27T15:05:34.410Z",
+    "@timestamp": "2023-06-29T17:58:25.465Z",
     "agent": {
-        "ephemeral_id": "379d06ef-b1a1-4aae-a292-1bd3bc8e78df",
-        "id": "ee8ce08e-6f80-467d-a633-e9ff4ffdb266",
+        "ephemeral_id": "c9e5ccdb-8232-4a7d-b161-ecd4f444bba5",
+        "id": "3bcbc378-6f9d-4e10-9bf2-b99c50493aa5",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
         "version": "8.8.0"
@@ -16,29 +16,29 @@
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "ee8ce08e-6f80-467d-a633-e9ff4ffdb266",
+        "id": "3bcbc378-6f9d-4e10-9bf2-b99c50493aa5",
         "snapshot": false,
         "version": "8.8.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "mongodb.collstats",
-        "duration": 5036250,
-        "ingested": "2023-06-27T15:05:35Z",
+        "duration": 2334698,
+        "ingested": "2023-06-29T17:58:26Z",
         "module": "mongodb"
     },
     "host": {
-        "architecture": "aarch64",
+        "architecture": "x86_64",
         "containerized": false,
         "hostname": "docker-fleet-agent",
-        "id": "b5cb0e96dbea41d19c3216af1f327d73",
-        "ip": "192.168.48.7",
-        "mac": "02-42-C0-A8-30-07",
+        "id": "e8978f2086c14e13b7a0af9ed0011d19",
+        "ip": "192.168.160.7",
+        "mac": "02-42-C0-A8-A0-07",
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.104-linuxkit",
+            "kernel": "5.19.0-43-generic",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -51,7 +51,7 @@
     },
     "mongodb": {
         "collstats": {
-            "collection": "oplog.rs",
+            "collection": "system.replset",
             "commands": {
                 "count": 0,
                 "time": {
@@ -73,19 +73,19 @@
             },
             "lock": {
                 "read": {
-                    "count": 17,
+                    "count": 1,
                     "time": {
-                        "us": 68
+                        "us": 1
                     }
                 },
                 "write": {
-                    "count": 1,
+                    "count": 0,
                     "time": {
-                        "us": 7516
+                        "us": 0
                     }
                 }
             },
-            "name": "local.oplog.rs",
+            "name": "local.system.replset",
             "queries": {
                 "count": 0,
                 "time": {
@@ -99,9 +99,9 @@
                 }
             },
             "total": {
-                "count": 18,
+                "count": 1,
                 "time": {
-                    "us": 7584
+                    "us": 1
                 }
             },
             "update": {
@@ -113,7 +113,7 @@
         }
     },
     "service": {
-        "address": "mongodb://elastic-package-service_mongodb_1",
+        "address": "mongodb://elastic-package-service-mongodb-1",
         "type": "mongodb"
     }
 }

test/packages/parallel/mongodb/data_stream/dbstats/fields/agent.yml

@@ -210,4 +210,4 @@
     - name: id
       type: keyword
       ignore_above: 1024
-      dimension: true
+      dimension: true

test/packages/parallel/mongodb/data_stream/dbstats/manifest.yml

@@ -13,4 +13,4 @@ streams:
     title: MongoDB dbstats metrics
     description: Collect MongoDB dbstats metrics
 elasticsearch:
-  index_mode: "time_series"
+  index_mode: "time_series"

test/packages/parallel/mongodb/data_stream/dbstats/sample_event.json

@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2023-06-27T09:46:40.443Z",
+    "@timestamp": "2023-06-29T17:59:01.652Z",
     "agent": {
-        "ephemeral_id": "edda44e9-e81b-4916-a6dd-ad6716976001",
-        "id": "ee8ce08e-6f80-467d-a633-e9ff4ffdb266",
+        "ephemeral_id": "9a272b3d-98cd-49d8-a116-2c89c7b75d03",
+        "id": "3bcbc378-6f9d-4e10-9bf2-b99c50493aa5",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
         "version": "8.8.0"
@@ -16,29 +16,29 @@
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "ee8ce08e-6f80-467d-a633-e9ff4ffdb266",
+        "id": "3bcbc378-6f9d-4e10-9bf2-b99c50493aa5",
         "snapshot": false,
         "version": "8.8.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "mongodb.dbstats",
-        "duration": 3543708,
-        "ingested": "2023-06-27T09:46:42Z",
+        "duration": 2384254,
+        "ingested": "2023-06-29T17:59:02Z",
         "module": "mongodb"
     },
     "host": {
-        "architecture": "aarch64",
+        "architecture": "x86_64",
         "containerized": false,
         "hostname": "docker-fleet-agent",
-        "id": "b5cb0e96dbea41d19c3216af1f327d73",
-        "ip": "192.168.48.7",
-        "mac": "02-42-C0-A8-30-07",
+        "id": "e8978f2086c14e13b7a0af9ed0011d19",
+        "ip": "192.168.160.7",
+        "mac": "02-42-C0-A8-A0-07",
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.104-linuxkit",
+            "kernel": "5.19.0-43-generic",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -52,26 +52,25 @@
     "mongodb": {
         "dbstats": {
             "avg_obj_size": {
-                "bytes": 522
+                "bytes": 59
             },
-            "collections": 3,
+            "collections": 1,
             "data_size": {
-                "bytes": 1566
+                "bytes": 59
             },
-            "db": "local",
+            "db": "admin",
             "index_size": {
-                "bytes": 12288
+                "bytes": 4096
             },
-            "indexes": 3,
-            "num_extents": 0,
-            "objects": 3,
+            "indexes": 1,
+            "objects": 1,
             "storage_size": {
-                "bytes": 12288
+                "bytes": 4096
             }
         }
     },
     "service": {
-        "address": "mongodb://elastic-package-service_mongodb_1",
+        "address": "mongodb://elastic-package-service-mongodb-1",
         "type": "mongodb"
     }
 }