Validate fully dynamic objects (preview) with dynamic templates

Author: mrodm

internal/fields/dynamic_template.go

@@ -30,15 +30,13 @@ func (d *dynamicTemplate) Matches(currentPath string, definition map[string]any)
 		name := fieldNameFromPath(currentPath)
 		if !slices.Contains(d.match, name) {
 			// If there is no an exact match, it is compared with patterns/wildcards
-
-			// logger.Warnf(">>>> no contained %s: %s", d.match, name)
 			matches, err := stringMatchesPatterns(d.match, name, fullRegex)
 			if err != nil {
-				return false, fmt.Errorf("failed to parse dynamic template %s: %w", d.name, err)
+				return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err)
 			}
 
 			if !matches {
-				// logger.Debugf(">> Issue match: not matches")
+				// logger.Debugf(">> Issue matchi %q: not matches", d.name)
 				return false, nil
 			}
 		}
@@ -52,11 +50,11 @@ func (d *dynamicTemplate) Matches(currentPath string, definition map[string]any)
 
 		matches, err := stringMatchesPatterns(d.unmatch, name, fullRegex)
 		if err != nil {
-			return false, fmt.Errorf("failed to parse dynamic template %s: %w", d.name, err)
+			return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err)
 		}
 
 		if matches {
-			// logger.Debugf(">> Issue unmatch: matches")
+			// logger.Debugf(">> Issue unmatchi %q: matches", d.name)
 			return false, nil
 		}
 	}
@@ -68,18 +66,18 @@ func (d *dynamicTemplate) Matches(currentPath string, definition map[string]any)
 			return false, fmt.Errorf("failed to parse dynamic template %s: %w", d.name, err)
 		}
 		if !matches {
-			logger.Debugf(">> Issue path_match: not matches (currentPath %s)", currentPath)
+			// logger.Debugf(">> Issue path_match %q: not matches (currentPath %s)", d.name, currentPath)
 			return false, nil
 		}
 	}
 
 	if len(d.unpathMatch) > 0 {
 		matches, err := stringMatchesPatterns(d.unpathMatch, currentPath, fullRegex)
 		if err != nil {
-			return false, fmt.Errorf("failed to parse dynamic template %s: %w", d.name, err)
+			return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err)
 		}
 		if matches {
-			// logger.Debugf(">> Issue unpath_match: matches")
+			// logger.Debugf(">> Issue unpath_matchi %q: matches", d.name)
 			return false, nil
 		}
 	}

internal/fields/mappings.go

@@ -450,8 +450,11 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit
 	// there could be "sub-fields" with name "properties" too
 	if couldBeParametersDefinition && isObject(actual) {
 		if isObjectFullyDynamic(preview) {
-			// TODO: Skip for now, it should be required to compare with dynamic templates
-			logger.Debugf("Pending to validate with the dynamic templates defined the path: %q", path)
+			dynamicErrors := v.validateMappingsNotInPreview(path, actual, dynamicTemplates)
+			errs = append(errs, dynamicErrors...)
+			if len(errs) > 0 {
+				return errs.Unique()
+			}
 			return nil
 		} else if !isObject(preview) {
 			errs = append(errs, fmt.Errorf("not found properties in preview mappings for path: %q", path))

internal/fields/mappings_test.go

@@ -437,7 +437,7 @@ func TestComparingMappings(t *testing.T) {
 			},
 		},
 		{
-			title: "skip dynamic objects", // TODO: should this be checked using dynamic templates?
+			title: "validate fully dynamic objects in preview",
 			preview: map[string]any{
 				"@timestamp": map[string]any{
 					"type": "keyword",
@@ -451,6 +451,13 @@ func TestComparingMappings(t *testing.T) {
 									"type":    "object",
 									"dynamic": "true",
 								},
+								"string": map[string]any{
+									"type":    "object",
+									"dynamic": "true",
+								},
+								"foo": map[string]any{
+									"type": "keyword",
+								},
 							},
 						},
 					},
@@ -473,61 +480,38 @@ func TestComparingMappings(t *testing.T) {
 										},
 									},
 								},
-							},
-						},
-					},
-				},
-			},
-			schema:         []FieldDefinition{},
-			expectedErrors: []string{},
-		},
-		{
-			title: "compare all objects even dynamic true", // TODO: should this be checked using dynamic templates?
-			preview: map[string]any{
-				"@timestamp": map[string]any{
-					"type": "keyword",
-				},
-				"sql": map[string]any{
-					"properties": map[string]any{
-						"metrics": map[string]any{
-							"properties": map[string]any{
-								"dynamic": "true",
-								"numeric": map[string]any{
-									"type":    "object",
-									"dynamic": "true",
-								},
-							},
-						},
-					},
-				},
-			},
-			actual: map[string]any{
-				"@timestamp": map[string]any{
-					"type": "keyword",
-				},
-				"sql": map[string]any{
-					"properties": map[string]any{
-						"metrics": map[string]any{
-							"properties": map[string]any{
-								"dynamic": "true",
-								"numeric": map[string]any{
+								"string": map[string]any{
 									"dynamic": "true",
 									"properties": map[string]any{
 										"innodb_data_fsyncs": map[string]any{
-											"type": "long",
+											"type": "keyword",
 										},
 									},
 								},
 								"example": map[string]any{
 									"type": "keyword",
 								},
+								"foo": map[string]any{
+									"type": "keyword",
+								},
 							},
 						},
 					},
 				},
 			},
+			dynamicTemplates: []map[string]any{
+				{
+					"sql.metrics.string.*": map[string]any{
+						"path_match": "sql.metrics.string.*",
+						"mapping": map[string]any{
+							"type": "keyword",
+						},
+					},
+				},
+			},
 			schema: []FieldDefinition{},
 			expectedErrors: []string{
+				`field "sql.metrics.numeric.innodb_data_fsyncs" is undefined: field definition not found`,
 				`field "sql.metrics.example" is undefined: field definition not found`,
 			},
 		},