Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
2c3f76e
Add dynamic templates as parameter
mrodm Dec 12, 2024
41d7b88
Add validation for each dynamic template depending on the parameters
mrodm Dec 16, 2024
ff5298e
Fixes for dynamic templates
mrodm Dec 16, 2024
af04d66
Compare fields with dynamic templates
mrodm Dec 17, 2024
f21c712
Remove multi_fields from flattened fields
mrodm Dec 17, 2024
aa28db6
Test without filtering dynamic templates
mrodm Dec 17, 2024
3767111
Ensure properties subfields are validated accordingly
mrodm Dec 18, 2024
9fb80b4
Restore filtering and add tests
mrodm Dec 18, 2024
2ad28ac
Disable unmatch_mapping_type and match_mapping_type and continue look…
mrodm Dec 19, 2024
8bdad65
Merge upstream/main into validate-dynamic-mappings
mrodm Dec 19, 2024
151e59e
Refactors and remove log statements
mrodm Dec 19, 2024
d0ff6b9
Fix function naming
mrodm Dec 19, 2024
b474b4b
Add test with match_pattern regex
mrodm Dec 19, 2024
066b5a0
Add comment in tests
mrodm Dec 19, 2024
a7eb191
Remove loading schema from create validator for mappings method
mrodm Jan 7, 2025
7666ac2
Separate parsing and validation of dynamic templates
mrodm Jan 7, 2025
a718796
Support match_pattern for the other settings
mrodm Jan 7, 2025
0a5e775
fix test
mrodm Jan 8, 2025
131cc9c
Update tests for multi-fields
mrodm Jan 8, 2025
38db8df
Review multi-fields logic
mrodm Jan 8, 2025
9ff3d0c
Revisit multi-fields logic in ECS
mrodm Jan 8, 2025
5165612
Report all errors related to multi-fields comparing with ECS
mrodm Jan 8, 2025
ce557c9
Ignored validation of multi-fields with ECS
mrodm Jan 8, 2025
ebda859
Fix multi-field test
mrodm Jan 8, 2025
019ffd4
Rephrase errors
mrodm Jan 9, 2025
8f36c18
Validate fully dynamic objects (preview) with dynamic templates
mrodm Jan 9, 2025
d96ffbf
Rephrase debug message
mrodm Jan 9, 2025
dbca4fe
Add logging - to be removed
mrodm Jan 9, 2025
8fce0ec
Merge remote-tracking branch 'upstream/main' into validate-dynamic-ma…
mrodm Jan 14, 2025
c333707
Revisited log messages
mrodm Jan 20, 2025
0569cfb
Remove comments
mrodm Jan 20, 2025
1aea303
Remove more debug statements
mrodm Jan 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add dynamic templates as parameter
  • Loading branch information
mrodm committed Dec 16, 2024
commit 2c3f76efffadeb4d80773c6c51705b4705f7e049
29 changes: 18 additions & 11 deletions internal/fields/mappings.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,14 @@ func (v *MappingValidator) ValidateIndexMappings(ctx context.Context) multierror
return errs.Unique()
}

mappingErrs := v.compareMappings("", rawPreview, rawActual)
var rawDynamicTemplates map[string]any
err = json.Unmarshal(actualDynamicTemplates, &rawDynamicTemplates)
if err != nil {
errs = append(errs, fmt.Errorf("failed to unmarshal actual dynamic templates (data stream %s): %w", v.dataStreamName, err))
return errs.Unique()
}

mappingErrs := v.compareMappings("", rawPreview, rawActual, rawDynamicTemplates)
errs = append(errs, mappingErrs...)

if len(errs) > 0 {
Expand Down Expand Up @@ -481,7 +488,7 @@ func validateConstantKeywordField(path string, preview, actual map[string]any) (
return isConstantKeyword, nil
}

func (v *MappingValidator) compareMappings(path string, preview, actual map[string]any) multierror.Error {
func (v *MappingValidator) compareMappings(path string, preview, actual, dynamicTemplates map[string]any) multierror.Error {
var errs multierror.Error

isConstantKeywordType, err := validateConstantKeywordField(path, preview, actual)
Expand Down Expand Up @@ -519,7 +526,7 @@ func (v *MappingValidator) compareMappings(path string, preview, actual map[stri
if err != nil {
errs = append(errs, fmt.Errorf("found invalid properties type in actual mappings for path %q: %w", path, err))
}
compareErrors := v.compareMappings(path, previewProperties, actualProperties)
compareErrors := v.compareMappings(path, previewProperties, actualProperties, dynamicTemplates)
errs = append(errs, compareErrors...)

if len(errs) == 0 {
Expand All @@ -542,21 +549,21 @@ func (v *MappingValidator) compareMappings(path string, preview, actual map[stri
if err != nil {
errs = append(errs, fmt.Errorf("found invalid multi_fields type in actual mappings for path %q: %w", path, err))
}
compareErrors := v.compareMappings(path, previewFields, actualFields)
compareErrors := v.compareMappings(path, previewFields, actualFields, dynamicTemplates)
errs = append(errs, compareErrors...)
// not returning here to keep validating the other fields of this object if any
}

// Compare and validate the elements under "properties": objects or fields and its parameters
propertiesErrs := v.validateObjectProperties(path, containsMultifield, actual, preview)
propertiesErrs := v.validateObjectProperties(path, containsMultifield, actual, preview, dynamicTemplates)
errs = append(errs, propertiesErrs...)
if len(errs) == 0 {
return nil
}
return errs.Unique()
}

func (v *MappingValidator) validateObjectProperties(path string, containsMultifield bool, actual, preview map[string]any) multierror.Error {
func (v *MappingValidator) validateObjectProperties(path string, containsMultifield bool, actual, preview, dynamicTemplates map[string]any) multierror.Error {
var errs multierror.Error
for key, value := range actual {
if containsMultifield && key == "fields" {
Expand All @@ -576,14 +583,14 @@ func (v *MappingValidator) validateObjectProperties(path string, containsMultifi
logger.Debugf("field %q is an empty object and it does not exist in the preview", currentPath)
continue
}
ecsErrors := v.validateMappingsNotInPreview(currentPath, childField)
ecsErrors := v.validateMappingsNotInPreview(currentPath, childField, dynamicTemplates)
errs = append(errs, ecsErrors...)
}

continue
}

fieldErrs := v.validateObjectMappingAndParameters(preview[key], value, currentPath)
fieldErrs := v.validateObjectMappingAndParameters(preview[key], value, currentPath, dynamicTemplates)
errs = append(errs, fieldErrs...)
}
if len(errs) == 0 {
Expand All @@ -594,7 +601,7 @@ func (v *MappingValidator) validateObjectProperties(path string, containsMultifi

// validateMappingsNotInPreview validates the object and the nested objects in the current path with other resources
// like ECS schema, dynamic templates or local fields defined in the package (type array).
func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, childField map[string]any) multierror.Error {
func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, childField, dynamicTemplates map[string]any) multierror.Error {
var errs multierror.Error
flattenFields, err := flattenMappings(currentPath, childField)
if err != nil {
Expand Down Expand Up @@ -634,7 +641,7 @@ func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, chil

// validateObjectMappingAndParameters validates the current object or field parameter (currentPath) comparing the values
// in the actual mapping with the values in the preview mapping.
func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string) multierror.Error {
func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string, dynamicTemplates map[string]any) multierror.Error {
var errs multierror.Error
switch actualValue.(type) {
case map[string]any:
Expand All @@ -647,7 +654,7 @@ func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actu
if !ok {
errs = append(errs, fmt.Errorf("unexpected type in actual mappings for path: %q", currentPath))
}
errs = append(errs, v.compareMappings(currentPath, previewField, actualField)...)
errs = append(errs, v.compareMappings(currentPath, previewField, actualField, dynamicTemplates)...)
case any:
// Validate each setting/parameter of the mapping
// If a mapping exist in both preview and actual, they should be the same. But forcing to compare each parameter just in case
Expand Down
4 changes: 3 additions & 1 deletion internal/fields/mappings_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,8 @@ func TestComparingMappings(t *testing.T) {

for _, c := range cases {
t.Run(c.title, func(t *testing.T) {
dynamicTemplates := map[string]any{}

logger.EnableDebugMode()
specVersion := defaultSpecVersion
if c.spec != "" {
Expand All @@ -647,7 +649,7 @@ func TestComparingMappings(t *testing.T) {
)
require.NoError(t, err)

errs := v.compareMappings("", c.preview, c.actual)
errs := v.compareMappings("", c.preview, c.actual, dynamicTemplates)
if len(c.expectedErrors) > 0 {
assert.Len(t, errs, len(c.expectedErrors))
for _, err := range errs {
Expand Down