diff --git a/internal/fields/dynamic_template.go b/internal/fields/dynamic_template.go new file mode 100644 index 0000000000..ff7df68080 --- /dev/null +++ b/internal/fields/dynamic_template.go @@ -0,0 +1,238 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fields + +import ( + "fmt" + "regexp" + "slices" + "strings" + + "github.com/elastic/elastic-package/internal/logger" +) + +type dynamicTemplate struct { + name string + matchPattern string + match []string + unmatch []string + pathMatch []string + unpathMatch []string + mapping any +} + +func (d *dynamicTemplate) Matches(currentPath string, definition map[string]any) (bool, error) { + fullRegex := d.matchPattern == "regex" + + if len(d.match) > 0 { + name := fieldNameFromPath(currentPath) + if !slices.Contains(d.match, name) { + // If there is no an exact match, it is compared with patterns/wildcards + matches, err := stringMatchesPatterns(d.match, name, fullRegex) + if err != nil { + return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err) + } + + if !matches { + return false, nil + } + } + } + + if len(d.unmatch) > 0 { + name := fieldNameFromPath(currentPath) + if slices.Contains(d.unmatch, name) { + return false, nil + } + + matches, err := stringMatchesPatterns(d.unmatch, name, fullRegex) + if err != nil { + return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err) + } + + if matches { + return false, nil + } + } + + if len(d.pathMatch) > 0 { + // logger.Debugf("path_match -> Comparing %s to %q", strings.Join(d.pathMatch, ";"), currentPath) + matches, err := stringMatchesPatterns(d.pathMatch, currentPath, fullRegex) + if err != nil { + return false, fmt.Errorf("failed to parse dynamic template %s: %w", d.name, err) + } + if !matches { + return false, nil + } + } + + if len(d.unpathMatch) > 0 { + matches, err := stringMatchesPatterns(d.unpathMatch, currentPath, fullRegex) + if err != nil { + return false, fmt.Errorf("failed to parse dynamic template %q: %w", d.name, err) + } + if matches { + return false, nil + } + } + return true, nil +} + +func stringMatchesRegex(regexes []string, elem string) (bool, error) { + applies := false + for _, v := range regexes { + if !strings.Contains(v, "*") { + // not a regex + continue + } + + match, err := regexp.MatchString(v, elem) + if err != nil { + return false, fmt.Errorf("failed to build regex %s: %w", v, err) + } + if match { + applies = true + break + } + } + return applies, nil +} + +func stringMatchesPatterns(regexes []string, elem string, fullRegex bool) (bool, error) { + if fullRegex { + return stringMatchesRegex(regexes, elem) + } + + // transform wildcards to valid regexes + updatedRegexes := []string{} + for _, v := range regexes { + r := strings.ReplaceAll(v, ".", "\\.") + r = strings.ReplaceAll(r, "*", ".*") + + // Force to match the beginning and ending of the given path + r = fmt.Sprintf("^%s$", r) + + updatedRegexes = append(updatedRegexes, r) + } + return stringMatchesRegex(updatedRegexes, elem) +} + +func parseDynamicTemplates(rawDynamicTemplates []map[string]any) ([]dynamicTemplate, error) { + dynamicTemplates := []dynamicTemplate{} + + for _, template := range rawDynamicTemplates { + if len(template) != 1 { + return nil, fmt.Errorf("unexpected number of dynamic template definitions found") + } + + // there is just one dynamic template per object + templateName := "" + var rawContents any + for key, value := range template { + templateName = key + rawContents = value + } + + if shouldSkipDynamicTemplate(templateName) { + continue + } + + aDynamicTemplate := dynamicTemplate{ + name: templateName, + } + + contents, ok := rawContents.(map[string]any) + if !ok { + return nil, fmt.Errorf("unexpected dynamic template format found for %q", templateName) + } + + for setting, value := range contents { + switch setting { + case "mapping": + aDynamicTemplate.mapping = value + case "match_pattern": + s, ok := value.(string) + if !ok { + return nil, fmt.Errorf("invalid type for \"match_pattern\": %T", value) + } + aDynamicTemplate.matchPattern = s + case "match": + values, err := parseDynamicTemplateParameter(value) + if err != nil { + logger.Warnf("failed to check match setting: %s", err) + return nil, fmt.Errorf("failed to check match setting: %w", err) + } + aDynamicTemplate.match = values + case "unmatch": + values, err := parseDynamicTemplateParameter(value) + if err != nil { + return nil, fmt.Errorf("failed to check unmatch setting: %w", err) + } + aDynamicTemplate.unmatch = values + case "path_match": + values, err := parseDynamicTemplateParameter(value) + if err != nil { + return nil, fmt.Errorf("failed to check path_match setting: %w", err) + } + aDynamicTemplate.pathMatch = values + case "path_unmatch": + values, err := parseDynamicTemplateParameter(value) + if err != nil { + return nil, fmt.Errorf("failed to check path_unmatch setting: %w", err) + } + aDynamicTemplate.unpathMatch = values + case "match_mapping_type", "unmatch_mapping_type": + // Do nothing + // These parameters require to check the original type (before the document is ingested) + // but the dynamic template just contains the type from the `mapping` field + default: + return nil, fmt.Errorf("unexpected setting found in dynamic template") + } + } + + dynamicTemplates = append(dynamicTemplates, aDynamicTemplate) + } + + return dynamicTemplates, nil +} + +func shouldSkipDynamicTemplate(templateName string) bool { + // Filter out dynamic templates created by elastic-package (import_mappings) + // or added automatically by ecs@mappings component template + if strings.HasPrefix(templateName, "_embedded_ecs-") { + return true + } + if strings.HasPrefix(templateName, "ecs_") { + return true + } + if slices.Contains([]string{"all_strings_to_keywords", "strings_as_keyword"}, templateName) { + return true + } + return false +} + +func parseDynamicTemplateParameter(value any) ([]string, error) { + all := []string{} + switch v := value.(type) { + case []any: + for _, elem := range v { + s, ok := elem.(string) + if !ok { + return nil, fmt.Errorf("failed to cast to string: %s", elem) + } + all = append(all, s) + } + case any: + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("failed to cast to string: %s", v) + } + all = append(all, s) + default: + return nil, fmt.Errorf("unexpected type for setting: %T", value) + + } + return all, nil +} diff --git a/internal/fields/mappings.go b/internal/fields/mappings.go index a0975494f9..68d9a8956a 100644 --- a/internal/fields/mappings.go +++ b/internal/fields/mappings.go @@ -7,13 +7,10 @@ package fields import ( "context" "encoding/json" - "errors" "fmt" - "path/filepath" "slices" "strings" - "github.com/Masterminds/semver/v3" "github.com/google/go-cmp/cmp" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -26,17 +23,6 @@ type MappingValidator struct { // Schema contains definition records. Schema []FieldDefinition - // SpecVersion contains the version of the spec used by the package. - specVersion semver.Version - - disabledDependencyManagement bool - - enabledImportAllECSSchema bool - - disabledNormalization bool - - injectFieldsOptions InjectFieldsOptions - esClient *elasticsearch.Client indexTemplateName string @@ -49,50 +35,6 @@ type MappingValidator struct { // MappingValidatorOption represents an optional flag that can be passed to CreateValidatorForMappings. type MappingValidatorOption func(*MappingValidator) error -// WithMappingValidatorSpecVersion enables validation dependant of the spec version used by the package. -func WithMappingValidatorSpecVersion(version string) MappingValidatorOption { - return func(v *MappingValidator) error { - sv, err := semver.NewVersion(version) - if err != nil { - return fmt.Errorf("invalid version %q: %v", version, err) - } - v.specVersion = *sv - return nil - } -} - -// WithMappingValidatorDisabledDependencyManagement configures the validator to ignore external fields and won't follow dependencies. -func WithMappingValidatorDisabledDependencyManagement() MappingValidatorOption { - return func(v *MappingValidator) error { - v.disabledDependencyManagement = true - return nil - } -} - -// WithMappingValidatorEnabledImportAllECSSchema configures the validator to check or not the fields with the complete ECS schema. -func WithMappingValidatorEnabledImportAllECSSChema(importSchema bool) MappingValidatorOption { - return func(v *MappingValidator) error { - v.enabledImportAllECSSchema = importSchema - return nil - } -} - -// WithMappingValidatorDisableNormalization configures the validator to disable normalization. -func WithMappingValidatorDisableNormalization(disabledNormalization bool) MappingValidatorOption { - return func(v *MappingValidator) error { - v.disabledNormalization = disabledNormalization - return nil - } -} - -// WithMappingValidatorInjectFieldsOptions configures fields injection. -func WithMappingValidatorInjectFieldsOptions(options InjectFieldsOptions) MappingValidatorOption { - return func(v *MappingValidator) error { - v.injectFieldsOptions = options - return nil - } -} - // WithMappingValidatorElasticsearchClient configures the Elasticsearch client. func WithMappingValidatorElasticsearchClient(esClient *elasticsearch.Client) MappingValidatorOption { return func(v *MappingValidator) error { @@ -134,13 +76,12 @@ func WithMappingValidatorExceptionFields(fields []string) MappingValidatorOption } // CreateValidatorForMappings function creates a validator for the mappings. -func CreateValidatorForMappings(fieldsParentDir string, esClient *elasticsearch.Client, opts ...MappingValidatorOption) (v *MappingValidator, err error) { - p := packageRoot{} +func CreateValidatorForMappings(esClient *elasticsearch.Client, opts ...MappingValidatorOption) (v *MappingValidator, err error) { opts = append(opts, WithMappingValidatorElasticsearchClient(esClient)) - return createValidatorForMappingsAndPackageRoot(fieldsParentDir, p, opts...) + return createValidatorForMappingsAndPackageRoot(opts...) } -func createValidatorForMappingsAndPackageRoot(fieldsParentDir string, finder packageRootFinder, opts ...MappingValidatorOption) (v *MappingValidator, err error) { +func createValidatorForMappingsAndPackageRoot(opts ...MappingValidatorOption) (v *MappingValidator, err error) { v = new(MappingValidator) for _, opt := range opts { if err := opt(v); err != nil { @@ -148,32 +89,6 @@ func createValidatorForMappingsAndPackageRoot(fieldsParentDir string, finder pac } } - if len(v.Schema) > 0 { - return v, nil - } - - fieldsDir := filepath.Join(fieldsParentDir, "fields") - - var fdm *DependencyManager - if !v.disabledDependencyManagement { - packageRoot, found, err := finder.FindPackageRoot() - if err != nil { - return nil, fmt.Errorf("can't find package root: %w", err) - } - if !found { - return nil, errors.New("package root not found and dependency management is enabled") - } - fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema) - if err != nil { - return nil, fmt.Errorf("failed to initialize dependency management: %w", err) - } - } - fields, err := loadFieldsFromDir(fieldsDir, fdm, v.injectFieldsOptions) - if err != nil { - return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err) - } - - v.Schema = append(fields, v.Schema...) return v, nil } @@ -241,7 +156,14 @@ func (v *MappingValidator) ValidateIndexMappings(ctx context.Context) multierror return errs.Unique() } - mappingErrs := v.compareMappings("", false, rawPreview, rawActual) + var rawDynamicTemplates []map[string]any + err = json.Unmarshal(actualDynamicTemplates, &rawDynamicTemplates) + if err != nil { + errs = append(errs, fmt.Errorf("failed to unmarshal actual dynamic templates (data stream %s): %w", v.dataStreamName, err)) + return errs.Unique() + } + + mappingErrs := v.compareMappings("", false, rawPreview, rawActual, rawDynamicTemplates) errs = append(errs, mappingErrs...) if len(errs) > 0 { @@ -258,6 +180,15 @@ func currentMappingPath(path, key string) string { return fmt.Sprintf("%s.%s", path, key) } +func fieldNameFromPath(path string) string { + if !strings.Contains(path, ".") { + return path + } + + elems := strings.Split(path, ".") + return elems[len(elems)-1] +} + func mappingParameter(field string, definition map[string]any) string { fieldValue, ok := definition[field] if !ok { @@ -369,28 +300,51 @@ func isNumberTypeField(previewType, actualType string) bool { return false } -func (v *MappingValidator) validateMappingInECSSchema(currentPath string, definition map[string]any) error { +func (v *MappingValidator) validateMappingInECSSchema(currentPath string, definition map[string]any) multierror.Error { found := FindElementDefinition(currentPath, v.Schema) if found == nil { - return fmt.Errorf("missing definition for path") + return multierror.Error{fmt.Errorf("field definition not found")} } if found.External != "ecs" { - return fmt.Errorf("missing definition for path (not in ECS)") + return multierror.Error{fmt.Errorf("field definition not found")} } - actualType := mappingParameter("type", definition) - if found.Type == actualType { - return nil + errs := compareFieldDefinitionWithECS(currentPath, found, definition) + if len(errs) > 0 { + return errs } - // exceptions related to numbers - if isNumberTypeField(found.Type, actualType) { - logger.Debugf("Allowed number fields with different types (ECS %s - actual %s)", string(found.Type), string(actualType)) - return nil + // Currently, validation of multi-fields with ECS is skipped + // Any multi-field found in the actual mapping must come from a definition from the preview or + // a dynamic template, therefore there is a definition for it. + // Example of this validation can be found at: + // https://github.com/elastic/elastic-package/pull/2285/commits/51656120 + return nil +} + +func compareFieldDefinitionWithECS(currentPath string, ecs *FieldDefinition, actual map[string]any) multierror.Error { + var errs multierror.Error + actualType := mappingParameter("type", actual) + if ecs.Type != actualType { + // exceptions related to numbers + if !isNumberTypeField(ecs.Type, actualType) { + errs = append(errs, fmt.Errorf("actual mapping type (%s) does not match with ECS definition type: %s", actualType, ecs.Type)) + } else { + logger.Debugf("Allowed number fields with different types (ECS %s - actual %s)", string(ecs.Type), string(actualType)) + } } - // any other field to validate here? - return fmt.Errorf("actual mapping type (%s) does not match with ECS definition type: %s", actualType, found.Type) + + // Compare other parameters + metricType := mappingParameter("time_series_metric", actual) + if ecs.MetricType != metricType { + errs = append(errs, fmt.Errorf("actual mapping \"time_series_metric\" (%s) does not match with ECS definition value: %s", metricType, ecs.MetricType)) + } + + if len(errs) > 0 { + return errs + } + return nil } // flattenMappings returns all the mapping definitions found at "path" flattened including @@ -398,22 +352,9 @@ func (v *MappingValidator) validateMappingInECSSchema(currentPath string, defini func flattenMappings(path string, definition map[string]any) (map[string]any, error) { newDefs := map[string]any{} if isMultiFields(definition) { - multifields, err := getMappingDefinitionsField("fields", definition) - if err != nil { - return nil, multierror.Error{fmt.Errorf("invalid multi_field mapping %q: %w", path, err)} - } - - // Include also the definition itself newDefs[path] = definition - - for key, object := range multifields { - currentPath := currentMappingPath(path, key) - def, ok := object.(map[string]any) - if !ok { - return nil, multierror.Error{fmt.Errorf("invalid multi_field mapping type: %q", path)} - } - newDefs[currentPath] = def - } + // multi_fields are going to be validated directly with the dynamic templates + // or with ECS fields return newDefs, nil } @@ -448,10 +389,13 @@ func flattenMappings(path string, definition map[string]any) (map[string]any, er } func getMappingDefinitionsField(field string, definition map[string]any) (map[string]any, error) { - anyValue := definition[field] + anyValue, ok := definition[field] + if !ok { + return nil, fmt.Errorf("field not found: %q", field) + } object, ok := anyValue.(map[string]any) if !ok { - return nil, fmt.Errorf("unexpected type found for %q: %T ", field, anyValue) + return nil, fmt.Errorf("unexpected type found for field %q: %T ", field, anyValue) } return object, nil } @@ -476,12 +420,12 @@ func validateConstantKeywordField(path string, preview, actual map[string]any) ( if previewValue != actualValue { // This should also be detected by the failure storage (if available) // or no documents being ingested - return isConstantKeyword, fmt.Errorf("constant_keyword value in preview %q does not match the actual mapping value %q for path: %q", previewValue, actualValue, path) + return isConstantKeyword, fmt.Errorf("invalid value in field %q: constant_keyword value in preview %q does not match the actual mapping value %q", path, previewValue, actualValue) } return isConstantKeyword, nil } -func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinition bool, preview, actual map[string]any) multierror.Error { +func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinition bool, preview, actual map[string]any, dynamicTemplates []map[string]any) multierror.Error { var errs multierror.Error isConstantKeywordType, err := validateConstantKeywordField(path, preview, actual) @@ -498,7 +442,7 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit } if isObjectFullyDynamic(actual) { - logger.Debugf("Dynamic object found but no fields ingested under path: \"%s.*\"", path) + logger.Warnf("Dynamic object found but no fields ingested under path: \"%s.*\"", path) return nil } @@ -506,8 +450,11 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit // there could be "sub-fields" with name "properties" too if couldBeParametersDefinition && isObject(actual) { if isObjectFullyDynamic(preview) { - // TODO: Skip for now, it should be required to compare with dynamic templates - logger.Debugf("Pending to validate with the dynamic templates defined the path: %q", path) + dynamicErrors := v.validateMappingsNotInPreview(path, actual, dynamicTemplates) + errs = append(errs, dynamicErrors...) + if len(errs) > 0 { + return errs.Unique() + } return nil } else if !isObject(preview) { errs = append(errs, fmt.Errorf("not found properties in preview mappings for path: %q", path)) @@ -521,7 +468,7 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit if err != nil { errs = append(errs, fmt.Errorf("found invalid properties type in actual mappings for path %q: %w", path, err)) } - compareErrors := v.compareMappings(path, false, previewProperties, actualProperties) + compareErrors := v.compareMappings(path, false, previewProperties, actualProperties, dynamicTemplates) errs = append(errs, compareErrors...) if len(errs) == 0 { @@ -533,7 +480,7 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit containsMultifield := isMultiFields(actual) if containsMultifield { if !isMultiFields(preview) { - errs = append(errs, fmt.Errorf("not found multi_fields in preview mappings for path: %q", path)) + errs = append(errs, fmt.Errorf("field %q is undefined: not found multi_fields definitions in preview mapping", path)) return errs.Unique() } previewFields, err := getMappingDefinitionsField("fields", preview) @@ -544,13 +491,13 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit if err != nil { errs = append(errs, fmt.Errorf("found invalid multi_fields type in actual mappings for path %q: %w", path, err)) } - compareErrors := v.compareMappings(path, false, previewFields, actualFields) + compareErrors := v.compareMappings(path, false, previewFields, actualFields, dynamicTemplates) errs = append(errs, compareErrors...) // not returning here to keep validating the other fields of this object if any } // Compare and validate the elements under "properties": objects or fields and its parameters - propertiesErrs := v.validateObjectProperties(path, true, containsMultifield, preview, actual) + propertiesErrs := v.validateObjectProperties(path, false, containsMultifield, preview, actual, dynamicTemplates) errs = append(errs, propertiesErrs...) if len(errs) == 0 { return nil @@ -558,7 +505,7 @@ func (v *MappingValidator) compareMappings(path string, couldBeParametersDefinit return errs.Unique() } -func (v *MappingValidator) validateObjectProperties(path string, couldBeParametersDefinition, containsMultifield bool, preview, actual map[string]any) multierror.Error { +func (v *MappingValidator) validateObjectProperties(path string, couldBeParametersDefinition, containsMultifield bool, preview, actual map[string]any, dynamicTemplates []map[string]any) multierror.Error { var errs multierror.Error for key, value := range actual { if containsMultifield && key == "fields" { @@ -576,19 +523,19 @@ func (v *MappingValidator) validateObjectProperties(path string, couldBeParamete if childField, ok := value.(map[string]any); ok { if isEmptyObject(childField) { // TODO: Should this be raised as an error instead? - logger.Debugf("field %q is an empty object and it does not exist in the preview", currentPath) + logger.Debugf("field %q skipped: empty object without definition in the preview", currentPath) continue } - ecsErrors := v.validateMappingsNotInPreview(currentPath, childField) + ecsErrors := v.validateMappingsNotInPreview(currentPath, childField, dynamicTemplates) errs = append(errs, ecsErrors...) continue } - // Parameter not defined + // Field or Parameter not defined errs = append(errs, fmt.Errorf("field %q is undefined", currentPath)) continue } - fieldErrs := v.validateObjectMappingAndParameters(preview[key], value, currentPath, true) + fieldErrs := v.validateObjectMappingAndParameters(preview[key], value, currentPath, dynamicTemplates, true) errs = append(errs, fieldErrs...) } if len(errs) == 0 { @@ -599,7 +546,7 @@ func (v *MappingValidator) validateObjectProperties(path string, couldBeParamete // validateMappingsNotInPreview validates the object and the nested objects in the current path with other resources // like ECS schema, dynamic templates or local fields defined in the package (type array). -func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, childField map[string]any) multierror.Error { +func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, childField map[string]any, rawDynamicTemplates []map[string]any) multierror.Error { var errs multierror.Error flattenFields, err := flattenMappings(currentPath, childField) if err != nil { @@ -607,9 +554,14 @@ func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, chil return errs } + dynamicTemplates, err := parseDynamicTemplates(rawDynamicTemplates) + if err != nil { + return multierror.Error{fmt.Errorf("failed to parse dynamic templates: %w", err)} + } + for fieldPath, object := range flattenFields { if slices.Contains(v.exceptionFields, fieldPath) { - logger.Warnf("Found exception field, skip its validation: %q", fieldPath) + logger.Warnf("Found exception field, skip its validation (not present in preview): %q", fieldPath) return nil } @@ -620,26 +572,59 @@ func (v *MappingValidator) validateMappingsNotInPreview(currentPath string, chil } if isEmptyObject(def) { - logger.Debugf("Skip empty object path: %q", fieldPath) + logger.Debugf("Skip field which value is an empty object: %q", fieldPath) continue } - // TODO: validate mapping with dynamic templates first than validating with ECS - // just raise an error if both validation processes fail + // validate whether or not the field has a corresponding dynamic template + if len(rawDynamicTemplates) > 0 { + err := v.matchingWithDynamicTemplates(fieldPath, def, dynamicTemplates) + if err == nil { + continue + } + } - // are all fields under this key defined in ECS? - err = v.validateMappingInECSSchema(fieldPath, def) - if err != nil { - logger.Warnf("undefined path %q (pending to check dynamic templates)", fieldPath) - errs = append(errs, fmt.Errorf("field %q is undefined: %w", fieldPath, err)) + // validate whether or not all fields under this key are defined in ECS + ecsErrs := v.validateMappingInECSSchema(fieldPath, def) + if len(ecsErrs) > 0 { + for _, e := range ecsErrs { + errs = append(errs, fmt.Errorf("field %q is undefined: %w", fieldPath, e)) + } } } return errs.Unique() } +// matchingWithDynamicTemplates validates a given definition (currentPath) with a set of dynamic templates. +// The dynamic templates parameters are based on https://www.elastic.co/guide/en/elasticsearch/reference/8.17/dynamic-templates.html +func (v *MappingValidator) matchingWithDynamicTemplates(currentPath string, definition map[string]any, dynamicTemplates []dynamicTemplate) error { + for _, template := range dynamicTemplates { + matches, err := template.Matches(currentPath, definition) + if err != nil { + return fmt.Errorf("failed to validate %q with dynamic template %q: %w", currentPath, template.name, err) + } + + if !matches { + // Look for another dynamic template + continue + } + + // Check that all parameters match (setting no dynamic templates to avoid recursion) + errs := v.validateObjectMappingAndParameters(template.mapping, definition, currentPath, []map[string]any{}, true) + if errs != nil { + // Look for another dynamic template + continue + } + + return nil + } + + return fmt.Errorf("no template matching for path: %q", currentPath) +} + // validateObjectMappingAndParameters validates the current object or field parameter (currentPath) comparing the values // in the actual mapping with the values in the preview mapping. -func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string, couldBeParametersDefinition bool) multierror.Error { +func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string, dynamicTemplates []map[string]any, couldBeParametersDefinition bool) multierror.Error { var errs multierror.Error switch actualValue.(type) { case map[string]any: @@ -652,7 +637,7 @@ func (v *MappingValidator) validateObjectMappingAndParameters(previewValue, actu if !ok { errs = append(errs, fmt.Errorf("unexpected type in actual mappings for path: %q", currentPath)) } - errs = append(errs, v.compareMappings(currentPath, couldBeParametersDefinition, previewField, actualField)...) + errs = append(errs, v.compareMappings(currentPath, couldBeParametersDefinition, previewField, actualField, dynamicTemplates)...) case any: // Validate each setting/parameter of the mapping // If a mapping exist in both preview and actual, they should be the same. But forcing to compare each parameter just in case diff --git a/internal/fields/mappings_test.go b/internal/fields/mappings_test.go index 76bf63c2d5..5682796762 100644 --- a/internal/fields/mappings_test.go +++ b/internal/fields/mappings_test.go @@ -14,15 +14,14 @@ import ( ) func TestComparingMappings(t *testing.T) { - defaultSpecVersion := "3.3.0" cases := []struct { - title string - preview map[string]any - actual map[string]any - schema []FieldDefinition - spec string - exceptionFields []string - expectedErrors []string + title string + preview map[string]any + actual map[string]any + schema []FieldDefinition + dynamicTemplates []map[string]any + exceptionFields []string + expectedErrors []string }{ { title: "same mappings", @@ -84,7 +83,7 @@ func TestComparingMappings(t *testing.T) { expectedErrors: []string{}, }, { - title: "validate field with ECS", + title: "validate fields with ECS", preview: map[string]any{ "foo": map[string]any{ "type": "keyword", @@ -100,6 +99,25 @@ func TestComparingMappings(t *testing.T) { "foo": map[string]any{ "type": "keyword", }, + "user": map[string]any{ + "type": "keyword", + }, + "time": map[string]any{ + "type": "keyword", + "fields": map[string]any{ + "text": map[string]any{ + "type": "match_only_text", + }, + // there should be a dynamic template in order to exist this multi-field + "other": map[string]any{ + "type": "match_only_text", + }, + }, + }, + // Should this fail since it has no multi-fields as in the ECS definition? + "name": map[string]any{ + "type": "keyword", + }, }, schema: []FieldDefinition{ { @@ -117,9 +135,34 @@ func TestComparingMappings(t *testing.T) { Type: "keyword", External: "", }, + { + Name: "time", + Type: "keyword", + External: "ecs", + MultiFields: []FieldDefinition{ + { + Name: "text", + Type: "match_only_text", + External: "ecs", + }, + }, + }, + { + Name: "name", + Type: "keyword", + External: "ecs", + MultiFields: []FieldDefinition{ + { + Name: "text", + Type: "match_only_text", + External: "ecs", + }, + }, + }, }, expectedErrors: []string{ `field "metrics" is undefined: actual mapping type (long) does not match with ECS definition type: keyword`, + `field "user" is undefined: field definition not found`, }, }, { @@ -172,7 +215,7 @@ func TestComparingMappings(t *testing.T) { }, schema: []FieldDefinition{}, expectedErrors: []string{ - `field "foo" is undefined: missing definition for path`, + `field "foo" is undefined: field definition not found`, }, }, { @@ -191,7 +234,7 @@ func TestComparingMappings(t *testing.T) { }, schema: []FieldDefinition{}, expectedErrors: []string{ - `constant_keyword value in preview "example" does not match the actual mapping value "bar" for path: "foo"`, + `invalid value in field "foo": constant_keyword value in preview "example" does not match the actual mapping value "bar"`, }, }, { @@ -231,6 +274,14 @@ func TestComparingMappings(t *testing.T) { { title: "validate multifields failure", preview: map[string]any{ + "time": map[string]any{ + "type": "keyword", + "fields": map[string]any{ + "text": map[string]any{ + "type": "match_only_text", + }, + }, + }, "foo": map[string]any{ "type": "keyword", "fields": map[string]any{ @@ -242,7 +293,7 @@ func TestComparingMappings(t *testing.T) { "bar": map[string]any{ "properties": map[string]any{ "type": map[string]any{ - "type": "constant_keyword", + "type": "keyword", }, "fields": map[string]any{ "type": "text", @@ -256,6 +307,14 @@ func TestComparingMappings(t *testing.T) { }, }, actual: map[string]any{ + "time": map[string]any{ + "type": "keyword", + "fields": map[string]any{ + "text": map[string]any{ + "type": "match_only_text", + }, + }, + }, "foo": map[string]any{ "type": "keyword", "fields": map[string]any{ @@ -270,7 +329,12 @@ func TestComparingMappings(t *testing.T) { "bar": map[string]any{ "properties": map[string]any{ "type": map[string]any{ - "type": "constant_keyword", + "type": "keyword", + "fields": map[string]any{ + "text": map[string]any{ + "type": "match_only_text", + }, + }, }, "fields": map[string]any{ "type": "text", @@ -285,7 +349,8 @@ func TestComparingMappings(t *testing.T) { }, schema: []FieldDefinition{}, expectedErrors: []string{ - `field "foo.text" is undefined: missing definition for path`, + `field "foo.text" is undefined: field definition not found`, + `field "bar.type" is undefined: not found multi_fields definitions in preview mapping`, }, }, { @@ -307,7 +372,7 @@ func TestComparingMappings(t *testing.T) { }, schema: []FieldDefinition{}, expectedErrors: []string{ - `not found multi_fields in preview mappings for path: "foo"`, + `field "foo" is undefined: not found multi_fields definitions in preview mapping`, }, }, { @@ -338,8 +403,8 @@ func TestComparingMappings(t *testing.T) { }, schema: []FieldDefinition{}, expectedErrors: []string{ - `field "file.path" is undefined: missing definition for path`, - `field "bar" is undefined: missing definition for path`, + `field "file.path" is undefined: field definition not found`, + `field "bar" is undefined: field definition not found`, }, }, { @@ -367,11 +432,11 @@ func TestComparingMappings(t *testing.T) { schema: []FieldDefinition{}, expectedErrors: []string{ // TODO: there is an exception in the logic to not raise this error - // `field "_tmp" is undefined: missing definition for path`, + // `field "_tmp" is undefined: field definition not found`, }, }, { - title: "skip dynamic objects", // TODO: should this be checked using dynamic templates? + title: "validate fully dynamic objects in preview", preview: map[string]any{ "@timestamp": map[string]any{ "type": "keyword", @@ -385,6 +450,13 @@ func TestComparingMappings(t *testing.T) { "type": "object", "dynamic": "true", }, + "string": map[string]any{ + "type": "object", + "dynamic": "true", + }, + "foo": map[string]any{ + "type": "keyword", + }, }, }, }, @@ -407,62 +479,39 @@ func TestComparingMappings(t *testing.T) { }, }, }, - }, - }, - }, - }, - }, - schema: []FieldDefinition{}, - expectedErrors: []string{}, - }, - { - title: "compare all objects even dynamic true", // TODO: should this be checked using dynamic templates? - preview: map[string]any{ - "@timestamp": map[string]any{ - "type": "keyword", - }, - "sql": map[string]any{ - "properties": map[string]any{ - "metrics": map[string]any{ - "properties": map[string]any{ - "dynamic": "true", - "numeric": map[string]any{ - "type": "object", - "dynamic": "true", - }, - }, - }, - }, - }, - }, - actual: map[string]any{ - "@timestamp": map[string]any{ - "type": "keyword", - }, - "sql": map[string]any{ - "properties": map[string]any{ - "metrics": map[string]any{ - "properties": map[string]any{ - "dynamic": "true", - "numeric": map[string]any{ + "string": map[string]any{ "dynamic": "true", "properties": map[string]any{ "innodb_data_fsyncs": map[string]any{ - "type": "long", + "type": "keyword", }, }, }, "example": map[string]any{ "type": "keyword", }, + "foo": map[string]any{ + "type": "keyword", + }, }, }, }, }, }, + dynamicTemplates: []map[string]any{ + { + "sql.metrics.string.*": map[string]any{ + "path_match": "sql.metrics.string.*", + "mapping": map[string]any{ + "type": "keyword", + }, + }, + }, + }, schema: []FieldDefinition{}, expectedErrors: []string{ - `field "sql.metrics.example" is undefined: missing definition for path`, + `field "sql.metrics.numeric.innodb_data_fsyncs" is undefined: field definition not found`, + `field "sql.metrics.example" is undefined: field definition not found`, }, }, { @@ -516,9 +565,8 @@ func TestComparingMappings(t *testing.T) { }, }, exceptionFields: []string{"access.field"}, - spec: "1.0.0", expectedErrors: []string{ - `field "error.field" is undefined: missing definition for path`, + `field "error.field" is undefined: field definition not found`, // should status.field return error ? or should it be ignored? `field "status.field" is undefined: actual mapping type (keyword) does not match with ECS definition type: array`, }, @@ -678,7 +726,6 @@ func TestComparingMappings(t *testing.T) { }, // foo is added to the exception list because it is type nested exceptionFields: []string{"foo"}, - spec: "3.0.0", schema: []FieldDefinition{}, expectedErrors: []string{}, }, @@ -700,30 +747,117 @@ func TestComparingMappings(t *testing.T) { }, }, exceptionFields: []string{}, - spec: "3.0.1", schema: []FieldDefinition{}, expectedErrors: []string{ `not found properties in preview mappings for path: "foo"`, }, }, + { + title: "fields matching dynamic templates", + preview: map[string]any{}, + actual: map[string]any{ + "foo": map[string]any{ + "type": "keyword", + }, + "foa": map[string]any{ + "type": "double", + }, + "fob": map[string]any{ + "type": "double", + "time_series_metric": "gauge", + }, + "bar": map[string]any{ + "type": "text", + "fields": map[string]any{ + "text": map[string]any{ + "type": "keyword", + }, + }, + }, + "bar_double": map[string]any{ + "type": "double", + }, + "full_regex_1": map[string]any{ + "type": "double", + }, + }, + dynamicTemplates: []map[string]any{ + { + "fo*_keyword": map[string]any{ + "path_match": "fo*", + "path_unmatch": []any{"foa", "fob"}, + "unmatch_mapping_type": []any{"long", "double"}, + "mapping": map[string]any{ + "type": "keyword", + }, + }, + }, + { + "fo*_number": map[string]any{ + "path_match": "fo*", + "path_unmatch": "foo", + "match_mapping_type": []any{"long", "double"}, + "mapping": map[string]any{ + "type": "double", + "time_series_metric": "counter", + }, + }, + }, + { + "bar_match": map[string]any{ + "unmatch": []any{"foo", "foo42", "*42"}, + "match": []any{"*ar", "bar42"}, + "match_mapping_type": "text", + "mapping": map[string]any{ + "type": "text", + "fields": map[string]any{ + "text": map[string]any{ + "type": "keyword", + }, + }, + }, + }, + }, + { + "bar_star_double": map[string]any{ + "match": "*", + "unmatch": "full*", + "unmatch_mapping_type": []any{"text"}, + "mapping": map[string]any{ + "type": "double", + }, + }, + }, + { + "full_regex_1": map[string]any{ + "match_pattern": "regex", + "match": "^full_.*\\d$", + "mapping": map[string]any{ + "type": "double", + }, + }, + }, + }, + exceptionFields: []string{}, + schema: []FieldDefinition{}, + expectedErrors: []string{ + // Should it be considered this error in "foa" "missing time_series_metric bar"? + // "fob" is failing because it does not have the expected value for the "time_series_metric" field + `field "fob" is undefined: field definition not found`, + }, + }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { logger.EnableDebugMode() - specVersion := defaultSpecVersion - if c.spec != "" { - specVersion = c.spec - } - v, err := CreateValidatorForMappings("", nil, - WithMappingValidatorSpecVersion(specVersion), + v, err := CreateValidatorForMappings(nil, WithMappingValidatorFallbackSchema(c.schema), - WithMappingValidatorDisabledDependencyManagement(), WithMappingValidatorExceptionFields(c.exceptionFields), ) require.NoError(t, err) - errs := v.compareMappings("", false, c.preview, c.actual) + errs := v.compareMappings("", false, c.preview, c.actual, c.dynamicTemplates) if len(c.expectedErrors) > 0 { assert.Len(t, errs, len(c.expectedErrors)) for _, err := range errs { diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 978f96781e..62fe012528 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -30,6 +30,8 @@ import ( "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) +const externalFieldAppendedTag = "ecs_component" + var ( semver2_0_0 = semver.MustParse("2.0.0") semver2_3_0 = semver.MustParse("2.3.0") @@ -448,7 +450,7 @@ func appendECSMappingMultifields(schema []FieldDefinition, prefix string) []Fiel { Name: "text", Type: "match_only_text", - External: "ecs", + External: externalFieldAppendedTag, }, }, }, diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 53327bac6e..7f82c13fd2 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1535,12 +1535,10 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re logger.Warn("Validate mappings found (technical preview)") exceptionFields := listExceptionFields(scenario.docs, fieldsValidator) - mappingsValidator, err := fields.CreateValidatorForMappings(r.dataStreamPath, r.esClient, + mappingsValidator, err := fields.CreateValidatorForMappings(r.esClient, fields.WithMappingValidatorFallbackSchema(fieldsValidator.Schema), fields.WithMappingValidatorIndexTemplate(scenario.indexTemplateName), fields.WithMappingValidatorDataStream(scenario.dataStream), - fields.WithMappingValidatorSpecVersion(r.pkgManifest.SpecVersion), - fields.WithMappingValidatorEnabledImportAllECSSChema(true), fields.WithMappingValidatorExceptionFields(exceptionFields), ) if err != nil {