From a372e7088977a671b39f76b62695d4bcf76a7e3f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 23 Jan 2025 13:01:12 +0100 Subject: [PATCH 1/2] Check index deprecation warnings on system tests --- internal/testrunner/runners/system/tester.go | 101 +++++++++++++++++-- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 97a01f4b73..f489f05a7e 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -874,18 +874,85 @@ func (r *tester) getFailureStoreDocs(ctx context.Context, dataStream string) ([] return docs, nil } +type deprecationWarning struct { + Level string `json:"level"` + Message string `json:"message"` + URL string `json:"url"` + Details string `json:"details"` + + ResolveDuringRollingUpgrade bool `json:"resolve_during_rolling_upgrade"` + + index string +} + +func (r *tester) getDeprecationWarnings(ctx context.Context, dataStream string) ([]deprecationWarning, error) { + resp, err := r.esAPI.Migration.Deprecations( + r.esAPI.Migration.Deprecations.WithContext(ctx), + r.esAPI.Migration.Deprecations.WithIndex(dataStream), + ) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.IsError() { + return nil, fmt.Errorf("unexpected status code in response: %s", resp.String()) + } + + // Apart from index_settings, there are also cluster_settings, node_settings and ml_settings. + // There is also a data_streams field in the response that is not documented and is empty. + // Here we are interested only on warnings on index settings. + var results struct { + IndexSettings map[string][]deprecationWarning `json:"index_settings"` + } + err = json.NewDecoder(resp.Body).Decode(&results) + if err != nil { + return nil, fmt.Errorf("cannot decode response: %w", err) + } + + var result []deprecationWarning + for index, warnings := range results.IndexSettings { + for _, warning := range warnings { + warning.index = index + result = append(result, warning) + } + } + return result, nil +} + +func (r *tester) checkDeprecationWarnings(dataStream string, warnings []deprecationWarning, configName string) []testrunner.TestResult { + var results []testrunner.TestResult + for _, warning := range warnings { + details := warning.Details + if warning.index != "" { + details = fmt.Sprintf("%s (index: %s)", details, warning.index) + } + tr := testrunner.TestResult{ + TestType: TestType, + Name: "Deprecation warnings - " + configName, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, + FailureMsg: warning.Message, + FailureDetails: details, + } + results = append(results, tr) + } + return results +} + type scenarioTest struct { - dataStream string - indexTemplateName string - policyTemplateName string - kibanaDataStream kibana.PackageDataStream - syntheticEnabled bool - docs []common.MapStr - failureStore []failureStoreDocument - ignoredFields []string - degradedDocs []common.MapStr - agent agentdeployer.DeployedAgent - startTestTime time.Time + dataStream string + indexTemplateName string + policyTemplateName string + kibanaDataStream kibana.PackageDataStream + syntheticEnabled bool + docs []common.MapStr + failureStore []failureStoreDocument + deprecationWarnings []deprecationWarning + ignoredFields []string + degradedDocs []common.MapStr + agent agentdeployer.DeployedAgent + startTestTime time.Time } type pipelineTrace []string @@ -1293,6 +1360,14 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, stackC return hits.size() > 0, nil }, 1*time.Second, waitForDataTimeout) + // Get deprecation warnings after ensuring that there are ingested docs and thus the + // data stream exists. + scenario.deprecationWarnings, err = r.getDeprecationWarnings(ctx, scenario.dataStream) + if err != nil { + return nil, fmt.Errorf("failed to get deprecation warnings for data stream %s: %w", scenario.dataStream, err) + } + logger.Debugf("Found %d deprecation warnings for data stream %s", len(scenario.deprecationWarnings), scenario.dataStream) + if service != nil && config.Service != "" && !config.IgnoreServiceError { exited, code, err := service.ExitCode(ctx, config.Service) if err != nil && !errors.Is(err, servicedeployer.ErrNotSupported) { @@ -1597,6 +1672,10 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } } + if results := r.checkDeprecationWarnings(scenario.dataStream, scenario.deprecationWarnings, config.Name()); len(results) > 0 { + return results, nil + } + if r.withCoverage { coverage, err := r.generateCoverageReport(result.CoveragePackageName()) if err != nil { From bff482d0fd7e57f25d2c4b85ab73d7aadabad9c6 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 23 Jan 2025 19:35:14 +0100 Subject: [PATCH 2/2] Ignore deprecation warning about source mode in 8.17.0 --- internal/testrunner/runners/system/tester.go | 53 +++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index f489f05a7e..9af5dc54c3 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -920,9 +920,12 @@ func (r *tester) getDeprecationWarnings(ctx context.Context, dataStream string) return result, nil } -func (r *tester) checkDeprecationWarnings(dataStream string, warnings []deprecationWarning, configName string) []testrunner.TestResult { +func (r *tester) checkDeprecationWarnings(stackVersion *semver.Version, dataStream string, warnings []deprecationWarning, configName string) []testrunner.TestResult { var results []testrunner.TestResult for _, warning := range warnings { + if ignoredDeprecationWarning(stackVersion, warning) { + continue + } details := warning.Details if warning.index != "" { details = fmt.Sprintf("%s (index: %s)", details, warning.index) @@ -940,6 +943,39 @@ func (r *tester) checkDeprecationWarnings(dataStream string, warnings []deprecat return results } +func mustParseConstraint(c string) *semver.Constraints { + constraint, err := semver.NewConstraint(c) + if err != nil { + panic(err) + } + return constraint +} + +var ignoredWarnings = []struct { + constraints *semver.Constraints + pattern *regexp.Regexp +}{ + { + // This deprecation warning was introduced in 8.17.0 and fixed in Fleet in 8.17.2. + // See https://github.com/elastic/kibana/pull/207133 + // Ignoring it because packages cannot do much about this on these versions. + constraints: mustParseConstraint(`>=8.17.0,<8.17.2`), + pattern: regexp.MustCompile(`^Configuring source mode in mappings is deprecated and will be removed in future versions.`), + }, +} + +func ignoredDeprecationWarning(stackVersion *semver.Version, warning deprecationWarning) bool { + for _, rule := range ignoredWarnings { + if rule.constraints != nil && !rule.constraints.Check(stackVersion) { + continue + } + if rule.pattern.MatchString(warning.Message) { + return true + } + } + return false +} + type scenarioTest struct { dataStream string indexTemplateName string @@ -1605,7 +1641,12 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } } - err = validateIgnoredFields(r.stackVersion.Number, scenario, config) + stackVersion, err := semver.NewVersion(r.stackVersion.Number) + if err != nil { + return result.WithErrorf("failed to parse stack version: %w", err) + } + + err = validateIgnoredFields(stackVersion, scenario, config) if err != nil { return result.WithError(err) } @@ -1672,7 +1713,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } } - if results := r.checkDeprecationWarnings(scenario.dataStream, scenario.deprecationWarnings, config.Name()); len(results) > 0 { + if results := r.checkDeprecationWarnings(stackVersion, scenario.dataStream, scenario.deprecationWarnings, config.Name()); len(results) > 0 { return results, nil } @@ -2259,12 +2300,8 @@ func listExceptionFields(docs []common.MapStr, fieldsValidator *fields.Validator return allFields } -func validateIgnoredFields(stackVersionString string, scenario *scenarioTest, config *testConfig) error { +func validateIgnoredFields(stackVersion *semver.Version, scenario *scenarioTest, config *testConfig) error { skipIgnoredFields := append([]string(nil), config.SkipIgnoredFields...) - stackVersion, err := semver.NewVersion(stackVersionString) - if err != nil { - return fmt.Errorf("failed to parse stack version: %w", err) - } if stackVersion.LessThan(semver.MustParse("8.14.0")) { // Pre 8.14 Elasticsearch commonly has event.original not mapped correctly, exclude from check: https://github.com/elastic/elasticsearch/pull/106714 skipIgnoredFields = append(skipIgnoredFields, "event.original")