diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 2d9276f604..9ec2bffbe1 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -110,22 +110,6 @@ echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package -# Add steps to test validation method mappings -while IFS= read -r -d '' package ; do - package_name=$(basename "${package}") - echo " - label: \":go: Integration test: ${package_name} (validate mappings)\"" - echo " key: \"integration-parallel-${package_name}-agent-validate-mappings\"" - echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" - echo " env:" - echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: mappings" - echo " agents:" - echo " provider: \"gcp\"" - echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\"" - echo " artifact_paths:" - echo " - build/test-results/*.xml" - echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package -done < <(find . -maxdepth 1 -mindepth 1 -type d -print0) popd > /dev/null # TODO: Missing docker & docker-compose in MACOS ARM agent image, skip installation of packages in the meantime. diff --git a/README.md b/README.md index d6d7fa4bb6..7af32d452f 100644 --- a/README.md +++ b/README.md @@ -689,10 +689,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the package (and unenrolled at the end of each test). Default: `true`. - - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this - variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to - `mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If - set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`. + - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `mappings` or `fields`. If this + variable is set to `fields`, then validation of fields will be based on the contents of the documents ingested into Elasticsearch. If this is set to + `mappings`, then validation of fields will be based on their mappings generated when the documents are ingested into Elasticsearch as well as + the contents of the documents ingested into Elasticsearch. + Default option: `mappings`. - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index c626e53053..8383dbed2f 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -18,6 +18,8 @@ Conceptually, running a system test involves the following steps: 1. Validate mappings are defined for the fields contained in the indexed documents. 1. Validate that the JSON data types contained `_source` are compatible with mappings declared for the field. +1. Validate mappings generated after ingesting documents are valid according to the definitions installed by the package. + - Requires `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD` to be unset or set to `mappings`. 1. If the Elastic Agent from the stack is not used, unenroll and remove the Elastic Agent as well as the test policies created. 1. Delete test artifacts and tear down the instance of the package's integration service. 1. Once the data stream have been system tested, unenroll and remove the Elastic Agent diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 6ac51a25bf..265f368e18 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -288,8 +288,8 @@ func NewSystemTester(options SystemTesterOptions) (*tester, error) { r.runIndependentElasticAgent = strings.ToLower(v) == "true" } - // default method using just fields - r.fieldValidationMethod = fieldsMethod + // default method to validate using mappings (along with fields) + r.fieldValidationMethod = mappingsMethod v, ok = os.LookupEnv(fieldValidationTestMethodEnv) if ok { method, ok := validationMethods[v] @@ -1646,7 +1646,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } if r.fieldValidationMethod == mappingsMethod { - logger.Warn("Validation based on mappings enabled (technical preview)") + logger.Debug("Performing validation based on mappings") exceptionFields := listExceptionFields(scenario.docs, fieldsValidator) mappingsValidator, err := fields.CreateValidatorForMappings(r.esClient, diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 8070afd434..0bbbea880b 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -243,10 +243,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the package (and unenrolled at the end of each test). Default: `true`. - - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this - variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to - `mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If - set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`. + - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `mappings` or `fields`. If this + variable is set to `fields`, then validation of fields will be based on the contents of the documents ingested into Elasticsearch. If this is set to + `mappings`, then validation of fields will be based on their mappings generated when the documents are ingested into Elasticsearch as well as + the contents of the documents ingested into Elasticsearch. + Default option: `mappings`. - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)