diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh index 2c5741b544..f97e51cd31 100755 --- a/scripts/test-check-false-positives.sh +++ b/scripts/test-check-false-positives.sh @@ -7,6 +7,11 @@ source "${SCRIPT_DIR}/stack_helpers.sh" set -euxo pipefail +# Add default values +export SUFFIX_FOLDER_DUMP_LOGS="${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" +export PACKAGE_TEST_TYPE="${PACKAGE_TEST_TYPE:-"false_positives"}" +export PACKAGE_UNDER_TEST="${PACKAGE_UNDER_TEST:-*}" + function cleanup() { r=$? if [ "${r}" -ne 0 ]; then @@ -18,14 +23,14 @@ function cleanup() { if is_stack_created ; then # Dump stack logs # Required containers could not be running, so ignore the error - elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" || true + elastic-package stack dump -v --output "build/elastic-stack-dump/check-${SUFFIX_FOLDER_DUMP_LOGS}}" || true # Take down the stack elastic-package stack down -v fi # Clean used resources - for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do elastic-package clean -C "$d" -v done @@ -117,9 +122,10 @@ elastic-package stack up -d -v ${stack_args} elastic-package stack status # Run package tests -for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do +for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do echo "--- Check build output: ${d}" check_build_output "$d" + echo "--- Check expected errors: ${d}" check_expected_errors "$d" done diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 0caaffe760..3c325c7b0c 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -7,6 +7,11 @@ source "${SCRIPT_DIR}/stack_helpers.sh" set -euxo pipefail +# Add default values +export SUFFIX_FOLDER_DUMP_LOGS="${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" +export PACKAGE_TEST_TYPE="${PACKAGE_TEST_TYPE:-"other"}" +export PACKAGE_UNDER_TEST="${PACKAGE_UNDER_TEST:-*}" + cleanup() { r=$? if [ "${r}" -ne 0 ]; then @@ -19,10 +24,10 @@ cleanup() { # Dump stack logs # Required containers could not be running, so ignore the error elastic-package stack dump -v \ - --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" || true + --output "build/elastic-stack-dump/check-${SUFFIX_FOLDER_DUMP_LOGS}" || true fi - if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then + if [ "${PACKAGE_TEST_TYPE}" == "with-kind" ]; then # Dump kubectl details kubectl describe pods --all-namespaces > build/kubectl-dump.txt kubectl logs -l app=elastic-agent -n kube-system >> build/kubectl-dump.txt @@ -44,13 +49,13 @@ cleanup() { fi fi - if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then + if [ "${PACKAGE_TEST_TYPE}" == "with-logstash" ]; then # Delete the logstash profile elastic-package profiles delete logstash -v fi # Clean used resources - for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do + for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do elastic-package clean -C "$d" -v done @@ -63,13 +68,56 @@ ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH export SERVERLESS=${SERVERLESS:-"false"} +run_system_benchmark() { + local package_name="$1" + local package_path="$2" + echo "--- Run system benchmarks for package ${package_name}" + elastic-package benchmark system -C "$package_path" --benchmark logs-benchmark -v --defer-cleanup 1s +} + +run_serverless_tests() { + local package_path="$1" + local test_options="-v --report-format xUnit --report-output file --defer-cleanup 1s" + local coverage_options="--test-coverage --coverage-format=generic" + + echo "--- Run tests for package ${package_path} in Serverless mode" + # skip system tests + elastic-package test asset -C "$package_path" $test_options $coverage_options + elastic-package test static -C "$package_path" $test_options $coverage_options + # FIXME: adding test-coverage for serverless results in errors like this: + # Error: error running package pipeline tests: could not complete test run: error calculating pipeline coverage: error fetching pipeline stats for code coverage calculations: need exactly one ES node in stats response (got 4) + elastic-package test pipeline -C "$package_path" $test_options +} + +run_pipeline_benchmark() { + local package_name="$1" + local package_path="$2" + local test_options="-v --report-format xUnit --report-output file --fail-on-missing" + + echo "--- Run pipeline benchmarks and report for package ${package_name}" + + rm -rf "${PWD}/build/benchmark-results" + elastic-package benchmark pipeline -C "$d" $test_options + + rm -rf "${PWD}/build/benchmark-results-old" + mv "${PWD}/build/benchmark-results" "${PWD}/build/benchmark-results-old" + + elastic-package benchmark pipeline -C "$d" $test_options + + elastic-package report -C "$d" --fail-on-missing benchmark \ + --new "${PWD}/build/benchmark-results" \ + --old "${PWD}/build/benchmark-results-old" \ + --threshold 1 --report-output-path="${PWD}/build/benchreport" +} + + # Build/check packages -for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do +for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do echo "--- Checking package ${d}" elastic-package check -C "$d" -v done -if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then +if [ "${PACKAGE_TEST_TYPE}" == "with-logstash" ]; then echo "--- Create logstash profile" # Create a logstash profile and use it @@ -103,58 +151,46 @@ if [[ "${SERVERLESS}" != "true" ]]; then elastic-package stack status fi -if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then +if [ "${PACKAGE_TEST_TYPE}" == "with-kind" ]; then # Boot up the kind cluster echo "--- Create kind cluster" kind create cluster --config "$PWD/scripts/kind-config.yaml" --image "kindest/node:${K8S_VERSION}" fi # Run package tests -for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do +for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do package_to_test=$(basename "${d}") - if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then - # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop - if [ "${package_to_test}" == "pipeline_benchmark" ]; then - echo "--- Run pipeline benchmarks and report for package ${package_to_test}" - - rm -rf "${PWD}/build/benchmark-results" - elastic-package benchmark pipeline -C "$d" -v --report-format xUnit --report-output file --fail-on-missing - - rm -rf "${PWD}/build/benchmark-results-old" - mv "${PWD}/build/benchmark-results" "${PWD}/build/benchmark-results-old" - - elastic-package benchmark pipeline -C "$d" -v --report-format json --report-output file --fail-on-missing - - elastic-package report -C "$d" --fail-on-missing benchmark \ - --new "${PWD}/build/benchmark-results" \ - --old "${PWD}/build/benchmark-results-old" \ - --threshold 1 --report-output-path="${PWD}/build/benchreport" - fi - if [ "${package_to_test}" == "system_benchmark" ]; then - echo "--- Run system benchmarks for package ${package_to_test}" + if [ "${PACKAGE_TEST_TYPE}" == "benchmarks" ]; then + # FIXME: There are other packages in test/packages/benchmarks folder that are not tested like rally_benchmark + case "${package_to_test}" in + pipeline_benchmark|use_pipeline_tests) + run_pipeline_benchmark "${package_to_test}" "$d" + ;; + system_benchmark) + run_system_benchmark "${package_to_test}" "$d" + ;; + esac + continue + fi - elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s - fi - elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${package_to_test}" == "system_benchmark" ]; then - echo "--- Run system benchmarks for package ${package_to_test}" - elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s - else - if [[ "${SERVERLESS}" == "true" ]]; then - echo "--- Run tests for package ${d} in Serverless mode" - # skip system tests - elastic-package test asset -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic - elastic-package test static -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic - # FIXME: adding test-coverage for serverless results in errors like this: - # Error: error running package pipeline tests: could not complete test run: error calculating pipeline coverage: error fetching pipeline stats for code coverage calculations: need exactly one ES node in stats response (got 4) - elastic-package test pipeline -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s - - continue - fi + if [ "${PACKAGE_TEST_TYPE}" == "with-logstash" ] && [ "${package_to_test}" == "system_benchmark" ]; then + run_system_benchmark "${package_to_test}" "$d" + continue + fi - echo "--- Run tests for package ${d}" - # Run all tests - # defer-cleanup is set to a short period to verify that the option is available - elastic-package test -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic + if [[ "${SERVERLESS}" == "true" ]]; then + run_serverless_tests "${d}" + continue fi + + echo "--- Run tests for package ${d}" + # Run all tests + # defer-cleanup is set to a short period to verify that the option is available + elastic-package test -C "$d" -v \ + --report-format xUnit \ + --report-output file \ + --defer-cleanup 1s \ + --test-coverage \ + --coverage-format=generic done