diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index ad4fc6450f..940c9454ac 100755 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -4,15 +4,20 @@ source .buildkite/scripts/tooling.sh set -euo pipefail +exit_code=0 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then echo "--- Take down the Elastic stack" # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package # is already installed in the test_serverless pipeline step, accessing # directly to the binary - "${HOME}"/go/bin/elastic-package stack down -v + if ! "${HOME}"/go/bin/elastic-package stack down -v ; then + # Ensure that even if this command fails, the script continues + exit_code=1 + fi fi echo "--- Cleanup" cleanup unset_secrets +exit "$exit_code" diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml index 674cb0ecdb..d00ea8664b 100644 --- a/.buildkite/pipeline.serverless.yml +++ b/.buildkite/pipeline.serverless.yml @@ -45,6 +45,7 @@ steps: provider: "gcp" image: "${UBUNTU_X86_64_AGENT_IMAGE}" env: + SERVERLESS: "true" SERVERLESS_PROJECT: "${SERVERLESS_PROJECT:-observability}" UPLOAD_SAFE_LOGS: 1 # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index c84889ce28..6648926557 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -18,9 +18,11 @@ FALSE_POSITIVES_TARGET="test-check-packages-false-positives" KIND_TARGET="test-check-packages-with-kind" SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags" TEST_BUILD_ZIP_TARGET="test-build-zip" +TEST_BUILD_INSTALL_ZIP_TARGET="test-build-install-zip" REPO_NAME=$(repo_name "${BUILDKITE_REPO}") -export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" +REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" +export REPO_BUILD_TAG TARGET="" PACKAGE="" SERVERLESS="false" @@ -58,50 +60,67 @@ if [[ "${TARGET}" == "" ]]; then exit 1 fi +if [[ "${SERVERLESS}" == "true" && "${TARGET}" != "${PARALLEL_TARGET}" ]]; then + # Just tested parallel target to run with Serverless projects, other Makefile targets + # have not been tested yet and could fail unexpectedly. For instance, "test-check-package-false-positives" + # target would require a different management to not stop Elastic stack after each package test. + echo "Target ${TARGET} is not supported for Serverless testing" + usage + exit 1 +fi + add_bin_path if [[ "$SERVERLESS" == "false" ]]; then - # If packages are tested with Serverless, these action are already performed + # If packages are tested with Serverless, these actions are already performed # here: .buildkite/scripts/test_packages_with_serverless.sh - echo "--- install go" + echo "--- Install go" with_go if [[ "${TARGET}" != "${TEST_BUILD_ZIP_TARGET}" ]]; then # Not supported in Macos ARM - echo "--- install docker" + echo "--- Install docker" with_docker - echo "--- install docker-compose plugin" + echo "--- Install docker-compose plugin" with_docker_compose_plugin fi -fi -echo "--- install yq" -with_yq + # In Serverless pipeline, elastic-package is installed in advance here: + # .buildkite/scripts/test_packages_with_serverless.sh + # No need to install it again for every package. + echo "--- Install elastic-package" + make install +fi if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then - echo "--- install kubectl & kind" + echo "--- Install kubectl & kind" with_kubernetes fi +if [[ "${TARGET}" == "${TEST_BUILD_INSTALL_ZIP_TARGET}" || "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then + echo "--- Install yq" + with_yq +fi + label="${TARGET}" if [ -n "${PACKAGE}" ]; then label="${label} - ${PACKAGE}" fi + echo "--- Run integration test ${label}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then - make install - # allow to fail this command, to be able to upload safe logs set +e make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}" testReturnCode=$? set -e - retry_count=${BUILDKITE_RETRY_COUNT:-"0"} if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then + retry_count=${BUILDKITE_RETRY_COUNT:-"0"} package_folder="${PACKAGE}" + if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-""}" == "false" ]]; then package_folder="${package_folder}-stack_agent" fi @@ -114,6 +133,8 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT package_folder="${package_folder}_retry_${retry_count}" fi + echo "--- Uploading safe logs to GCP bucket ${JOB_GCS_BUCKET_INTERNAL}" + upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ @@ -135,9 +156,10 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}" exit ${testReturnCode} fi - - make check-git-clean - exit 0 +else + make "${TARGET}" fi -make install "${TARGET}" check-git-clean +echo "--- Check git clean" +make check-git-clean +exit 0 diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh index 3aa20825c1..3491d7cd69 100755 --- a/.buildkite/scripts/release.sh +++ b/.buildkite/scripts/release.sh @@ -17,11 +17,11 @@ VERSION="" add_bin_path with_go -echo "--- fetching tags" +echo "--- Fetching tags" # Ensure that tags are present so goreleaser can build the changelog from the last release. git rev-parse --is-shallow-repository git fetch origin --tags -echo "--- running goreleaser" +echo "--- Running goreleaser" # Run latest version of goreleaser curl -sL https://git.io/goreleaser | bash diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh index 2723a46711..5cd6755e0a 100755 --- a/.buildkite/scripts/test-with-integrations.sh +++ b/.buildkite/scripts/test-with-integrations.sh @@ -5,10 +5,10 @@ set -euo pipefail add_bin_path -echo "--- install gh cli" +echo "--- Install gh cli" with_github_cli -echo "--- install jq" +echo "--- Install jq" with_jq @@ -87,7 +87,7 @@ create_integrations_pull_request() { update_dependency() { # it needs to set the Golang version from the integrations repository (.go-version file) - echo "--- install go for integrations repository :go:" + echo "--- Install go for integrations repository :go:" with_go echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:" @@ -167,7 +167,7 @@ create_or_update_pull_request() { rm -rf "${temp_path}" - echo "--- adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:" + echo "--- Adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:" add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link "${integrations_pr_number}")" } @@ -186,5 +186,5 @@ if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then exit 1 fi -echo "--- creating or updating integrations pull request" +echo "--- Creating or updating integrations pull request" create_or_update_pull_request diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh index 890b3d8b28..26d7ceafe3 100755 --- a/.buildkite/scripts/test_packages_with_serverless.sh +++ b/.buildkite/scripts/test_packages_with_serverless.sh @@ -10,7 +10,6 @@ UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"} SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt" FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt" -export SERVERLESS="true" SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"} add_pr_comment() { @@ -32,7 +31,7 @@ fi add_bin_path -echo "--- install go" +echo "--- Install go" with_go echo "--- Install docker" @@ -42,7 +41,7 @@ echo "--- Install docker-compose" with_docker_compose_plugin if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - echo "--- install gh cli" + echo "--- Install gh cli" with_github_cli add_pr_comment "${BUILDKITE_PULL_REQUEST}" "${BUILDKITE_BUILD_URL}" diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh index a9e846a72e..d1b85ad0be 100755 --- a/.buildkite/scripts/tooling.sh +++ b/.buildkite/scripts/tooling.sh @@ -101,8 +101,6 @@ upload_safe_logs() { local source="$2" local target="$3" - echo "--- Uploading safe logs to GCP bucket ${bucket}" - if ! ls ${source} 2>&1 > /dev/null ; then echo "upload_safe_logs: artifacts files not found, nothing will be archived" return diff --git a/.buildkite/scripts/unit_tests_macos_arm.sh b/.buildkite/scripts/unit_tests_macos_arm.sh index d8814d2076..5d8ae4a5f0 100755 --- a/.buildkite/scripts/unit_tests_macos_arm.sh +++ b/.buildkite/scripts/unit_tests_macos_arm.sh @@ -6,7 +6,7 @@ set -euo pipefail add_bin_path -echo "--- install go" +echo "--- Install go" with_go echo "--- Running unit tests" diff --git a/scripts/stack_helpers.sh b/scripts/stack_helpers.sh new file mode 100644 index 0000000000..08d768d7ab --- /dev/null +++ b/scripts/stack_helpers.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +is_stack_created() { + local files=0 + files=$(find ~/.elastic-package -type f -name "docker-compose.yml" | wc -l) + if [ "${files}" -gt 0 ]; then + return 0 + fi + return 1 +} + diff --git a/scripts/test-build-install-zip-file.sh b/scripts/test-build-install-zip-file.sh index a7597e3961..88ab084ea8 100755 --- a/scripts/test-build-install-zip-file.sh +++ b/scripts/test-build-install-zip-file.sh @@ -4,6 +4,11 @@ set -euxo pipefail cleanup() { local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" local output_path="build/elastic-stack-dump/install-zip" if [ ${USE_SHELLINIT} -eq 1 ]; then @@ -11,7 +16,8 @@ cleanup() { fi # Dump stack logs - elastic-package stack dump -v --output ${output_path} + # Required containers could not be running, so ignore the error + elastic-package stack dump -v --output ${output_path} || true # Take down the stack elastic-package stack down -v @@ -39,6 +45,7 @@ installAndVerifyPackage() { # e.g. "apache-999.999.999" becomes "apache/999.999.999" PACKAGE_NAME_VERSION="${PACKAGE_NAME_VERSION//-/\/}" + echo "--- Installing package: ${PACKAGE_NAME_VERSION}" elastic-package install -v --zip "${zipFile}" # check that the package is installed @@ -90,6 +97,7 @@ if [ "${STACK_VERSION}" != "default" ]; then ARG_VERSION="--version ${STACK_VERSION}" fi +echo "--- Prepare Elastic stack" # Update the stack elastic-package stack update -v ${ARG_VERSION} @@ -101,10 +109,14 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH # Build packages for d in test/packages/*/*/; do + # Added set +x in a sub-shell to avoid printing the testype command in the output + # This helps to keep the CI output cleaner + packageTestType=$(set +x ; testype "$d") # Packages in false_positives can have issues. - if [ "$(testype "$d")" == "false_positives" ]; then + if [ "${packageTestType}" == "false_positives" ]; then continue fi + echo "--- Building zip package: ${d}" elastic-package build -C "$d" done diff --git a/scripts/test-build-install-zip.sh b/scripts/test-build-install-zip.sh index 1c46076b03..d3bbd3d47d 100755 --- a/scripts/test-build-install-zip.sh +++ b/scripts/test-build-install-zip.sh @@ -3,10 +3,16 @@ set -euxo pipefail cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump/build-zip + # Required containers could not be running, so ignore the error + elastic-package stack dump -v --output build/elastic-stack-dump/build-zip || true # Take down the stack elastic-package stack down -v @@ -36,16 +42,21 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH go run ./scripts/gpgkey for d in test/packages/*/*/; do + # Added set +x in a sub-shell to avoid printing the testype command in the output + # This helps to keep the CI output cleaner + packageTestType=$(set +x ; testype "$d") # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "${packageTestType}" == "false_positives" ]; then continue fi + echo "--- Building package: ${d}" elastic-package build -C "$d" --zip --sign -v done # Remove unzipped built packages, leave .zip files rm -r build/packages/*/ +echo "--- Prepare Elastic stack" # Boot up the stack elastic-package stack up -d -v @@ -53,13 +64,17 @@ eval "$(elastic-package stack shellinit)" # Install packages from working copy for d in test/packages/*/*/; do + # Added set +x in a sub-shell to avoid printing the testype command in the output + # This helps to keep the CI output cleaner + packageTestType=$(set +x ; testype "$d") # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "${packageTestType}" == "false_positives" ]; then continue fi package_name=$(yq -r '.name' "${d}/manifest.yml") package_version=$(yq -r '.version' "${d}/manifest.yml") + echo "--- Installing package: ${package_name} (${package_version})" elastic-package install -C "$d" -v # check that the package is installed diff --git a/scripts/test-build-zip.sh b/scripts/test-build-zip.sh index bdd382cbba..b0514c0fbb 100755 --- a/scripts/test-build-zip.sh +++ b/scripts/test-build-zip.sh @@ -1,9 +1,13 @@ #!/bin/bash - set -euxo pipefail cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Clean used resources for d in test/packages/*/*/; do @@ -30,10 +34,14 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH go run ./scripts/gpgkey for d in test/packages/*/*/; do + # Added set +x in a sub-shell to avoid printing the testype command in the output + # This helps to keep the CI output cleaner + packageTestType=$(set +x ; testype "$d") # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "${packageTestType}" == "false_positives" ]; then continue fi + echo "--- Building zip package: ${d}" elastic-package build -C "$d" --zip --sign -v done diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh index f8f0bf0705..2c5741b544 100755 --- a/scripts/test-check-false-positives.sh +++ b/scripts/test-check-false-positives.sh @@ -3,17 +3,26 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${SCRIPT_DIR}/stack_parameters.sh" +source "${SCRIPT_DIR}/stack_helpers.sh" set -euxo pipefail function cleanup() { r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" - # Dump stack logs - elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" + if is_stack_created ; then + # Dump stack logs + # Required containers could not be running, so ignore the error + elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" || true - # Take down the stack - elastic-package stack down -v + # Take down the stack + elastic-package stack down -v + fi # Clean used resources for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do @@ -92,14 +101,15 @@ trap cleanup EXIT ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH -stack_args=$(stack_version_args) # --version +stack_args=$(set +x; stack_version_args) # --version +echo "--- Prepare Elastic stack" # Update the stack elastic-package stack update -v ${stack_args} # NOTE: if any provider argument is defined, the stack must be shutdown first to ensure # that all parameters are taken into account by the services -stack_args="${stack_args} $(stack_provider_args)" # -U +stack_args="${stack_args} $(set +x; stack_provider_args)" # -U # Boot up the stack elastic-package stack up -d -v ${stack_args} @@ -108,6 +118,8 @@ elastic-package stack status # Run package tests for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + echo "--- Check build output: ${d}" check_build_output "$d" + echo "--- Check expected errors: ${d}" check_expected_errors "$d" done diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index b0fe929682..0caaffe760 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -3,15 +3,24 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${SCRIPT_DIR}/stack_parameters.sh" +source "${SCRIPT_DIR}/stack_helpers.sh" set -euxo pipefail cleanup() { r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" - # Dump stack logs - elastic-package stack dump -v \ - --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" + if is_stack_created; then + # Dump stack logs + # Required containers could not be running, so ignore the error + elastic-package stack dump -v \ + --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" || true + fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Dump kubectl details @@ -29,8 +38,10 @@ cleanup() { # at the beginning of the pipeline and must be running for all packages without stopping it between # packages. if [[ "$SERVERLESS" != "true" ]]; then + if is_stack_created; then # Take down the stack elastic-package stack down -v + fi fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then @@ -54,10 +65,13 @@ export SERVERLESS=${SERVERLESS:-"false"} # Build/check packages for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do + echo "--- Checking package ${d}" elastic-package check -C "$d" -v done if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then + echo "--- Create logstash profile" + # Create a logstash profile and use it elastic-package profiles create logstash -v elastic-package profiles use logstash @@ -73,14 +87,15 @@ fi # started to test all packages. In our CI, this Elastic serverless stack is started # at the beginning of the pipeline and must be running for all packages. if [[ "${SERVERLESS}" != "true" ]]; then - stack_args=$(stack_version_args) # --version + echo "--- Prepare Elastic stack" + stack_args=$(set +x;stack_version_args) # --version # Update the stack elastic-package stack update -v ${stack_args} # NOTE: if any provider argument is defined, the stack must be shutdown first to ensure # that all parameters are taken into account by the services - stack_args="${stack_args} $(stack_provider_args)" # -U + stack_args="${stack_args} $(set +x; stack_provider_args)" # -U # Boot up the stack elastic-package stack up -d -v ${stack_args} @@ -90,6 +105,7 @@ fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Boot up the kind cluster + echo "--- Create kind cluster" kind create cluster --config "$PWD/scripts/kind-config.yaml" --image "kindest/node:${K8S_VERSION}" fi @@ -100,6 +116,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop if [ "${package_to_test}" == "pipeline_benchmark" ]; then + echo "--- Run pipeline benchmarks and report for package ${package_to_test}" + rm -rf "${PWD}/build/benchmark-results" elastic-package benchmark pipeline -C "$d" -v --report-format xUnit --report-output file --fail-on-missing @@ -114,12 +132,16 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do --threshold 1 --report-output-path="${PWD}/build/benchreport" fi if [ "${package_to_test}" == "system_benchmark" ]; then + echo "--- Run system benchmarks for package ${package_to_test}" + elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s fi elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${package_to_test}" == "system_benchmark" ]; then + echo "--- Run system benchmarks for package ${package_to_test}" elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s else if [[ "${SERVERLESS}" == "true" ]]; then + echo "--- Run tests for package ${d} in Serverless mode" # skip system tests elastic-package test asset -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic elastic-package test static -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic @@ -129,6 +151,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do continue fi + + echo "--- Run tests for package ${d}" # Run all tests # defer-cleanup is set to a short period to verify that the option is available elastic-package test -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic diff --git a/scripts/test-profiles-command.sh b/scripts/test-profiles-command.sh index 23eb2e90e1..7b3988e533 100755 --- a/scripts/test-profiles-command.sh +++ b/scripts/test-profiles-command.sh @@ -3,7 +3,7 @@ set -euxo pipefail cleanup() { - r=$? + local r=$? # Delete extra profiles elastic-package profiles delete test_default diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index 03f5cbeb85..7dec417547 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -1,20 +1,32 @@ #!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + set -euxo pipefail +source "${SCRIPT_DIR}/stack_helpers.sh" + VERSION=${1:-default} APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false} SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false} ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""} cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" - # Dump stack logs - elastic-package stack dump -v --output "build/elastic-stack-dump/stack/${VERSION}" + if is_stack_created; then + # Dump stack logs + # Required containers could not be running, so ignore the error + elastic-package stack dump -v --output "build/elastic-stack-dump/stack/${VERSION}" || true - # Take down the stack - elastic-package stack down -v + # Take down the stack + elastic-package stack down -v + fi if [ "${APM_SERVER_ENABLED}" = true ]; then elastic-package profiles delete with-apm-server @@ -52,6 +64,7 @@ fi OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}" if [ "${APM_SERVER_ENABLED}" = true ]; then + echo "--- Create APM server profile" OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}_with_apm_server" # Create an apm-server profile and use it @@ -66,6 +79,7 @@ EOF fi if [ "${SELF_MONITOR_ENABLED}" = true ]; then + echo "--- Create self-monitor profile" # Create a self-monitor profile and use it profile=with-self-monitor elastic-package profiles create -v ${profile} @@ -77,6 +91,7 @@ EOF fi if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then + echo "--- Create elastic subscription profile" profile=with-elastic-subscription elastic-package profiles create -v ${profile} elastic-package profiles use ${profile} @@ -88,6 +103,7 @@ fi mkdir -p "${OUTPUT_PATH_STATUS}" +echo "--- Check initial Elastic stack status" # Initial status empty elastic-package stack status 2> "${OUTPUT_PATH_STATUS}/initial.txt" grep "\- No service running" "${OUTPUT_PATH_STATUS}/initial.txt" @@ -100,12 +116,14 @@ if [[ "${EXPECTED_VERSION}" =~ ^7\.17 ]] ; then echo "Override elastic-agent docker image: ${ELASTIC_AGENT_IMAGE_REF_OVERRIDE}" fi +echo "--- Prepare Elastic stack" # Update the stack elastic-package stack update -v ${ARG_VERSION} # Boot up the stack elastic-package stack up -d -v ${ARG_VERSION} +echo "--- Check Elastic stack status" # Verify it's accessible eval "$(elastic-package stack shellinit)" curl --cacert "${ELASTIC_PACKAGE_CA_CERT}" -f "${ELASTIC_PACKAGE_KIBANA_HOST}/login" | grep kbn-injected-metadata >/dev/null # healthcheck @@ -133,10 +151,12 @@ clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > "${OUTPUT_PATH_STATUS} diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt" if [ "${APM_SERVER_ENABLED}" = true ]; then + echo "--- Check APM server status" curl http://localhost:8200/ fi if [ "${SELF_MONITOR_ENABLED}" = true ]; then + echo "--- Check self-monitor status" # Check that there is some data in the system indexes. curl -s -S --retry 5 --retry-all-errors --fail \ -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \ @@ -144,6 +164,7 @@ if [ "${SELF_MONITOR_ENABLED}" = true ]; then -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/metrics-system.*/_search?allow_no_indices=false&size=0" fi +echo "Check Elastic stack license" subscription=$(curl -s -S \ -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \ --cacert "${ELASTIC_PACKAGE_CA_CERT}" \ diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index b8665b7f69..d177025587 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -1,14 +1,27 @@ #!/usr/bin/env bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + set -euxo pipefail +source "${SCRIPT_DIR}/stack_helpers.sh" + cleanup() { local r=$? local container_id="" - local agent_ids + local agent_ids="" + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi - # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump/system-test-flags + echo "~~~ elastic-package cleanup" + + if is_stack_created; then + # Dump stack logs + # Required containers could not be running, so ignore the error + elastic-package stack dump -v --output build/elastic-stack-dump/system-test-flags || true + fi if is_service_container_running "${DEFAULT_AGENT_CONTAINER_NAME}" ; then container_id=$(container_ids "${DEFAULT_AGENT_CONTAINER_NAME}") @@ -32,8 +45,10 @@ cleanup() { kind delete cluster || true - # Take down the stack - elastic-package stack down -v + if is_stack_created; then + # Take down the stack + elastic-package stack down -v + fi # Clean used resources for d in test/packages/*/*/; do