[CI] Review output in CI builds

.buildkite/hooks/pre-exit

@@ -4,15 +4,20 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
+exit_code=0
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
     echo "--- Take down the Elastic stack"
     # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package
     # is already installed in the test_serverless pipeline step, accessing
     # directly to the binary
-    "${HOME}"/go/bin/elastic-package stack down -v
+    if ! "${HOME}"/go/bin/elastic-package stack down -v ; then
+        # Ensure that even if this command fails, the script continues
+        exit_code=1
+    fi
 fi
 
 echo "--- Cleanup"
 cleanup
 unset_secrets
 
+exit "$exit_code"

.buildkite/pipeline.serverless.yml

@@ -45,6 +45,7 @@ steps:
       provider: "gcp"
       image: "${UBUNTU_X86_64_AGENT_IMAGE}"
     env:
+      SERVERLESS: "true"
       SERVERLESS_PROJECT: "${SERVERLESS_PROJECT:-observability}"
       UPLOAD_SAFE_LOGS: 1
       # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf

.buildkite/scripts/integration_tests.sh

@@ -18,9 +18,11 @@ FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
 TEST_BUILD_ZIP_TARGET="test-build-zip"
+TEST_BUILD_INSTALL_ZIP_TARGET="test-build-install-zip"
 
 REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
-export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+export REPO_BUILD_TAG
 TARGET=""
 PACKAGE=""
 SERVERLESS="false"
@@ -58,50 +60,67 @@ if [[ "${TARGET}" == "" ]]; then
     exit 1
 fi
 
+if [[ "${SERVERLESS}" == "true" && "${TARGET}" != "${PARALLEL_TARGET}" ]]; then
+    # Just tested parallel target to run with Serverless projects, other Makefile targets
+    # have not been tested yet and could fail unexpectedly. For instance, "test-check-package-false-positives"
+    # target would require a different management to not stop Elastic stack after each package test.
+    echo "Target ${TARGET} is not supported for Serverless testing"
+    usage
+    exit 1
+fi
+
 add_bin_path
 
 if [[ "$SERVERLESS" == "false" ]]; then
-    # If packages are tested with Serverless, these action are already performed
+    # If packages are tested with Serverless, these actions are already performed
     # here: .buildkite/scripts/test_packages_with_serverless.sh
-    echo "--- install go"
+    echo "--- Install go"
     with_go
 
     if [[ "${TARGET}" != "${TEST_BUILD_ZIP_TARGET}" ]]; then
         # Not supported in Macos ARM
-        echo "--- install docker"
+        echo "--- Install docker"
         with_docker
 
-        echo "--- install docker-compose plugin"
+        echo "--- Install docker-compose plugin"
         with_docker_compose_plugin
     fi
-fi
 
-echo "--- install yq"
-with_yq
+    # In Serverless pipeline, elastic-package is installed in advance here:
+    # .buildkite/scripts/test_packages_with_serverless.sh
+    # No need to install it again for every package.
+    echo "--- Install elastic-package"
+    make install
+fi
 
 if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
-    echo "--- install kubectl & kind"
+    echo "--- Install kubectl & kind"
     with_kubernetes
 fi
 
+if [[ "${TARGET}" == "${TEST_BUILD_INSTALL_ZIP_TARGET}" || "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
+    echo "--- Install yq"
+    with_yq
+fi
+
 label="${TARGET}"
 if [ -n "${PACKAGE}" ]; then
     label="${label} - ${PACKAGE}"
 fi
+
 echo "--- Run integration test ${label}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
-    make install
-
     # allow to fail this command, to be able to upload safe logs
     set +e
     make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
     testReturnCode=$?
     set -e
 
-    retry_count=${BUILDKITE_RETRY_COUNT:-"0"}
 
     if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
+        retry_count=${BUILDKITE_RETRY_COUNT:-"0"}
         package_folder="${PACKAGE}"
+
         if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-""}" == "false" ]]; then
             package_folder="${package_folder}-stack_agent"
         fi
@@ -114,6 +133,8 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
             package_folder="${package_folder}_retry_${retry_count}"
         fi
 
+        echo "--- Uploading safe logs to GCP bucket ${JOB_GCS_BUCKET_INTERNAL}"
+
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
             "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \
@@ -135,9 +156,10 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
         echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
         exit ${testReturnCode}
     fi
-
-    make check-git-clean
-    exit 0
+else
+    make "${TARGET}"
 fi
 
-make install "${TARGET}" check-git-clean
+echo "--- Check git clean"
+make check-git-clean
+exit 0

.buildkite/scripts/release.sh

@@ -17,11 +17,11 @@ VERSION=""
 add_bin_path
 with_go
 
-echo "--- fetching tags"
+echo "--- Fetching tags"
 # Ensure that tags are present so goreleaser can build the changelog from the last release.
 git rev-parse --is-shallow-repository
 git fetch origin --tags
 
-echo "--- running goreleaser"
+echo "--- Running goreleaser"
 # Run latest version of goreleaser
 curl -sL https://git.io/goreleaser | bash

.buildkite/scripts/test-with-integrations.sh

@@ -5,10 +5,10 @@ set -euo pipefail
 
 add_bin_path
 
-echo "--- install gh cli"
+echo "--- Install gh cli"
 with_github_cli
 
-echo "--- install jq"
+echo "--- Install jq"
 with_jq
 
 
@@ -87,7 +87,7 @@ create_integrations_pull_request() {
 
 update_dependency() {
     # it needs to set the Golang version from the integrations repository (.go-version file)
-    echo "--- install go for integrations repository :go:"
+    echo "--- Install go for integrations repository :go:"
     with_go
 
     echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:"
@@ -167,7 +167,7 @@ create_or_update_pull_request() {
 
     rm -rf "${temp_path}"
 
-    echo "--- adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:"
+    echo "--- Adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:"
     add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link "${integrations_pr_number}")"
 }
 
@@ -186,5 +186,5 @@ if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
     exit 1
 fi
 
-echo "--- creating or updating integrations pull request"
+echo "--- Creating or updating integrations pull request"
 create_or_update_pull_request

.buildkite/scripts/test_packages_with_serverless.sh

@@ -10,7 +10,6 @@ UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 SKIPPED_PACKAGES_FILE_PATH="${WORKSPACE}/skipped_packages.txt"
 FAILED_PACKAGES_FILE_PATH="${WORKSPACE}/failed_packages.txt"
 
-export SERVERLESS="true"
 SERVERLESS_PROJECT=${SERVERLESS_PROJECT:-"observability"}
 
 add_pr_comment() {
@@ -32,7 +31,7 @@ fi
 
 add_bin_path
 
-echo "--- install go"
+echo "--- Install go"
 with_go
 
 echo "--- Install docker"
@@ -42,7 +41,7 @@ echo "--- Install docker-compose"
 with_docker_compose_plugin
 
 if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
-    echo "--- install gh cli"
+    echo "--- Install gh cli"
     with_github_cli
 
     add_pr_comment "${BUILDKITE_PULL_REQUEST}" "${BUILDKITE_BUILD_URL}"

.buildkite/scripts/tooling.sh

@@ -101,8 +101,6 @@ upload_safe_logs() {
     local source="$2"
     local target="$3"
 
-    echo "--- Uploading safe logs to GCP bucket ${bucket}"
-
     if ! ls ${source} 2>&1 > /dev/null ; then
         echo "upload_safe_logs: artifacts files not found, nothing will be archived"
         return

.buildkite/scripts/unit_tests_macos_arm.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 add_bin_path
 
-echo "--- install go"
+echo "--- Install go"
 with_go
 
 echo "--- Running unit tests"

scripts/stack_helpers.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+is_stack_created() {
+    local files=0
+    files=$(find ~/.elastic-package -type f -name "docker-compose.yml" | wc -l)
+    if [ "${files}" -gt 0 ]; then
+        return 0
+    fi
+    return 1
+}
+

scripts/test-build-install-zip-file.sh

@@ -4,14 +4,20 @@ set -euxo pipefail
 
 cleanup() {
   local r=$?
+  if [ "${r}" -ne 0 ]; then
+    # Ensure that the group where the failure happened is opened.
+    echo "^^^ +++"
+  fi
+  echo "~~~ elastic-package cleanup"
 
   local output_path="build/elastic-stack-dump/install-zip"
   if [ ${USE_SHELLINIT} -eq 1 ]; then
       output_path="${output_path}-shellinit"
   fi
 
   # Dump stack logs
-  elastic-package stack dump -v --output ${output_path}
+  # Required containers could not be running, so ignore the error
+  elastic-package stack dump -v --output ${output_path} || true
 
   # Take down the stack
   elastic-package stack down -v
@@ -39,6 +45,7 @@ installAndVerifyPackage() {
   # e.g. "apache-999.999.999" becomes "apache/999.999.999"
   PACKAGE_NAME_VERSION="${PACKAGE_NAME_VERSION//-/\/}"
 
+  echo "--- Installing package: ${PACKAGE_NAME_VERSION}"
   elastic-package install -v --zip "${zipFile}"
 
   # check that the package is installed
@@ -90,6 +97,7 @@ if [ "${STACK_VERSION}" != "default" ]; then
   ARG_VERSION="--version ${STACK_VERSION}"
 fi
 
+echo "--- Prepare Elastic stack"
 # Update the stack
 elastic-package stack update -v ${ARG_VERSION}
 
@@ -101,10 +109,14 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH
 
 # Build packages
 for d in test/packages/*/*/; do
+  # Added set +x in a sub-shell to avoid printing the testype command in the output
+  # This helps to keep the CI output cleaner
+  packageTestType=$(set +x ; testype "$d")
   # Packages in false_positives can have issues.
-  if [ "$(testype "$d")" == "false_positives" ]; then
+  if [ "${packageTestType}" == "false_positives" ]; then
     continue
   fi
+  echo "--- Building zip package: ${d}"
   elastic-package build -C "$d"
 done
 

scripts/test-build-install-zip.sh

@@ -3,10 +3,16 @@
 set -euxo pipefail
 
 cleanup() {
-  r=$?
+  local r=$?
+  if [ "${r}" -ne 0 ]; then
+    # Ensure that the group where the failure happened is opened.
+    echo "^^^ +++"
+  fi
+  echo "~~~ elastic-package cleanup"
 
   # Dump stack logs
-  elastic-package stack dump -v --output build/elastic-stack-dump/build-zip
+  # Required containers could not be running, so ignore the error
+  elastic-package stack dump -v --output build/elastic-stack-dump/build-zip || true
 
   # Take down the stack
   elastic-package stack down -v
@@ -36,16 +42,21 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH
 go run ./scripts/gpgkey
 
 for d in test/packages/*/*/; do
+  # Added set +x in a sub-shell to avoid printing the testype command in the output
+  # This helps to keep the CI output cleaner
+  packageTestType=$(set +x ; testype "$d")
   # Packages in false_positives can have issues.
-  if [ "$(testype $d)" == "false_positives" ]; then
+  if [ "${packageTestType}" == "false_positives" ]; then
     continue
   fi
+  echo "--- Building package: ${d}"
   elastic-package build -C "$d" --zip --sign -v
 done
 
 # Remove unzipped built packages, leave .zip files
 rm -r build/packages/*/
 
+echo "--- Prepare Elastic stack"
 # Boot up the stack
 elastic-package stack up -d -v
 
@@ -54,12 +65,13 @@ eval "$(elastic-package stack shellinit)"
 # Install packages from working copy
 for d in test/packages/*/*/; do
   # Packages in false_positives can have issues.
-  if [ "$(testype $d)" == "false_positives" ]; then
+  if [ "$(testype "$d")" == "false_positives" ]; then
     continue
   fi
   package_name=$(yq -r '.name' "${d}/manifest.yml")
   package_version=$(yq -r '.version' "${d}/manifest.yml")
 
+  echo "--- Installing package: ${package_name} (${package_version})"
   elastic-package install -C "$d" -v
 
   # check that the package is installed

scripts/test-build-zip.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
-
 set -euxo pipefail
 
 cleanup() {
-  r=$?
+  local r=$?
+  if [ "${r}" -ne 0 ]; then
+    # Ensure that the group where the failure happened is opened.
+    echo "^^^ +++"
+  fi
+  echo "~~~ elastic-package cleanup"
 
   # Clean used resources
   for d in test/packages/*/*/; do
@@ -30,10 +34,14 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH
 go run ./scripts/gpgkey
 
 for d in test/packages/*/*/; do
+  # Added set +x in a sub-shell to avoid printing the testype command in the output
+  # This helps to keep the CI output cleaner
+  packageTestType=$(set +x ; testype "$d")
   # Packages in false_positives can have issues.
-  if [ "$(testype $d)" == "false_positives" ]; then
+  if [ "${packageTestType}" == "false_positives" ]; then
     continue
   fi
+  echo "--- Building zip package: ${d}"
   elastic-package build -C "$d" --zip --sign -v
 done