better isolate integration test suites (improved specificity of pass/…

…fail/flake signal)

.github/workflows/serverless-integration.yml

@@ -12,8 +12,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         architecture: [amd64, arm64]
+        suite: [metric, log, trace, appsec]
+    name: ${{ matrix.suite }} on ${{ matrix.architecture }}
     steps:
       - name: Checkout datadog-agent repository
         uses: actions/checkout@v3
@@ -26,7 +29,7 @@ jobs:
           node-version: 14
 
       - name: Install Serverless Framework
-        run: sudo yarn global add serverless@^3.7.9 --prefix /usr/local
+        run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local
 
       - name: Checkout the datadog-lambda-extension repository
         uses: actions/checkout@v3
@@ -44,6 +47,12 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
+      - name: Create raw logs directory
+        id: rawlogs
+        run: |-
+          DIR=$(mktemp -d)
+          echo "dir=${DIR}" >> $GITHUB_OUTPUT
+
       - name: Run tests if AWS credentials are available
         id: test
         uses: nick-fields/retry@v2
@@ -54,15 +63,14 @@ jobs:
           timeout_minutes: 45
           max_attempts: 2
           command: |
-            export RAWLOGS_DIR=$(mktemp -d)
-            echo "rawlogs=$RAWLOGS_DIR" >> $GITHUB_OUTPUT
+            RAWLOGS_DIR="${{ steps.rawlogs.outputs.dir }}/${{ matrix.architecture }}}"
             cd go/src/github.com/DataDog/datadog-agent
             ARCHITECTURE=${{ matrix.architecture }} RAWLOGS_DIR=$RAWLOGS_DIR \
-                ./test/integration/serverless/run.sh
+                ./test/integration/serverless/run.sh ${{ matrix.suite }}
 
       - name: Archive raw logs
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: rawlogs
-          path: ${{ steps.test.outputs.rawlogs }}
+          path: ${{ steps.rawlogs.outputs.dir }}

test/integration/serverless/run.sh

@@ -165,7 +165,28 @@ appsec_functions=(
     "appsec-csharp"
 )
 
-all_functions=("${metric_functions[@]}" "${log_functions[@]}" "${trace_functions[@]}" "${appsec_functions[@]}")
+declare -a all_functions # This is an array
+if [ $# == 1 ]; then
+    case $1 in
+        metric)
+            all_functions=("${metric_functions[@]}")
+        ;;
+        log)
+            all_functions=("${log_functions[@]}")
+        ;;
+        trace)
+            all_functions=("${trace_functions[@]}")
+        ;;
+        appsec)
+            all_functions=("${appsec_functions[@]}")
+        ;;
+        *)
+            echo "Unknown test suite: $1 (valid names are: metric, log, trace, appsec)"
+        ;;
+    esac
+else
+    all_functions=("${metric_functions[@]}" "${log_functions[@]}" "${trace_functions[@]}" "${appsec_functions[@]}")
+fi
 
 # Add a function to this list to skip checking its results
 # This should only be used temporarily while we investigate and fix the test
@@ -218,6 +239,8 @@ failed_functions=()
 
 if [ -z $RAWLOGS_DIR ]; then
     RAWLOGS_DIR=$(mktemp -d)
+else
+    mkdir -p $RAWLOGS_DIR
 fi
 echo "Raw logs will be written to ${RAWLOGS_DIR}"