Add test package with docker and terraform

internal/servicedeployer/compose.go

@@ -80,6 +80,7 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 		variant:  d.variant,
 		env: []string{
 			fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local),
+			fmt.Sprintf("%s=%s", testRunIDEnv, svcInfo.Test.RunID),
 		},
 	}
 

test/packages/parallel/nginx_multiple_services/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/parallel/nginx_multiple_services/_dev/build/docs/README.md

@@ -0,0 +1,31 @@
+# Nginx Integration
+
+This integration periodically fetches metrics from [Nginx](https://nginx.org/) servers. It can parse access and error
+logs created by the HTTP server. 
+
+## Compatibility
+
+The logs were tested with version 1.19.5.
+On Windows, the module was tested with Nginx installed from the Chocolatey repository.
+
+## Logs
+
+**Timezone support**
+
+This datasource parses logs that don’t contain timezone information. For these logs, the Elastic Agent reads the local
+timezone and uses it when parsing to convert the timestamp to UTC. The timezone to be used for parsing is included
+in the event in the `event.timezone` field.
+
+To disable this conversion, the event.timezone field can be removed with the drop_fields processor.
+
+If logs are originated from systems or applications with a different timezone to the local one, the `event.timezone`
+field can be overwritten with the original timezone using the add_fields processor.
+
+### Access Docker & TF Logs
+
+Access logs collects the nginx access logs.
+
+{{event "access_docker_tf"}}
+
+{{fields "access_docker_tf"}}
+

test/packages/parallel/nginx_multiple_services/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,6 @@
+ARG SERVICE_VERSION=${SERVICE_VERSION:-1.19.5}
+FROM nginx:${SERVICE_VERSION}
+RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost/server-status
+COPY ./nginx.conf /etc/nginx/

test/packages/parallel/nginx_multiple_services/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+  nginx:
+    build: .
+    ports:
+      - 80
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/var/log/nginx

test/packages/parallel/nginx_multiple_services/_dev/deploy/docker/nginx.conf

@@ -0,0 +1,33 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /dev/stderr warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log  /var/log/nginx/access.log main;
+
+    server {
+        listen [::]:80;
+        listen 80;
+        server_name localhost;
+
+        location /server-status {
+            stub_status on;
+        }
+    }
+
+    include /etc/nginx/conf.d/*;
+}

test/packages/parallel/nginx_multiple_services/_dev/test/config.yml

@@ -0,0 +1,2 @@
+system:
+  parallel: false

test/packages/parallel/nginx_multiple_services/changelog.yml

@@ -0,0 +1,96 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: Change test public IPs to the supported subset
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2327
+- version: "1.2.1"
+  changes:
+    - description: Fix ML module manifest query to ignore frozen and cold tiers
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2219
+- version: "1.2.0"
+  changes:
+    - description: Release nginx package for v8.0.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2176
+- version: "1.1.2"
+  changes:
+    - description: Uniform with guidelines
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2052
+- version: "1.1.1"
+  changes:
+    - description: Fix logic that checks for the 'forwarded' tag
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1835
+- version: "1.1.0"
+  changes:
+    - description: Update to ECS 1.12.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1701
+- version: "1.0.0"
+  changes:
+    - description: Release Nginx as GA
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1614
+- version: "0.8.2"
+  changes:
+    - description: Convert to generated ECS fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1491
+- version: "0.8.1"
+  changes:
+    - description: update to ECS 1.11.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1398
+- version: "0.8.0"
+  changes:
+    - description: Update integration description
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1364
+- version: "0.7.0"
+  changes:
+    - description: Set "event.module" and "event.dataset"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1234
+- version: "0.6.2"
+  changes:
+    - description: Add support for Splunk authorization tokens
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1147
+- version: "0.6.1"
+  changes:
+    - description: Fix bug in Third Party REST API ingest pipeline
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1201
+- version: "0.6.0"
+  changes:
+    - description: update to ECS 1.10.0 and add event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1065
+- version: "0.5.0"
+  changes:
+    - description: Adds ML jobs for finding unusual activity in HTTP access logs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/912
+- version: "0.4.1"
+  changes:
+    - description: update to ECS 1.9.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/859
+- version: "0.3.11"
+  changes:
+    - description: Updating package owner
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/766
+- version: "0.3.10"
+  changes:
+    - description: Fix compatibility with Kibana
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/740
+- version: "0.1.0"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/21

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,6 @@
+ARG SERVICE_VERSION=${SERVICE_VERSION:-1.19.5}
+FROM nginx:${SERVICE_VERSION}
+RUN sed -i "/jessie-updates/d" /etc/apt/sources.list
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost/server-status
+COPY ./nginx.conf /etc/nginx/

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/Dockerfile.terraform

@@ -0,0 +1,26 @@
+FROM --platform=linux/amd64 ubuntu:24.04
+ENV TERRAFORM_VERSION=1.9.6
+
+# Based on this Dockerfile:
+# https://github.com/elastic/elastic-package/blob/d8e4300715af43b3b792d25960d75c3382948dcd/internal/servicedeployer/_static/Dockerfile.terraform_deployer
+# This dockerfile just installs terraform and sets up the environment to run terraform commands.
+
+RUN apt-get -qq update \
+  && apt-get install -yq curl apt-transport-https ca-certificates gnupg \
+  && apt-get clean
+
+RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \
+  && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com noble main" | tee /etc/apt/sources.list.d/hashicorp.list \
+  && apt-get update -qq \
+  && apt-get install -yq terraform=${TERRAFORM_VERSION}-1 \
+  && apt-get clean
+
+HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]"
+
+ENV TF_IN_AUTOMATION=true
+ENV TF_CLI_ARGS="-no-color"
+ADD terraform.run.sh /run.sh
+RUN chmod +x /run.sh
+WORKDIR /workspace
+
+ENTRYPOINT exec /run.sh

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,35 @@
+version: '2.3'
+services:
+  nginx:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 80
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/var/log/nginx
+    depends_on:
+      terraform:
+        condition: service_healthy
+  terraform:
+    tty: true
+    stop_grace_period: 5m
+    build:
+      context: .
+      dockerfile: Dockerfile.terraform
+    environment:
+      - TF_VAR_TEST_RUN_ID=${TEST_RUN_ID:-detached}
+      - TF_VAR_CREATED_DATE=${CREATED_DATE:-unknown}
+      - TF_VAR_BRANCH=${BRANCH_NAME_LOWER_CASE:-unknown}
+      - TF_VAR_BUILD_ID=${BUILD_ID:-unknown}
+      - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown}
+      - TF_VAR_REPO=${REPO:-unknown}
+      # Example to use credentials
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+      - AWS_PROFILE=${AWS_PROFILE}
+      - AWS_REGION=${AWS_REGION:-us-east-1}
+    volumes:
+      - ./tf/:/stage/
+      - ${SERVICE_LOGS_DIR}:/tmp/service_logs/

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/nginx.conf

@@ -0,0 +1,33 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /dev/stderr warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log  /var/log/nginx/access.log main;
+
+    server {
+        listen [::]:80;
+        listen 80;
+        server_name localhost;
+
+        location /server-status {
+            stub_status on;
+        }
+    }
+
+    include /etc/nginx/conf.d/*;
+}

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/terraform.run.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Terraform code may rely on content from other files than .tf files (es json, zip, html, text), so we copy all the content over
+# See more: https://github.com/elastic/elastic-package/pull/603
+# NOTE: must copy hidden files too (supported by "/.")
+# See more: https://github.com/elastic/package-spec/issues/269
+cp -r /stage/. /workspace
+
+cleanup() {
+  r=$?
+
+  set -x
+  terraform destroy -auto-approve
+
+  exit $r
+}
+trap cleanup EXIT INT TERM
+
+terraform init
+terraform plan
+terraform apply -auto-approve
+
+mkdir -p /output
+terraform output -json > /output/tfOutputValues.json
+
+touch /tmp/tf-applied # This file is used as indicator (healthcheck) that the service is UP, and so it must be placed as the last statement in the script
+
+echo "Terraform definitions applied."
+
+set +x
+while true; do sleep 1; done # wait for ctrl-c

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/tf/files/example.log

@@ -0,0 +1,21 @@
+::1 - - [03/Jun/2025:14:41:33 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:34 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:35 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:36 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:37 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:38 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:39 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:40 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:41 +0000] "GET /server-status-tf HTTP/1.1" 200 97 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:42 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:43 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:45 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:46 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:47 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:48 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:49 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:50 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:51 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:52 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:53 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"
+::1 - - [03/Jun/2025:14:41:54 +0000] "GET /server-status-tf HTTP/1.1" 200 100 "-" "curl/7.64.0" "-"

test/packages/parallel/nginx_multiple_services/data_stream/access_docker_tf/_dev/deploy/docker/tf/main.tf

@@ -0,0 +1,37 @@
+# Remember to set default tags in case the provider allows for that.
+# Example for the "aws" provider
+# provider "aws" {
+#   region = "us-east-1"
+#   default_tags {
+#     tags = {
+#       environment  = var.ENVIRONMENT
+#       repo         = var.REPO
+#       branch       = var.BRANCH
+#       build        = var.BUILD_ID
+#       created_date = var.CREATED_DATE
+#     }
+#   }
+# }
+
+resource "local_file" "log" {
+  source          = "./files/example.log"
+  filename        = "/tmp/service_logs/file.log"
+  file_permission = "0777"
+}
+
+locals {
+  items ={
+    environment  = "${var.ENVIRONMENT}"
+    repo         = "${var.REPO}"
+    branch       = "${var.BRANCH}"
+    build        = "${var.BUILD_ID}"
+    created_date = "${var.CREATED_DATE}"
+    test_run_id  = "${var.TEST_RUN_ID}"
+  }
+}
+
+resource "local_file" "log_variables" {
+  content         = format("%s\n", jsonencode(local.items))
+  filename        = "/tmp/service_logs/file_vars.log"
+  file_permission = "0777"
+}