From 90eb406a39aa8e978b4edf1329f5b6556088059d Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Thu, 19 Dec 2024 09:40:52 +0000 Subject: [PATCH 01/52] Test elastic-package from PR 2285 - 9fb80b4570ee4fdb85deca5ceaa24aa4ebdd635b --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 40d68e4d36e..a4c44a21d6f 100644 --- a/go.mod +++ b/go.mod @@ -231,3 +231,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee diff --git a/go.sum b/go.sum index 3cc690a3c6e..8b0b0129fb9 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY= github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk= -github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo= -github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A= @@ -372,6 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee h1:hjj7dK6vCUZLuPLwDckGPvtKQL8x8EqHaDK9FexIv9o= +github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee/go.mod h1:5u8SJgzT+VHGNkrMjGWeIbPUkx2Ej6rJOuc9wt5zJvE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 0bdbe2ff51179ff9059524c060a3fe9e47c5a62b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 19 Dec 2024 10:43:21 +0100 Subject: [PATCH 02/52] Test validation based on mappings --- .buildkite/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 29daa705014..8825ccd7bdd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -30,6 +30,8 @@ env: ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}" # Disable checking for newer versions ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true" + # Select method to validate fields are documented + ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: "mappings" steps: - label: "Get reference from target branch" From f0c0f78f3620a7cab62a3a3058601e04a1c9a2bd Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Thu, 19 Dec 2024 11:29:38 +0000 Subject: [PATCH 03/52] Test elastic-package from PR 2285 - 2ad28ac1c76f72209c7797fc6da8385946d1813a --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a4c44a21d6f..5e9fd51f124 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f diff --git a/go.sum b/go.sum index 8b0b0129fb9..4765bcfffa6 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee h1:hjj7dK6vCUZLuPLwDckGPvtKQL8x8EqHaDK9FexIv9o= -github.com/mrodm/elastic-package v0.53.1-0.20241218185655-9fb80b4570ee/go.mod h1:5u8SJgzT+VHGNkrMjGWeIbPUkx2Ej6rJOuc9wt5zJvE= +github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f h1:fQw+WYyG2ySvMR89/p8LIrgSRHOg7SHKeX9EdQGtZMs= +github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f/go.mod h1:5u8SJgzT+VHGNkrMjGWeIbPUkx2Ej6rJOuc9wt5zJvE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 009101fe1702bae845df20eb0dc1a494d0916032 Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Wed, 8 Jan 2025 09:17:01 +0000 Subject: [PATCH 04/52] Test elastic-package from PR 2285 - a718796431faedeefe457e525de4ae5f4ecb612f --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5e9fd51f124..f5c3c9ea709 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa diff --git a/go.sum b/go.sum index 4765bcfffa6..829140b555c 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f h1:fQw+WYyG2ySvMR89/p8LIrgSRHOg7SHKeX9EdQGtZMs= -github.com/mrodm/elastic-package v0.53.1-0.20241219103324-2ad28ac1c76f/go.mod h1:5u8SJgzT+VHGNkrMjGWeIbPUkx2Ej6rJOuc9wt5zJvE= +github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa h1:abT0mIRIxv+6bawIpxdinnl9XzUCxNLscBywTORV4vI= +github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 5e8abed6af9324a21f0c64bf8e6cfe88e5b6559c Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Wed, 8 Jan 2025 13:32:12 +0000 Subject: [PATCH 05/52] Test elastic-package from PR 2285 - 9ff3d0cf145ed8cd6c4563519d7bde7cfce7d926 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f5c3c9ea709..1fefc9b6792 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e diff --git a/go.sum b/go.sum index 829140b555c..b505e34f4ff 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa h1:abT0mIRIxv+6bawIpxdinnl9XzUCxNLscBywTORV4vI= -github.com/mrodm/elastic-package v0.53.1-0.20250107191047-a718796431fa/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= +github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e h1:KgU/PXqpToYLCgegPqvaZWakt502iG7ApRHHbBZXcbY= +github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From e5ceadd3af03e3925ce4f21d755ec742802155c6 Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Wed, 8 Jan 2025 18:31:50 +0000 Subject: [PATCH 06/52] Test elastic-package from PR 2285 - ebda8599dab64514208de7aab38ef45fd0870314 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1fefc9b6792..3bcaf481df6 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6 diff --git a/go.sum b/go.sum index b505e34f4ff..869a2b271c8 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e h1:KgU/PXqpToYLCgegPqvaZWakt502iG7ApRHHbBZXcbY= -github.com/mrodm/elastic-package v0.53.1-0.20250108130949-9ff3d0cf145e/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= +github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6 h1:v3WNsweCqs3IdcvRzbGS9P9gvKl5uQLqsJwbUprhXss= +github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From f100e53bfc967b98d67a17519ec926eac75d584e Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Thu, 9 Jan 2025 12:35:14 +0000 Subject: [PATCH 07/52] Test elastic-package from PR 2285 - 8f36c189acbd6ff0328b2154aec9168f9652f640 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3bcaf481df6..8a5e54053ef 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd diff --git a/go.sum b/go.sum index 869a2b271c8..b7351a75b55 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6 h1:v3WNsweCqs3IdcvRzbGS9P9gvKl5uQLqsJwbUprhXss= -github.com/mrodm/elastic-package v0.53.1-0.20250108175113-ebda8599dab6/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= +github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd h1:S4TpLXrapMQxGIsCJSKV3Q+7F5UVLw5goHsxpE8SQMw= +github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 6d7b2deb1f71d46c6aed098849c79f67e7a93dd5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 9 Jan 2025 16:18:19 +0100 Subject: [PATCH 08/52] Test subset packages --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9bcad94f6c6..8238f43cae7 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort | grep -E '^(crowdstrike|sublime_security)$' } check_package() { From b64079cb965970074d7e99e4f0a4354dd0659aef Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Thu, 9 Jan 2025 15:26:40 +0000 Subject: [PATCH 09/52] Test elastic-package from PR 2285 - dbca4feac02cb94243a6bbb0eb8bbd8889bafc01 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a5e54053ef..597db47e6e6 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c diff --git a/go.sum b/go.sum index b7351a75b55..43f589b6122 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd h1:S4TpLXrapMQxGIsCJSKV3Q+7F5UVLw5goHsxpE8SQMw= -github.com/mrodm/elastic-package v0.53.1-0.20250109120814-8f36c189acbd/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= +github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c h1:T5fpY6iP+/JPat4Q7zUayxjUbidnqDSXjUCuk55dMXg= +github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 0b451eb854697e29033c24f22bdf3b41abf2be1e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 9 Jan 2025 18:01:02 +0100 Subject: [PATCH 10/52] Revert "Test subset packages" This reverts commit 81c7871f7800c771ea1769495a969845910bc81f. --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 8238f43cae7..9bcad94f6c6 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort | grep -E '^(crowdstrike|sublime_security)$' + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort } check_package() { From 704fabcd68573b31e6474362960512defd6e51b8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 10 Jan 2025 17:29:32 +0100 Subject: [PATCH 11/52] Test subset packages --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9bcad94f6c6..ed93667c3a8 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas)$' } check_package() { From 6876013397526574c222346f44dde31292b8d5c5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 10 Jan 2025 17:34:51 +0100 Subject: [PATCH 12/52] Test with 8.18.0-SNAPSHOT --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8825ccd7bdd..00cadb1bc3a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,6 +8,7 @@ env: YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" + STACK_VERSION: "8.18.0-SNAPSHOT" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" From bae2b3f7f88ce71e9ae0174785774ea460fe04b1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:04:16 +0100 Subject: [PATCH 13/52] First batch of workarounds - to be confirmed --- .../data_stream/auditd/fields/fields.yml | 4 ++-- packages/box_events/data_stream/events/fields/ecs.yml | 8 ++++++++ packages/claroty_ctd/data_stream/event/fields/ecs.yml | 6 ++++++ packages/crowdstrike/data_stream/fdr/fields/fields.yml | 2 +- .../stats/_dev/test/system/test-default-config.yml | 5 +++++ .../envoyproxy/data_stream/stats/fields/fields.yml | 10 ++++++++++ .../threat_intel_malware_customer/fields/ecs.yml | 8 ++++++++ .../threat_intel_malware_grid/fields/ecs.yml | 8 ++++++++ .../data_stream/project/fields/fields.yml | 6 ++++++ .../data_stream/email_message/fields/fields.yml | 3 +++ .../ti_anomali/data_stream/intelligence/fields/ecs.yml | 6 ++++++ 11 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 packages/box_events/data_stream/events/fields/ecs.yml create mode 100644 packages/claroty_ctd/data_stream/event/fields/ecs.yml create mode 100644 packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml create mode 100644 packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml create mode 100644 packages/ti_anomali/data_stream/intelligence/fields/ecs.yml diff --git a/packages/auditd_manager/data_stream/auditd/fields/fields.yml b/packages/auditd_manager/data_stream/auditd/fields/fields.yml index ecfd2d66fbf..2475ccdac94 100644 --- a/packages/auditd_manager/data_stream/auditd/fields/fields.yml +++ b/packages/auditd_manager/data_stream/auditd/fields/fields.yml @@ -737,6 +737,6 @@ type: keyword - name: auditd.data.result type: keyword -- name: auditd.data +- name: auditd.data.* description: Auditd related data - type: flattened + type: keyword diff --git a/packages/box_events/data_stream/events/fields/ecs.yml b/packages/box_events/data_stream/events/fields/ecs.yml new file mode 100644 index 00000000000..953536f1aaa --- /dev/null +++ b/packages/box_events/data_stream/events/fields/ecs.yml @@ -0,0 +1,8 @@ +# This definition could be removed once Kibana constraint is updated +# to 8.15.2 or higher. "ecs@mappings" component template would define +# the correct dynamic template for it. +- external: ecs + name: threat.enrichments.indicator.first_seen +- external: ecs + name: threat.enrichments.indicator.last_seen +first_seen diff --git a/packages/claroty_ctd/data_stream/event/fields/ecs.yml b/packages/claroty_ctd/data_stream/event/fields/ecs.yml new file mode 100644 index 00000000000..7786c1b562c --- /dev/null +++ b/packages/claroty_ctd/data_stream/event/fields/ecs.yml @@ -0,0 +1,6 @@ +# This definition could be removed once Kibana constraint is updated +# to 8.15.2 or higher. "ecs@mappings" component template would define +# the correct dynamic template for it. +- external: ecs + name: threat.indicator.modified_at + diff --git a/packages/crowdstrike/data_stream/fdr/fields/fields.yml b/packages/crowdstrike/data_stream/fdr/fields/fields.yml index 7a27dc795e2..692a411263b 100644 --- a/packages/crowdstrike/data_stream/fdr/fields/fields.yml +++ b/packages/crowdstrike/data_stream/fdr/fields/fields.yml @@ -27,7 +27,7 @@ type: long - name: AsepWrittenCount type: long - - name: assessments.* + - name: assessments type: flattened - name: AssociatedFile type: keyword diff --git a/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml b/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml index c096c474b90..de3985fd779 100644 --- a/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml +++ b/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml @@ -3,3 +3,8 @@ service_notify_signal: SIGHUP vars: host: 0.0.0.0 port: 8125 +assert: + # force to ingest all documents to be created all mappings in the data stream + fields_present: + - envoy.cluster_manager.cluster_added.count + - envoy.thread_local_cluster_manager.main_thread_clusters_inflated.value diff --git a/packages/envoyproxy/data_stream/stats/fields/fields.yml b/packages/envoyproxy/data_stream/stats/fields/fields.yml index 7421a1e4e52..6f8097e236d 100644 --- a/packages/envoyproxy/data_stream/stats/fields/fields.yml +++ b/packages/envoyproxy/data_stream/stats/fields/fields.yml @@ -1065,6 +1065,16 @@ object_type_mapping_type: "*" metric_type: gauge description: Envoyproxy gauges + # Is this required or the Ingest pipeline requires to be revisited? + # - name: thread_local_cluster_manager + # type: group + # fields: + # - name: '*.value' + # type: object + # object_type: double + # object_type_mapping_type: "*" + # metric_type: gauge + # description: Envoyproxy gauges - name: tls_inspector type: group fields: diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml new file mode 100644 index 00000000000..90ed56f864c --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml @@ -0,0 +1,8 @@ +# This definition could be removed once Kibana constraint is updated +# to 8.15.2 or higher. "ecs@mappings" component template would define +# the correct dynamic template for it. +- external: ecs + name: threat.indicator.modified_at +- external: ecs + name: threat.indicator.first_seen + diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml new file mode 100644 index 00000000000..90ed56f864c --- /dev/null +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml @@ -0,0 +1,8 @@ +# This definition could be removed once Kibana constraint is updated +# to 8.15.2 or higher. "ecs@mappings" component template would define +# the correct dynamic template for it. +- external: ecs + name: threat.indicator.modified_at +- external: ecs + name: threat.indicator.first_seen + diff --git a/packages/mongodb_atlas/data_stream/project/fields/fields.yml b/packages/mongodb_atlas/data_stream/project/fields/fields.yml index c1fc56d274a..110ef5ce864 100644 --- a/packages/mongodb_atlas/data_stream/project/fields/fields.yml +++ b/packages/mongodb_atlas/data_stream/project/fields/fields.yml @@ -7,6 +7,12 @@ - name: additional_info.* type: flattened description: Additional meta information about the event. Only present when includeRaw query parameter is true. + - name: additional_info.hidden + # description? + type: boolean + - name: additional_info.is_mms_admin + # description? + type: boolean - name: alert type: group fields: diff --git a/packages/sublime_security/data_stream/email_message/fields/fields.yml b/packages/sublime_security/data_stream/email_message/fields/fields.yml index ebd5e6a90f5..c48217d3291 100644 --- a/packages/sublime_security/data_stream/email_message/fields/fields.yml +++ b/packages/sublime_security/data_stream/email_message/fields/fields.yml @@ -644,6 +644,9 @@ - name: fields type: object object_type: keyword + - name: fields.position + # description: ? + type: long # should be considered as keyword too? - name: index type: long description: Index indicates the order in which a hop occurred from sender to recipient. diff --git a/packages/ti_anomali/data_stream/intelligence/fields/ecs.yml b/packages/ti_anomali/data_stream/intelligence/fields/ecs.yml new file mode 100644 index 00000000000..7786c1b562c --- /dev/null +++ b/packages/ti_anomali/data_stream/intelligence/fields/ecs.yml @@ -0,0 +1,6 @@ +# This definition could be removed once Kibana constraint is updated +# to 8.15.2 or higher. "ecs@mappings" component template would define +# the correct dynamic template for it. +- external: ecs + name: threat.indicator.modified_at + From ad9e39ffa70b510b283e8b843b0e8951bbdc722d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:06:26 +0100 Subject: [PATCH 14/52] Remove asterisk from flattened types - to be confirmed --- packages/awsfirehose/data_stream/logs/fields/fields.yml | 2 +- packages/awsfirehose/data_stream/metrics/fields/fields.yml | 2 +- packages/linux/data_stream/memory/fields/fields.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/awsfirehose/data_stream/logs/fields/fields.yml b/packages/awsfirehose/data_stream/logs/fields/fields.yml index d16a8bfba99..dd30a812c3d 100644 --- a/packages/awsfirehose/data_stream/logs/fields/fields.yml +++ b/packages/awsfirehose/data_stream/logs/fields/fields.yml @@ -27,7 +27,7 @@ type: keyword description: | Firehose request ID. - - name: parameters.* + - name: parameters type: flattened description: | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. diff --git a/packages/awsfirehose/data_stream/metrics/fields/fields.yml b/packages/awsfirehose/data_stream/metrics/fields/fields.yml index 53cc7a2b50d..542f6bb1cf5 100644 --- a/packages/awsfirehose/data_stream/metrics/fields/fields.yml +++ b/packages/awsfirehose/data_stream/metrics/fields/fields.yml @@ -32,7 +32,7 @@ - name: arn type: keyword description: Amazon Resource Name (ARN) for the firehose stream. - - name: parameters.* + - name: parameters type: flattened description: | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. diff --git a/packages/linux/data_stream/memory/fields/fields.yml b/packages/linux/data_stream/memory/fields/fields.yml index b603987c1c2..d8f757a394f 100644 --- a/packages/linux/data_stream/memory/fields/fields.yml +++ b/packages/linux/data_stream/memory/fields/fields.yml @@ -82,7 +82,7 @@ metric_type: gauge description: | The percentage of used swap memory. - - name: vmstat.* + - name: vmstat type: flattened description: Raw data from /proc/vmstat on the host. - name: hugepages From ca101bce441e94ab8dd239840782662fc62df1ec Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:09:18 +0100 Subject: [PATCH 15/52] Update subset packages to test --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ed93667c3a8..2c9c3041365 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas)$' + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas|awsfirehose|linux|envoyproxy)$' } check_package() { From 8ecbe7a5f7fbb459340602f06527149bdb013880 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:22:45 +0100 Subject: [PATCH 16/52] Update docs --- packages/auditd_manager/docs/README.md | 2 +- packages/awsfirehose/docs/README.md | 4 ++-- packages/claroty_ctd/docs/README.md | 1 + packages/crowdstrike/docs/README.md | 2 +- packages/mimecast/docs/README.md | 4 ++++ packages/mongodb_atlas/docs/README.md | 4 +++- packages/sublime_security/docs/README.md | 1 + packages/ti_anomali/docs/README.md | 1 + 8 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/auditd_manager/docs/README.md b/packages/auditd_manager/docs/README.md index 6c70a8ca387..07f18308ea7 100644 --- a/packages/auditd_manager/docs/README.md +++ b/packages/auditd_manager/docs/README.md @@ -290,7 +290,7 @@ An example event for `auditd` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| auditd.data | Auditd related data | flattened | +| auditd.data.\* | Auditd related data | keyword | | auditd.data.a0-N | the arguments to a syscall | keyword | | auditd.data.acct | a user's account name | keyword | | auditd.data.acl | access mode of resource assigned to vm | keyword | diff --git a/packages/awsfirehose/docs/README.md b/packages/awsfirehose/docs/README.md index 61c86bed7fe..d994026299b 100644 --- a/packages/awsfirehose/docs/README.md +++ b/packages/awsfirehose/docs/README.md @@ -141,7 +141,7 @@ This is a current limitation in Firehose, which we are working with AWS to resol | aws.cloudwatch.log_group | CloudWatch log group name. | keyword | | aws.cloudwatch.log_stream | CloudWatch log stream name. | keyword | | aws.firehose.arn | Firehose ARN. | keyword | -| aws.firehose.parameters.\* | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. | flattened | +| aws.firehose.parameters | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. | flattened | | aws.firehose.request_id | Firehose request ID. | keyword | | aws.firehose.subscription_filters | Firehose request ID. | keyword | | aws.kinesis.name | Kinesis name. | keyword | @@ -163,7 +163,7 @@ This is a current limitation in Firehose, which we are working with AWS to resol | aws.dimensions.\* | Metric dimensions. | keyword | | | aws.exporter.arn | The metric stream Amazon Resource Name (ARN). | keyword | | | aws.firehose.arn | Amazon Resource Name (ARN) for the firehose stream. | keyword | | -| aws.firehose.parameters.\* | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. | flattened | | +| aws.firehose.parameters | Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. | flattened | | | aws.firehose.request_id | HTTP request opaque GUID. | keyword | | | aws.metrics_names_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names. For metrics coming in from Firehose, there can be cases two documents have the same timestamp, dimension, namespace, accountID, exportARN and region BUT from two different requests. With TSDB enabled, we will see documents missing if without aws.metrics_names_fingerprint field. | keyword | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | diff --git a/packages/claroty_ctd/docs/README.md b/packages/claroty_ctd/docs/README.md index 903a2e5bb0c..84c6d69986a 100644 --- a/packages/claroty_ctd/docs/README.md +++ b/packages/claroty_ctd/docs/README.md @@ -717,6 +717,7 @@ An example event for `event` looks as following: | log.offset | Log offset. | long | | log.source.address | Source address from which the log event read/sent. | keyword | | tags | User defined tags. | keyword | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | ### Assets diff --git a/packages/crowdstrike/docs/README.md b/packages/crowdstrike/docs/README.md index 3972576da20..dd7b1aff00d 100644 --- a/packages/crowdstrike/docs/README.md +++ b/packages/crowdstrike/docs/README.md @@ -1662,7 +1662,7 @@ and/or `session_token`. | crowdstrike.__mv_aip | | keyword | | crowdstrike.__mv_discoverer_aid | | keyword | | crowdstrike.aipCount | | integer | -| crowdstrike.assessments.\* | | flattened | +| crowdstrike.assessments | | flattened | | crowdstrike.cid | | keyword | | crowdstrike.discovererCount | | integer | | crowdstrike.discoverer_aid | | keyword | diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md index 09f856d38ee..1883c3d2aa6 100644 --- a/packages/mimecast/docs/README.md +++ b/packages/mimecast/docs/README.md @@ -1018,6 +1018,8 @@ An example event for `threat_intel_malware_customer` looks as following: | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | | mimecast.valid_from | The valid from date. | date | | mimecast.value | The value of the indicator. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | ### Threat Intel Feed Malware: Grid @@ -1134,6 +1136,8 @@ An example event for `threat_intel_malware_grid` looks as following: | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | | mimecast.valid_from | The valid from date. | date | | mimecast.value | The value of the indicator. | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | ### TTP Attachment Logs diff --git a/packages/mongodb_atlas/docs/README.md b/packages/mongodb_atlas/docs/README.md index 1eb842dbd86..0bd3283f76b 100644 --- a/packages/mongodb_atlas/docs/README.md +++ b/packages/mongodb_atlas/docs/README.md @@ -914,7 +914,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | input.type | Type of Filebeat input. | keyword | -| mongodb_atlas.project.additional_info.\* | Additional meta information about the event. Only present when includeRaw query parameter is true. | flattened | +| mongodb_atlas.project.additional_info.\* | Additional meta information about the event. Only present when includeRaw query parameter is true. | object | +| mongodb_atlas.project.additional_info.hidden | | boolean | +| mongodb_atlas.project.additional_info.is_mms_admin | | boolean | | mongodb_atlas.project.alert.config.id | Unique identifier for the alert configuration associated with the alertId. | keyword | | mongodb_atlas.project.alert.id | Unique identifier for the alert associated with this event. | keyword | | mongodb_atlas.project.api_key.id | Unique identifier for the API Key that triggered this event. If this field is present in the response, Cloud Manager does not return the userId field. | keyword | diff --git a/packages/sublime_security/docs/README.md b/packages/sublime_security/docs/README.md index 80299296e24..d0e5c789283 100644 --- a/packages/sublime_security/docs/README.md +++ b/packages/sublime_security/docs/README.md @@ -1223,6 +1223,7 @@ An example event for `email_message` looks as following: | sublime_security.email_message.headers.hops.authentication_results.spf_details.verdict | Verdict of the SPF. | keyword | | sublime_security.email_message.headers.hops.authentication_results.type | The type of authentication result, derived from the field name. | keyword | | sublime_security.email_message.headers.hops.fields | | object | +| sublime_security.email_message.headers.hops.fields.position | | long | | sublime_security.email_message.headers.hops.index | Index indicates the order in which a hop occurred from sender to recipient. | long | | sublime_security.email_message.headers.hops.received.additional.raw | The raw string for remaining additional clauses, such as transport information. | keyword | | sublime_security.email_message.headers.hops.received.id.raw | The raw string of 'id' section. | keyword | diff --git a/packages/ti_anomali/docs/README.md b/packages/ti_anomali/docs/README.md index 2a4012f3aa0..274cd88f7a4 100644 --- a/packages/ti_anomali/docs/README.md +++ b/packages/ti_anomali/docs/README.md @@ -180,6 +180,7 @@ An example event for `intelligence` looks as following: | labels.is_ioc_transform_source | Indicates whether an IOC is in the raw source data stream, or the in latest destination index. | constant_keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | ### Anomali ThreatStream via the Elastic Extension From 1868ddd6cc263b58443a9b6cbb0c9a9efbb433e4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:23:29 +0100 Subject: [PATCH 17/52] update docs box_events --- packages/box_events/data_stream/events/fields/ecs.yml | 1 - packages/box_events/docs/README.md | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/box_events/data_stream/events/fields/ecs.yml b/packages/box_events/data_stream/events/fields/ecs.yml index 953536f1aaa..68030c5511f 100644 --- a/packages/box_events/data_stream/events/fields/ecs.yml +++ b/packages/box_events/data_stream/events/fields/ecs.yml @@ -5,4 +5,3 @@ name: threat.enrichments.indicator.first_seen - external: ecs name: threat.enrichments.indicator.last_seen -first_seen diff --git a/packages/box_events/docs/README.md b/packages/box_events/docs/README.md index b24efb95035..ac56496e320 100644 --- a/packages/box_events/docs/README.md +++ b/packages/box_events/docs/README.md @@ -270,4 +270,6 @@ Preserves a raw copy of the original event, added to the field `event.original`. | related.description | Array of `description` derived from `threat[.enrichments].indicator.description` | keyword | | related.indicator_type | Array of `indicator_type` derived from `threat[.enrichments].indicator.type` | keyword | | related.location | Array of `location` derived from `related.ip` | geo_point | +| threat.enrichments.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.enrichments.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | From b1ffc45f90591b50cfcdc53d5a419a2aa06a53e0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 18:27:12 +0100 Subject: [PATCH 18/52] Test elastic-package from PR 2285 - 8fce0ec8 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 597db47e6e6..68a73254236 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba diff --git a/go.sum b/go.sum index 43f589b6122..4337e53a225 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c h1:T5fpY6iP+/JPat4Q7zUayxjUbidnqDSXjUCuk55dMXg= -github.com/mrodm/elastic-package v0.53.1-0.20250109151916-dbca4feac02c/go.mod h1:pnBlvzfA6wlZHIzYalN16OBM3t6YMEixdYfKTtywpmE= +github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba h1:f1oVWNLtaA4hA/8gDyFvJPEjyrJPHMDL/tif4LqkKSM= +github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba/go.mod h1:B8rWS6VFscWXHohOSek2FCSEzihlPuawQ4i5iLbEusk= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 4b72d15e9f00f380354afb725bde61e9a11678a8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 14 Jan 2025 19:02:11 +0100 Subject: [PATCH 19/52] Re-generate sample_event.json Re-generate sample_event file since there were errors related to a field missing the expected type (auditd.data.audit_pid). This was not failing previously probably because "auditd.data" was declared as flattened, and that is skipped in validation. --- .../data_stream/auditd/sample_event.json | 78 ++++++++++++------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/packages/auditd_manager/data_stream/auditd/sample_event.json b/packages/auditd_manager/data_stream/auditd/sample_event.json index e0d9f9ecd4e..bbff6f37e5b 100644 --- a/packages/auditd_manager/data_stream/auditd/sample_event.json +++ b/packages/auditd_manager/data_stream/auditd/sample_event.json @@ -1,22 +1,22 @@ { - "@timestamp": "2022-05-12T13:10:13.230Z", + "@timestamp": "2025-01-14T18:00:56.117Z", "agent": { - "ephemeral_id": "cfe4170e-f9b4-435f-b19c-a0e75b573b3a", - "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1", - "name": "custom-agent", + "ephemeral_id": "df24f825-df17-4f54-b3c4-3048edbdf123", + "id": "da084743-3b4d-43eb-a6c9-f26c44204375", + "name": "elastic-agent-90019", "type": "auditbeat", - "version": "8.2.0" + "version": "8.16.0" }, "auditd": { "data": { - "a0": "a", - "a1": "c00024e8c0", - "a2": "38", + "a0": "10", + "a1": "c001144140", + "a2": "3c", "a3": "0", "arch": "x86_64", - "audit_pid": "22501", + "audit_pid": 2532842, "auid": "unset", - "exit": "56", + "exit": "60", "old": "0", "op": "set", "result": "success", @@ -25,15 +25,16 @@ "family": "netlink", "saddr": "100000000000000000000000" }, + "subj_user": "docker-default", "syscall": "sendto", "tty": "(none)" }, "message_type": "config_change", "messages": [ - "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1", - "type=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)", - "type=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000", - "type=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C" + "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1", + "type=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)", + "type=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000", + "type=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65" ], "result": "success", "summary": { @@ -41,7 +42,7 @@ "primary": "unset", "secondary": "root" }, - "how": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat", + "how": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat", "object": { "primary": "set", "type": "audit-config" @@ -63,21 +64,24 @@ }, "id": "0", "name": "root" + }, + "selinux": { + "user": "docker-default" } } }, "data_stream": { "dataset": "auditd_manager.auditd", - "namespace": "ep", + "namespace": "73800", "type": "logs" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1", + "id": "da084743-3b4d-43eb-a6c9-f26c44204375", "snapshot": false, - "version": "8.2.0" + "version": "8.16.0" }, "event": { "action": "changed-audit-configuration", @@ -88,12 +92,12 @@ "network" ], "dataset": "auditd_manager.auditd", - "ingested": "2022-05-12T13:10:16Z", + "ingested": "2025-01-14T18:00:59Z", "kind": "event", "module": "auditd", - "original": "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1\ntype=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)\ntype=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C", + "original": "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1\ntype=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)\ntype=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65", "outcome": "success", - "sequence": 94471, + "sequence": 1197107, "type": [ "change", "connection", @@ -101,19 +105,37 @@ ] }, "host": { - "name": "custom-agent" + "architecture": "x86_64", + "containerized": false, + "hostname": "elastic-agent-90019", + "ip": [ + "192.168.176.2", + "192.168.144.5" + ], + "mac": [ + "02-42-C0-A8-90-05", + "02-42-C0-A8-B0-02" + ], + "name": "elastic-agent-90019", + "os": { + "kernel": "6.8.0-51-generic", + "name": "Wolfi", + "platform": "wolfi", + "type": "linux", + "version": "20230201" + } }, "network": { "direction": "egress" }, "process": { - "executable": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat", - "name": "auditbeat", + "executable": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat", + "name": "agentbeat", "parent": { - "pid": 9509 + "pid": 2531521 }, - "pid": 22501, - "title": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat -c auditbeat.elastic-agent.yml" + "pid": 2532842, + "title": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat auditbeat -E setup.ilm.enabled=false -E setup.template.e" }, "service": { "type": "auditd" @@ -130,4 +152,4 @@ "id": "0", "name": "root" } -} \ No newline at end of file +} From afbee355d2d282869b69343b5f5585a9265f6eec Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 15 Jan 2025 12:15:04 +0100 Subject: [PATCH 20/52] Upate README auditd_manager --- packages/auditd_manager/docs/README.md | 76 +++++++++++++++++--------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/packages/auditd_manager/docs/README.md b/packages/auditd_manager/docs/README.md index 07f18308ea7..1a5e186aee3 100644 --- a/packages/auditd_manager/docs/README.md +++ b/packages/auditd_manager/docs/README.md @@ -151,24 +151,24 @@ An example event for `auditd` looks as following: ```json { - "@timestamp": "2022-05-12T13:10:13.230Z", + "@timestamp": "2025-01-14T18:00:56.117Z", "agent": { - "ephemeral_id": "cfe4170e-f9b4-435f-b19c-a0e75b573b3a", - "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1", - "name": "custom-agent", + "ephemeral_id": "df24f825-df17-4f54-b3c4-3048edbdf123", + "id": "da084743-3b4d-43eb-a6c9-f26c44204375", + "name": "elastic-agent-90019", "type": "auditbeat", - "version": "8.2.0" + "version": "8.16.0" }, "auditd": { "data": { - "a0": "a", - "a1": "c00024e8c0", - "a2": "38", + "a0": "10", + "a1": "c001144140", + "a2": "3c", "a3": "0", "arch": "x86_64", - "audit_pid": "22501", + "audit_pid": 2532842, "auid": "unset", - "exit": "56", + "exit": "60", "old": "0", "op": "set", "result": "success", @@ -177,15 +177,16 @@ An example event for `auditd` looks as following: "family": "netlink", "saddr": "100000000000000000000000" }, + "subj_user": "docker-default", "syscall": "sendto", "tty": "(none)" }, "message_type": "config_change", "messages": [ - "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1", - "type=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)", - "type=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000", - "type=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C" + "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1", + "type=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)", + "type=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000", + "type=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65" ], "result": "success", "summary": { @@ -193,7 +194,7 @@ An example event for `auditd` looks as following: "primary": "unset", "secondary": "root" }, - "how": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat", + "how": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat", "object": { "primary": "set", "type": "audit-config" @@ -215,21 +216,24 @@ An example event for `auditd` looks as following: }, "id": "0", "name": "root" + }, + "selinux": { + "user": "docker-default" } } }, "data_stream": { "dataset": "auditd_manager.auditd", - "namespace": "ep", + "namespace": "73800", "type": "logs" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1", + "id": "da084743-3b4d-43eb-a6c9-f26c44204375", "snapshot": false, - "version": "8.2.0" + "version": "8.16.0" }, "event": { "action": "changed-audit-configuration", @@ -240,12 +244,12 @@ An example event for `auditd` looks as following: "network" ], "dataset": "auditd_manager.auditd", - "ingested": "2022-05-12T13:10:16Z", + "ingested": "2025-01-14T18:00:59Z", "kind": "event", "module": "auditd", - "original": "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1\ntype=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)\ntype=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C", + "original": "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1\ntype=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)\ntype=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65", "outcome": "success", - "sequence": 94471, + "sequence": 1197107, "type": [ "change", "connection", @@ -253,19 +257,37 @@ An example event for `auditd` looks as following: ] }, "host": { - "name": "custom-agent" + "architecture": "x86_64", + "containerized": false, + "hostname": "elastic-agent-90019", + "ip": [ + "192.168.176.2", + "192.168.144.5" + ], + "mac": [ + "02-42-C0-A8-90-05", + "02-42-C0-A8-B0-02" + ], + "name": "elastic-agent-90019", + "os": { + "kernel": "6.8.0-51-generic", + "name": "Wolfi", + "platform": "wolfi", + "type": "linux", + "version": "20230201" + } }, "network": { "direction": "egress" }, "process": { - "executable": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat", - "name": "auditbeat", + "executable": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat", + "name": "agentbeat", "parent": { - "pid": 9509 + "pid": 2531521 }, - "pid": 22501, - "title": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat -c auditbeat.elastic-agent.yml" + "pid": 2532842, + "title": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat auditbeat -E setup.ilm.enabled=false -E setup.template.e" }, "service": { "type": "auditd" From c6dfe63e0c5325f9668a2619e80b4275c3bcb978 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 22 Jan 2025 18:39:02 +0100 Subject: [PATCH 21/52] Test elastic-package from PR 2347 - 460b42027261 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 68a73254236..9d3b6d68cea 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261 diff --git a/go.sum b/go.sum index 4337e53a225..28529503935 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba h1:f1oVWNLtaA4hA/8gDyFvJPEjyrJPHMDL/tif4LqkKSM= -github.com/mrodm/elastic-package v0.53.1-0.20250114172511-8fce0ec825ba/go.mod h1:B8rWS6VFscWXHohOSek2FCSEzihlPuawQ4i5iLbEusk= +github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261 h1:+tkLdfGEh2LTIYBXyqb4r0j+j23RQ3ZjZc+ppskabuU= +github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 2240a37a52a6b4ef494e133863d337a4689ba341 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 23 Jan 2025 10:43:16 +0100 Subject: [PATCH 22/52] Update sync.time.field transform setting --- .../elasticsearch/transform/latest_ioc/transform.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml index eee5a3acaf3..864bc2b0cc4 100644 --- a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml +++ b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml @@ -9,7 +9,7 @@ source: # us that ability in order to prevent having duplicate IoC data and prevent query # time field type conflicts. dest: - index: "logs-ti_anomali_latest.threatstream-3" + index: "logs-ti_anomali_latest.threatstream-4" aliases: - alias: "logs-ti_anomali_latest.threatstream" move_on_creation: true @@ -22,7 +22,7 @@ description: Latest Anomali IoC data frequency: 30s sync: time: - field: "@timestamp" + field: "event.ingested" # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. delay: 120s retention_policy: @@ -32,4 +32,4 @@ retention_policy: _meta: managed: true # Bump this version to delete, reinstall, and restart the transform during package. - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.0 From c8d7b440d5e3050e6a819d526b8d1afe8c4a6e04 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 23 Jan 2025 10:48:58 +0100 Subject: [PATCH 23/52] Move all github.* field definitions to the same group field --- .../latest_code_scanning/fields/fields.yml | 56 ++++++++++++++++++ .../fields/package-fields.yml | 59 ------------------- .../latest_dependabot/fields/fields.yml | 57 ++++++++++++++++++ .../fields/package-fields.yml | 59 ------------------- .../transform/latest_issues/fields/fields.yml | 56 ++++++++++++++++++ .../latest_issues/fields/package-fields.yml | 59 ------------------- .../latest_secret_scanning/fields/fields.yml | 56 ++++++++++++++++++ .../fields/package-fields.yml | 59 ------------------- 8 files changed, 225 insertions(+), 236 deletions(-) delete mode 100644 packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml delete mode 100644 packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml delete mode 100644 packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml delete mode 100644 packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml index 614b8a8a7d8..ab51d9d1466 100644 --- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml @@ -131,3 +131,59 @@ - name: classifications type: keyword description: Classifications that have been applied to the file that triggered the alert.\nFor example identifying it as documentation, or a generated file. + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml deleted file mode 100644 index 39846f42be5..00000000000 --- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: github - type: group - fields: - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml b/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml index 07c05b0c926..170e2582a8d 100644 --- a/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml @@ -170,3 +170,60 @@ - name: vulnerable_requirements type: keyword description: The vulnerable requirements. + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. + diff --git a/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml deleted file mode 100644 index 39846f42be5..00000000000 --- a/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: github - type: group - fields: - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml b/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml index 0fee38e6d1f..f354e9ce0d1 100644 --- a/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml @@ -150,3 +150,59 @@ unit: s metric_type: gauge description: The time taken to close an issue in seconds. + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. \ No newline at end of file diff --git a/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml deleted file mode 100644 index 39846f42be5..00000000000 --- a/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: github - type: group - fields: - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml index bd83406466b..b798fc5c782 100644 --- a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml @@ -96,3 +96,59 @@ - name: push_protection_bypassed_at type: date description: The time that push protection was bypassed in ISO 8601 format - `YYYY-MM-DDTHH:MM:SSZ`. + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml deleted file mode 100644 index 39846f42be5..00000000000 --- a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml +++ /dev/null @@ -1,59 +0,0 @@ -- name: github - type: group - fields: - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. From 3274e5e5f698117993623ce7cc8efb15f967bd3e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 23 Jan 2025 18:58:39 +0100 Subject: [PATCH 24/52] Revert "Move all github.* field definitions to the same group field" This reverts commit 931299cfe300577176549dfe8450484238cae457. --- .../latest_code_scanning/fields/fields.yml | 56 ------------------ .../fields/package-fields.yml | 59 +++++++++++++++++++ .../latest_dependabot/fields/fields.yml | 57 ------------------ .../fields/package-fields.yml | 59 +++++++++++++++++++ .../transform/latest_issues/fields/fields.yml | 56 ------------------ .../latest_issues/fields/package-fields.yml | 59 +++++++++++++++++++ .../latest_secret_scanning/fields/fields.yml | 56 ------------------ .../fields/package-fields.yml | 59 +++++++++++++++++++ 8 files changed, 236 insertions(+), 225 deletions(-) create mode 100644 packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml create mode 100644 packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml create mode 100644 packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml create mode 100644 packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml index ab51d9d1466..614b8a8a7d8 100644 --- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/fields.yml @@ -131,59 +131,3 @@ - name: classifications type: keyword description: Classifications that have been applied to the file that triggered the alert.\nFor example identifying it as documentation, or a generated file. - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml new file mode 100644 index 00000000000..39846f42be5 --- /dev/null +++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/package-fields.yml @@ -0,0 +1,59 @@ +- name: github + type: group + fields: + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml b/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml index 170e2582a8d..07c05b0c926 100644 --- a/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_dependabot/fields/fields.yml @@ -170,60 +170,3 @@ - name: vulnerable_requirements type: keyword description: The vulnerable requirements. - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. - diff --git a/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml new file mode 100644 index 00000000000..39846f42be5 --- /dev/null +++ b/packages/github/elasticsearch/transform/latest_dependabot/fields/package-fields.yml @@ -0,0 +1,59 @@ +- name: github + type: group + fields: + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml b/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml index f354e9ce0d1..0fee38e6d1f 100644 --- a/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_issues/fields/fields.yml @@ -150,59 +150,3 @@ unit: s metric_type: gauge description: The time taken to close an issue in seconds. - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. \ No newline at end of file diff --git a/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml new file mode 100644 index 00000000000..39846f42be5 --- /dev/null +++ b/packages/github/elasticsearch/transform/latest_issues/fields/package-fields.yml @@ -0,0 +1,59 @@ +- name: github + type: group + fields: + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml index b798fc5c782..bd83406466b 100644 --- a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml +++ b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/fields.yml @@ -96,59 +96,3 @@ - name: push_protection_bypassed_at type: date description: The time that push protection was bypassed in ISO 8601 format - `YYYY-MM-DDTHH:MM:SSZ`. - - name: repository - type: group - description: Information about the GitHub repository. - fields: - - name: id - type: integer - description: A unique identifier of the repository. - - name: is_in_organization - type: boolean - description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. - - name: name - type: keyword - description: The name of the repository. - - name: full_name - type: keyword - description: The full, globally unique, name of the repository. - - name: private - type: boolean - description: Whether the repository is private. - - name: html_url - type: keyword - description: The URL to view the repository on GitHub.com. - - name: description - type: text - description: The repository description. - - name: fork - type: boolean - description: Whether the repository is a fork. - - name: url - type: keyword - description: The URL to get more information about the repository from the GitHub API. - - name: owner - type: group - description: Represents an owner of the repository. Owner could be an Organization or User. - fields: - - name: name - type: keyword - description: Name of repository owner. - - name: email - type: keyword - description: The public email of repository owner. - - name: login - type: keyword - description: Login username of repository owner. - - name: id - type: integer - description: ID of the repository owner. - - name: url - type: keyword - description: The URL to get more information about the repository owner from the GitHub API. - - name: html_url - type: keyword - description: The HTTP URL for the repository owner. - - name: type - type: keyword - description: The type of the repository owner. Example - User. diff --git a/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml new file mode 100644 index 00000000000..39846f42be5 --- /dev/null +++ b/packages/github/elasticsearch/transform/latest_secret_scanning/fields/package-fields.yml @@ -0,0 +1,59 @@ +- name: github + type: group + fields: + - name: repository + type: group + description: Information about the GitHub repository. + fields: + - name: id + type: integer + description: A unique identifier of the repository. + - name: is_in_organization + type: boolean + description: Indicates if a repository is either owned by an organization, or is a private fork of an organization repository. + - name: name + type: keyword + description: The name of the repository. + - name: full_name + type: keyword + description: The full, globally unique, name of the repository. + - name: private + type: boolean + description: Whether the repository is private. + - name: html_url + type: keyword + description: The URL to view the repository on GitHub.com. + - name: description + type: text + description: The repository description. + - name: fork + type: boolean + description: Whether the repository is a fork. + - name: url + type: keyword + description: The URL to get more information about the repository from the GitHub API. + - name: owner + type: group + description: Represents an owner of the repository. Owner could be an Organization or User. + fields: + - name: name + type: keyword + description: Name of repository owner. + - name: email + type: keyword + description: The public email of repository owner. + - name: login + type: keyword + description: Login username of repository owner. + - name: id + type: integer + description: ID of the repository owner. + - name: url + type: keyword + description: The URL to get more information about the repository owner from the GitHub API. + - name: html_url + type: keyword + description: The HTTP URL for the repository owner. + - name: type + type: keyword + description: The type of the repository owner. Example - User. From 1a0faa6e200a7df8a38ec68d088c781398a2ef88 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 12:05:25 +0100 Subject: [PATCH 25/52] Add missing field to ti_custom --- .../ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml index e67b0f76c91..3e947dce788 100644 --- a/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -42,6 +42,8 @@ type: keyword - name: threat.indicator.url.full type: keyword +- name: threat.indicator.url.original + type: wildcard # Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 # Related to fix: https://github.com/elastic/kibana/pull/177608 - name: event.module From d2505f42e6b27466823e62e62aedbf0a9a7c1243 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 13:17:32 +0100 Subject: [PATCH 26/52] Add external ecs for ecs message field in github.code_scanning --- .../elasticsearch/transform/latest_code_scanning/fields/ecs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml index 8cfb2793292..d3155a2d1cd 100644 --- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml +++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml @@ -38,3 +38,5 @@ name: rule.name - external: ecs name: tags +- external: ecs + name: message From 62cc8350f8a5938f7591242c53892c818f5f4f08 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 13:29:57 +0100 Subject: [PATCH 27/52] Add related.ip into tychon transforms --- packages/tychon/elasticsearch/transform/arp/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/browser/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/coams/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/cve/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/epp/fields/ecs.yml | 2 ++ .../elasticsearch/transform/exposedservice/fields/ecs.yml | 2 ++ .../transform/externaldevicecontrol/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/features/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/host/fields/ecs.yml | 2 ++ .../elasticsearch/transform/networkadapter/fields/ecs.yml | 2 ++ .../elasticsearch/transform/softwareinventory/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/stig/fields/ecs.yml | 2 ++ .../tychon/elasticsearch/transform/systemcerts/fields/ecs.yml | 2 ++ packages/tychon/elasticsearch/transform/volume/fields/ecs.yml | 2 ++ 17 files changed, 34 insertions(+) diff --git a/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml b/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml index 4e9268e07b5..4e57f8022fd 100644 --- a/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml @@ -68,3 +68,5 @@ name: network.type - external: ecs name: tags +- external: ecs + name: related.ip # should it be kept as keyword instead of IP ? Would that be a breaking change? diff --git a/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml b/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml index 48cfb3f77fc..bb8fd831b87 100644 --- a/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml @@ -80,3 +80,5 @@ name: tags - external: ecs name: tls.version_protocol +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml index 2c27a702b35..97ac22cb841 100644 --- a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml @@ -108,3 +108,5 @@ name: tls.client.supported_ciphers - external: ecs name: url.full +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml b/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml b/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml b/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml index e079a962770..c3ff3d48d04 100644 --- a/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml @@ -84,3 +84,5 @@ name: vulnerability.score.version - external: ecs name: vulnerability.severity +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml b/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml index 7d45d9509be..bc51fe56fe4 100644 --- a/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml @@ -70,3 +70,5 @@ name: package.type - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml b/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml index 0bfdefbb6c4..c94861ccf34 100644 --- a/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml @@ -86,3 +86,5 @@ name: tags - external: ecs name: user.name +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml b/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml index 22e6faaced3..7f69b33c3c4 100644 --- a/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml @@ -64,3 +64,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/features/fields/ecs.yml b/packages/tychon/elasticsearch/transform/features/fields/ecs.yml index 7d45d9509be..bc51fe56fe4 100644 --- a/packages/tychon/elasticsearch/transform/features/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/features/fields/ecs.yml @@ -70,3 +70,5 @@ name: package.type - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml b/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml index dafa90e8982..105db0e2f56 100644 --- a/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml @@ -66,3 +66,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/host/fields/ecs.yml b/packages/tychon/elasticsearch/transform/host/fields/ecs.yml index 857122fb420..36626e11ce6 100644 --- a/packages/tychon/elasticsearch/transform/host/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/host/fields/ecs.yml @@ -32,3 +32,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml b/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml index 2ac6aff0189..db2562fe89e 100644 --- a/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml @@ -32,3 +32,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml b/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml index b4846edeb05..1c3d6ba1689 100644 --- a/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml @@ -76,3 +76,5 @@ name: package.version - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml b/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml index 464da8ce398..628c74118ed 100644 --- a/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml @@ -74,3 +74,5 @@ name: rule.name - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml b/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml index f0f7dede28a..f7a8ed20a47 100644 --- a/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml @@ -106,3 +106,5 @@ name: tags - external: ecs name: url.full +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml b/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip From 25000b199fbdee87809dab0d174f63d252859c88 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 15:25:49 +0100 Subject: [PATCH 28/52] Added missing definitions in wiz transforms --- .../latest_cdr_misconfigurations/fields/ecs.yml | 4 ++++ .../latest_cdr_vulnerabilities/fields/fields.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml b/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml index 291b675502b..4cb860dea83 100644 --- a/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml +++ b/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml @@ -28,3 +28,7 @@ external: ecs - name: observer.vendor external: ecs +- name: message + external: ecs +- name: ecs.version + external: ecs diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml b/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml index 38aa91efa9e..b7c6b004465 100644 --- a/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml +++ b/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml @@ -6,6 +6,8 @@ external: ecs - name: cloud.region external: ecs +- name: device.id + external: ecs - name: package.name external: ecs - name: package.version @@ -14,6 +16,8 @@ external: ecs - name: vulnerability.id external: ecs +- name: vulnerability.reference + external: ecs - name: vulnerability.score.base external: ecs - name: vulnerability.score.version @@ -34,6 +38,14 @@ external: ecs - name: event.type external: ecs +- name: ecs.version + external: ecs +- name: tags + external: ecs +- name: related.ip + external: ecs # should it be keyword instead of IP ? Would this be breaking change? +- name: message + external: ecs - name: observer.vendor external: ecs - name: wiz From 07c164fc4d22dd39c6a78f041a0b402866b18078 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 15:29:02 +0100 Subject: [PATCH 29/52] Test elastic-package from PR 2347 - aff903b7 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9d3b6d68cea..2d3b215d1bb 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916 diff --git a/go.sum b/go.sum index 28529503935..8ce66e6814c 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261 h1:+tkLdfGEh2LTIYBXyqb4r0j+j23RQ3ZjZc+ppskabuU= -github.com/mrodm/elastic-package v0.53.1-0.20250122111242-460b42027261/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= +github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916 h1:fOJFEe6PqhmNGNdZfrCdhvGq17mMmvLU88ePoEA9Yyo= +github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 7f949b76312e45d9f83370a14d42c652878852bc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 16:11:18 +0100 Subject: [PATCH 30/52] Test with more packages --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2c9c3041365..a4aa9c243b1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas|awsfirehose|linux|envoyproxy)$' + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas|awsfirehose|linux|envoyproxy|ti_custom|ti_abusech|github|tychon|wiz)$' } check_package() { From e3001b15d65243ec640c649ab9fbafd1a17a64f4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 24 Jan 2025 18:02:23 +0100 Subject: [PATCH 31/52] Update missing fields in tychon --- packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml | 2 ++ .../tychon/elasticsearch/transform/harddrive/fields/ecs.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml index 97ac22cb841..0918c087d6b 100644 --- a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml @@ -96,6 +96,8 @@ name: process.user.name - external: ecs name: server.address +- external: ecs + name: server.ip # previously it was set as keyword but now it would be type IP, would that be a breaking change? - external: ecs name: server.port - external: ecs diff --git a/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml b/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml index 31a7235135f..634e60533fa 100644 --- a/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip # previously it was set as keyword but now it would be type IP, would that be a breaking change? From d2ca1e1afe800580010648b2dcccedc1a97e5bcd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 27 Jan 2025 13:31:12 +0100 Subject: [PATCH 32/52] Test elastic-package from PR 2347 - a65efc3156a0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2d3b215d1bb..5e860222020 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0 diff --git a/go.sum b/go.sum index 8ce66e6814c..2c450392a13 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916 h1:fOJFEe6PqhmNGNdZfrCdhvGq17mMmvLU88ePoEA9Yyo= -github.com/mrodm/elastic-package v0.53.1-0.20250123172336-aff903b76916/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= +github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0 h1:qrb+ddEQu0etElOKgVZv1Gv0mLri7v+xP51Cxe7CK1U= +github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 38c73da29d33042bc373f9b736f2c73a3227d21d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 27 Jan 2025 15:31:03 +0100 Subject: [PATCH 33/52] Add override parameter to some processors - teleport --- .../audit/elasticsearch/ingest_pipeline/event-groups.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml index c595b7a8369..7ebd3468c72 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml @@ -872,14 +872,20 @@ processors: field: teleport.audit.aws_region target_field: cloud.region ignore_missing: true + # This was failing due to `cloud.region` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_service target_field: cloud.service.name ignore_missing: true + # This was failing due to `cloud.service.name` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_host target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_assumed_role target_field: teleport.audit.app.aws.assumed_role From 1813e3abbb96945b45d6a80499419aa5a6415ac1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 27 Jan 2025 15:51:35 +0100 Subject: [PATCH 34/52] Report pipeline failures - to be removed --- .../data_stream/audit/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 3693bc502d8..6e7e8776341 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -55,7 +55,7 @@ processors: # Process most of the field groups. - pipeline: name: '{{ IngestPipeline "event-groups" }}' - ignore_failure: true + ignore_failure: false # App session request related metadata # The HTTP-related fields are used for other events as well. They work as catch-all # fields and should be at the end of the group processing. From a2d7589d81c6e0112069ae193700c8add840195b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 27 Jan 2025 16:22:03 +0100 Subject: [PATCH 35/52] Add workarounds for teleport fields --- .../elasticsearch/ingest_pipeline/event-groups.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml index 7ebd3468c72..7abc01d9be0 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml @@ -1413,10 +1413,13 @@ processors: field: teleport.audit.instance_id target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.exit_code target_field: process.exit_code ignore_missing: true + if: ctx.process != null && !ctx['process'].containsKey('exit_code') - set: field: event.outcome value: 'success' @@ -1432,11 +1435,18 @@ processors: field: teleport.audit.account_id target_field: cloud.account.id ignore_missing: true + # This was failing due to `cloud.account.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.region target_field: cloud.region ignore_missing: true ignore_failure: true + # in case of previous processor fails + - remove: + field: teleport.audit.region + ignore_missing: true + ignore_failure: true - rename: field: teleport.audit.stdout target_field: teleport.audit.database.aws.ssm_run.stdout From 9e163010e7a0100a987d981ccbcb487e8780be27 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 27 Jan 2025 18:25:32 +0100 Subject: [PATCH 36/52] Test elastic-package from PR 2347 - afac6f361e37 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5e860222020..8164c44965e 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37 diff --git a/go.sum b/go.sum index 2c450392a13..b0fc4b79cea 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0 h1:qrb+ddEQu0etElOKgVZv1Gv0mLri7v+xP51Cxe7CK1U= -github.com/mrodm/elastic-package v0.53.1-0.20250127122836-a65efc3156a0/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= +github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37 h1:T0On4tkAZA/2KDQUk4dvsKOQ+5o7y5pCk1PU4XwE1kk= +github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 92287ae8b0c8fe109220fa0016c7ce3a68b146df Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 28 Jan 2025 10:48:50 +0100 Subject: [PATCH 37/52] Remove asterisk from flattened field definition - mongodb_atlas --- packages/mongodb_atlas/data_stream/project/fields/fields.yml | 2 +- packages/mongodb_atlas/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mongodb_atlas/data_stream/project/fields/fields.yml b/packages/mongodb_atlas/data_stream/project/fields/fields.yml index 110ef5ce864..c4e799992b4 100644 --- a/packages/mongodb_atlas/data_stream/project/fields/fields.yml +++ b/packages/mongodb_atlas/data_stream/project/fields/fields.yml @@ -4,7 +4,7 @@ - name: project type: group fields: - - name: additional_info.* + - name: additional_info type: flattened description: Additional meta information about the event. Only present when includeRaw query parameter is true. - name: additional_info.hidden diff --git a/packages/mongodb_atlas/docs/README.md b/packages/mongodb_atlas/docs/README.md index 0bd3283f76b..0414c5822b3 100644 --- a/packages/mongodb_atlas/docs/README.md +++ b/packages/mongodb_atlas/docs/README.md @@ -914,7 +914,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | input.type | Type of Filebeat input. | keyword | -| mongodb_atlas.project.additional_info.\* | Additional meta information about the event. Only present when includeRaw query parameter is true. | object | +| mongodb_atlas.project.additional_info | Additional meta information about the event. Only present when includeRaw query parameter is true. | flattened | | mongodb_atlas.project.additional_info.hidden | | boolean | | mongodb_atlas.project.additional_info.is_mms_admin | | boolean | | mongodb_atlas.project.alert.config.id | Unique identifier for the alert configuration associated with the alertId. | keyword | From 4b918af791f6d62340f0053c06f57667d66e4171 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 28 Jan 2025 13:51:10 +0100 Subject: [PATCH 38/52] Update event-groups ingest pipeline - teleport --- .../audit/elasticsearch/ingest_pipeline/event-groups.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml index 7abc01d9be0..5dce8b6e750 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml @@ -974,6 +974,8 @@ processors: field: teleport.audit.db_gcp_instance_id target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.db_roles target_field: teleport.audit.database.roles @@ -1419,7 +1421,6 @@ processors: field: teleport.audit.exit_code target_field: process.exit_code ignore_missing: true - if: ctx.process != null && !ctx['process'].containsKey('exit_code') - set: field: event.outcome value: 'success' @@ -1441,12 +1442,11 @@ processors: field: teleport.audit.region target_field: cloud.region ignore_missing: true - ignore_failure: true - # in case of previous processor fails + ignore_failure: true # it could already exist this field + # in case it fails previous rename processor, remove the field (not defined in the package) - remove: field: teleport.audit.region ignore_missing: true - ignore_failure: true - rename: field: teleport.audit.stdout target_field: teleport.audit.database.aws.ssm_run.stdout From 9aa235953fddd8143f47683f915dfc0b4d2b83ee Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 28 Jan 2025 15:52:20 +0100 Subject: [PATCH 39/52] Remove another asterisk from flattened field definition - mongodb_atlas --- .../mongodb_atlas/data_stream/organization/fields/fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb_atlas/data_stream/organization/fields/fields.yml b/packages/mongodb_atlas/data_stream/organization/fields/fields.yml index 2ed4fa2f886..2104e4d2009 100644 --- a/packages/mongodb_atlas/data_stream/organization/fields/fields.yml +++ b/packages/mongodb_atlas/data_stream/organization/fields/fields.yml @@ -73,7 +73,7 @@ - name: public_key type: keyword description: Public key associated with the API Key that triggered this event. If this field is present in the response, Cloud Manager does not return the username field. - - name: additional_info.* + - name: additional_info type: flattened description: Additional meta information about the event. Only present when includeRaw query parameter is true. - name: replicaset.name From bea474f82a16930559e89518c49a2acae9f80107 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 28 Jan 2025 16:23:55 +0100 Subject: [PATCH 40/52] Update Readme mongodb_atlas --- packages/mongodb_atlas/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb_atlas/docs/README.md b/packages/mongodb_atlas/docs/README.md index 0414c5822b3..221a2a252f2 100644 --- a/packages/mongodb_atlas/docs/README.md +++ b/packages/mongodb_atlas/docs/README.md @@ -743,7 +743,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.type | Data stream type. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | mongodb_atlas.organization.access_list_entry | Access list entry of the API Key targeted by the event. | keyword | -| mongodb_atlas.organization.additional_info.\* | Additional meta information about the event. Only present when includeRaw query parameter is true. | flattened | +| mongodb_atlas.organization.additional_info | Additional meta information about the event. Only present when includeRaw query parameter is true. | flattened | | mongodb_atlas.organization.alert.config.id | Unique identifier for the alert configuration associated with the alertId. | keyword | | mongodb_atlas.organization.alert.id | Unique identifier for the alert associated with this event. | keyword | | mongodb_atlas.organization.api_key.id | Unique identifier for the API Key that triggered this event. | keyword | From 8546f23ed3ab142985bf62913cce145e035ac047 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 12:11:34 +0100 Subject: [PATCH 41/52] Add thread_local_cluster_manager field defs - envoyproxy.stats --- .../data_stream/stats/fields/fields.yml | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/envoyproxy/data_stream/stats/fields/fields.yml b/packages/envoyproxy/data_stream/stats/fields/fields.yml index 6f8097e236d..2333417fa92 100644 --- a/packages/envoyproxy/data_stream/stats/fields/fields.yml +++ b/packages/envoyproxy/data_stream/stats/fields/fields.yml @@ -1065,16 +1065,52 @@ object_type_mapping_type: "*" metric_type: gauge description: Envoyproxy gauges - # Is this required or the Ingest pipeline requires to be revisited? - # - name: thread_local_cluster_manager - # type: group - # fields: - # - name: '*.value' - # type: object - # object_type: double - # object_type_mapping_type: "*" - # metric_type: gauge - # description: Envoyproxy gauges + # Is this required to be added ? Or, the Ingest pipeline requires to be revisited? + - name: thread_local_cluster_manager + type: group + fields: + - name: '*.count' + type: object + object_type: double + object_type_mapping_type: "*" + metric_type: counter + description: Envoyproxy counters + - name: '*.max' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy max timers metric + - name: '*.mean_rate' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy mean rate timers metric + - name: '*.mean' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy mean timers metric + - name: '*.median' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy median timers metric + - name: '*.min' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy min timers metric + - name: '*.stddev' + type: object + object_type: double + object_type_mapping_type: "*" + description: Envoyproxy standard deviation timers metric + - name: '*.value' + type: object + object_type: double + object_type_mapping_type: "*" + metric_type: gauge + description: Envoyproxy gauges - name: tls_inspector type: group fields: From cb8069a2b6d314da7f1357f42d83e6078cdb727c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 16:20:14 +0100 Subject: [PATCH 42/52] Update envoyproxy docs --- packages/envoyproxy/docs/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/envoyproxy/docs/README.md b/packages/envoyproxy/docs/README.md index f25adf2c1ac..fffa247f615 100644 --- a/packages/envoyproxy/docs/README.md +++ b/packages/envoyproxy/docs/README.md @@ -434,6 +434,14 @@ An example event for `stats` looks as following: | envoy.tcp.\*.stddev | Envoyproxy standard deviation timers metric | object | | | envoy.tcp.\*.value | Envoyproxy gauges | object | gauge | | envoy.tcp.prefix | Stats prefix for the TCP Proxy network filter | keyword | | +| envoy.thread_local_cluster_manager.\*.count | Envoyproxy counters | object | counter | +| envoy.thread_local_cluster_manager.\*.max | Envoyproxy max timers metric | object | | +| envoy.thread_local_cluster_manager.\*.mean | Envoyproxy mean timers metric | object | | +| envoy.thread_local_cluster_manager.\*.mean_rate | Envoyproxy mean rate timers metric | object | | +| envoy.thread_local_cluster_manager.\*.median | Envoyproxy median timers metric | object | | +| envoy.thread_local_cluster_manager.\*.min | Envoyproxy min timers metric | object | | +| envoy.thread_local_cluster_manager.\*.stddev | Envoyproxy standard deviation timers metric | object | | +| envoy.thread_local_cluster_manager.\*.value | Envoyproxy gauges | object | gauge | | envoy.thrift.\*.count | Envoyproxy counters | object | counter | | envoy.thrift.\*.max | Envoyproxy max timers metric | object | | | envoy.thrift.\*.mean | Envoyproxy mean timers metric | object | | From cb15bb825ae6290b2030e5fcfdd6778cc67c5ade Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 18:57:02 +0100 Subject: [PATCH 43/52] Update mongodb_atlas - keep just flattened --- .../mongodb_atlas/data_stream/project/fields/fields.yml | 6 ------ packages/mongodb_atlas/docs/README.md | 2 -- 2 files changed, 8 deletions(-) diff --git a/packages/mongodb_atlas/data_stream/project/fields/fields.yml b/packages/mongodb_atlas/data_stream/project/fields/fields.yml index c4e799992b4..325ebc3be9e 100644 --- a/packages/mongodb_atlas/data_stream/project/fields/fields.yml +++ b/packages/mongodb_atlas/data_stream/project/fields/fields.yml @@ -7,12 +7,6 @@ - name: additional_info type: flattened description: Additional meta information about the event. Only present when includeRaw query parameter is true. - - name: additional_info.hidden - # description? - type: boolean - - name: additional_info.is_mms_admin - # description? - type: boolean - name: alert type: group fields: diff --git a/packages/mongodb_atlas/docs/README.md b/packages/mongodb_atlas/docs/README.md index 221a2a252f2..5e05fdaae67 100644 --- a/packages/mongodb_atlas/docs/README.md +++ b/packages/mongodb_atlas/docs/README.md @@ -915,8 +915,6 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.type | Data stream type. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | mongodb_atlas.project.additional_info | Additional meta information about the event. Only present when includeRaw query parameter is true. | flattened | -| mongodb_atlas.project.additional_info.hidden | | boolean | -| mongodb_atlas.project.additional_info.is_mms_admin | | boolean | | mongodb_atlas.project.alert.config.id | Unique identifier for the alert configuration associated with the alertId. | keyword | | mongodb_atlas.project.alert.id | Unique identifier for the alert associated with this event. | keyword | | mongodb_atlas.project.api_key.id | Unique identifier for the API Key that triggered this event. If this field is present in the response, Cloud Manager does not return the userId field. | keyword | From 693abdeba34f7f17a7a0048b198d8678433f6c30 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 19:09:36 +0100 Subject: [PATCH 44/52] Reverted changes in test configuration - envoyproxy --- .../stats/_dev/test/system/test-default-config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml b/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml index de3985fd779..c096c474b90 100644 --- a/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml +++ b/packages/envoyproxy/data_stream/stats/_dev/test/system/test-default-config.yml @@ -3,8 +3,3 @@ service_notify_signal: SIGHUP vars: host: 0.0.0.0 port: 8125 -assert: - # force to ingest all documents to be created all mappings in the data stream - fields_present: - - envoy.cluster_manager.cluster_added.count - - envoy.thread_local_cluster_manager.main_thread_clusters_inflated.value From 2ca62a868dbab5aa425cb04bd437870d9d3a3fe9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 19:23:01 +0100 Subject: [PATCH 45/52] Test updating dynamic template - sublime_security.email_message --- .../data_stream/email_message/fields/fields.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/sublime_security/data_stream/email_message/fields/fields.yml b/packages/sublime_security/data_stream/email_message/fields/fields.yml index c48217d3291..e7817808637 100644 --- a/packages/sublime_security/data_stream/email_message/fields/fields.yml +++ b/packages/sublime_security/data_stream/email_message/fields/fields.yml @@ -644,6 +644,10 @@ - name: fields type: object object_type: keyword + object_type_mapping_type: "*" + # - name: fields.position + # # description: ? + # type: long # should be considered as keyword too? - name: fields.position # description: ? type: long # should be considered as keyword too? From 542f0a1dcdee837be7ad722dd4796689f6338bd2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 19:24:25 +0100 Subject: [PATCH 46/52] Test elastic-package from PR 2347 - 1d539eef6799 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8164c44965e..cefe8708fb9 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799 diff --git a/go.sum b/go.sum index b0fc4b79cea..8c4a0620445 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37 h1:T0On4tkAZA/2KDQUk4dvsKOQ+5o7y5pCk1PU4XwE1kk= -github.com/mrodm/elastic-package v0.53.1-0.20250127171637-afac6f361e37/go.mod h1:J5i3KIJePmGWtVouUzmGvJa5y7eY4GrZg4dNG9ofesM= +github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799 h1:USh+J+2vsA9F1th79uIoZ9yMGeH5CFdthoNzDKh5PIc= +github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From 70dde4aaf7cf1be25fd9b5e487f86f620f5b117e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 19:34:43 +0100 Subject: [PATCH 47/52] Add another option tgo sublime_security.email_message --- .../email_message/fields/fields.yml | 23 +++++++++++++------ packages/sublime_security/docs/README.md | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/sublime_security/data_stream/email_message/fields/fields.yml b/packages/sublime_security/data_stream/email_message/fields/fields.yml index e7817808637..71481927b1b 100644 --- a/packages/sublime_security/data_stream/email_message/fields/fields.yml +++ b/packages/sublime_security/data_stream/email_message/fields/fields.yml @@ -641,16 +641,25 @@ - name: type type: keyword description: The type of authentication result, derived from the field name. - - name: fields - type: object - object_type: keyword - object_type_mapping_type: "*" + # https://github.com/elastic/kibana/pull/204104 + # Option 1: generate all keys as keywords under fields + # - name: fields + # type: object + # object_type: keyword + # object_type_mapping_type: "*" # - name: fields.position # # description: ? # type: long # should be considered as keyword too? - - name: fields.position - # description: ? - type: long # should be considered as keyword too? + # Option 2: keep position as long + - name: fields + type: group + fields: + - name: "*" + type: object + object_type: keyword + - name: position + # description: ? + type: long - name: index type: long description: Index indicates the order in which a hop occurred from sender to recipient. diff --git a/packages/sublime_security/docs/README.md b/packages/sublime_security/docs/README.md index d0e5c789283..23a1776031b 100644 --- a/packages/sublime_security/docs/README.md +++ b/packages/sublime_security/docs/README.md @@ -1222,7 +1222,7 @@ An example event for `email_message` looks as following: | sublime_security.email_message.headers.hops.authentication_results.spf_details.server.valid | Whether the domain is valid. | boolean | | sublime_security.email_message.headers.hops.authentication_results.spf_details.verdict | Verdict of the SPF. | keyword | | sublime_security.email_message.headers.hops.authentication_results.type | The type of authentication result, derived from the field name. | keyword | -| sublime_security.email_message.headers.hops.fields | | object | +| sublime_security.email_message.headers.hops.fields.\* | | object | | sublime_security.email_message.headers.hops.fields.position | | long | | sublime_security.email_message.headers.hops.index | Index indicates the order in which a hop occurred from sender to recipient. | long | | sublime_security.email_message.headers.hops.received.additional.raw | The raw string for remaining additional clauses, such as transport information. | keyword | From 98b98f9444588920853d99e1c81c2064544d009a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 29 Jan 2025 19:40:42 +0100 Subject: [PATCH 48/52] Add comment into transform settings - ti_anomali --- .../ti_anomali/elasticsearch/transform/latest_ioc/transform.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml index 864bc2b0cc4..1959a725e6f 100644 --- a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml +++ b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml @@ -22,6 +22,8 @@ description: Latest Anomali IoC data frequency: 30s sync: time: + # ensure that the field used to synchronize uses the ingested time of the documents + # this will also allow to process the documents defined in the test field: "event.ingested" # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. delay: 120s From 280e2e67978b11100bd4964801dd40fbe836d996 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 30 Jan 2025 10:15:49 +0100 Subject: [PATCH 49/52] Remove field definition - auditd_manager --- .../auditd_manager/data_stream/auditd/fields/fields.yml | 8 +++++--- packages/auditd_manager/docs/README.md | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/auditd_manager/data_stream/auditd/fields/fields.yml b/packages/auditd_manager/data_stream/auditd/fields/fields.yml index 2475ccdac94..cbbab67cca0 100644 --- a/packages/auditd_manager/data_stream/auditd/fields/fields.yml +++ b/packages/auditd_manager/data_stream/auditd/fields/fields.yml @@ -623,9 +623,11 @@ - name: auditd.data.perm_mask description: file permission mask that triggered a watch event type: keyword -- name: auditd.data.a0-N - description: the arguments to a syscall - type: keyword +# this mapping does not generate a dynamic template, and the expected fields do not match +# should it be kept for documentation purposes? +# - name: auditd.data.a0-N +# description: the arguments to a syscall +# type: keyword - name: auditd.data.ses description: login session ID type: keyword diff --git a/packages/auditd_manager/docs/README.md b/packages/auditd_manager/docs/README.md index 1a5e186aee3..a1660f80037 100644 --- a/packages/auditd_manager/docs/README.md +++ b/packages/auditd_manager/docs/README.md @@ -313,7 +313,6 @@ An example event for `auditd` looks as following: |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | | auditd.data.\* | Auditd related data | keyword | -| auditd.data.a0-N | the arguments to a syscall | keyword | | auditd.data.acct | a user's account name | keyword | | auditd.data.acl | access mode of resource assigned to vm | keyword | | auditd.data.action | netfilter packet disposition | keyword | From 0d86a5421a20a43297d6f81cbc3b2bb10826541f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 5 Feb 2025 10:40:06 +0100 Subject: [PATCH 50/52] Test elastic-package from PR 2381 - a82e4e12 include fields validation --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cefe8708fb9..caf2ba02424 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799 +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a diff --git a/go.sum b/go.sum index 8c4a0620445..04ba42a04b6 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799 h1:USh+J+2vsA9F1th79uIoZ9yMGeH5CFdthoNzDKh5PIc= -github.com/mrodm/elastic-package v0.53.1-0.20250129150823-1d539eef6799/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= +github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a h1:JtX6aMz9BtUskoPLUW6LF1lGxzPZ4kEEzXE2KJneSFg= +github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= From dd4535cbb8364a0c3d993ab2070683687e1ef1f4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 5 Feb 2025 19:43:08 +0100 Subject: [PATCH 51/52] Update logstash owner in manifest (cherry picked from commit fa96beb000d674ed0264ea61be713bd0109d3faf) --- packages/logstash/manifest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/logstash/manifest.yml b/packages/logstash/manifest.yml index 7ac30bfa4a5..b596b8f1980 100644 --- a/packages/logstash/manifest.yml +++ b/packages/logstash/manifest.yml @@ -18,7 +18,7 @@ conditions: elastic: subscription: basic owner: - github: elastic/stack-monitoring + github: elastic/logstash type: elastic screenshots: - src: /img/kibana-logstash-log.png @@ -131,4 +131,3 @@ policy_templates: multi: false required: false show_user: false - \ No newline at end of file From 88f838aac9de98c207a85aca52793f6f79528786 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 25 Feb 2025 18:03:02 +0100 Subject: [PATCH 52/52] Test elastic-package from PR 2381 - b0e11ddc include fields validation --- go.mod | 22 +++++++++++----------- go.sum | 46 +++++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index df0f8ef27ef..3a44b47747a 100644 --- a/go.mod +++ b/go.mod @@ -163,8 +163,8 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect @@ -213,17 +213,17 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - helm.sh/helm/v3 v3.17.0 // indirect + helm.sh/helm/v3 v3.17.1 // indirect howett.net/plist v1.0.0 // indirect - k8s.io/api v0.32.1 // indirect - k8s.io/apiextensions-apiserver v0.32.0 // indirect - k8s.io/apimachinery v0.32.1 // indirect - k8s.io/cli-runtime v0.32.1 // indirect - k8s.io/client-go v0.32.1 // indirect - k8s.io/component-base v0.32.0 // indirect + k8s.io/api v0.32.2 // indirect + k8s.io/apiextensions-apiserver v0.32.1 // indirect + k8s.io/apimachinery v0.32.2 // indirect + k8s.io/cli-runtime v0.32.2 // indirect + k8s.io/client-go v0.32.2 // indirect + k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/kubectl v0.32.0 // indirect + k8s.io/kubectl v0.32.1 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/kustomize/api v0.18.0 // indirect @@ -232,4 +232,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b diff --git a/go.sum b/go.sum index f61a992a1f4..533d894b574 100644 --- a/go.sum +++ b/go.sum @@ -108,7 +108,7 @@ github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q= github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -370,8 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a h1:JtX6aMz9BtUskoPLUW6LF1lGxzPZ4kEEzXE2KJneSFg= -github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= +github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b h1:zamEgHdRreoAh8Zd460OOc/0vl22QunM/9RsEFSVQOY= +github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b/go.mod h1:k/wq12XJyfvwr9mj/6xqTMVXf8IO2h6Lpu3EPnVQVZs= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= @@ -447,10 +447,10 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -681,30 +681,30 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -helm.sh/helm/v3 v3.17.0 h1:DUD4AGdNVn7PSTYfxe1gmQG7s18QeWv/4jI9TubnhT0= -helm.sh/helm/v3 v3.17.0/go.mod h1:Mo7eGyKPPHlS0Ml67W8z/lbkox/gD9Xt1XpD6bxvZZA= +helm.sh/helm/v3 v3.17.1 h1:gzVoAD+qVuoJU6KDMSAeo0xRJ6N1znRxz3wyuXRmJDk= +helm.sh/helm/v3 v3.17.1/go.mod h1:nvreuhuR+j78NkQcLC3TYoprCKStLyw5P4T7E5itv2w= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= -k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= -k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/cli-runtime v0.32.1 h1:19nwZPlYGJPUDbhAxDIS2/oydCikvKMHsxroKNGA2mM= -k8s.io/cli-runtime v0.32.1/go.mod h1:NJPbeadVFnV2E7B7vF+FvU09mpwYlZCu8PqjzfuOnkY= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= +k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/cli-runtime v0.32.2 h1:aKQR4foh9qeyckKRkNXUccP9moxzffyndZAvr+IXMks= +k8s.io/cli-runtime v0.32.2/go.mod h1:a/JpeMztz3xDa7GCyyShcwe55p8pbcCVQxvqZnIwXN8= +k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= +k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/kubectl v0.32.0 h1:rpxl+ng9qeG79YA4Em9tLSfX0G8W0vfaiPVrc/WR7Xw= -k8s.io/kubectl v0.32.0/go.mod h1:qIjSX+QgPQUgdy8ps6eKsYNF+YmFOAO3WygfucIqFiE= +k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= +k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=