diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 29daa705014..00cadb1bc3a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,6 +8,7 @@ env: YQ_VERSION: 'v4.35.2' JQ_VERSION: '1.7' GH_CLI_VERSION: "2.29.0" + STACK_VERSION: "8.18.0-SNAPSHOT" # Agent images used in pipeline steps LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" @@ -30,6 +31,8 @@ env: ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}" # Disable checking for newer versions ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true" + # Select method to validate fields are documented + ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: "mappings" steps: - label: "Get reference from target branch" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9bcad94f6c6..a4aa9c243b1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -757,7 +757,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas|awsfirehose|linux|envoyproxy|ti_custom|ti_abusech|github|tychon|wiz)$' } check_package() { diff --git a/go.mod b/go.mod index c57848481d7..3a44b47747a 100644 --- a/go.mod +++ b/go.mod @@ -163,8 +163,8 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect @@ -213,17 +213,17 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - helm.sh/helm/v3 v3.17.0 // indirect + helm.sh/helm/v3 v3.17.1 // indirect howett.net/plist v1.0.0 // indirect - k8s.io/api v0.32.1 // indirect - k8s.io/apiextensions-apiserver v0.32.0 // indirect - k8s.io/apimachinery v0.32.1 // indirect - k8s.io/cli-runtime v0.32.1 // indirect - k8s.io/client-go v0.32.1 // indirect - k8s.io/component-base v0.32.0 // indirect + k8s.io/api v0.32.2 // indirect + k8s.io/apiextensions-apiserver v0.32.1 // indirect + k8s.io/apimachinery v0.32.2 // indirect + k8s.io/cli-runtime v0.32.2 // indirect + k8s.io/client-go v0.32.2 // indirect + k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/kubectl v0.32.0 // indirect + k8s.io/kubectl v0.32.1 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/kustomize/api v0.18.0 // indirect @@ -231,3 +231,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b diff --git a/go.sum b/go.sum index 02837f20f1e..533d894b574 100644 --- a/go.sum +++ b/go.sum @@ -108,7 +108,7 @@ github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 h1:boJj011Hh+874zpIySeApCX4GeOjPl9qhRF3QuIZq+Q= github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -125,8 +125,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY= github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk= -github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo= -github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A= @@ -372,6 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b h1:zamEgHdRreoAh8Zd460OOc/0vl22QunM/9RsEFSVQOY= +github.com/mrodm/elastic-package v0.53.1-0.20250225170140-b0e11ddc9f0b/go.mod h1:k/wq12XJyfvwr9mj/6xqTMVXf8IO2h6Lpu3EPnVQVZs= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= @@ -447,10 +447,10 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -681,30 +681,30 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -helm.sh/helm/v3 v3.17.0 h1:DUD4AGdNVn7PSTYfxe1gmQG7s18QeWv/4jI9TubnhT0= -helm.sh/helm/v3 v3.17.0/go.mod h1:Mo7eGyKPPHlS0Ml67W8z/lbkox/gD9Xt1XpD6bxvZZA= +helm.sh/helm/v3 v3.17.1 h1:gzVoAD+qVuoJU6KDMSAeo0xRJ6N1znRxz3wyuXRmJDk= +helm.sh/helm/v3 v3.17.1/go.mod h1:nvreuhuR+j78NkQcLC3TYoprCKStLyw5P4T7E5itv2w= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= -k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= -k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/cli-runtime v0.32.1 h1:19nwZPlYGJPUDbhAxDIS2/oydCikvKMHsxroKNGA2mM= -k8s.io/cli-runtime v0.32.1/go.mod h1:NJPbeadVFnV2E7B7vF+FvU09mpwYlZCu8PqjzfuOnkY= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= +k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/cli-runtime v0.32.2 h1:aKQR4foh9qeyckKRkNXUccP9moxzffyndZAvr+IXMks= +k8s.io/cli-runtime v0.32.2/go.mod h1:a/JpeMztz3xDa7GCyyShcwe55p8pbcCVQxvqZnIwXN8= +k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= +k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/kubectl v0.32.0 h1:rpxl+ng9qeG79YA4Em9tLSfX0G8W0vfaiPVrc/WR7Xw= -k8s.io/kubectl v0.32.0/go.mod h1:qIjSX+QgPQUgdy8ps6eKsYNF+YmFOAO3WygfucIqFiE= +k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= +k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml index 8cfb2793292..d3155a2d1cd 100644 --- a/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml +++ b/packages/github/elasticsearch/transform/latest_code_scanning/fields/ecs.yml @@ -38,3 +38,5 @@ name: rule.name - external: ecs name: tags +- external: ecs + name: message diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 3693bc502d8..6e7e8776341 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -55,7 +55,7 @@ processors: # Process most of the field groups. - pipeline: name: '{{ IngestPipeline "event-groups" }}' - ignore_failure: true + ignore_failure: false # App session request related metadata # The HTTP-related fields are used for other events as well. They work as catch-all # fields and should be at the end of the group processing. diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml index c595b7a8369..5dce8b6e750 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/event-groups.yml @@ -872,14 +872,20 @@ processors: field: teleport.audit.aws_region target_field: cloud.region ignore_missing: true + # This was failing due to `cloud.region` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_service target_field: cloud.service.name ignore_missing: true + # This was failing due to `cloud.service.name` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_host target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.aws_assumed_role target_field: teleport.audit.app.aws.assumed_role @@ -968,6 +974,8 @@ processors: field: teleport.audit.db_gcp_instance_id target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.db_roles target_field: teleport.audit.database.roles @@ -1407,6 +1415,8 @@ processors: field: teleport.audit.instance_id target_field: cloud.instance.id ignore_missing: true + # This was failing due to `cloud.instance.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.exit_code target_field: process.exit_code @@ -1426,11 +1436,17 @@ processors: field: teleport.audit.account_id target_field: cloud.account.id ignore_missing: true + # This was failing due to `cloud.account.id` already existed + override: true # Should it be added an if condition? Should it be added a remove processor? - rename: field: teleport.audit.region target_field: cloud.region ignore_missing: true - ignore_failure: true + ignore_failure: true # it could already exist this field + # in case it fails previous rename processor, remove the field (not defined in the package) + - remove: + field: teleport.audit.region + ignore_missing: true - rename: field: teleport.audit.stdout target_field: teleport.audit.database.aws.ssm_run.stdout diff --git a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml index eee5a3acaf3..1959a725e6f 100644 --- a/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml +++ b/packages/ti_anomali/elasticsearch/transform/latest_ioc/transform.yml @@ -9,7 +9,7 @@ source: # us that ability in order to prevent having duplicate IoC data and prevent query # time field type conflicts. dest: - index: "logs-ti_anomali_latest.threatstream-3" + index: "logs-ti_anomali_latest.threatstream-4" aliases: - alias: "logs-ti_anomali_latest.threatstream" move_on_creation: true @@ -22,7 +22,9 @@ description: Latest Anomali IoC data frequency: 30s sync: time: - field: "@timestamp" + # ensure that the field used to synchronize uses the ingested time of the documents + # this will also allow to process the documents defined in the test + field: "event.ingested" # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. delay: 120s retention_policy: @@ -32,4 +34,4 @@ retention_policy: _meta: managed: true # Bump this version to delete, reinstall, and restart the transform during package. - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.0 diff --git a/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml index e67b0f76c91..3e947dce788 100644 --- a/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_custom/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -42,6 +42,8 @@ type: keyword - name: threat.indicator.url.full type: keyword +- name: threat.indicator.url.original + type: wildcard # Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 # Related to fix: https://github.com/elastic/kibana/pull/177608 - name: event.module diff --git a/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml b/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml index 4e9268e07b5..4e57f8022fd 100644 --- a/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/arp/fields/ecs.yml @@ -68,3 +68,5 @@ name: network.type - external: ecs name: tags +- external: ecs + name: related.ip # should it be kept as keyword instead of IP ? Would that be a breaking change? diff --git a/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml b/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml index 48cfb3f77fc..bb8fd831b87 100644 --- a/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/browser/fields/ecs.yml @@ -80,3 +80,5 @@ name: tags - external: ecs name: tls.version_protocol +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml index 2c27a702b35..0918c087d6b 100644 --- a/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/ciphers/fields/ecs.yml @@ -96,6 +96,8 @@ name: process.user.name - external: ecs name: server.address +- external: ecs + name: server.ip # previously it was set as keyword but now it would be type IP, would that be a breaking change? - external: ecs name: server.port - external: ecs @@ -108,3 +110,5 @@ name: tls.client.supported_ciphers - external: ecs name: url.full +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml b/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/coams/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml b/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/cpu/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml b/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml index e079a962770..c3ff3d48d04 100644 --- a/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/cve/fields/ecs.yml @@ -84,3 +84,5 @@ name: vulnerability.score.version - external: ecs name: vulnerability.severity +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml b/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml index 7d45d9509be..bc51fe56fe4 100644 --- a/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/epp/fields/ecs.yml @@ -70,3 +70,5 @@ name: package.type - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml b/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml index 0bfdefbb6c4..c94861ccf34 100644 --- a/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/exposedservice/fields/ecs.yml @@ -86,3 +86,5 @@ name: tags - external: ecs name: user.name +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml b/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml index 22e6faaced3..7f69b33c3c4 100644 --- a/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/externaldevicecontrol/fields/ecs.yml @@ -64,3 +64,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/features/fields/ecs.yml b/packages/tychon/elasticsearch/transform/features/fields/ecs.yml index 7d45d9509be..bc51fe56fe4 100644 --- a/packages/tychon/elasticsearch/transform/features/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/features/fields/ecs.yml @@ -70,3 +70,5 @@ name: package.type - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml b/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml index 31a7235135f..634e60533fa 100644 --- a/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/harddrive/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip # previously it was set as keyword but now it would be type IP, would that be a breaking change? diff --git a/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml b/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml index dafa90e8982..105db0e2f56 100644 --- a/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/hardware/fields/ecs.yml @@ -66,3 +66,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/host/fields/ecs.yml b/packages/tychon/elasticsearch/transform/host/fields/ecs.yml index 857122fb420..36626e11ce6 100644 --- a/packages/tychon/elasticsearch/transform/host/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/host/fields/ecs.yml @@ -32,3 +32,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml b/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml index 2ac6aff0189..db2562fe89e 100644 --- a/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/networkadapter/fields/ecs.yml @@ -32,3 +32,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml b/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml index b4846edeb05..1c3d6ba1689 100644 --- a/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/softwareinventory/fields/ecs.yml @@ -76,3 +76,5 @@ name: package.version - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml b/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml index 464da8ce398..628c74118ed 100644 --- a/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/stig/fields/ecs.yml @@ -74,3 +74,5 @@ name: rule.name - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml b/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml index f0f7dede28a..f7a8ed20a47 100644 --- a/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/systemcerts/fields/ecs.yml @@ -106,3 +106,5 @@ name: tags - external: ecs name: url.full +- external: ecs + name: related.ip diff --git a/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml b/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml index 31a7235135f..5cdec45a982 100644 --- a/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml +++ b/packages/tychon/elasticsearch/transform/volume/fields/ecs.yml @@ -60,3 +60,5 @@ name: log.file.path - external: ecs name: tags +- external: ecs + name: related.ip diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml b/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml index 291b675502b..4cb860dea83 100644 --- a/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml +++ b/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/fields/ecs.yml @@ -28,3 +28,7 @@ external: ecs - name: observer.vendor external: ecs +- name: message + external: ecs +- name: ecs.version + external: ecs diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml b/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml index 38aa91efa9e..b7c6b004465 100644 --- a/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml +++ b/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/fields/fields.yml @@ -6,6 +6,8 @@ external: ecs - name: cloud.region external: ecs +- name: device.id + external: ecs - name: package.name external: ecs - name: package.version @@ -14,6 +16,8 @@ external: ecs - name: vulnerability.id external: ecs +- name: vulnerability.reference + external: ecs - name: vulnerability.score.base external: ecs - name: vulnerability.score.version @@ -34,6 +38,14 @@ external: ecs - name: event.type external: ecs +- name: ecs.version + external: ecs +- name: tags + external: ecs +- name: related.ip + external: ecs # should it be keyword instead of IP ? Would this be breaking change? +- name: message + external: ecs - name: observer.vendor external: ecs - name: wiz