diff --git a/cmd/testrunner.go b/cmd/testrunner.go
index 057d9b52cc..8e46025904 100644
--- a/cmd/testrunner.go
+++ b/cmd/testrunner.go
@@ -543,6 +543,10 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
+ checkFailureStore, err := esClient.IsFailureStoreAvailable(ctx)
+ if err != nil {
+ return fmt.Errorf("can't check if failure store is available: %w", err)
+ }
if runTearDown || runTestsOnly {
if variantFlag != "" {
@@ -565,6 +569,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
PackageRootPath: packageRootPath,
KibanaClient: kibanaClient,
API: esClient.API,
+ ESClient: esClient,
ConfigFilePath: configFileFlag,
RunSetup: runSetup,
RunTearDown: runTearDown,
@@ -577,6 +582,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
GlobalTestConfig: globalTestConfig.System,
WithCoverage: testCoverage,
CoverageType: testCoverageFormat,
+ CheckFailureStore: checkFailureStore,
})
logger.Debugf("Running suite...")
diff --git a/internal/dump/indextemplates.go b/internal/dump/indextemplates.go
index 7fc50d3cf2..e1cba11b4e 100644
--- a/internal/dump/indextemplates.go
+++ b/internal/dump/indextemplates.go
@@ -6,119 +6,14 @@ package dump
import (
"context"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "slices"
"github.com/elastic/elastic-package/internal/elasticsearch"
+ "github.com/elastic/elastic-package/internal/elasticsearch/ingest"
)
-// IndexTemplate contains information related to an index template for exporting purpouses.
-// It contains a partially parsed index template and the original JSON from the response.
-type IndexTemplate struct {
- TemplateName string `json:"name"`
- IndexTemplate struct {
- Meta struct {
- ManagedBy string `json:"managed_by"`
- Managed bool `json:"managed"`
- Package struct {
- Name string `json:"name"`
- } `json:"package"`
- } `json:"_meta"`
- ComposedOf []string `json:"composed_of"`
- Template struct {
- Settings TemplateSettings `json:"settings"`
- } `json:"template"`
- } `json:"index_template"`
+type IndexTemplate = ingest.IndexTemplate
+type TemplateSettings = ingest.TemplateSettings
- raw json.RawMessage
-}
-
-// TemplateSettings are common settings to all kinds of templates.
-type TemplateSettings struct {
- Index struct {
- DefaultPipeline string `json:"default_pipeline"`
- FinalPipeline string `json:"final_pipeline"`
- Lifecycle struct {
- Name string `json:"name"`
- } `json:"lifecycle"`
- } `json:"index"`
-}
-
-// Name returns the name of the index template.
-func (t IndexTemplate) Name() string {
- return t.TemplateName
-}
-
-// JSON returns the JSON representation of the index template.
-func (t IndexTemplate) JSON() []byte {
- return []byte(t.raw)
-}
-
-// TemplateSettings returns the template settings of this template.
-func (t IndexTemplate) TemplateSettings() TemplateSettings {
- return t.IndexTemplate.Template.Settings
-}
-
-type getIndexTemplateResponse struct {
- IndexTemplates []json.RawMessage `json:"index_templates"`
-}
-
-func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]IndexTemplate, error) {
- resp, err := api.Indices.GetIndexTemplate(
- api.Indices.GetIndexTemplate.WithContext(ctx),
-
- // Wildcard may be too wide, we will double check below if it is a managed template.
- api.Indices.GetIndexTemplate.WithName(fmt.Sprintf("*-%s.*", packageName)),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to get index templates: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == http.StatusNotFound {
- // Some packages don't have index templates.
- return nil, nil
- }
- if resp.IsError() {
- return nil, fmt.Errorf("failed to get index templates: %s", resp.String())
- }
-
- d, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response body: %w", err)
- }
-
- var templateResponse getIndexTemplateResponse
- err = json.Unmarshal(d, &templateResponse)
- if err != nil {
- return nil, fmt.Errorf("failed to decode response: %w", err)
- }
-
- var indexTemplates []IndexTemplate
- for _, indexTemplateRaw := range templateResponse.IndexTemplates {
- var indexTemplate IndexTemplate
- err = json.Unmarshal(indexTemplateRaw, &indexTemplate)
- if err != nil {
- return nil, fmt.Errorf("failed to parse index template: %w", err)
- }
- indexTemplate.raw = indexTemplateRaw
-
- meta := indexTemplate.IndexTemplate.Meta
- if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) {
- // This is not the droid you are looking for.
- continue
- }
-
- indexTemplates = append(indexTemplates, indexTemplate)
- }
-
- return indexTemplates, nil
-}
-
-func managedByFleet(managedBy string) bool {
- var managers = []string{"ingest-manager", "fleet"}
- return slices.Contains(managers, managedBy)
+func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]ingest.IndexTemplate, error) {
+ return ingest.GetIndexTemplatesForPackage(ctx, api, packageName)
}
diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go
index 694306c14c..20dfff16be 100644
--- a/internal/elasticsearch/client.go
+++ b/internal/elasticsearch/client.go
@@ -176,6 +176,32 @@ func (client *Client) CheckHealth(ctx context.Context) error {
return nil
}
+// IsFailureStoreAvailable checks if the failure store is available.
+func (client *Client) IsFailureStoreAvailable(ctx context.Context) (bool, error) {
+ // FIXME: Using the low-level transport till the API SDK supports the failure store.
+ request, err := http.NewRequest(http.MethodGet, "/_search?failure_store=only", nil)
+ if err != nil {
+ return false, fmt.Errorf("failed to create search request: %w", err)
+ }
+ request.Header.Set("Content-Type", "application/json")
+
+ resp, err := client.Transport.Perform(request)
+ if err != nil {
+ return false, fmt.Errorf("failed to perform search request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ switch resp.StatusCode {
+ case http.StatusOK:
+ return true, nil
+ case http.StatusBadRequest:
+ // Error expected when using an unrecognized parameter.
+ return false, nil
+ default:
+ return false, fmt.Errorf("unexpected status code received: %d", resp.StatusCode)
+ }
+}
+
// redHealthCause tries to identify the cause of a cluster in red state. This could be
// also used as a replacement of CheckHealth, but keeping them separated because it uses
// internal undocumented APIs that might change.
diff --git a/internal/elasticsearch/ingest/failurestorage.go b/internal/elasticsearch/ingest/failurestorage.go
new file mode 100644
index 0000000000..52271d16a8
--- /dev/null
+++ b/internal/elasticsearch/ingest/failurestorage.go
@@ -0,0 +1,79 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package ingest
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "github.com/elastic/elastic-package/internal/elasticsearch"
+)
+
+func EnableFailureStore(ctx context.Context, api *elasticsearch.API, indexTemplateName string, enabled bool) error {
+ resp, err := api.Indices.GetIndexTemplate(
+ api.Indices.GetIndexTemplate.WithContext(ctx),
+ api.Indices.GetIndexTemplate.WithName(indexTemplateName),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to get index template %s: %w", indexTemplateName, err)
+ }
+ defer resp.Body.Close()
+ if resp.IsError() {
+ return fmt.Errorf("failed to get index template %s: %s", indexTemplateName, resp.String())
+ }
+
+ var templateResponse struct {
+ IndexTemplates []struct {
+ IndexTemplate map[string]any `json:"index_template"`
+ } `json:"index_templates"`
+ }
+ err = json.NewDecoder(resp.Body).Decode(&templateResponse)
+ if err != nil {
+ return fmt.Errorf("failed to decode response while getting index template %s: %w", indexTemplateName, err)
+ }
+ if n := len(templateResponse.IndexTemplates); n != 1 {
+ return fmt.Errorf("unexpected number of index templates obtained while getting %s, expected 1, found %d", indexTemplateName, err)
+ }
+
+ template := templateResponse.IndexTemplates[0].IndexTemplate
+ dsSettings, found := template["data_stream"]
+ if found {
+ dsMap, ok := dsSettings.(map[string]any)
+ if !ok {
+ return fmt.Errorf("unexpected type for data stream settings in index template %s, expected map, found %T", indexTemplateName, dsMap)
+ }
+ if current, found := dsMap["failure_store"].(bool); found && current == enabled {
+ // Nothing to do, it already has the expected value.
+ return nil
+ }
+ dsMap["failure_store"] = enabled
+ template["data_stream"] = dsMap
+ } else {
+ template["data_stream"] = map[string]any{
+ "failure_store": enabled,
+ }
+ }
+
+ d, err := json.Marshal(template)
+ if err != nil {
+ return fmt.Errorf("failed to marshal template after updating it: %w", err)
+ }
+
+ updateResp, err := api.Indices.PutIndexTemplate(indexTemplateName, bytes.NewReader(d),
+ api.Indices.PutIndexTemplate.WithContext(ctx),
+ api.Indices.PutIndexTemplate.WithCreate(false),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to update index template %s: %w", indexTemplateName, err)
+ }
+ defer updateResp.Body.Close()
+ if updateResp.IsError() {
+ return fmt.Errorf("failed to update index template %s: %s", indexTemplateName, resp.String())
+ }
+
+ return nil
+}
diff --git a/internal/elasticsearch/ingest/failurestorage_test.go b/internal/elasticsearch/ingest/failurestorage_test.go
new file mode 100644
index 0000000000..5e4ede4cff
--- /dev/null
+++ b/internal/elasticsearch/ingest/failurestorage_test.go
@@ -0,0 +1,88 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package ingest
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/elastic/elastic-package/internal/elasticsearch"
+ estest "github.com/elastic/elastic-package/internal/elasticsearch/test"
+)
+
+func TestEnableFailureStore(t *testing.T) {
+ client := estest.NewClient(t, "testdata/elasticsearch-8-enable-failure-store")
+
+ templateName := "ep-test-index-template"
+ templateBody := []byte(`{"index_patterns": ["metrics-eptest.failurestore-*"],"data_stream": {}}`)
+ createTempIndexTemplate(t, client.API, templateName, templateBody)
+ assertFailureStore(t, client.API, templateName, false)
+
+ err := EnableFailureStore(context.Background(), client.API, templateName, true)
+ assert.NoError(t, err)
+ assertFailureStore(t, client.API, templateName, true)
+
+ err = EnableFailureStore(context.Background(), client.API, templateName, false)
+ assert.NoError(t, err)
+ assertFailureStore(t, client.API, templateName, false)
+}
+
+func TestEnableFailureStoreNothingToDo(t *testing.T) {
+ client := estest.NewClient(t, "testdata/elasticsearch-8-enable-failure-store-noop")
+
+ templateName := "ep-test-index-template"
+ templateBody := []byte(`{"index_patterns": ["metrics-eptest.failurestore-*"],"data_stream": {"failure_store":true}}`)
+ createTempIndexTemplate(t, client.API, templateName, templateBody)
+ assertFailureStore(t, client.API, templateName, true)
+
+ err := EnableFailureStore(context.Background(), client.API, templateName, true)
+ assert.NoError(t, err)
+ assertFailureStore(t, client.API, templateName, true)
+}
+
+func createTempIndexTemplate(t *testing.T, api *elasticsearch.API, name string, body []byte) {
+ createResp, err := api.Indices.PutIndexTemplate(name, bytes.NewReader(body),
+ api.Indices.PutIndexTemplate.WithCreate(true),
+ )
+ require.NoError(t, err)
+ require.False(t, createResp.IsError(), createResp.String())
+ t.Cleanup(func() {
+ deleteResp, err := api.Indices.DeleteIndexTemplate(name)
+ require.NoError(t, err)
+ require.False(t, deleteResp.IsError())
+ })
+}
+
+func assertFailureStore(t *testing.T, api *elasticsearch.API, name string, expected bool) {
+ resp, err := api.Indices.GetIndexTemplate(
+ api.Indices.GetIndexTemplate.WithName(name),
+ )
+ require.NoError(t, err)
+ require.False(t, resp.IsError())
+ defer resp.Body.Close()
+
+ var templateResponse struct {
+ IndexTemplates []struct {
+ IndexTemplate struct {
+ DataStream struct {
+ FailureStore *bool `json:"failure_store"`
+ } `json:"data_stream"`
+ } `json:"index_template"`
+ } `json:"index_templates"`
+ }
+ err = json.NewDecoder(resp.Body).Decode(&templateResponse)
+ require.NoError(t, err)
+ require.Len(t, templateResponse.IndexTemplates, 1)
+ found := templateResponse.IndexTemplates[0].IndexTemplate.DataStream.FailureStore
+
+ if assert.NotNil(t, found) {
+ assert.Equal(t, expected, *found)
+ }
+}
diff --git a/internal/elasticsearch/ingest/packages.go b/internal/elasticsearch/ingest/packages.go
new file mode 100644
index 0000000000..0989e4bec6
--- /dev/null
+++ b/internal/elasticsearch/ingest/packages.go
@@ -0,0 +1,125 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package ingest
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "slices"
+
+ "github.com/elastic/elastic-package/internal/elasticsearch"
+)
+
+// IndexTemplate contains information related to an index template for exporting purpouses.
+// It contains a partially parsed index template and the original JSON from the response.
+type IndexTemplate struct {
+ TemplateName string `json:"name"`
+ IndexTemplate struct {
+ Meta struct {
+ ManagedBy string `json:"managed_by"`
+ Managed bool `json:"managed"`
+ Package struct {
+ Name string `json:"name"`
+ } `json:"package"`
+ } `json:"_meta"`
+ ComposedOf []string `json:"composed_of"`
+ Template struct {
+ Settings TemplateSettings `json:"settings"`
+ } `json:"template"`
+ } `json:"index_template"`
+
+ raw json.RawMessage
+}
+
+// TemplateSettings are common settings to all kinds of templates.
+type TemplateSettings struct {
+ Index struct {
+ DefaultPipeline string `json:"default_pipeline"`
+ FinalPipeline string `json:"final_pipeline"`
+ Lifecycle struct {
+ Name string `json:"name"`
+ } `json:"lifecycle"`
+ } `json:"index"`
+}
+
+// Name returns the name of the index template.
+func (t IndexTemplate) Name() string {
+ return t.TemplateName
+}
+
+// JSON returns the JSON representation of the index template.
+func (t IndexTemplate) JSON() []byte {
+ return []byte(t.raw)
+}
+
+// TemplateSettings returns the template settings of this template.
+func (t IndexTemplate) TemplateSettings() TemplateSettings {
+ return t.IndexTemplate.Template.Settings
+}
+
+type getIndexTemplateResponse struct {
+ IndexTemplates []json.RawMessage `json:"index_templates"`
+}
+
+// GetIndexTemplatesForPackage gets the index templates installed for a package.
+func GetIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]IndexTemplate, error) {
+ resp, err := api.Indices.GetIndexTemplate(
+ api.Indices.GetIndexTemplate.WithContext(ctx),
+
+ // Wildcard may be too wide, we will double check below if it is a managed template.
+ api.Indices.GetIndexTemplate.WithName(fmt.Sprintf("*-%s.*", packageName)),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to get index templates: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode == http.StatusNotFound {
+ // Some packages don't have index templates.
+ return nil, nil
+ }
+ if resp.IsError() {
+ return nil, fmt.Errorf("failed to get index templates: %s", resp.String())
+ }
+
+ d, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ var templateResponse getIndexTemplateResponse
+ err = json.Unmarshal(d, &templateResponse)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode response: %w", err)
+ }
+
+ var indexTemplates []IndexTemplate
+ for _, indexTemplateRaw := range templateResponse.IndexTemplates {
+ var indexTemplate IndexTemplate
+ err = json.Unmarshal(indexTemplateRaw, &indexTemplate)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse index template: %w", err)
+ }
+ indexTemplate.raw = indexTemplateRaw
+
+ meta := indexTemplate.IndexTemplate.Meta
+ if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) {
+ // This is not the droid you are looking for.
+ continue
+ }
+
+ indexTemplates = append(indexTemplates, indexTemplate)
+ }
+
+ return indexTemplates, nil
+}
+
+func managedByFleet(managedBy string) bool {
+ var managers = []string{"ingest-manager", "fleet"}
+ return slices.Contains(managers, managedBy)
+}
diff --git a/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store-noop.yaml b/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store-noop.yaml
new file mode 100644
index 0000000000..a83aabdce9
--- /dev/null
+++ b/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store-noop.yaml
@@ -0,0 +1,268 @@
+---
+version: 2
+interactions:
+ - id: 0
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 548
+ uncompressed: false
+ body: |
+ {
+ "name" : "a31464810e80",
+ "cluster_name" : "elasticsearch",
+ "cluster_uuid" : "zOcXOIv5R_qHrrA4HILi7g",
+ "version" : {
+ "number" : "8.15.0-SNAPSHOT",
+ "build_flavor" : "default",
+ "build_type" : "docker",
+ "build_hash" : "9092394b19dea9e0f20290a2571a82b1d3610987",
+ "build_date" : "2024-07-11T17:13:25.001995094Z",
+ "build_snapshot" : true,
+ "lucene_version" : "9.11.1",
+ "minimum_wire_compatibility_version" : "7.17.0",
+ "minimum_index_compatibility_version" : "7.0.0"
+ },
+ "tagline" : "You Know, for Search"
+ }
+ headers:
+ Content-Length:
+ - "548"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 4.25671ms
+ - id: 1
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 91
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: '{"index_patterns": ["metrics-eptest.failurestore-*"],"data_stream": {"failure_store":true}}'
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ Content-Type:
+ - application/json
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template?create=true
+ method: PUT
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 96.553464ms
+ - id: 2
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 223
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":true}}}]}'
+ headers:
+ Content-Length:
+ - "223"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 931.406µs
+ - id: 3
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 223
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":true}}}]}'
+ headers:
+ Content-Length:
+ - "223"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 808.886µs
+ - id: 4
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 223
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":true}}}]}'
+ headers:
+ Content-Length:
+ - "223"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 721.915µs
+ - id: 5
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: DELETE
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 83.638384ms
diff --git a/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store.yaml b/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store.yaml
new file mode 100644
index 0000000000..5674218baa
--- /dev/null
+++ b/internal/elasticsearch/ingest/testdata/elasticsearch-8-enable-failure-store.yaml
@@ -0,0 +1,436 @@
+---
+version: 2
+interactions:
+ - id: 0
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 548
+ uncompressed: false
+ body: |
+ {
+ "name" : "a31464810e80",
+ "cluster_name" : "elasticsearch",
+ "cluster_uuid" : "zOcXOIv5R_qHrrA4HILi7g",
+ "version" : {
+ "number" : "8.15.0-SNAPSHOT",
+ "build_flavor" : "default",
+ "build_type" : "docker",
+ "build_hash" : "9092394b19dea9e0f20290a2571a82b1d3610987",
+ "build_date" : "2024-07-11T17:13:25.001995094Z",
+ "build_snapshot" : true,
+ "lucene_version" : "9.11.1",
+ "minimum_wire_compatibility_version" : "7.17.0",
+ "minimum_index_compatibility_version" : "7.0.0"
+ },
+ "tagline" : "You Know, for Search"
+ }
+ headers:
+ Content-Length:
+ - "548"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 6.311531ms
+ - id: 1
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 71
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: '{"index_patterns": ["metrics-eptest.failurestore-*"],"data_stream": {}}'
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ Content-Type:
+ - application/json
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template?create=true
+ method: PUT
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 137.831843ms
+ - id: 2
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 224
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":false}}}]}'
+ headers:
+ Content-Length:
+ - "224"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 1.034594ms
+ - id: 3
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 224
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":false}}}]}'
+ headers:
+ Content-Length:
+ - "224"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 788.776µs
+ - id: 4
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 150
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: '{"composed_of":[],"data_stream":{"allow_custom_routing":false,"failure_store":true,"hidden":false},"index_patterns":["metrics-eptest.failurestore-*"]}'
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ Content-Type:
+ - application/json
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template?create=false
+ method: PUT
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 103.772284ms
+ - id: 5
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 223
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":true}}}]}'
+ headers:
+ Content-Length:
+ - "223"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 1.043947ms
+ - id: 6
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 223
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":true}}}]}'
+ headers:
+ Content-Length:
+ - "223"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 664.249µs
+ - id: 7
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 151
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: '{"composed_of":[],"data_stream":{"allow_custom_routing":false,"failure_store":false,"hidden":false},"index_patterns":["metrics-eptest.failurestore-*"]}'
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ Content-Type:
+ - application/json
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template?create=false
+ method: PUT
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 99.796675ms
+ - id: 8
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: GET
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 224
+ uncompressed: false
+ body: '{"index_templates":[{"name":"ep-test-index-template","index_template":{"index_patterns":["metrics-eptest.failurestore-*"],"composed_of":[],"data_stream":{"hidden":false,"allow_custom_routing":false,"failure_store":false}}}]}'
+ headers:
+ Content-Length:
+ - "224"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 1.317273ms
+ - id: 9
+ request:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ content_length: 0
+ transfer_encoding: []
+ trailer: {}
+ host: ""
+ remote_addr: ""
+ request_uri: ""
+ body: ""
+ form: {}
+ headers:
+ Authorization:
+ - Basic ZWxhc3RpYzpjaGFuZ2VtZQ==
+ User-Agent:
+ - go-elasticsearch/7.17.10 (linux amd64; Go 1.22.1)
+ X-Elastic-Client-Meta:
+ - es=7.17.10,go=1.22.1,t=7.17.10,hc=1.22.1
+ url: https://127.0.0.1:9200/_index_template/ep-test-index-template
+ method: DELETE
+ response:
+ proto: HTTP/1.1
+ proto_major: 1
+ proto_minor: 1
+ transfer_encoding: []
+ trailer: {}
+ content_length: 21
+ uncompressed: false
+ body: '{"acknowledged":true}'
+ headers:
+ Content-Length:
+ - "21"
+ Content-Type:
+ - application/json
+ X-Elastic-Product:
+ - Elasticsearch
+ status: 200 OK
+ code: 200
+ duration: 92.256183ms
diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl
index 7e182bfb9d..f07bcad9cf 100644
--- a/internal/stack/_static/docker-compose-stack.yml.tmpl
+++ b/internal/stack/_static/docker-compose-stack.yml.tmpl
@@ -17,7 +17,7 @@ services:
start_period: 300s
interval: 5s
environment:
- - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+ - "ES_JAVA_OPTS=-Xms1g -Xmx1g {{ if not (semverLessThan $version "8.15.0-SNAPSHOT") -}}-Des.failure_store_feature_flag_enabled=true{{- end -}}"
- "ELASTIC_PASSWORD={{ $password }}"
volumes:
- "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
diff --git a/internal/stack/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl
index 9007ed5584..43a8a981aa 100644
--- a/internal/stack/_static/elasticsearch.yml.tmpl
+++ b/internal/stack/_static/elasticsearch.yml.tmpl
@@ -25,11 +25,11 @@ script.context.template.max_compilations_rate: "unlimited"
script.context.ingest.cache_max_size: 2000
script.context.processor_conditional.cache_max_size: 2000
script.context.template.cache_max_size: 2000
-{{ end }}
+{{- end -}}
{{ $apm_enabled := fact "apm_enabled" }}
{{ if eq $apm_enabled "true" }}
tracing.apm.enabled: true
tracing.apm.agent.server_url: "http://fleet-server:8200"
tracing.apm.agent.environment: "dev"
-{{ end }}
+{{- end -}}
diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 1bea6ea80f..31e515c0a0 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/elastic/elastic-package/internal/elasticsearch"
+ "github.com/elastic/elastic-package/internal/elasticsearch/ingest"
"github.com/elastic/elastic-package/internal/kibana"
"github.com/elastic/elastic-package/internal/logger"
"github.com/elastic/elastic-package/internal/packages"
@@ -28,12 +29,14 @@ type runner struct {
packageRootPath string
kibanaClient *kibana.Client
esAPI *elasticsearch.API
+ esClient *elasticsearch.Client
dataStreams []string
serviceVariant string
globalTestConfig testrunner.GlobalRunnerTestConfig
failOnMissingTests bool
+ checkFailureStore bool
deferCleanup time.Duration
generateTestResult bool
withCoverage bool
@@ -57,6 +60,9 @@ type SystemTestRunnerOptions struct {
KibanaClient *kibana.Client
API *elasticsearch.API
+ // FIXME: Keeping Elasticsearch client to be able to do low-level requests for parameters not supported yet by the API.
+ ESClient *elasticsearch.Client
+
DataStreams []string
ServiceVariant string
@@ -68,6 +74,7 @@ type SystemTestRunnerOptions struct {
GlobalTestConfig testrunner.GlobalRunnerTestConfig
FailOnMissingTests bool
+ CheckFailureStore bool
GenerateTestResult bool
DeferCleanup time.Duration
WithCoverage bool
@@ -79,6 +86,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
packageRootPath: options.PackageRootPath,
kibanaClient: options.KibanaClient,
esAPI: options.API,
+ esClient: options.ESClient,
profile: options.Profile,
dataStreams: options.DataStreams,
serviceVariant: options.ServiceVariant,
@@ -87,6 +95,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
runTestsOnly: options.RunTestsOnly,
runTearDown: options.RunTearDown,
failOnMissingTests: options.FailOnMissingTests,
+ checkFailureStore: options.CheckFailureStore,
generateTestResult: options.GenerateTestResult,
deferCleanup: options.DeferCleanup,
globalTestConfig: options.GlobalTestConfig,
@@ -120,6 +129,34 @@ func (r *runner) SetupRunner(ctx context.Context) error {
return fmt.Errorf("can't install the package: %w", err)
}
+ if r.checkFailureStore {
+ err := r.setupFailureStore(ctx)
+ if err != nil {
+ return fmt.Errorf("can't enable the failure store: %w", err)
+ }
+ }
+
+ return nil
+}
+
+func (r *runner) setupFailureStore(ctx context.Context) error {
+ manifest, err := packages.ReadPackageManifestFromPackageRoot(r.packageRootPath)
+ if err != nil {
+ return fmt.Errorf("failed to read package manifest: %w", err)
+ }
+
+ indexTemplates, err := ingest.GetIndexTemplatesForPackage(ctx, r.esAPI, manifest.Name)
+ if err != nil {
+ return fmt.Errorf("failed to get index templates for package %s: %w", manifest.Name, err)
+ }
+
+ for _, template := range indexTemplates {
+ err := ingest.EnableFailureStore(ctx, r.esAPI, template.Name(), true)
+ if err != nil {
+ return fmt.Errorf("failed to enable failure store for index template %s: %w", template.Name(), err)
+ }
+ }
+
return nil
}
@@ -245,6 +282,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
PackageRootPath: r.packageRootPath,
KibanaClient: r.kibanaClient,
API: r.esAPI,
+ ESClient: r.esClient,
TestFolder: t,
ServiceVariant: variant,
GenerateTestResult: r.generateTestResult,
@@ -256,6 +294,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
GlobalTestConfig: r.globalTestConfig,
WithCoverage: r.withCoverage,
CoverageType: r.coverageType,
+ CheckFailureStore: r.checkFailureStore,
})
if err != nil {
return nil, fmt.Errorf(
diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go
index 74d0e4c0db..6025b68c83 100644
--- a/internal/testrunner/runners/system/tester.go
+++ b/internal/testrunner/runners/system/tester.go
@@ -5,6 +5,7 @@
package system
import (
+ "bytes"
"context"
"encoding/json"
"errors"
@@ -131,6 +132,7 @@ type tester struct {
packageRootPath string
generateTestResult bool
esAPI *elasticsearch.API
+ esClient *elasticsearch.Client
kibanaClient *kibana.Client
runIndependentElasticAgent bool
@@ -153,6 +155,7 @@ type tester struct {
dataStreamManifest *packages.DataStreamManifest
withCoverage bool
coverageType string
+ checkFailureStore bool
serviceStateFilePath string
@@ -176,12 +179,16 @@ type SystemTesterOptions struct {
API *elasticsearch.API
KibanaClient *kibana.Client
- DeferCleanup time.Duration
- ServiceVariant string
- ConfigFileName string
- GlobalTestConfig testrunner.GlobalRunnerTestConfig
- WithCoverage bool
- CoverageType string
+ // FIXME: Keeping Elasticsearch client to be able to do low-level requests for parameters not supported yet by the API.
+ ESClient *elasticsearch.Client
+
+ DeferCleanup time.Duration
+ ServiceVariant string
+ ConfigFileName string
+ GlobalTestConfig testrunner.GlobalRunnerTestConfig
+ WithCoverage bool
+ CoverageType string
+ CheckFailureStore bool
RunSetup bool
RunTearDown bool
@@ -195,6 +202,7 @@ func NewSystemTester(options SystemTesterOptions) (*tester, error) {
packageRootPath: options.PackageRootPath,
generateTestResult: options.GenerateTestResult,
esAPI: options.API,
+ esClient: options.ESClient,
kibanaClient: options.KibanaClient,
deferCleanup: options.DeferCleanup,
serviceVariant: options.ServiceVariant,
@@ -205,6 +213,7 @@ func NewSystemTester(options SystemTesterOptions) (*tester, error) {
globalTestConfig: options.GlobalTestConfig,
withCoverage: options.WithCoverage,
coverageType: options.CoverageType,
+ checkFailureStore: options.CheckFailureStore,
runIndependentElasticAgent: true,
}
r.resourcesManager = resources.NewManager()
@@ -755,18 +764,94 @@ func (r *tester) getDocs(ctx context.Context, dataStream string) (*hits, error)
return &hits, nil
}
+func (r *tester) getFailureStoreDocs(ctx context.Context, dataStream string) ([]failureStoreDocument, error) {
+ query := map[string]any{
+ "query": map[string]any{
+ "bool": map[string]any{
+ // Ignoring failures with error.type version_conflict_engine_exception because there are packages which
+ // explicitly set the _id with the fingerprint processor to avoid duplicates.
+ "must_not": map[string]any{
+ "term": map[string]any{
+ "error.type": "version_conflict_engine_exception",
+ },
+ },
+ },
+ },
+ }
+ body, err := json.Marshal(query)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encode search query: %w", err)
+ }
+ // FIXME: Using the low-level transport till the API SDK supports the failure store.
+ request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("/%s/_search?failure_store=only", dataStream), bytes.NewReader(body))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create search request: %w", err)
+ }
+ request.Header.Set("Content-Type", "application/json")
+
+ resp, err := r.esClient.Transport.Perform(request)
+ if err != nil {
+ return nil, fmt.Errorf("failed to perform search request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ switch {
+ case resp.StatusCode == http.StatusNotFound:
+ // Can happen if the data stream hasn't been created yet.
+ return nil, nil
+ case resp.StatusCode == http.StatusServiceUnavailable:
+ // Index is being created, but no shards are available yet.
+ // See https://github.com/elastic/elasticsearch/issues/65846
+ return nil, nil
+ case resp.StatusCode >= 400:
+ return nil, fmt.Errorf("search request returned status code %d", resp.StatusCode)
+ }
+
+ var results struct {
+ Hits struct {
+ Hits []struct {
+ Source failureStoreDocument `json:"_source"`
+ Fields common.MapStr `json:"fields"`
+ } `json:"hits"`
+ } `json:"hits"`
+ }
+ err = json.NewDecoder(resp.Body).Decode(&results)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode search response: %w", err)
+ }
+
+ var docs []failureStoreDocument
+ for _, hit := range results.Hits.Hits {
+ docs = append(docs, hit.Source)
+ }
+
+ return docs, nil
+}
+
type scenarioTest struct {
dataStream string
policyTemplateName string
kibanaDataStream kibana.PackageDataStream
syntheticEnabled bool
docs []common.MapStr
+ failureStore []failureStoreDocument
ignoredFields []string
degradedDocs []common.MapStr
agent agentdeployer.DeployedAgent
startTestTime time.Time
}
+type failureStoreDocument struct {
+ Error struct {
+ Type string `json:"type"`
+ Message string `json:"message"`
+ StackTrace string `json:"stack_trace"`
+ PipelineTrace []string `json:"pipeline_trace"`
+ Pipeline string `json:"pipeline"`
+ ProcessorType string `json:"processor_type"`
+ } `json:"error"`
+}
+
func (r *tester) deleteDataStream(ctx context.Context, dataStream string) error {
resp, err := r.esAPI.Indices.DeleteDataStream([]string{dataStream},
r.esAPI.Indices.DeleteDataStream.WithContext(ctx),
@@ -1087,6 +1172,18 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf
return false, err
}
+ if r.checkFailureStore {
+ failureStore, err := r.getFailureStoreDocs(ctx, scenario.dataStream)
+ if err != nil {
+ return false, fmt.Errorf("failed to check failure store: %w", err)
+ }
+ if n := len(failureStore); n > 0 {
+ // Interrupt loop earlier if there are failures in the document store.
+ logger.Debugf("Found %d hits in the failure store for %s", len(failureStore), scenario.dataStream)
+ return true, nil
+ }
+ }
+
if config.Assert.HitCount > 0 {
if hits.size() < config.Assert.HitCount {
return false, nil
@@ -1132,6 +1229,14 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf
scenario.docs = hits.getDocs(scenario.syntheticEnabled)
scenario.ignoredFields = hits.IgnoredFields
scenario.degradedDocs = hits.DegradedDocs
+ if r.checkFailureStore {
+ logger.Debugf("Checking failure store for data stream %s", scenario.dataStream)
+ scenario.failureStore, err = r.getFailureStoreDocs(ctx, scenario.dataStream)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get documents from the failure store for data stream %s: %w", scenario.dataStream, err)
+ }
+ logger.Debugf("Found %d docs in failure store for data stream %s", len(scenario.failureStore), scenario.dataStream)
+ }
if r.runSetup {
opts := scenarioStateOpts{
@@ -1269,6 +1374,10 @@ func (r *tester) createServiceStateDir() error {
}
func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.ResultComposer, scenario *scenarioTest, config *testConfig) ([]testrunner.TestResult, error) {
+ if err := validateFailureStore(scenario.failureStore); err != nil {
+ return result.WithError(err)
+ }
+
// Validate fields in docs
// when reroute processors are used, expectedDatasets should be set depends on the processor config
var expectedDatasets []string
@@ -1316,7 +1425,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
fields.WithDisableNormalization(scenario.syntheticEnabled),
)
if err != nil {
- return result.WithError(fmt.Errorf("creating fields validator for data stream failed (path: %s): %w", r.dataStreamPath, err))
+ return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", r.dataStreamPath, err)
}
if err := validateFields(scenario.docs, fieldsValidator, scenario.dataStream); err != nil {
return result.WithError(err)
@@ -1331,13 +1440,14 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
if scenario.syntheticEnabled {
docs, err = fieldsValidator.SanitizeSyntheticSourceDocs(scenario.docs)
if err != nil {
- return result.WithError(fmt.Errorf("failed to sanitize synthetic source docs: %w", err))
+ results, _ := result.WithErrorf("failed to sanitize synthetic source docs: %w", err)
+ return results, nil
}
}
specVersion, err := semver.NewVersion(r.pkgManifest.SpecVersion)
if err != nil {
- return result.WithError(fmt.Errorf("failed to parse format version %q: %w", r.pkgManifest.SpecVersion, err))
+ return result.WithErrorf("failed to parse format version %q: %w", r.pkgManifest.SpecVersion, err)
}
// Write sample events file from first doc, if requested
@@ -1352,7 +1462,8 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
// Check transforms if present
if err := r.checkTransforms(ctx, config, r.pkgManifest, scenario.kibanaDataStream, scenario.dataStream); err != nil {
- return result.WithError(err)
+ results, _ := result.WithError(err)
+ return results, nil
}
if scenario.agent != nil {
@@ -1856,6 +1967,30 @@ func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version
return nil
}
+func validateFailureStore(failureStore []failureStoreDocument) error {
+ var multiErr multierror.Error
+ for _, doc := range failureStore {
+ // TODO: Move this to the trace log level when available.
+ logger.Debug("Error found in failure store: ", doc.Error.StackTrace)
+ multiErr = append(multiErr,
+ fmt.Errorf("%s: %s (processor: %s, pipelines: %s)",
+ doc.Error.Type,
+ doc.Error.Message,
+ doc.Error.ProcessorType,
+ strings.Join(doc.Error.PipelineTrace, ",")))
+ }
+
+ if len(multiErr) > 0 {
+ multiErr = multiErr.Unique()
+ return testrunner.ErrTestCaseFailed{
+ Reason: "one or more documents found in the failure store",
+ Details: multiErr.Error(),
+ }
+ }
+
+ return nil
+}
+
func validateFields(docs []common.MapStr, fieldsValidator *fields.Validator, dataStream string) error {
var multiErr multierror.Error
for _, doc := range docs {
diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh
index ffb02aec1c..9751003841 100755
--- a/scripts/test-check-false-positives.sh
+++ b/scripts/test-check-false-positives.sh
@@ -76,16 +76,33 @@ function check_build_output() {
)
}
+function stack_version_args() {
+ if [[ -z "$PACKAGE_UNDER_TEST" ]]; then
+ # Don't force stack version if we are testing multiple packages.
+ return
+ fi
+
+ local package_root=test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST}/
+ local stack_version_file="${package_root%/}.stack_version"
+ if [[ ! -f $stack_version_file ]]; then
+ return
+ fi
+
+ echo -n "--version $(cat $stack_version_file)"
+}
+
trap cleanup EXIT
ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
export ELASTIC_PACKAGE_LINKS_FILE_PATH
+stack_args=$(stack_version_args)
+
# Update the stack
-elastic-package stack update -v
+elastic-package stack update -v ${stack_args}
# Boot up the stack
-elastic-package stack up -d -v
+elastic-package stack up -d -v ${stack_args}
elastic-package stack status
diff --git a/test/packages/false_positives/failure_store.expected_errors b/test/packages/false_positives/failure_store.expected_errors
new file mode 100644
index 0000000000..7e2f2d181c
--- /dev/null
+++ b/test/packages/false_positives/failure_store.expected_errors
@@ -0,0 +1 @@
+test case failed: one or more documents found in the failure store:.*
diff --git a/test/packages/false_positives/failure_store.stack_version b/test/packages/false_positives/failure_store.stack_version
new file mode 100644
index 0000000000..d95b929b33
--- /dev/null
+++ b/test/packages/false_positives/failure_store.stack_version
@@ -0,0 +1 @@
+8.15.0-SNAPSHOT
diff --git a/test/packages/false_positives/failure_store/LICENSE.txt b/test/packages/false_positives/failure_store/LICENSE.txt
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/test/packages/false_positives/failure_store/LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/test/packages/false_positives/failure_store/changelog.yml b/test/packages/false_positives/failure_store/changelog.yml
new file mode 100644
index 0000000000..bb0320a524
--- /dev/null
+++ b/test/packages/false_positives/failure_store/changelog.yml
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+ changes:
+ - description: Initial draft of the package
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link
diff --git a/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/docker-compose.yml b/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/docker-compose.yml
new file mode 100644
index 0000000000..ec376194ca
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/docker-compose.yml
@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+ logs:
+ image: alpine
+ volumes:
+ - ./logs:/logs:ro
+ - ${SERVICE_LOGS_DIR}:/var/log
+ command: /bin/sh -c "echo \"Copying files...\"; cp /logs/* /var/log/; echo \"Done.\"; sleep 500"
diff --git a/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/logs/logs.log b/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/logs/logs.log
new file mode 100644
index 0000000000..b915715817
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/_dev/deploy/docker/logs/logs.log
@@ -0,0 +1,8 @@
+1
+2
+3
+4
+five
+six
+seven
+eight
diff --git a/test/packages/false_positives/failure_store/data_stream/test/_dev/test/system/test-fail-config.yml b/test/packages/false_positives/failure_store/data_stream/test/_dev/test/system/test-fail-config.yml
new file mode 100644
index 0000000000..34521f5001
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/_dev/test/system/test-fail-config.yml
@@ -0,0 +1,6 @@
+input: logfile
+service: logs
+data_stream:
+ vars:
+ paths:
+ - "{{SERVICE_LOGS_DIR}}/*.log"
diff --git a/test/packages/false_positives/failure_store/data_stream/test/agent/stream/stream.yml.hbs b/test/packages/false_positives/failure_store/data_stream/test/agent/stream/stream.yml.hbs
new file mode 100644
index 0000000000..5845510de8
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/agent/stream/stream.yml.hbs
@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+ - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+ - add_locale: ~
diff --git a/test/packages/false_positives/failure_store/data_stream/test/elasticsearch/ingest_pipeline/default.yml b/test/packages/false_positives/failure_store/data_stream/test/elasticsearch/ingest_pipeline/default.yml
new file mode 100644
index 0000000000..3a361c646b
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,9 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- convert:
+ field: message
+ type: integer
+- rename:
+ field: message
+ target_field: number
diff --git a/test/packages/false_positives/failure_store/data_stream/test/fields/base-fields.yml b/test/packages/false_positives/failure_store/data_stream/test/fields/base-fields.yml
new file mode 100644
index 0000000000..5977a33eac
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/fields/base-fields.yml
@@ -0,0 +1,23 @@
+- name: data_stream.type
+ type: constant_keyword
+ description: Data stream type.
+- name: data_stream.dataset
+ type: constant_keyword
+ description: Data stream dataset.
+- name: data_stream.namespace
+ type: constant_keyword
+ description: Data stream namespace.
+- name: '@timestamp'
+ type: date
+ description: Event timestamp.
+- name: ecs.version
+ type: keyword
+- name: log.file.path
+ type: keyword
+- name: log.offset
+ type: long
+- name: input.type
+ type: keyword
+ description: Input type
+- name: number
+ type: long
diff --git a/test/packages/false_positives/failure_store/data_stream/test/manifest.yml b/test/packages/false_positives/failure_store/data_stream/test/manifest.yml
new file mode 100644
index 0000000000..d711b8ce30
--- /dev/null
+++ b/test/packages/false_positives/failure_store/data_stream/test/manifest.yml
@@ -0,0 +1,17 @@
+title: "Test Data Stream"
+type: logs
+streams:
+ - input: logfile
+ title: Sample logs
+ description: Collect sample logs
+ vars:
+ - name: paths
+ type: text
+ title: Paths
+ multi: true
+ default:
+ - /var/log/*.log
+elasticsearch:
+ index_template:
+ mappings:
+ subobjects: false
diff --git a/test/packages/false_positives/failure_store/docs/README.md b/test/packages/false_positives/failure_store/docs/README.md
new file mode 100644
index 0000000000..b2478003a5
--- /dev/null
+++ b/test/packages/false_positives/failure_store/docs/README.md
@@ -0,0 +1,84 @@
+
+
+
+# Failure goes to failure store
+
+
+
+## Data streams
+
+
+
+
+
+
+
+
+
+
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+
+
+## Setup
+
+
+
+For step-by-step instructions on how to set up an integration, see the
+[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/packages/false_positives/failure_store/img/sample-logo.svg b/test/packages/false_positives/failure_store/img/sample-logo.svg
new file mode 100644
index 0000000000..6268dd88f3
--- /dev/null
+++ b/test/packages/false_positives/failure_store/img/sample-logo.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/test/packages/false_positives/failure_store/img/sample-screenshot.png b/test/packages/false_positives/failure_store/img/sample-screenshot.png
new file mode 100644
index 0000000000..d7a56a3ecc
Binary files /dev/null and b/test/packages/false_positives/failure_store/img/sample-screenshot.png differ
diff --git a/test/packages/false_positives/failure_store/manifest.yml b/test/packages/false_positives/failure_store/manifest.yml
new file mode 100644
index 0000000000..41a4a38aca
--- /dev/null
+++ b/test/packages/false_positives/failure_store/manifest.yml
@@ -0,0 +1,36 @@
+format_version: 3.2.1
+name: failure_store
+title: "Failure goes to failure store"
+version: 0.0.1
+source:
+ license: "Apache-2.0"
+description: "This is a package whose system tests fail with documents in the failure store."
+type: integration
+categories:
+ - custom
+conditions:
+ kibana:
+ version: "^8.14.3"
+ elastic:
+ subscription: "basic"
+screenshots:
+ - src: /img/sample-screenshot.png
+ title: Sample screenshot
+ size: 600x600
+ type: image/png
+icons:
+ - src: /img/sample-logo.svg
+ title: Sample logo
+ size: 32x32
+ type: image/svg+xml
+policy_templates:
+ - name: sample
+ title: Sample logs
+ description: Collect sample logs
+ inputs:
+ - type: logfile
+ title: Collect sample logs from instances
+ description: Collecting sample logs
+owner:
+ github: elastic/integrations
+ type: elastic