Enable and check failure store on system tests

cmd/testrunner.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
@@ -534,6 +535,15 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}
+	versionInfo, err := kibanaClient.Version()
+	if err != nil {
+		return fmt.Errorf("can't get version info from Kibana client: %w", err)
+	}
+	stackVersion, err := semver.NewVersion(versionInfo.Number)
+	if err != nil {
+		return fmt.Errorf("can't parse Kibana version %q: %w", versionInfo.Number, err)
+	}
+	checkFailureStore := !stackVersion.LessThan(semver.MustParse("8.15.0"))
 
 	esClient, err := stack.NewElasticsearchClientFromProfile(profile)
 	if err != nil {
@@ -565,6 +575,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		PackageRootPath:    packageRootPath,
 		KibanaClient:       kibanaClient,
 		API:                esClient.API,
+		ESClient:           esClient,
 		ConfigFilePath:     configFileFlag,
 		RunSetup:           runSetup,
 		RunTearDown:        runTearDown,
@@ -577,6 +588,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		GlobalTestConfig:   globalTestConfig.System,
 		WithCoverage:       testCoverage,
 		CoverageType:       testCoverageFormat,
+		CheckFailureStore:  checkFailureStore,
 	})
 
 	logger.Debugf("Running suite...")

internal/dump/indextemplates.go

@@ -6,119 +6,14 @@ package dump
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"slices"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/elasticsearch/ingest"
 )
 
-// IndexTemplate contains information related to an index template for exporting purpouses.
-// It  contains a partially parsed index template and the original JSON from the response.
-type IndexTemplate struct {
-	TemplateName  string `json:"name"`
-	IndexTemplate struct {
-		Meta struct {
-			ManagedBy string `json:"managed_by"`
-			Managed   bool   `json:"managed"`
-			Package   struct {
-				Name string `json:"name"`
-			} `json:"package"`
-		} `json:"_meta"`
-		ComposedOf []string `json:"composed_of"`
-		Template   struct {
-			Settings TemplateSettings `json:"settings"`
-		} `json:"template"`
-	} `json:"index_template"`
+type IndexTemplate = ingest.IndexTemplate
+type TemplateSettings = ingest.TemplateSettings
 
-	raw json.RawMessage
-}
-
-// TemplateSettings are common settings to all kinds of templates.
-type TemplateSettings struct {
-	Index struct {
-		DefaultPipeline string `json:"default_pipeline"`
-		FinalPipeline   string `json:"final_pipeline"`
-		Lifecycle       struct {
-			Name string `json:"name"`
-		} `json:"lifecycle"`
-	} `json:"index"`
-}
-
-// Name returns the name of the index template.
-func (t IndexTemplate) Name() string {
-	return t.TemplateName
-}
-
-// JSON returns the JSON representation of the index template.
-func (t IndexTemplate) JSON() []byte {
-	return []byte(t.raw)
-}
-
-// TemplateSettings returns the template settings of this template.
-func (t IndexTemplate) TemplateSettings() TemplateSettings {
-	return t.IndexTemplate.Template.Settings
-}
-
-type getIndexTemplateResponse struct {
-	IndexTemplates []json.RawMessage `json:"index_templates"`
-}
-
-func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]IndexTemplate, error) {
-	resp, err := api.Indices.GetIndexTemplate(
-		api.Indices.GetIndexTemplate.WithContext(ctx),
-
-		// Wildcard may be too wide, we will double check below if it is a managed template.
-		api.Indices.GetIndexTemplate.WithName(fmt.Sprintf("*-%s.*", packageName)),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get index templates: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusNotFound {
-		// Some packages don't have index templates.
-		return nil, nil
-	}
-	if resp.IsError() {
-		return nil, fmt.Errorf("failed to get index templates: %s", resp.String())
-	}
-
-	d, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
-	}
-
-	var templateResponse getIndexTemplateResponse
-	err = json.Unmarshal(d, &templateResponse)
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode response: %w", err)
-	}
-
-	var indexTemplates []IndexTemplate
-	for _, indexTemplateRaw := range templateResponse.IndexTemplates {
-		var indexTemplate IndexTemplate
-		err = json.Unmarshal(indexTemplateRaw, &indexTemplate)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse index template: %w", err)
-		}
-		indexTemplate.raw = indexTemplateRaw
-
-		meta := indexTemplate.IndexTemplate.Meta
-		if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) {
-			// This is not the droid you are looking for.
-			continue
-		}
-
-		indexTemplates = append(indexTemplates, indexTemplate)
-	}
-
-	return indexTemplates, nil
-}
-
-func managedByFleet(managedBy string) bool {
-	var managers = []string{"ingest-manager", "fleet"}
-	return slices.Contains(managers, managedBy)
+func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]ingest.IndexTemplate, error) {
+	return ingest.GetIndexTemplatesForPackage(ctx, api, packageName)
 }

internal/elasticsearch/ingest/packages.go

@@ -0,0 +1,125 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package ingest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"slices"
+
+	"github.com/elastic/elastic-package/internal/elasticsearch"
+)
+
+// IndexTemplate contains information related to an index template for exporting purpouses.
+// It contains a partially parsed index template and the original JSON from the response.
+type IndexTemplate struct {
+	TemplateName  string `json:"name"`
+	IndexTemplate struct {
+		Meta struct {
+			ManagedBy string `json:"managed_by"`
+			Managed   bool   `json:"managed"`
+			Package   struct {
+				Name string `json:"name"`
+			} `json:"package"`
+		} `json:"_meta"`
+		ComposedOf []string `json:"composed_of"`
+		Template   struct {
+			Settings TemplateSettings `json:"settings"`
+		} `json:"template"`
+	} `json:"index_template"`
+
+	raw json.RawMessage
+}
+
+// TemplateSettings are common settings to all kinds of templates.
+type TemplateSettings struct {
+	Index struct {
+		DefaultPipeline string `json:"default_pipeline"`
+		FinalPipeline   string `json:"final_pipeline"`
+		Lifecycle       struct {
+			Name string `json:"name"`
+		} `json:"lifecycle"`
+	} `json:"index"`
+}
+
+// Name returns the name of the index template.
+func (t IndexTemplate) Name() string {
+	return t.TemplateName
+}
+
+// JSON returns the JSON representation of the index template.
+func (t IndexTemplate) JSON() []byte {
+	return []byte(t.raw)
+}
+
+// TemplateSettings returns the template settings of this template.
+func (t IndexTemplate) TemplateSettings() TemplateSettings {
+	return t.IndexTemplate.Template.Settings
+}
+
+type getIndexTemplateResponse struct {
+	IndexTemplates []json.RawMessage `json:"index_templates"`
+}
+
+// GetIndexTemplatesForPackage gets the index templates installed for a package.
+func GetIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]IndexTemplate, error) {
+	resp, err := api.Indices.GetIndexTemplate(
+		api.Indices.GetIndexTemplate.WithContext(ctx),
+
+		// Wildcard may be too wide, we will double check below if it is a managed template.
+		api.Indices.GetIndexTemplate.WithName(fmt.Sprintf("*-%s.*", packageName)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get index templates: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotFound {
+		// Some packages don't have index templates.
+		return nil, nil
+	}
+	if resp.IsError() {
+		return nil, fmt.Errorf("failed to get index templates: %s", resp.String())
+	}
+
+	d, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var templateResponse getIndexTemplateResponse
+	err = json.Unmarshal(d, &templateResponse)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode response: %w", err)
+	}
+
+	var indexTemplates []IndexTemplate
+	for _, indexTemplateRaw := range templateResponse.IndexTemplates {
+		var indexTemplate IndexTemplate
+		err = json.Unmarshal(indexTemplateRaw, &indexTemplate)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse index template: %w", err)
+		}
+		indexTemplate.raw = indexTemplateRaw
+
+		meta := indexTemplate.IndexTemplate.Meta
+		if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) {
+			// This is not the droid you are looking for.
+			continue
+		}
+
+		indexTemplates = append(indexTemplates, indexTemplate)
+	}
+
+	return indexTemplates, nil
+}
+
+func managedByFleet(managedBy string) bool {
+	var managers = []string{"ingest-manager", "fleet"}
+	return slices.Contains(managers, managedBy)
+}

internal/testrunner/runners/system/runner.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/elasticsearch/ingest"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
@@ -28,12 +29,14 @@ type runner struct {
 	packageRootPath string
 	kibanaClient    *kibana.Client
 	esAPI           *elasticsearch.API
+	esClient        *elasticsearch.Client
 
 	dataStreams    []string
 	serviceVariant string
 
 	globalTestConfig   testrunner.GlobalRunnerTestConfig
 	failOnMissingTests bool
+	checkFailureStore  bool
 	deferCleanup       time.Duration
 	generateTestResult bool
 	withCoverage       bool
@@ -57,6 +60,9 @@ type SystemTestRunnerOptions struct {
 	KibanaClient    *kibana.Client
 	API             *elasticsearch.API
 
+	// FIXME: Keeping Elasticsearch client to be able to do low-level requests for parameters not supported yet by the API.
+	ESClient *elasticsearch.Client
+
 	DataStreams    []string
 	ServiceVariant string
 
@@ -68,6 +74,7 @@ type SystemTestRunnerOptions struct {
 	GlobalTestConfig testrunner.GlobalRunnerTestConfig
 
 	FailOnMissingTests bool
+	CheckFailureStore  bool
 	GenerateTestResult bool
 	DeferCleanup       time.Duration
 	WithCoverage       bool
@@ -79,6 +86,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
 		packageRootPath:    options.PackageRootPath,
 		kibanaClient:       options.KibanaClient,
 		esAPI:              options.API,
+		esClient:           options.ESClient,
 		profile:            options.Profile,
 		dataStreams:        options.DataStreams,
 		serviceVariant:     options.ServiceVariant,
@@ -87,6 +95,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner {
 		runTestsOnly:       options.RunTestsOnly,
 		runTearDown:        options.RunTearDown,
 		failOnMissingTests: options.FailOnMissingTests,
+		checkFailureStore:  options.CheckFailureStore,
 		generateTestResult: options.GenerateTestResult,
 		deferCleanup:       options.DeferCleanup,
 		globalTestConfig:   options.GlobalTestConfig,
@@ -120,6 +129,34 @@ func (r *runner) SetupRunner(ctx context.Context) error {
 		return fmt.Errorf("can't install the package: %w", err)
 	}
 
+	if r.checkFailureStore {
+		err := r.setupFailureStore(ctx)
+		if err != nil {
+			return fmt.Errorf("can't enable the failure store: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (r *runner) setupFailureStore(ctx context.Context) error {
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(r.packageRootPath)
+	if err != nil {
+		return fmt.Errorf("failed to read package manifest: %w", err)
+	}
+
+	indexTemplates, err := ingest.GetIndexTemplatesForPackage(ctx, r.esAPI, manifest.Name)
+	if err != nil {
+		return fmt.Errorf("failed to get index templates for package %s: %w", manifest.Name, err)
+	}
+
+	for _, template := range indexTemplates {
+		err := ingest.EnableFailureStore(ctx, r.esAPI, template.Name(), true)
+		if err != nil {
+			return fmt.Errorf("failed to enable failure store for index template %s: %w", template.Name(), err)
+		}
+	}
+
 	return nil
 }
 
@@ -245,6 +282,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
 					PackageRootPath:    r.packageRootPath,
 					KibanaClient:       r.kibanaClient,
 					API:                r.esAPI,
+					ESClient:           r.esClient,
 					TestFolder:         t,
 					ServiceVariant:     variant,
 					GenerateTestResult: r.generateTestResult,
@@ -256,6 +294,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) {
 					GlobalTestConfig:   r.globalTestConfig,
 					WithCoverage:       r.withCoverage,
 					CoverageType:       r.coverageType,
+					CheckFailureStore:  r.checkFailureStore,
 				})
 				if err != nil {
 					return nil, fmt.Errorf(