Remove unused API keys

internal/serverless/project.go

@@ -31,7 +31,6 @@ type Project struct {
 	Region string `json:"region_id"`
 
 	Credentials struct {
-		APIKey   string `json:"apiKey,omitempty"`
 		Username string `json:"username,omitempty"`
 		Password string `json:"password,omitempty"`
 	} `json:"credentials"`

internal/stack/serverless.go

@@ -9,8 +9,11 @@ import (
 	"errors"
 	"fmt"
 	"slices"
+	"strings"
 	"time"
 
+	"github.com/elastic/elastic-package/internal/compose"
+	"github.com/elastic/elastic-package/internal/docker"
 	"github.com/elastic/elastic-package/internal/elasticsearch"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/logger"
@@ -76,7 +79,6 @@ func (sp *serverlessProvider) createProject(ctx context.Context, settings projec
 	}
 	config.ElasticsearchHost = project.Endpoints.Elasticsearch
 	config.KibanaHost = project.Endpoints.Kibana
-	config.ElasticsearchAPIKey = project.Credentials.APIKey
 	config.ElasticsearchUsername = project.Credentials.Username
 	config.ElasticsearchPassword = project.Credentials.Password
 
@@ -288,10 +290,7 @@ func (sp *serverlessProvider) BootUp(ctx context.Context, options Options) error
 	}
 
 	logger.Infof("Starting local services")
-	localServices := &localServicesManager{
-		profile: sp.profile,
-	}
-	err = localServices.start(ctx, options, config)
+	err = sp.startLocalServices(ctx, options, config)
 	if err != nil {
 		return fmt.Errorf("failed to start local services: %w", err)
 	}
@@ -308,6 +307,55 @@ func (sp *serverlessProvider) BootUp(ctx context.Context, options Options) error
 	return nil
 }
 
+func (sp *serverlessProvider) composeProjectName() string {
+	return DockerComposeProjectName(sp.profile)
+}
+
+func (sp *serverlessProvider) localServicesComposeProject() (*compose.Project, error) {
+	composeFile := sp.profile.Path(ProfileStackPath, ComposeFile)
+	return compose.NewProject(sp.composeProjectName(), composeFile)
+}
+
+func (sp *serverlessProvider) startLocalServices(ctx context.Context, options Options, config Config) error {
+	err := applyServerlessResources(sp.profile, options.StackVersion, config)
+	if err != nil {
+		return fmt.Errorf("could not initialize compose files for local services: %w", err)
+	}
+
+	project, err := sp.localServicesComposeProject()
+	if err != nil {
+		return fmt.Errorf("could not initialize local services compose project")
+	}
+
+	opts := compose.CommandOptions{
+		ExtraArgs: []string{},
+	}
+	err = project.Build(ctx, opts)
+	if err != nil {
+		return fmt.Errorf("failed to build images for local services: %w", err)
+	}
+
+	if options.DaemonMode {
+		opts.ExtraArgs = append(opts.ExtraArgs, "-d")
+	}
+	if err := project.Up(ctx, opts); err != nil {
+		// At least starting on 8.6.0, fleet-server may be reconfigured or
+		// restarted after being healthy. If elastic-agent tries to enroll at
+		// this moment, it fails inmediately, stopping and making `docker-compose up`
+		// to fail too.
+		// As a workaround, try to give another chance to docker-compose if only
+		// elastic-agent failed.
+		if onlyElasticAgentFailed(ctx, options) && !errors.Is(err, context.Canceled) {
+			fmt.Println("Elastic Agent failed to start, trying again.")
+			if err := project.Up(ctx, opts); err != nil {
+				return fmt.Errorf("failed to start local services: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (sp *serverlessProvider) TearDown(ctx context.Context, options Options) error {
 	config, err := LoadConfig(sp.profile)
 	if err != nil {
@@ -316,20 +364,13 @@ func (sp *serverlessProvider) TearDown(ctx context.Context, options Options) err
 
 	var errs error
 
-	localServices := &localServicesManager{
-		profile: sp.profile,
-	}
-	err = localServices.destroy(ctx)
+	err = sp.destroyLocalServices(ctx)
 	if err != nil {
 		logger.Errorf("failed to destroy local services: %v", err)
 		errs = fmt.Errorf("failed to destroy local services: %w", err)
 	}
 
 	project, err := sp.currentProject(ctx, config)
-	if errors.Is(err, serverless.ErrProjectNotExist) {
-		logger.Debug("Project does not exist")
-		return nil
-	}
 	if err != nil {
 		return fmt.Errorf("failed to find current project: %w", err)
 	}
@@ -347,6 +388,24 @@ func (sp *serverlessProvider) TearDown(ctx context.Context, options Options) err
 	return errs
 }
 
+func (sp *serverlessProvider) destroyLocalServices(ctx context.Context) error {
+	project, err := sp.localServicesComposeProject()
+	if err != nil {
+		return fmt.Errorf("could not initialize local services compose project")
+	}
+
+	opts := compose.CommandOptions{
+		// Remove associated volumes.
+		ExtraArgs: []string{"--volumes", "--remove-orphans"},
+	}
+	err = project.Down(ctx, opts)
+	if err != nil {
+		return fmt.Errorf("failed to destroy local services: %w", err)
+	}
+
+	return nil
+}
+
 func (sp *serverlessProvider) Update(ctx context.Context, options Options) error {
 	return fmt.Errorf("not implemented")
 }
@@ -393,10 +452,7 @@ func (sp *serverlessProvider) Status(ctx context.Context, options Options) ([]Se
 		})
 	}
 
-	localServices := &localServicesManager{
-		profile: sp.profile,
-	}
-	agentStatus, err := localServices.status()
+	agentStatus, err := sp.localAgentStatus()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get local agent status: %w", err)
 	}
@@ -405,3 +461,66 @@ func (sp *serverlessProvider) Status(ctx context.Context, options Options) ([]Se
 
 	return serviceStatus, nil
 }
+
+func (sp *serverlessProvider) localAgentStatus() ([]ServiceStatus, error) {
+	var services []ServiceStatus
+	serviceStatusFunc := func(description docker.ContainerDescription) error {
+		service, err := newServiceStatus(&description)
+		if err != nil {
+			return err
+		}
+		services = append(services, *service)
+		return nil
+	}
+
+	err := runOnLocalServices(sp.composeProjectName(), serviceStatusFunc)
+	if err != nil {
+		return nil, err
+	}
+
+	return services, nil
+}
+
+func localServiceNames(project string) ([]string, error) {
+	services := []string{}
+	serviceFunc := func(description docker.ContainerDescription) error {
+		services = append(services, description.Config.Labels.ComposeService)
+		return nil
+	}
+
+	err := runOnLocalServices(project, serviceFunc)
+	if err != nil {
+		return nil, err
+	}
+
+	return services, nil
+}
+
+func runOnLocalServices(project string, serviceFunc func(docker.ContainerDescription) error) error {
+	// query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles
+	containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, project)
+	if err != nil {
+		return err
+	}
+
+	if len(containerIDs) == 0 {
+		return nil
+	}
+
+	containerDescriptions, err := docker.InspectContainers(containerIDs...)
+	if err != nil {
+		return err
+	}
+
+	for _, containerDescription := range containerDescriptions {
+		serviceName := containerDescription.Config.Labels.ComposeService
+		if strings.HasSuffix(serviceName, readyServicesSuffix) {
+			continue
+		}
+		err := serviceFunc(containerDescription)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}