Extend client APIs for support of existing stacks in more cases #2338
Remove code copied from serverless
- Loading branch information
commit 4e1cceaae0f0fdbcb65b6c714aabf119c060de7a
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package stack | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/elastic/elastic-package/internal/kibana" | ||
| "github.com/elastic/elastic-package/internal/logger" | ||
| "github.com/elastic/elastic-package/internal/profile" | ||
| "github.com/elastic/elastic-package/internal/registry" | ||
| ) | ||
|
|
||
| const ( | ||
| managedAgentPolicyID = "elastic-agent-managed-ep" | ||
| fleetLogstashOutput = "fleet-logstash-output" | ||
| fleetElasticsearchOutput = "fleet-elasticsearch-output" | ||
| ) | ||
|
|
||
| // createAgentPolicy creates an agent policy with the initial configuration used for | ||
| // agents managed by elastic-package. | ||
| func createAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, stackVersion string, outputId string, selfMonitor bool) (*kibana.Policy, error) { | ||
| policy := kibana.Policy{ | ||
| ID: managedAgentPolicyID, | ||
| Name: "Elastic-Agent (elastic-package)", | ||
| Description: "Policy created by elastic-package", | ||
| Namespace: "default", | ||
| MonitoringEnabled: []string{}, | ||
| DataOutputID: outputId, | ||
| } | ||
| if selfMonitor { | ||
| policy.MonitoringEnabled = []string{"logs", "metrics"} | ||
| } | ||
|
|
||
| newPolicy, err := kibanaClient.CreatePolicy(ctx, policy) | ||
| if errors.Is(err, kibana.ErrConflict) { | ||
| newPolicy, err = kibanaClient.GetPolicy(ctx, policy.ID) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error while getting existing policy: %w", err) | ||
| } | ||
| return newPolicy, nil | ||
| } | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error while creating agent policy: %w", err) | ||
| } | ||
|
|
||
| if selfMonitor { | ||
| err := createSystemPackagePolicy(ctx, kibanaClient, stackVersion, newPolicy.ID, newPolicy.Namespace) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| return newPolicy, nil | ||
| } | ||
|
|
||
| func createSystemPackagePolicy(ctx context.Context, kibanaClient *kibana.Client, stackVersion, agentPolicyID, namespace string) error { | ||
| systemPackages, err := registry.Production.Revisions("system", registry.SearchOptions{ | ||
| KibanaVersion: strings.TrimSuffix(stackVersion, kibana.SNAPSHOT_SUFFIX), | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("could not get the system package version for Kibana %v: %w", stackVersion, err) | ||
| } | ||
| if len(systemPackages) != 1 { | ||
| return fmt.Errorf("unexpected number of system package versions for Kibana %s - found %d expected 1", stackVersion, len(systemPackages)) | ||
| } | ||
| logger.Debugf("Found %s package - version %s", systemPackages[0].Name, systemPackages[0].Version) | ||
| packagePolicy := kibana.PackagePolicy{ | ||
| Name: "system-1", | ||
| PolicyID: agentPolicyID, | ||
| Namespace: namespace, | ||
| } | ||
| packagePolicy.Package.Name = "system" | ||
| packagePolicy.Package.Version = systemPackages[0].Version | ||
|
|
||
| _, err = kibanaClient.CreatePackagePolicy(ctx, packagePolicy) | ||
| if err != nil { | ||
| return fmt.Errorf("error while creating package policy: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func addFleetOutput(ctx context.Context, client *kibana.Client, outputType, host, id string) error { | ||
| output := kibana.FleetOutput{ | ||
| Name: id, | ||
| ID: id, | ||
|
Comment on lines
+92
to
+93
There was a problem hiding this comment. Could this fail if tests are run in parallel? Or maybe this will not fail but it will use an unexpected output. Or maybe it is good to reuse the same output when running in parallel. Or well, the caller is the one to take into account assigning the right/expected IDs. There was a problem hiding this comment. Yes, this can be an issue if we use the same stack for different environments, for example if a serverless cluster with two environments, one with logstash and the other one without it. But it should not be an issue though while running tests in parallel with a single environment. For reusing a stack for different executions we have more conflicts. I would leave fixing them for future improvements. One issue we have is that we are relying on default Fleet configurations (output, fleet server, fleet server policy...), and I think we could avoid it. I think that ideally each environment should use its own resources, and they should be explicitly set in agent policies, and don't rely on defaults. |
||
| Type: outputType, | ||
| Hosts: []string{host}, | ||
| } | ||
|
|
||
| err := client.AddFleetOutput(ctx, output) | ||
| if errors.Is(err, kibana.ErrConflict) { | ||
| // Output already exists. | ||
| return nil | ||
| } | ||
| if err != nil { | ||
| return fmt.Errorf("failed to add %s fleet output of type %s: %w", id, outputType, err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func addLogstashFleetOutput(ctx context.Context, client *kibana.Client) error { | ||
| return addFleetOutput(ctx, client, "logstash", "logstash:5044", fleetLogstashOutput) | ||
| } | ||
|
|
||
| func updateLogstashFleetOutput(ctx context.Context, profile *profile.Profile, kibanaClient *kibana.Client) error { | ||
| certsDir := filepath.Join(profile.ProfilePath, "certs", "elastic-agent") | ||
|
|
||
| caFile, err := os.ReadFile(filepath.Join(certsDir, "ca-cert.pem")) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read ca certificate: %w", err) | ||
| } | ||
|
|
||
| certFile, err := os.ReadFile(filepath.Join(certsDir, "cert.pem")) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read client certificate: %w", err) | ||
| } | ||
|
|
||
| keyFile, err := os.ReadFile(filepath.Join(certsDir, "key.pem")) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read client certificate private key: %w", err) | ||
| } | ||
|
|
||
| logstashFleetOutput := kibana.FleetOutput{ | ||
| SSL: &kibana.AgentSSL{ | ||
| CertificateAuthorities: []string{string(caFile)}, | ||
| Certificate: string(certFile), | ||
| Key: string(keyFile), | ||
| }, | ||
| } | ||
|
|
||
| if err := kibanaClient.UpdateFleetOutput(ctx, logstashFleetOutput, fleetLogstashOutput); err != nil { | ||
| return fmt.Errorf("failed to update logstash fleet output: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.