From f867ac0417f00db26bbb4829b4cab5aaad06075a Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Sat, 31 Aug 2024 20:28:37 +0100 Subject: [PATCH] :sparkles: add metrics check and further helpers for the e2e tests Provide further improvements for e2e tests test to help users be aware of how to tests using the metrics endpoint and validate if the metrics are properly expose. --- .../project/test/e2e/e2e_suite_test.go | 3 + .../testdata/project/test/e2e/e2e_test.go | 188 ++++++++++++++++- .../testdata/project/test/utils/utils.go | 51 +++++ .../project/test/e2e/e2e_suite_test.go | 3 + .../testdata/project/test/e2e/e2e_test.go | 188 ++++++++++++++++- .../testdata/project/test/utils/utils.go | 51 +++++ .../internal/templates/test/e2e/suite.go | 3 + .../internal/templates/test/e2e/test.go | 192 +++++++++++++++++- .../internal/templates/test/utils/utils.go | 52 +++++ .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 188 ++++++++++++++++- .../test/utils/utils.go | 51 +++++ .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 188 ++++++++++++++++- .../project-v4-multigroup/test/utils/utils.go | 51 +++++ .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 188 ++++++++++++++++- .../test/utils/utils.go | 51 +++++ .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 188 ++++++++++++++++- .../test/utils/utils.go | 51 +++++ .../project-v4/test/e2e/e2e_suite_test.go | 3 + testdata/project-v4/test/e2e/e2e_test.go | 188 ++++++++++++++++- testdata/project-v4/test/utils/utils.go | 51 +++++ 24 files changed, 1881 insertions(+), 60 deletions(-) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go index 141ebe9d5f4..265d1829d2c 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go index 2812ef5b973..1e0c327f192 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "tutorial.kubebuilder.io/project/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go index 2f582ed11f9..5e3806b98ee 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go index 66f7dcef8fd..6a263bbda02 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "example.com/memcached/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/getting-started/testdata/project/test/utils/utils.go b/docs/book/src/getting-started/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/getting-started/testdata/project/test/utils/utils.go +++ b/docs/book/src/getting-started/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go index b34f9d855eb..5334efdfbbe 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go @@ -83,6 +83,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go index 6288f9d4cfd..b7ad89f64a1 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go @@ -40,24 +40,33 @@ func (f *Test) SetTemplateDefaults() error { var TestTemplate = `{{ .Boilerplate }} - package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "{{ .Repo }}/test/utils" ) +// namespace where the project is deployed in const namespace = "{{ .ProjectName }}-system" +// serviceAccountName created for the project +const serviceAccountName = "{{ .ProjectName }}-controller-manager" +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "{{ .ProjectName }}-controller-manager-metrics-service" +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "{{ .ProjectName }}-metrics-binding" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -80,8 +89,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -93,11 +106,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -137,6 +149,164 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole={{ .ProjectName}}-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(` + "`controller_runtime_reconcile_total{controller=\"%s\",result=\"success\"} 1`" + `, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = ` + "`" + `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + "`" + ` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string ` + "`json:\"token\"`" + ` + } ` + "`json:\"status\"`" + ` +} ` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go index f34d5cfbbc7..fd68964be6f 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go @@ -42,6 +42,8 @@ var utilsTemplate = `{{ .Boilerplate }} package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -223,4 +225,54 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} ` diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go index ae4e19086c6..2404cc6ea74 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go index fe3a6e4d75d..0d692a856f3 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup-with-deploy-image/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-with-deploy-image-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-with-deploy-image-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-with-deploy-image-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-with-deploy-image-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-with-deploy-image-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go b/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go index b68ec10b389..be68fdc2df9 100644 --- a/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup/test/e2e/e2e_test.go b/testdata/project-v4-multigroup/test/e2e/e2e_test.go index 580c8745b03..9d2b4147e8c 100644 --- a/testdata/project-v4-multigroup/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup/test/utils/utils.go b/testdata/project-v4-multigroup/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup/test/utils/utils.go +++ b/testdata/project-v4-multigroup/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go index 68830827b0b..c2020988060 100644 --- a/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go b/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go index 144c8c76d1d..775ca731cfa 100644 --- a/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-deploy-image/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-deploy-image-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-deploy-image-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-deploy-image-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-deploy-image-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-deploy-image-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-deploy-image/test/utils/utils.go b/testdata/project-v4-with-deploy-image/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-deploy-image/test/utils/utils.go +++ b/testdata/project-v4-with-deploy-image/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go index f006e422db7..24357e32ee0 100644 --- a/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-grafana/test/e2e/e2e_test.go b/testdata/project-v4-with-grafana/test/e2e/e2e_test.go index a26e626abbe..aee12265402 100644 --- a/testdata/project-v4-with-grafana/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-grafana/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-grafana/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-grafana-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-grafana-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-grafana-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-grafana-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-grafana-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-grafana/test/utils/utils.go b/testdata/project-v4-with-grafana/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-grafana/test/utils/utils.go +++ b/testdata/project-v4-with-grafana/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4/test/e2e/e2e_suite_test.go b/testdata/project-v4/test/e2e/e2e_suite_test.go index 048651805bc..59afad48a06 100644 --- a/testdata/project-v4/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4/test/e2e/e2e_test.go b/testdata/project-v4/test/e2e/e2e_test.go index 690c711c470..63343796722 100644 --- a/testdata/project-v4/test/e2e/e2e_test.go +++ b/testdata/project-v4/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() { // TODO(user): Customize the e2e test suite to include // additional scenarios specific to your project. + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4/test/utils/utils.go b/testdata/project-v4/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4/test/utils/utils.go +++ b/testdata/project-v4/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +}