From 94633ccd16737cbc4921a3518aa543c110220fd4 Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Sat, 31 Aug 2024 20:28:37 +0100 Subject: [PATCH] :sparkles: add metrics check and further helpers for the e2e tests Provide further improvements for e2e tests test to help users be aware of how to tests using the metrics endpoint and validate if the metrics are properly expose. --- .github/workflows/test-e2e-samples.yml | 2 + .../project/test/e2e/e2e_suite_test.go | 3 + .../testdata/project/test/e2e/e2e_test.go | 189 +++++++++++++++- .../testdata/project/test/utils/utils.go | 51 +++++ .../project/test/e2e/e2e_suite_test.go | 3 + .../testdata/project/test/e2e/e2e_test.go | 189 +++++++++++++++- .../testdata/project/test/utils/utils.go | 51 +++++ .../internal/templates/test/e2e/suite.go | 3 + .../internal/templates/test/e2e/test.go | 195 ++++++++++++++-- .../internal/templates/test/utils/utils.go | 52 +++++ .../example.com/busybox_controller.go | 15 +- .../example.com/memcached_controller.go | 15 +- .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 189 +++++++++++++++- .../test/utils/utils.go | 51 +++++ .../project-v4-with-deploy-image/Makefile | 212 ------------------ testdata/project-v4-with-grafana/Makefile | 212 ------------------ .../test/e2e/e2e_suite_test.go | 3 + .../test/e2e/e2e_test.go | 189 +++++++++++++++- .../test/utils/utils.go | 51 +++++ .../project-v4/test/e2e/e2e_suite_test.go | 3 + testdata/project-v4/test/e2e/e2e_test.go | 189 +++++++++++++++- testdata/project-v4/test/utils/utils.go | 51 +++++ 23 files changed, 1434 insertions(+), 487 deletions(-) delete mode 100644 testdata/project-v4-with-deploy-image/Makefile delete mode 100644 testdata/project-v4-with-grafana/Makefile diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index 8137407e814..24fa09c74a1 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -38,6 +38,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '55,151s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ @@ -58,6 +59,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-with-plugins/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go index 141ebe9d5f4..7a1062cb6ae 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go index ba8ca4eadae..b524c318b0f 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "tutorial.kubebuilder.io/project/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go index 2f582ed11f9..6a037f724ff 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go index 030110d2d50..64534e4590d 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "example.com/memcached/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,7 +126,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/getting-started/testdata/project/test/utils/utils.go b/docs/book/src/getting-started/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/getting-started/testdata/project/test/utils/utils.go +++ b/docs/book/src/getting-started/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go index b34f9d855eb..3060008b005 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go @@ -83,6 +83,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go index 11f2873a172..31e7e2b5daf 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go @@ -150,24 +150,33 @@ const validatingWebhookChecksFragment = `It("should validate that the validating var TestTemplate = `{{ .Boilerplate }} - package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "{{ .Repo }}/test/utils" ) +// namespace where the project is deployed in const namespace = "{{ .ProjectName }}-system" - -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "{{ .ProjectName }}-controller-manager" +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "{{ .ProjectName }}-controller-manager-metrics-service" +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "{{ .ProjectName }}-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -190,8 +199,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -203,11 +216,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -247,8 +259,163 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole={{ .ProjectName}}-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(` + "`controller_runtime_reconcile_total{controller=\"%s\",result=\"success\"} 1`" + `, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = ` + "`" + `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + "`" + ` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string ` + "`json:\"token\"`" + ` + } ` + "`json:\"status\"`" + ` +} ` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go index f34d5cfbbc7..fd68964be6f 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go @@ -42,6 +42,8 @@ var utilsTemplate = `{{ .Boilerplate }} package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -223,4 +225,54 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} ` diff --git a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go index 0d259d3241c..1df858c2c02 100644 --- a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go +++ b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go @@ -421,11 +421,22 @@ func imageForBusybox() (string, error) { } // SetupWithManager sets up the controller with the Manager. -// Note that the Deployment will be also watched in order to ensure its -// desirable state on the cluster +// The whole idea is to be watching the resources that matter for the controller. +// When a resource that the controller is interested in changes, the Watch triggers +// the controller’s reconciliation loop, ensuring that the actual state of the resource +// matches the desired state as defined in the controller’s logic. +// +// Notice how we configured the Manager to monitor events such as the creation, update, +// or deletion of a Custom Resource (CR) of the Busybox kind, as well as any changes +// to the Deployment that the controller manages and owns. func (r *BusyboxReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + // Watch the Busybox CR(s) and trigger reconciliation whenever it + // is created, updated, or deleted For(&examplecomv1alpha1.Busybox{}). + // Watch the Deployment managed by the BusyboxReconciler. If any changes occur to the Deployment + // owned and managed by this controller, it will trigger reconciliation, ensuring that the cluster + // state aligns with the desired state. See that the ownerRef was set when the Deployment was created. Owns(&appsv1.Deployment{}). Complete(r) } diff --git a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go index 0202b73892f..a1a24ac0507 100644 --- a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go +++ b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go @@ -427,11 +427,22 @@ func imageForMemcached() (string, error) { } // SetupWithManager sets up the controller with the Manager. -// Note that the Deployment will be also watched in order to ensure its -// desirable state on the cluster +// The whole idea is to be watching the resources that matter for the controller. +// When a resource that the controller is interested in changes, the Watch triggers +// the controller’s reconciliation loop, ensuring that the actual state of the resource +// matches the desired state as defined in the controller’s logic. +// +// Notice how we configured the Manager to monitor events such as the creation, update, +// or deletion of a Custom Resource (CR) of the Memcached kind, as well as any changes +// to the Deployment that the controller manages and owns. func (r *MemcachedReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + // Watch the Memcached CR(s) and trigger reconciliation whenever it + // is created, updated, or deleted For(&examplecomv1alpha1.Memcached{}). + // Watch the Deployment managed by the MemcachedReconciler. If any changes occur to the Deployment + // owned and managed by this controller, it will trigger reconciliation, ensuring that the cluster + // state aligns with the desired state. See that the ownerRef was set when the Deployment was created. Owns(&appsv1.Deployment{}). Complete(r) } diff --git a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go index 6841341caff..f8ced72b3cc 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go index 417e347ef5a..f35db955f0b 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup-with-plugins/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-with-plugins-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-with-plugins-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-with-plugins-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-with-plugins-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-with-plugins-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go b/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go +++ b/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-deploy-image/Makefile b/testdata/project-v4-with-deploy-image/Makefile deleted file mode 100644 index 93640d0ea39..00000000000 --- a/testdata/project-v4-with-deploy-image/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -# Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'kind' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - go test ./test/e2e/ -v -ginkgo.v - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v4-with-deploy-image-builder - $(CONTAINER_TOOL) buildx use project-v4-with-deploy-image-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v4-with-deploy-image-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef diff --git a/testdata/project-v4-with-grafana/Makefile b/testdata/project-v4-with-grafana/Makefile deleted file mode 100644 index 6c48d493969..00000000000 --- a/testdata/project-v4-with-grafana/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -# Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'kind' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - go test ./test/e2e/ -v -ginkgo.v - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v4-with-grafana-builder - $(CONTAINER_TOOL) buildx use project-v4-with-grafana-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v4-with-grafana-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef diff --git a/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go index b3b83243304..d27933b1f53 100644 --- a/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-plugins/test/e2e/e2e_test.go b/testdata/project-v4-with-plugins/test/e2e/e2e_test.go index 3d9a33401e4..d1fffcacd2a 100644 --- a/testdata/project-v4-with-plugins/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-plugins/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-plugins/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-plugins-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-plugins-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-plugins-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-plugins-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -134,7 +150,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-plugins-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-plugins/test/utils/utils.go b/testdata/project-v4-with-plugins/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-plugins/test/utils/utils.go +++ b/testdata/project-v4-with-plugins/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4/test/e2e/e2e_suite_test.go b/testdata/project-v4/test/e2e/e2e_suite_test.go index 048651805bc..ac06e98c700 100644 --- a/testdata/project-v4/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4/test/e2e/e2e_test.go b/testdata/project-v4/test/e2e/e2e_test.go index cf40d485885..6191e01d1cf 100644 --- a/testdata/project-v4/test/e2e/e2e_test.go +++ b/testdata/project-v4/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4/test/utils/utils.go b/testdata/project-v4/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4/test/utils/utils.go +++ b/testdata/project-v4/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +}