diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml index 8137407e814..24fa09c74a1 100644 --- a/.github/workflows/test-e2e-samples.yml +++ b/.github/workflows/test-e2e-samples.yml @@ -38,6 +38,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '55,151s/^#//' $KUSTOMIZATION_FILE_PATH cd testdata/project-v4/ @@ -58,6 +59,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4-with-plugins/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH # Uncomment only ValidatingWebhookConfiguration # from cert-manager replaces diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go index 141ebe9d5f4..7a1062cb6ae 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go index ba8ca4eadae..b524c318b0f 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "tutorial.kubebuilder.io/project/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go index 2f582ed11f9..6a037f724ff 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go index 030110d2d50..64534e4590d 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "example.com/memcached/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -110,7 +126,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/getting-started/testdata/project/test/utils/utils.go b/docs/book/src/getting-started/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/getting-started/testdata/project/test/utils/utils.go +++ b/docs/book/src/getting-started/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go index b34f9d855eb..3060008b005 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go @@ -83,6 +83,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go index 11f2873a172..31e7e2b5daf 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go @@ -150,24 +150,33 @@ const validatingWebhookChecksFragment = `It("should validate that the validating var TestTemplate = `{{ .Boilerplate }} - package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "{{ .Repo }}/test/utils" ) +// namespace where the project is deployed in const namespace = "{{ .ProjectName }}-system" - -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "{{ .ProjectName }}-controller-manager" +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "{{ .ProjectName }}-controller-manager-metrics-service" +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "{{ .ProjectName }}-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -190,8 +199,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -203,11 +216,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -247,8 +259,163 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole={{ .ProjectName}}-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(` + "`controller_runtime_reconcile_total{controller=\"%s\",result=\"success\"} 1`" + `, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = ` + "`" + `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + "`" + ` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string ` + "`json:\"token\"`" + ` + } ` + "`json:\"status\"`" + ` +} ` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go index f34d5cfbbc7..fd68964be6f 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go @@ -42,6 +42,8 @@ var utilsTemplate = `{{ .Boilerplate }} package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -223,4 +225,54 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} ` diff --git a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go index 0d259d3241c..1df858c2c02 100644 --- a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go +++ b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/busybox_controller.go @@ -421,11 +421,22 @@ func imageForBusybox() (string, error) { } // SetupWithManager sets up the controller with the Manager. -// Note that the Deployment will be also watched in order to ensure its -// desirable state on the cluster +// The whole idea is to be watching the resources that matter for the controller. +// When a resource that the controller is interested in changes, the Watch triggers +// the controller’s reconciliation loop, ensuring that the actual state of the resource +// matches the desired state as defined in the controller’s logic. +// +// Notice how we configured the Manager to monitor events such as the creation, update, +// or deletion of a Custom Resource (CR) of the Busybox kind, as well as any changes +// to the Deployment that the controller manages and owns. func (r *BusyboxReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + // Watch the Busybox CR(s) and trigger reconciliation whenever it + // is created, updated, or deleted For(&examplecomv1alpha1.Busybox{}). + // Watch the Deployment managed by the BusyboxReconciler. If any changes occur to the Deployment + // owned and managed by this controller, it will trigger reconciliation, ensuring that the cluster + // state aligns with the desired state. See that the ownerRef was set when the Deployment was created. Owns(&appsv1.Deployment{}). Complete(r) } diff --git a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go index 0202b73892f..a1a24ac0507 100644 --- a/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go +++ b/testdata/project-v4-multigroup-with-plugins/internal/controller/example.com/memcached_controller.go @@ -427,11 +427,22 @@ func imageForMemcached() (string, error) { } // SetupWithManager sets up the controller with the Manager. -// Note that the Deployment will be also watched in order to ensure its -// desirable state on the cluster +// The whole idea is to be watching the resources that matter for the controller. +// When a resource that the controller is interested in changes, the Watch triggers +// the controller’s reconciliation loop, ensuring that the actual state of the resource +// matches the desired state as defined in the controller’s logic. +// +// Notice how we configured the Manager to monitor events such as the creation, update, +// or deletion of a Custom Resource (CR) of the Memcached kind, as well as any changes +// to the Deployment that the controller manages and owns. func (r *MemcachedReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + // Watch the Memcached CR(s) and trigger reconciliation whenever it + // is created, updated, or deleted For(&examplecomv1alpha1.Memcached{}). + // Watch the Deployment managed by the MemcachedReconciler. If any changes occur to the Deployment + // owned and managed by this controller, it will trigger reconciliation, ensuring that the cluster + // state aligns with the desired state. See that the ownerRef was set when the Deployment was created. Owns(&appsv1.Deployment{}). Complete(r) } diff --git a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go index 6841341caff..f8ced72b3cc 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go index 417e347ef5a..f35db955f0b 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup-with-plugins/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup-with-plugins/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-with-plugins-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-with-plugins-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-with-plugins-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-with-plugins-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-with-plugins-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go b/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go +++ b/testdata/project-v4-multigroup-with-plugins/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-deploy-image/Makefile b/testdata/project-v4-with-deploy-image/Makefile deleted file mode 100644 index 93640d0ea39..00000000000 --- a/testdata/project-v4-with-deploy-image/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -# Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'kind' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - go test ./test/e2e/ -v -ginkgo.v - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v4-with-deploy-image-builder - $(CONTAINER_TOOL) buildx use project-v4-with-deploy-image-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v4-with-deploy-image-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef diff --git a/testdata/project-v4-with-grafana/Makefile b/testdata/project-v4-with-grafana/Makefile deleted file mode 100644 index 6c48d493969..00000000000 --- a/testdata/project-v4-with-grafana/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -# Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'kind' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - go test ./test/e2e/ -v -ginkgo.v - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v4-with-grafana-builder - $(CONTAINER_TOOL) buildx use project-v4-with-grafana-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v4-with-grafana-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef diff --git a/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go index b3b83243304..d27933b1f53 100644 --- a/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-plugins/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-plugins/test/e2e/e2e_test.go b/testdata/project-v4-with-plugins/test/e2e/e2e_test.go index 3d9a33401e4..d1fffcacd2a 100644 --- a/testdata/project-v4-with-plugins/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-plugins/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-plugins/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-plugins-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-plugins-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-plugins-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-plugins-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -134,7 +150,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-plugins-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-plugins/test/utils/utils.go b/testdata/project-v4-with-plugins/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-plugins/test/utils/utils.go +++ b/testdata/project-v4-with-plugins/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4/test/e2e/e2e_suite_test.go b/testdata/project-v4/test/e2e/e2e_suite_test.go index 048651805bc..ac06e98c700 100644 --- a/testdata/project-v4/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enable") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4/test/e2e/e2e_test.go b/testdata/project-v4/test/e2e/e2e_test.go index cf40d485885..6191e01d1cf 100644 --- a/testdata/project-v4/test/e2e/e2e_test.go +++ b/testdata/project-v4/test/e2e/e2e_test.go @@ -17,8 +17,12 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -27,10 +31,19 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { @@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() { // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string It("should run successfully", func() { - var controllerPodName string - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get the name of the controller-manager pod @@ -149,7 +165,162 @@ var _ = Describe("controller", Ordered, func() { // +kubebuilder:scaffold:e2e-webhooks-checks - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func() error { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(output), "8443") { + return fmt.Errorf("metrics endpoint is not ready") + } + return nil + } + EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + Eventually(func() error { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + if err != nil { + return err + } + if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") { + return fmt.Errorf("metrics server not yet started") + } + return nil + }, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func() error { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + status, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if string(status) != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // ExpectWithOffset(1, metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return "", err + } + + var out string + var rawJson string + Eventually(func() error { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4/test/utils/utils.go b/testdata/project-v4/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4/test/utils/utils.go +++ b/testdata/project-v4/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +}