diff --git a/test/extended/prometheus/alerts.go b/test/extended/prometheus/alerts.go new file mode 100644 index 000000000000..5e1c8b27cad9 --- /dev/null +++ b/test/extended/prometheus/alerts.go @@ -0,0 +1,94 @@ +package prometheus + +import ( + "time" + + g "github.com/onsi/ginkgo" + + v1 "k8s.io/api/core/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + intstr "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + clientset "k8s.io/client-go/kubernetes" + e2e "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/pod" + + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[Feature:PodDisruptionBudgetAtLimitAlert][Conformance] Prometheus", func() { + defer g.GinkgoRecover() + var ( + oc = exutil.NewCLIWithoutNamespace("prometheus") + + url, bearerToken string + ) + + g.BeforeEach(func() { + var ok bool + url, bearerToken, ok = locatePrometheus(oc) + if !ok { + e2e.Skipf("Prometheus could not be located on this cluster, skipping prometheus test") + } + }) + // This alert is managed by cluster-kube-controller-manager-operator + // https://github.com/openshift/cluster-kube-controller-manager-operator/blob/master/manifests/0000_90_kube-controller-manager-operator_05_alert-pdb.yaml + // Check for 'pending' rather than 'firing' because alert will remain pending for 15m according to the alert definition above. + g.Describe("when installed on the cluster", func() { + g.It("should have a PodDisruptionBudgetAtLimit alert in pending state if pdbMinAvailable exists and MinAvailable pods", func() { + oc.SetupProject() + ns := oc.Namespace() + labels := map[string]string{"app": "pdbtest"} + execPod := createPodOrFail(oc.AdminKubeClient(), ns, "execpod", labels) + defer func() { oc.AdminKubeClient().CoreV1().Pods(ns).Delete(execPod.Name, metav1.NewDeleteOptions(1)) }() + pdbCreateMinAvailable(oc, ns, labels) + + tests := map[string]bool{ + // should have pdb alert if pdb created and at limit + `ALERTS{alertstate="pending",alertname="PodDisruptionBudgetAtLimit",severity="warning"} == 1`: true, + } + runQueries(tests, oc, ns, execPod.Name, url, bearerToken) + + e2e.Logf("PodDisruptionBudget alert is firing") + }) + }) +}) + +func pdbCreateMinAvailable(oc *exutil.CLI, ns string, labels map[string]string) { + minAvailable := intstr.FromInt(1) + pdb := policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + MinAvailable: &minAvailable, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + }, + } + _, err := oc.AdminPolicyClient().PodDisruptionBudgets(ns).Create(&pdb) + e2e.ExpectNoError(err, "Waiting for the pdb to be created with minAvailable %d in namespace %s", minAvailable, ns) + wait.PollImmediate(10*time.Second, 4*time.Minute, func() (bool, error) { + pdb, err := oc.AdminPolicyClient().PodDisruptionBudgets(ns).Get(ns, metav1.GetOptions{}) + if err != nil { + return false, err + } + if pdb.Status.ObservedGeneration < pdb.Generation { + return false, nil + } + return true, nil + }) + e2e.ExpectNoError(err, "Waiting for the pdb in namespace %s", ns) +} + +func createPodOrFail(client clientset.Interface, ns, generateName string, labels map[string]string) *v1.Pod { + return pod.CreateExecPodOrFail(client, ns, generateName, func(pod *v1.Pod) { + pod.ObjectMeta.Labels = labels + pod.Spec.Containers[0].Image = "centos:7" + pod.Spec.Containers[0].Command = []string{"sh", "-c", "trap exit TERM; while true; do sleep 5; done"} + pod.Spec.Containers[0].Args = nil + + }) +} diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index 380ed6b9ac5b..aea0b2b1f2a5 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -2,13 +2,11 @@ package prometheus import ( "bytes" - "context" "encoding/json" "fmt" "os" "regexp" "strconv" - "strings" "time" g "github.com/onsi/ginkgo" @@ -23,18 +21,14 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - watchtools "k8s.io/client-go/tools/watch" kapi "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/client/conditions" e2e "k8s.io/kubernetes/test/e2e/framework" "github.com/openshift/origin/test/extended/networking" exutil "github.com/openshift/origin/test/extended/util" ) -const waitForPrometheusStartSeconds = 240 - var _ = g.Describe("[Feature:Prometheus][Conformance] Prometheus", func() { defer g.GinkgoRecover() var ( @@ -403,15 +397,6 @@ func expectBearerTokenURLStatusCodeExec(ns, execPodName, url, bearer string, sta return nil } -func getBearerTokenURLViaPod(ns, execPodName, url, bearer string) (string, error) { - cmd := fmt.Sprintf("curl -s -k -H 'Authorization: Bearer %s' %q", bearer, url) - output, err := e2e.RunHostCmd(ns, execPodName, cmd) - if err != nil { - return "", fmt.Errorf("host command failed: %v\n%s", err, output) - } - return output, nil -} - func getAuthenticatedURLViaPod(ns, execPodName, url, user, pass string) (string, error) { cmd := fmt.Sprintf("curl -s -u %s:%s %q", user, pass, url) output, err := e2e.RunHostCmd(ns, execPodName, cmd) @@ -430,48 +415,6 @@ func getInsecureURLViaPod(ns, execPodName, url string) (string, error) { return output, nil } -func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountName string, timeout time.Duration) error { - w, err := c.CoreV1().ServiceAccounts(ns).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: serviceAccountName})) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - _, err = watchtools.UntilWithoutRetry(ctx, w, conditions.ServiceAccountHasSecrets) - return err -} - -func locatePrometheus(oc *exutil.CLI) (url, bearerToken string, ok bool) { - _, err := oc.AdminKubeClient().CoreV1().Services("openshift-monitoring").Get("prometheus-k8s", metav1.GetOptions{}) - if kapierrs.IsNotFound(err) { - return "", "", false - } - - waitForServiceAccountInNamespace(oc.AdminKubeClient(), "openshift-monitoring", "prometheus-k8s", 2*time.Minute) - for i := 0; i < 30; i++ { - secrets, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-monitoring").List(metav1.ListOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - for _, secret := range secrets.Items { - if secret.Type != v1.SecretTypeServiceAccountToken { - continue - } - if !strings.HasPrefix(secret.Name, "prometheus-") { - continue - } - bearerToken = string(secret.Data[v1.ServiceAccountTokenKey]) - break - } - if len(bearerToken) == 0 { - e2e.Logf("Waiting for prometheus service account secret to show up") - time.Sleep(time.Second) - continue - } - } - o.Expect(bearerToken).ToNot(o.BeEmpty()) - - return "https://prometheus-k8s.openshift-monitoring.svc:9091", bearerToken, true -} - func hasPullSecret(client clientset.Interface, name string) bool { scrt, err := client.CoreV1().Secrets("openshift-config").Get("pull-secret", metav1.GetOptions{}) if err != nil { diff --git a/test/extended/prometheus/prometheus_builds.go b/test/extended/prometheus/prometheus_builds.go index 9184018b6c6c..704acb919746 100644 --- a/test/extended/prometheus/prometheus_builds.go +++ b/test/extended/prometheus/prometheus_builds.go @@ -1,14 +1,11 @@ package prometheus import ( - "encoding/json" "fmt" - "net/url" "time" g "github.com/onsi/ginkgo" o "github.com/onsi/gomega" - "github.com/prometheus/common/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" e2e "k8s.io/kubernetes/test/e2e/framework" @@ -88,52 +85,6 @@ var _ = g.Describe("[Feature:Prometheus][Feature:Builds] Prometheus", func() { }) }) -type prometheusResponse struct { - Status string `json:"status"` - Data prometheusResponseData `json:"data"` -} - -type prometheusResponseData struct { - ResultType string `json:"resultType"` - Result model.Vector `json:"result"` -} - -func runQueries(promQueries map[string]bool, oc *exutil.CLI, ns, execPodName, baseURL, bearerToken string) { - // expect all correct metrics within a reasonable time period - errsMap := map[string]error{} - for i := 0; i < waitForPrometheusStartSeconds; i++ { - for query, expected := range promQueries { - //TODO when the http/query apis discussed at https://github.com/prometheus/client_golang#client-for-the-prometheus-http-api - // and introduced at https://github.com/prometheus/client_golang/blob/master/api/prometheus/v1/api.go are vendored into - // openshift/origin, look to replace this homegrown http request / query param with that API - g.By("perform prometheus metric query " + query) - contents, err := getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("%s/api/v1/query?%s", baseURL, (url.Values{"query": []string{query}}).Encode()), bearerToken) - o.Expect(err).NotTo(o.HaveOccurred()) - result := prometheusResponse{} - json.Unmarshal([]byte(contents), &result) - metrics := result.Data.Result - - delete(errsMap, query) // clear out any prior failures - if (len(metrics) > 0 && !expected) || (len(metrics) == 0 && expected) { - dbg := fmt.Sprintf("promQL query: %s had reported incorrect results: %v", query, metrics) - fmt.Fprintf(g.GinkgoWriter, dbg) - errsMap[query] = fmt.Errorf(dbg) - } - - } - - if len(errsMap) == 0 { - break - } - time.Sleep(time.Second) - } - - if len(errsMap) != 0 { - exutil.DumpPodLogsStartingWith("prometheus-0", oc) - } - o.Expect(errsMap).To(o.BeEmpty()) -} - func startOpenShiftBuild(oc *exutil.CLI, appTemplate string) *exutil.BuildResult { g.By(fmt.Sprintf("calling oc create -f %s ", appTemplate)) err := oc.Run("create").Args("-f", appTemplate).Execute() diff --git a/test/extended/prometheus/util.go b/test/extended/prometheus/util.go new file mode 100644 index 000000000000..8e78b1ad3a64 --- /dev/null +++ b/test/extended/prometheus/util.go @@ -0,0 +1,123 @@ +package prometheus + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + "github.com/prometheus/common/model" + + v1 "k8s.io/api/core/v1" + kapierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + watchtools "k8s.io/client-go/tools/watch" + "k8s.io/kubernetes/pkg/client/conditions" + e2e "k8s.io/kubernetes/test/e2e/framework" + + exutil "github.com/openshift/origin/test/extended/util" +) + +const waitForPrometheusStartSeconds = 240 + +type prometheusResponse struct { + Status string `json:"status"` + Data prometheusResponseData `json:"data"` +} + +type prometheusResponseData struct { + ResultType string `json:"resultType"` + Result model.Vector `json:"result"` +} + +func getBearerTokenURLViaPod(ns, execPodName, url, bearer string) (string, error) { + cmd := fmt.Sprintf("curl -s -k -H 'Authorization: Bearer %s' %q", bearer, url) + output, err := e2e.RunHostCmd(ns, execPodName, cmd) + if err != nil { + return "", fmt.Errorf("host command failed: %v\n%s", err, output) + } + return output, nil +} + +func runQueries(promQueries map[string]bool, oc *exutil.CLI, ns, execPodName, baseURL, bearerToken string) { + // expect all correct metrics within a reasonable time period + errsMap := map[string]error{} + for i := 0; i < waitForPrometheusStartSeconds; i++ { + for query, expected := range promQueries { + //TODO when the http/query apis discussed at https://github.com/prometheus/client_golang#client-for-the-prometheus-http-api + // and introduced at https://github.com/prometheus/client_golang/blob/master/api/prometheus/v1/api.go are vendored into + // openshift/origin, look to replace this homegrown http request / query param with that API + g.By("perform prometheus metric query " + query) + contents, err := getBearerTokenURLViaPod(ns, execPodName, fmt.Sprintf("%s/api/v1/query?%s", baseURL, (url.Values{"query": []string{query}}).Encode()), bearerToken) + o.Expect(err).NotTo(o.HaveOccurred()) + result := prometheusResponse{} + json.Unmarshal([]byte(contents), &result) + metrics := result.Data.Result + + delete(errsMap, query) // clear out any prior failures + if (len(metrics) > 0 && !expected) || (len(metrics) == 0 && expected) { + dbg := fmt.Sprintf("promQL query: %s had reported incorrect results: %v", query, metrics) + fmt.Fprintf(g.GinkgoWriter, dbg) + errsMap[query] = fmt.Errorf(dbg) + } + + } + + if len(errsMap) == 0 { + break + } + time.Sleep(time.Second) + } + + if len(errsMap) != 0 { + exutil.DumpPodLogsStartingWith("prometheus-0", oc) + } + o.Expect(errsMap).To(o.BeEmpty()) +} + +func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountName string, timeout time.Duration) error { + w, err := c.CoreV1().ServiceAccounts(ns).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: serviceAccountName})) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + _, err = watchtools.UntilWithoutRetry(ctx, w, conditions.ServiceAccountHasSecrets) + return err +} + +func locatePrometheus(oc *exutil.CLI) (url, bearerToken string, ok bool) { + _, err := oc.AdminKubeClient().CoreV1().Services("openshift-monitoring").Get("prometheus-k8s", metav1.GetOptions{}) + if kapierrs.IsNotFound(err) { + return "", "", false + } + + waitForServiceAccountInNamespace(oc.AdminKubeClient(), "openshift-monitoring", "prometheus-k8s", 2*time.Minute) + for i := 0; i < 30; i++ { + secrets, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-monitoring").List(metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, secret := range secrets.Items { + if secret.Type != v1.SecretTypeServiceAccountToken { + continue + } + if !strings.HasPrefix(secret.Name, "prometheus-") { + continue + } + bearerToken = string(secret.Data[v1.ServiceAccountTokenKey]) + break + } + if len(bearerToken) == 0 { + e2e.Logf("Waiting for prometheus service account secret to show up") + time.Sleep(time.Second) + continue + } + } + o.Expect(bearerToken).ToNot(o.BeEmpty()) + + return "https://prometheus-k8s.openshift-monitoring.svc:9091", bearerToken, true +} diff --git a/test/extended/util/client.go b/test/extended/util/client.go index 83d33d7dffb8..74b013aacc51 100644 --- a/test/extended/util/client.go +++ b/test/extended/util/client.go @@ -37,6 +37,7 @@ import ( "k8s.io/apiserver/pkg/storage/names" memory "k8s.io/client-go/discovery/cached" "k8s.io/client-go/kubernetes" + policyv1beta1client "k8s.io/client-go/kubernetes/typed/policy/v1beta1" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/cache" @@ -391,6 +392,10 @@ func (c *CLI) AdminQuotaClient() quotav1client.Interface { return quotav1client.NewForConfigOrDie(c.AdminConfig()) } +func (c *CLI) AdminPolicyClient() policyv1beta1client.PolicyV1beta1Interface { + return policyv1beta1client.NewForConfigOrDie(c.AdminConfig()) +} + func (c *CLI) AdminOAuthClient() oauthv1client.Interface { return oauthv1client.NewForConfigOrDie(c.AdminConfig()) }