Configure metrics collection for operator

go.mod

@@ -10,8 +10,9 @@ require (
 	github.com/jteeuwen/go-bindata v3.0.8-0.20151023091102-a0ff2567cfb7+incompatible
 	github.com/openshift/api v3.9.1-0.20191212095247-c1898f32de35+incompatible
 	github.com/openshift/client-go v0.0.0-20191216194936-57f413491e9e
-	github.com/openshift/library-go v0.0.0-20200106191802-9821002633e8
+	github.com/openshift/library-go v0.0.0-20200113183004-f2ca9aafdf5a
 	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/common v0.6.0
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect

go.sum

@@ -290,8 +290,8 @@ github.com/openshift/api v3.9.1-0.20191212095247-c1898f32de35+incompatible h1:LU
 github.com/openshift/api v3.9.1-0.20191212095247-c1898f32de35+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
 github.com/openshift/client-go v0.0.0-20191216194936-57f413491e9e h1:l+fwEFa4Voy9u+6pVOJI33sxwddz1oJ6rbVYpDfGytY=
 github.com/openshift/client-go v0.0.0-20191216194936-57f413491e9e/go.mod h1:nLJaHFCQ5Mavh98g2ejEnWYFWBMGVdphrKNjLErOn/w=
-github.com/openshift/library-go v0.0.0-20200106191802-9821002633e8 h1:gjmVJ0XiETkdUbVm4Fu5Hg7S9mlXcEuG6DztuQ4KiYM=
-github.com/openshift/library-go v0.0.0-20200106191802-9821002633e8/go.mod h1:+EzNb8oA3fnhC613pNcAU0DJ9s3m6WaIMECIVQm2ork=
+github.com/openshift/library-go v0.0.0-20200113183004-f2ca9aafdf5a h1:Sq7Kv7VeWApt2WZVQMyIE64NHqaQcM/GkWDPZZnfhP0=
+github.com/openshift/library-go v0.0.0-20200113183004-f2ca9aafdf5a/go.mod h1:+EzNb8oA3fnhC613pNcAU0DJ9s3m6WaIMECIVQm2ork=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

manifests/0000_90_service-ca-operator_01_prometheusrole.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-service-ca-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch

manifests/0000_90_service-ca-operator_02_prometheusrolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-service-ca-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

manifests/0000_90_service-ca-operator_03_servicemonitor.yaml

@@ -0,0 +1,21 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: service-ca-operator
+  namespace: openshift-service-ca-operator
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    port: https
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: metrics.openshift-service-ca-operator.svc
+  jobLabel: component
+  namespaceSelector:
+    matchNames:
+    - openshift-service-ca-operator
+  selector:
+    matchLabels:
+      app: service-ca-operator

manifests/01_namespace.yaml

@@ -3,6 +3,7 @@ kind: Namespace
 metadata:
   labels:
     openshift.io/run-level: "1"
+    openshift.io/cluster-monitoring: "true"
   name: openshift-service-ca-operator
   annotations:
     openshift.io/node-selector: ""

manifests/02_service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: serving-cert
+  labels:
+    app: service-ca-operator
+  name: metrics
+  namespace: openshift-service-ca-operator
+spec:
+  ports:
+  - name: https
+    port: 443
+    protocol: TCP
+    targetPort: 8443
+  selector:
+    app: service-ca-operator
+  sessionAffinity: None
+  type: ClusterIP

manifests/05_deploy.yaml

@@ -37,6 +37,8 @@ spec:
         volumeMounts:
         - mountPath: /var/run/configmaps/config
           name: config
+        - mountPath: /var/run/secrets/serving-cert
+          name: serving-cert
       volumes:
       - name: serving-cert
         secret:

test/e2e/e2e_test.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"bytes"
+	"context"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -11,6 +12,9 @@ import (
 	"testing"
 	"time"
 
+	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -22,9 +26,11 @@ import (
 	"k8s.io/client-go/util/cert"
 
 	operatorv1client "github.com/openshift/client-go/operator/clientset/versioned"
+	routeclient "github.com/openshift/client-go/route/clientset/versioned"
 	"github.com/openshift/library-go/pkg/crypto"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
+	"github.com/openshift/library-go/test/library/metrics"
 	"github.com/openshift/service-ca-operator/pkg/controller/api"
 	"github.com/openshift/service-ca-operator/pkg/operator"
 	"github.com/openshift/service-ca-operator/pkg/operator/operatorclient"
@@ -585,6 +591,57 @@ func pollForResource(t *testing.T, resourceID string, timeout time.Duration, acc
 	return obj, err
 }
 
+// newPrometheusClientForConfig returns a new prometheus client for
+// the provided kubeconfig.
+func newPrometheusClientForConfig(config *rest.Config) (prometheusv1.API, error) {
+	routeClient, err := routeclient.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating route client: %v", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating kube client: %v", err)
+	}
+	return metrics.NewPrometheusClient(kubeClient, routeClient)
+}
+
+// checkMetricsCollection tests whether metrics are being successfully scraped from at
+// least one target in a namespace.
+func checkMetricsCollection(t *testing.T, promClient prometheusv1.API, namespace string) {
+	// Metrics are scraped every 30s. Wait as long as 2 intervals to avoid failing if
+	// the target is temporarily unhealthy.
+	timeout := 60 * time.Second
+
+	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		query := fmt.Sprintf("up{namespace=\"%s\"}", namespace)
+		rawResult, warnings, err := promClient.Query(context.Background(), query, time.Now())
+		if err != nil {
+			t.Errorf("failed to execute prometheus query: %v", err)
+			return false, nil
+		}
+		if len(warnings) > 0 {
+			t.Logf("prometheus query emitted warnings: %v", warnings)
+		}
+
+		resultVector, ok := rawResult.(model.Vector)
+		if !ok {
+			t.Fatalf("expected prometheus query to return type Vector, got %T", resultVector)
+		}
+		metricsCollected := false
+		for _, sample := range resultVector {
+			metricsCollected = sample.Value == 1
+			if metricsCollected {
+				// Metrics are successfully being scraped for at least one target in the namespace
+				break
+			}
+		}
+		return metricsCollected, nil
+	})
+	if err != nil {
+		t.Fatalf("Health check of metrics collection in namespace %s did not succeed within %v", serviceCAOperatorNamespace, timeout)
+	}
+}
+
 func TestE2E(t *testing.T) {
 	// use /tmp/admin.conf (placed by ci-operator) or KUBECONFIG env
 	confPath := "/tmp/admin.conf"
@@ -836,6 +893,15 @@ func TestE2E(t *testing.T) {
 		}
 	})
 
+	// Test that the operator's metrics endpoint is being read by prometheus
+	t.Run("metrics", func(t *testing.T) {
+		promClient, err := newPrometheusClientForConfig(adminConfig)
+		if err != nil {
+			t.Fatalf("error initializing prometheus client: %v", err)
+		}
+		checkMetricsCollection(t, promClient, "openshift-service-ca-operator")
+	})
+
 	// This test triggers rotation by updating the CA to have an
 	// expiry that is less than the minimum required duration and then
 	// validates that both refreshed and unrefreshed clients and