Add webhook total and in-flight metrics

Add a gauge for requests currently being handled and a counter for the total amount of requests processed.

Used [prometheus/promhttp](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp?tab=doc) package which provides nice decorators for `Http.Handler` to handle the metrics instead of the custom code.

pkg/webhook/internal/metrics/metrics.go

@@ -30,11 +30,32 @@ var (
 			Name: "controller_runtime_webhook_latency_seconds",
 			Help: "Histogram of the latency of processing admission requests",
 		},
-		[]string{"webhook"},
+		[]string{"webhook", "code"},
 	)
+
+	// RequestTotal is a prometheus metric which is a counter of the total processed admission requests.
+	RequestTotal = func() *prometheus.CounterVec {
+		return prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "controller_runtime_webhook_requests_total",
+				Help: "Total number of admission requests by HTTP status code.",
+			},
+			[]string{"webhook", "code"},
+		)
+	}()
+
+	// RequestInFlight is a prometheus metric which is a gauge of the in-flight admission requests.
+	RequestInFlight = func() *prometheus.GaugeVec {
+		return prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "controller_runtime_webhook_requests_in_flight",
+				Help: "Current number of admission requests being served.",
+			},
+			[]string{"webhook"},
+		)
+	}()
 )
 
 func init() {
-	metrics.Registry.MustRegister(
-		RequestLatency)
+	metrics.Registry.MustRegister(RequestLatency, RequestTotal, RequestInFlight)
 }

pkg/webhook/server.go

@@ -28,8 +28,9 @@ import (
 	"path/filepath"
 	"strconv"
 	"sync"
-	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/internal/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
@@ -123,13 +124,19 @@ func (s *Server) Register(path string, hook http.Handler) {
 
 // instrumentedHook adds some instrumentation on top of the given webhook.
 func instrumentedHook(path string, hookRaw http.Handler) http.Handler {
-	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		startTS := time.Now()
-		defer func() { metrics.RequestLatency.WithLabelValues(path).Observe(time.Since(startTS).Seconds()) }()
-		hookRaw.ServeHTTP(resp, req)
-
-		// TODO(directxman12): add back in metric about total requests broken down by result?
-	})
+	lbl := prometheus.Labels{"webhook": path}
+
+	lat := metrics.RequestLatency.MustCurryWith(lbl)
+	cnt := metrics.RequestTotal.MustCurryWith(lbl)
+	gge := metrics.RequestInFlight.With(lbl)
+
+	return promhttp.InstrumentHandlerDuration(
+		lat,
+		promhttp.InstrumentHandlerCounter(
+			cnt,
+			promhttp.InstrumentHandlerInFlight(gge, hookRaw),
+		),
+	)
 }
 
 // Start runs the server.