chore: aws client metrics

pkg/kcp/provider/aws/client/config.go

@@ -4,10 +4,13 @@ import (
 	"context"
 	"errors"
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	smithymiddleware "github.com/aws/smithy-go/middleware"
+	"github.com/kyma-project/cloud-manager/pkg/metrics"
 )
 
 func NewGardenConfig(ctx context.Context, region, key, secret string) (cfg aws.Config, err error) {
@@ -19,7 +22,16 @@ func NewGardenConfig(ctx context.Context, region, key, secret string) (cfg aws.C
 		ctx,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(key, secret, "")),
+		config.WithRetryer(
+			func() aws.Retryer {
+				// https://github.com/aws/aws-sdk-go-v2/issues/1744
+				return retry.NewStandard()
+			},
+		),
 	)
+	cfg.APIOptions = append(cfg.APIOptions, func(stack *smithymiddleware.Stack) error {
+		return stack.Deserialize.Add(metrics.AwsReportMetricsMiddleware(), smithymiddleware.After)
+	})
 	return
 }
 
@@ -47,6 +59,15 @@ func NewSkrConfig(ctx context.Context, region, key, secret, assumeRole string) (
 				assumeRole,
 			)),
 		),
+		config.WithRetryer(
+			func() aws.Retryer {
+				// https://github.com/aws/aws-sdk-go-v2/issues/1744
+				return retry.NewStandard()
+			},
+		),
 	)
+	cfg.APIOptions = append(cfg.APIOptions, func(stack *smithymiddleware.Stack) error {
+		return stack.Deserialize.Add(metrics.AwsReportMetricsMiddleware(), smithymiddleware.After)
+	})
 	return
 }

pkg/metrics/cloudProvider.go

@@ -0,0 +1,30 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	CloudProviderAWS = "aws"
+)
+
+var (
+	CloudProviderCallCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "cloud_manager_cloud_provider_api_call_total",
+		Help: "Total number of cloud provider API calls per provider, method, response code, and region",
+	}, []string{"provider", "method", "response_code", "region"})
+
+	CloudProviderCallTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "cloud_manager_cloud_provider_api_time_seconds",
+		Help:    "Length of cloud provider call time per provider, method, and region",
+		Buckets: []float64{0.2, 0.4, 0.6, 0.8, 1.0, 1.5, 2, 3, 4, 5, 7, 9},
+	}, []string{"provider", "method", "region"})
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		CloudProviderCallCount,
+		CloudProviderCallTime,
+	)
+}

pkg/metrics/cloudProviderAws.go

@@ -0,0 +1,68 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	sdkmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	smithymiddleware "github.com/aws/smithy-go/middleware"
+	"github.com/aws/smithy-go/transport/http"
+	"time"
+)
+
+type awsRequestMetricTuple struct {
+	ServiceName   string
+	OperationName string
+	Region        string
+	Latency       time.Duration
+	ResponseCode  int
+}
+
+func awsReportMetrics(metrics *awsRequestMetricTuple) {
+	CloudProviderCallCount.WithLabelValues(
+		CloudProviderAWS,
+		fmt.Sprintf("%s/%s", metrics.ServiceName, metrics.OperationName),
+		fmt.Sprintf("%d", metrics.ResponseCode),
+		metrics.Region,
+	).Inc()
+
+	CloudProviderCallTime.WithLabelValues(
+		CloudProviderAWS,
+		fmt.Sprintf("%s/%s", metrics.ServiceName, metrics.OperationName),
+		metrics.Region,
+	).Observe(metrics.Latency.Seconds())
+}
+
+func AwsReportMetricsMiddleware() smithymiddleware.DeserializeMiddleware {
+	// look at https://github.com/aws/aws-sdk-go-v2/issues/1744
+	reportRequestMetrics := smithymiddleware.DeserializeMiddlewareFunc("ReportRequestMetrics", func(
+		ctx context.Context, in smithymiddleware.DeserializeInput, next smithymiddleware.DeserializeHandler,
+	) (
+		out smithymiddleware.DeserializeOutput, metadata smithymiddleware.Metadata, err error,
+	) {
+		requestMadeTime := time.Now()
+		out, metadata, err = next.HandleDeserialize(ctx, in)
+		if err != nil {
+			return out, metadata, err
+		}
+
+		responseStatusCode := -1
+		switch resp := out.RawResponse.(type) {
+		case *http.Response:
+			responseStatusCode = resp.StatusCode
+		}
+
+		latency := time.Now().Sub(requestMadeTime)
+		metrics := awsRequestMetricTuple{
+			ServiceName:   sdkmiddleware.GetServiceID(ctx),
+			OperationName: sdkmiddleware.GetOperationName(ctx),
+			Region:        sdkmiddleware.GetRegion(ctx),
+			Latency:       latency,
+			ResponseCode:  responseStatusCode,
+		}
+		awsReportMetrics(&metrics)
+
+		return out, metadata, nil
+	})
+
+	return reportRequestMetrics
+}