From 610e235e22ab919e65d362aa9295fd11dfcb1eb2 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 19 Nov 2025 14:21:50 -0500 Subject: [PATCH] chore(otel): add support for http/protobuf OTLP metrics export Signed-off-by: Milas Bowman --- docs/metrics.md | 2 ++ go.mod | 1 + go.sum | 2 ++ util/telemetry/metrics.go | 32 ++++++++++++++++++++++++++------ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index bec07b4d3f3f..cc4101aa7e3f 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -31,6 +31,8 @@ It will not be enabled if left blank, unlike some other implementations. You can configure the protocol using the environment variables documented in [standard environment variables](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/). +By default, GRPC is used; to switch to HTTP, set either the `OTEL_EXPORTER_OTLP_PROTOCOL` or `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL` environment variable to `http/protobuf`. + The [configuration options](#common) in the controller ConfigMap `metricsTTL`, `modifiers` and `temporality` affect the OpenTelemetry behavior, but the other parameters do not. To use the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) you can configure it diff --git a/go.mod b/go.mod index 81764889ead3..d5280934580f 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/runtime v0.61.0 go.opentelemetry.io/otel v1.36.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 go.opentelemetry.io/otel/exporters/prometheus v0.58.0 go.opentelemetry.io/otel/metric v1.36.0 go.opentelemetry.io/otel/sdk v1.36.0 diff --git a/go.sum b/go.sum index 42c386b0734b..0dd6b151690e 100644 --- a/go.sum +++ b/go.sum @@ -872,6 +872,8 @@ go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 h1:zwdo1gS2eH26Rg+CoqVQpEK1h8gvt5qyU5Kk5Bixvow= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0/go.mod h1:rUKCPscaRWWcqGT6HnEmYrK+YNe5+Sw64xgQTOJ5b30= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 h1:gAU726w9J8fwr4qRDqu1GYMNNs4gXrU+Pv20/N1UpB4= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0/go.mod h1:RboSDkp7N292rgu+T0MgVt2qgFGu6qa1RpZDOtpL76w= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= diff --git a/util/telemetry/metrics.go b/util/telemetry/metrics.go index 3d64682235f9..253b95852882 100644 --- a/util/telemetry/metrics.go +++ b/util/telemetry/metrics.go @@ -3,13 +3,14 @@ package telemetry import ( "context" "os" + "strings" "sync" "time" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/metric" metricsdk "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" @@ -74,13 +75,32 @@ func NewMetrics(ctx context.Context, serviceName, prometheusName string, config _, otlpEnabled := os.LookupEnv(`OTEL_EXPORTER_OTLP_ENDPOINT`) _, otlpMetricsEnabled := os.LookupEnv(`OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`) logger := logging.RequireLoggerFromContext(ctx) + if otlpEnabled || otlpMetricsEnabled { logger.Info(ctx, "Starting OTLP metrics exporter") - otelExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithTemporalitySelector(config.Temporality)) - if err != nil { - return nil, err + + // NOTE: The OTel SDK default changed from gRPC to http/protobuf. For backwards compatibility, + // gRPC is preserved as the default in workflows controller, but http/protobuf can be opted-in + // to by setting the _PROTOCOL env var explicitly. + // These env vars match the official SDK: https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#otel_exporter_otlp_metrics_protocol. + otlpProtocol := os.Getenv(`OTEL_EXPORTER_OTLP_METRICS_PROTOCOL`) + if otlpProtocol == "" { + otlpProtocol = os.Getenv(`OTEL_EXPORTER_OTLP_PROTOCOL`) + } + + if otlpProtocol == "" || otlpProtocol == "grpc" { + httpExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithTemporalitySelector(config.Temporality)) + if err != nil { + return nil, err + } + options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(httpExporter))) + } else if strings.HasPrefix(otlpProtocol, "http/") { + grpcExporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTemporalitySelector(config.Temporality)) + if err != nil { + return nil, err + } + options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(grpcExporter))) } - options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(otelExporter))) } if config.Enabled {