feat(inputs.promql): Add plugin

Makefile

@@ -131,7 +131,7 @@ config:
 	go run ./cmd/telegraf config > etc/telegraf.conf
 
 .PHONY: docs
-docs: build_tools embed_readme_inputs embed_readme_outputs embed_readme_processors embed_readme_aggregators embed_readme_secretstores
+docs: build_tools embed_readme_common embed_readme_inputs embed_readme_outputs embed_readme_processors embed_readme_aggregators embed_readme_secretstores
 
 .PHONY: build
 build:

go.mod

@@ -466,6 +466,7 @@ require (
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muhlemmer/gu v0.3.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/nats-io/jwt/v2 v2.8.0 // indirect
 	github.com/nats-io/nkeys v0.4.11 // indirect

plugins/common/http/config.go

@@ -1,3 +1,4 @@
+//go:generate ../../../tools/config_includer/generator "common.http" "transport.conf.in"
 package httpconfig
 
 import (
@@ -17,14 +18,44 @@ import (
 	"github.com/influxdata/telegraf/plugins/common/tls"
 )
 
+// TransportConfig is configuration structure for HTTP transports
+type TransportConfig struct {
+	IdleConnTimeout       config.Duration `toml:"idle_conn_timeout"`
+	MaxIdleConns          int             `toml:"max_idle_conn"`
+	MaxIdleConnsPerHost   int             `toml:"max_idle_conn_per_host"`
+	ResponseHeaderTimeout config.Duration `toml:"response_timeout"`
+	proxy.HTTPProxy
+	tls.ClientConfig
+}
+
+func (h *TransportConfig) CreateTransport() (*http.Transport, error) {
+	tlsCfg, err := h.ClientConfig.TLSConfig()
+	if err != nil {
+		return nil, fmt.Errorf("creating TLS configuration failed: %w", err)
+	}
+
+	prox, err := h.HTTPProxy.Proxy()
+	if err != nil {
+		return nil, fmt.Errorf("setting up proxy failed: %w", err)
+	}
+
+	return &http.Transport{
+		TLSClientConfig:       tlsCfg,
+		Proxy:                 prox,
+		IdleConnTimeout:       time.Duration(h.IdleConnTimeout),
+		MaxIdleConns:          h.MaxIdleConns,
+		MaxIdleConnsPerHost:   h.MaxIdleConnsPerHost,
+		ResponseHeaderTimeout: time.Duration(h.ResponseHeaderTimeout),
+	}, nil
+}
+
 // HTTPClientConfig is a common HTTP client struct.
 type HTTPClientConfig struct {
 	Timeout               config.Duration `toml:"timeout"`
 	IdleConnTimeout       config.Duration `toml:"idle_conn_timeout"`
 	MaxIdleConns          int             `toml:"max_idle_conn"`
 	MaxIdleConnsPerHost   int             `toml:"max_idle_conn_per_host"`
 	ResponseHeaderTimeout config.Duration `toml:"response_timeout"`
-
 	proxy.HTTPProxy
 	tls.ClientConfig
 	oauth.OAuth2Config
@@ -34,12 +65,12 @@ type HTTPClientConfig struct {
 func (h *HTTPClientConfig) CreateClient(ctx context.Context, log telegraf.Logger) (*http.Client, error) {
 	tlsCfg, err := h.ClientConfig.TLSConfig()
 	if err != nil {
-		return nil, fmt.Errorf("failed to set TLS config: %w", err)
+		return nil, fmt.Errorf("creating TLS configuration failed: %w", err)
 	}
 
 	prox, err := h.HTTPProxy.Proxy()
 	if err != nil {
-		return nil, fmt.Errorf("failed to set proxy: %w", err)
+		return nil, fmt.Errorf("setting up proxy failed: %w", err)
 	}
 
 	transport := &http.Transport{

plugins/common/http/transport.conf

@@ -0,0 +1,35 @@
+  ## HTTP connection settings
+  # idle_conn_timeout = "0s"
+  # max_idle_conn = 0
+  # max_idle_conn_per_host = 0
+  # response_timeout ="0s"
+
+  ## Optional proxy settings
+  # use_system_proxy = false
+  # http_proxy_url = ""
+
+  ## Optional TLS settings
+  ## Set to true/false to enforce TLS being enabled/disabled. If not set,
+  ## enable TLS only if any of the other options are specified.
+  # tls_enable =
+  ## Trusted root certificates for server
+  # tls_ca = "/path/to/cafile"
+  ## Used for TLS client certificate authentication
+  # tls_cert = "/path/to/certfile"
+  ## Used for TLS client certificate authentication
+  # tls_key = "/path/to/keyfile"
+  ## Password for the key file if it is encrypted
+  # tls_key_pwd = ""
+  ## Send the specified TLS server name via SNI
+  # tls_server_name = "kubernetes.example.com"
+  ## Minimal TLS version to accept by the client
+  # tls_min_version = "TLS12"
+  ## List of ciphers to accept, by default all secure ciphers will be accepted
+  ## See https://pkg.go.dev/crypto/tls#pkg-constants for supported values.
+  ## Use "all", "secure" and "insecure" to add all support ciphers, secure
+  ## suites or insecure suites respectively.
+  # tls_cipher_suites = ["secure"]
+  ## Renegotiation method, "never", "once" or "freely"
+  # tls_renegotiation_method = "never"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false

plugins/common/http/transport.conf.in

@@ -0,0 +1,11 @@
+  ## HTTP connection settings
+  # idle_conn_timeout = "0s"
+  # max_idle_conn = 0
+  # max_idle_conn_per_host = 0
+  # response_timeout ="0s"
+
+  ## Optional proxy settings
+{{template "/plugins/common/proxy/proxy.conf"}}
+
+  ## Optional TLS settings
+{{template "/plugins/common/tls/client.conf"}}

plugins/common/proxy/proxy.conf

@@ -0,0 +1,2 @@
+  # use_system_proxy = false
+  # http_proxy_url = ""

plugins/inputs/all/promql.go

@@ -0,0 +1,5 @@
+//go:build !custom || inputs || inputs.promql
+
+package all
+
+import _ "github.com/influxdata/telegraf/plugins/inputs/promql" // register plugin

plugins/inputs/promql/README.md

@@ -0,0 +1,151 @@
+# PromQL Input Plugin
+
+This plugin gathers metrics from a [Prometheus][prometheus] endpoint using
+[PromQL queries][promql] via the [HTTP API][http_api].
+
+⭐ Telegraf v1.37.0
+🏷️ datastore
+💻 all
+
+[prometheus]: https://prometheus.io/
+[promql]: https://prometheus.io/docs/prometheus/latest/querying/basics/
+[http_api]: https://prometheus.io/docs/prometheus/latest/querying/api/
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Secret-store support
+
+This plugin supports secrets from secret-stores for the `username`, `password`
+and `token` option. See the [secret-store documentation][SECRETSTORE] for
+more details on how to use them.
+
+[SECRETSTORE]: ../../../docs/CONFIGURATION.md#secret-store-secrets
+
+## Configuration
+
+```toml @sample.conf
+# Query prometheus endpoints using PromQL
+[[inputs.promql]]
+  ## URL of the prometheus endpoint
+  url = "http://localhost:9090"
+
+  ## Basic authentication properties
+  # username = ""
+  # password = ""
+
+  ## Bearer token based authentication
+  # token = ""
+
+  ## Timeout for executing queries with zero meaning no timeout
+  # timeout = "5s"
+
+  ## HTTP connection settings
+  # idle_conn_timeout = "0s"
+  # max_idle_conn = 0
+  # max_idle_conn_per_host = 0
+  # response_timeout ="0s"
+
+  ## Optional proxy settings
+  # use_system_proxy = false
+  # http_proxy_url = ""
+
+  ## Optional TLS settings
+  ## Set to true/false to enforce TLS being enabled/disabled. If not set,
+  ## enable TLS only if any of the other options are specified.
+  # tls_enable =
+  ## Trusted root certificates for server
+  # tls_ca = "/path/to/cafile"
+  ## Used for TLS client certificate authentication
+  # tls_cert = "/path/to/certfile"
+  ## Used for TLS client certificate authentication
+  # tls_key = "/path/to/keyfile"
+  ## Password for the key file if it is encrypted
+  # tls_key_pwd = ""
+  ## Send the specified TLS server name via SNI
+  # tls_server_name = "kubernetes.example.com"
+  ## Minimal TLS version to accept by the client
+  # tls_min_version = "TLS12"
+  ## List of ciphers to accept, by default all secure ciphers will be accepted
+  ## See https://pkg.go.dev/crypto/tls#pkg-constants for supported values.
+  ## Use "all", "secure" and "insecure" to add all support ciphers, secure
+  ## suites or insecure suites respectively.
+  # tls_cipher_suites = ["secure"]
+  ## Renegotiation method, "never", "once" or "freely"
+  # tls_renegotiation_method = "never"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+
+  ## Instant queries, multiple instances are allowed
+  # [[inputs.promql.instant]]
+  #   ## Query to execute
+  #   query = 'prometheus_http_requests_total'
+  #
+  #   ## Limit for the number of results returned by the server with zero
+  #   ## meaning no limit
+  #   # limit = 0
+
+  ## Rangequeries, multiple instances are allowed
+  # [[inputs.promql.range]]
+  #   ## Query to execute
+  #   query = 'prometheus_http_requests_total{job="prometheus"}'
+  #   ## Range parameters relative to the gathering time with positive values
+  #   ## refer to BEFORE and negative to AFTER the gathering time
+  #   start = "5m"
+  #   # end = "0s"
+  #   step = "1m"
+  #
+  #   ## Limit for the number of results returned by the server with zero
+  #   ## meaning no limit
+  #   # limit = 0
+```
+
+> [!NOTE]
+> You can either use no authentication _or_ basic authentication _or_ Bearer
+> token based authentication. Uncommenting both basic and Bearer token based
+> authentication will fail.
+
+## Metrics
+
+The metrics collected by this input plugin will depend on the specified queries.
+However, the resulting metrics will have the following structure for the
+returned results.
+
+### Scalar and String Results
+
+A scalar or string result will produce a single metrics named after the value of
+the Prometheus `__name__` label. Other labels will be kept as tags. The
+resulting metric used the Prometheus timestamp and will have the value stored in
+a `value` field.
+
+### Vector and Matrix Results
+
+A vector result will produce one or more metrics with the metric named after the
+value of the Prometheus `__name__` label for each element of the vector. Other
+labels will be kept as tags. All metrics will use the Prometheus timestamp.
+
+Non-histogram results will have the value stored in a `value` field. Histogram
+results will contain multiple fields with the fieldname being the upper bound
+of the bin and a value with the bin count. Additionally, the metric will have a
+`count` and a `sum` field.
+
+## Example Output
+
+For example, a range-query for
+`prometheus_http_requests_total{job="prometheus", handler="/api/v1/query"}`
+starting 5 minutes in the past with 1 minute stepping returns
+
+```text
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806201000000000
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806261000000000
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806321000000000
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806381000000000
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806441000000000
+prometheus_http_requests_total,app=prometheus,code=200,handler=/api/v1/query,instance=localhost:9090,job=prometheus value=28 1758806501000000000
+```

plugins/inputs/promql/client.go

@@ -0,0 +1,92 @@
+package promql
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	common_http "github.com/influxdata/telegraf/plugins/common/http"
+
+	"github.com/prometheus/client_golang/api"
+	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	promcfg "github.com/prometheus/common/config"
+)
+
+type client struct {
+	url      string
+	username config.Secret
+	password config.Secret
+	token    config.Secret
+	cfg      common_http.TransportConfig
+
+	client api.Client
+	apiv1.API
+}
+
+func (c *client) init() (*client, error) {
+	// Create a round-tripper suitable for the given configuration based on the
+	// http-client transport...
+	transport, err := c.cfg.CreateTransport()
+	if err != nil {
+		return nil, fmt.Errorf("creating transport failed: %w", err)
+	}
+	rt := promcfg.NewUserAgentRoundTripper(internal.ProductToken(), transport)
+	if !c.username.Empty() {
+		rt = promcfg.NewBasicAuthRoundTripper(
+			&secretReader{"username", &c.username},
+			&secretReader{"password", &c.password},
+			rt,
+		)
+	} else if !c.token.Empty() {
+		rt = promcfg.NewAuthorizationCredentialsRoundTripper(
+			"Bearer",
+			&secretReader{"token", &c.token},
+			rt,
+		)
+	}
+
+	// Create API client
+	apiClient, err := api.NewClient(api.Config{
+		Address:      c.url,
+		RoundTripper: rt,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("creating API client failed: %w", err)
+	}
+	c.client = apiClient
+	c.API = apiv1.NewAPI(c.client)
+
+	return c, nil
+}
+
+func (c *client) close() {
+	if c.client != nil {
+		c.client.(api.CloseIdler).CloseIdleConnections()
+	}
+}
+
+// Wrapper for reading secrets from Prometheus API client
+type secretReader struct {
+	desc   string
+	secret *config.Secret
+}
+
+func (r *secretReader) Fetch(ctx context.Context) (string, error) {
+	raw, err := r.secret.Get()
+	if err != nil {
+		return "", fmt.Errorf("getting %s failed: %w", r.desc, err)
+	}
+	s := raw.String()
+	raw.Destroy()
+
+	return s, nil
+}
+
+func (r *secretReader) Description() string {
+	return r.desc
+}
+
+func (*secretReader) Immutable() bool {
+	return true
+}