diff --git a/prometheus-ksonnet/grafana/dashboards.libsonnet b/prometheus-ksonnet/grafana/dashboards.libsonnet index b7af2d997..41d763b42 100644 --- a/prometheus-ksonnet/grafana/dashboards.libsonnet +++ b/prometheus-ksonnet/grafana/dashboards.libsonnet @@ -7,11 +7,38 @@ dashboard_config_maps: 8, }, - // Extension point for you to add your own dashboards. - dashboards+:: {}, - grafana_dashboards+:: {}, - grafanaDashboards+:: $.dashboards + $.grafana_dashboards, - dashboardsByFolder+:: {}, + // New API: Mixins go in the mixins map. + mixins+:: {}, + + // emptyMixin allows us to reliably do `mixin.grafanaDashboards` without + // having to check the field exists first. Some mixins don't declare all + // the fields, and thats fine. + local emptyMixin = { + grafanaDashboards+: {}, + }, + + // Legacy extension points for you to add your own dashboards. + grafanaDashboards+:: std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + if !std.objectHas(mixin, 'grafanaDashboardFolder') + then acc + mixin.grafanaDashboards + else acc, + std.objectFields($.mixins), + {} + ), + + dashboardsByFolder+:: std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + if std.objectHas(mixin, 'grafanaDashboardFolder') + then acc { + [mixin.grafanaDashboardFolder]: mixin.grafanaDashboards, + } + else acc, + std.objectFields($.mixins), + {} + ), local materialise_config_map(config_map_name, dashboards) = configMap.new(config_map_name) + diff --git a/prometheus-ksonnet/lib/config.libsonnet b/prometheus-ksonnet/lib/config.libsonnet index 243d9bec5..69f5411cb 100644 --- a/prometheus-ksonnet/lib/config.libsonnet +++ b/prometheus-ksonnet/lib/config.libsonnet @@ -81,25 +81,6 @@ // Node exporter options. node_exporter_mount_root: true, - // Kubernetes mixin overrides. - cadvisorSelector: 'job="kube-system/cadvisor"', - kubeletSelector: 'job="kube-system/kubelet"', - kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - notKubeDnsSelector: 'job!="kube-system/kube-dns"', - kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', - kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', - kubeApiserverSelector: 'job="kube-system/kube-apiserver"', - podLabel: 'instance', - grafanaPrefix: '/grafana', // Also used by node-mixin. - - // Prometheus mixin overrides. - prometheusSelector: 'job="default/prometheus"', - alertmanagerSelector: 'job="default/alertmanager"', - - // Node mixin overrides. - nodeCriticalSeverity: 'warning', // Do not page if nodes run out of disk space. - // oauth2-proxy oauth_enabled: false, }, diff --git a/prometheus-ksonnet/lib/prometheus-config.libsonnet b/prometheus-ksonnet/lib/prometheus-config.libsonnet index 8522cfee3..e993b2d60 100644 --- a/prometheus-ksonnet/lib/prometheus-config.libsonnet +++ b/prometheus-ksonnet/lib/prometheus-config.libsonnet @@ -363,75 +363,92 @@ ], }, - // Extension points for adding alerts, recording rules and prometheus config. - prometheus_alerts:: { - groups+: [ - { - name: 'prometheus-extra', - rules: [ - { - alert: 'PromScrapeFailed', - expr: ||| - up != 1 - |||, - 'for': '15m', - labels: { - severity: 'warning', - }, - annotations: { - message: 'Prometheus failed to scrape a target {{ $labels.job }} / {{ $labels.instance }}', - }, - }, - { - alert: 'PromScrapeFlapping', - expr: ||| - avg_over_time(up[5m]) < 1 - |||, - 'for': '15m', - labels: { - severity: 'warning', + // Legacy Extension points for adding alerts, recording rules and prometheus config. + local emptyMixin = { + prometheusAlerts+:: {}, + prometheusRules+:: {}, + }, + + prometheusAlerts:: + { + groups+: [ + { + name: 'prometheus-extra', + rules: [ + { + alert: 'PromScrapeFailed', + expr: ||| + up != 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: 'Prometheus failed to scrape a target {{ $labels.job }} / {{ $labels.instance }}', + }, }, - annotations: { - message: 'Prometheus target flapping {{ $labels.job }} / {{ $labels.instance }}', + { + alert: 'PromScrapeFlapping', + expr: ||| + avg_over_time(up[5m]) < 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: 'Prometheus target flapping {{ $labels.job }} / {{ $labels.instance }}', + }, }, - }, - { - alert: 'PromScrapeTooLong', - expr: ||| - scrape_duration_seconds > 60 - |||, - 'for': '15m', - labels: { - severity: 'warning', + { + alert: 'PromScrapeTooLong', + expr: ||| + scrape_duration_seconds > 60 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: '{{ $labels.job }} / {{ $labels.instance }} is taking too long to scrape ({{ printf "%.1f" $value }}s)', + }, }, - annotations: { - message: '{{ $labels.job }} / {{ $labels.instance }} is taking too long to scrape ({{ printf "%.1f" $value }}s)', + ], + }, + ], + } + + std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + acc + mixin.prometheusAlerts, + std.objectFields($.mixins), + {} + ), + + prometheusRules:: + { + groups+: [ + { + // Add mapping from namespace, pod -> node with node name as pod, as + // we use the node name as the node-exporter instance label. + name: 'instance_override', + rules: [ + { + record: 'node_namespace_pod:kube_pod_info:', + expr: ||| + max by(node, namespace, instance) (label_replace(kube_pod_info{job="default/kube-state-metrics"}, "instance", "$1", "node", "(.*)")) + |||, }, - }, - ], - }, - ], - }, - - prometheus_rules:: { - groups+: [ - { - // Add mapping from namespace, pod -> node with node name as pod, as - // we use the node name as the node-exporter instance label. - name: 'instance_override', - rules: [ - { - record: 'node_namespace_pod:kube_pod_info:', - expr: ||| - max by(node, namespace, instance) (label_replace(kube_pod_info{job="default/kube-state-metrics"}, "instance", "$1", "node", "(.*)")) - |||, - }, - ], - }, - ], - }, - - // We changes to using camelCase, but here we try and make it backwards compatible. - prometheusAlerts+:: $.prometheus_alerts, - prometheusRules+:: $.prometheus_rules, + ], + }, + ], + } + + std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + acc + mixin.prometheusRules, + std.objectFields($.mixins), + {}, + ), } diff --git a/prometheus-ksonnet/mixins.libsonnet b/prometheus-ksonnet/mixins.libsonnet new file mode 100644 index 000000000..fdcaedc5f --- /dev/null +++ b/prometheus-ksonnet/mixins.libsonnet @@ -0,0 +1,44 @@ +{ + // Add you mixins here. + mixins+:: { + kubernetes: + (import 'kubernetes-mixin/mixin.libsonnet') { + _config+:: { + cadvisorSelector: 'job="kube-system/cadvisor"', + kubeletSelector: 'job="kube-system/kubelet"', + kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + notKubeDnsSelector: 'job!="kube-system/kube-dns"', + kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', + kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', + kubeApiserverSelector: 'job="kube-system/kube-apiserver"', + podLabel: 'instance', + }, + }, + + prometheus: + (import 'prometheus-mixin/mixin.libsonnet') { + _config+:: { + prometheusSelector: 'job="default/prometheus"', + }, + }, + + alertmanager: + (import 'alertmanager-mixin/mixin.libsonnet') { + _config+:: { + alertmanagerSelector: 'job="default/alertmanager"', + }, + }, + + node_exporter: + (import 'node-mixin/mixin.libsonnet') { + _config+:: { + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + + // Do not page if nodes run out of disk space. + nodeCriticalSeverity: 'warning', + grafanaPrefix: '/grafana', + }, + }, + }, +} diff --git a/prometheus-ksonnet/prometheus-ksonnet.libsonnet b/prometheus-ksonnet/prometheus-ksonnet.libsonnet index a9e71b0bb..a6a648175 100644 --- a/prometheus-ksonnet/prometheus-ksonnet.libsonnet +++ b/prometheus-ksonnet/prometheus-ksonnet.libsonnet @@ -8,8 +8,5 @@ (import 'lib/prometheus.libsonnet') + (import 'lib/prometheus-config.libsonnet') + (import 'lib/prometheus-configmap.libsonnet') + -(import 'kubernetes-mixin/mixin.libsonnet') + -(import 'prometheus-mixin/mixin.libsonnet') + -(import 'alertmanager-mixin/mixin.libsonnet') + -(import 'node-mixin/mixin.libsonnet') + -(import 'lib/config.libsonnet') +(import 'lib/config.libsonnet') + +(import 'mixins.libsonnet')