From 479097a28af3e7be699ff4b6f33e4c4379e11722 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 12:19:51 +0100 Subject: [PATCH 1/6] Don't merge mixins in the global namespace anymore. Instead, put them in the $.mixins map. This allows for mixins to be places in specific Grafana directories, and also removes the horrible bug around playbook links. Still support merging into the global namespace for the time being, but this will be going away. Signed-off-by: Tom Wilkie --- .../grafana/dashboards.libsonnet | 34 +++++++++++-- prometheus-ksonnet/lib/config.libsonnet | 19 ------- .../lib/prometheus-config.libsonnet | 23 ++++++++- .../prometheus-ksonnet.libsonnet | 49 +++++++++++++++++-- 4 files changed, 94 insertions(+), 31 deletions(-) diff --git a/prometheus-ksonnet/grafana/dashboards.libsonnet b/prometheus-ksonnet/grafana/dashboards.libsonnet index b7af2d997..8dbff8aef 100644 --- a/prometheus-ksonnet/grafana/dashboards.libsonnet +++ b/prometheus-ksonnet/grafana/dashboards.libsonnet @@ -7,11 +7,35 @@ dashboard_config_maps: 8, }, - // Extension point for you to add your own dashboards. - dashboards+:: {}, - grafana_dashboards+:: {}, - grafanaDashboards+:: $.dashboards + $.grafana_dashboards, - dashboardsByFolder+:: {}, + // New API: Mixins go in the mixins map. + mixins+:: {}, + + // Legacy extension points for you to add your own dashboards. + local emptyMixin = { + grafanaDashboards+: {}, + }, + + grafanaDashboards+:: std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + if !std.objectHas(mixin, 'grafanaDashboardFolder') + then acc + mixin.grafanaDashboards + else acc, + std.objectFields($.mixins), + {} + ), + + dashboardsByFolder+:: std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + if std.objectHas(mixin, 'grafanaDashboardFolder') + then acc + { + [mixin.grafanaDashboardFolder]: mixin.grafanaDashboards, + } + else acc, + std.objectFields($.mixins), + {} + ), local materialise_config_map(config_map_name, dashboards) = configMap.new(config_map_name) + diff --git a/prometheus-ksonnet/lib/config.libsonnet b/prometheus-ksonnet/lib/config.libsonnet index 243d9bec5..69f5411cb 100644 --- a/prometheus-ksonnet/lib/config.libsonnet +++ b/prometheus-ksonnet/lib/config.libsonnet @@ -81,25 +81,6 @@ // Node exporter options. node_exporter_mount_root: true, - // Kubernetes mixin overrides. - cadvisorSelector: 'job="kube-system/cadvisor"', - kubeletSelector: 'job="kube-system/kubelet"', - kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - notKubeDnsSelector: 'job!="kube-system/kube-dns"', - kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', - kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', - kubeApiserverSelector: 'job="kube-system/kube-apiserver"', - podLabel: 'instance', - grafanaPrefix: '/grafana', // Also used by node-mixin. - - // Prometheus mixin overrides. - prometheusSelector: 'job="default/prometheus"', - alertmanagerSelector: 'job="default/alertmanager"', - - // Node mixin overrides. - nodeCriticalSeverity: 'warning', // Do not page if nodes run out of disk space. - // oauth2-proxy oauth_enabled: false, }, diff --git a/prometheus-ksonnet/lib/prometheus-config.libsonnet b/prometheus-ksonnet/lib/prometheus-config.libsonnet index 8522cfee3..60a047c7b 100644 --- a/prometheus-ksonnet/lib/prometheus-config.libsonnet +++ b/prometheus-ksonnet/lib/prometheus-config.libsonnet @@ -432,6 +432,25 @@ }, // We changes to using camelCase, but here we try and make it backwards compatible. - prometheusAlerts+:: $.prometheus_alerts, - prometheusRules+:: $.prometheus_rules, + prometheusAlerts+:: + $.prometheus_alerts + + std.foldr( + function(m, acc) m + acc, + [ + $.mixins[mixinName].prometheusAlerts + for mixinName in std.objectFields($.mixins) + ], + {} + ), + + prometheusRules+:: + $.prometheus_rules + + std.foldr( + function(m, acc) m + acc, + [ + $.mixins[mixinName].prometheusAlerts + for mixinName in std.objectFields($.mixins) + ], + {} + ), } diff --git a/prometheus-ksonnet/prometheus-ksonnet.libsonnet b/prometheus-ksonnet/prometheus-ksonnet.libsonnet index a9e71b0bb..ed1a5c8a6 100644 --- a/prometheus-ksonnet/prometheus-ksonnet.libsonnet +++ b/prometheus-ksonnet/prometheus-ksonnet.libsonnet @@ -8,8 +8,47 @@ (import 'lib/prometheus.libsonnet') + (import 'lib/prometheus-config.libsonnet') + (import 'lib/prometheus-configmap.libsonnet') + -(import 'kubernetes-mixin/mixin.libsonnet') + -(import 'prometheus-mixin/mixin.libsonnet') + -(import 'alertmanager-mixin/mixin.libsonnet') + -(import 'node-mixin/mixin.libsonnet') + -(import 'lib/config.libsonnet') +(import 'lib/config.libsonnet') + +{ + mixins+:: { + kubernetes+: + (import 'kubernetes-mixin/mixin.libsonnet') { + _config+:: { + cadvisorSelector: 'job="kube-system/cadvisor"', + kubeletSelector: 'job="kube-system/kubelet"', + kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + notKubeDnsSelector: 'job!="kube-system/kube-dns"', + kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', + kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', + kubeApiserverSelector: 'job="kube-system/kube-apiserver"', + podLabel: 'instance', + }, + }, + + prometheus+: + (import 'prometheus-mixin/mixin.libsonnet') { + _config+:: { + prometheusSelector: 'job="default/prometheus"', + }, + }, + + alertmanager+: + (import 'alertmanager-mixin/mixin.libsonnet') { + _config+:: { + alertmanagerSelector: 'job="default/alertmanager"', + }, + }, + + node_exporter+: + (import 'node-mixin/mixin.libsonnet') { + _config+:: { + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + + // Do not page if nodes run out of disk space. + nodeCriticalSeverity: 'warning', + grafanaPrefix: '/grafana', + }, + }, + }, +} From bb31466ae90d5bb9c42530ce78b9e6326b785837 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 12:26:18 +0100 Subject: [PATCH 2/6] make fmt Signed-off-by: Tom Wilkie --- .../grafana/dashboards.libsonnet | 2 +- .../prometheus-ksonnet.libsonnet | 74 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/prometheus-ksonnet/grafana/dashboards.libsonnet b/prometheus-ksonnet/grafana/dashboards.libsonnet index 8dbff8aef..d3bb3b8a9 100644 --- a/prometheus-ksonnet/grafana/dashboards.libsonnet +++ b/prometheus-ksonnet/grafana/dashboards.libsonnet @@ -29,7 +29,7 @@ function(mixinName, acc) local mixin = $.mixins[mixinName] + emptyMixin; if std.objectHas(mixin, 'grafanaDashboardFolder') - then acc + { + then acc { [mixin.grafanaDashboardFolder]: mixin.grafanaDashboards, } else acc, diff --git a/prometheus-ksonnet/prometheus-ksonnet.libsonnet b/prometheus-ksonnet/prometheus-ksonnet.libsonnet index ed1a5c8a6..3c95bf2f7 100644 --- a/prometheus-ksonnet/prometheus-ksonnet.libsonnet +++ b/prometheus-ksonnet/prometheus-ksonnet.libsonnet @@ -10,45 +10,45 @@ (import 'lib/prometheus-configmap.libsonnet') + (import 'lib/config.libsonnet') + { - mixins+:: { - kubernetes+: - (import 'kubernetes-mixin/mixin.libsonnet') { - _config+:: { - cadvisorSelector: 'job="kube-system/cadvisor"', - kubeletSelector: 'job="kube-system/kubelet"', - kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - notKubeDnsSelector: 'job!="kube-system/kube-dns"', - kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', - kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', - kubeApiserverSelector: 'job="kube-system/kube-apiserver"', - podLabel: 'instance', - }, - }, + mixins+:: { + kubernetes+: + (import 'kubernetes-mixin/mixin.libsonnet') { + _config+:: { + cadvisorSelector: 'job="kube-system/cadvisor"', + kubeletSelector: 'job="kube-system/kubelet"', + kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + notKubeDnsSelector: 'job!="kube-system/kube-dns"', + kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', + kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', + kubeApiserverSelector: 'job="kube-system/kube-apiserver"', + podLabel: 'instance', + }, + }, - prometheus+: - (import 'prometheus-mixin/mixin.libsonnet') { - _config+:: { - prometheusSelector: 'job="default/prometheus"', - }, - }, + prometheus+: + (import 'prometheus-mixin/mixin.libsonnet') { + _config+:: { + prometheusSelector: 'job="default/prometheus"', + }, + }, - alertmanager+: - (import 'alertmanager-mixin/mixin.libsonnet') { - _config+:: { - alertmanagerSelector: 'job="default/alertmanager"', - }, - }, + alertmanager+: + (import 'alertmanager-mixin/mixin.libsonnet') { + _config+:: { + alertmanagerSelector: 'job="default/alertmanager"', + }, + }, - node_exporter+: - (import 'node-mixin/mixin.libsonnet') { - _config+:: { - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + node_exporter+: + (import 'node-mixin/mixin.libsonnet') { + _config+:: { + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - // Do not page if nodes run out of disk space. - nodeCriticalSeverity: 'warning', - grafanaPrefix: '/grafana', - }, - }, - }, + // Do not page if nodes run out of disk space. + nodeCriticalSeverity: 'warning', + grafanaPrefix: '/grafana', + }, + }, + }, } From 5b130291ad752707ee2ace64eb6698039e03997d Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 13:14:09 +0100 Subject: [PATCH 3/6] Put mixins + overrides in separate file. Signed-off-by: Tom Wilkie --- .../lib/prometheus-config.libsonnet | 160 +++++++++--------- prometheus-ksonnet/mixins.libsonnet | 44 +++++ .../prometheus-ksonnet.libsonnet | 44 +---- 3 files changed, 124 insertions(+), 124 deletions(-) create mode 100644 prometheus-ksonnet/mixins.libsonnet diff --git a/prometheus-ksonnet/lib/prometheus-config.libsonnet b/prometheus-ksonnet/lib/prometheus-config.libsonnet index 60a047c7b..6f332da63 100644 --- a/prometheus-ksonnet/lib/prometheus-config.libsonnet +++ b/prometheus-ksonnet/lib/prometheus-config.libsonnet @@ -363,94 +363,92 @@ ], }, - // Extension points for adding alerts, recording rules and prometheus config. - prometheus_alerts:: { - groups+: [ - { - name: 'prometheus-extra', - rules: [ - { - alert: 'PromScrapeFailed', - expr: ||| - up != 1 - |||, - 'for': '15m', - labels: { - severity: 'warning', - }, - annotations: { - message: 'Prometheus failed to scrape a target {{ $labels.job }} / {{ $labels.instance }}', - }, - }, - { - alert: 'PromScrapeFlapping', - expr: ||| - avg_over_time(up[5m]) < 1 - |||, - 'for': '15m', - labels: { - severity: 'warning', - }, - annotations: { - message: 'Prometheus target flapping {{ $labels.job }} / {{ $labels.instance }}', + // Legacy Extension points for adding alerts, recording rules and prometheus config. + local emptyMixin = { + prometheusAlerts+:: {}, + prometheusRules+:: {}, + }, + + prometheusAlerts:: + { + groups+: [ + { + name: 'prometheus-extra', + rules: [ + { + alert: 'PromScrapeFailed', + expr: ||| + up != 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: 'Prometheus failed to scrape a target {{ $labels.job }} / {{ $labels.instance }}', + }, }, - }, - { - alert: 'PromScrapeTooLong', - expr: ||| - scrape_duration_seconds > 60 - |||, - 'for': '15m', - labels: { - severity: 'warning', + { + alert: 'PromScrapeFlapping', + expr: ||| + avg_over_time(up[5m]) < 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: 'Prometheus target flapping {{ $labels.job }} / {{ $labels.instance }}', + }, }, - annotations: { - message: '{{ $labels.job }} / {{ $labels.instance }} is taking too long to scrape ({{ printf "%.1f" $value }}s)', + { + alert: 'PromScrapeTooLong', + expr: ||| + scrape_duration_seconds > 60 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + message: '{{ $labels.job }} / {{ $labels.instance }} is taking too long to scrape ({{ printf "%.1f" $value }}s)', + }, }, - }, - ], - }, - ], - }, - - prometheus_rules:: { - groups+: [ - { - // Add mapping from namespace, pod -> node with node name as pod, as - // we use the node name as the node-exporter instance label. - name: 'instance_override', - rules: [ - { - record: 'node_namespace_pod:kube_pod_info:', - expr: ||| - max by(node, namespace, instance) (label_replace(kube_pod_info{job="default/kube-state-metrics"}, "instance", "$1", "node", "(.*)")) - |||, - }, - ], - }, - ], - }, - - // We changes to using camelCase, but here we try and make it backwards compatible. - prometheusAlerts+:: - $.prometheus_alerts + - std.foldr( - function(m, acc) m + acc, - [ - $.mixins[mixinName].prometheusAlerts - for mixinName in std.objectFields($.mixins) + ], + }, ], + } + + std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + acc + mixin.prometheusAlerts, + std.objectFields($.mixins), {} ), - prometheusRules+:: - $.prometheus_rules + - std.foldr( - function(m, acc) m + acc, - [ - $.mixins[mixinName].prometheusAlerts - for mixinName in std.objectFields($.mixins) + prometheusRules:: + { + groups+: [ + { + // Add mapping from namespace, pod -> node with node name as pod, as + // we use the node name as the node-exporter instance label. + name: 'instance_override', + rules: [ + { + record: 'node_namespace_pod:kube_pod_info:', + expr: ||| + max by(node, namespace, instance) (label_replace(kube_pod_info{job="default/kube-state-metrics"}, "instance", "$1", "node", "(.*)")) + |||, + }, + ], + }, ], - {} + } + + std.foldr( + function(mixinName, acc) + local mixin = $.mixins[mixinName] + emptyMixin; + acc + mixin.prometheusRules, + std.objectFields($.mixins), + {}, ), } diff --git a/prometheus-ksonnet/mixins.libsonnet b/prometheus-ksonnet/mixins.libsonnet new file mode 100644 index 000000000..7eaebd5a8 --- /dev/null +++ b/prometheus-ksonnet/mixins.libsonnet @@ -0,0 +1,44 @@ +{ + // Add you mixins here. + mixins+:: { + kubernetes+: + (import 'kubernetes-mixin/mixin.libsonnet') { + _config+:: { + cadvisorSelector: 'job="kube-system/cadvisor"', + kubeletSelector: 'job="kube-system/kubelet"', + kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + notKubeDnsSelector: 'job!="kube-system/kube-dns"', + kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', + kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', + kubeApiserverSelector: 'job="kube-system/kube-apiserver"', + podLabel: 'instance', + }, + }, + + prometheus+: + (import 'prometheus-mixin/mixin.libsonnet') { + _config+:: { + prometheusSelector: 'job="default/prometheus"', + }, + }, + + alertmanager+: + (import 'alertmanager-mixin/mixin.libsonnet') { + _config+:: { + alertmanagerSelector: 'job="default/alertmanager"', + }, + }, + + node_exporter+: + (import 'node-mixin/mixin.libsonnet') { + _config+:: { + nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. + + // Do not page if nodes run out of disk space. + nodeCriticalSeverity: 'warning', + grafanaPrefix: '/grafana', + }, + }, + }, +} diff --git a/prometheus-ksonnet/prometheus-ksonnet.libsonnet b/prometheus-ksonnet/prometheus-ksonnet.libsonnet index 3c95bf2f7..a6a648175 100644 --- a/prometheus-ksonnet/prometheus-ksonnet.libsonnet +++ b/prometheus-ksonnet/prometheus-ksonnet.libsonnet @@ -9,46 +9,4 @@ (import 'lib/prometheus-config.libsonnet') + (import 'lib/prometheus-configmap.libsonnet') + (import 'lib/config.libsonnet') + -{ - mixins+:: { - kubernetes+: - (import 'kubernetes-mixin/mixin.libsonnet') { - _config+:: { - cadvisorSelector: 'job="kube-system/cadvisor"', - kubeletSelector: 'job="kube-system/kubelet"', - kubeStateMetricsSelector: 'job="%s/kube-state-metrics"' % $._config.namespace, - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - notKubeDnsSelector: 'job!="kube-system/kube-dns"', - kubeSchedulerSelector: 'job="kube-system/kube-scheduler"', - kubeControllerManagerSelector: 'job="kube-system/kube-controller-manager"', - kubeApiserverSelector: 'job="kube-system/kube-apiserver"', - podLabel: 'instance', - }, - }, - - prometheus+: - (import 'prometheus-mixin/mixin.libsonnet') { - _config+:: { - prometheusSelector: 'job="default/prometheus"', - }, - }, - - alertmanager+: - (import 'alertmanager-mixin/mixin.libsonnet') { - _config+:: { - alertmanagerSelector: 'job="default/alertmanager"', - }, - }, - - node_exporter+: - (import 'node-mixin/mixin.libsonnet') { - _config+:: { - nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. - - // Do not page if nodes run out of disk space. - nodeCriticalSeverity: 'warning', - grafanaPrefix: '/grafana', - }, - }, - }, -} +(import 'mixins.libsonnet') From 6cc735ef693ce008d846aeb5a629055f3864ac17 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 13:27:00 +0100 Subject: [PATCH 4/6] Review feedback. Signed-off-by: Tom Wilkie --- prometheus-ksonnet/mixins.libsonnet | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prometheus-ksonnet/mixins.libsonnet b/prometheus-ksonnet/mixins.libsonnet index 7eaebd5a8..fdcaedc5f 100644 --- a/prometheus-ksonnet/mixins.libsonnet +++ b/prometheus-ksonnet/mixins.libsonnet @@ -1,7 +1,7 @@ { // Add you mixins here. mixins+:: { - kubernetes+: + kubernetes: (import 'kubernetes-mixin/mixin.libsonnet') { _config+:: { cadvisorSelector: 'job="kube-system/cadvisor"', @@ -16,21 +16,21 @@ }, }, - prometheus+: + prometheus: (import 'prometheus-mixin/mixin.libsonnet') { _config+:: { prometheusSelector: 'job="default/prometheus"', }, }, - alertmanager+: + alertmanager: (import 'alertmanager-mixin/mixin.libsonnet') { _config+:: { alertmanagerSelector: 'job="default/alertmanager"', }, }, - node_exporter+: + node_exporter: (import 'node-mixin/mixin.libsonnet') { _config+:: { nodeExporterSelector: 'job="%s/node-exporter"' % $._config.namespace, // Also used by node-mixin. From 683150533c59c294c08b0fa923e9e24bc59156b6 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 13:27:26 +0100 Subject: [PATCH 5/6] make fmt Signed-off-by: Tom Wilkie --- prometheus-ksonnet/lib/prometheus-config.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus-ksonnet/lib/prometheus-config.libsonnet b/prometheus-ksonnet/lib/prometheus-config.libsonnet index 6f332da63..e993b2d60 100644 --- a/prometheus-ksonnet/lib/prometheus-config.libsonnet +++ b/prometheus-ksonnet/lib/prometheus-config.libsonnet @@ -420,8 +420,8 @@ } + std.foldr( function(mixinName, acc) - local mixin = $.mixins[mixinName] + emptyMixin; - acc + mixin.prometheusAlerts, + local mixin = $.mixins[mixinName] + emptyMixin; + acc + mixin.prometheusAlerts, std.objectFields($.mixins), {} ), From 5a61339873152059d38bd3a2121bb2a40288c0c2 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 7 Apr 2020 13:40:12 +0100 Subject: [PATCH 6/6] Clarify comment. Signed-off-by: Tom Wilkie --- prometheus-ksonnet/grafana/dashboards.libsonnet | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/prometheus-ksonnet/grafana/dashboards.libsonnet b/prometheus-ksonnet/grafana/dashboards.libsonnet index d3bb3b8a9..41d763b42 100644 --- a/prometheus-ksonnet/grafana/dashboards.libsonnet +++ b/prometheus-ksonnet/grafana/dashboards.libsonnet @@ -10,11 +10,14 @@ // New API: Mixins go in the mixins map. mixins+:: {}, - // Legacy extension points for you to add your own dashboards. + // emptyMixin allows us to reliably do `mixin.grafanaDashboards` without + // having to check the field exists first. Some mixins don't declare all + // the fields, and thats fine. local emptyMixin = { grafanaDashboards+: {}, }, + // Legacy extension points for you to add your own dashboards. grafanaDashboards+:: std.foldr( function(mixinName, acc) local mixin = $.mixins[mixinName] + emptyMixin;