From a9f13f6eac941e57d33f3214d335be2cd0da6fa8 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Mon, 20 Apr 2020 21:24:43 +0530 Subject: [PATCH 1/4] support for generating slack receivers with buttons for alerts Signed-off-by: Sandeep Sukhani --- prometheus-ksonnet/lib/alertmanager.libsonnet | 55 ++++++++++++++++--- .../lib/files/alertmanager_config.tmpl | 12 ++++ 2 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 prometheus-ksonnet/lib/files/alertmanager_config.tmpl diff --git a/prometheus-ksonnet/lib/alertmanager.libsonnet b/prometheus-ksonnet/lib/alertmanager.libsonnet index 07aeb4f8f..4e05364e8 100644 --- a/prometheus-ksonnet/lib/alertmanager.libsonnet +++ b/prometheus-ksonnet/lib/alertmanager.libsonnet @@ -17,20 +17,58 @@ ] else [], + slackAlertTitle:: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.cluster }}: {{ .GroupLabels.alertname }} ({{ .GroupLabels.namespace }})', + slackAlertText:: ||| + {{ .CommonAnnotations.summary }} + {{ if .Alerts.Firing | len }}Firing alerts: + {{ range .Alerts.Firing }}- {{ .Annotations.message }}{{ .Annotations.description }} + {{ end }}{{ end }}{{ if .Alerts.Resolved | len }}Resolved alerts: + {{ range .Alerts.Resolved }}- {{ .Annotations.message }}{{ .Annotations.description }} + {{ end }}{{ end }} + |||, + + build_slack_receiver(name, slack_channel):: + { + name: name, + slack_configs: [{ + api_url: $._config.slack_url, + channel: slack_channel, + send_resolved: true, + title: $.slackAlertTitle, + text: $.slackAlertText, + actions: [ + { + type: 'button', + text: 'Runbook :green_book:', + url: '{{ (index .Alerts 0).Annotations.runbook_url }}', + }, + { + type: 'button', + text: 'Source :information_source:', + url: '{{ (index .Alerts 0).GeneratorURL }}', + }, + { + type: 'button', + text: 'Silence :no_bell:', + url: '{{ template "__alert_silence_link" . }}', + }, + ], + }], + }, + alertmanager_config:: { - templates: ['/etc/alertmanager/*.tmpl'], + templates: [ + '/etc/alertmanager/*.tmpl', + '/etc/alertmanager/config/templates.tmpl', + ], route: { group_by: ['alertname'], receiver: 'slack', }, - receivers: [{ - name: 'slack', - slack_configs: [{ - api_url: $._config.slack_url, - channel: $._config.slack_channel, - }], - }], + receivers: [ + $.build_slack_receiver('slack', $._config.slack_channel), + ], }, local configMap = $.core.v1.configMap, @@ -40,6 +78,7 @@ configMap.new('alertmanager-config') + configMap.withData({ 'alertmanager.yml': $.util.manifestYaml($.alertmanager_config), + 'templates.tmpl': (importstr 'files/alertmanager_config.tmpl'), }) else {}, diff --git a/prometheus-ksonnet/lib/files/alertmanager_config.tmpl b/prometheus-ksonnet/lib/files/alertmanager_config.tmpl new file mode 100644 index 000000000..ddd6d329d --- /dev/null +++ b/prometheus-ksonnet/lib/files/alertmanager_config.tmpl @@ -0,0 +1,12 @@ +# This builds the silence URL. We exclude the alertname in the range +# to avoid the issue of having trailing comma separator (%2C) at the end +# of the generated URL +{{ define "__alert_silence_link" -}} + {{ .ExternalURL }}/#/silences/new?filter=%7B + {{- range .CommonLabels.SortedPairs -}} + {{- if ne .Name "alertname" -}} + {{- .Name }}%3D"{{- .Value -}}"%2C%20 + {{- end -}} + {{- end -}} + alertname%3D"{{ .CommonLabels.alertname }}"%7D +{{- end }} From 3ac303c6dc52d600aa3a0f90250db6cf9f345f1d Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Tue, 21 Apr 2020 12:01:48 +0530 Subject: [PATCH 2/4] moved slack alert title and text to template and added link for source of silence button template Signed-off-by: Sandeep Sukhani --- prometheus-ksonnet/lib/alertmanager.libsonnet | 11 ++--------- .../lib/files/alertmanager_config.tmpl | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/prometheus-ksonnet/lib/alertmanager.libsonnet b/prometheus-ksonnet/lib/alertmanager.libsonnet index 4e05364e8..005107d3e 100644 --- a/prometheus-ksonnet/lib/alertmanager.libsonnet +++ b/prometheus-ksonnet/lib/alertmanager.libsonnet @@ -17,15 +17,8 @@ ] else [], - slackAlertTitle:: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.cluster }}: {{ .GroupLabels.alertname }} ({{ .GroupLabels.namespace }})', - slackAlertText:: ||| - {{ .CommonAnnotations.summary }} - {{ if .Alerts.Firing | len }}Firing alerts: - {{ range .Alerts.Firing }}- {{ .Annotations.message }}{{ .Annotations.description }} - {{ end }}{{ end }}{{ if .Alerts.Resolved | len }}Resolved alerts: - {{ range .Alerts.Resolved }}- {{ .Annotations.message }}{{ .Annotations.description }} - {{ end }}{{ end }} - |||, + slackAlertTitle:: '{{ template "__alert_title" . }}', + slackAlertText:: {{ template "__alert_text" . }}, build_slack_receiver(name, slack_channel):: { diff --git a/prometheus-ksonnet/lib/files/alertmanager_config.tmpl b/prometheus-ksonnet/lib/files/alertmanager_config.tmpl index ddd6d329d..a44867d36 100644 --- a/prometheus-ksonnet/lib/files/alertmanager_config.tmpl +++ b/prometheus-ksonnet/lib/files/alertmanager_config.tmpl @@ -1,6 +1,19 @@ +{{ define "__alert_title" -}} + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.cluster }}: {{ .GroupLabels.alertname }} ({{ .GroupLabels.namespace }}) +{{- end }} + +{{ define "__alert_text" -}} +{{ .CommonAnnotations.summary }} +{{ if .Alerts.Firing | len }}Firing alerts: +{{ range .Alerts.Firing }}- {{ .Annotations.message }}{{ .Annotations.description }} +{{ end }}{{ end }}{{ if .Alerts.Resolved | len }}Resolved alerts: +{{ range .Alerts.Resolved }}- {{ .Annotations.message }}{{ .Annotations.description }} +{{ end }}{{ end }} +{{- end }} + # This builds the silence URL. We exclude the alertname in the range # to avoid the issue of having trailing comma separator (%2C) at the end -# of the generated URL +# of the generated URL. Inspired from https://gist.github.com/milesbxf/e2744fc90e9c41b47aa47925f8ff6512 {{ define "__alert_silence_link" -}} {{ .ExternalURL }}/#/silences/new?filter=%7B {{- range .CommonLabels.SortedPairs -}} From 71095f2bb1256c6266a106c62bfc3cb6d3c57ce3 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Tue, 21 Apr 2020 12:05:46 +0530 Subject: [PATCH 3/4] add left out change --- prometheus-ksonnet/lib/alertmanager.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus-ksonnet/lib/alertmanager.libsonnet b/prometheus-ksonnet/lib/alertmanager.libsonnet index 005107d3e..1e5384148 100644 --- a/prometheus-ksonnet/lib/alertmanager.libsonnet +++ b/prometheus-ksonnet/lib/alertmanager.libsonnet @@ -18,7 +18,7 @@ else [], slackAlertTitle:: '{{ template "__alert_title" . }}', - slackAlertText:: {{ template "__alert_text" . }}, + slackAlertText:: '{{ template "__alert_text" . }}', build_slack_receiver(name, slack_channel):: { From 06a6a42f9e5f021d05c25f267d918b58d6997c88 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Wed, 22 Apr 2020 11:37:57 +0530 Subject: [PATCH 4/4] using slack alert title and text template directly while building receiver --- prometheus-ksonnet/lib/alertmanager.libsonnet | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/prometheus-ksonnet/lib/alertmanager.libsonnet b/prometheus-ksonnet/lib/alertmanager.libsonnet index 1e5384148..7cae51071 100644 --- a/prometheus-ksonnet/lib/alertmanager.libsonnet +++ b/prometheus-ksonnet/lib/alertmanager.libsonnet @@ -17,9 +17,6 @@ ] else [], - slackAlertTitle:: '{{ template "__alert_title" . }}', - slackAlertText:: '{{ template "__alert_text" . }}', - build_slack_receiver(name, slack_channel):: { name: name, @@ -27,8 +24,8 @@ api_url: $._config.slack_url, channel: slack_channel, send_resolved: true, - title: $.slackAlertTitle, - text: $.slackAlertText, + title: '{{ template "__alert_title" . }}', + text: '{{ template "__alert_text" . }}', actions: [ { type: 'button',