From 504f5d610ade4ab6e4f97753271ac0de008cfea2 Mon Sep 17 00:00:00 2001 From: v-zhuravlev Date: Mon, 3 Jun 2024 20:13:39 +0800 Subject: [PATCH 1/2] Add NodeSystemdServiceCrashlooping alert Signed-off-by: Vitaly Zhuravlev --- docs/node-mixin/alerts/alerts.libsonnet | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index c346480906..4ed6e7f30b 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -407,6 +407,20 @@ description: 'Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}', }, }, + { + alert: 'NodeSystemdServiceCrashlooping', + expr: ||| + increase(node_systemd_service_restart_total{%(filteringSelector)s}[5m]) > 2 + ||| % this.config, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Systemd service keeps restaring, possibly crash looping.', + description: 'Systemd service {{ $labels.name }} has being restarted too many times at {{ $labels.instance }} for the last 15 minutes. Please check if service is crash looping.', + }, + }, { alert: 'NodeBondingDegraded', expr: ||| From c753942b7b077a6a9cc5bb3018d412f2d6aed09a Mon Sep 17 00:00:00 2001 From: Vitaly Zhuravlev Date: Mon, 3 Jun 2024 12:19:38 +0000 Subject: [PATCH 2/2] Fix alert Signed-off-by: Vitaly Zhuravlev --- docs/node-mixin/alerts/alerts.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 4ed6e7f30b..a0fa7a4f9b 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -410,8 +410,8 @@ { alert: 'NodeSystemdServiceCrashlooping', expr: ||| - increase(node_systemd_service_restart_total{%(filteringSelector)s}[5m]) > 2 - ||| % this.config, + increase(node_systemd_service_restart_total{%(nodeExporterSelector)s}[5m]) > 2 + ||| % $._config, 'for': '15m', labels: { severity: 'warning',