Add alerts and dashboards for the new replication metric

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

Author: kakkoyun

examples/alerts/alerts.md

@@ -412,16 +412,18 @@ rules:
   for: 10m
   labels:
     severity: critical
-- alert: ThanosReceiveHighForwardRequestFailures
+- alert: ThanosReceiveHighReplicationFailures
   annotations:
-    message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize
+    message: Thanos Receive {{$labels.job}} is failing to replicate {{ $value | humanize
       }}% of requests.
   expr: |
+    thanos_receive_replication_factor > 1
+      and
     (
       (
-        sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m]))
+        sum by (job) (rate(thanos_receive_replications_total{result="error", job=~"thanos-receive.*"}[5m]))
       /
-        sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m]))
+        sum by (job) (rate(thanos_receive_replications_total{job=~"thanos-receive.*"}[5m]))
       )
       >
       (
@@ -433,6 +435,19 @@ rules:
   for: 5m
   labels:
     severity: warning
+- alert: ThanosReceiveHighForwardRequestFailures
+  annotations:
+    message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize
+      }}% of requests.
+  expr: |
+    (
+      sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m]))
+    /
+      sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m]))
+    ) * 100 > 20
+  for: 5m
+  labels:
+    severity: warning
 - alert: ThanosReceiveHighHashringFileRefreshFailures
   annotations:
     message: Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{

examples/alerts/alerts.yaml

@@ -173,16 +173,18 @@ groups:
     for: 10m
     labels:
       severity: critical
-  - alert: ThanosReceiveHighForwardRequestFailures
+  - alert: ThanosReceiveHighReplicationFailures
     annotations:
-      message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize
-        }}% of requests.
+      message: Thanos Receive {{$labels.job}} is failing to replicate {{ $value |
+        humanize }}% of requests.
     expr: |
+      thanos_receive_replication_factor > 1
+        and
       (
         (
-          sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m]))
+          sum by (job) (rate(thanos_receive_replications_total{result="error", job=~"thanos-receive.*"}[5m]))
         /
-          sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m]))
+          sum by (job) (rate(thanos_receive_replications_total{job=~"thanos-receive.*"}[5m]))
         )
         >
         (
@@ -194,6 +196,19 @@ groups:
     for: 5m
     labels:
       severity: warning
+  - alert: ThanosReceiveHighForwardRequestFailures
+    annotations:
+      message: Thanos Receive {{$labels.job}} is failing to forward {{ $value | humanize
+        }}% of requests.
+    expr: |
+      (
+        sum by (job) (rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m]))
+      /
+        sum by (job) (rate(thanos_receive_forward_requests_total{job=~"thanos-receive.*"}[5m]))
+      ) * 100 > 20
+    for: 5m
+    labels:
+      severity: warning
   - alert: ThanosReceiveHighHashringFileRefreshFailures
     annotations:
       message: Thanos Receive {{$labels.job}} is failing to refresh hashring file,

examples/alerts/rules.yaml

@@ -72,6 +72,14 @@ groups:
     labels:
       quantile: "0.99"
     record: :http_request_duration_seconds:histogram_quantile
+  - expr: |
+      (
+        sum(rate(thanos_receive_replications_total{result="error", job=~"thanos-receive.*"}[5m]))
+      /
+        sum(rate(thanos_receive_replications_total{job=~"thanos-receive.*"}[5m]))
+      )
+    labels: {}
+    record: :thanos_receive_replication_failure_per_requests:sum_rate
   - expr: |
       (
         sum(rate(thanos_receive_forward_requests_total{result="error", job=~"thanos-receive.*"}[5m]))