Introduce service levels and change alert routing based on this

2023-01-05 23:16:50 +01:00
parent 30e22dff8d
commit cfd746fddb
5 changed files with 53 additions and 2 deletions
--- a/hosts/monitoring-3/rules.yml
+++ b/hosts/monitoring-3/rules.yml
@@ -31,3 +31,12 @@ groups:
    annotations:
      summary: "Uptime of {{ $labels.instance }} less than 5 min"
      description: "{{ $labels.instance }} just booted"
+  - alert: EventInstanceDown
+    expr: up{job="node-exporter", service_level="event"} == 0
+    for: 2h
+    labels:
+      mute: instance
+      severity: muted
+    annotations:
+      summary: "Event instance {{ $labels.instance }} down"
+      description: "{{ $labels.instance }} has been down for more than 2 hours."