Improve monitoring rules

hosts/monitoring-3/configuration.nix

@@ -63,6 +63,40 @@ in {
 
   services.prometheus.exporters.node.enable = true;
 
+  services.prometheus.xmpp-alerts = {
+    enable = true;
+    settings = {
+      jid = "feuer@fem-net.de";
+      password_command = "cat /var/src/secrets/xmpp-alert/password";
+      listen_address = "::1";
+      listen_port = 9199;
+      to_jid = "clerie@fem-net.de";
+      format = "short";
+    };
+  };
+
+  services.prometheus.alertmanager = {
+    enable = true;
+    listenAddress = "[::1]";
+    port = 9093;
+    configuration = {
+      route = {
+        receiver = "xmpp-receiver";
+        group_by = [ "instance" ];
+      };
+      receivers = [
+        {
+          name = "xmpp-receiver";
+          webhook_configs = [
+            {
+              url = "http://[::1]:9199/alert";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
   services.prometheus = {
     enable = true;
     listenAddress = "[::1]";
@@ -118,6 +152,16 @@ in {
         ];
       }
     ];
+    alertmanagers = [
+      {
+        static_configs = [ {
+          targets = [
+            "[::1]:9093"
+          ];
+        } ];
+      }
+    ];
+    rules = [ (readFile ./rules.yml) ];
   };
 
   services.grafana = {

hosts/monitoring-3/rules.yml

@@ -0,0 +1,11 @@
+groups:
+- name: alert.rules
+  rules:
+  - alert: InstanceDown
+    expr: up{job="node-exporter"} == 0
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Endpoint {{ $labels.instance }} down"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."