Add alerting to monitoring

hosts/monitoring-3/configuration.nix

@@ -63,6 +63,40 @@ in {
 
   services.prometheus.exporters.node.enable = true;
 
+  services.prometheus.xmpp-alerts = {
+    enable = true;
+    settings = {
+      jid = "feuer@fem-net.de";
+      password_command = "cat /var/src/secrets/xmpp-alert/password";
+      listen_address = "::1";
+      listen_port = 9199;
+      to_jid = "clerie@fem-net.de";
+      format = "short";
+    };
+  };
+
+  services.prometheus.alertmanager = {
+    enable = true;
+    listenAddress = "[::1]";
+    port = 9093;
+    configuration = {
+      route = {
+        receiver = "xmpp-receiver";
+        group_by = [ "instance" ];
+      };
+      receivers = [
+        {
+          name = "xmpp-receiver";
+          webhook_configs = [
+            {
+              url = "http://[::1]:9199/alert";
+            }
+          ];
+        }
+      ];
+    };
+  };
+
   services.prometheus = {
     enable = true;
     listenAddress = "[::1]";
@@ -118,6 +152,28 @@ in {
         ];
       }
     ];
+    alertmanagers = [
+      {
+        static_configs = [ {
+          targets = [
+            "[::1]:9093"
+          ];
+        } ];
+      }
+    ];
+    rules = [ ''
+      groups:
+      - name: alert.rules
+        rules:
+        - alert: InstanceDown
+          expr: up == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Endpoint {{ $labels.instance }} down"
+            description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."
+    '' ];
   };
 
   services.grafana = {