hosts/monitoring-3: Monitor alertmanager

hosts/monitoring-3/prometheus.nix

@@ -104,6 +104,21 @@ in {
           relabelAddressToInstance
         ];
       }
+      {
+        job_name = "alertmanager";
+        scrape_interval = "20s";
+        scheme = "http";
+        static_configs = [
+          {
+            targets = [
+              "monitoring-3.mon.clerie.de:9093"
+            ];
+          }
+        ];
+        relabel_configs = [
+          relabelAddressToInstance
+        ];
+      }
       {
         job_name = "node-exporter";
         scrape_interval = "20s";
@@ -526,7 +541,7 @@ in {
       {
         static_configs = [ {
           targets = [
-            "[::1]:9093"
+            "monitoring-3.mon.clerie.de:9093"
           ];
         } ];
       }

hosts/monitoring-3/rules.yml

@@ -95,3 +95,10 @@ groups:
     annotations:
       summary: "blog.nadja.top unreachable via IPv4"
       description: "blog.nadja.top unreachable IPv4, but reachable via IPv6"
+  - alert: AlertmanagerNotificationRequestsFailed
+    expr: rate(alertmanager_notification_requests_failed_total[5m]) > 0
+    labels:
+      severity: critical
+    annotations:
+      summary: "Too many notification requests failed"
+      description: "Too many notification requests to Alertmanager integration {{ $labels.integration }} failed"