hosts/monitoring-3: Warn if storages are almost full

hosts/monitoring-3/alertmanager.nix

@@ -63,6 +63,18 @@
             "instance"
           ];
         }
+        {
+          target_matchers = [
+            ''alertname = "StorageAlmostFull"''
+          ];
+          source_matchers = [
+            ''alertname = "StorageFull"''
+          ];
+          equal = [
+            "instance"
+            "mountpoint"
+          ];
+        }
       ];
     };
   };

hosts/monitoring-3/rules.yml

@@ -17,7 +17,7 @@ groups:
     annotations:
       summary: "Current system of {{ $labels.instance }} not in sync with config"
       description: "The current system hash of {{ $labels.instance }} does not match the one generated by hydra based on the current config"
-  - alert: BackupStorageFull
+  - alert: StorageFull
     expr: ((last_over_time(node_filesystem_avail_bytes{job="node-exporter"}[5m]) / last_over_time(node_filesystem_size_bytes{job="node-exporter"}[5m])) * 100) < 5
     for: 30m
     labels:
@@ -25,6 +25,14 @@ groups:
     annotations:
       summary: "Storage of {{ $labels.instance }} is full"
       description: "Storage of {{ $labels.instance }} for {{ $labels.mountpoint }} on {{ $labels.device }} is full"
+  - alert: StorageAlmostFull
+    expr: ((last_over_time(node_filesystem_avail_bytes{job="node-exporter"}[5m]) / last_over_time(node_filesystem_size_bytes{job="node-exporter"}[5m])) * 100) < 10
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Storage of {{ $labels.instance }} is almost full"
+      description: "Storage of {{ $labels.instance }} for {{ $labels.mountpoint }} on {{ $labels.device }} is almost full"
   - alert: ClerieBackupJobLastSuccessfulRunBehind
     expr: time() - last_over_time(clerie_backup_last_successful_run_time{}[5m]) >= 9000
     for: 5m