From 38567829f13ab3b10ac604c363071437007a4e7a Mon Sep 17 00:00:00 2001 From: clerie Date: Sat, 4 Feb 2023 01:15:07 +0100 Subject: [PATCH] hosts/monitoring-3: alert on out of sync host system --- hosts/monitoring-3/rules.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hosts/monitoring-3/rules.yml b/hosts/monitoring-3/rules.yml index 3982832..26d45b2 100644 --- a/hosts/monitoring-3/rules.yml +++ b/hosts/monitoring-3/rules.yml @@ -16,6 +16,14 @@ groups: annotations: summary: "Uptime of {{ $labels.instance }} more than 90 days" description: "The last reboot of {{ $labels.instance }} was more than 90 days ago" + - alert: InstanceConfigNotSync + expr: nixos_current_system_is_sync{} == 0 + for: 2h + labels: + severity: warning + annotations: + summary: "Current system of {{ $labels.instance }} not in sync with config" + description: "The current system hash of {{ $labels.instance }} does not match the one generated by hydra based on the current config" - alert: BackupStorageFull expr: ((node_filesystem_avail_bytes{instance="clerie-backup.mon.clerie.de:9100", job="node-exporter", mountpoint="/mnt/clerie-backup"} / node_filesystem_size_bytes{instance="clerie-backup.mon.clerie.de:9100", job="node-exporter", mountpoint="/mnt/clerie-backup"}) * 100) < 5 for: 30m