From cff95863fdb00a00a2b658d05f61a1bbbed013a3 Mon Sep 17 00:00:00 2001
From: clerie <git@clerie.de>
Date: Fri, 1 Dec 2023 17:50:54 +0100
Subject: [PATCH] hosts/monitoring-3: Add alert for Synapse

---
 hosts/monitoring-3/prometheus.nix | 5 +++--
 hosts/monitoring-3/rules.yml      | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/hosts/monitoring-3/prometheus.nix b/hosts/monitoring-3/prometheus.nix
index e5b7286..d8ca1d9 100644
--- a/hosts/monitoring-3/prometheus.nix
+++ b/hosts/monitoring-3/prometheus.nix
@@ -273,7 +273,7 @@ in {
         static_configs = [
           {
             targets = [
-              "https://matrix.entr0py.de/_matrix/static/"
+              "matrix.entr0py.de"
             ];
           }
         ];
@@ -281,9 +281,10 @@ in {
           {
             source_labels = [ "__address__" ];
             target_label = "__param_target";
+            replacement = "https://\${1}/_matrix/static/";
           }
           {
-            source_labels = [ "__param_target" ];
+            source_labels = [ "__address__" ];
             target_label = "target";
           }
           {
diff --git a/hosts/monitoring-3/rules.yml b/hosts/monitoring-3/rules.yml
index e0b3cd5..ae906a7 100644
--- a/hosts/monitoring-3/rules.yml
+++ b/hosts/monitoring-3/rules.yml
@@ -57,3 +57,11 @@ groups:
     annotations:
       summary: "Kernel of {{ $labels.instance }} changed"
       description: "The Kernel {{ $labels.instance }} booted with, isn't the target Kernel. A reboot may be required."
+  - alert: SynapseUnavailable
+    expr: last_over_time(probe_success{instance="monitoring-3.net.clerie.de", job="blackbox_local_synapse", target="matrix.entr0py.de"}[5m]) == 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Synapse of {{ $labels.target }} unavailable"
+      description: "The Synapse backend of {{ $labels.instance }} is unreachable or returns garbage"