From be5b1c1baf34607c3ca3425e2d0104074f28dbc1 Mon Sep 17 00:00:00 2001
From: clerie <git@clerie.de>
Date: Mon, 31 Oct 2022 22:54:06 +0100
Subject: [PATCH] hosts/monitoring-3: move to blackbox monitoring

---
 hosts/gatekeeper/configuration.nix   |   1 +
 hosts/monitoring-3/configuration.nix | 119 +++++++++++++++++++++------
 hosts/porter/configuration.nix       |   1 +
 modules/monitoring/default.nix       |  20 +++++
 4 files changed, 117 insertions(+), 24 deletions(-)

diff --git a/hosts/gatekeeper/configuration.nix b/hosts/gatekeeper/configuration.nix
index e0f979e..507a9e7 100644
--- a/hosts/gatekeeper/configuration.nix
+++ b/hosts/gatekeeper/configuration.nix
@@ -123,6 +123,7 @@
     enable = true;
     id = "101";
     pubkey = "H9Pvx/BzwEMM7acT9mioT8zBD2Yn13L82EKKqdAfeGM=";
+    blackbox = true;
   };
 
   system.stateVersion = "21.03";
diff --git a/hosts/monitoring-3/configuration.nix b/hosts/monitoring-3/configuration.nix
index 826536b..4ce1595 100644
--- a/hosts/monitoring-3/configuration.nix
+++ b/hosts/monitoring-3/configuration.nix
@@ -29,6 +29,14 @@ let
     attrByPath ["clerie" "monitoring" "bird"] false host.config)
     monitoringHosts);
 
+  blackboxMonitoringTargets = mapAttrsToList (name: host:
+    "${host.config.networking.hostName}.mon.clerie.de:9115")
+    (filterAttrs (name: host:
+    attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
+    monitoringHosts);
+
+  eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
+
 in {
   imports =
     [
@@ -68,19 +76,6 @@ in {
   networking.firewall.allowedUDPPorts = [ 54523 ];
 
   services.prometheus.exporters.node.enable = true;
-  services.prometheus.exporters.smokeping = {
-    enable = true;
-    listenAddress = "[::1]";
-    port = 9374;
-    hosts = [
-      "clerie.de"
-      "fluorine.net.clerie.de"
-      "www.fem.tu-ilmenau.de"
-      "www.heise.de"
-      "212.218.154.100" # ie10-ffm1
-      "matrix.bau-ha.us"
-    ];
-  };
 
   services.prometheus.xmpp-alerts = {
     enable = true;
@@ -163,6 +158,93 @@ in {
           }
         ];
       }
+      {
+        job_name = "blackbox_icmp6";
+        scrape_interval = "20s";
+        metrics_path = "/probe";
+        params = {
+          module = [ "icmp6" ];
+        };
+        static_configs = [
+          {
+            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
+              "clerie.de"
+              "tagesschau.de"
+              "google.com"
+              "achtbaan.nikhef.nl"
+              "fluorine.net.clerie.de"
+              "www.fem.tu-ilmenau.de"
+              "www.heise.de"
+            ];
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            regex = "(.+);(.+)";
+            target_label = "__param_target";
+            replacement = "\${2}";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "target";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "(.+);(.+)";
+            target_label = "__address__";
+            replacement = "\${1}";
+          }
+          {
+            source_labels = [ "__address__" ];
+            target_label = "instance";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox_icmp4";
+        scrape_interval = "20s";
+        metrics_path = "/probe";
+        params = {
+          module = [ "icmp4" ];
+        };
+        static_configs = [
+          {
+            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
+              "clerie.de"
+              "tagesschau.de"
+              "google.com"
+              "achtbaan.nikhef.nl"
+              "www.fem.tu-ilmenau.de"
+              "www.heise.de"
+              "ie10-ffm2.nodes.nethinks.com"
+              "matrix.bau-ha.us"
+            ];
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            regex = "(.+);(.+)";
+            target_label = "__param_target";
+            replacement = "\${2}";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "target";
+          }
+          {
+            source_labels = [ "__address__" ];
+            regex = "(.+);(.+)";
+            target_label = "__address__";
+            replacement = "\${1}";
+          }
+          {
+            source_labels = [ "__address__" ];
+            target_label = "instance";
+          }
+        ];
+      }
       {
         job_name = "zimmer-temp";
         scrape_interval = "20s";
@@ -200,17 +282,6 @@ in {
           }
         ];
       }
-      {
-        job_name = "smokeping";
-        scrape_interval = "20s";
-        static_configs = [
-          {
-            targets = [
-              "monitoring-3.mon.clerie.de:9374"
-            ];
-          }
-        ];
-      }
     ];
     alertmanagers = [
       {
diff --git a/hosts/porter/configuration.nix b/hosts/porter/configuration.nix
index b54f1aa..81df9f9 100644
--- a/hosts/porter/configuration.nix
+++ b/hosts/porter/configuration.nix
@@ -58,6 +58,7 @@
     enable = true;
     id = "102";
     pubkey = "+mJN+ustPo2ehP0wqajYs3nTdJ0SPuIDyiZQSHFIK3o=";
+    blackbox = true;
   };
 
   system.stateVersion = "21.03";
diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix
index 3e34324..d610adf 100644
--- a/modules/monitoring/default.nix
+++ b/modules/monitoring/default.nix
@@ -22,6 +22,7 @@ in
         description = "Public Key of the monitoring wireguard interface of this host";
       };
       bird = mkEnableOption "Monitor bird";
+      blackbox = mkEnableOption "Monitor blackbox";
     };
   };
 
@@ -54,5 +55,24 @@ in
       openFirewall = true;
       firewallFilter = "-i wg-monitoring -p tcp -m tcp --dport 9324";
     };
+
+    services.prometheus.exporters.blackbox = mkIf cfg.blackbox {
+      enable = true;
+      openFirewall = true;
+      firewallFilter = "-i wg-monitoring -p tcp -m tcp --dport 9115";
+      configFile = pkgs.writeText "blackbox.yml" ''
+        modules:
+          icmp6:
+            prober: icmp
+            icmp:
+              preferred_ip_protocol: ip6
+              ip_protocol_fallback: false
+          icmp4:
+            prober: icmp
+            icmp:
+              preferred_ip_protocol: ip4
+              ip_protocol_fallback: false
+      '';
+    };
   };
 }