hosts/monitoring-3: alert on averaged metrics

flake.lock

@@ -43,6 +43,42 @@
         "type": "github"
       }
     },
+    "fernglas": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1680199136,
+        "narHash": "sha256-dlPzsVkEjhQjrV0++UOX+eaOaO1FTSNoHQ2JHQz9qcA=",
+        "owner": "wobcom",
+        "repo": "fernglas",
+        "rev": "71766d09dc6a43c2219d57c6372a1b5934b61a99",
+        "type": "github"
+      },
+      "original": {
+        "owner": "wobcom",
+        "repo": "fernglas",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nixos-exporter": {
       "inputs": {
         "nixpkgs": [
@@ -98,6 +134,7 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "fernglas": "fernglas",
         "nixos-exporter": "nixos-exporter",
         "nixpkgs": "nixpkgs",
         "nixpkgs-schule": "nixpkgs-schule",

flake.nix

@@ -6,6 +6,10 @@
       url = "github:ryantm/agenix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    fernglas = {
+      url = "github:wobcom/fernglas";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nixos-exporter = {
       url = "git+https://git.clerie.de/clerie/nixos-exporter.git";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -15,7 +19,7 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };
-  outputs = { self, nixpkgs, nixos-exporter, solid-xmpp-alarm, ... }@inputs: let
+  outputs = { self, nixpkgs, fernglas, nixos-exporter, solid-xmpp-alarm, ... }@inputs: let
     helper = (import ./lib/flake-helper.nix) inputs;
   in {
     clerie.hosts = {

hosts/dn42-ildix-clerie/configuration.nix

@@ -120,21 +120,19 @@
   # Internal
   protocol bgp peer_2953_dn42_ildix_service {
     local as 4242422953;
-    graceful restart on;
-    neighbor fd81:edb3:71d8:ffff:2953::1 as 4242422953;
+    neighbor fd81:edb3:71d8:ffff:2953::1 port 1179 as 4242422953;
     source address fd81:edb3:71d8:ffff::13;
+    multihop 64;
+    rr client;
     ipv6 {
       table master6;
       next hop keep;
-      import keep filtered;
+      add paths tx;
       import filter {
-        if net ~ [fd81:edb3:71d8::/48{48,128}] then bgp_path.prepend(4242422953);
-        if net ~ [fd81:edb3:71d8::/48{48,64}] then accept;
         reject;
       };
       export filter {
-        if net ~ [fd00::/8{8,64}] then accept;
-        reject;
+        accept;
       };
     };
   }

hosts/dn42-ildix-service/configuration.nix

@@ -6,6 +6,7 @@
       ./hardware-configuration.nix
       ../../configuration/proxmox-vm
       ./bird.nix
+      ./fernglas.nix
     ];
 
   # Use the GRUB 2 boot loader.
@@ -36,6 +37,10 @@
   networking.defaultGateway = { address = "192.168.10.1"; interface = "ens18"; };
   networking.nameservers = [ "2001:638:904:ffcc::3" "2001:638:904:ffcc::4" "141.24.40.3" "141.24.40.4" ];
 
+  services.nginx.enable = true;
+
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+
   system.stateVersion = "23.05";
 }
 

hosts/dn42-ildix-service/fernglas.nix

@@ -0,0 +1,37 @@
+{ config, lib, inputs, ... }:
+
+{
+  networking.firewall.allowedTCPPorts = [ 3000 1179 ];
+
+  services.fernglas = {
+    enable = true;
+    settings = {
+      api.bind = "[::1]:3000";
+      collectors = [
+        {
+          collector_type = "Bgp";
+          bind = "[::]:1179";
+          default_peer_config = {
+            asn = 4242422953;
+            router_id = "${(lib.head config.networking.interfaces.ens18.ipv4.addresses).address}";
+            route_state = "Accepted";
+            add_path = true;
+          };
+        }
+      ];
+    };
+  };
+
+  services.nginx.virtualHosts = {
+    "lg.ildix.clerie.de" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        root = inputs.fernglas.packages."x86_64-linux"."fernglas-frontend";
+      };
+      locations."/api/" = {
+        proxyPass = "http://${config.services.fernglas.settings.api.bind}";
+      };
+    };
+  };
+}

hosts/monitoring-3/rules.yml

@@ -18,7 +18,7 @@ groups:
       summary: "Current system of {{ $labels.instance }} not in sync with config"
       description: "The current system hash of {{ $labels.instance }} does not match the one generated by hydra based on the current config"
   - alert: BackupStorageFull
-    expr: ((node_filesystem_avail_bytes{job="node-exporter"} / node_filesystem_size_bytes{job="node-exporter"}) * 100) < 5
+    expr: ((avg_over_time(node_filesystem_avail_bytes{job="node-exporter"}[5m]) / avg_over_time(node_filesystem_size_bytes{job="node-exporter"}[5m])) * 100) < 5
     for: 30m
     labels:
       severity: critical
@@ -42,7 +42,7 @@ groups:
       summary: "Event instance {{ $labels.instance }} down"
       description: "{{ $labels.instance }} has been down for more than 2 hours."
   - alert: KernelChanged
-    expr: nixos_current_system_kernel_is_booted_system_kernel{job="nixos-exporter"} == 0
+    expr: avg_over_time(nixos_current_system_kernel_is_booted_system_kernel{job="nixos-exporter"}[5m]) == 0
     for: 2h
     labels:
       severity: warning

lib/flake-helper.nix

@@ -1,4 +1,4 @@
-{ self, nixpkgs, agenix, nixos-exporter, solid-xmpp-alarm, ... }@inputs:
+{ self, nixpkgs, agenix, fernglas, nixos-exporter, solid-xmpp-alarm, ... }@inputs:
 
 rec {
   generateNixosSystem = {
@@ -16,7 +16,10 @@ rec {
           Useful for having the monitoring server scraping the
           target config from all other servers automatically.
         */
-        _module.args._nixfiles = self;
+        _module.args = {
+          inputs = inputs;
+          _nixfiles = self;
+        };
       })
       ../configuration/common
       ../users/clerie
@@ -32,6 +35,7 @@ rec {
         clerie.monitoring = nixpkgs.lib.attrsets.optionalAttrs (group != null) { serviceLevel = group; };
       })
       agenix.nixosModules.default
+      fernglas.nixosModules.default
       solid-xmpp-alarm.nixosModules.solid-xmpp-alarm
       (../hosts + "/${name}/configuration.nix")
       # Automatically load secrets from the hosts secrets directory