nixfiles/hosts/monitoring-3/configuration.nix

{ config, pkgs, lib, ... }:

with lib;

let
  hosts = (import ../../lib/hosts.nix { inherit pkgs; }).hosts;

  monitoringHosts = filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "enable"] false host.config)
    hosts;

  monitoringHostsNames = mapAttrs' (name: host:
    nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}"  ["${host.config.networking.hostName}.mon.clerie.de"])
    monitoringHosts;

  monitoringPeers = mapAttrsToList (name: host: {
      allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ];
      publicKey = host.config.clerie.monitoring.pubkey;
    })
    monitoringHosts;

  monitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}")
    monitoringHosts;

  nixosMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9152")
    (filterAttrs (name: host:
    # assume this is a NixOS system if not specified
    attrByPath ["clerie" "monitoring" "nixos"] true host.config)
    monitoringHosts);

  birdMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9324")
    (filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "bird"] false host.config)
    monitoringHosts);

  blackboxMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9115")
    (filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
    monitoringHosts);

  eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));

in {
  imports =
    [
      ./hardware-configuration.nix
      ../../configuration/common
      ../../configuration/proxmox-vm
    ];

  boot.loader.grub.enable = true;
  boot.loader.grub.version = 2;
  boot.loader.grub.device = "/dev/sda";

  networking.hostName = "monitoring-3";

  networking.useDHCP = false;
  networking.interfaces.ens18.ipv4.addresses = [ { address = "192.168.10.32"; prefixLength = 24; } ];
  networking.interfaces.ens19.ipv6.addresses = [ { address = "2001:638:904:ffca::7"; prefixLength = 64; } ];
  networking.defaultGateway = { address = "192.168.10.1"; interface = "ens18"; };
  networking.defaultGateway6 = { address = "2001:638:904:ffca::1"; interface = "ens19"; };
  networking.nameservers = [ "2001:638:904:ffcc::3" "2001:638:904:ffcc::4" "141.24.40.3" "141.24.40.4" ];

  networking.hosts = {
    "::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1
  }
  // monitoringHostsNames;

  networking.wireguard.enable = true;
  networking.wireguard.interfaces = {
    wg-monitoring = {
      ips = [ "fd00:327:327:327::1/64" ];
      listenPort = 54523;
      peers = monitoringPeers;
      privateKeyFile = "/var/src/secrets/wireguard/wg-monitoring";
    };
  };

  networking.firewall.allowedUDPPorts = [ 54523 ];

  services.prometheus.exporters.node.enable = true;

  services.prometheus.xmpp-alerts = {
    enable = true;
    settings = {
      jid = "feuer@fem-net.de";
      password_command = "cat /var/src/secrets/xmpp-alert/password";
      listen_address = "::1";
      listen_port = 9199;
      to_jid = "clerie@fem-net.de";
      format = "short";
    };
  };

  services.prometheus.alertmanager = {
    enable = true;
    listenAddress = "[::1]";
    port = 9093;
    configuration = {
      route = {
        receiver = "xmpp-receiver";
        repeat_interval = "4h";
        group_by = [ "instance" ];
        routes = [
          {
            matchers = [
              ''severity = "muted"''
            ];
            receiver = "muted";
          }
          {
            receiver = "xmpp-receiver";
            matchers = [
              ''severity = "warning"''
            ];
            repeat_interval = "70h";
          }
        ];
      };
      receivers = [
        {
          name = "muted";
        }
        {
          name = "xmpp-receiver";
          webhook_configs = [
            {
              url = "http://[::1]:9199/alert";
            }
          ];
        }
      ];
      inhibit_rules = [
        {
          # Mute all alerts for an instance which also has an alert starting with MuteInstanceOn
          target_matchers = [
            ''alertname =~ ".+"''
          ];
          source_matchers = [
            ''mute = "instance"''
          ];
          equal = [
            "instance"
          ];
        }
      ];
    };
  };

  services.prometheus = {
    enable = true;
    listenAddress = "[::1]";
    scrapeConfigs = let
      relabelAddressToInstance = {
        source_labels = [ "__address__" ];
        target_label = "instance";
        regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
        replacement = "\${1}.net.clerie.de";
      };
    in [
      {
        job_name = "prometheus";
        scrape_interval = "20s";
        scheme = "http";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9090"
            ];
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "node-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9100;infra"
            ]
            ++ monitoringTargets;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "service_level";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "nixos-exporter";
        scrape_interval = "1m";
        static_configs = [
          {
            targets = nixosMonitoringTargets;
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "bird-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = birdMonitoringTargets;
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_icmp6";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp6" ];
        };
        static_configs = [
          {
            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
              "clerie.de"
              "tagesschau.de"
              "google.com"
              "achtbaan.nikhef.nl"
              "fluorine.net.clerie.de"
              "www.fem.tu-ilmenau.de"
              "www.heise.de"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__param_target";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "target";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_icmp4";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp4" ];
        };
        static_configs = [
          {
            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
              "clerie.de"
              "tagesschau.de"
              "google.com"
              "achtbaan.nikhef.nl"
              "www.fem.tu-ilmenau.de"
              "www.heise.de"
              "ie10-ffm2.nodes.nethinks.com"
              "matrix.bau-ha.us"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__param_target";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "target";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "zimmer-temp";
        scrape_interval = "20s";
        scheme = "https";
        metrics_path = "/data/zimmer-temp/";
        static_configs = [
          {
            targets = [
              "iot-data.clerie.de"
            ];
          }
        ];
      }
      {
        job_name = "outdoor-temp";
        scrape_interval = "20s";
        scheme = "https";
        metrics_path = "/data/outdoor-temp/";
        static_configs = [
          {
            targets = [
              "iot-data.clerie.de"
            ];
          }
        ];
      }
      {
        job_name = "xmpp-alerts";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9199"
            ];
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
    ];
    alertmanagers = [
      {
        static_configs = [ {
          targets = [
            "[::1]:9093"
          ];
        } ];
      }
    ];
    rules = [ (readFile ./rules.yml) ];
  };

  services.grafana = {
    enable  = true;
    domain  = "grafana.monitoring.clerie.de";
    rootUrl = "https://grafana.monitoring.clerie.de";
    port    = 3001;
    addr = "::1";
    auth.anonymous.enable = true;

    provision = {
      enable = true;
      datasources = [
        {
          type = "prometheus";
          name = "Prometheus";
          url = "http://[::1]:9090";
          isDefault = true;
        }
      ];
      dashboards = [
        {
          options.path = ./dashboards;
        }
      ];
    };
  };

  users.users.uptimestatus = {
    description = "Uptime Status Service";
    group = "uptimestatus";
    home = "/var/lib/uptimestatus/";
    useDefaultShell = true;
    isSystemUser = true;
  };
  users.groups.uptimestatus = {};

  systemd.services.uptimestatus = {
    wantedBy = [ "multi-user.target" ];
    serviceConfig = {
      RuntimeDirectory = "uptimestatus";
      StateDirectory = "uptimestatus";
      User = "uptimestatus";
      Group = "uptimestatus";
    };
    script = "gunicorn -w 4 -b [::1]:8235 uptimestatus:app";
    path = with pkgs; [ (python3.withPackages (ps: [ ps.gunicorn uptimestatus ])) ];
  };

  services.nginx = {
    enable = true;

    virtualHosts = {
      "prometheus.monitoring.clerie.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:9090/";
      };
      "grafana.monitoring.clerie.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:3001/";
      };
      "status.monitoring.clerie.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:8235/";
      };
    };
  };

  networking.firewall.allowedTCPPorts = [ 80 443 ];

  system.stateVersion = "21.03";
}