nixfiles/hosts/monitoring-3/prometheus.nix

{ config, pkgs, lib, _nixfiles, ... }:

with lib;

let
  hosts = _nixfiles.nixosConfigurations // {
    "mercury".config = {
      networking.hostName = "mercury";
      clerie.monitoring = {
        enable = true;
        id = "401";
        pubkey = "jrwc4gMZh/Ss1MEXWkLhXcAp4Z++Xnl4noYIZnCfWQg=";
        nixos = false;
      };
    };
  };

  monitoringHosts = filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "enable"] false host.config)
    hosts;

  monitoringHostsNames = mapAttrs' (name: host:
    nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}"  ["${host.config.networking.hostName}.mon.clerie.de"])
    monitoringHosts;

  monitoringPeers = mapAttrsToList (name: host: {
      allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ];
      publicKey = host.config.clerie.monitoring.pubkey;
    })
    monitoringHosts;

  monitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}")
    monitoringHosts;

  nixosMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9152")
    (filterAttrs (name: host:
    # assume this is a NixOS system if not specified
    attrByPath ["clerie" "monitoring" "nixos"] true host.config)
    monitoringHosts);

  birdMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9324")
    (filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "bird"] false host.config)
    monitoringHosts);

  blackboxMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9115")
    (filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
    monitoringHosts);

  nginxlogMonitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.clerie.de:9117")
    (filterAttrs (name: host:
    attrByPath ["services" "prometheus" "exporters" "nginxlog" "enable"] false host.config)
    monitoringHosts);

  eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));

in {
  sops.secrets.uberspace-monitor-password = {
    owner = "prometheus";
    group = "prometheus";
  };

  networking.hosts = {
    "::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1
  }
  // monitoringHostsNames;

  networking.wireguard.enable = true;
  networking.wireguard.interfaces = {
    wg-monitoring = {
      ips = [ "fd00:327:327:327::1/64" ];
      listenPort = 54523;
      peers = monitoringPeers;
      privateKeyFile = config.sops.secrets.wg-monitoring.path;
    };
  };

  networking.firewall.allowedUDPPorts = [ 54523 ];

  services.prometheus = {
    enable = true;
    enableReload = true;
    listenAddress = "[::1]";
    scrapeConfigs = let
      relabelAddressToInstance = {
        source_labels = [ "__address__" ];
        target_label = "instance";
        regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
        replacement = "\${1}.net.clerie.de";
      };
    in [
      {
        job_name = "prometheus";
        scrape_interval = "20s";
        scheme = "http";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9090"
            ];
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "alertmanager";
        scrape_interval = "20s";
        scheme = "http";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9093"
            ];
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "node-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:9100;infra"
            ]
            ++ monitoringTargets;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "service_level";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "node-exporter-uberspace";
        scrape_interval = "20s";
        metrics_path = "/.node-exporter/metrics";
        basic_auth = {
          username = "monitor";
          password_file = config.sops.secrets.uberspace-monitor-password.path;
        };
        static_configs = [
          {
            targets = map (target: "${target};infra") config.profiles.clerie.monitoring-server.probeTargets.node-exporter-uberspace;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "service_level";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "instance";
          }
          {
            target_label = "job";
            replacement = "node-exporter";
          }
        ];
      }
      {
        job_name = "nixos-exporter";
        scrape_interval = "1m";
        static_configs = [
          {
            targets = nixosMonitoringTargets;
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "nixos-validator";
        scrape_interval = "1m";
        static_configs = [
          {
            targets = nixosMonitoringTargets;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
            replacement = "\${1}";
          }
          relabelAddressToInstance
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9153";
          }
        ];
      }
      {
        job_name = "bird-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = birdMonitoringTargets;
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_icmp6";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp6" ];
        };
        static_configs = [
          {
            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets config.profiles.clerie.monitoring-server.probeTargets.blackbox-icmp6;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__param_target";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "target";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_icmp4";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp4" ];
        };
        static_configs = [
          {
            targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets config.profiles.clerie.monitoring-server.probeTargets.blackbox-icmp4;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__param_target";
            replacement = "\${2}";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "target";
          }
          {
            source_labels = [ "__address__" ];
            regex = "(.+);(.+)";
            target_label = "__address__";
            replacement = "\${1}";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_synapse";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "synapse" ];
        };
        static_configs = [
          {
            targets = config.profiles.clerie.monitoring-server.probeTargets.blackbox-local-synapse;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            replacement = "https://\${1}/_matrix/static/";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_icmp6_1s";
        scrape_interval = "1s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp6" ];
        };
        static_configs = [
          {
            targets = [
              "carbon.net.clerie.de"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_icmp4_1s";
        scrape_interval = "1s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp4" ];
        };
        static_configs = [
          {
            targets = [
              "carbon.net.clerie.de"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_http6";
        scrape_interval = "100s";
        metrics_path = "/probe";
        params = {
          module = [ "http6" ];
        };
        static_configs = [
          {
            targets = config.profiles.clerie.monitoring-server.probeTargets.blackbox-local-http6;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            replacement = "http://\${1}";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_http4";
        scrape_interval = "100s";
        metrics_path = "/probe";
        params = {
          module = [ "http4" ];
        };
        static_configs = [
          {
            targets = config.profiles.clerie.monitoring-server.probeTargets.blackbox-local-http4;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            replacement = "http://\${1}";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_https6";
        scrape_interval = "100s";
        metrics_path = "/probe";
        params = {
          module = [ "https6" ];
        };
        static_configs = [
          {
            targets = config.profiles.clerie.monitoring-server.probeTargets.blackbox-local-http6;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            replacement = "https://\${1}";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "blackbox_local_https4";
        scrape_interval = "100s";
        metrics_path = "/probe";
        params = {
          module = [ "https4" ];
        };
        static_configs = [
          {
            targets = config.profiles.clerie.monitoring-server.probeTargets.blackbox-local-http4;
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
            replacement = "https://\${1}";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "target";
          }
          {
            target_label = "__address__";
            replacement = "monitoring-3.mon.clerie.de:9115";
          }
          relabelAddressToInstance
        ];
      }
      {
        job_name = "hydra";
        scrape_interval = "20s";
        scheme = "https";
        static_configs = [
          {
            targets = [
              "hydra.clerie.de"
            ];
          }
        ];
      }
      {
        job_name = "berlinerbaeder-exporter";
        scrape_interval = "5m";
        static_configs = [
          {
            targets = [
              "monitoring-3.mon.clerie.de:57382"
            ];
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
      {
        job_name = "clerie_keys";
        scrape_interval = "5m";
        scheme = "https";
        metrics_path = "/gpg/clerie@clerie.de.metrics.txt";
        static_configs = [
          {
            targets = [
              "clerie.de"
            ];
          }
        ];
      }
      {
        job_name = "nginxlog-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = nginxlogMonitoringTargets;
          }
        ];
        relabel_configs = [
          relabelAddressToInstance
        ];
      }
    ];
    alertmanagers = [
      {
        static_configs = [ {
          targets = [
            "monitoring-3.mon.clerie.de:9093"
          ];
        } ];
      }
    ];
    rules = [ (readFile ./rules.yml) ];
  };

  services.nginx = {
    virtualHosts = {
      "prometheus.monitoring.clerie.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:9090/";
      };
    };
  };

}