{ config, lib, pkgs, ... }: with lib; let cfg = config.clerie.monitoring; monitoring-network-base = "fd00:327:327:327::"; in { options = { clerie.monitoring = { enable = mkEnableOption "clerie's Monitoring"; id = mkOption { type = types.str; description = "ID of the Monitoring Interface (it is actually a part of an ip address)"; }; pubkey = mkOption { type = types.str; description = "Public Key of the monitoring wireguard interface of this host"; }; privateKeyFile = mkOption { type = with types; nullOr str; default = null; description = "Path to private key file, pulls secret from secret store when null"; }; serviceLevel = mkOption { type = types.str; default = "infra"; description = "Service level this instance is assigned to"; }; bird = mkEnableOption "Monitor bird"; blackbox = mkEnableOption "Monitor blackbox"; nixos = mkOption { type = types.bool; default = true; description = "Monitor NixOS"; }; }; }; config = mkIf cfg.enable { networking.wireguard.enable = true; networking.wireguard.interfaces = { wg-monitoring = { ips = [ "${monitoring-network-base}${cfg.id}/64" ]; peers = [ { endpoint = "[2001:638:904:ffca::7]:54523"; persistentKeepalive = 25; allowedIPs = [ "${monitoring-network-base}/64" ]; publicKey = "eyhJKV41E1F0gZHBNqyzUnj72xg5f3bdDduVtpPN4AY="; } ]; privateKeyFile = if cfg.privateKeyFile != null then cfg.privateKeyFile else config.sops.secrets.wg-monitoring.path; }; }; services.prometheus.exporters.node = { enable = true; enabledCollectors = [ "systemd" ]; extraFlags = [ "--collector.textfile.directory=/var/lib/prometheus-node-exporter/textfiles" ]; }; systemd.tmpfiles.rules = [ "d /var/lib/prometheus-node-exporter/textfiles - - - - -" ]; systemd.services."prometheus-node-exporter".serviceConfig.RestrictAddressFamilies = [ "AF_NETLINK" ]; services.bijwerken.nodeExporterTextfilePath = "/var/lib/prometheus-node-exporter/textfiles/bijwerken-system-upgrade.prom"; services.prometheus.exporters.bird = mkIf cfg.bird { enable = true; }; services.prometheus.exporters.blackbox = mkIf cfg.blackbox { enable = true; configFile = pkgs.writeText "blackbox.yml" '' modules: icmp6: prober: icmp icmp: preferred_ip_protocol: ip6 ip_protocol_fallback: false icmp4: prober: icmp icmp: preferred_ip_protocol: ip4 ip_protocol_fallback: false ''; }; services.nixos-exporter = { enable = true; listen = "[::]:9152"; }; services.prometheus.exporters.nginxlog = mkIf config.services.nginx.enable { enable = true; settings = { namespaces = [ { name = "nginxlog"; format = ''$host: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$server_name" rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"''; source = { files = [ "/var/log/nginx/access.log" ]; }; relabel_configs = [ { target_label = "server_name"; from = "server_name"; } ]; } ]; }; }; systemd.services."prometheus-nginxlog-exporter".serviceConfig = { SupplementaryGroups = "nginx"; }; networking.firewall.interfaces."wg-monitoring".allowedTCPPorts = [ 9100 # node-exporter 9152 # nixos-exporter ] ++ (if cfg.bird then [ 9324 # bird-exporter ] else []) ++ (if cfg.blackbox then [ 9115 # blackbox-exporter ] else []) ++ (if config.services.prometheus.exporters.nginxlog.enable then [ config.services.prometheus.exporters.nginxlog.port ] else []); }; }