{ config, pkgs, lib, _nixfiles, ... }: with lib; let hosts = _nixfiles.nixosConfigurations // { "mercury".config = { networking.hostName = "mercury"; clerie.monitoring = { enable = true; id = "401"; pubkey = "jrwc4gMZh/Ss1MEXWkLhXcAp4Z++Xnl4noYIZnCfWQg="; nixos = false; }; }; }; monitoringHosts = filterAttrs (name: host: attrByPath ["clerie" "monitoring" "enable"] false host.config) hosts; monitoringHostsNames = mapAttrs' (name: host: nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}" ["${host.config.networking.hostName}.mon.clerie.de"]) monitoringHosts; monitoringPeers = mapAttrsToList (name: host: { allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ]; publicKey = host.config.clerie.monitoring.pubkey; }) monitoringHosts; monitoringTargets = mapAttrsToList (name: host: "${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}") monitoringHosts; nixosMonitoringTargets = mapAttrsToList (name: host: "${host.config.networking.hostName}.mon.clerie.de:9152") (filterAttrs (name: host: # assume this is a NixOS system if not specified attrByPath ["clerie" "monitoring" "nixos"] true host.config) monitoringHosts); birdMonitoringTargets = mapAttrsToList (name: host: "${host.config.networking.hostName}.mon.clerie.de:9324") (filterAttrs (name: host: attrByPath ["clerie" "monitoring" "bird"] false host.config) monitoringHosts); blackboxMonitoringTargets = mapAttrsToList (name: host: "${host.config.networking.hostName}.mon.clerie.de:9115") (filterAttrs (name: host: attrByPath ["clerie" "monitoring" "blackbox"] false host.config) monitoringHosts); eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b)))); in { networking.hosts = { "::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1 } // monitoringHostsNames; networking.wireguard.enable = true; networking.wireguard.interfaces = { wg-monitoring = { ips = [ "fd00:327:327:327::1/64" ]; listenPort = 54523; peers = monitoringPeers; privateKeyFile = config.age.secrets.wg-monitoring.path; }; }; networking.firewall.allowedUDPPorts = [ 54523 ]; services.prometheus = { enable = true; enableReload = true; listenAddress = "[::1]"; scrapeConfigs = let relabelAddressToInstance = { source_labels = [ "__address__" ]; target_label = "instance"; regex = ''([\w-]+)\.mon\.clerie\.de\:\d+''; replacement = "\${1}.net.clerie.de"; }; in [ { job_name = "prometheus"; scrape_interval = "20s"; scheme = "http"; static_configs = [ { targets = [ "monitoring-3.mon.clerie.de:9090" ]; } ]; relabel_configs = [ relabelAddressToInstance ]; } { job_name = "node-exporter"; scrape_interval = "20s"; static_configs = [ { targets = [ "monitoring-3.mon.clerie.de:9100;infra" ] ++ monitoringTargets; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "service_level"; replacement = "\${2}"; } { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "__address__"; replacement = "\${1}"; } relabelAddressToInstance ]; } { job_name = "nixos-exporter"; scrape_interval = "1m"; static_configs = [ { targets = nixosMonitoringTargets; } ]; relabel_configs = [ relabelAddressToInstance ]; } { job_name = "nixos-validator"; scrape_interval = "1m"; static_configs = [ { targets = nixosMonitoringTargets; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; regex = ''([\w-]+)\.mon\.clerie\.de\:\d+''; replacement = "\${1}"; } relabelAddressToInstance { target_label = "__address__"; replacement = "[::1]:9153"; } ]; } { job_name = "bird-exporter"; scrape_interval = "20s"; static_configs = [ { targets = birdMonitoringTargets; } ]; relabel_configs = [ relabelAddressToInstance ]; } { job_name = "blackbox_icmp6"; scrape_interval = "20s"; metrics_path = "/probe"; params = { module = [ "icmp6" ]; }; static_configs = [ { targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [ "clerie.de" "tagesschau.de" "google.com" "achtbaan.nikhef.nl" "fluorine.net.clerie.de" "www.fem.tu-ilmenau.de" "www.heise.de" "dyon.net.entr0py.de" ]; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "__param_target"; replacement = "\${2}"; } { source_labels = [ "__param_target" ]; target_label = "target"; } { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "__address__"; replacement = "\${1}"; } relabelAddressToInstance ]; } { job_name = "blackbox_icmp4"; scrape_interval = "20s"; metrics_path = "/probe"; params = { module = [ "icmp4" ]; }; static_configs = [ { targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [ "clerie.de" "tagesschau.de" "google.com" "achtbaan.nikhef.nl" "www.fem.tu-ilmenau.de" "www.heise.de" "pe10-fd2.nodes.nethinks.com" "pe20-fd2.nodes.nethinks.com" "pe10-pet1.nodes.nethinks.com" "pe20-pet1.nodes.nethinks.com" "pe10-ffm1.nodes.nethinks.com" "ie10-ffm2.nodes.nethinks.com" "pe10-ffm2.nodes.nethinks.com" "ie10-due1.nodes.nethinks.com" "pe10-due1.nodes.nethinks.com" "matrix.bau-ha.us" "dyon.net.entr0py.de" "matrix.entr0py.de" ]; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "__param_target"; replacement = "\${2}"; } { source_labels = [ "__param_target" ]; target_label = "target"; } { source_labels = [ "__address__" ]; regex = "(.+);(.+)"; target_label = "__address__"; replacement = "\${1}"; } relabelAddressToInstance ]; } { job_name = "blackbox_local_synapse"; scrape_interval = "20s"; metrics_path = "/probe"; params = { module = [ "synapse" ]; }; static_configs = [ { targets = [ "matrix.entr0py.de" ]; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; replacement = "https://\${1}/_matrix/static/"; } { source_labels = [ "__address__" ]; target_label = "target"; } { target_label = "__address__"; replacement = "monitoring-3.mon.clerie.de:9115"; } relabelAddressToInstance ]; } { job_name = "blackbox_local_icmp6_1s"; scrape_interval = "1s"; metrics_path = "/probe"; params = { module = [ "icmp6" ]; }; static_configs = [ { targets = [ "carbon.net.clerie.de" ]; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__address__" ]; target_label = "target"; } { target_label = "__address__"; replacement = "monitoring-3.mon.clerie.de:9115"; } relabelAddressToInstance ]; } { job_name = "blackbox_local_icmp4_1s"; scrape_interval = "1s"; metrics_path = "/probe"; params = { module = [ "icmp4" ]; }; static_configs = [ { targets = [ "carbon.net.clerie.de" ]; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__address__" ]; target_label = "target"; } { target_label = "__address__"; replacement = "monitoring-3.mon.clerie.de:9115"; } relabelAddressToInstance ]; } { job_name = "zimmer-temp"; scrape_interval = "20s"; scheme = "https"; metrics_path = "/data/zimmer-temp/"; static_configs = [ { targets = [ "iot-data.clerie.de" ]; } ]; } { job_name = "outdoor-temp"; scrape_interval = "20s"; scheme = "https"; metrics_path = "/data/outdoor-temp/"; static_configs = [ { targets = [ "iot-data.clerie.de" ]; } ]; } { job_name = "xmpp-alerts"; scrape_interval = "20s"; static_configs = [ { targets = [ "monitoring-3.mon.clerie.de:9199" ]; } ]; relabel_configs = [ relabelAddressToInstance ]; } { job_name = "hydra"; scrape_interval = "20s"; scheme = "https"; static_configs = [ { targets = [ "hydra.clerie.de" ]; } ]; } ]; alertmanagers = [ { static_configs = [ { targets = [ "[::1]:9093" ]; } ]; } ]; rules = [ (readFile ./rules.yml) ]; }; services.nginx = { virtualHosts = { "prometheus.monitoring.clerie.de" = { enableACME = true; forceSSL = true; locations."/".proxyPass = "http://[::1]:9090/"; }; }; }; }