1
0
nixfiles/hosts/monitoring-3/prometheus.nix

348 lines
9.1 KiB
Nix

{ config, pkgs, lib, _nixfiles, ... }:
with lib;
let
hosts = _nixfiles.nixosConfigurations // {
"mail-1".config = {
networking.hostName = "mail-1";
clerie.monitoring = {
enable = true;
id = "208";
pubkey = "s2GT9iYQmuiwkZ04KzSqqlIEbUJ6CT6i4htCjWYt2xs=";
nixos = false;
};
};
"mercury".config = {
networking.hostName = "mercury";
clerie.monitoring = {
enable = true;
id = "401";
pubkey = "jrwc4gMZh/Ss1MEXWkLhXcAp4Z++Xnl4noYIZnCfWQg=";
nixos = false;
};
};
};
monitoringHosts = filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "enable"] false host.config)
hosts;
monitoringHostsNames = mapAttrs' (name: host:
nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}" ["${host.config.networking.hostName}.mon.clerie.de"])
monitoringHosts;
monitoringPeers = mapAttrsToList (name: host: {
allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ];
publicKey = host.config.clerie.monitoring.pubkey;
})
monitoringHosts;
monitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}")
monitoringHosts;
nixosMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9152")
(filterAttrs (name: host:
# assume this is a NixOS system if not specified
attrByPath ["clerie" "monitoring" "nixos"] true host.config)
monitoringHosts);
birdMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9324")
(filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "bird"] false host.config)
monitoringHosts);
blackboxMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9115")
(filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
monitoringHosts);
eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
in {
networking.hosts = {
"::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1
}
// monitoringHostsNames;
networking.wireguard.enable = true;
networking.wireguard.interfaces = {
wg-monitoring = {
ips = [ "fd00:327:327:327::1/64" ];
listenPort = 54523;
peers = monitoringPeers;
privateKeyFile = config.age.secrets.wg-monitoring.path;
};
};
networking.firewall.allowedUDPPorts = [ 54523 ];
services.prometheus = {
enable = true;
listenAddress = "[::1]";
scrapeConfigs = let
relabelAddressToInstance = {
source_labels = [ "__address__" ];
target_label = "instance";
regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
replacement = "\${1}.net.clerie.de";
};
in [
{
job_name = "prometheus";
scrape_interval = "20s";
scheme = "http";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9090"
];
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "node-exporter";
scrape_interval = "20s";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9100;infra"
]
++ monitoringTargets;
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "service_level";
replacement = "\${2}";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
}
{
job_name = "nixos-exporter";
scrape_interval = "1m";
static_configs = [
{
targets = nixosMonitoringTargets;
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "nixos-validator";
scrape_interval = "1m";
static_configs = [
{
targets = nixosMonitoringTargets;
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
replacement = "\${1}";
}
relabelAddressToInstance
{
target_label = "__address__";
replacement = "[::1]:9153";
}
];
}
{
job_name = "bird-exporter";
scrape_interval = "20s";
static_configs = [
{
targets = birdMonitoringTargets;
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "blackbox_icmp6";
scrape_interval = "20s";
metrics_path = "/probe";
params = {
module = [ "icmp6" ];
};
static_configs = [
{
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
"clerie.de"
"tagesschau.de"
"google.com"
"achtbaan.nikhef.nl"
"fluorine.net.clerie.de"
"www.fem.tu-ilmenau.de"
"www.heise.de"
];
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__param_target";
replacement = "\${2}";
}
{
source_labels = [ "__param_target" ];
target_label = "target";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
}
{
job_name = "blackbox_icmp4";
scrape_interval = "20s";
metrics_path = "/probe";
params = {
module = [ "icmp4" ];
};
static_configs = [
{
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
"clerie.de"
"tagesschau.de"
"google.com"
"achtbaan.nikhef.nl"
"www.fem.tu-ilmenau.de"
"www.heise.de"
"pe10-fd2.nodes.nethinks.com"
"pe20-fd2.nodes.nethinks.com"
"pe10-pet1.nodes.nethinks.com"
"pe20-pet1.nodes.nethinks.com"
"pe10-ffm1.nodes.nethinks.com"
"ie10-ffm2.nodes.nethinks.com"
"pe10-ffm2.nodes.nethinks.com"
"ie10-due1.nodes.nethinks.com"
"pe10-due1.nodes.nethinks.com"
"matrix.bau-ha.us"
];
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__param_target";
replacement = "\${2}";
}
{
source_labels = [ "__param_target" ];
target_label = "target";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
}
{
job_name = "zimmer-temp";
scrape_interval = "20s";
scheme = "https";
metrics_path = "/data/zimmer-temp/";
static_configs = [
{
targets = [
"iot-data.clerie.de"
];
}
];
}
{
job_name = "outdoor-temp";
scrape_interval = "20s";
scheme = "https";
metrics_path = "/data/outdoor-temp/";
static_configs = [
{
targets = [
"iot-data.clerie.de"
];
}
];
}
{
job_name = "xmpp-alerts";
scrape_interval = "20s";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9199"
];
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "hydra";
scrape_interval = "20s";
scheme = "https";
static_configs = [
{
targets = [
"hydra.clerie.de"
];
}
];
}
];
alertmanagers = [
{
static_configs = [ {
targets = [
"[::1]:9093"
];
} ];
}
];
rules = [ (readFile ./rules.yml) ];
};
services.nginx = {
virtualHosts = {
"prometheus.monitoring.clerie.de" = {
enableACME = true;
forceSSL = true;
locations."/".proxyPass = "http://[::1]:9090/";
};
};
};
}