425 lines
11 KiB
Nix
425 lines
11 KiB
Nix
{ config, pkgs, lib, _nixfiles, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
hosts = _nixfiles.nixosConfigurations // {
|
|
"mercury".config = {
|
|
networking.hostName = "mercury";
|
|
clerie.monitoring = {
|
|
enable = true;
|
|
id = "401";
|
|
pubkey = "jrwc4gMZh/Ss1MEXWkLhXcAp4Z++Xnl4noYIZnCfWQg=";
|
|
nixos = false;
|
|
};
|
|
};
|
|
};
|
|
|
|
monitoringHosts = filterAttrs (name: host:
|
|
attrByPath ["clerie" "monitoring" "enable"] false host.config)
|
|
hosts;
|
|
|
|
monitoringHostsNames = mapAttrs' (name: host:
|
|
nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}" ["${host.config.networking.hostName}.mon.clerie.de"])
|
|
monitoringHosts;
|
|
|
|
monitoringPeers = mapAttrsToList (name: host: {
|
|
allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ];
|
|
publicKey = host.config.clerie.monitoring.pubkey;
|
|
})
|
|
monitoringHosts;
|
|
|
|
monitoringTargets = mapAttrsToList (name: host:
|
|
"${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}")
|
|
monitoringHosts;
|
|
|
|
nixosMonitoringTargets = mapAttrsToList (name: host:
|
|
"${host.config.networking.hostName}.mon.clerie.de:9152")
|
|
(filterAttrs (name: host:
|
|
# assume this is a NixOS system if not specified
|
|
attrByPath ["clerie" "monitoring" "nixos"] true host.config)
|
|
monitoringHosts);
|
|
|
|
birdMonitoringTargets = mapAttrsToList (name: host:
|
|
"${host.config.networking.hostName}.mon.clerie.de:9324")
|
|
(filterAttrs (name: host:
|
|
attrByPath ["clerie" "monitoring" "bird"] false host.config)
|
|
monitoringHosts);
|
|
|
|
blackboxMonitoringTargets = mapAttrsToList (name: host:
|
|
"${host.config.networking.hostName}.mon.clerie.de:9115")
|
|
(filterAttrs (name: host:
|
|
attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
|
|
monitoringHosts);
|
|
|
|
eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
|
|
|
|
in {
|
|
networking.hosts = {
|
|
"::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1
|
|
}
|
|
// monitoringHostsNames;
|
|
|
|
networking.wireguard.enable = true;
|
|
networking.wireguard.interfaces = {
|
|
wg-monitoring = {
|
|
ips = [ "fd00:327:327:327::1/64" ];
|
|
listenPort = 54523;
|
|
peers = monitoringPeers;
|
|
privateKeyFile = config.sops.secrets.wg-monitoring.path;
|
|
};
|
|
};
|
|
|
|
networking.firewall.allowedUDPPorts = [ 54523 ];
|
|
|
|
services.prometheus = {
|
|
enable = true;
|
|
enableReload = true;
|
|
listenAddress = "[::1]";
|
|
scrapeConfigs = let
|
|
relabelAddressToInstance = {
|
|
source_labels = [ "__address__" ];
|
|
target_label = "instance";
|
|
regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
|
|
replacement = "\${1}.net.clerie.de";
|
|
};
|
|
in [
|
|
{
|
|
job_name = "prometheus";
|
|
scrape_interval = "20s";
|
|
scheme = "http";
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"monitoring-3.mon.clerie.de:9090"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "node-exporter";
|
|
scrape_interval = "20s";
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"monitoring-3.mon.clerie.de:9100;infra"
|
|
]
|
|
++ monitoringTargets;
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "service_level";
|
|
replacement = "\${2}";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "__address__";
|
|
replacement = "\${1}";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "nixos-exporter";
|
|
scrape_interval = "1m";
|
|
static_configs = [
|
|
{
|
|
targets = nixosMonitoringTargets;
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "nixos-validator";
|
|
scrape_interval = "1m";
|
|
static_configs = [
|
|
{
|
|
targets = nixosMonitoringTargets;
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "__param_target";
|
|
regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
|
|
replacement = "\${1}";
|
|
}
|
|
relabelAddressToInstance
|
|
{
|
|
target_label = "__address__";
|
|
replacement = "[::1]:9153";
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "bird-exporter";
|
|
scrape_interval = "20s";
|
|
static_configs = [
|
|
{
|
|
targets = birdMonitoringTargets;
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "blackbox_icmp6";
|
|
scrape_interval = "20s";
|
|
metrics_path = "/probe";
|
|
params = {
|
|
module = [ "icmp6" ];
|
|
};
|
|
static_configs = [
|
|
{
|
|
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
|
|
"clerie.de"
|
|
"tagesschau.de"
|
|
"google.com"
|
|
"achtbaan.nikhef.nl"
|
|
"fluorine.net.clerie.de"
|
|
"www.fem.tu-ilmenau.de"
|
|
"www.heise.de"
|
|
"dyon.net.entr0py.de"
|
|
"matrix.fachschaften.org"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "__param_target";
|
|
replacement = "\${2}";
|
|
}
|
|
{
|
|
source_labels = [ "__param_target" ];
|
|
target_label = "target";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "__address__";
|
|
replacement = "\${1}";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "blackbox_icmp4";
|
|
scrape_interval = "20s";
|
|
metrics_path = "/probe";
|
|
params = {
|
|
module = [ "icmp4" ];
|
|
};
|
|
static_configs = [
|
|
{
|
|
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
|
|
"clerie.de"
|
|
"tagesschau.de"
|
|
"google.com"
|
|
"achtbaan.nikhef.nl"
|
|
"www.fem.tu-ilmenau.de"
|
|
"www.heise.de"
|
|
"pe10-fd2.nodes.nethinks.com"
|
|
"pe20-fd2.nodes.nethinks.com"
|
|
"pe10-pet1.nodes.nethinks.com"
|
|
"pe20-pet1.nodes.nethinks.com"
|
|
"pe10-ffm1.nodes.nethinks.com"
|
|
"ie10-ffm2.nodes.nethinks.com"
|
|
"pe10-ffm2.nodes.nethinks.com"
|
|
"ie10-due1.nodes.nethinks.com"
|
|
"pe10-due1.nodes.nethinks.com"
|
|
"matrix.bau-ha.us"
|
|
"dyon.net.entr0py.de"
|
|
"matrix.entr0py.de"
|
|
"matrix.fachschaften.org"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "__param_target";
|
|
replacement = "\${2}";
|
|
}
|
|
{
|
|
source_labels = [ "__param_target" ];
|
|
target_label = "target";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
regex = "(.+);(.+)";
|
|
target_label = "__address__";
|
|
replacement = "\${1}";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "blackbox_local_synapse";
|
|
scrape_interval = "20s";
|
|
metrics_path = "/probe";
|
|
params = {
|
|
module = [ "synapse" ];
|
|
};
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"matrix.entr0py.de"
|
|
"matrix.fachschaften.org"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "__param_target";
|
|
replacement = "https://\${1}/_matrix/static/";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "target";
|
|
}
|
|
{
|
|
target_label = "__address__";
|
|
replacement = "monitoring-3.mon.clerie.de:9115";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "blackbox_local_icmp6_1s";
|
|
scrape_interval = "1s";
|
|
metrics_path = "/probe";
|
|
params = {
|
|
module = [ "icmp6" ];
|
|
};
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"carbon.net.clerie.de"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "__param_target";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "target";
|
|
}
|
|
{
|
|
target_label = "__address__";
|
|
replacement = "monitoring-3.mon.clerie.de:9115";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "blackbox_local_icmp4_1s";
|
|
scrape_interval = "1s";
|
|
metrics_path = "/probe";
|
|
params = {
|
|
module = [ "icmp4" ];
|
|
};
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"carbon.net.clerie.de"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "__param_target";
|
|
}
|
|
{
|
|
source_labels = [ "__address__" ];
|
|
target_label = "target";
|
|
}
|
|
{
|
|
target_label = "__address__";
|
|
replacement = "monitoring-3.mon.clerie.de:9115";
|
|
}
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "xmpp-alerts";
|
|
scrape_interval = "20s";
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"monitoring-3.mon.clerie.de:9199"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
{
|
|
job_name = "hydra";
|
|
scrape_interval = "20s";
|
|
scheme = "https";
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"hydra.clerie.de"
|
|
];
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "berlinerbaeder-exporter";
|
|
scrape_interval = "5m";
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"monitoring-3.mon.clerie.de:57382"
|
|
];
|
|
}
|
|
];
|
|
relabel_configs = [
|
|
relabelAddressToInstance
|
|
];
|
|
}
|
|
];
|
|
alertmanagers = [
|
|
{
|
|
static_configs = [ {
|
|
targets = [
|
|
"[::1]:9093"
|
|
];
|
|
} ];
|
|
}
|
|
];
|
|
rules = [ (readFile ./rules.yml) ];
|
|
};
|
|
|
|
services.nginx = {
|
|
virtualHosts = {
|
|
"prometheus.monitoring.clerie.de" = {
|
|
enableACME = true;
|
|
forceSSL = true;
|
|
locations."/".proxyPass = "http://[::1]:9090/";
|
|
};
|
|
};
|
|
};
|
|
|
|
}
|