1
0
nixfiles/hosts/monitoring-3/configuration.nix

438 lines
11 KiB
Nix
Raw Normal View History

2021-02-24 00:16:30 +01:00
{ config, pkgs, lib, ... }:
2021-02-24 00:16:30 +01:00
with lib;
let
hosts = (import ../../lib/hosts.nix { inherit pkgs; }).hosts;
monitoringHosts = filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "enable"] false host.config)
hosts;
monitoringHostsNames = mapAttrs' (name: host:
nameValuePair "fd00:327:327:327::${host.config.clerie.monitoring.id}" ["${host.config.networking.hostName}.mon.clerie.de"])
monitoringHosts;
monitoringPeers = mapAttrsToList (name: host: {
allowedIPs = [ "fd00:327:327:327::${host.config.clerie.monitoring.id}/128" ];
publicKey = host.config.clerie.monitoring.pubkey;
})
monitoringHosts;
monitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9100;${attrByPath ["clerie" "monitoring" "serviceLevel"] "infra" host.config}")
2021-02-24 00:16:30 +01:00
monitoringHosts;
nixosMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9152")
(filterAttrs (name: host:
# assume this is a NixOS system if not specified
attrByPath ["clerie" "monitoring" "nixos"] true host.config)
monitoringHosts);
2022-03-22 12:16:28 +01:00
birdMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9324")
(filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "bird"] false host.config)
monitoringHosts);
blackboxMonitoringTargets = mapAttrsToList (name: host:
"${host.config.networking.hostName}.mon.clerie.de:9115")
(filterAttrs (name: host:
attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
monitoringHosts);
eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
2021-02-24 00:16:30 +01:00
in {
imports =
[
./hardware-configuration.nix
../../configuration/common
../../configuration/proxmox-vm
];
boot.loader.grub.enable = true;
boot.loader.grub.version = 2;
boot.loader.grub.device = "/dev/sda";
networking.hostName = "monitoring-3";
networking.useDHCP = false;
networking.interfaces.ens18.ipv4.addresses = [ { address = "192.168.10.32"; prefixLength = 24; } ];
networking.interfaces.ens19.ipv6.addresses = [ { address = "2001:638:904:ffca::7"; prefixLength = 64; } ];
networking.defaultGateway = { address = "192.168.10.1"; interface = "ens18"; };
networking.defaultGateway6 = { address = "2001:638:904:ffca::1"; interface = "ens19"; };
networking.nameservers = [ "2001:638:904:ffcc::3" "2001:638:904:ffcc::4" "141.24.40.3" "141.24.40.4" ];
networking.hosts = {
"::1" = [ "monitoring-3.mon.clerie.de" ]; # fd00:327:327:327::1
2021-02-24 00:16:30 +01:00
}
// monitoringHostsNames;
networking.wireguard.enable = true;
networking.wireguard.interfaces = {
wg-monitoring = {
ips = [ "fd00:327:327:327::1/64" ];
2021-02-21 22:39:57 +01:00
listenPort = 54523;
2021-02-24 00:16:30 +01:00
peers = monitoringPeers;
privateKeyFile = "/var/src/secrets/wireguard/wg-monitoring";
};
};
2021-02-21 22:39:57 +01:00
networking.firewall.allowedUDPPorts = [ 54523 ];
services.prometheus.exporters.node.enable = true;
2021-10-22 23:21:26 +02:00
services.prometheus.xmpp-alerts = {
enable = true;
settings = {
jid = "feuer@fem-net.de";
password_command = "cat /var/src/secrets/xmpp-alert/password";
listen_address = "::1";
listen_port = 9199;
to_jid = "clerie@fem-net.de";
format = "short";
};
};
services.prometheus.alertmanager = {
enable = true;
listenAddress = "[::1]";
port = 9093;
configuration = {
route = {
receiver = "xmpp-receiver";
repeat_interval = "4h";
2021-10-22 23:21:26 +02:00
group_by = [ "instance" ];
routes = [
{
matchers = [
''severity = "muted"''
];
receiver = "muted";
}
{
receiver = "xmpp-receiver";
matchers = [
''severity = "warning"''
];
repeat_interval = "70h";
}
];
2021-10-22 23:21:26 +02:00
};
receivers = [
{
name = "muted";
}
2021-10-22 23:21:26 +02:00
{
name = "xmpp-receiver";
webhook_configs = [
{
url = "http://[::1]:9199/alert";
}
];
}
];
inhibit_rules = [
{
# Mute all alerts for an instance which also has an alert starting with MuteInstanceOn
target_matchers = [
''alertname =~ ".+"''
];
source_matchers = [
''mute = "instance"''
];
equal = [
"instance"
];
}
];
2021-10-22 23:21:26 +02:00
};
};
services.prometheus = {
enable = true;
listenAddress = "[::1]";
scrapeConfigs = let
relabelAddressToInstance = {
source_labels = [ "__address__" ];
target_label = "instance";
regex = ''([\w-]+)\.mon\.clerie\.de\:\d+'';
replacement = "\${1}.net.clerie.de";
};
in [
{
job_name = "prometheus";
scrape_interval = "20s";
scheme = "http";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9090"
];
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "node-exporter";
scrape_interval = "20s";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9100;infra"
2021-02-24 00:16:30 +01:00
]
++ monitoringTargets;
}
2022-03-22 12:16:28 +01:00
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "service_level";
replacement = "\${2}";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
2022-03-22 12:16:28 +01:00
}
{
job_name = "nixos-exporter";
scrape_interval = "1m";
static_configs = [
{
targets = nixosMonitoringTargets;
}
];
relabel_configs = [
relabelAddressToInstance
];
}
2022-03-22 12:16:28 +01:00
{
job_name = "bird-exporter";
scrape_interval = "20s";
static_configs = [
{
targets = birdMonitoringTargets;
}
];
relabel_configs = [
relabelAddressToInstance
];
}
{
job_name = "blackbox_icmp6";
scrape_interval = "20s";
metrics_path = "/probe";
params = {
module = [ "icmp6" ];
};
static_configs = [
{
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
"clerie.de"
"tagesschau.de"
"google.com"
"achtbaan.nikhef.nl"
"fluorine.net.clerie.de"
"www.fem.tu-ilmenau.de"
"www.heise.de"
];
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__param_target";
replacement = "\${2}";
}
{
source_labels = [ "__param_target" ];
target_label = "target";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
}
{
job_name = "blackbox_icmp4";
scrape_interval = "20s";
metrics_path = "/probe";
params = {
module = [ "icmp4" ];
};
static_configs = [
{
targets = eachWithEachOther (instance: target: "${instance};${target}") blackboxMonitoringTargets [
"clerie.de"
"tagesschau.de"
"google.com"
"achtbaan.nikhef.nl"
"www.fem.tu-ilmenau.de"
"www.heise.de"
"ie10-ffm2.nodes.nethinks.com"
"matrix.bau-ha.us"
];
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__param_target";
replacement = "\${2}";
}
{
source_labels = [ "__param_target" ];
target_label = "target";
}
{
source_labels = [ "__address__" ];
regex = "(.+);(.+)";
target_label = "__address__";
replacement = "\${1}";
}
relabelAddressToInstance
];
}
{
job_name = "zimmer-temp";
scrape_interval = "20s";
scheme = "https";
metrics_path = "/data/zimmer-temp/";
static_configs = [
{
targets = [
"iot-data.clerie.de"
];
2021-12-20 16:47:57 +01:00
}
];
}
{
job_name = "outdoor-temp";
2021-12-20 16:47:57 +01:00
scrape_interval = "20s";
scheme = "https";
metrics_path = "/data/outdoor-temp/";
2021-12-20 16:47:57 +01:00
static_configs = [
{
targets = [
"iot-data.clerie.de"
2021-12-20 16:47:57 +01:00
];
2021-12-20 17:49:06 +01:00
}
];
}
{
job_name = "xmpp-alerts";
2021-12-20 17:49:06 +01:00
scrape_interval = "20s";
static_configs = [
{
targets = [
"monitoring-3.mon.clerie.de:9199"
2021-12-20 17:49:06 +01:00
];
}
];
relabel_configs = [
relabelAddressToInstance
];
}
];
2021-10-22 23:21:26 +02:00
alertmanagers = [
{
static_configs = [ {
targets = [
"[::1]:9093"
];
} ];
}
];
2021-10-22 23:53:42 +02:00
rules = [ (readFile ./rules.yml) ];
};
services.grafana = {
enable = true;
domain = "grafana.monitoring.clerie.de";
rootUrl = "https://grafana.monitoring.clerie.de";
port = 3001;
addr = "::1";
auth.anonymous.enable = true;
provision = {
enable = true;
datasources = [
{
type = "prometheus";
name = "Prometheus";
url = "http://[::1]:9090";
isDefault = true;
}
];
dashboards = [
{
options.path = ./dashboards;
}
];
};
};
2021-05-15 18:43:15 +02:00
users.users.uptimestatus = {
description = "Uptime Status Service";
group = "uptimestatus";
home = "/var/lib/uptimestatus/";
useDefaultShell = true;
isSystemUser = true;
};
users.groups.uptimestatus = {};
systemd.services.uptimestatus = {
wantedBy = [ "multi-user.target" ];
serviceConfig = {
RuntimeDirectory = "uptimestatus";
StateDirectory = "uptimestatus";
User = "uptimestatus";
Group = "uptimestatus";
};
script = "gunicorn -w 4 -b [::1]:8235 uptimestatus:app";
2021-12-07 18:29:16 +01:00
path = with pkgs; [ (python3.withPackages (ps: [ ps.gunicorn uptimestatus ])) ];
2021-05-15 18:43:15 +02:00
};
services.nginx = {
enable = true;
virtualHosts = {
"prometheus.monitoring.clerie.de" = {
enableACME = true;
forceSSL = true;
locations."/".proxyPass = "http://[::1]:9090/";
};
"grafana.monitoring.clerie.de" = {
enableACME = true;
forceSSL = true;
locations."/".proxyPass = "http://[::1]:3001/";
};
2021-05-15 18:43:15 +02:00
"status.monitoring.clerie.de" = {
enableACME = true;
forceSSL = true;
locations."/".proxyPass = "http://[::1]:8235/";
};
};
};
networking.firewall.allowedTCPPorts = [ 80 443 ];
system.stateVersion = "21.03";
}