vcp-bula-nixfiles/hosts/monitoring/configuration.nix

{ config, pkgs, lib, ... }:
with lib;

{
  imports =
    [
      ./hardware-configuration.nix
    ];
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  networking.hostName = "monitoring";
  networking.useDHCP = false;
  networking.interfaces.ens18.ipv4.addresses = [
    {address = "10.42.10.7"; prefixLength = 24; }
  ];
  networking.interfaces.ens18.ipv6.addresses = [
    {address = "2a01:4f8:1c0c:8221::7"; prefixLength = 64; }
    {address = "2a01:4f8:1c0c:8221::1984"; prefixLength = 64; }
  ];
  networking.defaultGateway = { address = "10.42.10.1"; interface = "ens18"; };
  networking.defaultGateway6 = { address = "2a01:4f8:1c0c:8221::1"; interface = "ens18"; };

  environment = {
    etc = {
      "snmp-exporter/snmp.yml".source = ./snmp.yml;
    };
  };

  services.influxdb2.enable = true;

  services.prometheus.exporters = {
    blackbox = {
      enable = true;
      listenAddress = "[::1]";
      port = 9115;
      configFile = ./blackbox.yml;
    };
    snmp = {
      enable = true;
      port= 9116;
      listenAddress = "[::1]";
      configurationPath = "/etc/snmp-exporter/snmp.yml";
    };
  };
  services.prometheus.exporters.node.enable = true;

  systemd.services.waldbrandgefahrenstufen-exporter = {
    description = "Waldbrandgefahrenstufen Exporter";
    wantedBy = [ "multi-user.target" ];

    serviceConfig = {
      DynamicUser = "yes";
    };

    script = "${pkgs.python3}/bin/python ${./waldbrandgefahrenstufen-exporter.py}";
  };

  systemd.services.sipgate-balance-exporter = {
    description = "Sipgate Balance Exporter";
    wantedBy = [ "multi-user.target" ];

    serviceConfig = {
      DynamicUser = "yes";
      Environment = "SIPGATE_TOKEN_PATH=/var/src/secrets/sipgate-balance/apitoken";
    };

    script = "${pkgs.python3}/bin/python ${./sipgate-balance-exporter.py}";
  };

  services.prometheus.alertmanager = {
    enable = true;
    listenAddress = "[::1]";
    port = 9093;
    configuration = {
      route = {
        receiver = "default";
        group_by = [ "instance" ];
      };
      receivers = [
        {
          name = "default";
        }
      ];
    };
  };
  services.prometheus = {
    enable = true;
    retentionTime = "90d";
    listenAddress = "[::1]";
    scrapeConfigs = [
      {
        job_name = "prometheus";
        scrape_interval = "20s";
        scheme = "http";
        static_configs = [
          {
            targets = [
              "[::1]:9090"
            ];
          }
        ];
      }
      {
        job_name = "node-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = [
              "[::1]:9100"
              "radius.bula22.de:9100"
              "router.bula22.de:9100"
            ];
          }
        ];
      }
      {
        job_name = "blackbox_icmp6";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp6" ];
        };
        static_configs = [
          {
            targets = [
              # Internet Probes
              "clerie.de"

              # Vlan Probes
              "gw.services.net.bula22.de"
              "gw.technik-iot.net.bula22.de"
              "gw.technik.net.bula22.de"
              "gw.hospital.net.bula22.de"
              "gw.zoll.net.bula22.de"
              "gw.leitstelle.net.bula22.de"
              "gw.verwaltung.net.bula22.de"
              "gw.yolo.net.bula22.de"
              "gw.infojurte.net.bula22.de"
              "gw.international.net.bula22.de"
              "gw.programmtresen.net.bula22.de"
              "gw.open-office.net.bula22.de"

              # Service Probes
              "router.bula22.de"
              "radius.bula22.de"
              "yate.bula22.de"
              "monitoring.bula22.de"
              "dns.bula22.de"
              "yate-dialup.bula22.de"
              "nerd.bula22.de"
              "pre-router.bula22.de"
              "nixdeploy.bula22.de"

              # External Service Probes
              "vcp-bula-mon.bula22.de"
              "vcp-bula-telko.bula22.de"
            ];
            labels = {
              "module" = "icmp6";
            };
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:9115";
          }
        ];
      }
      {
        job_name = "blackbox_icmp4";
        scrape_interval = "20s";
        metrics_path = "/probe";
        params = {
          module = [ "icmp4" ];
        };
        static_configs = [
          {
            targets = [
              # Internet Probes
              "clerie.de"
              "vcp.de"
              "bundeslager.vcp.de"

              # Vlan Probes
              "gw.services.net.bula22.de"
              "gw.management.net.bula22.de"
              "gw.technik-iot.net.bula22.de"
              "gw.technik.net.bula22.de"
              "gw.hospital.net.bula22.de"
              "gw.zoll.net.bula22.de"
              "gw.leitstelle.net.bula22.de"
              "gw.verwaltung.net.bula22.de"
              "gw.yolo.net.bula22.de"
              "gw.infojurte.net.bula22.de"
              "gw.international.net.bula22.de"
              "gw.programmtresen.net.bula22.de"
              "gw.open-office.net.bula22.de"

              # Management Probes
              "wlan-controller.bula22.de"
              "sw-zentral-1.bula22.de"
              "sw-zentral-2.bula22.de"
              "sw-verwaltung.bula22.de"
              "sw-zoll-container.bula22.de"
              # "sw-zoll-zelt.bula22.de" # Wird doch nicht benötigt
              "sw-programmtresen.bula22.de"
              "sw-post.bula22.de"
              "sw-international.bula22.de"
              "sw-leitstelle.bula22.de"
              "sw-infojurte.bula22.de"
              "sw-technik-zelt.bula22.de"
              "sw-technik-container.bula22.de"
              "sw-hospital.bula22.de"
              "sw-fuf.bula22.de"
              "sw-waschhaus.bula22.de"
              "sw-buehne.bula22.de"
              "sw-trabantenstadt.bula22.de"
              "sw-bll.bula22.de"
              "sw-finanzen.bula22.de"

              # Service Probes
              "router.bula22.de"
              "radius.bula22.de"
              "yate.bula22.de"
              "monitoring.bula22.de"
              "dns.bula22.de"
              "yate-dialup.bula22.de"
              "nerd.bula22.de"
              "pre-router.bula22.de"
              "nixdeploy.bula22.de"
              "omm.bula22.de"
              "webcam.bula22.de"
            ];
            labels = {
              "module" = "icmp4";
            };
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:9115";
          }
        ];
      }
      {
        job_name = "dns";
        scrape_interval = "5s";
        static_configs = [
          {
            targets = [
              "dns.bula22.de:9253"
            ];
          }
        ];
      }
      {
        job_name = "waldbrandgefahrenstufen";
        scrape_interval = "1h";
	      scrape_timeout = "60s";
        static_configs = [
          {
            targets = [
              "[::1]:9242"
            ];
          }
        ];
      }
      {
        job_name = "sipgate-balance";
        scrape_interval = "120s";
	      scrape_timeout = "20s";
        static_configs = [
          {
            targets = [
              "[::1]:9243"
            ];
          }
        ];
      }
      {
        job_name = "snmp";
        scrape_interval = "120s";
        scrape_timeout = "40s";
        metrics_path = "/snmp";
        params = {
          module = [
            "if_mib"
          ];
        };
        static_configs = [
          {
            targets = [
              "sw-zentral-1.bula22.de"
              "sw-zentral-2.bula22.de"
              "sw-verwaltung.bula22.de"
              "sw-zoll-container.bula22.de"
              # "sw-zoll-zelt.bula22.de" # Wird doch nicht benötigt
              "sw-programmtresen.bula22.de"
              "sw-international.bula22.de"
              "sw-post.bula22.de"
              "sw-leitstelle.bula22.de"
              "sw-infojurte.bula22.de"
              "sw-technik-zelt.bula22.de"
              "sw-technik-container.bula22.de"
              "sw-hospital.bula22.de"
              "sw-fuf.bula22.de"
              "sw-waschhaus.bula22.de"
              "sw-buehne.bula22.de"
              "sw-trabantenstadt.bula22.de"
              "sw-bll.bula22.de"
              "sw-finanzen.bula22.de"
              "wlan-controller.bula22.de"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:9116";
          }
        ];
      }
      {
        job_name = "aruba";
        scrape_interval = "120s";
        scrape_timeout = "40s";
        metrics_path = "/snmp";
        params = {
          module = [
            "aruba"
          ];
        };
        static_configs = [
          {
            targets = [
              "wlan-controller.bula22.de"
            ];
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:9116";
          }
        ];
      }
      {
        job_name = "yate";
        scrape_interval = "30s";
        scrape_timeout = "20s";
        metrics_path = "/snmp";
        params = {
          module = [
            "yate"
          ];
        };
        static_configs = [
          {
            targets = [
                "10.42.10.6"
            ];
	    labels = {
                instance = "yate.bula22.de";
	    };
          }
          {
            targets = [
                "10.42.10.9"
            ];
	    labels = {
                instance = "yate-dialup.bula22.de";
	    };
          }
        ];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:9116";
          }
        ];
      }
      {
        job_name = "energy";
        scrape_interval = "1s";
        scrape_timeout = "1s";
        static_configs = [
          {
            targets = [
              "pbs.bula22.de:9555"
            ];
          }
        ];
      }
    ];
    alertmanagers = [
      {
        static_configs = [ {
          targets = [
            "[::1]:9093"
          ];
        } ];
      }
    ];
    rules = [ (readFile ./alertmanager-rules.yml) ];
  };

  systemd.services.prometheus-snmp-exporter.restartTriggers = [
    config.environment.etc."snmp-exporter/snmp.yml".source
  ];

  services.grafana = {
    enable  = true;
    domain  = "grafana.bula22.de";
    rootUrl = "https://grafana.bula22.de";
    port    = 3001;
    addr = "::1";
    auth.anonymous.enable = true;
    provision = {
      enable = true;
      datasources = [
        {
          type = "prometheus";
          name = "Prometheus";
          url = "http://[::1]:9090";
          isDefault = true;
        }
      ];
      dashboards = [];
    };
  };
  services.nginx = {
    enable = true;
    virtualHosts = {
      "prometheus.bula22.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:9090/";
      };
      "grafana.bula22.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:3001/";
      };
      "influxdb.bula22.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:8086/";
      };
    };
  };
  networking.firewall.allowedTCPPorts = [ 80 443 ];

  system.stateVersion = "22.05";
}