1
0

Compare commits

...

2 Commits

Author SHA1 Message Date
398067f533 hosts/monitoring-3: alert on averaged metrics 2023-05-04 14:43:14 +02:00
4834e5b91d hosts/dn42-ildix-service: add lookingglass 2023-05-04 14:35:30 +02:00
7 changed files with 97 additions and 12 deletions

View File

@ -43,6 +43,42 @@
"type": "github" "type": "github"
} }
}, },
"fernglas": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1680199136,
"narHash": "sha256-dlPzsVkEjhQjrV0++UOX+eaOaO1FTSNoHQ2JHQz9qcA=",
"owner": "wobcom",
"repo": "fernglas",
"rev": "71766d09dc6a43c2219d57c6372a1b5934b61a99",
"type": "github"
},
"original": {
"owner": "wobcom",
"repo": "fernglas",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixos-exporter": { "nixos-exporter": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
@ -98,6 +134,7 @@
"root": { "root": {
"inputs": { "inputs": {
"agenix": "agenix", "agenix": "agenix",
"fernglas": "fernglas",
"nixos-exporter": "nixos-exporter", "nixos-exporter": "nixos-exporter",
"nixpkgs": "nixpkgs", "nixpkgs": "nixpkgs",
"nixpkgs-schule": "nixpkgs-schule", "nixpkgs-schule": "nixpkgs-schule",

View File

@ -6,6 +6,10 @@
url = "github:ryantm/agenix"; url = "github:ryantm/agenix";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
fernglas = {
url = "github:wobcom/fernglas";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-exporter = { nixos-exporter = {
url = "git+https://git.clerie.de/clerie/nixos-exporter.git"; url = "git+https://git.clerie.de/clerie/nixos-exporter.git";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
@ -15,7 +19,7 @@
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
}; };
outputs = { self, nixpkgs, nixos-exporter, solid-xmpp-alarm, ... }@inputs: let outputs = { self, nixpkgs, fernglas, nixos-exporter, solid-xmpp-alarm, ... }@inputs: let
helper = (import ./lib/flake-helper.nix) inputs; helper = (import ./lib/flake-helper.nix) inputs;
in { in {
clerie.hosts = { clerie.hosts = {

View File

@ -120,21 +120,19 @@
# Internal # Internal
protocol bgp peer_2953_dn42_ildix_service { protocol bgp peer_2953_dn42_ildix_service {
local as 4242422953; local as 4242422953;
graceful restart on; neighbor fd81:edb3:71d8:ffff:2953::1 port 1179 as 4242422953;
neighbor fd81:edb3:71d8:ffff:2953::1 as 4242422953;
source address fd81:edb3:71d8:ffff::13; source address fd81:edb3:71d8:ffff::13;
multihop 64;
rr client;
ipv6 { ipv6 {
table master6; table master6;
next hop keep; next hop keep;
import keep filtered; add paths tx;
import filter { import filter {
if net ~ [fd81:edb3:71d8::/48{48,128}] then bgp_path.prepend(4242422953);
if net ~ [fd81:edb3:71d8::/48{48,64}] then accept;
reject; reject;
}; };
export filter { export filter {
if net ~ [fd00::/8{8,64}] then accept; accept;
reject;
}; };
}; };
} }

View File

@ -6,6 +6,7 @@
./hardware-configuration.nix ./hardware-configuration.nix
../../configuration/proxmox-vm ../../configuration/proxmox-vm
./bird.nix ./bird.nix
./fernglas.nix
]; ];
# Use the GRUB 2 boot loader. # Use the GRUB 2 boot loader.
@ -36,6 +37,10 @@
networking.defaultGateway = { address = "192.168.10.1"; interface = "ens18"; }; networking.defaultGateway = { address = "192.168.10.1"; interface = "ens18"; };
networking.nameservers = [ "2001:638:904:ffcc::3" "2001:638:904:ffcc::4" "141.24.40.3" "141.24.40.4" ]; networking.nameservers = [ "2001:638:904:ffcc::3" "2001:638:904:ffcc::4" "141.24.40.3" "141.24.40.4" ];
services.nginx.enable = true;
networking.firewall.allowedTCPPorts = [ 80 443 ];
system.stateVersion = "23.05"; system.stateVersion = "23.05";
} }

View File

@ -0,0 +1,37 @@
{ config, lib, inputs, ... }:
{
networking.firewall.allowedTCPPorts = [ 3000 1179 ];
services.fernglas = {
enable = true;
settings = {
api.bind = "[::1]:3000";
collectors = [
{
collector_type = "Bgp";
bind = "[::]:1179";
default_peer_config = {
asn = 4242422953;
router_id = "${(lib.head config.networking.interfaces.ens18.ipv4.addresses).address}";
route_state = "Accepted";
add_path = true;
};
}
];
};
};
services.nginx.virtualHosts = {
"lg.ildix.clerie.de" = {
enableACME = true;
forceSSL = true;
locations."/" = {
root = inputs.fernglas.packages."x86_64-linux"."fernglas-frontend";
};
locations."/api/" = {
proxyPass = "http://${config.services.fernglas.settings.api.bind}";
};
};
};
}

View File

@ -18,7 +18,7 @@ groups:
summary: "Current system of {{ $labels.instance }} not in sync with config" summary: "Current system of {{ $labels.instance }} not in sync with config"
description: "The current system hash of {{ $labels.instance }} does not match the one generated by hydra based on the current config" description: "The current system hash of {{ $labels.instance }} does not match the one generated by hydra based on the current config"
- alert: BackupStorageFull - alert: BackupStorageFull
expr: ((node_filesystem_avail_bytes{job="node-exporter"} / node_filesystem_size_bytes{job="node-exporter"}) * 100) < 5 expr: ((avg_over_time(node_filesystem_avail_bytes{job="node-exporter"}[5m]) / avg_over_time(node_filesystem_size_bytes{job="node-exporter"}[5m])) * 100) < 5
for: 30m for: 30m
labels: labels:
severity: critical severity: critical
@ -42,7 +42,7 @@ groups:
summary: "Event instance {{ $labels.instance }} down" summary: "Event instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} has been down for more than 2 hours." description: "{{ $labels.instance }} has been down for more than 2 hours."
- alert: KernelChanged - alert: KernelChanged
expr: nixos_current_system_kernel_is_booted_system_kernel{job="nixos-exporter"} == 0 expr: avg_over_time(nixos_current_system_kernel_is_booted_system_kernel{job="nixos-exporter"}[5m]) == 0
for: 2h for: 2h
labels: labels:
severity: warning severity: warning

View File

@ -1,4 +1,4 @@
{ self, nixpkgs, agenix, nixos-exporter, solid-xmpp-alarm, ... }@inputs: { self, nixpkgs, agenix, fernglas, nixos-exporter, solid-xmpp-alarm, ... }@inputs:
rec { rec {
generateNixosSystem = { generateNixosSystem = {
@ -16,7 +16,10 @@ rec {
Useful for having the monitoring server scraping the Useful for having the monitoring server scraping the
target config from all other servers automatically. target config from all other servers automatically.
*/ */
_module.args._nixfiles = self; _module.args = {
inputs = inputs;
_nixfiles = self;
};
}) })
../configuration/common ../configuration/common
../users/clerie ../users/clerie
@ -32,6 +35,7 @@ rec {
clerie.monitoring = nixpkgs.lib.attrsets.optionalAttrs (group != null) { serviceLevel = group; }; clerie.monitoring = nixpkgs.lib.attrsets.optionalAttrs (group != null) { serviceLevel = group; };
}) })
agenix.nixosModules.default agenix.nixosModules.default
fernglas.nixosModules.default
solid-xmpp-alarm.nixosModules.solid-xmpp-alarm solid-xmpp-alarm.nixosModules.solid-xmpp-alarm
(../hosts + "/${name}/configuration.nix") (../hosts + "/${name}/configuration.nix")
# Automatically load secrets from the hosts secrets directory # Automatically load secrets from the hosts secrets directory