From eddb365ae5f23d15b2e824e83205398841717919 Mon Sep 17 00:00:00 2001 From: clerie Date: Sun, 17 Aug 2025 10:17:43 +0200 Subject: [PATCH 1/6] hosts/monitoring-3: Alert nadja.top down after 15min only --- hosts/monitoring-3/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/monitoring-3/rules.yml b/hosts/monitoring-3/rules.yml index 7cb9d07..f9bb742 100644 --- a/hosts/monitoring-3/rules.yml +++ b/hosts/monitoring-3/rules.yml @@ -89,7 +89,7 @@ groups: description: "GPG with fingerprint {{ $labels.fingerprint }} is expiring in less then six weeks" - alert: NadjaTopIPv4ProxyBroken expr: probe_success{job="blackbox_local_http6", target="blog.nadja.top"} != on (target) probe_success{job="blackbox_local_http4", target="blog.nadja.top"} - for: 5m + for: 15m labels: severity: critical annotations: From 9357981ff3486b1979f7a4574ec147e2fcce35a7 Mon Sep 17 00:00:00 2001 From: clerie Date: Sun, 17 Aug 2025 10:39:01 +0200 Subject: [PATCH 2/6] hosts/monitoring-3: Alert on fem.social unavailable --- hosts/monitoring-3/rules.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hosts/monitoring-3/rules.yml b/hosts/monitoring-3/rules.yml index f9bb742..2ed0b6b 100644 --- a/hosts/monitoring-3/rules.yml +++ b/hosts/monitoring-3/rules.yml @@ -102,3 +102,11 @@ groups: annotations: summary: "Too many notification requests failed" description: "Too many notification requests to Alertmanager integration {{ $labels.integration }} failed" + - alert: FemSocialDown + expr: min(probe_success{target="fem.social", job=~"blackbox_local_http.*"}) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "fem.social unavailable via HTTP" + description: "fem.social is not fully reachable via HTTP" From 22c7cb451b90b0b99fd81835480b3e9c65030622 Mon Sep 17 00:00:00 2001 From: clerie Date: Sun, 17 Aug 2025 19:05:22 +0200 Subject: [PATCH 3/6] pkgs/nixfiles: Add helper script to trigger system upgrades --- pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix | 10 ++++++++++ pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh | 5 +++++ pkgs/overlay.nix | 1 + 3 files changed, 16 insertions(+) create mode 100644 pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix create mode 100755 pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh diff --git a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix b/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix new file mode 100644 index 0000000..2c980d9 --- /dev/null +++ b/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix @@ -0,0 +1,10 @@ +{ pkgs, ... }: + +pkgs.writeShellApplication { + name = "nixfiles-trigger-system-upgrade"; + text = builtins.readFile ./nixfiles-trigger-system-upgrade.sh; + runtimeInputs = with pkgs; [ + pssh + ]; +} + diff --git a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh b/pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh new file mode 100755 index 0000000..a91d7be --- /dev/null +++ b/pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +TARGETS="$(nix --extra-experimental-features "nix-command flakes" eval --raw ".#nixosConfigurations" --apply "nixosConfigurations: builtins.concatStringsSep \"\\n\" (builtins.attrValues (builtins.mapAttrs (name: host: host.config.networking.fqdn) nixosConfigurations))")" + +pssh -h <(echo "${TARGETS}") -i -- sudo systemctl start clerie-system-auto-upgrade.service --no-block diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 5bb0aca..a6b0cd9 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -22,6 +22,7 @@ final: prev: { nixfiles-auto-install = final.callPackage ./nixfiles/nixfiles-auto-install.nix {}; nixfiles-generate-config = final.callPackage ./nixfiles/nixfiles-generate-config.nix {}; nixfiles-generate-backup-secrets = final.callPackage ./nixfiles/nixfiles-generate-backup-secrets.nix {}; + nixfiles-trigger-system-upgrade = final.callPackage ./nixfiles/nixfiles-trigger-system-upgrade.nix {}; nixfiles-update-ssh-host-keys = final.callPackage ./nixfiles/nixfiles-update-ssh-host-keys.nix {}; pipewire-all-bluetooth = final.callPackage ./pipewire-all-bluetooth {}; print-afra = final.callPackage ./print-afra {}; From 72cdef91d9be7a5a63b2f7b5fa3ee4eef1c62897 Mon Sep 17 00:00:00 2001 From: clerie Date: Sun, 17 Aug 2025 20:02:34 +0200 Subject: [PATCH 4/6] profiles/common-nix: Remove guests group from trusted nix users --- profiles/common-nix/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/common-nix/default.nix b/profiles/common-nix/default.nix index 0a50688..9cfe182 100644 --- a/profiles/common-nix/default.nix +++ b/profiles/common-nix/default.nix @@ -22,7 +22,7 @@ in { clerie.system-auto-upgrade.enable = true; nix.settings = { - trusted-users = [ "@wheel" "@guests" ]; + trusted-users = [ "@wheel" ]; auto-optimise-store = true; # Keep buildtime dependencies keep-outputs = true; From dd76691f7da3f860b25f770b65e602f90e1a1de8 Mon Sep 17 00:00:00 2001 From: clerie Date: Sun, 17 Aug 2025 21:49:24 +0200 Subject: [PATCH 5/6] pkgs/bijwerken-*,modules/bijwerken: Consolidate system update management and refactor under the same name --- hosts/dn42-il-gw1/configuration.nix | 2 +- hosts/dn42-il-gw5/configuration.nix | 2 +- hosts/dn42-il-gw6/configuration.nix | 2 +- hosts/dn42-ildix-clerie/configuration.nix | 2 +- hosts/dn42-ildix-service/configuration.nix | 2 +- hosts/nonat/configuration.nix | 2 +- hosts/porter/configuration.nix | 2 +- hosts/storage-2/configuration.nix | 2 +- .../default.nix | 19 ++++++++++++------- modules/default.nix | 2 +- modules/monitoring/default.nix | 2 ++ .../bijwerken-poke.sh} | 2 +- pkgs/bijwerken-poke/default.nix | 10 ++++++++++ .../bijwerken-system-upgrade.sh} | 2 +- .../default.nix} | 4 ++-- .../nixfiles-trigger-system-upgrade.nix | 10 ---------- pkgs/overlay.nix | 4 ++-- profiles/common-nix/default.nix | 2 +- 18 files changed, 40 insertions(+), 33 deletions(-) rename modules/{clerie-system-upgrade => bijwerken}/default.nix (62%) rename pkgs/{nixfiles/nixfiles-trigger-system-upgrade.sh => bijwerken-poke/bijwerken-poke.sh} (74%) create mode 100644 pkgs/bijwerken-poke/default.nix rename pkgs/{clerie-system-upgrade/clerie-system-upgrade.sh => bijwerken-system-upgrade/bijwerken-system-upgrade.sh} (95%) rename pkgs/{clerie-system-upgrade/clerie-system-upgrade.nix => bijwerken-system-upgrade/default.nix} (52%) delete mode 100644 pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix diff --git a/hosts/dn42-il-gw1/configuration.nix b/hosts/dn42-il-gw1/configuration.nix index 8328e3d..f7b22bf 100644 --- a/hosts/dn42-il-gw1/configuration.nix +++ b/hosts/dn42-il-gw1/configuration.nix @@ -237,7 +237,7 @@ ]; }; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/hosts/dn42-il-gw5/configuration.nix b/hosts/dn42-il-gw5/configuration.nix index b232429..b5d7ec6 100644 --- a/hosts/dn42-il-gw5/configuration.nix +++ b/hosts/dn42-il-gw5/configuration.nix @@ -111,7 +111,7 @@ ''; }; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; startAt = "*-*-* 06:22:00"; }; diff --git a/hosts/dn42-il-gw6/configuration.nix b/hosts/dn42-il-gw6/configuration.nix index 6491bda..b64f2bb 100644 --- a/hosts/dn42-il-gw6/configuration.nix +++ b/hosts/dn42-il-gw6/configuration.nix @@ -105,7 +105,7 @@ ''; }; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; startAt = "*-*-* 07:22:00"; }; diff --git a/hosts/dn42-ildix-clerie/configuration.nix b/hosts/dn42-ildix-clerie/configuration.nix index b6c4600..051cfa0 100644 --- a/hosts/dn42-ildix-clerie/configuration.nix +++ b/hosts/dn42-ildix-clerie/configuration.nix @@ -161,7 +161,7 @@ } ''; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/hosts/dn42-ildix-service/configuration.nix b/hosts/dn42-ildix-service/configuration.nix index 843da06..b56a280 100644 --- a/hosts/dn42-ildix-service/configuration.nix +++ b/hosts/dn42-ildix-service/configuration.nix @@ -70,7 +70,7 @@ networking.firewall.allowedTCPPorts = [ 80 443 ]; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/hosts/nonat/configuration.nix b/hosts/nonat/configuration.nix index 8d1e79f..b85b898 100644 --- a/hosts/nonat/configuration.nix +++ b/hosts/nonat/configuration.nix @@ -41,7 +41,7 @@ networking.firewall.allowedUDPPorts = []; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/hosts/porter/configuration.nix b/hosts/porter/configuration.nix index ae30793..1054d17 100644 --- a/hosts/porter/configuration.nix +++ b/hosts/porter/configuration.nix @@ -58,7 +58,7 @@ networking.firewall.allowedTCPPorts = [ 80 443 ]; networking.firewall.allowedUDPPorts = []; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/hosts/storage-2/configuration.nix b/hosts/storage-2/configuration.nix index ade3ece..91fff67 100644 --- a/hosts/storage-2/configuration.nix +++ b/hosts/storage-2/configuration.nix @@ -52,7 +52,7 @@ }; }; - clerie.system-auto-upgrade = { + services.bijwerken = { autoUpgrade = true; }; diff --git a/modules/clerie-system-upgrade/default.nix b/modules/bijwerken/default.nix similarity index 62% rename from modules/clerie-system-upgrade/default.nix rename to modules/bijwerken/default.nix index d2e7a26..ade6ee9 100644 --- a/modules/clerie-system-upgrade/default.nix +++ b/modules/bijwerken/default.nix @@ -3,13 +3,13 @@ with lib; let - cfg = config.clerie.system-auto-upgrade; + cfg = config.services.bijwerken; in { options = { - clerie.system-auto-upgrade = { - enable = mkEnableOption "clerie system upgrade"; + services.bijwerken = { + enable = mkEnableOption "Automatic system upgrades"; autoUpgrade = mkOption { type = types.bool; default = false; @@ -20,10 +20,15 @@ in default = null; description = "Systemd time string for starting the unit"; }; + nodeExporterTextfilePath = mkOption { + type = with types; nullOr str; + default = null; + description = "Path to node exporter textfile for putting metrics"; + }; }; }; config = mkIf cfg.enable { - systemd.services.clerie-system-auto-upgrade = { + systemd.services.bijwerken-system-upgrade = { requires = [ "network-online.target" ]; after = [ "network-online.target" ]; @@ -33,10 +38,10 @@ in serviceConfig = { Type = "oneshot"; - ExecStart = pkgs.clerie-system-upgrade + "/bin/clerie-system-upgrade --no-confirm${optionalString (config.clerie.monitoring.enable) " --node-exporter-metrics-path /var/lib/prometheus-node-exporter/textfiles/clerie-system-upgrade.prom"}"; + ExecStart = (getExe pkgs.bijwerken-system-upgrade) + " --no-confirm${optionalString (cfg.nodeExporterTextfilePath != null) " --node-exporter-metrics-path ${cfg.nodeExporterTextfilePath}"}"; }; }; - systemd.timers.clerie-system-auto-upgrade = mkIf cfg.autoUpgrade { + systemd.timers.bijwerken-system-upgrade = mkIf cfg.autoUpgrade { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = if cfg.startAt == null then "*-*-* 05:37:00" else cfg.startAt; @@ -46,7 +51,7 @@ in after = [ "network-online.target" ]; }; environment.systemPackages = with pkgs; [ - clerie-system-upgrade + bijwerken-system-upgrade ]; }; } diff --git a/modules/default.nix b/modules/default.nix index 41ba192..4952395 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -5,9 +5,9 @@ ./policyrouting ./akne ./backup + ./bijwerken ./clerie-firewall ./clerie-gc-dir - ./clerie-system-upgrade ./dhcpcd-prefixdelegation ./minecraft-server ./monitoring diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix index fd8bf99..483a716 100644 --- a/modules/monitoring/default.nix +++ b/modules/monitoring/default.nix @@ -75,6 +75,8 @@ in systemd.services."prometheus-node-exporter".serviceConfig.RestrictAddressFamilies = [ "AF_NETLINK" ]; + services.bijwerken.nodeExporterTextfilePath = "/var/lib/prometheus-node-exporter/textfiles/bijwerken-system-upgrade.prom"; + services.prometheus.exporters.bird = mkIf cfg.bird { enable = true; }; diff --git a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh b/pkgs/bijwerken-poke/bijwerken-poke.sh similarity index 74% rename from pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh rename to pkgs/bijwerken-poke/bijwerken-poke.sh index a91d7be..f17d968 100755 --- a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.sh +++ b/pkgs/bijwerken-poke/bijwerken-poke.sh @@ -2,4 +2,4 @@ TARGETS="$(nix --extra-experimental-features "nix-command flakes" eval --raw ".#nixosConfigurations" --apply "nixosConfigurations: builtins.concatStringsSep \"\\n\" (builtins.attrValues (builtins.mapAttrs (name: host: host.config.networking.fqdn) nixosConfigurations))")" -pssh -h <(echo "${TARGETS}") -i -- sudo systemctl start clerie-system-auto-upgrade.service --no-block +pssh -h <(echo "${TARGETS}") -i -- sudo systemctl start bijwerken-system-upgrade.service --no-block diff --git a/pkgs/bijwerken-poke/default.nix b/pkgs/bijwerken-poke/default.nix new file mode 100644 index 0000000..08e8750 --- /dev/null +++ b/pkgs/bijwerken-poke/default.nix @@ -0,0 +1,10 @@ +{ pkgs, ... }: + +pkgs.writeShellApplication { + name = "bijwerken-poke"; + text = builtins.readFile ./bijwerken-poke.sh; + runtimeInputs = with pkgs; [ + pssh + ]; +} + diff --git a/pkgs/clerie-system-upgrade/clerie-system-upgrade.sh b/pkgs/bijwerken-system-upgrade/bijwerken-system-upgrade.sh similarity index 95% rename from pkgs/clerie-system-upgrade/clerie-system-upgrade.sh rename to pkgs/bijwerken-system-upgrade/bijwerken-system-upgrade.sh index c5eee1b..729c5b1 100755 --- a/pkgs/clerie-system-upgrade/clerie-system-upgrade.sh +++ b/pkgs/bijwerken-system-upgrade/bijwerken-system-upgrade.sh @@ -50,7 +50,7 @@ echo "Set as boot target" if [[ -n "$NODE_EXPORTER_METRICS_PATH" ]]; then echo "Write monitoring check data" - echo "clerie_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH" + echo "bijwerken_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH" fi BOOTED_SYSTEM_KERNEL="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})" diff --git a/pkgs/clerie-system-upgrade/clerie-system-upgrade.nix b/pkgs/bijwerken-system-upgrade/default.nix similarity index 52% rename from pkgs/clerie-system-upgrade/clerie-system-upgrade.nix rename to pkgs/bijwerken-system-upgrade/default.nix index 5088be4..b8dc6c0 100644 --- a/pkgs/clerie-system-upgrade/clerie-system-upgrade.nix +++ b/pkgs/bijwerken-system-upgrade/default.nix @@ -1,8 +1,8 @@ { pkgs, ... }: pkgs.writeShellApplication { - name = "clerie-system-upgrade"; - text = builtins.readFile ./clerie-system-upgrade.sh; + name = "bijwerken-system-upgrade"; + text = builtins.readFile ./bijwerken-system-upgrade.sh; runtimeInputs = with pkgs; [ curl jq diff --git a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix b/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix deleted file mode 100644 index 2c980d9..0000000 --- a/pkgs/nixfiles/nixfiles-trigger-system-upgrade.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ pkgs, ... }: - -pkgs.writeShellApplication { - name = "nixfiles-trigger-system-upgrade"; - text = builtins.readFile ./nixfiles-trigger-system-upgrade.sh; - runtimeInputs = with pkgs; [ - pssh - ]; -} - diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index a6b0cd9..1d66db8 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -1,10 +1,11 @@ final: prev: { + bijwerken-poke = final.callPackage ./bijwerken-poke {}; + bijwerken-system-upgrade = final.callPackage ./bijwerken-system-upgrade {}; clerie-backup = final.callPackage ./clerie-backup {}; clerie-cleanup-branches = final.callPackage ./clerie-update-nixfiles/clerie-cleanup-branches.nix {}; clerie-keys = final.callPackage ./clerie-keys {}; clerie-ssh-known-hosts = final.callPackage ./clerie-ssh-known-hosts {}; clerie-system-remote-install = final.callPackage ./clerie-system-remote-install {}; - clerie-system-upgrade = final.callPackage ./clerie-system-upgrade/clerie-system-upgrade.nix {}; clerie-merge-nixfiles-update = final.callPackage ./clerie-update-nixfiles/clerie-merge-nixfiles-update.nix {}; clerie-sops = final.callPackage ./clerie-sops/clerie-sops.nix {}; clerie-sops-config = final.callPackage ./clerie-sops/clerie-sops-config.nix {}; @@ -22,7 +23,6 @@ final: prev: { nixfiles-auto-install = final.callPackage ./nixfiles/nixfiles-auto-install.nix {}; nixfiles-generate-config = final.callPackage ./nixfiles/nixfiles-generate-config.nix {}; nixfiles-generate-backup-secrets = final.callPackage ./nixfiles/nixfiles-generate-backup-secrets.nix {}; - nixfiles-trigger-system-upgrade = final.callPackage ./nixfiles/nixfiles-trigger-system-upgrade.nix {}; nixfiles-update-ssh-host-keys = final.callPackage ./nixfiles/nixfiles-update-ssh-host-keys.nix {}; pipewire-all-bluetooth = final.callPackage ./pipewire-all-bluetooth {}; print-afra = final.callPackage ./print-afra {}; diff --git a/profiles/common-nix/default.nix b/profiles/common-nix/default.nix index 9cfe182..58abb99 100644 --- a/profiles/common-nix/default.nix +++ b/profiles/common-nix/default.nix @@ -19,7 +19,7 @@ in { clerie.nixfiles.enable = true; - clerie.system-auto-upgrade.enable = true; + services.bijwerken.enable = true; nix.settings = { trusted-users = [ "@wheel" ]; From 168a349eedb5d72059bd14b6dd3836a9a9673042 Mon Sep 17 00:00:00 2001 From: Flake Update Bot Date: Mon, 18 Aug 2025 03:04:29 +0200 Subject: [PATCH 6/6] Update nixpkgs 2025-08-18-01-03 --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index ba0c6b0..57bd04e 100644 --- a/flake.lock +++ b/flake.lock @@ -646,11 +646,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1753549186, - "narHash": "sha256-Znl7rzuxKg/Mdm6AhimcKynM7V3YeNDIcLjBuoBcmNs=", + "lastModified": 1755186698, + "narHash": "sha256-wNO3+Ks2jZJ4nTHMuks+cxAiVBGNuEBXsT29Bz6HASo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "17f6bd177404d6d43017595c5264756764444ab8", + "rev": "fbcf476f790d8a217c3eab4e12033dc4a0f6d23c", "type": "github" }, "original": {