From cff95863fdb00a00a2b658d05f61a1bbbed013a3 Mon Sep 17 00:00:00 2001 From: clerie Date: Fri, 1 Dec 2023 17:50:54 +0100 Subject: [PATCH 1/5] hosts/monitoring-3: Add alert for Synapse --- hosts/monitoring-3/prometheus.nix | 5 +++-- hosts/monitoring-3/rules.yml | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hosts/monitoring-3/prometheus.nix b/hosts/monitoring-3/prometheus.nix index e5b7286..d8ca1d9 100644 --- a/hosts/monitoring-3/prometheus.nix +++ b/hosts/monitoring-3/prometheus.nix @@ -273,7 +273,7 @@ in { static_configs = [ { targets = [ - "https://matrix.entr0py.de/_matrix/static/" + "matrix.entr0py.de" ]; } ]; @@ -281,9 +281,10 @@ in { { source_labels = [ "__address__" ]; target_label = "__param_target"; + replacement = "https://\${1}/_matrix/static/"; } { - source_labels = [ "__param_target" ]; + source_labels = [ "__address__" ]; target_label = "target"; } { diff --git a/hosts/monitoring-3/rules.yml b/hosts/monitoring-3/rules.yml index e0b3cd5..ae906a7 100644 --- a/hosts/monitoring-3/rules.yml +++ b/hosts/monitoring-3/rules.yml @@ -57,3 +57,11 @@ groups: annotations: summary: "Kernel of {{ $labels.instance }} changed" description: "The Kernel {{ $labels.instance }} booted with, isn't the target Kernel. A reboot may be required." + - alert: SynapseUnavailable + expr: last_over_time(probe_success{instance="monitoring-3.net.clerie.de", job="blackbox_local_synapse", target="matrix.entr0py.de"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Synapse of {{ $labels.target }} unavailable" + description: "The Synapse backend of {{ $labels.instance }} is unreachable or returns garbage" From 01b1ce3caaf26f90c084d4b356053a2d79fcb9fa Mon Sep 17 00:00:00 2001 From: clerie Date: Fri, 1 Dec 2023 18:02:44 +0100 Subject: [PATCH 2/5] pkgs/nixfiles: rename nixfiles-system-sync-to-hydra to nixfiles-system-upgrade --- flake.nix | 2 +- hosts/dn42-il-gw1/configuration.nix | 2 +- hosts/dn42-il-gw5/configuration.nix | 2 +- hosts/dn42-il-gw6/configuration.nix | 2 +- hosts/nonat/configuration.nix | 2 +- modules/default.nix | 2 +- ...auto-sync-to-hydra.nix => nixfiles-system-upgrade.nix} | 8 ++++---- ...stem-sync-to-hydra.nix => nixfiles-system-upgrade.nix} | 4 ++-- ...system-sync-to-hydra.sh => nixfiles-system-upgrade.sh} | 0 pkgs/overlay.nix | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) rename modules/nixfiles/{system-auto-sync-to-hydra.nix => nixfiles-system-upgrade.nix} (55%) rename pkgs/nixfiles/{nixfiles-system-sync-to-hydra.nix => nixfiles-system-upgrade.nix} (50%) rename pkgs/nixfiles/{nixfiles-system-sync-to-hydra.sh => nixfiles-system-upgrade.sh} (100%) diff --git a/flake.nix b/flake.nix index 9830c65..3ae7d60 100644 --- a/flake.nix +++ b/flake.nix @@ -107,7 +107,7 @@ nixfiles-auto-install nixfiles-generate-backup-secrets nixfiles-generate-config - nixfiles-system-sync-to-hydra + nixfiles-system-upgrade nixfiles-updated-inputs nixfiles-update-ssh-host-keys nixos-firewall-tool diff --git a/hosts/dn42-il-gw1/configuration.nix b/hosts/dn42-il-gw1/configuration.nix index ea58042..35118d9 100644 --- a/hosts/dn42-il-gw1/configuration.nix +++ b/hosts/dn42-il-gw1/configuration.nix @@ -329,7 +329,7 @@ } ''; - clerie.nixfiles.system-auto-sync-to-hydra = { + clerie.nixfiles.system-auto-upgrade = { enable = true; allowReboot = true; }; diff --git a/hosts/dn42-il-gw5/configuration.nix b/hosts/dn42-il-gw5/configuration.nix index 4d28052..9b0295d 100644 --- a/hosts/dn42-il-gw5/configuration.nix +++ b/hosts/dn42-il-gw5/configuration.nix @@ -183,7 +183,7 @@ } ''; - clerie.nixfiles.system-auto-sync-to-hydra = { + clerie.nixfiles.system-auto-upgrade = { enable = true; allowReboot = true; }; diff --git a/hosts/dn42-il-gw6/configuration.nix b/hosts/dn42-il-gw6/configuration.nix index 15837c2..03be38d 100644 --- a/hosts/dn42-il-gw6/configuration.nix +++ b/hosts/dn42-il-gw6/configuration.nix @@ -183,7 +183,7 @@ } ''; - clerie.nixfiles.system-auto-sync-to-hydra = { + clerie.nixfiles.system-auto-upgrade = { enable = true; allowReboot = true; }; diff --git a/hosts/nonat/configuration.nix b/hosts/nonat/configuration.nix index 3dde127..8c8e4e8 100644 --- a/hosts/nonat/configuration.nix +++ b/hosts/nonat/configuration.nix @@ -39,7 +39,7 @@ networking.firewall.allowedUDPPorts = []; - clerie.nixfiles.system-auto-sync-to-hydra = { + clerie.nixfiles.system-auto-upgrade = { enable = true; allowReboot = true; }; diff --git a/modules/default.nix b/modules/default.nix index c43aede..08b5ed4 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -13,7 +13,7 @@ ./monitoring ./nginx-port-forward ./nixfiles - ./nixfiles/system-auto-sync-to-hydra.nix + ./nixfiles/nixfiles-system-upgrade.nix ./update-from-hydra ./wg-clerie ./wireguard-initrd diff --git a/modules/nixfiles/system-auto-sync-to-hydra.nix b/modules/nixfiles/nixfiles-system-upgrade.nix similarity index 55% rename from modules/nixfiles/system-auto-sync-to-hydra.nix rename to modules/nixfiles/nixfiles-system-upgrade.nix index 3d41ea1..a848a92 100644 --- a/modules/nixfiles/system-auto-sync-to-hydra.nix +++ b/modules/nixfiles/nixfiles-system-upgrade.nix @@ -3,12 +3,12 @@ with lib; let - cfg = config.clerie.nixfiles.system-auto-sync-to-hydra; + cfg = config.clerie.nixfiles.system-auto-upgrade; in { options = { - clerie.nixfiles.system-auto-sync-to-hydra = { + clerie.nixfiles.system-auto-upgrade = { enable = mkEnableOption "clerie nixfiles tools"; allowReboot = mkOption { type = types.bool; @@ -18,10 +18,10 @@ in }; }; config = mkIf cfg.enable { - systemd.services.nixfiles-system-auto-sync-to-hydra = { + systemd.services.nixfiles-system-auto-upgrade = { serviceConfig = { Type = "oneshot"; - ExecStart = pkgs.nixfiles-system-sync-to-hydra + "/bin/nixfiles-system-sync-to-hydra${optionalString cfg.allowReboot " --allow-reboot"}"; + ExecStart = pkgs.nixfiles-system-upgrade + "/bin/nixfiles-system-upgrade${optionalString cfg.allowReboot " --allow-reboot"}"; }; startAt = "*-*-* 06:47:00"; }; diff --git a/pkgs/nixfiles/nixfiles-system-sync-to-hydra.nix b/pkgs/nixfiles/nixfiles-system-upgrade.nix similarity index 50% rename from pkgs/nixfiles/nixfiles-system-sync-to-hydra.nix rename to pkgs/nixfiles/nixfiles-system-upgrade.nix index 41c32d2..9d494f4 100644 --- a/pkgs/nixfiles/nixfiles-system-sync-to-hydra.nix +++ b/pkgs/nixfiles/nixfiles-system-upgrade.nix @@ -1,8 +1,8 @@ { pkgs, ... }: pkgs.writeShellApplication { - name = "nixfiles-system-sync-to-hydra"; - text = builtins.readFile ./nixfiles-system-sync-to-hydra.sh; + name = "nixfiles-system-upgrade"; + text = builtins.readFile ./nixfiles-system-upgrade.sh; runtimeInputs = with pkgs; [ curl jq diff --git a/pkgs/nixfiles/nixfiles-system-sync-to-hydra.sh b/pkgs/nixfiles/nixfiles-system-upgrade.sh similarity index 100% rename from pkgs/nixfiles/nixfiles-system-sync-to-hydra.sh rename to pkgs/nixfiles/nixfiles-system-upgrade.sh diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 2d5ffdf..743818a 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -6,7 +6,7 @@ self: super: { nixfiles-auto-install = self.callPackage ./nixfiles/nixfiles-auto-install.nix {}; nixfiles-generate-backup-secrets = self.callPackage ./nixfiles/nixfiles-generate-backup-secrets.nix {}; nixfiles-generate-config = self.callPackage ./nixfiles/nixfiles-generate-config.nix {}; - nixfiles-system-sync-to-hydra = self.callPackage ./nixfiles/nixfiles-system-sync-to-hydra.nix {}; + nixfiles-system-upgrade = self.callPackage ./nixfiles/nixfiles-system-upgrade.nix {}; nixfiles-updated-inputs = self.callPackage ./nixfiles/nixfiles-updated-inputs.nix {}; nixfiles-update-ssh-host-keys = self.callPackage ./nixfiles/nixfiles-update-ssh-host-keys.nix {}; nixos-firewall-tool = self.callPackage ./nixos-firewall-tool {}; From b37bdf88e100532e7eec760fec0dbfd4b6fe70c9 Mon Sep 17 00:00:00 2001 From: clerie Date: Fri, 1 Dec 2023 18:14:25 +0100 Subject: [PATCH 3/5] pkgs/nixfiles: Ask before doing a system upgrade from Hydra --- modules/nixfiles/nixfiles-system-upgrade.nix | 2 +- pkgs/nixfiles/nixfiles-system-upgrade.sh | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/nixfiles/nixfiles-system-upgrade.nix b/modules/nixfiles/nixfiles-system-upgrade.nix index a848a92..546478b 100644 --- a/modules/nixfiles/nixfiles-system-upgrade.nix +++ b/modules/nixfiles/nixfiles-system-upgrade.nix @@ -21,7 +21,7 @@ in systemd.services.nixfiles-system-auto-upgrade = { serviceConfig = { Type = "oneshot"; - ExecStart = pkgs.nixfiles-system-upgrade + "/bin/nixfiles-system-upgrade${optionalString cfg.allowReboot " --allow-reboot"}"; + ExecStart = pkgs.nixfiles-system-upgrade + "/bin/nixfiles-system-upgrade --no-confirm${optionalString cfg.allowReboot " --allow-reboot"}"; }; startAt = "*-*-* 06:47:00"; }; diff --git a/pkgs/nixfiles/nixfiles-system-upgrade.sh b/pkgs/nixfiles/nixfiles-system-upgrade.sh index d3da0c0..9ad1b96 100755 --- a/pkgs/nixfiles/nixfiles-system-upgrade.sh +++ b/pkgs/nixfiles/nixfiles-system-upgrade.sh @@ -3,17 +3,22 @@ set -euo pipefail ALLOW_REBOOT= +NO_CONFIRM= while [[ $# -gt 0 ]]; do case $1 in --allow-reboot) ALLOW_REBOOT=1 shift - ;; + ;; + --no-confirm) + NO_CONFIRM=1 + shift + ;; *) echo "Unknown option $1" exit 1 - ;; + ;; esac done @@ -22,6 +27,17 @@ HYDRA_JOB_URL="https://hydra.clerie.de/job/nixfiles/nixfiles/nixosConfigurations echo "Fetching job output from ${HYDRA_JOB_URL}" STORE_PATH="$(curl --fail -s -L -H "Accept: application/json" "${HYDRA_JOB_URL}" | jq -r ".buildoutputs.out.path")" +if [[ -z $NO_CONFIRM ]]; then + echo "" + echo " ! WARNING !" + echo "" + echo " You are about to upgrade ${HOSTNAME} to ${STORE_PATH}." + echo " This can be an older version than currently running on this system." + echo "" + read -e -r -p "Continue?" confirm + echo "$confirm" > /dev/null +fi + echo "Download ${STORE_PATH}" nix copy --from "https://nix-cache.clerie.de" "${STORE_PATH}" From 3168a58878eb92aac1b5c83f6caa588995a9446a Mon Sep 17 00:00:00 2001 From: clerie Date: Fri, 1 Dec 2023 18:14:59 +0100 Subject: [PATCH 4/5] configuration/common: Add nixfiles-system-upgrade to environment --- configuration/common/programs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/common/programs.nix b/configuration/common/programs.nix index 2eafc83..d0e80cc 100644 --- a/configuration/common/programs.nix +++ b/configuration/common/programs.nix @@ -11,6 +11,7 @@ vim agenix nixos-firewall-tool + nixfiles-system-upgrade ]; programs.mtr.enable = true; From b093ae28be1ec9b1054f2e15e337fb818dc55949 Mon Sep 17 00:00:00 2001 From: clerie Date: Fri, 1 Dec 2023 18:28:36 +0100 Subject: [PATCH 5/5] flake.lock: Update input nixpkgs-krypton --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 0359302..f204604 100644 --- a/flake.lock +++ b/flake.lock @@ -196,11 +196,11 @@ }, "nixpkgs-krypton": { "locked": { - "lastModified": 1699099776, - "narHash": "sha256-X09iKJ27mGsGambGfkKzqvw5esP1L/Rf8H3u3fCqIiU=", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "85f1ba3e51676fa8cc604a3d863d729026a6b8eb", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", "type": "github" }, "original": {