1
0

7 Commits

22 changed files with 480 additions and 26 deletions

6
flake.lock generated
View File

@@ -646,11 +646,11 @@
},
"nixpkgs_5": {
"locked": {
"lastModified": 1755186698,
"narHash": "sha256-wNO3+Ks2jZJ4nTHMuks+cxAiVBGNuEBXsT29Bz6HASo=",
"lastModified": 1756386758,
"narHash": "sha256-1wxxznpW2CKvI9VdniaUnTT2Os6rdRJcRUf65ZK9OtE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "fbcf476f790d8a217c3eab4e12033dc4a0f6d23c",
"rev": "dfb2f12e899db4876308eba6d93455ab7da304cd",
"type": "github"
},
"original": {

View File

@@ -237,7 +237,7 @@
];
};
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -111,7 +111,7 @@
'';
};
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
startAt = "*-*-* 06:22:00";
};

View File

@@ -105,7 +105,7 @@
'';
};
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
startAt = "*-*-* 07:22:00";
};

View File

@@ -161,7 +161,7 @@
}
'';
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -70,7 +70,7 @@
networking.firewall.allowedTCPPorts = [ 80 443 ];
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -0,0 +1,77 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 10,
"links": [],
"panels": [
{
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"includeVars": false,
"keepTime": false,
"maxItems": 10,
"query": "",
"showFolderNames": true,
"showHeadings": false,
"showRecentlyViewed": false,
"showSearch": true,
"showStarred": false,
"tags": []
},
"pluginVersion": "12.0.2+security-01",
"title": "Dashboards",
"type": "dashlist"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 41,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"hidden": true
},
"timezone": "browser",
"title": "Home",
"uid": "OqTN9p2nz",
"version": 1
}

View File

@@ -0,0 +1,135 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 15,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-RdYlGr"
},
"custom": {
"axisPlacement": "auto",
"fillOpacity": 70,
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineWidth": 0,
"spanNulls": false
},
"mappings": [
{
"options": {
"0": {
"index": 1,
"text": "mismatch"
},
"1": {
"index": 0,
"text": "sync"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red"
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 23,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"alignValue": "left",
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"mergeValues": true,
"rowHeight": 0.9,
"showValue": "auto",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.0.2+security-01",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "builder",
"expr": "nixos_current_system_is_sync",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Config is Sync",
"type": "state-timeline"
}
],
"preload": false,
"refresh": "5m",
"schemaVersion": 41,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-7d",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "NixOS Status",
"uid": "W4j3nz1Vz",
"version": 3
}

View File

@@ -0,0 +1,211 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 11,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 22,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.0.2+security-01",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "probe_icmp_duration_seconds{job=\"blackbox_icmp6\", target=~\"$target\", instance=~\"$instance\", phase=\"rtt\"} > 0 ",
"interval": "",
"legendFormat": "IPv6 {{target}} ({{instance}})",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "probe_icmp_duration_seconds{job=\"blackbox_icmp4\", target=~\"$target\", instance=~\"$instance\", phase=\"rtt\"} > 0",
"hide": false,
"interval": "",
"legendFormat": "IPv4 {{target}} ({{instance}})",
"range": true,
"refId": "B"
}
],
"title": "Smokeping",
"type": "timeseries"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 41,
"tags": [],
"templating": {
"list": [
{
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"definition": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, target)",
"includeAll": true,
"label": "Target:",
"multi": true,
"name": "target",
"options": [],
"query": {
"query": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, target)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
},
{
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"definition": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, instance)",
"includeAll": true,
"label": "Instance:",
"multi": true,
"name": "instance",
"options": [],
"query": {
"query": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Smokeping",
"uid": "IytTVZL7z",
"version": 9
}

View File

@@ -89,7 +89,7 @@ groups:
description: "GPG with fingerprint {{ $labels.fingerprint }} is expiring in less then six weeks"
- alert: NadjaTopIPv4ProxyBroken
expr: probe_success{job="blackbox_local_http6", target="blog.nadja.top"} != on (target) probe_success{job="blackbox_local_http4", target="blog.nadja.top"}
for: 5m
for: 15m
labels:
severity: critical
annotations:
@@ -102,3 +102,11 @@ groups:
annotations:
summary: "Too many notification requests failed"
description: "Too many notification requests to Alertmanager integration {{ $labels.integration }} failed"
- alert: FemSocialDown
expr: min(probe_success{target="fem.social", job=~"blackbox_local_http.*"}) == 0
for: 5m
labels:
severity: critical
annotations:
summary: "fem.social unavailable via HTTP"
description: "fem.social is not fully reachable via HTTP"

View File

@@ -41,7 +41,7 @@
networking.firewall.allowedUDPPorts = [];
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -58,7 +58,7 @@
networking.firewall.allowedTCPPorts = [ 80 443 ];
networking.firewall.allowedUDPPorts = [];
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -52,7 +52,7 @@
};
};
clerie.system-auto-upgrade = {
services.bijwerken = {
autoUpgrade = true;
};

View File

@@ -3,13 +3,13 @@
with lib;
let
cfg = config.clerie.system-auto-upgrade;
cfg = config.services.bijwerken;
in
{
options = {
clerie.system-auto-upgrade = {
enable = mkEnableOption "clerie system upgrade";
services.bijwerken = {
enable = mkEnableOption "Automatic system upgrades";
autoUpgrade = mkOption {
type = types.bool;
default = false;
@@ -20,10 +20,15 @@ in
default = null;
description = "Systemd time string for starting the unit";
};
nodeExporterTextfilePath = mkOption {
type = with types; nullOr str;
default = null;
description = "Path to node exporter textfile for putting metrics";
};
};
};
config = mkIf cfg.enable {
systemd.services.clerie-system-auto-upgrade = {
systemd.services.bijwerken-system-upgrade = {
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
@@ -33,10 +38,10 @@ in
serviceConfig = {
Type = "oneshot";
ExecStart = pkgs.clerie-system-upgrade + "/bin/clerie-system-upgrade --no-confirm${optionalString (config.clerie.monitoring.enable) " --node-exporter-metrics-path /var/lib/prometheus-node-exporter/textfiles/clerie-system-upgrade.prom"}";
ExecStart = (getExe pkgs.bijwerken-system-upgrade) + " --no-confirm${optionalString (cfg.nodeExporterTextfilePath != null) " --node-exporter-metrics-path ${cfg.nodeExporterTextfilePath}"}";
};
};
systemd.timers.clerie-system-auto-upgrade = mkIf cfg.autoUpgrade {
systemd.timers.bijwerken-system-upgrade = mkIf cfg.autoUpgrade {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = if cfg.startAt == null then "*-*-* 05:37:00" else cfg.startAt;
@@ -46,7 +51,7 @@ in
after = [ "network-online.target" ];
};
environment.systemPackages = with pkgs; [
clerie-system-upgrade
bijwerken-system-upgrade
];
};
}

View File

@@ -5,9 +5,9 @@
./policyrouting
./akne
./backup
./bijwerken
./clerie-firewall
./clerie-gc-dir
./clerie-system-upgrade
./dhcpcd-prefixdelegation
./minecraft-server
./monitoring

View File

@@ -75,6 +75,8 @@ in
systemd.services."prometheus-node-exporter".serviceConfig.RestrictAddressFamilies = [ "AF_NETLINK" ];
services.bijwerken.nodeExporterTextfilePath = "/var/lib/prometheus-node-exporter/textfiles/bijwerken-system-upgrade.prom";
services.prometheus.exporters.bird = mkIf cfg.bird {
enable = true;
};

View File

@@ -0,0 +1,5 @@
#!/usr/bin/env bash
TARGETS="$(nix --extra-experimental-features "nix-command flakes" eval --raw ".#nixosConfigurations" --apply "nixosConfigurations: builtins.concatStringsSep \"\\n\" (builtins.attrValues (builtins.mapAttrs (name: host: host.config.networking.fqdn) nixosConfigurations))")"
pssh -h <(echo "${TARGETS}") -i -- sudo systemctl start bijwerken-system-upgrade.service --no-block

View File

@@ -0,0 +1,10 @@
{ pkgs, ... }:
pkgs.writeShellApplication {
name = "bijwerken-poke";
text = builtins.readFile ./bijwerken-poke.sh;
runtimeInputs = with pkgs; [
pssh
];
}

View File

@@ -50,7 +50,7 @@ echo "Set as boot target"
if [[ -n "$NODE_EXPORTER_METRICS_PATH" ]]; then
echo "Write monitoring check data"
echo "clerie_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH"
echo "bijwerken_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH"
fi
BOOTED_SYSTEM_KERNEL="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"

View File

@@ -1,8 +1,8 @@
{ pkgs, ... }:
pkgs.writeShellApplication {
name = "clerie-system-upgrade";
text = builtins.readFile ./clerie-system-upgrade.sh;
name = "bijwerken-system-upgrade";
text = builtins.readFile ./bijwerken-system-upgrade.sh;
runtimeInputs = with pkgs; [
curl
jq

View File

@@ -1,10 +1,11 @@
final: prev: {
bijwerken-poke = final.callPackage ./bijwerken-poke {};
bijwerken-system-upgrade = final.callPackage ./bijwerken-system-upgrade {};
clerie-backup = final.callPackage ./clerie-backup {};
clerie-cleanup-branches = final.callPackage ./clerie-update-nixfiles/clerie-cleanup-branches.nix {};
clerie-keys = final.callPackage ./clerie-keys {};
clerie-ssh-known-hosts = final.callPackage ./clerie-ssh-known-hosts {};
clerie-system-remote-install = final.callPackage ./clerie-system-remote-install {};
clerie-system-upgrade = final.callPackage ./clerie-system-upgrade/clerie-system-upgrade.nix {};
clerie-merge-nixfiles-update = final.callPackage ./clerie-update-nixfiles/clerie-merge-nixfiles-update.nix {};
clerie-sops = final.callPackage ./clerie-sops/clerie-sops.nix {};
clerie-sops-config = final.callPackage ./clerie-sops/clerie-sops-config.nix {};

View File

@@ -19,10 +19,10 @@ in {
clerie.nixfiles.enable = true;
clerie.system-auto-upgrade.enable = true;
services.bijwerken.enable = true;
nix.settings = {
trusted-users = [ "@wheel" "@guests" ];
trusted-users = [ "@wheel" ];
auto-optimise-store = true;
# Keep buildtime dependencies
keep-outputs = true;