Update nixpkgs 2025-08-28-01-03

flake.lock

@@ -646,11 +646,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1755027561,
+        "lastModified": 1756266583,
-        "narHash": "sha256-IVft239Bc8p8Dtvf7UAACMG5P3ZV+3/aO28gXpGtMXI=",
+        "narHash": "sha256-cr748nSmpfvnhqSXPiCfUPxRz2FJnvf/RjJGvFfaCsM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "005433b926e16227259a1843015b5b2b7f7d1fc3",
+        "rev": "8a6d5427d99ec71c64f0b93d45778c889005d9c2",
         "type": "github"
       },
       "original": {

hosts/dn42-il-gw1/configuration.nix

@@ -237,7 +237,7 @@
     ];
   };
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/dn42-il-gw5/configuration.nix

@@ -111,7 +111,7 @@
     '';
   };
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
     startAt = "*-*-* 06:22:00";
   };

hosts/dn42-il-gw6/configuration.nix

@@ -105,7 +105,7 @@
     '';
   };
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
     startAt = "*-*-* 07:22:00";
   };

hosts/dn42-ildix-clerie/configuration.nix

@@ -161,7 +161,7 @@
   }
   '';
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/dn42-ildix-service/configuration.nix

@@ -70,7 +70,7 @@
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/monitoring-3/dashboards/home.json

@@ -0,0 +1,77 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 10,
+  "links": [],
+  "panels": [
+    {
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "includeVars": false,
+        "keepTime": false,
+        "maxItems": 10,
+        "query": "",
+        "showFolderNames": true,
+        "showHeadings": false,
+        "showRecentlyViewed": false,
+        "showSearch": true,
+        "showStarred": false,
+        "tags": []
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "title": "Dashboards",
+      "type": "dashlist"
+    }
+  ],
+  "preload": false,
+  "refresh": "",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "hidden": true
+  },
+  "timezone": "browser",
+  "title": "Home",
+  "uid": "OqTN9p2nz",
+  "version": 1
+}

hosts/monitoring-3/dashboards/nginx-exporter.json

@@ -0,0 +1,355 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 16,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(server_name) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Total requests",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(server_name, method) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "{{server_name}}: {{method}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Status codes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(server_name, status) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "{{server_name}}: HTTP {{status}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Response codes",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "refresh": "30s",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "All",
+          "value": [
+            "$__all"
+          ]
+        },
+        "definition": "label_values(nginxlog_http_response_count_total,server_name)",
+        "includeAll": true,
+        "label": "vHost",
+        "multi": true,
+        "name": "server_name",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(nginxlog_http_response_count_total,server_name)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-3h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Nginx Exporter",
+  "uid": "b042a880-3cb0-4dd3-ae48-4745a58af698",
+  "version": 7
+}

hosts/monitoring-3/dashboards/nixos-status.json

@@ -0,0 +1,135 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 15,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-RdYlGr"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "index": 1,
+                  "text": "mismatch"
+                },
+                "1": {
+                  "index": 0,
+                  "text": "sync"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red"
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 23,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "auto",
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "builder",
+          "expr": "nixos_current_system_is_sync",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Config is Sync",
+      "type": "state-timeline"
+    }
+  ],
+  "preload": false,
+  "refresh": "5m",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-7d",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "NixOS Status",
+  "uid": "W4j3nz1Vz",
+  "version": 3
+}

hosts/monitoring-3/dashboards/smokeping.json

@@ -0,0 +1,211 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 11,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 22,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2+security-01",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "probe_icmp_duration_seconds{job=\"blackbox_icmp6\", target=~\"$target\", instance=~\"$instance\", phase=\"rtt\"} > 0 ",
+          "interval": "",
+          "legendFormat": "IPv6 {{target}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "probe_icmp_duration_seconds{job=\"blackbox_icmp4\", target=~\"$target\", instance=~\"$instance\", phase=\"rtt\"} > 0",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "IPv4 {{target}} ({{instance}})",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Smokeping",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "refresh": "",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, target)",
+        "includeAll": true,
+        "label": "Target:",
+        "multi": true,
+        "name": "target",
+        "options": [],
+        "query": {
+          "query": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, target)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, instance)",
+        "includeAll": true,
+        "label": "Instance:",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "label_values(probe_icmp_duration_seconds{phase=\"rtt\"}, instance)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Smokeping",
+  "uid": "IytTVZL7z",
+  "version": 9
+}

hosts/monitoring-3/prometheus.nix

@@ -52,6 +52,12 @@ let
     attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
     monitoringHosts);
 
+  nginxlogMonitoringTargets = mapAttrsToList (name: host:
+    "${host.config.networking.hostName}.mon.clerie.de:9117")
+    (filterAttrs (name: host:
+    attrByPath ["services" "prometheus" "exporters" "nginxlog" "enable"] false host.config)
+    monitoringHosts);
+
   eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
 
 in {
@@ -536,6 +542,18 @@ in {
           }
         ];
       }
+      {
+        job_name = "nginxlog-exporter";
+        scrape_interval = "20s";
+        static_configs = [
+          {
+            targets = nginxlogMonitoringTargets;
+          }
+        ];
+        relabel_configs = [
+          relabelAddressToInstance
+        ];
+      }
     ];
     alertmanagers = [
       {

hosts/monitoring-3/rules.yml

@@ -89,7 +89,7 @@ groups:
       description: "GPG with fingerprint {{ $labels.fingerprint }} is expiring in less then six weeks"
   - alert: NadjaTopIPv4ProxyBroken
     expr: probe_success{job="blackbox_local_http6", target="blog.nadja.top"} != on (target) probe_success{job="blackbox_local_http4", target="blog.nadja.top"}
-    for: 5m
+    for: 15m
     labels:
       severity: critical
     annotations:
@@ -102,3 +102,11 @@ groups:
     annotations:
       summary: "Too many notification requests failed"
       description: "Too many notification requests to Alertmanager integration {{ $labels.integration }} failed"
+  - alert: FemSocialDown
+    expr: min(probe_success{target="fem.social", job=~"blackbox_local_http.*"}) == 0
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: "fem.social unavailable via HTTP"
+      description: "fem.social is not fully reachable via HTTP"

hosts/nonat/configuration.nix

@@ -41,7 +41,7 @@
 
   networking.firewall.allowedUDPPorts = [];
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/porter/configuration.nix

@@ -58,7 +58,7 @@
   networking.firewall.allowedTCPPorts = [ 80 443 ];
   networking.firewall.allowedUDPPorts = [];
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/storage-2/configuration.nix

@@ -52,7 +52,7 @@
     };
   };
 
-  clerie.system-auto-upgrade = {
+  services.bijwerken = {
     autoUpgrade = true;
   };
 

hosts/web-2/clerie.nix

@@ -53,9 +53,6 @@
         '';
         return = "200 ''";
       };
-      extraConfig = ''
-        access_log /var/log/nginx/clerie.de.log combined_anon;
-      '';
     };
   };
 }

hosts/web-2/gitea.nix

@@ -83,9 +83,6 @@
           proxyPass = "http://[::1]:3000";
         };
       };
-      extraConfig = ''
-        access_log /var/log/nginx/git.clerie.de.log combined_anon;
-      '';
     };
   };
 }

hosts/web-2/ip.nix

@@ -53,9 +53,6 @@
           types { } default_type "text/html; charset=utf-8";
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
     "ip4.clerie.de" = {
       enableACME = true;
@@ -67,9 +64,6 @@
           add_header Access-Control-Allow-Origin *;
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
     "ip6.clerie.de" = {
       enableACME = true;
@@ -81,9 +75,6 @@
           add_header Access-Control-Allow-Origin *;
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
   };
 }

modules/bijwerken/default.nix

@@ -3,13 +3,13 @@
 with lib;
 
 let
-  cfg = config.clerie.system-auto-upgrade;
+  cfg = config.services.bijwerken;
 in
 
 {
   options = {
-    clerie.system-auto-upgrade = {
+    services.bijwerken = {
-      enable = mkEnableOption "clerie system upgrade";
+      enable = mkEnableOption "Automatic system upgrades";
       autoUpgrade = mkOption {
         type = types.bool;
         default = false;
@@ -20,10 +20,15 @@ in
         default = null;
         description = "Systemd time string for starting the unit";
       };
+      nodeExporterTextfilePath = mkOption {
+        type = with types; nullOr str;
+        default = null;
+        description = "Path to node exporter textfile for putting metrics";
+      };
     };
   };
   config = mkIf cfg.enable {
-    systemd.services.clerie-system-auto-upgrade = {
+    systemd.services.bijwerken-system-upgrade = {
       requires = [ "network-online.target" ];
       after = [ "network-online.target" ];
 
@@ -33,10 +38,10 @@ in
 
       serviceConfig = {
         Type = "oneshot";
-        ExecStart = pkgs.clerie-system-upgrade + "/bin/clerie-system-upgrade --no-confirm${optionalString (config.clerie.monitoring.enable) " --node-exporter-metrics-path /var/lib/prometheus-node-exporter/textfiles/clerie-system-upgrade.prom"}";
+        ExecStart = (getExe pkgs.bijwerken-system-upgrade) + " --no-confirm${optionalString (cfg.nodeExporterTextfilePath != null) " --node-exporter-metrics-path ${cfg.nodeExporterTextfilePath}"}";
       };
     };
-    systemd.timers.clerie-system-auto-upgrade = mkIf cfg.autoUpgrade {
+    systemd.timers.bijwerken-system-upgrade = mkIf cfg.autoUpgrade {
       wantedBy = [ "timers.target" ];
       timerConfig = {
         OnCalendar = if cfg.startAt == null then "*-*-* 05:37:00" else cfg.startAt;
@@ -46,7 +51,7 @@ in
       after = [ "network-online.target" ];
     };
     environment.systemPackages = with pkgs; [
-      clerie-system-upgrade
+      bijwerken-system-upgrade
     ];
   };
 }

modules/default.nix

@@ -5,9 +5,9 @@
     ./policyrouting
     ./akne
     ./backup
+    ./bijwerken
     ./clerie-firewall
     ./clerie-gc-dir
-    ./clerie-system-upgrade
     ./dhcpcd-prefixdelegation
     ./minecraft-server
     ./monitoring

modules/monitoring/default.nix

@@ -75,6 +75,8 @@ in
 
     systemd.services."prometheus-node-exporter".serviceConfig.RestrictAddressFamilies = [ "AF_NETLINK" ];
 
+    services.bijwerken.nodeExporterTextfilePath = "/var/lib/prometheus-node-exporter/textfiles/bijwerken-system-upgrade.prom";
+
     services.prometheus.exporters.bird = mkIf cfg.bird {
       enable = true;
     };
@@ -102,6 +104,33 @@ in
       listen = "[::]:9152";
     };
 
+    services.prometheus.exporters.nginxlog = mkIf config.services.nginx.enable {
+      enable = true;
+      settings = {
+        namespaces = [
+          {
+            name = "nginxlog";
+            format = ''$host: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$server_name" rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
+            source = {
+              files = [
+                "/var/log/nginx/access.log"
+              ];
+            };
+            relabel_configs = [
+              {
+                target_label = "server_name";
+                from = "server_name";
+              }
+            ];
+          }
+        ];
+      };
+    };
+
+    systemd.services."prometheus-nginxlog-exporter".serviceConfig = {
+      SupplementaryGroups = "nginx";
+    };
+
     networking.firewall.interfaces."wg-monitoring".allowedTCPPorts = [
       9100 # node-exporter
       9152 # nixos-exporter
@@ -109,6 +138,8 @@ in
       9324 # bird-exporter
     ] else []) ++ (if cfg.blackbox then [
       9115 # blackbox-exporter
+    ] else []) ++ (if config.services.prometheus.exporters.nginxlog.enable then [
+      config.services.prometheus.exporters.nginxlog.port
     ] else []);
   };
 }

pkgs/bijwerken-poke/bijwerken-poke.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+TARGETS="$(nix --extra-experimental-features "nix-command flakes" eval --raw ".#nixosConfigurations" --apply "nixosConfigurations: builtins.concatStringsSep \"\\n\" (builtins.attrValues (builtins.mapAttrs (name: host: host.config.networking.fqdn) nixosConfigurations))")"
+
+pssh -h <(echo "${TARGETS}") -i -- sudo systemctl start bijwerken-system-upgrade.service --no-block

pkgs/bijwerken-poke/default.nix

@@ -0,0 +1,10 @@
+{ pkgs, ... }:
+
+pkgs.writeShellApplication {
+  name = "bijwerken-poke";
+  text = builtins.readFile ./bijwerken-poke.sh;
+  runtimeInputs = with pkgs; [
+    pssh
+  ];
+}
+

pkgs/bijwerken-system-upgrade/bijwerken-system-upgrade.sh

@@ -50,7 +50,7 @@ echo "Set as boot target"
 
 if [[ -n "$NODE_EXPORTER_METRICS_PATH" ]]; then
 	echo "Write monitoring check data"
-	echo "clerie_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH"
+	echo "bijwerken_system_upgrade_last_check $(date +%s)" > "$NODE_EXPORTER_METRICS_PATH"
 fi
 
 BOOTED_SYSTEM_KERNEL="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"

pkgs/bijwerken-system-upgrade/default.nix

@@ -1,8 +1,8 @@
 { pkgs, ... }:
 
 pkgs.writeShellApplication {
-  name = "clerie-system-upgrade";
+  name = "bijwerken-system-upgrade";
-  text = builtins.readFile ./clerie-system-upgrade.sh;
+  text = builtins.readFile ./bijwerken-system-upgrade.sh;
   runtimeInputs = with pkgs; [
     curl
     jq

pkgs/overlay.nix

@@ -1,10 +1,11 @@
 final: prev: {
+  bijwerken-poke = final.callPackage ./bijwerken-poke {};
+  bijwerken-system-upgrade = final.callPackage ./bijwerken-system-upgrade {};
   clerie-backup = final.callPackage ./clerie-backup {};
   clerie-cleanup-branches = final.callPackage ./clerie-update-nixfiles/clerie-cleanup-branches.nix {};
   clerie-keys = final.callPackage ./clerie-keys {};
   clerie-ssh-known-hosts = final.callPackage ./clerie-ssh-known-hosts {};
   clerie-system-remote-install = final.callPackage ./clerie-system-remote-install {};
-  clerie-system-upgrade = final.callPackage ./clerie-system-upgrade/clerie-system-upgrade.nix {};
   clerie-merge-nixfiles-update = final.callPackage ./clerie-update-nixfiles/clerie-merge-nixfiles-update.nix {};
   clerie-sops = final.callPackage ./clerie-sops/clerie-sops.nix {};
   clerie-sops-config = final.callPackage ./clerie-sops/clerie-sops-config.nix {};

profiles/common-nix/default.nix

@@ -19,10 +19,10 @@ in {
 
     clerie.nixfiles.enable = true;
 
-    clerie.system-auto-upgrade.enable = true;
+    services.bijwerken.enable = true;
 
     nix.settings = {
-      trusted-users = [ "@wheel" "@guests" ];
+      trusted-users = [ "@wheel" ];
       auto-optimise-store = true;
       # Keep buildtime dependencies
       keep-outputs = true;

profiles/common-webserver/default.nix

@@ -40,7 +40,12 @@ in {
         log_format vcombined_anon '$host: $remote_addr_anon - $remote_user [$time_local] '
                                   '"$request" $status $body_bytes_sent '
                                   '"$http_referer" "$http_user_agent"';
-        access_log /var/log/nginx/access.log vcombined_anon;
+        log_format vcombined_anon_monitoring '$host: $remote_addr_anon - $remote_user [$time_local] '
+                                  '"$request" $status $body_bytes_sent '
+                                  '"$http_referer" "$http_user_agent" '
+                                  '"$server_name" '
+                                  'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+        access_log /var/log/nginx/access.log vcombined_anon_monitoring;
       '';
 
       virtualHosts = mkIf cfg.httpDefaultVirtualHost {