diff --git a/flake.lock b/flake.lock index 303d7b5..57bd04e 100644 --- a/flake.lock +++ b/flake.lock @@ -646,11 +646,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1755027561, - "narHash": "sha256-IVft239Bc8p8Dtvf7UAACMG5P3ZV+3/aO28gXpGtMXI=", + "lastModified": 1755186698, + "narHash": "sha256-wNO3+Ks2jZJ4nTHMuks+cxAiVBGNuEBXsT29Bz6HASo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "005433b926e16227259a1843015b5b2b7f7d1fc3", + "rev": "fbcf476f790d8a217c3eab4e12033dc4a0f6d23c", "type": "github" }, "original": { diff --git a/hosts/monitoring-3/dashboards/nginx-exporter.json b/hosts/monitoring-3/dashboards/nginx-exporter.json new file mode 100644 index 0000000..227d7ab --- /dev/null +++ b/hosts/monitoring-3/dashboards/nginx-exporter.json @@ -0,0 +1,355 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 16, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(server_name) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(server_name, method) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{server_name}}: {{method}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status codes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(server_name, status) (rate(nginxlog_http_response_count_total{server_name=~\"$server_name\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{server_name}}: HTTP {{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Response codes", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(nginxlog_http_response_count_total,server_name)", + "includeAll": true, + "label": "vHost", + "multi": true, + "name": "server_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(nginxlog_http_response_count_total,server_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Nginx Exporter", + "uid": "b042a880-3cb0-4dd3-ae48-4745a58af698", + "version": 7 +} \ No newline at end of file diff --git a/hosts/monitoring-3/prometheus.nix b/hosts/monitoring-3/prometheus.nix index 72392f7..0997ef9 100644 --- a/hosts/monitoring-3/prometheus.nix +++ b/hosts/monitoring-3/prometheus.nix @@ -52,6 +52,12 @@ let attrByPath ["clerie" "monitoring" "blackbox"] false host.config) monitoringHosts); + nginxlogMonitoringTargets = mapAttrsToList (name: host: + "${host.config.networking.hostName}.mon.clerie.de:9117") + (filterAttrs (name: host: + attrByPath ["services" "prometheus" "exporters" "nginxlog" "enable"] false host.config) + monitoringHosts); + eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b)))); in { @@ -536,6 +542,18 @@ in { } ]; } + { + job_name = "nginxlog-exporter"; + scrape_interval = "20s"; + static_configs = [ + { + targets = nginxlogMonitoringTargets; + } + ]; + relabel_configs = [ + relabelAddressToInstance + ]; + } ]; alertmanagers = [ { diff --git a/hosts/web-2/clerie.nix b/hosts/web-2/clerie.nix index 7961a54..6a6e7e6 100644 --- a/hosts/web-2/clerie.nix +++ b/hosts/web-2/clerie.nix @@ -53,9 +53,6 @@ ''; return = "200 ''"; }; - extraConfig = '' - access_log /var/log/nginx/clerie.de.log combined_anon; - ''; }; }; } diff --git a/hosts/web-2/gitea.nix b/hosts/web-2/gitea.nix index 49e23b8..3d07b7f 100644 --- a/hosts/web-2/gitea.nix +++ b/hosts/web-2/gitea.nix @@ -83,9 +83,6 @@ proxyPass = "http://[::1]:3000"; }; }; - extraConfig = '' - access_log /var/log/nginx/git.clerie.de.log combined_anon; - ''; }; }; } diff --git a/hosts/web-2/ip.nix b/hosts/web-2/ip.nix index bde306b..28708e4 100644 --- a/hosts/web-2/ip.nix +++ b/hosts/web-2/ip.nix @@ -53,9 +53,6 @@ types { } default_type "text/html; charset=utf-8"; ''; }; - extraConfig = '' - access_log /var/log/nginx/ip.clerie.de.log vcombined_anon; - ''; }; "ip4.clerie.de" = { enableACME = true; @@ -67,9 +64,6 @@ add_header Access-Control-Allow-Origin *; ''; }; - extraConfig = '' - access_log /var/log/nginx/ip.clerie.de.log vcombined_anon; - ''; }; "ip6.clerie.de" = { enableACME = true; @@ -81,9 +75,6 @@ add_header Access-Control-Allow-Origin *; ''; }; - extraConfig = '' - access_log /var/log/nginx/ip.clerie.de.log vcombined_anon; - ''; }; }; } diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix index c24107f..fd8bf99 100644 --- a/modules/monitoring/default.nix +++ b/modules/monitoring/default.nix @@ -102,6 +102,33 @@ in listen = "[::]:9152"; }; + services.prometheus.exporters.nginxlog = mkIf config.services.nginx.enable { + enable = true; + settings = { + namespaces = [ + { + name = "nginxlog"; + format = ''$host: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$server_name" rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"''; + source = { + files = [ + "/var/log/nginx/access.log" + ]; + }; + relabel_configs = [ + { + target_label = "server_name"; + from = "server_name"; + } + ]; + } + ]; + }; + }; + + systemd.services."prometheus-nginxlog-exporter".serviceConfig = { + SupplementaryGroups = "nginx"; + }; + networking.firewall.interfaces."wg-monitoring".allowedTCPPorts = [ 9100 # node-exporter 9152 # nixos-exporter @@ -109,6 +136,8 @@ in 9324 # bird-exporter ] else []) ++ (if cfg.blackbox then [ 9115 # blackbox-exporter + ] else []) ++ (if config.services.prometheus.exporters.nginxlog.enable then [ + config.services.prometheus.exporters.nginxlog.port ] else []); }; } diff --git a/profiles/common-webserver/default.nix b/profiles/common-webserver/default.nix index f64ac8e..7033c1e 100644 --- a/profiles/common-webserver/default.nix +++ b/profiles/common-webserver/default.nix @@ -40,7 +40,12 @@ in { log_format vcombined_anon '$host: $remote_addr_anon - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"'; - access_log /var/log/nginx/access.log vcombined_anon; + log_format vcombined_anon_monitoring '$host: $remote_addr_anon - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '"$server_name" ' + 'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + access_log /var/log/nginx/access.log vcombined_anon_monitoring; ''; virtualHosts = mkIf cfg.httpDefaultVirtualHost {