modules/monitoring: Extract metrics from nginx logs

hosts/monitoring-3/prometheus.nix

@@ -52,6 +52,12 @@ let
     attrByPath ["clerie" "monitoring" "blackbox"] false host.config)
     monitoringHosts);
 
+  nginxlogMonitoringTargets = mapAttrsToList (name: host:
+    "${host.config.networking.hostName}.mon.clerie.de:9117")
+    (filterAttrs (name: host:
+    attrByPath ["services" "prometheus" "exporters" "nginxlog" "enable"] false host.config)
+    monitoringHosts);
+
   eachWithEachOther = (f: x: y: lib.lists.flatten (lib.lists.forEach x (a: lib.lists.forEach y (b: f a b))));
 
 in {
@@ -536,6 +542,18 @@ in {
           }
         ];
       }
+      {
+        job_name = "nginxlog-exporter";
+        scrape_interval = "20s";
+        static_configs = [
+          {
+            targets = nginxlogMonitoringTargets;
+          }
+        ];
+        relabel_configs = [
+          relabelAddressToInstance
+        ];
+      }
     ];
     alertmanagers = [
       {

hosts/web-2/clerie.nix

@@ -53,9 +53,6 @@
         '';
         return = "200 ''";
       };
-      extraConfig = ''
-        access_log /var/log/nginx/clerie.de.log combined_anon;
-      '';
     };
   };
 }

hosts/web-2/gitea.nix

@@ -83,9 +83,6 @@
           proxyPass = "http://[::1]:3000";
         };
       };
-      extraConfig = ''
-        access_log /var/log/nginx/git.clerie.de.log combined_anon;
-      '';
     };
   };
 }

hosts/web-2/ip.nix

@@ -53,9 +53,6 @@
           types { } default_type "text/html; charset=utf-8";
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
     "ip4.clerie.de" = {
       enableACME = true;
@@ -67,9 +64,6 @@
           add_header Access-Control-Allow-Origin *;
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
     "ip6.clerie.de" = {
       enableACME = true;
@@ -81,9 +75,6 @@
           add_header Access-Control-Allow-Origin *;
         '';
       };
-      extraConfig = ''
-        access_log /var/log/nginx/ip.clerie.de.log vcombined_anon;
-      '';
     };
   };
 }

modules/monitoring/default.nix

@@ -102,6 +102,33 @@ in
       listen = "[::]:9152";
     };
 
+    services.prometheus.exporters.nginxlog = mkIf config.services.nginx.enable {
+      enable = true;
+      settings = {
+        namespaces = [
+          {
+            name = "nginxlog";
+            format = ''$host: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$server_name" rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
+            source = {
+              files = [
+                "/var/log/nginx/access.log"
+              ];
+            };
+            relabel_configs = [
+              {
+                target_label = "server_name";
+                from = "server_name";
+              }
+            ];
+          }
+        ];
+      };
+    };
+
+    systemd.services."prometheus-nginxlog-exporter".serviceConfig = {
+      SupplementaryGroups = "nginx";
+    };
+
     networking.firewall.interfaces."wg-monitoring".allowedTCPPorts = [
       9100 # node-exporter
       9152 # nixos-exporter
@@ -109,6 +136,8 @@ in
       9324 # bird-exporter
     ] else []) ++ (if cfg.blackbox then [
       9115 # blackbox-exporter
+    ] else []) ++ (if config.services.prometheus.exporters.nginxlog.enable then [
+      config.services.prometheus.exporters.nginxlog.port
     ] else []);
   };
 }

profiles/common-webserver/default.nix

@@ -40,7 +40,12 @@ in {
         log_format vcombined_anon '$host: $remote_addr_anon - $remote_user [$time_local] '
                                   '"$request" $status $body_bytes_sent '
                                   '"$http_referer" "$http_user_agent"';
-        access_log /var/log/nginx/access.log vcombined_anon;
+        log_format vcombined_anon_monitoring '$host: $remote_addr_anon - $remote_user [$time_local] '
+                                  '"$request" $status $body_bytes_sent '
+                                  '"$http_referer" "$http_user_agent" '
+                                  '"$server_name" '
+                                  'rt="$request_time" uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+        access_log /var/log/nginx/access.log vcombined_anon_monitoring;
       '';
 
       virtualHosts = mkIf cfg.httpDefaultVirtualHost {