4 files changed, 94 insertions, 22 deletions

diff --git a/modules/foundation/monitoring/client.nix b/modules/foundation/monitoring/client.nix
index f3f6873..2b0bc84 100644
--- a/modules/foundation/monitoring/client.nix
+++ b/modules/foundation/monitoring/client.nix
@@ -1,7 +1,9 @@
 {
+  services,
   listenAddress,
   nodeExporterPort,
   nginxExporterPort,
+  nginxLogExporterPort,
   cadvisorExporterPort,
   ...
 }:
@@ -22,6 +24,33 @@
         port = nginxExporterPort;
         inherit listenAddress;
       };
+
+      nginxlog = {
+        enable = true;
+        openFirewall = false;
+        port = nginxLogExporterPort;
+        inherit listenAddress;
+
+        user = "nginx";
+        group = "nginx";
+
+        settings.namespaces =
+          let
+            namespace = name: log: {
+              inherit name;
+              source_files = [ log ];
+              parser = "json";
+              metrics_override.prefix = "nginx";
+              namespace_label = "vhost";
+            };
+
+            vhost = name: (namespace name "/var/log/nginx/${name}.access.log");
+          in
+          [
+            (namespace "default" "/var/log/nginx/access.log")
+          ]
+          ++ (map vhost services);
+      };
     };
 
     cadvisor = {
diff --git a/modules/foundation/monitoring/default.nix b/modules/foundation/monitoring/default.nix
index a7bc185..2489948 100644
--- a/modules/foundation/monitoring/default.nix
+++ b/modules/foundation/monitoring/default.nix
@@ -1,4 +1,9 @@
-{ me, config, lib, ... }:
+{
+  me,
+  config,
+  lib,
+  ...
+}:
 
 let
   inherit (lib) mkOption mkEnableOption types;
@@ -10,17 +15,25 @@ let
     nodeExporterPort = 9001;
     cadvisorExporterPort = 9002;
     nginxExporterPort = 9113;
+    nginxLogExporterPort = 9117;
     dockerExporterPort = 9323;
   };
 
-  serverConfiguration = import ./server.nix ({
-    hosts = cfg.server.hosts;
-    victoriaAddress = me.tailscale.ip;
-  } // ports);
+  serverConfiguration = import ./server.nix (
+    {
+      hosts = cfg.server.hosts;
+      victoriaAddress = "${me.tailscale.ip}:${toString ports.victoriaDefaultPort}";
+    }
+    // ports
+  );
 
-  clientConfiguration = import ./client.nix ({
-    listenAddress = me.tailscale.ip;
-  } // ports);
+  clientConfiguration = import ./client.nix (
+    {
+      services = cfg.services;
+      listenAddress = me.tailscale.ip;
+    }
+    // ports
+  );
 in
 {
   imports = [ ../../../services/monitoring/wrapper.nix ];
@@ -30,12 +43,14 @@ in
       enable = mkEnableOption "monitoring server";
 
       hosts = mkOption {
-        type = with types; listOf (submodule {
-          options = {
-            name = mkOption { type = str; };
-            ip = mkOption { type = str; };
-          };
-        });
+        type =
+          with types;
+          listOf (submodule {
+            options = {
+              name = mkOption { type = str; };
+              ip = mkOption { type = str; };
+            };
+          });
         default = [ ];
       };
     };
@@ -43,6 +58,11 @@ in
     client = {
       enable = mkEnableOption "monitoring client";
     };
+
+    services = mkOption {
+      type = with types; listOf str;
+      default = [ ];
+    };
   };
 
   config = lib.mkMerge [
diff --git a/modules/foundation/monitoring/server.nix b/modules/foundation/monitoring/server.nix
index 4c922b9..1d1afb3 100644
--- a/modules/foundation/monitoring/server.nix
+++ b/modules/foundation/monitoring/server.nix
@@ -1,11 +1,11 @@
 {
   hosts,
   victoriaAddress,
-  victoriaDefaultPort,
   nodeExporterPort,
   dockerExporterPort,
   cadvisorExporterPort,
   nginxExporterPort,
+  nginxLogExporterPort,
   ...
 }:
 
@@ -14,7 +14,7 @@
 
   services.vmagent = {
     enable = true;
-    remoteWrite.url = "http://${victoriaAddress}:${toString victoriaDefaultPort}/api/v1/write";
+    remoteWrite.url = "http://${victoriaAddress}/api/v1/write";
 
     prometheusConfig = {
       global = {
@@ -23,12 +23,12 @@
 
       scrape_configs =
         let
-          exporter = job: port: {
+          exporter = job: ports: {
             job_name = job;
             static_configs = map (
               { name, ip }:
               {
-                targets = [ "${ip}:${toString port}" ];
+                targets = map (p: "${ip}:${toString p}") ports;
                 labels = {
                   instance = name;
                 };
@@ -37,10 +37,13 @@
           };
         in
         [
-          (exporter "node" nodeExporterPort)
-          (exporter "docker" dockerExporterPort)
-          (exporter "cadvisor" cadvisorExporterPort)
-          (exporter "nginx" nginxExporterPort)
+          (exporter "node" [ nodeExporterPort ])
+          (exporter "docker" [ dockerExporterPort ])
+          (exporter "cadvisor" [ cadvisorExporterPort ])
+          (exporter "nginx" [
+            nginxExporterPort
+            nginxLogExporterPort
+          ])
         ];
     };
   };
diff --git a/modules/www.nix b/modules/www.nix
index a1c2b55..c50019a 100644
--- a/modules/www.nix
+++ b/modules/www.nix
@@ -21,6 +21,26 @@ in
 
     statusPage = true;
 
+    commonHttpConfig = ''
+      log_format json_combined escape=json '{'
+	'"time_local":"$time_local",'
+	'"remote_addr":"$remote_addr",'
+	'"remote_user":"$remote_user",'
+	'"request":"$request",'
+	'"status": "$status",'
+	'"body_bytes_sent":"$body_bytes_sent",'
+	'"request_length":"$request_length",'
+	'"request_time":"$request_time",'
+	'"http_referrer":"$http_referer",'
+	'"http_user_agent":"$http_user_agent",'
+	'"upstream_response_time":"$upstream_response_time",'
+	'"upstream_addr":"$upstream_addr",'
+	'"upstream_status":"$upstream_status"'
+      '}';
+      access_log /var/log/nginx/access.log json_combined;
+      error_log /var/log/nginx/error.log warn;
+    '';
+
     virtualHosts = {
       default = { default = true; };
       ${rnrdUrl} = {