more monitoring & scrape config shortcuts

9 files changed, 63 insertions, 68 deletions

diff --git a/lib/default.nix b/lib/default.nix
index 61c0760..5aecf51 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -4,10 +4,11 @@
   flake = {
     lib =
       let
-        callLibs = file: import file { inherit lib; };
+        callLibs = file: import file { inherit lib config; };
       in
       {
         nginx = callLibs ./nginx;
+        mon = callLibs ./monitoring;
       };
   };
 }
diff --git a/lib/monitoring/default.nix b/lib/monitoring/default.nix
new file mode 100644
index 0000000..2c7c3cf
--- /dev/null
+++ b/lib/monitoring/default.nix
@@ -0,0 +1,20 @@
+{ config, lib }:
+
+rec {
+  mkTarget = host: port: {
+    targets = [ "${host}:${toString port}" ];
+  };
+
+  mkScrapeConfig = name: hosts: port:  {
+    job_name = name; 
+    static_configs = map (host: mkTarget host port) hosts; 
+    relabel_configs = relabelConfig;
+  };
+
+  relabelConfig = [{
+    source_labels = ["__address__"];
+    target_label = "host";
+    regex = "([^:]+)(:[0-9]+)?";
+    replacement = "\${1}";
+  }];
+}
diff --git a/lib/nginx/default.nix b/lib/nginx/default.nix
index 1f0f482..c6b6da2 100644
--- a/lib/nginx/default.nix
+++ b/lib/nginx/default.nix
@@ -1,4 +1,4 @@
-{ lib }:
+{ lib, ... }:
 
 {
   proxyDomain = cert: proxyPass: {
diff --git a/modules/bgp/default.nix b/modules/bgp/default.nix
index 5a726b6..c551ee8 100644
--- a/modules/bgp/default.nix
+++ b/modules/bgp/default.nix
@@ -86,4 +86,10 @@
         !
     '';
   };
+
+  services.prometheus.exporters.frr = {
+    enable = false;
+    enabledCollectors = [ "BGP" ];
+    disabledCollectors = [ "OSPFv4" "BFD" "Route" ];
+  };
 }
diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix
index 05ca338..afcffa5 100644
--- a/modules/monitoring/default.nix
+++ b/modules/monitoring/default.nix
@@ -1,7 +1,9 @@
 { self, config, lib, pkgs, ... }:
 
 with self.lib.nginx;
-{
+with self.lib.mon;
+let exp = config.services.prometheus.exporters;
+in {
   services.prometheus = {
     enable = true;
     enableReload = true;
@@ -10,70 +12,32 @@ with self.lib.nginx;
     listenAddress = "10.10.0.1";
     webExternalUrl = "https://prometheus.vapor.systems";
 
-    exporters = {
-      postgres = {
-        enable = true;
-      };
-    };
-
     globalConfig = {
       scrape_interval = "10s";
     };
 
     scrapeConfigs = [
-      { 
-        job_name = "node"; 
-        static_configs = [
-          { targets = [ "ettves:${toString config.services.prometheus.exporters.node.port}" ]; }
-          { targets = [ "phaenn:${toString config.services.prometheus.exporters.node.port}" ]; }
-        ]; 
-      }
-      { 
-        job_name = "zfs"; 
-        static_configs = [
-          { targets = [ "ettves:${toString config.services.prometheus.exporters.zfs.port}" ]; }
-          { targets = [ "phaenn:${toString config.services.prometheus.exporters.zfs.port}" ]; }
-        ]; 
-      }
-      { 
-        job_name = "smartctl"; 
-        static_configs = [
-          { targets = [ "ettves:${toString config.services.prometheus.exporters.smartctl.port}" ]; }
-          { targets = [ "phaenn:${toString config.services.prometheus.exporters.smartctl.port}" ]; }
-        ]; 
-      }
-      { 
-        job_name = "postgres"; 
-        static_configs = [
-          { targets = [ "ettves:${toString config.services.prometheus.exporters.postgres.port}" ]; }
-        ]; 
-      }
-      { 
-        job_name = "nginx"; 
-        static_configs = [
-          { targets = [ "ettves:${toString config.services.prometheus.exporters.nginx.port}" ]; }
-          { targets = [ "phaenn:${toString config.services.prometheus.exporters.nginx.port}" ]; }
-        ]; 
-      }
-      { 
-        job_name = "quassel"; 
-        static_configs = [
-          { targets = [ "localhost:${toString config.services.quassel.settings.metrics.port}" ]; }
-        ]; 
-      }
+      (mkScrapeConfig "node" [ "ettves" "phaenn" "fra01" "nyc01" "sin01" ] exp.node.port)
+      (mkScrapeConfig "zfs" [ "ettves" "phaenn" ] exp.zfs.port)
+      (mkScrapeConfig "smartctl" [ "ettves" "phaenn" ] exp.smartctl.port)
+      (mkScrapeConfig "nginx" [ "ettves" "phaenn" "fra01" "nyc01" "sin01" ] exp.nginx.port)
+      
+      (mkScrapeConfig "postgres" [ "ettves" ] exp.postgres.port)
+      (mkScrapeConfig "quassel" [ "localhost" ] config.services.quassel.settings.metrics.port)
+
       { 
         job_name = "authentik"; 
         static_configs = [
-          { targets = [ "ettves:9300" ]; }
-          { targets = [ "ettves:9303" ]; }
-          { targets = [ "ettves:9304" ]; }
+          { targets = [ "ettves:9300" "ettves:9303" "ettves:9304" ]; }
         ]; 
+        relabel_configs = relabelConfig;
       }
+      
+      (mkScrapeConfig "garage" [ "fra01" "nyc01" "sin01" ] 3903)
+      (mkScrapeConfig "pdns" [ "ettves" "fra01" "nyc01" "sin01" ] 8081)
     ];
   };
 
-  services.nginx.statusPage = true;
-
   services.udev.extraRules = ''
     SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
   '';
@@ -87,7 +51,7 @@ with self.lib.nginx;
   services.nginx = {
     enable = true;
     virtualHosts = {
-      "prometheus.vapor.systems" = (proxyDomain "vapor.systems" "http://10.10.0.1:9090/");
+      "prometheus.vapor.systems" = (proxyDomainAuth "vapor.systems" "http://10.10.0.1:9090/");
     };
   };
 }
diff --git a/modules/monitoring/node-exporter.nix b/modules/monitoring/node-exporter.nix
index 1b7481d..7508953 100644
--- a/modules/monitoring/node-exporter.nix
+++ b/modules/monitoring/node-exporter.nix
@@ -9,23 +9,11 @@ with self.lib.nginx;
         enabledCollectors = [ "systemd" ];
       };
 
-      zfs = {
-        enable = true;
-      };
-
       nginx = {
         enable = true;
       };
-
-      smartctl = {
-        enable = true;
-      };
     };
   };
 
   services.nginx.statusPage = true;
-
-  services.udev.extraRules = ''
-    SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
-  '';
 }
diff --git a/modules/postgresql/default.nix b/modules/postgresql/default.nix
index 0e6d372..20c1906 100644
--- a/modules/postgresql/default.nix
+++ b/modules/postgresql/default.nix
@@ -29,4 +29,6 @@
       listen_addresses = lib.mkForce "127.0.0.1,10.10.0.1,::1";
     };
   };
+
+  services.prometheus.exporters.postgres.enable = true;
 }
diff --git a/modules/powerdns/default.nix b/modules/powerdns/default.nix
index 5a920ec..cce4abb 100644
--- a/modules/powerdns/default.nix
+++ b/modules/powerdns/default.nix
@@ -19,7 +19,7 @@ let
     local-address = 0.0.0.0 [::]
     primary = yes
     webserver = yes
-    webserver-address = 10.10.0.1
+    webserver-address = 0.0.0.0
     webserver-allow-from = 10.0.0.0/8,127.0.0.0/8
     api-key = $PDNS_API_KEY
   '';
@@ -44,6 +44,10 @@ let
     secondary = yes
 
     allow-notify-from = 10.10.0.1/32
+
+    webserver = yes
+    webserver-address = 0.0.0.0
+    webserver-allow-from = 10.0.0.0/8,127.0.0.0/8
   '';
 in
 {
diff --git a/modules/zfs/default.nix b/modules/zfs/default.nix
index 75018ea..ac28159 100644
--- a/modules/zfs/default.nix
+++ b/modules/zfs/default.nix
@@ -15,6 +15,16 @@
     autoScrub.pools = [ "rpool" ];
   };
 
+  services.prometheus.exporters = {
+    zfs.enable = true;
+    smartctl.enable = true;
+  };
+
+  # For allowing smartctl prometheus exporter access to nvme disks
+  services.udev.extraRules = ''
+    SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
+  '';
+
   fileSystems = {
     "/" = {
       device = "rpool/root";