Get some metrics and a dashboard for dovecot

nix/modules/nixos/infrastructure/hetzner-instance/default.nix

@@ -81,6 +81,16 @@ let
         ];
         description = "dns";
       }
+      {
+        direction = "out";
+        protocol = "udp";
+        port = 123;
+        destination_ips = [
+          "0.0.0.0/0"
+          "::/0"
+        ];
+        description = "ntp";
+      }
       {
         direction = "out";
         protocol = "tcp";

nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/prometheus.nix

@@ -1,9 +1,108 @@
+let
+  dovecotPromPort = 9900;
+in
 {
   config = {
-
     services.prometheus.exporters.postfix = {
       enable = true;
     };
-    khscodes.infrastructure.vault-prometheus-sender.exporters.enabled = [ "postfix" ];
+    khscodes.infrastructure.vault-prometheus-sender.exporters.enabled = [
+      "postfix"
+    ];
+    # From https://alfaexploit.com/en/posts/monitoring_dovecot_with_prometheus/
+    services.dovecot2.extraConfig = ''
+      ##
+      ## Statistics and metrics
+      ##
+
+      # Dovecot supports gathering statistics from events.
+      # Currently there are no statistics logged by default, and therefore they must
+      # be explicitly added using the metric configuration blocks.
+      #
+      # Unlike old stats, the new statistics do not require any plugins loaded.
+      #
+      # See https://doc.dovecot.org/configuration_manual/stats/ for more details.
+
+      ##
+      ## Example metrics
+      ##
+
+      metric auth_success {
+        filter = event=auth_request_finished AND success=yes
+      }
+
+      metric auth_failures {
+        filter = event=auth_request_finished AND NOT success=yes
+      }
+
+      metric imap_command {
+        filter = event=imap_command_finished
+        group_by = cmd_name tagged_reply_state
+      }
+
+      metric smtp_command {
+        filter = event=smtp_server_command_finished
+        group_by = cmd_name status_code duration:exponential:1:5:10
+      }
+
+      metric mail_delivery {
+        filter = event=mail_delivery_finished
+        group_by = duration:exponential:1:5:10
+      }
+
+      ##
+      ## Prometheus
+      ##
+
+      # To allow access to statistics with Prometheus, enable http listener
+      # on stats process. Stats will be available on /metrics path.
+      #
+      # See https://doc.dovecot.org/configuration_manual/stats/openmetrics/ for more
+      # details.
+
+      service stats {
+        inet_listener http {
+          port = ${toString dovecotPromPort}
+        }
+      }
+
+      ##
+      ## Event exporting
+      ##
+
+      # You can also export individual events.
+      #
+      # See https://doc.dovecot.org/configuration_manual/event_export/ for more
+      # details.
+
+      #event_exporter log {
+      #  format = json
+      #  format_args = time-rfc3339
+      #  transport = log
+      #}
+      #
+      #metric imap_commands {
+      #  exporter = log
+      #  filter = event=imap_command_finished
+      #}
+    '';
+
+    khscodes.infrastructure.vault-prometheus-sender.exporters.external = [ "dovecot" ];
+    environment.etc."alloy/dovecot_prometheus.alloy" = {
+      text = ''
+        prometheus.scrape "dovecot_exporter" {
+          scrape_interval = "1m"
+          targets    = [
+            {
+              "__address__" = "127.0.0.1:${toString dovecotPromPort}",
+              "instance" = constants.hostname,
+              "job" = "dovecot",
+            },
+          ]
+          metrics_path = "/metrics"
+          forward_to = [otelcol.receiver.prometheus.default.receiver]
+        }
+      '';
+    };
   };
 }

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/dovecot.yaml

@@ -0,0 +1,10 @@
+- name: Dovecot
+  rules:
+    - alert: DovecotDown
+      expr: >
+        dovecot_up{job="dovecot"} == 0
+      for: 10m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Dovecot on {{ $labels.instance }} is down"

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix

@@ -10,19 +10,58 @@ let
   loki = config.services.loki;
   prometheus = config.services.prometheus;
   nginxExporterSrc = "${pkgs.prometheus-nginx-exporter.src}/grafana/dashboard.json";
+  promDataSource = {
+    current = { };
+    includeAll = false;
+    label = "Datasource";
+    name = "DS_PROMETHEUS";
+    options = [ ];
+    query = "prometheus";
+    refresh = 1;
+    regex = "";
+    type = "datasource";
+  };
+  srcName = src: if lib.isPath src then builtins.baseNameOf src else src.name;
+  patchPrometheusTemplatingDataSource =
+    src:
+    let
+      name = srcName src;
+    in
+    pkgs.stdenvNoCC.mkDerivation {
+      inherit src;
+      name = "patched-${name}";
+      dontUnpack = true;
+      buildPhase = ''
+        cat ${src} | ${lib.getExe pkgs.jq} --raw-output --argjson src ${lib.escapeShellArg (builtins.toJSON promDataSource)} '.templating.list = [$src] + .templating.list' > ${lib.escapeShellArg name}
+      '';
+      installPhase = ''
+        mv ${lib.escapeShellArg name} $out
+      '';
+    };
   postgresqlDashboard = pkgs.fetchurl {
     url = "https://grafana.com/api/dashboards/9628/revisions/8/download";
     hash = "sha256-UhusNAZbyt7fJV/DhFUK4FKOmnTpG0R15YO2r+nDnMc=";
+    name = "postgresql.json";
   };
-  postfixDashboard = pkgs.fetchurl {
+  postfixDashboard = patchPrometheusTemplatingDataSource (
-    url = "https://grafana.com/api/dashboards/10013/revisions/2/download";
+    pkgs.fetchurl {
-    hash = "sha256-SIKL1V+sJ5F7vPOwp/LuOjrGm8nCsscEX8LcLFMotfc=";
+      url = "https://grafana.com/api/dashboards/10013/revisions/2/download";
-  };
+      hash = "sha256-SIKL1V+sJ5F7vPOwp/LuOjrGm8nCsscEX8LcLFMotfc=";
+      name = "postfix.json";
+    }
+  );
+
+  oauthCredentialFile = config.khscodes.infrastructure.kanidm-client-application.secretFile;
 in
 {
   imports = [
     "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix"
   ];
+  systemd.services.grafana = {
+    unitConfig.ConditionPathExists = [
+      oauthCredentialFile
+    ];
+  };
   services.grafana = {
     enable = true;
     settings = {
@@ -53,7 +92,7 @@ in
         token_url = "https://login.kaareskovgaard.net/oauth2/token";
         api_url = "https://login.kaareskovgaard.net/oauth2/openid/monitoring/userinfo";
         client_id = "monitoring";
-        client_secret = "$__file{${config.khscodes.infrastructure.kanidm-client-application.secretFile}}";
+        client_secret = "$__file{${oauthCredentialFile}}";
         scopes = "openid profile email";
         use_pkce = true;
         skip_org_role_sync = false;
@@ -81,13 +120,13 @@ in
         {
           url = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}";
           type = "loki";
-          name = "Logs";
+          name = "Loki";
           uid = "loki";
         }
         {
           url = "http://${prometheus.listenAddress}:${toString prometheus.port}";
           type = "prometheus";
-          name = "Metrics";
+          name = "Prometheus";
           uid = "prometheus";
           jsonData = {
             manageAlerts = true;
@@ -99,6 +138,10 @@ in
           name = "Node Exporter";
           options.path = ./grafana/dashboards/node_exporter;
         }
+        {
+          name = "Dovecot";
+          options.path = patchPrometheusTemplatingDataSource ./grafana/dashboards/dovecot/dovecot_prometheus.json;
+        }
         {
           name = "Nginx";
           options.path = nginxExporterSrc;
@@ -127,6 +170,7 @@ in
         ${builtins.readFile ./alerts/postfix.yaml}
         ${builtins.readFile ./alerts/postgres.yaml}
         ${builtins.readFile ./alerts/systemd.yaml}
+        ${builtins.readFile ./alerts/dovecot.yaml}
         ${import ./alerts/job_up.nix { inherit inputs lib; }}
       ''
     ];