Add some ZFS alerting rules to prometheus

nix/modules/nixos/fs/zfs/default.nix

@@ -138,6 +138,7 @@ in
       forceImportRoot = false;
       requestEncryptionCredentials = false;
     };
+    services.zfs.autoScrub.enable = true;
     systemd.services.zfs-mount.enable = false;
     systemd.services.zfs-import-zroot.enable = false;
     systemd.services.khscodes-zpool-setup = {
@@ -197,5 +198,7 @@ in
         restartUnits = [ "khscodes-zpool-setup.service" ];
       }
     ];
+    services.prometheus.exporters.zfs.enable = true;
+    khscodes.infrastructure.vault-prometheus-sender.exporters.enabled = [ "zfs" ];
   };
 }

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml

@@ -7,4 +7,4 @@
       labels:
         severity: warn
       annotations:
-        summary: "Unit {{ $labels.name}} on {{ $labels.instance }} is in failed state"
+        summary: "Unit {{ $labels.name }} on {{ $labels.instance }} is in failed state"

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/zfs.yaml

@@ -0,0 +1,26 @@
+- name: ZFS
+  rules:
+    - alert: ScrapeFailed
+      expr: >
+        zfs_scrape_collector_success{job="zfs"} == 0
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: "Could not scrape zfs metrics on {{ $labels.instance }}"
+    - alert: PoolNotOnline
+      expr: >
+        zfs_pool_health{job="zfs"} != 0
+      for: 10m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Pool {{ $labels.pool }} on {{ $labels.instance }} is not ONLINE."
+    - alert: PoolReadOnly
+      expr: >
+        zfs_pool_readonly{job="zfs"} == 1
+      for: 10m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Pool {{ $labels.pool }} on {{ $labels.instance }} is in read only mode."

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix

@@ -171,6 +171,7 @@ in
         ${builtins.readFile ./alerts/postgres.yaml}
         ${builtins.readFile ./alerts/systemd.yaml}
         ${builtins.readFile ./alerts/dovecot.yaml}
+        ${builtins.readFile ./alerts/zfs.yaml}
         ${import ./alerts/job_up.nix { inherit inputs lib; }}
       ''
     ];