From b9e7228520f292fa5ff4ef6f1b36231927356d3b Mon Sep 17 00:00:00 2001
From: Kaare Hoff Skovgaard <kaare@kaareskovgaard.net>
Date: Thu, 14 Aug 2025 22:48:14 +0200
Subject: [PATCH] Update scrape intervals

---
 .../infrastructure/vault-prometheus-sender/default.nix     | 2 +-
 .../vault-prometheus-sender/prometheus.alloy               | 2 +-
 nix/systems/aarch64-linux/kas.codes/forgejo/default.nix    | 2 +-
 .../mx.kaareskovgaard.net/mailserver/prometheus.nix        | 2 +-
 .../monitoring.kaareskovgaard.net/alerts/http.yaml         | 2 +-
 .../x86_64-linux/monitoring.kaareskovgaard.net/default.nix | 7 +++++--
 .../x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix     | 1 -
 7 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix b/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix
index df89a7a..082d5d4 100644
--- a/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix
+++ b/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix
@@ -109,7 +109,7 @@ in
                   {"__address__" = "127.0.0.1:${toString value.port}", "instance" = constants.hostname, "job" = "${name}"},
                 ]
 
-                scrape_interval = "1m"
+                scrape_interval = "15s"
 
                 forward_to = [otelcol.receiver.prometheus.default.receiver]
               }
diff --git a/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy
index a2ff1d8..ea762b1 100644
--- a/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy
+++ b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy
@@ -50,7 +50,7 @@ prometheus.exporter.unix "integrations_node_exporter" {
 
 // Define how to scrape metrics from the node_exporter
 prometheus.scrape "integrations_node_exporter" {
-  scrape_interval = "1m"
+  scrape_interval = "15s"
   // Use the targets with labels from the discovery.relabel component
   targets    = discovery.relabel.integrations_node_exporter.output
   // Send the scraped metrics to the relabeling component
diff --git a/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix b/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix
index 26d70aa..af58ace 100644
--- a/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix
+++ b/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix
@@ -185,7 +185,7 @@ in
   environment.etc."alloy/forgejo_prometheus.alloy" = {
     text = ''
       prometheus.scrape "forgejo_exporter" {
-        scrape_interval = "1m"
+        scrape_interval = "15s"
         targets    = [
           {
             "__address__" = "127.0.0.1:${toString config.services.forgejo.settings.server.HTTP_PORT}",
diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/prometheus.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/prometheus.nix
index 5b8b7dd..0828c4c 100644
--- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/prometheus.nix
+++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/prometheus.nix
@@ -91,7 +91,7 @@ in
     environment.etc."alloy/dovecot_prometheus.alloy" = {
       text = ''
         prometheus.scrape "dovecot_exporter" {
-          scrape_interval = "1m"
+          scrape_interval = "15s"
           targets    = [
             {
               "__address__" = "127.0.0.1:${toString dovecotPromPort}",
diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/http.yaml b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/http.yaml
index bca404e..a0c6e88 100644
--- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/http.yaml
+++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/http.yaml
@@ -17,4 +17,4 @@
       labels:
         severity: critical
       annotations:
-        summary: "Nginx VHost {{ $labels.vhost }} on {{ $labels.instance }} is generating many internal server errors over 1 hour"
+        summary: "Nginx VHost {{ $labels.vhost }} on {{ $labels.instance }} is generating many internal server errors over 30 minutes"
diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix
index 4baf538..6cf40f1 100644
--- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix
+++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix
@@ -161,7 +161,10 @@ in
   services.prometheus = {
     enable = true;
     listenAddress = "127.0.0.1";
-    extraFlags = [ "--web.enable-otlp-receiver" ];
+    extraFlags = [
+      "--web.enable-otlp-receiver"
+      "--storage.tsdb.retention.time=15d"
+    ];
     # alertmanager.enable = true;
     rules = [
       ''
@@ -224,7 +227,7 @@ in
       compactor = {
         retention_enabled = true;
         compaction_interval = "24h";
-        retention_delete_delay = "24h";
+        retention_delete_delay = "15d";
         delete_request_store = "filesystem";
         working_directory = "${config.services.loki.dataDir}/retention";
       };
diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix
index dfb0e26..b6b861e 100644
--- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix
+++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix
@@ -1,7 +1,6 @@
 {
   pkgs,
   lib,
-  config,
   ...
 }:
 {