machines/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix

{
  inputs,
  config,
  pkgs,
  ...
}:
let
  grafana = config.services.grafana;
  loki = config.services.loki;
  prometheus = config.services.prometheus;
  nginxExporterSrc = "${pkgs.prometheus-nginx-exporter.src}/grafana/dashboard.json";
  postgresqlDashboard = pkgs.fetchurl {
    url = "https://grafana.com/api/dashboards/9628/revisions/8/download";
    hash = "sha256-UhusNAZbyt7fJV/DhFUK4FKOmnTpG0R15YO2r+nDnMc=";
  };
  postfixDashboard = pkgs.fetchurl {
    url = "https://grafana.com/api/dashboards/10013/revisions/2/download";
    hash = "sha256-SIKL1V+sJ5F7vPOwp/LuOjrGm8nCsscEX8LcLFMotfc=";
  };
in
{
  imports = [
    "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix"
  ];
  services.grafana = {
    enable = true;
    settings = {
      security = {
        disable_initial_admin_creation = true;
      };
      server = {
        http_addr = "127.0.0.1";
        http_port = 3000;
        domain = "monitoring.kaareskovgaard.net";
        root_url = "https://monitoring.kaareskovgaard.net";
        serve_from_sub_path = false;
      };
      "auth" = {
        disable_login_form = true;
      };
      "auth.basic" = {
        enabled = false;
      };
      "auth.generic_oauth" = {
        enabled = true;
        allow_sign_up = true;
        auto_login = false;
        team_ids = null;
        allowed_organizations = null;
        name = "Kanidm";
        auth_url = "https://login.kaareskovgaard.net/ui/oauth2";
        token_url = "https://login.kaareskovgaard.net/oauth2/token";
        api_url = "https://login.kaareskovgaard.net/oauth2/openid/monitoring/userinfo";
        client_id = "monitoring";
        client_secret = "$__file{/var/lib/vault-agent/grafana/kanidm_client_secret}";
        scopes = "openid profile email";
        use_pkce = true;
        skip_org_role_sync = false;
        allow_assign_grafana_admin = true;
        org_attribute_path = "['Main org.']";
        org_mapping = "*:*:Admin";
        role_attribute_path = "'GrafanaAdmin'";
      };
    };
    provision = {
      enable = true;
      alerting = {
        rules = {
          settings = {
            deleteRules = [
              {
                uid = "desmw56u3jfgga";
                orgId = 1;
              }
            ];
          };
        };
      };
      datasources.settings.datasources = [
        {
          url = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}";
          type = "loki";
          name = "Logs";
          uid = "loki";
        }
        {
          url = "http://${prometheus.listenAddress}:${toString prometheus.port}";
          type = "prometheus";
          name = "Metrics";
          uid = "prometheus";
          jsonData = {
            manageAlerts = true;
          };
        }
      ];
      dashboards.settings.providers = [
        {
          name = "Node Exporter";
          options.path = ./grafana/dashboards/node_exporter;
        }
        {
          name = "Nginx";
          options.path = nginxExporterSrc;
        }
        {
          name = "Postgresql";
          options.path = postgresqlDashboard;
        }
        {
          name = "Postfix";
          options.path = postfixDashboard;
        }
      ];
    };
  };
  services.prometheus = {
    enable = true;
    listenAddress = "127.0.0.1";
    extraFlags = [ "--web.enable-otlp-receiver" ];
    # alertmanager.enable = true;
    rules = [
      ''
        groups:
        ${builtins.readFile ./alerts/http.yaml}
        ${builtins.readFile ./alerts/instance.yaml}
        ${builtins.readFile ./alerts/postfix.yaml}
        ${builtins.readFile ./alerts/postgres.yaml}
        ${builtins.readFile ./alerts/systemd.yaml}
      ''
    ];
  };
  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
        http_listen_address = "127.0.0.1";
      };
      common = {
        ring = {
          instance_addr = "127.0.0.1";
          kvstore = {
            store = "inmemory";
          };
        };
        replication_factor = 1;
        path_prefix = "${config.services.loki.dataDir}/common";
      };
      schema_config = {
        configs = [
          {
            from = "2025-07-11";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };
      storage_config = {
        tsdb_shipper = {
          active_index_directory = "${config.services.loki.dataDir}/index";
          cache_location = "${config.services.loki.dataDir}/index_cache";
        };
        filesystem = {
          directory = "${config.services.loki.dataDir}/chunks";
        };
      };
      pattern_ingester = {
        enabled = true;
      };
      compactor = {
        retention_enabled = true;
        compaction_interval = "24h";
        retention_delete_delay = "24h";
        delete_request_store = "filesystem";
        working_directory = "${config.services.loki.dataDir}/retention";
      };
      limits_config = {
        allow_structured_metadata = true;
        discover_log_levels = true;
      };
    };
  };
  khscodes = {
    infrastructure.khs-openstack-instance = {
      enable = true;
      flavor = "m.large";
    };
    services.nginx = {
      enable = true;
      virtualHosts."monitoring.kaareskovgaard.net" = {
        rateLimit.enable = false;
        locations."/" = {
          proxyPass = "http://${grafana.settings.server.http_addr}:${toString grafana.settings.server.http_port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
      virtualHosts."loki.kaareskovgaard.net" = {
        mtls = {
          verify = "on";
          certificate = "/etc/loki/client-signer.pem";
        };
        locations."/" = {
          proxyPass = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
      virtualHosts."prometheus.kaareskovgaard.net" = {
        mtls = {
          verify = "on";
          certificate = "/etc/prometheus/client-signer.pem";
        };
        locations."/" = {
          proxyPass = "http://${prometheus.listenAddress}:${toString prometheus.port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
    };
    services.vault-agent.templates = [
      {
        contents = ''
          {{- with secret "loki-mtls/cert/ca_chain" -}}
          {{ .Data.certificate }}
          {{- end -}}
        '';
        destination = "/etc/loki/client-signer.pem";
        owner = "loki";
        group = "loki";
        perms = "0644";
        reloadOrRestartUnits = [ "nginx.service" ];
      }
      {
        contents = ''
          {{- with secret "prometheus-mtls/cert/ca_chain" -}}
          {{ .Data.certificate }}
          {{- end -}}
        '';
        destination = "/etc/prometheus/client-signer.pem";
        owner = "prometheus";
        group = "prometheus";
        perms = "0644";
        reloadOrRestartUnits = [ "nginx.service" ];
      }
      {
        contents = ''
          {{- with secret "kanidm/data/apps/monitoring" -}}
          {{ .Data.data.basic_secret }}
          {{- end -}}
        '';
        destination = "/var/lib/vault-agent/grafana/kanidm_client_secret";
        owner = "grafana";
        group = "grafana";
        perms = "0600";
        reloadOrRestartUnits = [ "grafana.service" ];
      }
    ];
    infrastructure.vault-server-approle.policy = {
      "kanidm/data/apps/monitoring" = {
        capabilities = [ "read" ];
      };
    };
  };
  khscodes.networking.fqdn = "monitoring.kaareskovgaard.net";
  system.stateVersion = "25.05";
}