{
  inputs,
  config,
  ...
}:
let
  grafana = config.services.grafana;
  loki = config.services.loki;
  prometheus = config.services.prometheus;
in
{
  imports = [
    "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix"
  ];
  services.grafana = {
    enable = true;
    settings = {
      security = {
        disable_initial_admin_creation = true;
      };
      server = {
        http_addr = "127.0.0.1";
        http_port = 3000;
        domain = "monitoring.kaareskovgaard.net";
        root_url = "https://monitoring.kaareskovgaard.net";
        serve_from_sub_path = false;
      };
      "auth" = {
        disable_login_form = true;
      };
      "auth.basic" = {
        enabled = false;
      };
      "auth.generic_oauth" = {
        enabled = true;
        allow_sign_up = true;
        auto_login = false;
        team_ids = null;
        allowed_organizations = null;
        name = "Kanidm";
        auth_url = "https://login.kaareskovgaard.net/ui/oauth2";
        token_url = "https://login.kaareskovgaard.net/oauth2/token";
        api_url = "https://login.kaareskovgaard.net/oauth2/openid/monitoring/userinfo";
        client_id = "monitoring";
        client_secret = "$__file{/var/lib/vault-agent/grafana/kanidm_client_secret}";
        scopes = "openid profile email";
        use_pkce = true;
        skip_org_role_sync = false;
        allow_assign_grafana_admin = true;
        org_attribute_path = "['Main org.']";
        org_mapping = "*:*:Admin";
        role_attribute_path = "'GrafanaAdmin'";
      };
    };
    provision = {
      enable = true;
      datasources.settings.datasources = [
        {
          url = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}";
          type = "loki";
          name = "Logs";
        }
        {
          url = "http://${prometheus.listenAddress}:${toString prometheus.port}";
          type = "prometheus";
          name = "Metrics";
        }
      ];
      dashboards.settings.providers = [
        {
          name = "Node Exporter";
          options.path = ./grafana/dashboards/node_exporter;
        }
      ];
    };
  };
  services.prometheus = {
    enable = true;
    listenAddress = "127.0.0.1";
    extraFlags = [ "--web.enable-otlp-receiver" ];
  };
  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
        http_listen_address = "127.0.0.1";
      };
      common = {
        ring = {
          instance_addr = "127.0.0.1";
          kvstore = {
            store = "inmemory";
          };
        };
        replication_factor = 1;
        path_prefix = "${config.services.loki.dataDir}/common";
      };
      schema_config = {
        configs = [
          {
            from = "2025-07-11";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };
      storage_config = {
        tsdb_shipper = {
          active_index_directory = "${config.services.loki.dataDir}/index";
          cache_location = "${config.services.loki.dataDir}/index_cache";
        };
        filesystem = {
          directory = "${config.services.loki.dataDir}/chunks";
        };
      };
      pattern_ingester = {
        enabled = true;
      };
      compactor = {
        retention_enabled = true;
        compaction_interval = "24h";
        retention_delete_delay = "24h";
        delete_request_store = "filesystem";
        working_directory = "${config.services.loki.dataDir}/retention";
      };
    };
  };
  khscodes = {
    infrastructure.khs-openstack-instance = {
      enable = true;
      flavor = "m.medium";
    };
    services.nginx = {
      enable = true;
      virtualHosts."monitoring.kaareskovgaard.net" = {
        locations."/" = {
          proxyPass = "http://${grafana.settings.server.http_addr}:${toString grafana.settings.server.http_port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
      virtualHosts."loki.kaareskovgaard.net" = {
        mtls = {
          verify = "on";
          certificate = "/etc/loki/client-signer.pem";
        };
        locations."/" = {
          proxyPass = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
      virtualHosts."prometheus.kaareskovgaard.net" = {
        mtls = {
          verify = "on";
          certificate = "/etc/prometheus/client-signer.pem";
        };
        locations."/" = {
          proxyPass = "http://${prometheus.listenAddress}:${toString prometheus.port}";
          proxyWebsockets = true;
          recommendedProxySettings = true;
        };
      };
    };
    services.vault-agent.templates = [
      {
        contents = ''
          {{- with secret "loki-mtls/cert/ca_chain" -}}
          {{ .Data.certificate }}
          {{- end -}}
        '';
        destination = "/etc/loki/client-signer.pem";
        owner = "loki";
        group = "loki";
        perms = "0644";
        reloadOrRestartUnits = [ "nginx.service" ];
      }
      {
        contents = ''
          {{- with secret "prometheus-mtls/cert/ca_chain" -}}
          {{ .Data.certificate }}
          {{- end -}}
        '';
        destination = "/etc/prometheus/client-signer.pem";
        owner = "prometheus";
        group = "prometheus";
        perms = "0644";
        reloadOrRestartUnits = [ "nginx.service" ];
      }
      {
        contents = ''
          {{- with secret "kanidm/data/apps/monitoring" -}}
          {{ .Data.data.basic_secret }}
          {{- end -}}
        '';
        destination = "/var/lib/vault-agent/grafana/kanidm_client_secret";
        owner = "grafana";
        group = "grafana";
        perms = "0600";
        reloadOrRestartUnits = [ "grafana.service" ];
      }
    ];
    infrastructure.vault-server-approle.policy = {
      "kanidm/data/apps/monitoring" = {
        capabilities = [ "read" ];
      };
    };
  };
  snowfallorg.users.khs.admin = true;
  users.users.khs = {
    initialPassword = "changeme";
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw=="
    ];
  };
  khscodes.networking.fqdn = "monitoring.kaareskovgaard.net";
  system.stateVersion = "25.05";
}
