From af583fcbd46545ff98b05b6a028cf9068e077842 Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Fri, 1 Aug 2025 22:21:21 +0200 Subject: [PATCH] Add fleet handling and ensure all instances are up to date --- .../khs-openstack-instance/default.nix | 10 +++-- .../vault-loki-sender/loki.alloy | 2 +- .../vault-prometheus-sender/prometheus.alloy | 2 +- nix/packages/configure-fleet/default.nix | 38 +++++++++++++++++++ nix/packages/provision-fleet/default.nix | 37 ++++++++++++++++++ nix/packages/update-fleet/default.nix | 30 +++++++++++++++ .../kas.codes/forgejo/default.nix | 2 + .../alerts/instance.yaml | 6 +-- .../alerts/job_up.yaml | 0 .../alerts/systemd.yaml | 2 +- 10 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 nix/packages/configure-fleet/default.nix create mode 100644 nix/packages/provision-fleet/default.nix create mode 100644 nix/packages/update-fleet/default.nix create mode 100644 nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/job_up.yaml diff --git a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix index e1b5ab7..8a6433c 100644 --- a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix +++ b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix @@ -162,10 +162,12 @@ in content = config.khscodes.openstack.output.compute_instance.compute.ipv4_address; } ]; - aaaaRecords = lib.lists.map (d: { - fqdn = d; - content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address; - }) cfg.dnsAliases; + aaaaRecords = [ + { + fqdn = cfg.dnsName; + content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address; + } + ]; cnameRecords = lib.lists.map (domain: { fqdn = domain; content = cfg.dnsName; diff --git a/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy b/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy index 27f3d9a..deada10 100644 --- a/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy +++ b/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy @@ -1,5 +1,5 @@ loki_send "node_exporter" { - job = "integrations/node_exporter" + job = "node_exporter" } // Collect logs from systemd journal for node_exporter integration diff --git a/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy index d84ece5..a2ff1d8 100644 --- a/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy +++ b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy @@ -11,7 +11,7 @@ discovery.relabel "integrations_node_exporter" { rule { // Set a standard job name for all node_exporter metrics target_label = "job" - replacement = "integrations/node_exporter" + replacement = "node_exporter" } } // diff --git a/nix/packages/configure-fleet/default.nix b/nix/packages/configure-fleet/default.nix new file mode 100644 index 0000000..a2b8d2b --- /dev/null +++ b/nix/packages/configure-fleet/default.nix @@ -0,0 +1,38 @@ +{ + inputs, + pkgs, + lib, + ... +}: +let + fleet = lib.attrsets.foldlAttrs ( + acc: name: nixos: + acc + ++ ( + if + ( + nixos.config.khscodes.infrastructure.hetzner-instance.enable + || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable + ) + && ((lib.lists.length nixos.config.khscodes.infrastructure.provisioning.post.modules) > 0) + then + [ ">&2 echo \"Configuring ${name}\n\"\nconfigure-instance ${lib.escapeShellArg name}" ] + else + [ ] + ) + ) [ ] inputs.self.nixosConfigurations; +in +pkgs.writeShellApplication { + name = "configure-fleet"; + runtimeInputs = [ + pkgs.khscodes.configure-instance + ]; + text = '' + if [[ "''${BW_SESSION:-}" == "" ]]; then + BW_SESSION="$(bw unlock --raw)" + export BW_SESSION + trap "bw lock" EXIT + fi + ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet} + ''; +} diff --git a/nix/packages/provision-fleet/default.nix b/nix/packages/provision-fleet/default.nix new file mode 100644 index 0000000..38f6d25 --- /dev/null +++ b/nix/packages/provision-fleet/default.nix @@ -0,0 +1,37 @@ +{ + inputs, + pkgs, + lib, + ... +}: +let + fleet = lib.attrsets.foldlAttrs ( + acc: name: nixos: + acc + ++ ( + if + nixos.config.khscodes.infrastructure.hetzner-instance.enable + || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable + then + [ ">&2 echo \"Provisioning ${name}\n\"\nprovision-instance ${lib.escapeShellArg name}" ] + else + [ ] + ) + ) [ ] inputs.self.nixosConfigurations; +in +pkgs.writeShellApplication { + name = "provision-fleet"; + runtimeInputs = [ + pkgs.khscodes.provision-instance + pkgs.bitwarden-cli + ]; + text = '' + if [[ "''${BW_SESSION:-}" == "" ]]; then + BW_SESSION="$(bw unlock --raw)" + >&2 echo "" + export BW_SESSION + trap "bw lock" EXIT + fi + ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet} + ''; +} diff --git a/nix/packages/update-fleet/default.nix b/nix/packages/update-fleet/default.nix new file mode 100644 index 0000000..21745ca --- /dev/null +++ b/nix/packages/update-fleet/default.nix @@ -0,0 +1,30 @@ +{ + inputs, + pkgs, + lib, + ... +}: +let + fleet = lib.attrsets.foldlAttrs ( + acc: name: nixos: + acc + ++ ( + if + nixos.config.khscodes.infrastructure.hetzner-instance.enable + || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable + then + [ ">&2 echo \"Updating ${name}\n\"\nupdate-instance ${lib.escapeShellArg name}" ] + else + [ ] + ) + ) [ ] inputs.self.nixosConfigurations; +in +pkgs.writeShellApplication { + name = "update-fleet"; + runtimeInputs = [ + pkgs.khscodes.update-instance + ]; + text = '' + ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet} + ''; +} diff --git a/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix b/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix index b1f257c..7e4b274 100644 --- a/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix +++ b/nix/systems/aarch64-linux/kas.codes/forgejo/default.nix @@ -188,6 +188,8 @@ in targets = [ { "__address__" = "127.0.0.1:${toString config.services.forgejo.settings.server.HTTP_PORT}", + "instance" = constants.hostname, + "job" = "forgejo", }, ] metrics_path = "/metrics" diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/instance.yaml b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/instance.yaml index da03a80..6a6e419 100644 --- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/instance.yaml +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/instance.yaml @@ -3,12 +3,12 @@ - alert: DiskPressure expr: > ( - node_filesystem_avail_bytes{fstype!="ramfs",job="integrations/node_exporter"} / - node_filesystem_size_bytes{fstype!="ramfs",job="integrations/node_exporter"} + node_filesystem_avail_bytes{fstype!="ramfs",job="node_exporter"} / + node_filesystem_size_bytes{fstype!="ramfs",job="node_exporter"} ) < 0.2 and ( - node_filesystem_avail_bytes{fstype!="ramfs",job="integrations/node_exporter"} / + node_filesystem_avail_bytes{fstype!="ramfs",job="node_exporter"} / 1024 / 1024 / 1024 ) < 20 for: 10m diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/job_up.yaml b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/job_up.yaml new file mode 100644 index 0000000..e69de29 diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml index 019eda2..e2910ae 100644 --- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml @@ -2,7 +2,7 @@ rules: - alert: UnitFailed expr: > - node_systemd_unit_state{job="integrations/node_exporter",state="failed"} == 1 + node_systemd_unit_state{job="node_exporter",state="failed"} == 1 for: 10m labels: severity: warn