Add fleet handling and ensure all instances are up to date

nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix

@@ -162,10 +162,12 @@ in
                       content = config.khscodes.openstack.output.compute_instance.compute.ipv4_address;
                     }
                   ];
-                  aaaaRecords = lib.lists.map (d: {
+                  aaaaRecords = [
-                    fqdn = d;
+                    {
-                    content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address;
+                      fqdn = cfg.dnsName;
-                  }) cfg.dnsAliases;
+                      content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address;
+                    }
+                  ];
                   cnameRecords = lib.lists.map (domain: {
                     fqdn = domain;
                     content = cfg.dnsName;

nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy

@@ -1,5 +1,5 @@
 loki_send "node_exporter" {
-	job = "integrations/node_exporter"
+	job = "node_exporter"
 }
 
 // Collect logs from systemd journal for node_exporter integration

nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy

@@ -11,7 +11,7 @@ discovery.relabel "integrations_node_exporter" {
   rule {
     // Set a standard job name for all node_exporter metrics
     target_label = "job"
-    replacement = "integrations/node_exporter"
+    replacement = "node_exporter"
   }
 }
 //

nix/packages/configure-fleet/default.nix

@@ -0,0 +1,38 @@
+{
+  inputs,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  fleet = lib.attrsets.foldlAttrs (
+    acc: name: nixos:
+    acc
+    ++ (
+      if
+        (
+          nixos.config.khscodes.infrastructure.hetzner-instance.enable
+          || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable
+        )
+        && ((lib.lists.length nixos.config.khscodes.infrastructure.provisioning.post.modules) > 0)
+      then
+        [ ">&2 echo \"Configuring ${name}\n\"\nconfigure-instance ${lib.escapeShellArg name}" ]
+      else
+        [ ]
+    )
+  ) [ ] inputs.self.nixosConfigurations;
+in
+pkgs.writeShellApplication {
+  name = "configure-fleet";
+  runtimeInputs = [
+    pkgs.khscodes.configure-instance
+  ];
+  text = ''
+    if [[ "''${BW_SESSION:-}" == "" ]]; then
+      BW_SESSION="$(bw unlock --raw)"
+      export BW_SESSION
+      trap "bw lock" EXIT
+    fi
+    ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet}
+  '';
+}

nix/packages/provision-fleet/default.nix

@@ -0,0 +1,37 @@
+{
+  inputs,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  fleet = lib.attrsets.foldlAttrs (
+    acc: name: nixos:
+    acc
+    ++ (
+      if
+        nixos.config.khscodes.infrastructure.hetzner-instance.enable
+        || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable
+      then
+        [ ">&2 echo \"Provisioning ${name}\n\"\nprovision-instance ${lib.escapeShellArg name}" ]
+      else
+        [ ]
+    )
+  ) [ ] inputs.self.nixosConfigurations;
+in
+pkgs.writeShellApplication {
+  name = "provision-fleet";
+  runtimeInputs = [
+    pkgs.khscodes.provision-instance
+    pkgs.bitwarden-cli
+  ];
+  text = ''
+    if [[ "''${BW_SESSION:-}" == "" ]]; then
+      BW_SESSION="$(bw unlock --raw)"
+      >&2 echo ""
+      export BW_SESSION
+      trap "bw lock" EXIT
+    fi
+    ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet}
+  '';
+}

nix/packages/update-fleet/default.nix

@@ -0,0 +1,30 @@
+{
+  inputs,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  fleet = lib.attrsets.foldlAttrs (
+    acc: name: nixos:
+    acc
+    ++ (
+      if
+        nixos.config.khscodes.infrastructure.hetzner-instance.enable
+        || nixos.config.khscodes.infrastructure.khs-openstack-instance.enable
+      then
+        [ ">&2 echo \"Updating ${name}\n\"\nupdate-instance ${lib.escapeShellArg name}" ]
+      else
+        [ ]
+    )
+  ) [ ] inputs.self.nixosConfigurations;
+in
+pkgs.writeShellApplication {
+  name = "update-fleet";
+  runtimeInputs = [
+    pkgs.khscodes.update-instance
+  ];
+  text = ''
+    ${lib.strings.concatStringsSep "\n>&2 echo \"\"\n" fleet}
+  '';
+}

nix/systems/aarch64-linux/kas.codes/forgejo/default.nix

@@ -188,6 +188,8 @@ in
         targets    = [
           {
             "__address__" = "127.0.0.1:${toString config.services.forgejo.settings.server.HTTP_PORT}",
+            "instance" = constants.hostname,
+            "job" = "forgejo",
           },
         ]
         metrics_path = "/metrics"

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/instance.yaml

@@ -3,12 +3,12 @@
     - alert: DiskPressure
       expr: >
         (
-          node_filesystem_avail_bytes{fstype!="ramfs",job="integrations/node_exporter"} /
+          node_filesystem_avail_bytes{fstype!="ramfs",job="node_exporter"} /
-          node_filesystem_size_bytes{fstype!="ramfs",job="integrations/node_exporter"}
+          node_filesystem_size_bytes{fstype!="ramfs",job="node_exporter"}
         ) < 0.2
         and
         (
-          node_filesystem_avail_bytes{fstype!="ramfs",job="integrations/node_exporter"} /
+          node_filesystem_avail_bytes{fstype!="ramfs",job="node_exporter"} /
           1024 / 1024 / 1024
         ) < 20
       for: 10m

nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/alerts/systemd.yaml

@@ -2,7 +2,7 @@
   rules:
     - alert: UnitFailed
       expr: >
-        node_systemd_unit_state{job="integrations/node_exporter",state="failed"} == 1
+        node_systemd_unit_state{job="node_exporter",state="failed"} == 1
       for: 10m
       labels:
         severity: warn