From 1bf63cc7350442c9fc9aea0b1627d2bcdc0bbe12 Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Thu, 14 Aug 2025 00:26:01 +0200 Subject: [PATCH] Fix a bug with opentofu credentials and add zfs support to openstack and add it to monitoring --- .../khs-openstack-instance/default.nix | 125 +++++++++++++++++- nix/modules/terranix/openstack/default.nix | 1 + .../monitoring.kaareskovgaard.net/default.nix | 10 ++ .../monitoring.kaareskovgaard.net/zfs.nix | 55 ++++++++ .../infrastructure/src/secrets/endpoints.rs | 16 ++- .../infrastructure/src/secrets/openbao.rs | 4 - 6 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix diff --git a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix index 0588db6..6c087f9 100644 --- a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix +++ b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix @@ -6,6 +6,18 @@ }: let cfg = config.khscodes.infrastructure.khs-openstack-instance; + mainConfig = config; + hasDisks = cfg.dataDisks != [ ]; + hasZfsDisk = lib.lists.foldl (acc: d: acc || d.zfs) false cfg.dataDisks; + diskZpools = lib.mkMerge ( + lib.lists.map (d: { + "${d.zpoolName}".vdevs = [ + { + members = [ d.name ]; + } + ]; + }) (lib.lists.filter (d: d.zfs) cfg.dataDisks) + ); fqdn = config.khscodes.networking.fqdn; provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData; firewallTcpRules = lib.lists.flatten ( @@ -58,6 +70,30 @@ let remote_subnet = "::/0"; } ]; + diskModule = lib.khscodes.mkSubmodule' ( + { config }: + { + description = "Persistent disk"; + options = { + name = lib.mkOption { + type = lib.types.str; + }; + nameSanitized = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = lib.khscodes.sanitize-terraform-name config.name; + }; + zfs = lib.mkEnableOption "Enables adding the disk to a zpool as its own vdev"; + zpoolName = lib.mkOption { + type = lib.types.str; + default = mainConfig.khscodes.fs.zfs.mainPoolName; + }; + size = lib.mkOption { + type = lib.types.int; + }; + }; + } + ); firewallRules = firewallTcpRules ++ firewallUdpRules ++ firewallIcmpRules ++ cfg.extraFirewallRules; in { @@ -73,6 +109,11 @@ in lib.lists.filter (alias: alias != cfg.dnsName) config.khscodes.networking.aliases ); }; + dataDisks = lib.mkOption { + type = lib.types.listOf diskModule; + description = "Extra data disks to add to the instance, these will be added in the persistence phase"; + default = [ ]; + }; bucket = { key = lib.mkOption { type = lib.types.str; @@ -118,7 +159,7 @@ in config = lib.mkIf cfg.enable ( let tags = [ fqdn ]; - modules = [ + computeModules = [ ( { config, ... }: { @@ -187,6 +228,73 @@ in } ) ]; + persistenceModules = lib.lists.optional hasDisks ( + { ... }: + { + imports = [ + inputs.self.terranixModules.openstack + inputs.self.terranixModules.s3 + ]; + khscodes.s3 = { + enable = true; + bucket.key = "persistence-" + cfg.bucket.key; + }; + khscodes.openstack.enable = true; + resource.openstack_blockstorage_volume_v3 = lib.listToAttrs ( + lib.lists.map (disk: { + name = disk.nameSanitized; + value = { + inherit (disk) name size; + enable_online_resize = true; + volume_type = "__DEFAULT__"; + }; + }) cfg.dataDisks + ); + } + ); + persistenceAttachModules = lib.lists.optional hasDisks ( + { config, ... }: + { + config = { + data.openstack_blockstorage_volume_v3 = lib.listToAttrs ( + lib.lists.map (disk: { + name = disk.nameSanitized; + value = { + name = disk.name; + }; + }) cfg.dataDisks + ); + resource.openstack_compute_volume_attach_v2 = lib.listToAttrs ( + lib.lists.map (disk: { + name = disk.nameSanitized; + value = { + volume_id = "\${ data.openstack_blockstorage_volume_v3.${disk.nameSanitized}.id }"; + instance_id = config.khscodes.openstack.output.compute_instance.compute.id; + }; + }) cfg.dataDisks + ); + resource.vault_kv_secret_v2.data_disks = { + mount = "data-disks"; + name = fqdn; + data_json = '' + { + "template": "{id}", + "disks": ''${ jsonencode({ ${ + lib.strings.concatStringsSep ", " ( + lib.lists.map (disk: '' + ${builtins.toJSON disk.name} = { + "linuxDevice" = resource.openstack_compute_volume_attach_v2.${disk.nameSanitized}.device, + "size" = ${builtins.toString disk.size} + } + '') cfg.dataDisks + ) + } }) } + } + ''; + }; + }; + } + ); in { assertions = [ @@ -207,10 +315,23 @@ in khscodes.security.acme.dns01Enabled = true; khscodes.infrastructure.provisioning = { compute = { - modules = modules; + modules = computeModules; + }; + persistence = { + modules = persistenceModules; + }; + persistenceAttach = { + modules = persistenceAttachModules; }; imageUsername = "debian"; }; + khscodes.infrastructure.vault-server-approle.policy = lib.mkIf hasDisks { + "data-disks/data/${fqdn}" = { + capabilities = [ "read" ]; + }; + }; + khscodes.fs.zfs.enable = lib.mkIf hasZfsDisk true; + khscodes.fs.zfs.zpools = diskZpools; } ); } diff --git a/nix/modules/terranix/openstack/default.nix b/nix/modules/terranix/openstack/default.nix index c38acd6..ca2e540 100644 --- a/nix/modules/terranix/openstack/default.nix +++ b/nix/modules/terranix/openstack/default.nix @@ -386,6 +386,7 @@ in size = value.volume_size; image_id = "\${ data.openstack_images_image_v2.${sanitizedName}.id }"; volume_type = value.volume_type; + lifecycle.ignore_changes = [ "image_id" ]; }; } ) cfg.compute_instance; diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix index cbe142a..4baf538 100644 --- a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix @@ -56,6 +56,7 @@ in { imports = [ "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix" + ./zfs.nix ]; systemd.services.grafana = { unitConfig.ConditionPathExists = [ @@ -234,12 +235,21 @@ in }; }; khscodes = { + fs.zfs.zpools.zroot.encryptionKeyOpenbao.field = + "MONITORING_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY"; infrastructure.khs-openstack-instance = { enable = true; flavor = "m.large"; network = { router = null; }; + dataDisks = [ + { + name = "monitoring.kaareskovgaard.net-zroot-disk1"; + size = 15; + zfs = true; + } + ]; }; services.nginx = { enable = true; diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix new file mode 100644 index 0000000..dfb0e26 --- /dev/null +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/zfs.nix @@ -0,0 +1,55 @@ +{ + pkgs, + lib, + config, + ... +}: +{ + khscodes.fs.zfs.zpools.zroot.datasets = { + "monitoring/grafana" = { + mountpoint = "/var/lib/grafana"; + }; + "monitoring/loki" = { + mountpoint = "/var/lib/loki"; + }; + "monitoring/prometheus2" = { + mountpoint = "/var/lib/prometheus2"; + }; + }; + systemd.services = { + grafana = { + after = [ "khscodes-zpool-setup.service" ]; + wants = [ "khscodes-zpool-setup.service" ]; + unitConfig.RequiresMountsFor = [ + "/var/lib/grafana" + ]; + }; + loki = { + after = [ "khscodes-zpool-setup.service" ]; + wants = [ "khscodes-zpool-setup.service" ]; + unitConfig.RequiresMountsFor = [ + "/var/lib/loki" + ]; + }; + prometheus = { + after = [ "khscodes-zpool-setup.service" ]; + wants = [ "khscodes-zpool-setup.service" ]; + unitConfig.RequiresMountsFor = [ + "/var/lib/prometheus2" + ]; + }; + khscodes-zpool-setup = { + serviceConfig = { + ExecStartPost = [ + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chown"} grafana:grafana /var/lib/grafana" + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chmod"} 0700 /var/lib/grafana" + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chown"} loki:loki /var/lib/loki" + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chmod"} 0700 /var/lib/loki" + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chown"} prometheus:prometheus /var/lib/prometheus2" + "${lib.getExe' pkgs.uutils-coreutils-noprefix "chmod"} 0700 /var/lib/prometheus2" + ]; + }; + }; + }; + networking.hostId = "313166d7"; +} diff --git a/rust/program/infrastructure/src/secrets/endpoints.rs b/rust/program/infrastructure/src/secrets/endpoints.rs index 366844e..d1827eb 100644 --- a/rust/program/infrastructure/src/secrets/endpoints.rs +++ b/rust/program/infrastructure/src/secrets/endpoints.rs @@ -109,6 +109,19 @@ impl Endpoint for MxKaareskovgaardNet { const BITWARDEN_KEYS: &'static [BitwardenKey] = &[BitwardenKey::Field("ZROOT_ENCRYPTION_KEY")]; } +pub struct MonitoringKaareskovgaardNet; + +impl Endpoint for MonitoringKaareskovgaardNet { + const NAME: &'static str = "monitoring.kaareskovgaard.net"; + + const BITWARDEN_KEY: &'static str = "monitoring.kaareskovgaard.net"; + + const ENV_KEYS: &'static [&'static str] = + &["MONITORING_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY"]; + + const BITWARDEN_KEYS: &'static [BitwardenKey] = &[BitwardenKey::Field("ZROOT_ENCRYPTION_KEY")]; +} + #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)] pub enum CliEndpoint { #[serde(rename = "openstack")] @@ -155,7 +168,7 @@ impl CliEndpoint { Self::Cloudflare => Cloudflare.read_from_openbao(map), Self::Hcloud => Hcloud.read_from_openbao(map), Self::Openstack => Openstack.read_from_openbao(map), - Self::Unifi => Openstack.read_from_openbao(map), + Self::Unifi => Unifi.read_from_openbao(map), // We don't transfer the root token to openbao itself, but relies on the user being authenticated // through oauth. Self::Vault => Ok(()), @@ -173,6 +186,7 @@ pub fn transfer_from_bitwarden_to_vault(session: &mut BitwardenSession) -> anyho transfer_endpoint(Aws, session, &mut all_entries)?; transfer_endpoint(Cloudflare, session, &mut all_entries)?; transfer_endpoint(MxKaareskovgaardNet, session, &mut all_entries)?; + transfer_endpoint(MonitoringKaareskovgaardNet, session, &mut all_entries)?; for entry in all_entries { let mut delete_entry_proc = common::proc::Command::new("bao"); diff --git a/rust/program/infrastructure/src/secrets/openbao.rs b/rust/program/infrastructure/src/secrets/openbao.rs index 237e3bd..e532207 100644 --- a/rust/program/infrastructure/src/secrets/openbao.rs +++ b/rust/program/infrastructure/src/secrets/openbao.rs @@ -19,10 +19,6 @@ impl EnvEntry { fn new_from_values(values: Vec<(&'static str, String)>) -> Self { Self(values, PhantomData) } - - pub fn read_from_bao() -> anyhow::Result { - read_bao_data::() - } } impl From> for Vec<(&'static str, String)> {