From fa8320b8052a49c43bf0c4b0f241bdfcab9e6c18 Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Sun, 3 Aug 2025 22:29:19 +0200 Subject: [PATCH] Mount IMAP data in zfs volume, which should be easily backed up by TrueNAS. Also enable full text search --- .../infrastructure/nixos-install/default.nix | 10 + .../read-vault-auth-from-userdata/default.nix | 3 +- .../nixos/services/vault-agent/default.nix | 4 +- nix/modules/terranix/cloudflare/default.nix | 29 -- nix/packages/bw-opentofu/secrets-map.nix | 3 + nix/packages/nixos-install/default.nix | 13 +- .../mx.kaareskovgaard.net/default.nix | 26 ++ .../mx.kaareskovgaard.net/disko.nix | 295 +++++++++++------- .../mailserver/default.nix | 5 + rust/program/openbao-helper/src/main.rs | 15 + 10 files changed, 249 insertions(+), 154 deletions(-) create mode 100644 nix/modules/nixos/infrastructure/nixos-install/default.nix diff --git a/nix/modules/nixos/infrastructure/nixos-install/default.nix b/nix/modules/nixos/infrastructure/nixos-install/default.nix new file mode 100644 index 0000000..46c838f --- /dev/null +++ b/nix/modules/nixos/infrastructure/nixos-install/default.nix @@ -0,0 +1,10 @@ +{ lib, ... }: +{ + options.khscodes.infrastructure.nixos-install = { + preScript = lib.mkOption { + type = lib.types.anything; + default = ""; + description = "Script to run before running nixos-anywhere."; + }; + }; +} diff --git a/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix b/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix index a5c4553..2edca5f 100644 --- a/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix +++ b/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix @@ -27,7 +27,8 @@ in { systemd.services."read-vault-auth-from-userdata" = { enable = true; - wantedBy = [ "multi-user.target" ]; + wantedBy = [ "vault-agent-openbao.service" ]; + before = [ "vault-agent-openbao.service" ]; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; environment = { diff --git a/nix/modules/nixos/services/vault-agent/default.nix b/nix/modules/nixos/services/vault-agent/default.nix index 9f87199..02297e6 100644 --- a/nix/modules/nixos/services/vault-agent/default.nix +++ b/nix/modules/nixos/services/vault-agent/default.nix @@ -19,12 +19,12 @@ let restartUnits = svcs: lib.strings.concatStringsSep "\n" ( - lib.lists.map (svc: "systemctl restart ${lib.escapeShellArg svc}") svcs + lib.lists.map (svc: "systemctl restart ${lib.escapeShellArg svc} || true") svcs ); reloadOrRestartUnits = svcs: lib.strings.concatStringsSep "\n" ( - lib.lists.map (svc: "systemctl reload-or-restart ${lib.escapeShellArg svc}") svcs + lib.lists.map (svc: "systemctl reload-or-restart ${lib.escapeShellArg svc} || true") svcs ); mapTemplate = template: diff --git a/nix/modules/terranix/cloudflare/default.nix b/nix/modules/terranix/cloudflare/default.nix index fcf774a..2411f5e 100644 --- a/nix/modules/terranix/cloudflare/default.nix +++ b/nix/modules/terranix/cloudflare/default.nix @@ -19,35 +19,6 @@ let top = lib.lists.takeEnd 2 split; in lib.strings.concatStringsSep "." top; - serviceFromFqdn = - fqdn: - let - split = lib.strings.splitString "." fqdn; - in - assert - lib.strings.hasPrefix "_" (builtins.elemAt split 0) - && lib.strings.hasPrefix "_" (builtins.elemAt split 1); - builtins.elemAt split 0; - protocolFromFqdn = - fqdn: - let - split = lib.strings.splitString "." fqdn; - in - assert - lib.strings.hasPrefix "_" (builtins.elemAt split 0) - && lib.strings.hasPrefix "_" (builtins.elemAt split 1); - builtins.elemAt split 1; - nameFromFqdn = - fqdn: - let - split = lib.strings.splitString "." fqdn; - in - assert - lib.strings.hasPrefix "_" (builtins.elemAt split 0) - && lib.strings.hasPrefix "_" (builtins.elemAt split 1); - lib.strings.concatStringsSep "." ( - lib.lists.removePrefix [ (builtins.elemAt split 0) (builtins.elemAt split 1) ] split - ); dnsARecordModule = lib.khscodes.mkSubmodule { description = "Module for defining dns A/AAAA record"; options = { diff --git a/nix/packages/bw-opentofu/secrets-map.nix b/nix/packages/bw-opentofu/secrets-map.nix index 5346efc..9f53065 100644 --- a/nix/packages/bw-opentofu/secrets-map.nix +++ b/nix/packages/bw-opentofu/secrets-map.nix @@ -28,4 +28,7 @@ "secrets.kaareskovgaard.net" = { "VAULT_TOKEN" = "Initial root token"; }; + "mx.kaareskovgaard.net" = { + "MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY" = "ZROOT_ENCRYPTION_KEY"; + }; } diff --git a/nix/packages/nixos-install/default.nix b/nix/packages/nixos-install/default.nix index 9cf68d0..a34d72b 100644 --- a/nix/packages/nixos-install/default.nix +++ b/nix/packages/nixos-install/default.nix @@ -18,14 +18,17 @@ pkgs.writeShellApplication { # Build the configuration to ensure it doesn't fail when trying to install it on the host nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' fi - baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning' - config="$(nix build --no-link --print-out-paths "''${baseAttr}.preConfig")" - username="$(nix eval --raw "''${baseAttr}.preImageUsername")" + baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure' + config="$(nix build --no-link --print-out-paths "''${baseAttr}.provisioning.preConfig")" + preScript="$(nix eval --raw "''${baseAttr}.nixos-install.preScript")" + username="$(nix eval --raw "''${baseAttr}.provisioning.preImageUsername")" if [[ "$config" == "null" ]]; then echo "No preprovisioning needed" exit 0 fi - echo -n "tempkey" | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$username@$host" -- "cat >/tmp/tempkey" - nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" + + INSTALL_ARGS=() + eval "$preScript" + nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" "''${INSTALL_ARGS[@]}" ''; } diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix index 509e557..e8f5783 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix @@ -1,10 +1,21 @@ { + lib, inputs, ... }: +let + locationFromDatacenter = + datacenter: + let + split = lib.strings.splitString "-" datacenter; + in + assert (lib.lists.length split) == 2; + lib.lists.head split; +in { imports = [ "${inputs.self}/nix/profiles/nixos/hetzner-server.nix" + ./disko.nix ./mailserver ]; khscodes = { @@ -15,6 +26,21 @@ server_type = "cax11"; }; provisioning.pre.modules = [ + ( + { config, ... }: + { + resource.hcloud_volume.zroot-disk1 = { + name = "mx.kaareskovgaard.net-zroot-disk1"; + size = 100; + location = locationFromDatacenter config.khscodes.hcloud.server.compute.datacenter; + }; + resource.hcloud_volume_attachment.zroot-disk1 = { + volume_id = "\${ resource.hcloud_volume.zroot-disk1.id }"; + server_id = config.khscodes.hcloud.output.server.compute.id; + automount = false; + }; + } + ) ( { ... }: { diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix index f566ba2..46d2ce0 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix @@ -1,4 +1,9 @@ -{ pkgs, lib, ... }: +{ + config, + lib, + pkgs, + ... +}: let diskName = "nixos"; espSize = "500M"; @@ -7,161 +12,217 @@ let volumeGroupName = "mainpool"; rootLvName = "root"; zrootKey = "/run/secret/zroot.key"; - zfsLoadKeyScript = pkgs.writeShellApplication { - name = "load-zfs-key"; - runtimeInputs = [ pkgs.zfs ]; + # Don't ask me why this changes when there's more than one volume attached. + nixosDisk = "/dev/sdb"; + zrootDisk1Disk = "/dev/sda"; + vmailUser = config.mailserver.vmailUserName; + vmailGroup = config.mailserver.vmailGroupName; + + downloadZrootKey = pkgs.writeShellApplication { + name = "zfs-download-zroot-key"; + runtimeInputs = [ + pkgs.openbao + pkgs.zfs + pkgs.uutils-coreutils-noprefix + pkgs.jq + ]; text = '' - if ! zfs load-key -L ${zrootKey} zroot; then - echo -n "tempkey" /tmp/tempkey - zfs load-key -L /temp/tempkey zroot - zfs change-key -o keylocation=${zrootKey} zroot + if [[ "$(zfs list -j -o keystatus zroot/mailserver | jq --raw-output '.datasets."zroot/mailserver".properties.keystatus.value')" == "available" ]]; then + >&2 echo "Key already loaded, exiting" + exit 0 fi + # The vault cli insists on needing a token helper, disable it + HOME="$(mktemp -d)" + export HOME + trap 'rm -rf $HOME' EXIT + echo 'token_helper = "/bin/true"' > "$HOME/.vault" + role_id="$(cat /var/lib/vault-agent/role-id)" + secret_id="$(cat /var/lib/vault-agent/secret-id)" + VAULT_TOKEN="$(bao write -field=token auth/approle/login "role_id=$role_id" "secret_id=$secret_id")" + export VAULT_TOKEN + + encryption_key="$(bao kv get -mount=opentofu -field=MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY mx.kaareskovgaard.net)" + rm -rf "$HOME" + + echo "$encryption_key" | zfs load-key -L file:///dev/stdin zroot/mailserver ''; }; in { + systemd.services = { + dovecot2 = { + unitConfig.RequiresMountsFor = [ + "/var/mailserver/vmail" + "/var/mailserver/indices" + ]; + }; + }; + khscodes.infrastructure.nixos-install.preScript = '' + encryption_key="$(bao kv get -mount=opentofu -field=MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY mx.kaareskovgaard.net)" + tmpfile="$(mktemp)" + touch "$tmpfile" + chmod 0600 "$tmpfile" + trap "rm -f $tmpfile" EXIT + echo "$encryption_key" > "$tmpfile" + INSTALL_ARGS+=("--disk-encryption-keys") + INSTALL_ARGS+=("/run/secret/zroot.key") + INSTALL_ARGS+=("$tmpfile") + ''; + systemd.services.zfs-download-zroot-key = { + after = [ + "network-online.target" + "zfs-import-zroot.service" + "read-vault-auth-from-userdata.service" + ]; + wants = [ + "network-online.target" + "zfs-import-zroot.service" + "read-vault-auth-from-userdata.service" + ]; + wantedBy = [ "zfs-mount.target" ]; + before = [ "zfs-mount.target" ]; + environment = { + BAO_ADDR = config.khscodes.services.vault-agent.vault.address; + }; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = lib.getExe downloadZrootKey; + }; + }; khscodes.services.vault-agent.templates = [ { contents = '' - {{- with pkiCert "mx.kaareskovgaard.net/data/zroot_encryption" -}} - {{ .Data.data.key }} + {{- with secret "opentofu/data/mx.kaareskovgaard.net" -}} + {{ .Data.data.MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY }} {{- end -}} ''; destination = zrootKey; owner = "root"; group = "root"; perms = "0600"; - exec = lib.getExe zfsLoadKeyScript; + exec = '' + chown ${lib.escapeShellArg vmailUser}:${lib.escapeShellArg vmailGroup} /var/mailserver/vmail + chmod 2770 /var/mailserver/vmail + chown ${lib.escapeShellArg vmailUser}:${lib.escapeShellArg vmailGroup} /var/mailserver/indices + chmod 0700 /var/mailserver/indices + ''; restartUnits = [ + "zfs-mount.service" "postfix.service" "dovecot2.service" "rspamd.service" ]; } ]; - khscodes.infrastructure.provisioning.pre.modules = [ - { - resource.random_password.zroot_encryption_key = { - length = 48; - numeric = true; - lower = true; - upper = true; - special = false; - }; - resource.vault_kv_secret_v2.test = { - mount = "mx.kaareskovgaard.net"; - name = "zroot_encryption"; - data_json = '' - { - "key": ''${ jsonencode(resource.random_password.zroot_encryption_key.result) } - } - ''; - }; - } - ]; + mailserver.mailDirectory = "/var/mailserver/vmail"; + mailserver.indexDir = "/var/mailserver/indices"; khscodes.infrastructure.vault-server-approle.policy = { - "mx.kaareskovgaard.net/data/zroot_encryption" = { + "opentofu/data/mx.kaareskovgaard.net" = { capabilities = [ "read" ]; }; }; - disko.devices.disk = { - "${diskName}" = { - device = "/dev/sda"; - type = "disk"; - content = { - type = "gpt"; - partitions = { - "${bootPartName}" = { - size = espSize; - type = "EF00"; + networking.hostId = "9af535e4"; + disko.devices = { + disk = { + "${diskName}" = { + device = nixosDisk; + type = "disk"; + content = { + type = "gpt"; + partitions = { + "${bootPartName}" = { + size = espSize; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + }; + }; + "${rootPartName}" = { + size = "100%"; + content = { + type = "lvm_pv"; + vg = volumeGroupName; + }; + }; + }; + }; + }; + zroot-disk1 = { + device = zrootDisk1Disk; + type = "disk"; + content = { + type = "gpt"; + partitions = { + zfs = { + size = "100%"; + content = { + type = "zfs"; + pool = "zroot"; + }; + }; + }; + }; + }; + }; + lvm_vg = { + "${volumeGroupName}" = { + type = "lvm_vg"; + lvs = { + "${rootLvName}" = { + size = "100%"; content = { type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; - mountOptions = [ "umask=0077" ]; - }; - }; - "${rootPartName}" = { - size = "100%"; - content = { - type = "lvm_pv"; - vg = volumeGroupName; + format = "ext4"; + mountpoint = "/"; + mountOptions = [ "defaults" ]; }; }; }; }; }; - zroot1 = { - device = "/dev/sdb"; - type = "disk"; - content = { - type = "gpt"; - partitions = { - zfs = { - size = "100%"; - content = { - type = "zfs"; - pool = "zroot"; + zpool = { + zroot = { + type = "zpool"; + rootFsOptions = { + mountpoint = "none"; + compression = "zstd"; + acltype = "posixacl"; + xattr = "sa"; + "com.sun:auto-snapshot" = "true"; + }; + options.ashift = "12"; + datasets = { + "mailserver" = { + type = "zfs_fs"; + options = { + encryption = "aes-256-gcm"; + keyformat = "passphrase"; + keylocation = "file:///run/secret/zroot.key"; }; }; - }; - }; - }; - }; - devices.lvm_vg = { - "${volumeGroupName}" = { - type = "lvm_vg"; - lvs = { - "${rootLvName}" = { - size = "100%"; - content = { - type = "filesystem"; - format = "ext4"; - mountpoint = "/"; - mountOptions = [ "defaults" ]; + "mailserver/vmail" = { + type = "zfs_fs"; + mountpoint = "/var/mailserver/vmail"; + }; + "mailserver/indices" = { + type = "zfs_fs"; + mountpoint = "/var/mailserver/indices"; }; }; - }; - }; - }; - zpool = { - zroot = { - type = "zpool"; - rootFsOptions = { - mountpoint = "none"; - compression = "zstd"; - acltype = "posixacl"; - xattr = "sa"; - "com.sun:auto-snapshot" = "true"; - }; - options.ashift = "12"; - datasets = { - "mailserver" = { - type = "zfs_fs"; - options = { - encryption = "aes-256-gcm"; - keyformat = "passphrase"; - keylocation = "file:///tmp/tempkey"; + mode = { + topology = { + type = "topology"; + vdev = [ + { + members = [ "zroot-disk1" ]; + } + ]; }; }; - "mailserver/vmail" = { - type = "zfs_fs"; - mountpoint = "/var/mailserver/vmail"; - }; - "mailserver/indices" = { - type = "zfs_fs"; - mountpoint = "/var/mailserver/indices"; - }; - }; - mode = { - topology = { - type = "topology"; - vdev = [ - { - members = [ "zroot1" ]; - } - ]; - }; }; }; }; diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/default.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/default.nix index cbb60be..a514865 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/default.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/mailserver/default.nix @@ -95,6 +95,11 @@ in fqdn = config.khscodes.networking.fqdn; useUTF8FolderNames = true; certificateScheme = "acme"; + fullTextSearch = { + enable = true; + autoIndex = true; + enforced = "body"; + }; }; services.fail2ban.jails = { postfix = { diff --git a/rust/program/openbao-helper/src/main.rs b/rust/program/openbao-helper/src/main.rs index 9e574d5..2993082 100644 --- a/rust/program/openbao-helper/src/main.rs +++ b/rust/program/openbao-helper/src/main.rs @@ -54,6 +54,8 @@ pub enum Endpoint { Unifi, #[value(name = "vault")] Vault, + #[value(name = "mx.kaareskovgaard.net")] + MxKaareSkovgaardNet, } impl Endpoint { @@ -83,6 +85,10 @@ impl Endpoint { let data = VaultData::read_from_bao()?; Ok(data.into()) } + Self::MxKaareSkovgaardNet => { + let data = MxKaareSkovgaardNetData::read_from_bao()?; + Ok(data.into()) + } } } } @@ -147,6 +153,13 @@ entry_definition!( ); entry_definition!(VaultDataConfig, VaultData, "vault", &["VAULT_TOKEN"]); +entry_definition!( + MxKaareSkovgaardNetDataConfig, + MxKaareSkovgaardNetData, + "mx.kaareskovgaard.net", + &["MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY"] +); + fn transfer() -> anyhow::Result<()> { let openstack = OpenstackData::try_new_from_env()?; let cloudflare = CloudflareData::try_new_from_env()?; @@ -154,6 +167,7 @@ fn transfer() -> anyhow::Result<()> { let hcloud = HcloudData::try_new_from_env()?; let unifi = UnifiData::try_new_from_env()?; let vault = VaultData::try_new_from_env()?; + let mx_kaareskovgaard_net = MxKaareSkovgaardNetData::try_new_from_env()?; write_kv_data(openstack)?; write_kv_data(cloudflare)?; @@ -161,6 +175,7 @@ fn transfer() -> anyhow::Result<()> { write_kv_data(hcloud)?; write_kv_data(unifi)?; write_kv_data(vault)?; + write_kv_data(mx_kaareskovgaard_net)?; Ok(()) }