From 4fa553db5646df2fcf1cd0fd631d72c50d573bca Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Wed, 6 Aug 2025 23:27:26 +0200 Subject: [PATCH] Begin initial attempt at getting zfs setup working --- nix/checks/zfs/default.nix | 113 ++++++++++ nix/modules/nixos/fs/zfs/default.nix | 201 ++++++++++++++++++ .../hetzner-instance/default.nix | 18 ++ nix/packages/infrastructure/default.nix | 1 + .../default.nix | 4 +- .../mx.kaareskovgaard.net/default.nix | 4 +- .../mx.kaareskovgaard.net/disko.nix | 139 ------------ .../mx.kaareskovgaard.net/zfs.nix | 23 ++ rust/Cargo.lock | 172 +++++++++++---- rust/program/disko-zpool-expand/src/main.rs | 117 ---------- .../Cargo.toml | 4 +- rust/program/zpool-setup/src/cli.rs | 93 ++++++++ rust/program/zpool-setup/src/disk_mapping.rs | 41 ++++ rust/program/zpool-setup/src/main.rs | 200 +++++++++++++++++ rust/program/zpool-setup/src/zfs.rs | 174 +++++++++++++++ 15 files changed, 996 insertions(+), 308 deletions(-) create mode 100644 nix/checks/zfs/default.nix create mode 100644 nix/modules/nixos/fs/zfs/default.nix rename nix/packages/{disko-zpool-expand => zpool-setup}/default.nix (52%) delete mode 100644 nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix create mode 100644 nix/systems/aarch64-linux/mx.kaareskovgaard.net/zfs.nix delete mode 100644 rust/program/disko-zpool-expand/src/main.rs rename rust/program/{disko-zpool-expand => zpool-setup}/Cargo.toml (79%) create mode 100644 rust/program/zpool-setup/src/cli.rs create mode 100644 rust/program/zpool-setup/src/disk_mapping.rs create mode 100644 rust/program/zpool-setup/src/main.rs create mode 100644 rust/program/zpool-setup/src/zfs.rs diff --git a/nix/checks/zfs/default.nix b/nix/checks/zfs/default.nix new file mode 100644 index 0000000..0d984b4 --- /dev/null +++ b/nix/checks/zfs/default.nix @@ -0,0 +1,113 @@ +{ + inputs, + lib, + pkgs, + ... +}: +let + sharedModule = { + # Since it's common for CI not to have $DISPLAY available, explicitly disable graphics support + virtualisation.graphics = false; + }; + diskMapping = { + disks = { + "disk1" = { + linuxDevice = "/dev/vdb"; + size = 2048; + }; + }; + template = "{id}"; + }; + diskMappingScript = pkgs.writeShellApplication { + name = "disk-mapping"; + runtimeInputs = [ pkgs.util-linux ]; + text = '' + df -h + lsblk + ${lib.getExe' pkgs.uutils-coreutils-noprefix "mkdir"} -p /run/secret + echo ${lib.escapeShellArg (builtins.toJSON diskMapping)} > /run/secret/disk-mapping.json + ''; + }; + diskMappingModule = { + systemd.services.disk-mapping = { + enable = true; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = lib.getExe diskMappingScript; + }; + }; + systemd.services.khscodes-zpool-setup = { + after = [ "disk-mapping.service" ]; + wants = [ "disk-mapping.service" ]; + environment = { + LOGLEVEL = "trace"; + }; + }; + }; +in +pkgs.nixosTest { + name = "zfs"; + nodes = { + machine = + { ... }: + { + imports = [ + inputs.self.nixosModules.systemd-boot + inputs.self.nixosModules.hetzner + inputs.self.nixosModules."fs/zfs" + inputs.self.nixosModules."networking/fqdn" + inputs.self.nixosModules."infrastructure/vault-server-approle" + inputs.self.nixosModules."infrastructure/provisioning" + inputs.self.nixosModules."infrastructure/openbao" + inputs.self.nixosModules."services/vault-agent" + inputs.self.nixosModules."services/read-vault-auth-from-userdata" + inputs.self.nixosModules."services/openssh" + inputs.self.nixosModules."virtualisation/qemu-guest" + inputs.disko.nixosModules.disko + sharedModule + diskMappingModule + { + virtualisation.emptyDiskImages = [ diskMapping.disks.disk1.size ]; + khscodes.networking.fqdn = "machine"; + networking.hostId = "deadbeef"; + khscodes.fs.zfs = { + enable = true; + test = true; + zpools.zroot = { + vdevs = [ + { + mode = "mirror"; + members = [ "disk1" ]; + } + ]; + datasets = { + "mailserver/vmail" = { + mountpoint = "/var/mailserver/vmail"; + }; + }; + }; + }; + } + ]; + system.stateVersion = "25.05"; + }; + }; + testScript = '' + machine.start() + machine.wait_for_unit("disk-mapping.service") + machine.wait_for_unit("khscodes-zpool-setup.service") + machine.succeed("zpool status zroot") + machine.succeed("df -h | grep /var/mailserver/vmail") + machine.succeed("echo 'test' > /var/mailserver/vmail/test") + # machine.succeed("systemctl restart khscodes-zpool-setup.service") + machine.shutdown() + machine.start() + machine.wait_for_unit("khscodes-zpool-setup.service") + machine.succeed("df -h | grep /var/mailserver/vmail") + machine.succeed('if [[ "$(cat /var/mailserver/vmail/test)" == "test" ]]; then exit 0; else exit 1; fi') + machine.succeed("systemctl restart khscodes-zpool-setup.service") + machine.reboot() + ''; +} diff --git a/nix/modules/nixos/fs/zfs/default.nix b/nix/modules/nixos/fs/zfs/default.nix new file mode 100644 index 0000000..92bf4c9 --- /dev/null +++ b/nix/modules/nixos/fs/zfs/default.nix @@ -0,0 +1,201 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.khscodes.fs.zfs; + isTest = cfg.test; + zpoolSetup = lib.getExe pkgs.khscodes.zpool-setup; + setupZpool = + { name, value }: + let + enc = lib.strings.optionalString (!isTest) '' + + --encryption-key-mount=${lib.escapeShellArg value.encryptionKeyOpenbao.mount} \ + --encryption-key-name=${lib.escapeShellArg value.encryptionKeyOpenbao.name} \ + --encryption-key-field=${lib.escapeShellArg value.encryptionKeyOpenbao.field} \ + ''; + in + '' + ${zpoolSetup} setup ${enc} \ + --vdevs=${lib.escapeShellArg (builtins.toJSON value.vdevs)} \ + --root-fs-options=${lib.escapeShellArg (builtins.toJSON value.rootFsOptions)} \ + --zpool-options=${lib.escapeShellArg (builtins.toJSON value.zpoolOptions)} \ + --datasets=${lib.escapeShellArg (builtins.toJSON value.datasets)} \ + ${lib.escapeShellArg name} + ''; + setupZpools = lib.lists.map setupZpool (lib.attrsToList cfg.zpools); + vdevModule = lib.khscodes.mkSubmodule { + description = "vdev"; + options = { + mode = lib.mkOption { + type = lib.types.enum [ + "mirror" + "raidz" + "raidz1" + "raidz2" + "raidz3" + ]; + description = "Mode of the vdev"; + default = "mirror"; + }; + members = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Member disks of the vdev. Given as symbolic names, expected to be mapped to actual disks elsewhere."; + }; + }; + }; + datasetModule = lib.khscodes.mkSubmodule { + description = "dataset"; + options = { + options = lib.mkOption { + description = "Options for the dataset"; + type = lib.types.attrsOf lib.types.str; + default = { }; + }; + mountpoint = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Path to mount the dataset to"; + }; + }; + }; + zpoolModule = lib.khscodes.mkSubmodule { + description = "zpool"; + options = { + vdevs = lib.mkOption { + type = lib.types.listOf vdevModule; + default = [ ]; + }; + encryptionKeyOpenbao = { + mount = lib.mkOption { + type = lib.types.str; + default = "opentofu"; + description = "The mountpoint of the encryption key"; + }; + name = lib.mkOption { + type = lib.types.str; + description = "The name of the encryption key in the mount"; + default = config.khscodes.networking.fqdn; + }; + field = lib.mkOption { + type = lib.types.str; + description = "Field name of the encryption key"; + }; + }; + rootFsOptions = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + }; + zpoolOptions = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + }; + datasets = lib.mkOption { + type = lib.types.attrsOf datasetModule; + description = "Datasets for the zpool"; + default = { }; + }; + }; + }; +in +{ + options.khscodes.fs.zfs = { + enable = lib.mkEnableOption "Enables support for ZFS filesystem"; + test = lib.mkOption { + type = lib.types.bool; + description = "Enables test mode. In test mode no encryption keys are needed and no additional setup is added for them"; + default = false; + }; + mainPoolName = lib.mkOption { + type = lib.types.str; + default = "zroot"; + description = "The name of the main pool to create"; + }; + zpools = lib.mkOption { + type = lib.types.attrsOf zpoolModule; + description = "List of zpools and their layout"; + default = { + "${cfg.mainPoolName}" = { }; + }; + }; + }; + config = lib.mkIf cfg.enable { + # TODO: Verify that each member disk is uniquely named, and exists somewhere? + assertions = lib.lists.map ( + { name, value }: + { + assertion = (lib.lists.length value.vdevs) > 0; + message = "Zpool ${name} contains no vdevs"; + } + ) (lib.attrsToList cfg.zpools); + boot.supportedFilesystems = { + zfs = true; + }; + # On servers, we handle importing, creating and mounting of the pool manually. + boot.zfs = { + forceImportRoot = false; + requestEncryptionCredentials = false; + }; + systemd.services.zfs-mount.enable = false; + systemd.services.zfs-import-zroot.enable = false; + systemd.services.khscodes-zpool-setup = { + after = [ + "network-online.target" + ]; + wants = [ + "network-online.target" + ]; + wantedBy = [ + "multi-user.target" + ]; + environment = { + BAO_ADDR = config.khscodes.services.vault-agent.vault.address; + VAULT_ROLE_ID_FILE = "/var/lib/vault-agent/role-id"; + VAULT_SECRET_ID_FILE = "/var/lib/vault-agent/secret-id"; + DISK_MAPPING_FILE = "/run/secret/disk-mapping.json"; + } + // (lib.attrsets.optionalAttrs isTest { + ZFS_TEST = "true"; + }); + unitConfig.ConditionPathExists = [ + "/run/secret/disk-mapping.json" + ] + ++ lib.lists.optionals (!isTest) [ + "/var/lib/vault-agent/role-id" + "/var/lib/vault-agent/secret-id" + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = '' + ${lib.strings.concatStringsSep "\n" setupZpools} + ''; + }; + }; + khscodes.infrastructure.vault-server-approle.policy = lib.mapAttrs' (name: value: { + name = "${value.encryptionKeyOpenbao.mount}/data/${value.encryptionKeyOpenbao.name}"; + value = { + capabilities = [ "read" ]; + }; + }) cfg.zpools; + # Reading the disk setup through anopenbao secret allows + # the service to be restarted when adding new disks, or resizing existing disks. + khscodes.services.vault-agent.templates = [ + { + contents = '' + {{- with secret "data-disks/data/${config.khscodes.networking.fqdn}" -}} + {{ .Data.data | toUnescapedJSON }} + {{- end -}} + ''; + destination = "/run/secret/disk-mapping.json"; + owner = "root"; + group = "root"; + perms = "0644"; + restartUnits = [ "khscodes-zpool-setup.service" ]; + } + ]; + }; +} diff --git a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix index e7c566a..62e8861 100644 --- a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix +++ b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix @@ -6,7 +6,18 @@ }: let cfg = config.khscodes.infrastructure.hetzner-instance; + mainConfig = config; hasDisks = cfg.dataDisks != [ ]; + hasZfsDisk = lib.lists.foldl (acc: d: acc || d.zfs) false cfg.dataDisks; + diskZpools = lib.mkMerge ( + lib.lists.map (d: { + "${d.zpoolName}".vdevs = [ + { + members = [ d.name ]; + } + ]; + }) (lib.lists.filter (d: d.zfs) cfg.dataDisks) + ); fqdn = config.khscodes.networking.fqdn; provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData; locationFromDatacenter = @@ -30,6 +41,11 @@ let readOnly = true; default = lib.khscodes.sanitize-terraform-name config.name; }; + zfs = lib.mkEnableOption "Enables adding the disk to a zpool as its own vdev"; + zpoolName = lib.mkOption { + type = lib.types.str; + default = mainConfig.khscodes.fs.zfs.mainPoolName; + }; size = lib.mkOption { type = lib.types.int; }; @@ -384,6 +400,8 @@ in capabilities = [ "read" ]; }; }; + khscodes.fs.zfs.enable = lib.mkIf hasZfsDisk true; + khscodes.fs.zfs.zpools = diskZpools; khscodes.infrastructure.provisioning = { compute.modules = computeModules; persistence.modules = persistenceModules; diff --git a/nix/packages/infrastructure/default.nix b/nix/packages/infrastructure/default.nix index 872ef99..9b20839 100644 --- a/nix/packages/infrastructure/default.nix +++ b/nix/packages/infrastructure/default.nix @@ -6,6 +6,7 @@ (lib.khscodes.mkRust pkgs "${inputs.self}/rust").buildRustPackage { crateName = "infrastructure"; runtimeInputs = [ + pkgs.openssl # Needed when doing dkim keys pkgs.openssh pkgs.openbao pkgs.khscodes.opentofu diff --git a/nix/packages/disko-zpool-expand/default.nix b/nix/packages/zpool-setup/default.nix similarity index 52% rename from nix/packages/disko-zpool-expand/default.nix rename to nix/packages/zpool-setup/default.nix index a6c790f..41eec54 100644 --- a/nix/packages/disko-zpool-expand/default.nix +++ b/nix/packages/zpool-setup/default.nix @@ -4,11 +4,9 @@ inputs, }: (lib.khscodes.mkRust pkgs "${inputs.self}/rust").buildRustPackage { - crateName = "disko-zpool-expand"; + crateName = "zpool-setup"; replacePath = true; runtimeInputs = [ pkgs.zfs - pkgs.cloud-utils - pkgs.uutils-coreutils-noprefix # Needed for readlink which growpart ends up calling ]; } diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix index e31efc6..18b8af8 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix @@ -14,10 +14,11 @@ in { imports = [ "${inputs.self}/nix/profiles/nixos/hetzner-server.nix" - ./disko.nix + ./zfs.nix ./mailserver ]; khscodes = { + fs.zfs.zpools.zroot.encryptionKeyOpenbao.field = "MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY"; infrastructure = { hetzner-instance = { enable = true; @@ -26,6 +27,7 @@ in { name = "mx.kaareskovgaard.net-zroot-disk1"; size = 10; + zfs = true; } ]; server_type = "cax11"; diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix deleted file mode 100644 index a0f067d..0000000 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix +++ /dev/null @@ -1,139 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: -let - downloadZrootKey = pkgs.writeShellApplication { - name = "zfs-download-zroot-key"; - runtimeInputs = [ - pkgs.openbao - pkgs.zfs - pkgs.uutils-coreutils-noprefix - pkgs.jq - pkgs.gawk - ]; - text = '' - poolReady() { - pool="$1" - state="$(zpool import -d "/dev/disk/by-id" 2>/dev/null | awk "/pool: $pool/ { found = 1 }; /state:/ { if (found == 1) { print \$2; exit } }; END { if (found == 0) { print \"MISSING\" } }")" - if [[ "$state" = "ONLINE" ]]; then - return 0 - else - echo "Pool $pool in state $state, waiting" - return 1 - fi - } - poolImported() { - pool="$1" - zpool list "$pool" >/dev/null 2>/dev/null - } - poolImport() { - pool="$1" - zpool import -d "/dev/disk/by-id" -N "$pool" - } - if ! poolImported "zroot"; then - echo -n "importing ZFS pool \"zroot\"..." - # Loop across the import until it succeeds, because the devices needed may not be discovered yet. - for _ in $(seq 1 60); do - poolReady "zroot" && poolImport "zroot" && break - sleep 1 - done - poolImported "zroot" || poolImport "zroot" # Try one last time, e.g. to import a degraded pool. - fi - if ! poolImported "zroot"; then - echo "Could not import zroot" - exit 1 - fi - if [[ "$(zfs list -j -o keystatus zroot/mailserver | jq --raw-output '.datasets."zroot/mailserver".properties.keystatus.value')" == "unavailable" ]]; then - # The vault cli insists on needing a token helper, disable it - HOME="$(mktemp -d)" - export HOME - trap 'rm -rf $HOME' EXIT - echo 'token_helper = "/bin/true"' > "$HOME/.vault" - role_id="$(cat /var/lib/vault-agent/role-id)" - secret_id="$(cat /var/lib/vault-agent/secret-id)" - VAULT_TOKEN="$(bao write -field=token auth/approle/login "role_id=$role_id" "secret_id=$secret_id")" - export VAULT_TOKEN - - encryption_key="$(bao kv get -mount=opentofu -field=MX_KAARESKOVGAARD_NET_ZROOT_ENCRYPTION_KEY mx.kaareskovgaard.net)" - rm -rf "$HOME" - - echo "$encryption_key" | zfs load-key -L file:///dev/stdin zroot/mailserver - fi - zfs mount -a - ''; - }; -in -{ - systemd.services = { - dovecot2 = { - after = [ "zfs-download-zroot-key.service" ]; - wants = [ "zfs-download-zroot-key.service" ]; - unitConfig.RequiresMountsFor = [ - "/var/mailserver/vmail" - "/var/mailserver/indices" - ]; - }; - }; - boot.supportedFilesystems = { - zfs = true; - }; - boot.zfs = { - forceImportRoot = false; - requestEncryptionCredentials = false; - }; - systemd.services.zfs-mount.enable = false; - systemd.services.zfs-import-zroot.enable = false; - fileSystems = { - "/var/mailserver/vmail" = { - enable = lib.mkForce false; - }; - "/var/mailserver/indices" = { - enable = lib.mkForce false; - }; - }; - systemd.services.zfs-download-zroot-key = { - after = [ - "network-online.target" - ]; - wants = [ - "network-online.target" - ]; - wantedBy = [ - "multi-user.target" - ]; - environment = { - BAO_ADDR = config.khscodes.services.vault-agent.vault.address; - }; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = lib.getExe downloadZrootKey; - }; - unitConfig.ConditionPathExists = [ - "/var/lib/vault-agent/role-id" - "/var/lib/vault-agent/secret-id" - ]; - }; - systemd.services.disko-zpool-expand-zroot = { - after = [ "zfs-download-zroot-key.service" ]; - wants = [ "zfs-download-zroot-key.service" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "oneshot"; - ExecStart = '' - ${lib.getExe pkgs.khscodes.disko-zpool-expand} expand-zpool zroot - ''; - }; - }; - mailserver.mailDirectory = "/var/mailserver/vmail"; - mailserver.indexDir = "/var/mailserver/indices"; - khscodes.infrastructure.vault-server-approle.policy = { - "opentofu/data/mx.kaareskovgaard.net" = { - capabilities = [ "read" ]; - }; - }; - networking.hostId = "9af535e4"; -} diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/zfs.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/zfs.nix new file mode 100644 index 0000000..3c6e6df --- /dev/null +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/zfs.nix @@ -0,0 +1,23 @@ +{ + systemd.services = { + dovecot2 = { + after = [ "khscodes-zpool-setup.service" ]; + wants = [ "khscodes-zpool-setup.service" ]; + unitConfig.RequiresMountsFor = [ + "/var/mailserver/vmail" + "/var/mailserver/indices" + ]; + }; + }; + khscodes.fs.zfs.zpools.zroot.datasets = { + "mailserver/vmail" = { + mountpoint = "/var/mailserver/vmail"; + }; + "mailserver/indices" = { + mountpoint = "/var/mailserver/indices"; + }; + }; + mailserver.mailDirectory = "/var/mailserver/vmail"; + mailserver.indexDir = "/var/mailserver/indices"; + networking.hostId = "9af535e4"; +} diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 78faf43..28ec09f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -43,9 +43,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -73,22 +73,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -126,9 +126,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "cc" -version = "1.2.30" +version = "1.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2" dependencies = [ "shlex", ] @@ -147,9 +147,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "clap" -version = "4.5.40" +version = "4.5.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "50fd97c9dc2399518aa331917ac6f274280ec5eb34e555dd291899745c48ec6f" dependencies = [ "clap_builder", "clap_derive", @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "c35b5830294e1fa0462034af85cc95225a4cb07092c088c55bda3147cfcd8f65" dependencies = [ "anstream", "anstyle", @@ -169,9 +169,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", @@ -270,18 +270,6 @@ dependencies = [ "syn", ] -[[package]] -name = "disko-zpool-expand" -version = "1.0.0" -dependencies = [ - "anyhow", - "clap", - "common", - "hakari", - "log", - "serde", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -356,7 +344,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -368,7 +356,7 @@ dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -637,9 +625,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360e552c93fa0e8152ab463bc4c4837fce76a225df11dfaeea66c313de5e61f7" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ "bitflags", "libc", @@ -859,7 +847,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -890,9 +878,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "itoa", "memchr", @@ -1078,13 +1066,28 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", ] [[package]] @@ -1093,14 +1096,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -1109,48 +1129,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "writeable" version = "0.6.1" @@ -1265,9 +1333,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke", "zerofrom", @@ -1313,3 +1381,15 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zpool-setup" +version = "1.0.0" +dependencies = [ + "anyhow", + "clap", + "common", + "hakari", + "log", + "serde", +] diff --git a/rust/program/disko-zpool-expand/src/main.rs b/rust/program/disko-zpool-expand/src/main.rs deleted file mode 100644 index 0a8e422..0000000 --- a/rust/program/disko-zpool-expand/src/main.rs +++ /dev/null @@ -1,117 +0,0 @@ -use serde::Deserialize; -use std::{collections::BTreeMap, path::PathBuf}; - -use anyhow::Context as _; -use clap::{Parser, Subcommand}; - -fn main() { - common::entrypoint(program); -} - -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -pub struct Args { - #[command(subcommand)] - pub command: Commands, -} - -#[derive(Debug, Subcommand)] -pub enum Commands { - /// Expands the partitions based on a zpool and brings the pool up to the new size. - #[command(name = "expand-zpool")] - ExpandZpool(ExpandZpool), -} - -#[derive(Debug, Clone, clap::Args)] -pub struct ExpandZpool { - /// Name of the pool to expand - pool_name: String, -} - -fn program() -> anyhow::Result<()> { - let args = Args::parse(); - match args.command { - Commands::ExpandZpool(pool) => expand_zpool(pool), - } -} - -#[derive(Deserialize)] -struct ZpoolStatus { - pools: BTreeMap, -} - -#[derive(Deserialize)] -struct ZpoolStatusPool { - state: Option, - vdevs: BTreeMap, -} - -#[derive(Clone, Copy, Deserialize, PartialEq)] -enum ZpoolState { - #[serde(rename = "ONLINE")] - Online, -} - -#[derive(Deserialize)] -struct ZpoolStatusVdev { - vdevs: BTreeMap, -} -#[derive(Deserialize)] -struct ZpoolStatusVdevVdev { - path: PathBuf, -} - -fn expand_zpool(p: ExpandZpool) -> anyhow::Result<()> { - let mut proc = common::proc::Command::new("zpool"); - proc.args(["status", "--json", &p.pool_name]); - let result: ZpoolStatus = proc - .try_spawn_to_json() - .context("Could not get zpool status")?; - - let pool = result - .pools - .get(&p.pool_name) - .context("Could not find requested pool in status output")?; - - if !pool - .state - .as_ref() - .is_some_and(|st| *st == ZpoolState::Online) - { - return Err(anyhow::format_err!("Zpool {} is not online", p.pool_name)); - } - - for vdev in pool.vdevs.values() { - for vdev in vdev.vdevs.values() { - let partition_dev = vdev.path.display().to_string(); - let Some(dev) = partition_dev.strip_suffix("-part1") else { - return Err(anyhow::format_err!( - "Expected vdev path {} to end with -part1", - vdev.path.display() - )); - }; - let mut proc = common::proc::Command::new("growpart"); - proc.args([dev, "1"]); - let (stdout, _stderr, status) = proc.spawn_into_parts()?; - if !status.success() && !stdout.starts_with("NOCHANGE: ") { - return Err(anyhow::format_err!( - "Could not resize partitin for {}, err: {stdout}", - vdev.path.display() - )); - } - // let name = partition_dev - // .split("/") - // .last() - // .expect("Should always have at least one element"); - let mut proc = common::proc::Command::new("zpool"); - proc.args(["online", "-e", &p.pool_name, &partition_dev]); - proc.try_spawn_to_string().with_context(|| { - format!( - "Could not bring zpool {} online with expand flag", - p.pool_name - ) - })?; - } - } - Ok(()) -} diff --git a/rust/program/disko-zpool-expand/Cargo.toml b/rust/program/zpool-setup/Cargo.toml similarity index 79% rename from rust/program/disko-zpool-expand/Cargo.toml rename to rust/program/zpool-setup/Cargo.toml index f63aab1..c101082 100644 --- a/rust/program/disko-zpool-expand/Cargo.toml +++ b/rust/program/zpool-setup/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "disko-zpool-expand" +name = "zpool-setup" edition = "2024" version = "1.0.0" -metadata.crane.name = "disko-zpool-expand" +metadata.crane.name = "zpool-setup" [dependencies] anyhow = { workspace = true } diff --git a/rust/program/zpool-setup/src/cli.rs b/rust/program/zpool-setup/src/cli.rs new file mode 100644 index 0000000..2169509 --- /dev/null +++ b/rust/program/zpool-setup/src/cli.rs @@ -0,0 +1,93 @@ +use std::{borrow::Cow, collections::BTreeMap, path::PathBuf, str::FromStr}; + +use serde::Deserialize; + +use crate::disk_mapping::DiskMapping; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] +pub enum VdevMode { + #[serde(rename = "mirror")] + Mirror, + #[serde(rename = "raidz")] + Raidz, + #[serde(rename = "raidz1")] + Raidz1, + #[serde(rename = "raidz2")] + Raidz2, + #[serde(rename = "raidz3")] + Raidz3, +} + +impl VdevMode { + fn str(&self) -> &'static str { + match self { + Self::Mirror => "mirror", + Self::Raidz => "raidz", + Self::Raidz1 => "raidz1", + Self::Raidz2 => "raidz2", + Self::Raidz3 => "raidz3", + } + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Vdev { + pub mode: VdevMode, + pub members: Vec, +} + +impl Vdev { + pub fn cli_args(&self, disk_mapper: &DiskMapping) -> anyhow::Result>> { + let mut args = Vec::with_capacity(self.members.len() + 1); + if self.members.len() > 1 || self.mode != VdevMode::Mirror { + args.push(Cow::Borrowed(self.mode.str())); + } + for member in self.members.iter() { + let resolved = disk_mapper.resolve(member)?; + args.push(resolved.into()); + } + Ok(args) + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(transparent)] +pub struct Vdevs(pub Vec); + +impl FromStr for Vdevs { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + common::json::from_str(s) + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(transparent)] +pub struct Options(pub BTreeMap); + +impl FromStr for Options { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + common::json::from_str(s) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Dataset { + pub options: Options, + pub mountpoint: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(transparent)] +pub struct Datasets(pub BTreeMap); + +impl FromStr for Datasets { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + common::json::from_str(s) + } +} diff --git a/rust/program/zpool-setup/src/disk_mapping.rs b/rust/program/zpool-setup/src/disk_mapping.rs new file mode 100644 index 0000000..a31d403 --- /dev/null +++ b/rust/program/zpool-setup/src/disk_mapping.rs @@ -0,0 +1,41 @@ +use std::collections::BTreeMap; + +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +pub struct DiskMapping(DiskmappingFile); + +impl DiskMapping { + pub fn resolve(&self, name: &str) -> anyhow::Result { + let resolved = self + .0 + .disks + .get(name) + .ok_or_else(|| anyhow::format_err!("No mapping for disk named {}", name))?; + + Ok(self.0.template.execute(resolved.linux_device.as_str())) + } +} + +#[derive(Debug, Deserialize)] +struct DiskmappingFile { + disks: BTreeMap, + template: DeviceTemplate, +} + +#[derive(Debug, Deserialize)] +struct Disk { + #[serde(rename = "linuxDevice")] + linux_device: String, +} + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +struct DeviceTemplate(String); + +impl DeviceTemplate { + pub fn execute(&self, name: &str) -> String { + self.0.replace("{id}", name) + } +} diff --git a/rust/program/zpool-setup/src/main.rs b/rust/program/zpool-setup/src/main.rs new file mode 100644 index 0000000..a1104ff --- /dev/null +++ b/rust/program/zpool-setup/src/main.rs @@ -0,0 +1,200 @@ +use serde::Deserialize; +use std::{collections::BTreeMap, path::PathBuf}; + +use anyhow::Context as _; +use clap::{Parser, Subcommand}; + +mod cli; +mod disk_mapping; +mod zfs; + +fn main() { + common::entrypoint(program); +} + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + /// Expands the partitions based on a zpool and brings the pool up to the new size. + #[command(name = "setup")] + Setup(SetupZpool), +} + +#[derive(Debug, Clone, clap::Args)] +pub struct SetupZpool { + /// Openbao mount of the encryption key for the pool. Can only omit during test. + #[arg(long = "encryption-key-mount")] + encryption_key_mount: Option, + + /// Openbao name of the encryption key for the pool. Can only omit during test. + #[arg(long = "encryption-key-name")] + encryption_key_name: Option, + + /// Openbao name of the encryption field for the pool. Can only omit during test. + #[arg(long = "encryption-key-name")] + encryption_key_field: Option, + + /// Vdevs of the pool + #[arg(long = "vdevs")] + vdevs: cli::Vdevs, + + /// Options of the pool + #[arg(long = "zpool-options")] + zpool_options: cli::Options, + + /// Options of the root file system + #[arg(long = "root-fs-options")] + root_fs_options: cli::Options, + + /// Datasets the pool should have + #[arg(long = "datasets")] + datasets: cli::Datasets, + + /// Name of the pool to expand + pool_name: String, +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + pub fn try_new(template: &str) -> anyhow::Result { + let mut proc = common::proc::Command::new("mktemp"); + proc.args(["-dt", template]); + let path: PathBuf = proc.try_spawn_to_string()?.into(); + common::fs::create_dir_recursive(&path)?; + Ok(Self { path }) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + common::fs::remove_dir_recursive(&self.path).expect("Could not clean up after temp dir"); + } +} + +impl SetupZpool { + fn encryption_key(&self) -> anyhow::Result { + let is_test = common::env::read_env("ZFS_TEST").is_ok_and(|t| t == "true"); + if is_test { + return Ok(String::from("testtest")); + } + let role_id_file = common::env::read_path_env("VAULT_ROLE_ID_FILE")?; + let role_id = common::fs::read_to_string(&role_id_file)?; + let secret_id_file = common::env::read_path_env("VAULT_SECRET_ID_FILE")?; + let secret_id = common::fs::read_to_string(&secret_id_file)?; + let tmpdir = TempDir::try_new("zpool-setup.XXXXXX")?; + common::fs::write_file_string( + &tmpdir.path.join(".vault"), + "token_helper = \"/bin/true\"", + common::fs::user_only_file_permissions(), + )?; + let mut login_proc = common::proc::Command::new("bao"); + login_proc.env("HOME", tmpdir.path.display().to_string()); + login_proc.args(["write", "-field=token", "auth/approle/login"]); + login_proc.args([ + format!("role_id={role_id}"), + format!("secret_id={secret_id}"), + ]); + let vault_token = login_proc.try_spawn_to_string()?; + let (field, name, mount) = match ( + self.encryption_key_field.as_deref(), + self.encryption_key_name.as_deref(), + self.encryption_key_mount.as_deref(), + ) { + (Some(field), Some(name), Some(mount)) => (field, name, mount), + _ => { + return Err(anyhow::format_err!( + "Missing one of --encryption-key-mount, --encryption-key-name, --encryption-key-field" + )); + } + }; + let mut proc = common::proc::Command::new("bao"); + proc.env("HOME", tmpdir.path.display().to_string()); + proc.env_sensitive("VAULT_TOKEN", vault_token); + proc.args(["kv", "get"]); + proc.arg(format!("-field={field}")); + proc.arg(format!("-mount={mount}")); + proc.arg(name); + + proc.try_spawn_to_string() + } +} + +fn program() -> anyhow::Result<()> { + let args = Args::parse(); + match args.command { + Commands::Setup(setup) => setup_zpool(setup), + } +} + +#[derive(Deserialize)] +struct ZpoolStatus { + pools: BTreeMap, +} + +#[derive(Deserialize)] +struct ZpoolStatusPool { + state: Option, +} + +#[derive(Clone, Copy, Deserialize, PartialEq)] +enum ZpoolState { + #[serde(rename = "ONLINE")] + Online, +} + +fn setup_zpool(p: SetupZpool) -> anyhow::Result<()> { + let disk_mapping_file = common::env::read_path_env("DISK_MAPPING_FILE")?; + let disk_mapping = common::fs::read_to_string(&disk_mapping_file)?; + let disk_mapping = common::json::from_str(&disk_mapping)?; + if !zfs::import_pool(&p.pool_name)? { + let encryption_key = p.encryption_key()?; + zfs::create_pool(&p, &disk_mapping, &encryption_key)?; + for (name, dataset) in p.datasets.0.iter() { + zfs::create_dataset_recursive(&p.pool_name, name, dataset)?; + } + zfs::mount_all(&p.pool_name)?; + return Ok(()); + } + let mut proc: common::proc::Command = common::proc::Command::new("zpool"); + proc.args(["status", "--json", &p.pool_name]); + let result: ZpoolStatus = proc + .try_spawn_to_json() + .context("Could not get zpool status")?; + + let pool = result + .pools + .get(&p.pool_name) + .context("Could not find requested pool in status output")?; + + if !pool + .state + .as_ref() + .is_some_and(|st| *st == ZpoolState::Online) + { + return Err(anyhow::format_err!("Zpool {} is not online", p.pool_name)); + } + + for vdev in p.vdevs.0.iter() { + for member in vdev.members.iter() { + let resolved = disk_mapping.resolve(member)?; + zfs::resize_disk(&p.pool_name, &resolved)?; + } + } + if zfs::encryption_key_needs_load(&p.pool_name)? { + let encryption_key = p.encryption_key()?; + zfs::load_key(&p.pool_name, &encryption_key)?; + } + // TODO: Update pool options, and all fs options, and create missing datasets. + // Maybe for extranous datasets, set mountpoint=none ? + zfs::mount_all(&p.pool_name)?; + Ok(()) +} diff --git a/rust/program/zpool-setup/src/zfs.rs b/rust/program/zpool-setup/src/zfs.rs new file mode 100644 index 0000000..6dd85cb --- /dev/null +++ b/rust/program/zpool-setup/src/zfs.rs @@ -0,0 +1,174 @@ +use std::collections::BTreeMap; + +use anyhow::Context as _; +use common::proc::Command; +use serde::Deserialize; + +use crate::{SetupZpool, cli::Dataset, disk_mapping::DiskMapping}; + +#[derive(Debug, Deserialize, PartialEq)] +enum ZpoolState { + #[serde(rename = "ONLINE")] + Online, +} + +pub fn import_pool(name: &str) -> anyhow::Result { + // Test if the pool exists and is already imported + let mut exists_proc = Command::new("zpool"); + exists_proc.args(["status", name]); + if exists_proc.try_spawn_to_bytes().is_ok() { + return Ok(true); + } + // Try to import the pool if it exists + let mut proc = Command::new("zpool"); + proc.args(["import", name]); + + let (_stdout, stderr, exit_code) = proc.spawn_into_parts()?; + if exit_code.success() { + return Ok(true); + } + if stderr.contains("no such pool available") { + // The pool doesn't exist + return Ok(false); + } + Err(anyhow::format_err!( + "Could not import pool {name}, stderr: {stderr}" + )) +} + +pub fn create_pool( + zpool: &SetupZpool, + disk_mapping: &DiskMapping, + encryption_key: &str, +) -> anyhow::Result<()> { + let mut proc = Command::new("zpool"); + proc.args([ + "create", + zpool.pool_name.as_str(), + "-m", + "none", + "-o", + "feature@device_removal=enabled", + "-o", + "feature@draid=enabled", + "-o", + "feature@raidz_expansion=enabled", + "-o", + "feature@zilsaxattr=enabled", + "-o", + "feature@zstd_compress=enabled", + "-o", + "cachefile=none", + ]); + + for (key, value) in zpool.zpool_options.0.iter() { + proc.args(["-o", &format!("{key}={value}")]); + } + + for (key, value) in zpool.root_fs_options.0.iter() { + proc.args(["-O", &format!("{key}={value}")]); + } + + proc.args([ + "-O", + "encryption=aes-256-gcm", + "-O", + "keyformat=passphrase", + "-O", + "keylocation=prompt", + ]); + + for vdev in zpool.vdevs.0.iter() { + proc.args(vdev.cli_args(disk_mapping)?.into_iter()); + } + + proc.stdin_string(encryption_key); + proc.try_spawn_to_bytes()?; + + Ok(()) +} + +pub fn create_dataset_recursive( + pool_name: &str, + dataset_name: &str, + dataset: &Dataset, +) -> anyhow::Result<()> { + let mut proc = Command::new("zfs"); + let name = format!("{pool_name}/{dataset_name}"); + proc.args(["create", "-p", "-u"]); + if let Some(mountpoint) = dataset.mountpoint.as_deref() { + proc.arg("-o"); + proc.arg(format!("mountpoint={}", mountpoint.display())); + } + for (key, value) in dataset.options.0.iter() { + proc.arg("-o"); + proc.arg(format!("{key}={value}")); + } + + proc.arg(name); + + let _ = proc.try_spawn_to_bytes()?; + Ok(()) +} + +pub fn mount_all(pool: &str) -> anyhow::Result<()> { + let mut proc = Command::new("zfs"); + proc.args(["mount", "-R", pool]); + proc.try_spawn_to_bytes()?; + Ok(()) +} + +pub fn resize_disk(pool_name: &str, device: &str) -> anyhow::Result<()> { + let mut proc = Command::new("zpool"); + proc.args(["online", "-e", pool_name, device]); + let _ = proc.try_spawn_to_bytes().with_context(|| { + format!("Could not bring zpool {pool_name} online with expand flag for device {device}",) + })?; + Ok(()) +} + +pub fn load_key(pool_name: &str, encryption_key: &str) -> anyhow::Result<()> { + let mut proc = Command::new("zfs"); + proc.args(["load-key", "-r", "-L", "prompt", pool_name]); + proc.stdin_bytes(encryption_key); + proc.try_spawn_to_string()?; + Ok(()) +} + +pub fn encryption_key_needs_load(pool_name: &str) -> anyhow::Result { + #[derive(Deserialize)] + struct PoolEncStatus { + datasets: BTreeMap, + } + + #[derive(Deserialize)] + struct PoolEncStatusDataset { + properties: PoolEncStatusDatasetProperties, + } + + #[derive(Deserialize)] + struct PoolEncStatusDatasetProperties { + keystatus: PoolEncStatusDatasetProperty, + } + + #[derive(Deserialize)] + struct PoolEncStatusDatasetProperty { + value: Option, + } + + // "$(zfs list -j -o keystatus zroot/mailserver | jq --raw-output '.datasets."zroot/mailserver".properties.keystatus.value')" == "unavailable" + let mut proc = Command::new("zfs"); + proc.args(["list", "-j", "-o", "keystatus", pool_name]); + let json: PoolEncStatus = proc.try_spawn_to_json()?; + let pool = json + .datasets + .get(pool_name) + .ok_or_else(|| anyhow::format_err!("Pool {pool_name} not found in status output"))?; + + Ok(pool + .properties + .keystatus + .value + .as_deref() + .is_some_and(|v| v == "unavailable")) +}