diff --git a/nix/modules/nixos/hetzner/default.nix b/nix/modules/nixos/hetzner/default.nix index 51beb57..0f6e469 100644 --- a/nix/modules/nixos/hetzner/default.nix +++ b/nix/modules/nixos/hetzner/default.nix @@ -30,7 +30,7 @@ in diskName = cfg.diskName; } ); - + boot.growPartition = lib.mkDefault true; boot.tmp.cleanOnBoot = lib.mkDefault true; boot.initrd.kernelModules = lib.mkIf (system == "aarch64-linux") [ "virtio_gpu" ]; boot.kernelParams = lib.mkIf (system == "aarch64-linux") [ "console=tty" ]; @@ -67,13 +67,12 @@ in ${lib.getExe pkgs.khscodes.hetzner-static-ip} configure ''; }; - environment = - { - PATH = lib.mkForce ""; - } - // lib.attrsets.optionalAttrs (cfg.metadataApiUri != null) { - INSTANCE_API_URI = cfg.metadataApiUri; - }; + environment = { + PATH = lib.mkForce ""; + } + // lib.attrsets.optionalAttrs (cfg.metadataApiUri != null) { + INSTANCE_API_URI = cfg.metadataApiUri; + }; }; }; } diff --git a/nix/modules/nixos/openstack/default.nix b/nix/modules/nixos/openstack/default.nix index 9a54b5b..11e912a 100644 --- a/nix/modules/nixos/openstack/default.nix +++ b/nix/modules/nixos/openstack/default.nix @@ -24,6 +24,7 @@ in ); # When this is set as the default, outbound ipv6 doesn't work on the instance. networking.tempAddresses = "disabled"; + boot.growPartition = lib.mkDefault true; boot.loader.grub.efiSupport = false; boot.loader.timeout = 1; khscodes.virtualisation.qemu-guest.enable = true; diff --git a/nix/packages/disko-zpool-expand/default.nix b/nix/packages/disko-zpool-expand/default.nix new file mode 100644 index 0000000..a6c790f --- /dev/null +++ b/nix/packages/disko-zpool-expand/default.nix @@ -0,0 +1,14 @@ +{ + lib, + pkgs, + inputs, +}: +(lib.khscodes.mkRust pkgs "${inputs.self}/rust").buildRustPackage { + crateName = "disko-zpool-expand"; + replacePath = true; + runtimeInputs = [ + pkgs.zfs + pkgs.cloud-utils + pkgs.uutils-coreutils-noprefix # Needed for readlink which growpart ends up calling + ]; +} diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/README.md b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/README.md index 4daaa15..2b46fb4 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/README.md +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/README.md @@ -14,3 +14,7 @@ To add new domains to the MX, simply add them to domains in `default.nix`. This ## Loading of encryption key for the zpool The encryption key for the zpool is stored in OpenBAO, and is loaded during boot and should just work. The key is never stored on the server disk itself, and should never hit the disk itself. + +## Resizing zfs pool + +Simply changing the disk size in the nix code and running the `provision-instance` script and then rebooting the instance should be enough. diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix index ab2c77b..a522a99 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/default.nix @@ -39,7 +39,7 @@ in { resource.hcloud_volume.zroot-disk1 = { name = "mx.kaareskovgaard.net-zroot-disk1"; - size = 10; + size = 30; location = locationFromDatacenter config.khscodes.hcloud.server.compute.datacenter; }; resource.hcloud_volume_attachment.zroot-disk1 = { diff --git a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix index 3940eab..79c6b23 100644 --- a/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix +++ b/nix/systems/aarch64-linux/mx.kaareskovgaard.net/disko.nix @@ -138,6 +138,17 @@ in "/var/lib/vault-agent/secret-id" ]; }; + systemd.services.disko-zpool-expand-zroot = { + after = [ "zfs-download-zroot-key.service" ]; + wants = [ "zfs-download-zroot-key.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = '' + ${lib.getExe pkgs.khscodes.disko-zpool-expand} expand-zpool zroot + ''; + }; + }; mailserver.mailDirectory = "/var/mailserver/vmail"; mailserver.indexDir = "/var/mailserver/indices"; khscodes.infrastructure.vault-server-approle.policy = { diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6e12c7e..59ef90e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -270,6 +270,18 @@ dependencies = [ "syn", ] +[[package]] +name = "disko-zpool-expand" +version = "1.0.0" +dependencies = [ + "anyhow", + "clap", + "common", + "hakari", + "log", + "serde", +] + [[package]] name = "displaydoc" version = "0.2.5" diff --git a/rust/lib/common/src/proc.rs b/rust/lib/common/src/proc.rs index 9513c1b..7594235 100644 --- a/rust/lib/common/src/proc.rs +++ b/rust/lib/common/src/proc.rs @@ -308,21 +308,67 @@ impl Command { cmd.status() .with_context(|| format!("Could not spawn command: {}", command_to_string(self))) } + + pub fn spawn_into_parts(&mut self) -> anyhow::Result<(String, String, ExitStatus)> { + let mut cmd = self.as_command(); + let mut child = cmd + .stderr(self.stderr.as_std_stdio()) + .stdin(self.stdin.as_std_stdio()) + .stdout(Stdio::piped()) + .spawn() + .with_context(|| format!("Could not spawn command: {}", command_to_string(self)))?; + let mut stdin = Stdin::Null; + std::mem::swap(&mut self.stdin, &mut stdin); + let join_handle = if let Some(data) = stdin.into_data() { + let mut stdin_pipe = child.stdin.take().expect("Child has no stdin"); + Some(std::thread::spawn(move || { + stdin_pipe + .write_all(data.as_slice()) + .expect("Could not write to child"); + })) + } else { + None + }; + let output: Result<(Vec, Vec, ExitStatus), anyhow::Error> = + wait_with_output_into_parts(child, || command_to_string(self)); + if let Some(join_handle) = join_handle { + join_handle + .join() + .map_err(|e| anyhow::format_err!("Thread sending stdin panicked: {e:?}"))?; + } + let (stdout, stderr, status) = output?; + let stdout = + String::from_utf8(stdout).context("Could not read stdout of command as UTF-8")?; + let stderr = + String::from_utf8(stderr).context("Could not read stderr of command as UTF-8")?; + Ok((stdout, stderr, status)) + } } -fn wait_with_output(child: Child, cmd_str: impl Fn() -> String) -> anyhow::Result> { - let output = child - .wait_with_output() - .with_context(|| format!("Could not wait for output of commnad: {}", cmd_str()))?; - if !output.status.success() { +fn wait_with_output(child: Child, cmd_str: impl Fn() -> String + Clone) -> anyhow::Result> { + let (stdout, stderr, status) = wait_with_output_into_parts(child, cmd_str.clone())?; + if !status.success() { return Err(anyhow::format_err!( "Command {}, exited unexpectedly: {:?}. With stderr: {}", cmd_str(), - output.status, - String::from_utf8_lossy(&output.stderr), + status, + String::from_utf8_lossy(&stderr), )); } - Ok(output.stdout) + Ok(stdout) +} + +fn wait_with_output_into_parts( + child: Child, + cmd_str: impl Fn() -> String, +) -> anyhow::Result<(Vec, Vec, ExitStatus)> { + let output = child + .wait_with_output() + .with_context(|| format!("Could not wait for output of commnad: {}", cmd_str()))?; + let status = output.status; + let stdout = output.stdout; + let stderr = output.stderr; + Ok((stdout, stderr, status)) } #[cfg(test)] diff --git a/rust/program/disko-zpool-expand/Cargo.toml b/rust/program/disko-zpool-expand/Cargo.toml new file mode 100644 index 0000000..f63aab1 --- /dev/null +++ b/rust/program/disko-zpool-expand/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "disko-zpool-expand" +edition = "2024" +version = "1.0.0" +metadata.crane.name = "disko-zpool-expand" + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +common = { path = "../../lib/common" } +log = { workspace = true } +serde = { workspace = true } +hakari = { version = "0.1", path = "../../lib/hakari" } diff --git a/rust/program/disko-zpool-expand/src/main.rs b/rust/program/disko-zpool-expand/src/main.rs new file mode 100644 index 0000000..0a8e422 --- /dev/null +++ b/rust/program/disko-zpool-expand/src/main.rs @@ -0,0 +1,117 @@ +use serde::Deserialize; +use std::{collections::BTreeMap, path::PathBuf}; + +use anyhow::Context as _; +use clap::{Parser, Subcommand}; + +fn main() { + common::entrypoint(program); +} + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + /// Expands the partitions based on a zpool and brings the pool up to the new size. + #[command(name = "expand-zpool")] + ExpandZpool(ExpandZpool), +} + +#[derive(Debug, Clone, clap::Args)] +pub struct ExpandZpool { + /// Name of the pool to expand + pool_name: String, +} + +fn program() -> anyhow::Result<()> { + let args = Args::parse(); + match args.command { + Commands::ExpandZpool(pool) => expand_zpool(pool), + } +} + +#[derive(Deserialize)] +struct ZpoolStatus { + pools: BTreeMap, +} + +#[derive(Deserialize)] +struct ZpoolStatusPool { + state: Option, + vdevs: BTreeMap, +} + +#[derive(Clone, Copy, Deserialize, PartialEq)] +enum ZpoolState { + #[serde(rename = "ONLINE")] + Online, +} + +#[derive(Deserialize)] +struct ZpoolStatusVdev { + vdevs: BTreeMap, +} +#[derive(Deserialize)] +struct ZpoolStatusVdevVdev { + path: PathBuf, +} + +fn expand_zpool(p: ExpandZpool) -> anyhow::Result<()> { + let mut proc = common::proc::Command::new("zpool"); + proc.args(["status", "--json", &p.pool_name]); + let result: ZpoolStatus = proc + .try_spawn_to_json() + .context("Could not get zpool status")?; + + let pool = result + .pools + .get(&p.pool_name) + .context("Could not find requested pool in status output")?; + + if !pool + .state + .as_ref() + .is_some_and(|st| *st == ZpoolState::Online) + { + return Err(anyhow::format_err!("Zpool {} is not online", p.pool_name)); + } + + for vdev in pool.vdevs.values() { + for vdev in vdev.vdevs.values() { + let partition_dev = vdev.path.display().to_string(); + let Some(dev) = partition_dev.strip_suffix("-part1") else { + return Err(anyhow::format_err!( + "Expected vdev path {} to end with -part1", + vdev.path.display() + )); + }; + let mut proc = common::proc::Command::new("growpart"); + proc.args([dev, "1"]); + let (stdout, _stderr, status) = proc.spawn_into_parts()?; + if !status.success() && !stdout.starts_with("NOCHANGE: ") { + return Err(anyhow::format_err!( + "Could not resize partitin for {}, err: {stdout}", + vdev.path.display() + )); + } + // let name = partition_dev + // .split("/") + // .last() + // .expect("Should always have at least one element"); + let mut proc = common::proc::Command::new("zpool"); + proc.args(["online", "-e", &p.pool_name, &partition_dev]); + proc.try_spawn_to_string().with_context(|| { + format!( + "Could not bring zpool {} online with expand flag", + p.pool_name + ) + })?; + } + } + Ok(()) +}