From e6a152e95c7e7ece8017c775cb968b2d71182564 Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Tue, 5 Aug 2025 01:42:57 +0200 Subject: [PATCH] Begin working on porting much of the opentofu related code into rust. Mainly this should give proper argument parsing and error handling, and also remove some of all the scattered shell scripts. --- .../hetzner-instance/default.nix | 11 +- .../infrastructure/nixos-install/default.nix | 10 - nix/packages/instance-opentofu/default.nix | 2 +- nix/packages/nixos-install/default.nix | 10 +- nix/packages/provision-instance/default.nix | 1 + rust/Cargo.lock | 13 + rust/lib/common/src/fs.rs | 2 +- rust/program/provision/Cargo.toml | 14 + rust/program/provision/src/main.rs | 281 ++++++++++++++++++ 9 files changed, 319 insertions(+), 25 deletions(-) delete mode 100644 nix/modules/nixos/infrastructure/nixos-install/default.nix create mode 100644 rust/program/provision/Cargo.toml create mode 100644 rust/program/provision/src/main.rs diff --git a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix index 1671e2c..e7c566a 100644 --- a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix +++ b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix @@ -275,11 +275,14 @@ in data_json = '' { "template": "{id}", - "mapping": ''${ jsonencode({ ${ + "disks": ''${ jsonencode({ ${ lib.strings.concatStringsSep ", " ( - lib.lists.map ( - disk: "${builtins.toJSON disk.name} = data.hcloud_volume.${disk.nameSanitized}.linux_device" - ) cfg.dataDisks + lib.lists.map (disk: '' + ${builtins.toJSON disk.name} = { + "linuxDevice" = data.hcloud_volume.${disk.nameSanitized}.linux_device, + "size" = ${builtins.toString disk.size} + } + '') cfg.dataDisks ) } }) } } diff --git a/nix/modules/nixos/infrastructure/nixos-install/default.nix b/nix/modules/nixos/infrastructure/nixos-install/default.nix deleted file mode 100644 index 46c838f..0000000 --- a/nix/modules/nixos/infrastructure/nixos-install/default.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ lib, ... }: -{ - options.khscodes.infrastructure.nixos-install = { - preScript = lib.mkOption { - type = lib.types.anything; - default = ""; - description = "Script to run before running nixos-anywhere."; - }; - }; -} diff --git a/nix/packages/instance-opentofu/default.nix b/nix/packages/instance-opentofu/default.nix index 67103fe..686d8e6 100644 --- a/nix/packages/instance-opentofu/default.nix +++ b/nix/packages/instance-opentofu/default.nix @@ -18,7 +18,7 @@ pkgs.writeShellApplication { fqdn="$1" config="$2" cmd="''${3:-apply}" - dir="$(mktemp -dt "$fqdn-compute-provision.XXXXXX")" + dir="$(mktemp -dt "$fqdn-provision.XXXXXX")" mkdir -p "$dir" cat "''${config}" > "$dir/config.tf.json" diff --git a/nix/packages/nixos-install/default.nix b/nix/packages/nixos-install/default.nix index cc74cf1..4d1b428 100644 --- a/nix/packages/nixos-install/default.nix +++ b/nix/packages/nixos-install/default.nix @@ -19,16 +19,8 @@ pkgs.writeShellApplication { nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' fi baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure' - config="$(nix build --no-link --print-out-paths "''${baseAttr}.provisioning.compute.config")" - preScript="$(nix eval --raw "''${baseAttr}.nixos-install.preScript")" username="$(nix eval --raw "''${baseAttr}.provisioning.imageUsername")" - if [[ "$config" == "null" ]]; then - echo "No preprovisioning needed" - exit 0 - fi - INSTALL_ARGS=() - eval "$preScript" - nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" "''${INSTALL_ARGS[@]}" + nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" ''; } diff --git a/nix/packages/provision-instance/default.nix b/nix/packages/provision-instance/default.nix index c53f53e..1dd64a5 100644 --- a/nix/packages/provision-instance/default.nix +++ b/nix/packages/provision-instance/default.nix @@ -4,6 +4,7 @@ pkgs.writeShellApplication { runtimeInputs = [ pkgs.khscodes.provision ]; text = '' instance="''${1:-}" + provision "$instance" persistence apply provision "$instance" combinedPersistenceAttachAndCompute apply ''; } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 59ef90e..20fd84f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -795,6 +795,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "provision" +version = "1.0.0" +dependencies = [ + "anyhow", + "clap", + "common", + "hakari", + "log", + "serde", + "serde_json", +] + [[package]] name = "quote" version = "1.0.40" diff --git a/rust/lib/common/src/fs.rs b/rust/lib/common/src/fs.rs index 9f855af..5b55259 100644 --- a/rust/lib/common/src/fs.rs +++ b/rust/lib/common/src/fs.rs @@ -141,7 +141,7 @@ pub fn read_to_string(path: &Path) -> anyhow::Result { #[cfg(target_family = "unix")] pub fn create_link(from: &Path, to: &Path) -> anyhow::Result<()> { - std::os::unix::fs::symlink(to, from).with_context(|| { + std::os::unix::fs::symlink(from, to).with_context(|| { format!( "Could not create symbolic link from {from} to {to}", from = from.display(), diff --git a/rust/program/provision/Cargo.toml b/rust/program/provision/Cargo.toml new file mode 100644 index 0000000..b7954de --- /dev/null +++ b/rust/program/provision/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "provision" +edition = "2024" +version = "1.0.0" +metadata.crane.name = "provision" + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +common = { path = "../../lib/common" } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +hakari = { version = "0.1", path = "../../lib/hakari" } diff --git a/rust/program/provision/src/main.rs b/rust/program/provision/src/main.rs new file mode 100644 index 0000000..02efec0 --- /dev/null +++ b/rust/program/provision/src/main.rs @@ -0,0 +1,281 @@ +use serde::Deserialize; +use std::{ + collections::BTreeMap, + net::Ipv4Addr, + path::{Path, PathBuf}, +}; + +use clap::{Parser, Subcommand}; + +fn main() { + common::entrypoint(program); +} + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + /// Creates the instance. + CreateInstance(CreateInstance), + /// Updates the provisioned resources for the instance. + Update(UpdateInstance), + /// Configures the software installed on he instance. + Configure(ConfigureInstance), + /// Destroys the instance, but leaves behind persistent data, unless forcing deletion. + Destroy(DestroyInstance), +} + +#[derive(Debug, Clone, clap::Args)] +pub struct CreateInstance { + /// Name of the instance to create. + instance: String, + + #[arg(short = 's')] + skip_sanitity_checks: bool, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct UpdateInstance { + /// Name of the instance to update. + instance: String, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct ConfigureInstance { + /// Name of the instance to configure. + instance: String, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct DestroyInstance { + /// Name of the instance to destroy. + instance: String, +} + +fn program() -> anyhow::Result<()> { + let args = Args::parse(); + let flake_path = common::env::read_path_env("FLAKE_PATH")?; + match args.command { + Commands::CreateInstance(instance) => create_instance(instance, &flake_path), + _ => Ok(()), + } +} + +#[derive(Deserialize, Debug)] +struct ProvisioningData { + persistence: OpenTofuConfig, + compute: OpenTofuConfig, + compute_with_persistence_attached: OpenTofuConfig, + configuration: OpenTofuConfig, + secrets_source: SecretsSource, + + image_username: String, +} + +#[derive(Deserialize, Debug, Clone, Copy, PartialEq)] +enum SecretsSource { + #[serde(rename = "bitwarden")] + Bitwarden, + #[serde(rename = "vault")] + Vault, +} + +#[derive(Deserialize, Debug)] +struct OpenTofuConfig { + config: Option, + endpoints: Vec, +} + +struct WorkDir { + path: PathBuf, +} + +impl WorkDir { + pub fn try_new(template: &str) -> anyhow::Result { + let mut proc = common::proc::Command::new("mktemp"); + proc.args(["-dt", template]); + let path: PathBuf = proc.try_spawn_to_string()?.into(); + common::fs::create_dir_recursive(&path)?; + Ok(Self { path }) + } + + pub fn cleanup(self) -> anyhow::Result<()> { + common::fs::remove_dir_recursive(&self.path)?; + Ok(()) + } +} + +impl OpenTofuConfig { + pub fn init<'e, K: AsRef, V: AsRef>( + &self, + instance: &str, + env_map: &'e BTreeMap, + ) -> anyhow::Result>> { + let Some(config) = self.config.as_ref() else { + return Ok(None); + }; + let work_dir = WorkDir::try_new(&format!("{instance}-provision.XXXXXX"))?; + let config_path: PathBuf = work_dir.path.join("config.tf.json"); + common::fs::create_link(config, &config_path)?; + + let mut init_proc = common::proc::Command::new("tofu"); + let chdir_arg = format!("-chdir={}", work_dir.path.display()); + for (key, value) in env_map { + init_proc.env(key.as_ref(), value.as_ref()); + } + init_proc.args([&chdir_arg, "init"]); + init_proc.try_spawn_to_bytes()?; + Ok(Some(OpenTofuInstance { + work_dir, + env_map, + chdir_arg, + })) + } +} + +struct OpenTofuInstance<'e, K, V> { + work_dir: WorkDir, + chdir_arg: String, + env_map: &'e BTreeMap, +} + +impl<'e, K: AsRef, V: AsRef> OpenTofuInstance<'e, K, V> { + pub fn run(&mut self, action: &str) -> anyhow::Result<()> { + let mut proc = common::proc::Command::new("tofu"); + for (key, value) in self.env_map { + proc.env(key.as_ref(), value.as_ref()); + } + proc.args([&self.chdir_arg, action]); + proc.try_spawn_to_bytes()?; + Ok(()) + } + + pub fn output serde::Deserialize<'de>>(&mut self) -> anyhow::Result { + let mut proc = common::proc::Command::new("tofu"); + for (key, value) in self.env_map { + proc.env(key.as_ref(), value.as_ref()); + } + proc.args([&self.chdir_arg, "output", "-json"]); + proc.try_spawn_to_json() + } + + pub fn cleanup(self) -> anyhow::Result<()> { + self.work_dir.cleanup() + } +} + +impl ProvisioningData { + pub fn try_new(instance: &str, flake_path: &Path) -> anyhow::Result { + let mut proc = common::proc::Command::new("nix"); + let base_attr = format!( + "{}#nixosConfigurations.\"{instance}\".config.khscodes.infrastructure.provisioning", + flake_path.display() + ); + let script = r#" + let + data = prov: { inherit (prov) config endpoints; }; + in + p: { + persistence = data p.persistence; + compute = data p.compute; + compute_with_persistence_attached = data p.combinedPersistenceAttachAndCompute; + configuration = data p.configuration; + secrets_source = p.secretsSource; + image_username = p.imageUsername; + } + "#; + proc.args(["eval", "--json", &base_attr, "--apply", script]); + + let result = proc.stderr_inherit().try_spawn_to_json()?; + Ok(result) + } +} + +fn nixos_install( + flake_path: &Path, + instance: &str, + username: &str, + ip_addr: &Ipv4Addr, +) -> anyhow::Result<()> { + let mut proc = common::proc::Command::new("nixos-install"); + let flake_arg = format!("{}#{instance}", flake_path.display()); + let host_arg = format!("{username}@{ip_addr}"); + proc.args(["--flake", &flake_arg, "--target-host", &host_arg]); + + if !proc.stderr_inherit().try_spawn_stdout_inherit()?.success() { + return Err(anyhow::format_err!("nixos-install didn't succeed")); + } + Ok(()) +} + +fn create_instance(c: CreateInstance, flake_path: &Path) -> anyhow::Result<()> { + let data = ProvisioningData::try_new(&c.instance, flake_path)?; + if data.compute.config.is_none() { + return Err(anyhow::format_err!( + "No compute resources allocated for {}", + c.instance + )); + } + if !c.skip_sanitity_checks { + log::info!("Building system configuration to ensure it is installable"); + // First lets make sure we can build what we're trying to create + let mut proc = common::proc::Command::new("nix"); + proc.args([ + "build", + "--no-link", + &format!( + "{}#nixosConfigurations.\"{}\".config.system.build.toplevel", + flake_path.display(), + c.instance + ), + ]); + proc.stderr_inherit().try_spawn_to_bytes()?; + } + // TODO: Gather all needed endpoints and load the environments as needed from either Bitwarden or + // vault. Can probably get rid of the nix bw implementation and merge the rust helper program into this one + // to remove unneeded stuff. + let env = BTreeMap::::new(); + if let Some(mut persistence) = data.persistence.init(&c.instance, &env)? { + persistence.run("apply")?; + persistence.cleanup()?; + } + let mut compute = data + .compute + .init(&c.instance, &env)? + .expect("Verified earlier that config is not none"); + compute.run("apply")?; + + #[derive(Deserialize)] + struct Output { + ipv4_address: Ipv4AddrValue, + } + + #[derive(Deserialize)] + struct Ipv4AddrValue { + value: Ipv4Addr, + } + + let output: Output = compute.output()?; + compute.cleanup()?; + + nixos_install( + flake_path, + &c.instance, + &data.image_username, + &output.ipv4_address.value, + )?; + + if let Some(mut compute_with_persistence) = data + .compute_with_persistence_attached + .init(&c.instance, &env)? + { + compute_with_persistence.run("apply")?; + compute_with_persistence.cleanup()?; + } + Ok(()) +}