diff --git a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix index 1671e2c..e7c566a 100644 --- a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix +++ b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix @@ -275,11 +275,14 @@ in data_json = '' { "template": "{id}", - "mapping": ''${ jsonencode({ ${ + "disks": ''${ jsonencode({ ${ lib.strings.concatStringsSep ", " ( - lib.lists.map ( - disk: "${builtins.toJSON disk.name} = data.hcloud_volume.${disk.nameSanitized}.linux_device" - ) cfg.dataDisks + lib.lists.map (disk: '' + ${builtins.toJSON disk.name} = { + "linuxDevice" = data.hcloud_volume.${disk.nameSanitized}.linux_device, + "size" = ${builtins.toString disk.size} + } + '') cfg.dataDisks ) } }) } } diff --git a/nix/modules/nixos/infrastructure/nixos-install/default.nix b/nix/modules/nixos/infrastructure/nixos-install/default.nix deleted file mode 100644 index 46c838f..0000000 --- a/nix/modules/nixos/infrastructure/nixos-install/default.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ lib, ... }: -{ - options.khscodes.infrastructure.nixos-install = { - preScript = lib.mkOption { - type = lib.types.anything; - default = ""; - description = "Script to run before running nixos-anywhere."; - }; - }; -} diff --git a/nix/packages/instance-opentofu/default.nix b/nix/packages/instance-opentofu/default.nix index 67103fe..686d8e6 100644 --- a/nix/packages/instance-opentofu/default.nix +++ b/nix/packages/instance-opentofu/default.nix @@ -18,7 +18,7 @@ pkgs.writeShellApplication { fqdn="$1" config="$2" cmd="''${3:-apply}" - dir="$(mktemp -dt "$fqdn-compute-provision.XXXXXX")" + dir="$(mktemp -dt "$fqdn-provision.XXXXXX")" mkdir -p "$dir" cat "''${config}" > "$dir/config.tf.json" diff --git a/nix/packages/nixos-install/default.nix b/nix/packages/nixos-install/default.nix index cc74cf1..4d1b428 100644 --- a/nix/packages/nixos-install/default.nix +++ b/nix/packages/nixos-install/default.nix @@ -19,16 +19,8 @@ pkgs.writeShellApplication { nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' fi baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure' - config="$(nix build --no-link --print-out-paths "''${baseAttr}.provisioning.compute.config")" - preScript="$(nix eval --raw "''${baseAttr}.nixos-install.preScript")" username="$(nix eval --raw "''${baseAttr}.provisioning.imageUsername")" - if [[ "$config" == "null" ]]; then - echo "No preprovisioning needed" - exit 0 - fi - INSTALL_ARGS=() - eval "$preScript" - nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" "''${INSTALL_ARGS[@]}" + nixos-anywhere --flake "${inputs.self}#$hostname" --target-host "$username@$host" ''; } diff --git a/nix/packages/provision-instance/default.nix b/nix/packages/provision-instance/default.nix index c53f53e..1dd64a5 100644 --- a/nix/packages/provision-instance/default.nix +++ b/nix/packages/provision-instance/default.nix @@ -4,6 +4,7 @@ pkgs.writeShellApplication { runtimeInputs = [ pkgs.khscodes.provision ]; text = '' instance="''${1:-}" + provision "$instance" persistence apply provision "$instance" combinedPersistenceAttachAndCompute apply ''; } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 59ef90e..20fd84f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -795,6 +795,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "provision" +version = "1.0.0" +dependencies = [ + "anyhow", + "clap", + "common", + "hakari", + "log", + "serde", + "serde_json", +] + [[package]] name = "quote" version = "1.0.40" diff --git a/rust/lib/common/src/fs.rs b/rust/lib/common/src/fs.rs index 9f855af..5b55259 100644 --- a/rust/lib/common/src/fs.rs +++ b/rust/lib/common/src/fs.rs @@ -141,7 +141,7 @@ pub fn read_to_string(path: &Path) -> anyhow::Result { #[cfg(target_family = "unix")] pub fn create_link(from: &Path, to: &Path) -> anyhow::Result<()> { - std::os::unix::fs::symlink(to, from).with_context(|| { + std::os::unix::fs::symlink(from, to).with_context(|| { format!( "Could not create symbolic link from {from} to {to}", from = from.display(), diff --git a/rust/program/provision/Cargo.toml b/rust/program/provision/Cargo.toml new file mode 100644 index 0000000..b7954de --- /dev/null +++ b/rust/program/provision/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "provision" +edition = "2024" +version = "1.0.0" +metadata.crane.name = "provision" + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +common = { path = "../../lib/common" } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +hakari = { version = "0.1", path = "../../lib/hakari" } diff --git a/rust/program/provision/src/main.rs b/rust/program/provision/src/main.rs new file mode 100644 index 0000000..02efec0 --- /dev/null +++ b/rust/program/provision/src/main.rs @@ -0,0 +1,281 @@ +use serde::Deserialize; +use std::{ + collections::BTreeMap, + net::Ipv4Addr, + path::{Path, PathBuf}, +}; + +use clap::{Parser, Subcommand}; + +fn main() { + common::entrypoint(program); +} + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + /// Creates the instance. + CreateInstance(CreateInstance), + /// Updates the provisioned resources for the instance. + Update(UpdateInstance), + /// Configures the software installed on he instance. + Configure(ConfigureInstance), + /// Destroys the instance, but leaves behind persistent data, unless forcing deletion. + Destroy(DestroyInstance), +} + +#[derive(Debug, Clone, clap::Args)] +pub struct CreateInstance { + /// Name of the instance to create. + instance: String, + + #[arg(short = 's')] + skip_sanitity_checks: bool, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct UpdateInstance { + /// Name of the instance to update. + instance: String, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct ConfigureInstance { + /// Name of the instance to configure. + instance: String, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct DestroyInstance { + /// Name of the instance to destroy. + instance: String, +} + +fn program() -> anyhow::Result<()> { + let args = Args::parse(); + let flake_path = common::env::read_path_env("FLAKE_PATH")?; + match args.command { + Commands::CreateInstance(instance) => create_instance(instance, &flake_path), + _ => Ok(()), + } +} + +#[derive(Deserialize, Debug)] +struct ProvisioningData { + persistence: OpenTofuConfig, + compute: OpenTofuConfig, + compute_with_persistence_attached: OpenTofuConfig, + configuration: OpenTofuConfig, + secrets_source: SecretsSource, + + image_username: String, +} + +#[derive(Deserialize, Debug, Clone, Copy, PartialEq)] +enum SecretsSource { + #[serde(rename = "bitwarden")] + Bitwarden, + #[serde(rename = "vault")] + Vault, +} + +#[derive(Deserialize, Debug)] +struct OpenTofuConfig { + config: Option, + endpoints: Vec, +} + +struct WorkDir { + path: PathBuf, +} + +impl WorkDir { + pub fn try_new(template: &str) -> anyhow::Result { + let mut proc = common::proc::Command::new("mktemp"); + proc.args(["-dt", template]); + let path: PathBuf = proc.try_spawn_to_string()?.into(); + common::fs::create_dir_recursive(&path)?; + Ok(Self { path }) + } + + pub fn cleanup(self) -> anyhow::Result<()> { + common::fs::remove_dir_recursive(&self.path)?; + Ok(()) + } +} + +impl OpenTofuConfig { + pub fn init<'e, K: AsRef, V: AsRef>( + &self, + instance: &str, + env_map: &'e BTreeMap, + ) -> anyhow::Result>> { + let Some(config) = self.config.as_ref() else { + return Ok(None); + }; + let work_dir = WorkDir::try_new(&format!("{instance}-provision.XXXXXX"))?; + let config_path: PathBuf = work_dir.path.join("config.tf.json"); + common::fs::create_link(config, &config_path)?; + + let mut init_proc = common::proc::Command::new("tofu"); + let chdir_arg = format!("-chdir={}", work_dir.path.display()); + for (key, value) in env_map { + init_proc.env(key.as_ref(), value.as_ref()); + } + init_proc.args([&chdir_arg, "init"]); + init_proc.try_spawn_to_bytes()?; + Ok(Some(OpenTofuInstance { + work_dir, + env_map, + chdir_arg, + })) + } +} + +struct OpenTofuInstance<'e, K, V> { + work_dir: WorkDir, + chdir_arg: String, + env_map: &'e BTreeMap, +} + +impl<'e, K: AsRef, V: AsRef> OpenTofuInstance<'e, K, V> { + pub fn run(&mut self, action: &str) -> anyhow::Result<()> { + let mut proc = common::proc::Command::new("tofu"); + for (key, value) in self.env_map { + proc.env(key.as_ref(), value.as_ref()); + } + proc.args([&self.chdir_arg, action]); + proc.try_spawn_to_bytes()?; + Ok(()) + } + + pub fn output serde::Deserialize<'de>>(&mut self) -> anyhow::Result { + let mut proc = common::proc::Command::new("tofu"); + for (key, value) in self.env_map { + proc.env(key.as_ref(), value.as_ref()); + } + proc.args([&self.chdir_arg, "output", "-json"]); + proc.try_spawn_to_json() + } + + pub fn cleanup(self) -> anyhow::Result<()> { + self.work_dir.cleanup() + } +} + +impl ProvisioningData { + pub fn try_new(instance: &str, flake_path: &Path) -> anyhow::Result { + let mut proc = common::proc::Command::new("nix"); + let base_attr = format!( + "{}#nixosConfigurations.\"{instance}\".config.khscodes.infrastructure.provisioning", + flake_path.display() + ); + let script = r#" + let + data = prov: { inherit (prov) config endpoints; }; + in + p: { + persistence = data p.persistence; + compute = data p.compute; + compute_with_persistence_attached = data p.combinedPersistenceAttachAndCompute; + configuration = data p.configuration; + secrets_source = p.secretsSource; + image_username = p.imageUsername; + } + "#; + proc.args(["eval", "--json", &base_attr, "--apply", script]); + + let result = proc.stderr_inherit().try_spawn_to_json()?; + Ok(result) + } +} + +fn nixos_install( + flake_path: &Path, + instance: &str, + username: &str, + ip_addr: &Ipv4Addr, +) -> anyhow::Result<()> { + let mut proc = common::proc::Command::new("nixos-install"); + let flake_arg = format!("{}#{instance}", flake_path.display()); + let host_arg = format!("{username}@{ip_addr}"); + proc.args(["--flake", &flake_arg, "--target-host", &host_arg]); + + if !proc.stderr_inherit().try_spawn_stdout_inherit()?.success() { + return Err(anyhow::format_err!("nixos-install didn't succeed")); + } + Ok(()) +} + +fn create_instance(c: CreateInstance, flake_path: &Path) -> anyhow::Result<()> { + let data = ProvisioningData::try_new(&c.instance, flake_path)?; + if data.compute.config.is_none() { + return Err(anyhow::format_err!( + "No compute resources allocated for {}", + c.instance + )); + } + if !c.skip_sanitity_checks { + log::info!("Building system configuration to ensure it is installable"); + // First lets make sure we can build what we're trying to create + let mut proc = common::proc::Command::new("nix"); + proc.args([ + "build", + "--no-link", + &format!( + "{}#nixosConfigurations.\"{}\".config.system.build.toplevel", + flake_path.display(), + c.instance + ), + ]); + proc.stderr_inherit().try_spawn_to_bytes()?; + } + // TODO: Gather all needed endpoints and load the environments as needed from either Bitwarden or + // vault. Can probably get rid of the nix bw implementation and merge the rust helper program into this one + // to remove unneeded stuff. + let env = BTreeMap::::new(); + if let Some(mut persistence) = data.persistence.init(&c.instance, &env)? { + persistence.run("apply")?; + persistence.cleanup()?; + } + let mut compute = data + .compute + .init(&c.instance, &env)? + .expect("Verified earlier that config is not none"); + compute.run("apply")?; + + #[derive(Deserialize)] + struct Output { + ipv4_address: Ipv4AddrValue, + } + + #[derive(Deserialize)] + struct Ipv4AddrValue { + value: Ipv4Addr, + } + + let output: Output = compute.output()?; + compute.cleanup()?; + + nixos_install( + flake_path, + &c.instance, + &data.image_username, + &output.ipv4_address.value, + )?; + + if let Some(mut compute_with_persistence) = data + .compute_with_persistence_attached + .init(&c.instance, &env)? + { + compute_with_persistence.run("apply")?; + compute_with_persistence.cleanup()?; + } + Ok(()) +}