From 5037d791db559fe86913ff5f427f4480a6f812a1 Mon Sep 17 00:00:00 2001 From: Kaare Hoff Skovgaard Date: Fri, 11 Jul 2025 00:38:31 +0200 Subject: [PATCH] Get basic nginx and acme setup working This should enable DNS-01 acme for all khs openstack servers, thus removing the pain of setting up acme for those servers. Do note that this might not really be needed that much anymore, as I should be able to hit them over IPv6, but for ease of mind, this will enable ACME trivially, also for non https workloads, as well as servers without open ports. Do note that currently there's a global unifi firewall rule in place to allow port 80 and 443 to my own servers over ipv6, I'd like to remove this and have Nix configure firewall rules for each server individually, as requested in the setup. Former-commit-id: c402ada8f70329eb37966b3343ae3d8b347f7836 --- README.md | 22 ++-- .../khs-openstack-instance/default.nix | 60 +--------- .../vault-server-approle/default.nix | 7 +- nix/modules/nixos/openstack/default.nix | 2 + nix/modules/nixos/security/acme/default.nix | 18 ++- nix/modules/nixos/services/nginx/default.nix | 104 +++++++++++++++++- .../default.nix | 1 - nix/packages/create-instance/default.nix | 16 ++- nix/packages/instance-opentofu/default.nix | 7 +- nix/packages/nixos-install/default.nix | 7 +- nix/packages/provision-instance/default.nix | 9 ++ .../test.kaareskovgaard.net/default.nix | 15 ++- rust/program/openbao-helper/src/main.rs | 17 ++- 13 files changed, 184 insertions(+), 101 deletions(-) create mode 100644 nix/packages/provision-instance/default.nix diff --git a/README.md b/README.md index 54f72ca..17b0fab 100644 --- a/README.md +++ b/README.md @@ -8,43 +8,35 @@ When running on a desktop machine, simply running `nixos-install` as per usual s ## Servers -To provision the cloud resources needed, the following can be run: +To provision the cloud resources needed, and install NixOS, the following can be run: ```bash nix run '.#create-instance' -- ``` -This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO. +This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO. Then it will install NixOS. -Once the instance has been created it will _not_ run NixOS, but rather something like Debian, which can then be provisioned into a NixOS installation. Run the following command to enroll NixOS on the instance: +When making changes to eg. the approle needed, and needing to provision the instance again (but not installing NixOS again, as that won't work), run: ```bash -nix run '.#inxos-install' -- +nix run '.#provision-instance' -- ``` -
-NOTE -If you're creating and destroying instances on the same host name and have DNS caching trouble, you can run the following to connect using an IP address: +To update the NixOS config on an instance: ```bash -nix run '.#nixos-install' -- +nix run '.#update-instance` -- ``` -
- -TODO: Here should be some guidance on how to transfer RoleID/SecretID to the server, as well as running the post provisioning scripts for the servers that need it. - To delete the resources again run: ```bash nix run '.#destroy-instance' -- ``` -NOTE: It is normal for the secret id associated with vault/openbao roles to not be deletable. Simply run the destroy-instance command a 2nd time and everything should work just fine. - ## Secrets -To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO run: +To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO/Vault run: ```bash nix run '.#bitwarden-to-vault' diff --git a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix index c4f283d..12d70a1 100644 --- a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix +++ b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix @@ -97,62 +97,7 @@ in extraFirewallRules = lib.mkOption { type = lib.types.listOf lib.types.attrs; description = "Extra firewall rules added to the instance"; - default = [ - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "tcp"; - port = 80; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "tcp"; - port = 80; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "tcp"; - port = 443; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "tcp"; - port = 443; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "udp"; - port = 443; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "udp"; - port = 443; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "icmp"; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "icmp"; - remote_subnet = "::/0"; - } - ]; + default = [ ]; }; }; config = lib.mkIf cfg.enable ( @@ -236,6 +181,9 @@ in enable = true; }; }; + # khs openstack hosted servers are cannot use http-01 challenges (or maybe they can through ipv6?) + # so enable dns-01. + khscodes.security.acme.dns01Enabled = true; khscodes.infrastructure.provisioning = { pre = { modules = modules; diff --git a/nix/modules/nixos/infrastructure/vault-server-approle/default.nix b/nix/modules/nixos/infrastructure/vault-server-approle/default.nix index c3bc8b8..57d0596 100644 --- a/nix/modules/nixos/infrastructure/vault-server-approle/default.nix +++ b/nix/modules/nixos/infrastructure/vault-server-approle/default.nix @@ -53,6 +53,7 @@ in config = lib.mkIf cfg.enable { khscodes.services.openstack-read-vault-auth-from-userdata.enable = true; + khscodes.services.vault-agent.enable = true; khscodes.infrastructure.provisioning.${cfg.stage} = { endpoints = [ "vault" ]; modules = [ @@ -79,10 +80,10 @@ in # on the role being created first, which is needed. role_name = config.khscodes.vault.output.approle_auth_backend_role.${cfg.role_name}.role_name; # Should only be 5-10 mins once done testing - wrapping_ttl = 5 * 60 * 60; + wrapping_ttl = 5 * 60; - # All of this simply tries to ensure that I never recreate this secret id - # even if the original wrapped secret id is expired (which I expect it to be). + # This should simply mean that we never attempt to recreate the secret id, as we don't want a rerun of the + # provisioning to invalidate the existing secret id, nor recreate the entire server. with_wrapped_accessor = true; lifecycle = { ignore_changes = [ diff --git a/nix/modules/nixos/openstack/default.nix b/nix/modules/nixos/openstack/default.nix index bbc4bcb..9a54b5b 100644 --- a/nix/modules/nixos/openstack/default.nix +++ b/nix/modules/nixos/openstack/default.nix @@ -22,6 +22,8 @@ in diskName = cfg.diskName; } ); + # When this is set as the default, outbound ipv6 doesn't work on the instance. + networking.tempAddresses = "disabled"; boot.loader.grub.efiSupport = false; boot.loader.timeout = 1; khscodes.virtualisation.qemu-guest.enable = true; diff --git a/nix/modules/nixos/security/acme/default.nix b/nix/modules/nixos/security/acme/default.nix index 8b12928..d4955d0 100644 --- a/nix/modules/nixos/security/acme/default.nix +++ b/nix/modules/nixos/security/acme/default.nix @@ -4,7 +4,7 @@ let vaultAgentCredentialsFile = "/var/lib/vault-agent/acme/cloudflare-api-token"; cloudflareSecret = "opentofu/data/cloudflare"; acmeServicesToRestart = lib.lists.map (a: "acme-${a}.service") ( - lib.attrsets.attrNames config.security.certs + lib.attrsets.attrNames config.security.acme.certs ); in { @@ -25,21 +25,19 @@ in } // lib.attrsets.optionalAttrs cfg.dns01Enabled { dnsProvider = "cloudflare"; - dnsResolver = "1.1.1.1:53"; + dnsResolver = null; credentialsFile = vaultAgentCredentialsFile; }; }; khscodes.infrastructure.vault-server-approle = { enable = true; - policy = [ - { - "${cloudflareSecret}" = { - capabilities = [ "read" ]; - }; - } - ]; + policy = { + "${cloudflareSecret}" = { + capabilities = [ "read" ]; + }; + }; }; - khscodes.services.vault-agent = (cfg.dns01Enabled && acmeServicesToRestart != [ ]) { + khscodes.services.vault-agent = lib.mkIf (cfg.dns01Enabled && acmeServicesToRestart != [ ]) { enable = true; templates = [ { diff --git a/nix/modules/nixos/services/nginx/default.nix b/nix/modules/nixos/services/nginx/default.nix index dfb1dd5..ed07690 100644 --- a/nix/modules/nixos/services/nginx/default.nix +++ b/nix/modules/nixos/services/nginx/default.nix @@ -2,19 +2,66 @@ config, lib, pkgs, + modulesPath, ... }: let cfg = config.khscodes.services.nginx; + locationOptions = import "${modulesPath}/services/web-servers/nginx/location-options.nix" { + inherit lib config; + }; vhostOption = lib.khscodes.mkSubmodule { description = "nginx vhost"; options = { - useACMEHost = lib.mkOption { + acme = lib.mkOption { + description = "If a simple certificate for the virtual host name itself is not desired auto configured, then set this option. If set to a string it will be used as `useAcmeHost` from NixOS nginx service configuration. Otherwise set to the acme submodule and configure the desired certificate that way"; + type = lib.types.nullOr ( + lib.types.oneOf [ + lib.types.str + (lib.types.mkSubmodule { + description = "acme certificate"; + options = { + domains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Domain names the certificate should be requested for, should include the virtual host itself"; + }; + }; + }) + ] + ); + default = null; + }; + globalRedirect = lib.mkOption { type = lib.types.nullOr lib.types.str; - description = "Makes the virtual host use the certificate of another acme host"; + default = null; + description = "If set, all requests for this host are redirected (defaults to 301, configurable with redirectCode) to the given hostname."; + }; + redirectCode = lib.mkOption { + type = lib.types.int; + default = 301; + description = "HTTP status used by globalRedirect and forceSSL. Possible usecases include temporary (302, 307) redirects, keeping the request method and body (307, 308), or explicitly resetting the method to GET (303). See https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections."; + }; + extraConfig = lib.mkOption { + type = lib.types.lines; + description = "Extra configuration to inject into the generated nginx config"; + default = ''''; + }; + locations = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + description = "nginx virtual host location"; + options = locationOptions; + } + ); + default = { }; }; }; }; + dns01Enabled = config.khscodes.security.acme.dns01Enabled; + useAcmeConfiguration = lib.attrsets.foldlAttrs ( + acc: name: item: + acc || (item.acme != null && !lib.attrsets.isAttrs item.acme) + ) false cfg.virtualHosts; in { options.khscodes.services.nginx = { @@ -26,7 +73,15 @@ in }; }; config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = !useAcmeConfiguration || dns01Enabled; + message = "Cannot use `config.khscodes.services.nginx.virtualHosts..acme = {}` without setting config.khscodes.security.acme.dns01Enabled"; + } + ]; khscodes.security.acme.enable = true; + security.dhparams.enable = true; + security.dhparams.params."nginx".bits = 4096; services.nginx = { enable = true; package = lib.mkDefault pkgs.nginxStable; @@ -36,6 +91,51 @@ in recommendedOptimisation = lib.mkDefault true; recommendedZstdSettings = lib.mkDefault true; recommendedProxySettings = lib.mkDefault true; + virtualHosts = lib.attrsets.mapAttrs (name: value: { + inherit (value) + extraConfig + locations + globalRedirect + redirectCode + ; + forceSSL = true; + enableACME = value.acme == null && !dns01Enabled; + useACMEHost = + if lib.strings.isString value.acme then + value.acme + else if lib.attrsets.isAttrs value.acme || dns01Enabled then + name + else + null; + }) cfg.virtualHosts; }; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + networking.firewall.allowedUDPPorts = [ 443 ]; + users.users.nginx.extraGroups = lib.lists.optional dns01Enabled "acme"; + security.acme.certs = lib.mkIf dns01Enabled ( + lib.attrsets.foldlAttrs ( + acc: name: value: + ( + acc + // (lib.attrsets.optionalAttrs (lib.attrsets.isAttrs value.acme || dns01Enabled) { + "${name}" = + if value.acme == null then + { + domain = name; + reloadServices = [ "nginx" ]; + } + else + { + domain = lib.lists.head value.acme.domains; + extraDomainNames = lib.lists.tail value.acme.domains; + reloadServices = [ "nginx" ]; + }; + }) + ) + ) { } cfg.virtualHosts + ); }; } diff --git a/nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix b/nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix index 4e4135e..480c60e 100644 --- a/nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix +++ b/nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix @@ -19,7 +19,6 @@ in roleIdFilePath = config.khscodes.services.vault-agent.vault.roleIdFilePath; in { - services.khscodes.vault-agent.enable = true; systemd.services."openstack-read-vault-auth-from-userdata" = { enable = true; wantedBy = [ "multi-user.target" ]; diff --git a/nix/packages/create-instance/default.nix b/nix/packages/create-instance/default.nix index b3e1b38..6474d9d 100644 --- a/nix/packages/create-instance/default.nix +++ b/nix/packages/create-instance/default.nix @@ -1,9 +1,17 @@ -{ pkgs, ... }: +{ pkgs, inputs, ... }: pkgs.writeShellApplication { name = "create-instance"; - runtimeInputs = [ pkgs.khscodes.pre-provisioning ]; + runtimeInputs = [ + pkgs.khscodes.provision-instance + pkgs.khscodes.nixos-install + pkgs.jq + ]; text = '' - instance="''${1:-}" - pre-provisioning "$instance" apply + hostname="$1" + # Build the configuration to ensure it doesn't fail when trying to install it on the host + nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' + output="$(provision-instance "$hostname")" + ipv4_addr="$(echo "$output" | jq --raw-output '.ipv4_address.value')" + nixos-install "$hostname" "$ipv4_addr" "no" ''; } diff --git a/nix/packages/instance-opentofu/default.nix b/nix/packages/instance-opentofu/default.nix index a959fae..35a2438 100644 --- a/nix/packages/instance-opentofu/default.nix +++ b/nix/packages/instance-opentofu/default.nix @@ -17,6 +17,11 @@ pkgs.writeShellApplication { cat "''${config}" > "$dir/config.tf.json" tofu -chdir="$dir" init > /dev/null - tofu -chdir="$dir" "$cmd" + if [[ "$cmd" == "apply" ]]; then + tofu -chdir="$dir" "$cmd" >&2 + tofu -chdir="$dir" output -json + else + tofu -chdir="$dir" "$cmd" + fi ''; } diff --git a/nix/packages/nixos-install/default.nix b/nix/packages/nixos-install/default.nix index 7edf965..ac2ac5b 100644 --- a/nix/packages/nixos-install/default.nix +++ b/nix/packages/nixos-install/default.nix @@ -11,10 +11,13 @@ pkgs.writeShellApplication { # TODO: Use secret source and required secrets to set up the correct env variables text = '' hostname="$1" - # Build the configuration to ensure it doesn't fail when trying to install it on the host - nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' # Allow overriding the host to connec tto, this is useful when testing and the DNS entries are stale with older IPs. host="''${2:-$1}" + verify="''${3:-yes}" + if [[ "$verify" == "yes" ]]; then + # Build the configuration to ensure it doesn't fail when trying to install it on the host + nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' + fi baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning' config="$(nix build --no-link --print-out-paths "''${baseAttr}.preConfig")" username="$(nix eval --raw "''${baseAttr}.preImageUsername")" diff --git a/nix/packages/provision-instance/default.nix b/nix/packages/provision-instance/default.nix new file mode 100644 index 0000000..0acf0d5 --- /dev/null +++ b/nix/packages/provision-instance/default.nix @@ -0,0 +1,9 @@ +{ pkgs, ... }: +pkgs.writeShellApplication { + name = "provision-instance"; + runtimeInputs = [ pkgs.khscodes.pre-provisioning ]; + text = '' + instance="''${1:-}" + pre-provisioning "$instance" apply + ''; +} diff --git a/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix b/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix index 8aa3a36..2e30925 100644 --- a/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix +++ b/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix @@ -4,9 +4,18 @@ }: { imports = [ "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix" ]; - khscodes.infrastructure.khs-openstack-instance = { - enable = true; - flavor = "m.medium"; + khscodes = { + infrastructure.khs-openstack-instance = { + enable = true; + flavor = "m.medium"; + }; + services.nginx = { + enable = true; + virtualHosts."test.kaareskovgaard.net" = { + globalRedirect = "khs.codes"; + redirectCode = 302; + }; + }; }; snowfallorg.users.khs.admin = true; users.users.khs = { diff --git a/rust/program/openbao-helper/src/main.rs b/rust/program/openbao-helper/src/main.rs index a6c35ef..f76dd52 100644 --- a/rust/program/openbao-helper/src/main.rs +++ b/rust/program/openbao-helper/src/main.rs @@ -232,13 +232,22 @@ unsafe fn execvpe, SEK: AsRef, SEV: AsRef>( args: &[SA], environ: &[(SEK, SEV)], ) -> anyhow::Result { - let environ: Vec<_> = environ + let environ = environ .iter() .map(|(k, v)| { - CString::new(Format!("{k}={v}")) - .with_context(|| format!("Environment variable {k} contains null bytes"))? + CString::new(format!( + "{k}={v}", + k = k.as_ref().display(), + v = v.as_ref().display() + )) + .with_context(|| { + format!( + "Environment variable {k} contains null bytes", + k = k.as_ref().display() + ) + }) }) - .collect(); + .collect::>>()?; Ok(nix::unistd::execvpe(filename, args, &environ)?) }