Get basic nginx and acme setup working

This should enable DNS-01 acme for all khs openstack servers,
thus removing the pain of setting up acme for those servers.

Do note that this might not really be needed that much anymore,
as I should be able to hit them over IPv6, but for ease of mind,
this will enable ACME trivially, also for non https workloads, as well
as servers without open ports.

Do note that currently there's a global unifi firewall rule in place to
allow port 80 and 443 to my own servers over ipv6, I'd like to remove this
and have Nix configure firewall rules for each server individually, as
requested in the setup.


Former-commit-id: c402ada8f7

README.md

@@ -8,43 +8,35 @@ When running on a desktop machine, simply running `nixos-install` as per usual s
 
 ## Servers
 
-To provision the cloud resources needed, the following can be run:
+To provision the cloud resources needed, and install NixOS, the following can be run:
 
 ```bash
 nix run '.#create-instance' -- <hostname>
 ```
 
-This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO.
+This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO. Then it will install NixOS.
 
-Once the instance has been created it will _not_ run NixOS, but rather something like Debian, which can then be provisioned into a NixOS installation. Run the following command to enroll NixOS on the instance:
+When making changes to eg. the approle needed, and needing to provision the instance again (but not installing NixOS again, as that won't work), run:
 
 ```bash
-nix run '.#inxos-install' -- <hostname>
+nix run '.#provision-instance' -- <hostname>
 ```
 
-<details>
+To update the NixOS config on an instance:
-<summary>NOTE</summary>
-If you're creating and destroying instances on the same host name and have DNS caching trouble, you can run the following to connect using an IP address:
 
 ```bash
-nix run '.#nixos-install' -- <hostname> <ip>
+nix run '.#update-instance` -- <hostname>
 ```
 
-</details>
-
-TODO: Here should be some guidance on how to transfer RoleID/SecretID to the server, as well as running the post provisioning scripts for the servers that need it.
-
 To delete the resources again run:
 
 ```bash
 nix run '.#destroy-instance' -- <hostname>
 ```
 
-NOTE: It is normal for the secret id associated with vault/openbao roles to not be deletable. Simply run the destroy-instance command a 2nd time and everything should work just fine.
-
 ## Secrets
 
-To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO run:
+To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO/Vault run:
 
 ```bash
 nix run '.#bitwarden-to-vault'

nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix

@@ -97,62 +97,7 @@ in
     extraFirewallRules = lib.mkOption {
       type = lib.types.listOf lib.types.attrs;
       description = "Extra firewall rules added to the instance";
-      default = [
+      default = [ ];
-        {
-          direction = "egress";
-          ethertype = "IPv4";
-          protocol = "tcp";
-          port = 80;
-          remote_subnet = "0.0.0.0/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv6";
-          protocol = "tcp";
-          port = 80;
-          remote_subnet = "::/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv4";
-          protocol = "tcp";
-          port = 443;
-          remote_subnet = "0.0.0.0/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv6";
-          protocol = "tcp";
-          port = 443;
-          remote_subnet = "::/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv4";
-          protocol = "udp";
-          port = 443;
-          remote_subnet = "0.0.0.0/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv6";
-          protocol = "udp";
-          port = 443;
-          remote_subnet = "::/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv4";
-          protocol = "icmp";
-          remote_subnet = "0.0.0.0/0";
-        }
-        {
-          direction = "egress";
-          ethertype = "IPv6";
-          protocol = "icmp";
-          remote_subnet = "::/0";
-        }
-      ];
     };
   };
   config = lib.mkIf cfg.enable (
@@ -236,6 +181,9 @@ in
           enable = true;
         };
       };
+      # khs openstack hosted servers are cannot use http-01 challenges (or maybe they can through ipv6?)
+      # so enable dns-01.
+      khscodes.security.acme.dns01Enabled = true;
       khscodes.infrastructure.provisioning = {
         pre = {
           modules = modules;

nix/modules/nixos/infrastructure/vault-server-approle/default.nix

@@ -53,6 +53,7 @@ in
 
   config = lib.mkIf cfg.enable {
     khscodes.services.openstack-read-vault-auth-from-userdata.enable = true;
+    khscodes.services.vault-agent.enable = true;
     khscodes.infrastructure.provisioning.${cfg.stage} = {
       endpoints = [ "vault" ];
       modules = [
@@ -79,10 +80,10 @@ in
                 # on the role being created first, which is needed.
                 role_name = config.khscodes.vault.output.approle_auth_backend_role.${cfg.role_name}.role_name;
                 # Should only be 5-10 mins once done testing
-                wrapping_ttl = 5 * 60 * 60;
+                wrapping_ttl = 5 * 60;
 
-                # All of this simply tries to ensure that I never recreate this secret id
+                # This should simply mean that we never attempt to recreate the secret id, as we don't want a rerun of the
-                # even if the original wrapped secret id is expired (which I expect it to be).
+                # provisioning to invalidate the existing secret id, nor recreate the entire server.
                 with_wrapped_accessor = true;
                 lifecycle = {
                   ignore_changes = [

nix/modules/nixos/openstack/default.nix

@@ -22,6 +22,8 @@ in
         diskName = cfg.diskName;
       }
     );
+    # When this is set as the default, outbound ipv6 doesn't work on the instance.
+    networking.tempAddresses = "disabled";
     boot.loader.grub.efiSupport = false;
     boot.loader.timeout = 1;
     khscodes.virtualisation.qemu-guest.enable = true;

nix/modules/nixos/security/acme/default.nix

@@ -4,7 +4,7 @@ let
   vaultAgentCredentialsFile = "/var/lib/vault-agent/acme/cloudflare-api-token";
   cloudflareSecret = "opentofu/data/cloudflare";
   acmeServicesToRestart = lib.lists.map (a: "acme-${a}.service") (
-    lib.attrsets.attrNames config.security.certs
+    lib.attrsets.attrNames config.security.acme.certs
   );
 in
 {
@@ -25,21 +25,19 @@ in
         }
         // lib.attrsets.optionalAttrs cfg.dns01Enabled {
           dnsProvider = "cloudflare";
-          dnsResolver = "1.1.1.1:53";
+          dnsResolver = null;
           credentialsFile = vaultAgentCredentialsFile;
         };
     };
     khscodes.infrastructure.vault-server-approle = {
       enable = true;
-      policy = [
+      policy = {
-        {
+        "${cloudflareSecret}" = {
-          "${cloudflareSecret}" = {
+          capabilities = [ "read" ];
-            capabilities = [ "read" ];
+        };
-          };
+      };
-        }
-      ];
     };
-    khscodes.services.vault-agent = (cfg.dns01Enabled && acmeServicesToRestart != [ ]) {
+    khscodes.services.vault-agent = lib.mkIf (cfg.dns01Enabled && acmeServicesToRestart != [ ]) {
       enable = true;
       templates = [
         {

nix/modules/nixos/services/nginx/default.nix

@@ -2,19 +2,66 @@
   config,
   lib,
   pkgs,
+  modulesPath,
   ...
 }:
 let
   cfg = config.khscodes.services.nginx;
+  locationOptions = import "${modulesPath}/services/web-servers/nginx/location-options.nix" {
+    inherit lib config;
+  };
   vhostOption = lib.khscodes.mkSubmodule {
     description = "nginx vhost";
     options = {
-      useACMEHost = lib.mkOption {
+      acme = lib.mkOption {
+        description = "If a simple certificate for the virtual host name itself is not desired auto configured, then set this option. If set to a string it will be used as `useAcmeHost` from NixOS nginx service configuration. Otherwise set to the acme submodule and configure the desired certificate that way";
+        type = lib.types.nullOr (
+          lib.types.oneOf [
+            lib.types.str
+            (lib.types.mkSubmodule {
+              description = "acme certificate";
+              options = {
+                domains = lib.mkOption {
+                  type = lib.types.listOf lib.types.str;
+                  description = "Domain names the certificate should be requested for, should include the virtual host itself";
+                };
+              };
+            })
+          ]
+        );
+        default = null;
+      };
+      globalRedirect = lib.mkOption {
         type = lib.types.nullOr lib.types.str;
-        description = "Makes the virtual host use the certificate of another acme host";
+        default = null;
+        description = "If set, all requests for this host are redirected (defaults to 301, configurable with redirectCode) to the given hostname.";
+      };
+      redirectCode = lib.mkOption {
+        type = lib.types.int;
+        default = 301;
+        description = "HTTP status used by globalRedirect and forceSSL. Possible usecases include temporary (302, 307) redirects, keeping the request method and body (307, 308), or explicitly resetting the method to GET (303). See https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections.";
+      };
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra configuration to inject into the generated nginx config";
+        default = '''';
+      };
+      locations = lib.mkOption {
+        type = lib.types.attrsOf (
+          lib.khscodes.mkSubmodule {
+            description = "nginx virtual host location";
+            options = locationOptions;
+          }
+        );
+        default = { };
       };
     };
   };
+  dns01Enabled = config.khscodes.security.acme.dns01Enabled;
+  useAcmeConfiguration = lib.attrsets.foldlAttrs (
+    acc: name: item:
+    acc || (item.acme != null && !lib.attrsets.isAttrs item.acme)
+  ) false cfg.virtualHosts;
 in
 {
   options.khscodes.services.nginx = {
@@ -26,7 +73,15 @@ in
     };
   };
   config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = !useAcmeConfiguration || dns01Enabled;
+        message = "Cannot use `config.khscodes.services.nginx.virtualHosts.<name>.acme = {}` without setting config.khscodes.security.acme.dns01Enabled";
+      }
+    ];
     khscodes.security.acme.enable = true;
+    security.dhparams.enable = true;
+    security.dhparams.params."nginx".bits = 4096;
     services.nginx = {
       enable = true;
       package = lib.mkDefault pkgs.nginxStable;
@@ -36,6 +91,51 @@ in
       recommendedOptimisation = lib.mkDefault true;
       recommendedZstdSettings = lib.mkDefault true;
       recommendedProxySettings = lib.mkDefault true;
+      virtualHosts = lib.attrsets.mapAttrs (name: value: {
+        inherit (value)
+          extraConfig
+          locations
+          globalRedirect
+          redirectCode
+          ;
+        forceSSL = true;
+        enableACME = value.acme == null && !dns01Enabled;
+        useACMEHost =
+          if lib.strings.isString value.acme then
+            value.acme
+          else if lib.attrsets.isAttrs value.acme || dns01Enabled then
+            name
+          else
+            null;
+      }) cfg.virtualHosts;
     };
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+    networking.firewall.allowedUDPPorts = [ 443 ];
+    users.users.nginx.extraGroups = lib.lists.optional dns01Enabled "acme";
+    security.acme.certs = lib.mkIf dns01Enabled (
+      lib.attrsets.foldlAttrs (
+        acc: name: value:
+        (
+          acc
+          // (lib.attrsets.optionalAttrs (lib.attrsets.isAttrs value.acme || dns01Enabled) {
+            "${name}" =
+              if value.acme == null then
+                {
+                  domain = name;
+                  reloadServices = [ "nginx" ];
+                }
+              else
+                {
+                  domain = lib.lists.head value.acme.domains;
+                  extraDomainNames = lib.lists.tail value.acme.domains;
+                  reloadServices = [ "nginx" ];
+                };
+          })
+        )
+      ) { } cfg.virtualHosts
+    );
   };
 }

nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix

@@ -19,7 +19,6 @@ in
       roleIdFilePath = config.khscodes.services.vault-agent.vault.roleIdFilePath;
     in
     {
-      services.khscodes.vault-agent.enable = true;
       systemd.services."openstack-read-vault-auth-from-userdata" = {
         enable = true;
         wantedBy = [ "multi-user.target" ];

nix/packages/create-instance/default.nix

@@ -1,9 +1,17 @@
-{ pkgs, ... }:
+{ pkgs, inputs, ... }:
 pkgs.writeShellApplication {
   name = "create-instance";
-  runtimeInputs = [ pkgs.khscodes.pre-provisioning ];
+  runtimeInputs = [
+    pkgs.khscodes.provision-instance
+    pkgs.khscodes.nixos-install
+    pkgs.jq
+  ];
   text = ''
-    instance="''${1:-}"
+    hostname="$1"
-    pre-provisioning "$instance" apply
+    # Build the configuration to ensure it doesn't fail when trying to install it on the host
+    nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel'
+    output="$(provision-instance "$hostname")"
+    ipv4_addr="$(echo "$output" | jq --raw-output '.ipv4_address.value')"
+    nixos-install "$hostname" "$ipv4_addr" "no"
   '';
 }

nix/packages/instance-opentofu/default.nix

@@ -17,6 +17,11 @@ pkgs.writeShellApplication {
     cat "''${config}" > "$dir/config.tf.json"
 
     tofu -chdir="$dir" init > /dev/null
-    tofu -chdir="$dir" "$cmd"
+    if [[ "$cmd" == "apply" ]]; then
+      tofu -chdir="$dir" "$cmd" >&2
+      tofu -chdir="$dir" output -json
+    else
+      tofu -chdir="$dir" "$cmd"
+    fi
   '';
 }

nix/packages/nixos-install/default.nix

@@ -11,10 +11,13 @@ pkgs.writeShellApplication {
   # TODO: Use secret source and required secrets to set up the correct env variables
   text = ''
     hostname="$1"
-    # Build the configuration to ensure it doesn't fail when trying to install it on the host
-    nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel'
     # Allow overriding the host to connec tto, this is useful when testing and the DNS entries are stale with older IPs.
     host="''${2:-$1}"
+    verify="''${3:-yes}"
+    if [[ "$verify" == "yes" ]]; then
+      # Build the configuration to ensure it doesn't fail when trying to install it on the host
+      nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel'
+    fi
     baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning'
     config="$(nix build --no-link --print-out-paths "''${baseAttr}.preConfig")"
     username="$(nix eval --raw "''${baseAttr}.preImageUsername")"

nix/packages/provision-instance/default.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+pkgs.writeShellApplication {
+  name = "provision-instance";
+  runtimeInputs = [ pkgs.khscodes.pre-provisioning ];
+  text = ''
+    instance="''${1:-}"
+    pre-provisioning "$instance" apply
+  '';
+}

nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix

@@ -4,9 +4,18 @@
 }:
 {
   imports = [ "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix" ];
-  khscodes.infrastructure.khs-openstack-instance = {
+  khscodes = {
-    enable = true;
+    infrastructure.khs-openstack-instance = {
-    flavor = "m.medium";
+      enable = true;
+      flavor = "m.medium";
+    };
+    services.nginx = {
+      enable = true;
+      virtualHosts."test.kaareskovgaard.net" = {
+        globalRedirect = "khs.codes";
+        redirectCode = 302;
+      };
+    };
   };
   snowfallorg.users.khs.admin = true;
   users.users.khs = {

rust/program/openbao-helper/src/main.rs

@@ -232,13 +232,22 @@ unsafe fn execvpe<SA: AsRef<CStr>, SEK: AsRef<OsStr>, SEV: AsRef<OsStr>>(
     args: &[SA],
     environ: &[(SEK, SEV)],
 ) -> anyhow::Result<Infallible> {
-    let environ: Vec<_> = environ
+    let environ = environ
         .iter()
         .map(|(k, v)| {
-            CString::new(Format!("{k}={v}"))
+            CString::new(format!(
-                .with_context(|| format!("Environment variable {k} contains null bytes"))?
+                "{k}={v}",
+                k = k.as_ref().display(),
+                v = v.as_ref().display()
+            ))
+            .with_context(|| {
+                format!(
+                    "Environment variable {k} contains null bytes",
+                    k = k.as_ref().display()
+                )
+            })
         })
-        .collect();
+        .collect::<anyhow::Result<Vec<CString>>>()?;
     Ok(nix::unistd::execvpe(filename, args, &environ)?)
 }