nix/machines
1
0
Fork 0
Code Issues Releases Packages Activity Actions

Begin testing bootstrapping of vault authentication

Browse source 
However, the nixos-install script fails on khs openstack
as the system won't boot up after installation due it
being unable to locate the root disk. I am not sure what disk
it ends up finding.
This commit is contained in:
Kaare Hoff Skovgaard 2025-07-09 23:53:42 +02:00
parent f7d4bef46c
commit 608d758f30
Signed by: khs
GPG key ID: C7D890804F01E9F0
32 changed files with 705 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/modules/nixos/infrastructure/hetzner-instance/default.nix
View file

@ -7,6 +7,7 @@
let
cfg = config.khscodes.infrastructure.hetzner-instance;
fqdn = config.khscodes.networking.fqdn;
provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData;
firewallTcpRules = lib.lists.map (p: {
direction = "in";
protocol = "tcp";
@ -160,6 +161,7 @@ in
initial_image = "debian-12";
rdns = fqdn;
ssh_keys = [ config.khscodes.hcloud.output.data.ssh_key.khs.id ];
user_data = provisioningUserData;
};
khscodes.cloudflare = {
enable = true;

18
nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix
View file

@ -7,6 +7,7 @@
let
cfg = config.khscodes.infrastructure.khs-openstack-instance;
fqdn = config.khscodes.networking.fqdn;
provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData;
firewallTcpRules = lib.lists.flatten (
lib.lists.map (p: [
{
@ -83,14 +84,6 @@ in
default = "${fqdn}.tfstate";
};
};
secretsSource = lib.mkOption {
type = lib.types.enum [
"bitwarden"
"vault"
];
description = "Whether to load opentofu secrets from Bitwarden or Vault";
default = "vault";
};
flavor = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "The server type to create";
@ -188,6 +181,7 @@ in
flavor = cfg.flavor;
ssh_public_key = cfg.ssh_key;
firewall_rules = firewallRules;
user_data = provisioningUserData;
};
khscodes.unifi.enable = true;
khscodes.unifi.static_route.compute = {
@ -236,11 +230,15 @@ in
message = "Must set config.khscodes.networking.fqdn when using opentofu";
}
];
khscodes.services.openssh = {
enable = true;
hostCertificate = {
enable = true;
};
};
khscodes.infrastructure.provisioning = {
pre = {
modules = modules;
secretsSource = cfg.secretsSource;
endpoints = [
"aws"
"cloudflare"

7
nix/modules/nixos/infrastructure/provisioning/default.nix
View file

@ -29,6 +29,8 @@ let
"unifi"
"hcloud"
"cloudflare"
"vault"
"authentik"
]
);
description = "Needed endpoints to be used during provisioning";
@ -40,6 +42,11 @@ in
options.khscodes.infrastructure.provisioning = {
pre = provisioning;
post = provisioning;
instanceUserData = lib.mkOption {
type = lib.types.str;
description = "User data that should be added to the instance during provisioning";
default = "";
};
preConfig = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "The generated config for the pre provisioning, if any was specified";

120
nix/modules/nixos/infrastructure/vault-server-approle/default.nix Normal file
View file

@ -0,0 +1,120 @@
{
config,
lib,
inputs,
...
}:
let
cfg = config.khscodes.infrastructure.vault-server-approle;
in
{
options.khscodes.infrastructure.vault-server-approle = {
enable = lib.mkEnableOption "Enables creating an OpenBAO role for the server";
stage = lib.mkOption {
type = lib.types.enum [
"pre"
"post"
];
description = "The provisioning stage that should include the provisioning. This should be pre for every server except the OpenBAO server itself";
default = "pre";
};
role_name = lib.mkOption {
type = lib.types.str;
description = "Name of the role being created";
default = config.networking.fqdnOrHostName;
};
policy = lib.mkOption {
type = lib.types.attrsOf (
lib.khscodes.mkSubmodule {
options = {
capabilities = lib.mkOption {
type = lib.types.listOf (
lib.types.enum [
"create"
"update"
"patch"
"read"
"delete"
"list"
]
);
};
};
description = "Vault role policy";
}
);
};
stageModules = lib.mkOption {
type = lib.types.listOf lib.types.anything;
description = "Extra modules to add to the configured stage";
default = [ ];
};
};
config = lib.mkIf cfg.enable {
khscodes.infrastructure.provisioning.${cfg.stage} = {
endpoints = [ "vault" ];
modules = [
(
{ config, ... }:
{
imports = [ inputs.self.terranixModules.vault ];
khscodes.vault = {
enable = true;
approle_auth_backend_role.${cfg.role_name} = {
backend = "approle";
role_name = cfg.role_name;
# I keep the secret ids alive for quite long, as I have no way of
# automatically bootstrapping a new secret id.
secret_id_ttl = 5 * 60 * 60;
secret_id_num_uses = 5 * 60;
token_ttl = 20 * 60;
token_max_ttl = 30 * 60;
token_policies = [ cfg.role_name ];
};
approle_auth_backend_role_secret_id.${cfg.role_name} = {
backend = "approle";
# Not hardcoding the role name here, as reading it like this will create a dependency
# on the role being created first, which is needed.
role_name = config.khscodes.vault.output.approle_auth_backend_role.${cfg.role_name}.role_name;
# Should only be 5-10 mins once done testing
wrapping_ttl = 5 * 60 * 60;
# All of this simply tries to ensure that I never recreate this secret id
# even if the original wrapped secret id is expired (which I expect it to be).
with_wrapped_accessor = true;
lifecycle = {
ignore_changes = [
"num_uses"
"ttl"
];
};
};
policy.${cfg.role_name} = {
name = cfg.role_name;
policy = lib.strings.concatStringsSep "\n\n" (
lib.lists.map (
{ name, value }:
''
path "${name}" {
capabilities = ${builtins.toJSON value.capabilities}
}
''
) (lib.attrsToList cfg.policy)
);
};
};
}
)
] ++ cfg.stageModules;
};
# I can only provide the user data if the stage is pre (along with the instance creation)
# Also I should probably find a way of injecting this in a nicer way than this mess.
khscodes.infrastructure.provisioning.instanceUserData = lib.mkIf (cfg.stage == "pre") ''
{
"VAULT_ROLE_ID": "''${ vault_approle_auth_backend_role.${lib.khscodes.sanitize-terraform-name cfg.role_name}.role_id }",
"VAULT_SECRET_ID_WRAPPED": "''${ vault_approle_auth_backend_role_secret_id.${lib.khscodes.sanitize-terraform-name cfg.role_name}.wrapping_token }"
}
'';
};
}

1
nix/modules/nixos/networking/fqdn/default.nix
View file

@ -21,6 +21,7 @@ in
{
networking.hostName = lib.mkForce hostname;
networking.domain = lib.mkForce domain;
networking.fqdn = cfg;
# Add the name of the server to the ssh host certificate domains, but let other configs enable getting the host certificates.
khscodes.services.openssh.hostCertificate.hostNames = [ cfg ];
boot.kernel.sysctl = {

12
nix/modules/nixos/openstack/default.nix
View file

@ -16,12 +16,14 @@ in
};
};
config = lib.mkIf cfg.enable {
disko = lib.khscodes.disko-root-lvm-bios {
device = "/dev/sda";
diskName = cfg.diskName;
};
disko = lib.mkDefault (
lib.khscodes.disko-root-bios {
device = "/dev/sda";
diskName = cfg.diskName;
}
);
boot.loader.grub.efiSupport = false;
boot.loader.timeout = 1;
khscodes.virtualisation.qemu-guest.enable = true;
# khscodes.virtualisation.qemu-guest.enable = true;
};
}

38
nix/modules/nixos/services/openssh/default.nix
View file

@ -7,11 +7,6 @@ in
enable = lib.mkEnableOption "Enables openssh service for the instance";
hostCertificate = {
enable = lib.mkEnableOption "Enables getting host certificates from OpenBAO";
secretName = lib.mkOption {
type = lib.types.str;
description = "Secret where the certificate is stored";
example = "ssh-host/sign/ca-kaareskovgaard.net";
};
hostNames = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "The list of host names to get certificates for";
@ -24,6 +19,9 @@ in
let
certificateNames = lib.lists.unique cfg.hostCertificate.hostNames;
hostCertificatEnable = cfg.hostCertificate.enable && cfg.hostCertificate.hostNames != [ ];
vaultRoleName = config.khscodes.infrastructure.vault-server-approle.role_name;
fqdn = config.networking.fqdnOrHostName;
sshHostBackend = "ssh-host";
in
{
services.openssh = {
@ -37,6 +35,34 @@ in
HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub
'';
};
khscodes.infrastructure.vault-server-approle = {
enable = true;
policy."${sshHostBackend}/sign/${vaultRoleName}" = {
capabilities = [
"read"
"update"
"create"
];
};
stageModules = [
{
khscodes.vault.ssh_secret_backend_role.${vaultRoleName} = {
name = fqdn;
backend = sshHostBackend;
key_type = "ca";
allow_host_certificates = true;
allow_bare_domains = true;
allowed_domains = certificateNames;
allowed_user_key_config = [
{
type = "ed25519";
lengths = [ 0 ];
}
];
};
}
];
};
khscodes.services.vault-agent = lib.mkIf hostCertificatEnable {
enable = true;
templates = [
@ -44,7 +70,7 @@ in
contents = ''
{{- $public_key := file "/etc/ssh/ssh_host_ed25519_key.pub" -}}
{{- $public_key = printf "public_key=%s" $public_key -}}
{{- with secret "${cfg.hostCertificate.secretName}" "cert_type=host" $public_key "valid_principals=${lib.strings.concatStringsSep "," certificateNames}" -}}
{{- with secret "ssh-host/sign/${fqdn}" "cert_type=host" $public_key "valid_principals=${lib.strings.concatStringsSep "," certificateNames}" -}}
{{ .Data.signed_key }}
{{- end -}}
'';

63
nix/modules/nixos/services/openstack-read-vault-auth-from-userdata/default.nix Normal file
View file

@ -0,0 +1,63 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.khscodes.services.openstack-read-vault-auth-from-userdata;
in
{
options.khscodes.services.openstack-read-vault-auth-from-userdata = {
enable = lib.mkEnableOption "Enables reading vault auth information from instance userdata";
};
config = lib.mkIf (cfg.enable && config.khscodes.services.vault-agent.enable) (
let
vault_addr = config.khscodes.services.vault-agent.vault.address;
secretIdFilePath = config.khscodes.services.vault-agent.secretIdFilePath;
roleIdFilePath = config.khscodes.services.vault-agent.roleIdFilePath;
in
{
systemd.services."openstack-read-vault-auth-from-userdata" = {
enable = true;
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = lib.getExe (
pkgs.writeShellApplication {
name = "openstack-read-vault-auth-from-userdata";
runtimeInputs = [
pkgs.curl
pkgs.jq
pkgs.openbao
];
text = ''
if [[ -f "${lib.escapeShellArg secretIdFilePath}" ]]; then
echo "Secret id already found, not copying new id"
exit 0
fi
userdata="$(curl http://169.254.169.254/openstack/2012-08-10/user_data)"
role_id="$(echo "$userdata" | jq --raw '.VAULT_ROLE_ID')"
secret_id_wrapped="$(echo "$userdata" | jq --raw '.VAULT_SECRET_ID_WRAPPED')"
secret_id="$(BAO_ADDR=${lib.escapeShellArg vault_addr} bao unwrap -field=secret_id "$secret_id_wrapped")"
mkdir -p "$(dirname ${lib.escapeShellArg secretIdFilePath})"
mkdir -p "$(dirname ${lib.escapeShellArg roleIdFilePath})"
echo -n "$role_id" > ${lib.escapeShellArg roleIdFilePath}
echo -n "$secret_id" > ${lib.escapeShellArg secretIdFilePath}
chown root:root "$${lib.escapeShellArg secretIdFilePath}"
chmod 0600 "$${lib.escapeShellArg secretIdFilePath}"
chown root:root "$${lib.escapeShellArg roleIdFilePath}"
chmod 0600 "$${lib.escapeShellArg roleIdFilePath}"
echo "Role id and secret id copied, restart vault-agent"
systemctl restart vault-agent-openbao.service
'';
}
);
};
};
}
);
}

64
nix/modules/nixos/services/vault-agent/default.nix
View file

@ -5,9 +5,7 @@
...
}:
let
cfg = config.input-output.openbao.agent;
secretIdFilePath = "/var/lib/vault-agent/secret-id";
roleIdFilePath = "/var/lib/vault-agent/role-id";
cfg = config.khscodes.services.vault-agent;
mkSubmodule =
{
options,
@ -62,8 +60,8 @@ let
type = "approle";
config = {
mount_path = "auth/approle";
role_id_file_path = roleIdFilePath;
secret_id_file_path = secretIdFilePath;
role_id_file_path = cfg.vault.roleIdFilePath;
secret_id_file_path = cfg.vault.secretIdFilePath;
remove_secret_id_file_after_reading = false;
};
}
@ -90,10 +88,22 @@ in
default = pkgs.openbao;
defaultText = "pkgs.openbao";
};
vault.address = lib.mkOption {
type = lib.types.str;
description = "Address of the Vault/OpenBAO service";
default = "https://vault.kaareskovgaard.net";
vault = {
address = lib.mkOption {
type = lib.types.str;
description = "Address of the Vault/OpenBAO service";
default = "https://vault.kaareskovgaard.net";
};
roleIdFilePath = lib.mkOption {
type = lib.types.str;
description = "Location of the role id";
default = "/var/lib/vault-agent/role-id";
};
secretIdFilePath = lib.mkOption {
type = lib.types.str;
description = "Location of the secret id";
default = "/var/lib/vault-agent/secret-id";
};
};
templates = lib.mkOption {
default = [ ];
@ -146,42 +156,10 @@ in
wantedBy = unitsDependsOnAgent;
unitConfig = {
ConditionPathExists = [
secretIdFilePath
roleIdFilePath
cfg.vault.secretIdFilePath
cfg.vault.roleIdFilePath
];
};
};
environment.systemPackages = [
(pkgs.writeShellApplication {
name = "vault-agent-load-credentials";
meta = {
mainProgram = "vault-agent-load-credentials";
};
runtimeInputs = [
pkgs.systemd
pkgs.openbao
];
text = ''
if [[ -z "''${1:-}" || -z "''${2:-}" ]]; then
>&2 echo "Usage: vault-agent-load-credentials <role-id> <secret-id-wrapped>"
exit 1
fi
role_id="$1"
secret_id_wrapped="$2"
secret_id="$(BAO_ADDR=${lib.escapeShellArg cfg.vault.address} bao unwrap -field=secret_id "$secret_id_wrapped")"
mkdir -p "$(dirname ${lib.escapeShellArg secretIdFilePath})"
mkdir -p "$(dirname ${lib.escapeShellArg roleIdFilePath})"
echo -n "$role_id" > ${lib.escapeShellArg roleIdFilePath}
echo -n "$secret_id" > ${lib.escapeShellArg secretIdFilePath}
chown root:root "$${lib.escapeShellArg secretIdFilePath}"
chmod 0600 "$${lib.escapeShellArg secretIdFilePath}"
chown root:root "$${lib.escapeShellArg roleIdFilePath}"
chmod 0600 "$${lib.escapeShellArg roleIdFilePath}"
systemctl restart vault-agent-openbao.service
${restartUnits unitsDependsOnAgent}
${reloadOrRestartUnits unitsDependsOnAgent}
'';
})
];
};
}

24
nix/modules/nixos/virtualisation/qemu-guest/default.nix
View file

@ -1,7 +1,6 @@
{
config,
lib,
modulesPath,
...
}:
let
@ -24,22 +23,21 @@ in
enableWhenVmTarget = lib.mkEnableOption "Enables some enhancement settings when building as a vm";
};
imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ];
config = lib.mkIf cfg.enableWhenVmTarget {
virtualisation = {
vmVariant = {
services.qemuGuest.enable = true;
services.spice-vdagentd.enable = true;
khscodes.virtualisation.qemu-guest.enable = true;
};
memorySize = 1024 * 8;
qemu = {
options = [
"-smp 8"
"-vga none -device virtio-gpu-gl,hostmem=2G,blob=true,venus=true"
rng
] ++ spice;
services.spice-vdagentd.enable = true;
virtualisation = {
memorySize = 1024 * 8;
qemu = {
options = [
"-smp 8"
"-vga none -device virtio-gpu-gl,hostmem=2G,blob=true,venus=true"
rng
] ++ spice;
};
};
};
};
};

7
nix/modules/nixos/virtualisation/qemu-guest/profile.nix
View file

@ -8,5 +8,10 @@ let
cfg = config.khscodes.virtualisation.qemu-guest;
in
{
config = lib.mkIf cfg.enable (import "${modulesPath}/profiles/qemu-guest.nix" { });
config = lib.mkIf cfg.enable (
(import "${modulesPath}/profiles/qemu-guest.nix" { })
// {
services.qemuGuest.enable = true;
}
);
}