From a6c4012f7993e37cfa2ce4c7d033a511f7faaeca Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 9 May 2025 10:20:43 +0200 Subject: [PATCH 1/4] tests: extract potentially common test code into lib.nix --- nixos/lib.nix | 169 +++++++++++++++++ nixos/vault-test.nix | 423 +++++++++++++++---------------------------- 2 files changed, 319 insertions(+), 273 deletions(-) create mode 100644 nixos/lib.nix diff --git a/nixos/lib.nix b/nixos/lib.nix new file mode 100644 index 0000000..dbce32f --- /dev/null +++ b/nixos/lib.nix @@ -0,0 +1,169 @@ +{ pkgs, lib }: +let + nixos-lib = import (pkgs.path + "/nixos/lib") { }; + acme-test-module = "${pkgs.path}/nixos/tests/common/acme/server"; + + domain = "faythe.test"; + ns_host = "ns.${domain}"; + + # Pebble in version > 2.3.1 (NixOS 24.11 and up) is ramping up towards ACME + # profiles and not issuing CNs for tlsserver profile certs. We want to test + # against behaviour that matches the current letsencrypt behaviour, so stick + # to 2.3.1. + pebble-cn-overlay = self: super: { + pebble = super.pebble.overrideAttrs (oa: rec { + version = "2.3.1"; + src = self.fetchFromGitHub { + owner = "letsencrypt"; + repo = "pebble"; + rev = "v${version}"; + hash = "sha256-S9+iRaTSRt4F6yMKK0OJO6Zto9p0dZ3q/mULaipudVo="; + }; + }); + }; +in +{ + inherit domain ns_host; + mkFaytheTest = faytheTest: + nixos-lib.runTest ( + test@{ nodes, ... }: + let + args = faytheTest test; + optionalExtraModules = name: + (args.extraModules or {}).${name} or []; + in + { + hostPkgs = pkgs; + name = args.name; + defaults = { + nixpkgs.overlays = [ pebble-cn-overlay ]; + nixpkgs.pkgs = pkgs; + networking.nameservers = lib.mkForce [ nodes.ns.networking.primaryIPAddress ]; + networking.dhcpcd.enable = false; + security.pki.certificateFiles = [ nodes.acme.test-support.acme.caCert ]; + networking.hosts."${nodes.acme.networking.primaryIPAddress}" = [ nodes.acme.test-support.acme.caDomain ]; + }; + nodes = { + acme = + { pkgs, ... }: + { + imports = [ acme-test-module ] ++ (optionalExtraModules "acme"); + }; + + ns = + { pkgs, ... }: + { + imports = optionalExtraModules "ns"; + + environment.systemPackages = with pkgs; [ + dig + dnsutils + ]; + + environment.etc."bind/zones/${domain}.zone" = { + mode = "0644"; + user = "named"; + group = "named"; + text = '' + $TTL 60 + ${domain}. IN SOA ${ns_host}. admin.${domain}. ( 1 3h 1h 1w 1d ) + + @ IN NS ${ns_host}. + + ${ns_host}. IN A ${nodes.ns.networking.primaryIPAddress} + '' + args.extraBindZoneFileLines; + }; + + networking.firewall.allowedTCPPorts = [ 53 ]; + networking.firewall.allowedUDPPorts = [ 53 ]; + + services.bind.enable = true; + + services.bind.zones."${domain}" = { + master = true; + file = "/etc/bind/zones/${domain}.zone"; + # the bind zone module is very opinionated and this sets allow-transfer. + slaves = [ nodes.client.networking.primaryIPAddress ]; + extraConfig = '' + allow-update { ${nodes.client.networking.primaryIPAddress}; }; + ''; + }; + + # Hack to allow access to the directory copied from environment.etc + systemd.services.bind.serviceConfig.ExecStartPre = "+${pkgs.coreutils}/bin/chown named /etc/bind/zones"; + }; + + client = + { pkgs, config, ... }: + let + faytheConfig = { + lets_encrypt_url = "https://${nodes.acme.test-support.acme.caDomain}/dir"; + lets_encrypt_email = "test_mail@${domain}"; + zones = { + "${domain}" = { + auth_dns_server = ns_host; + challenge_driver.nsupdate ={ + server = ns_host; + key = "test"; + }; + }; + }; + val_dns_servers = [ ns_host ]; + } // args.faytheExtraConfig; + + faytheConfigFile = pkgs.writeText "faythe.config.json" (builtins.toJSON faytheConfig); + + faytheConfigFileChecked = pkgs.runCommand "faythe.config.checked.json" { } '' + ${pkgs.faythe}/bin/faythe --config-check ${faytheConfigFile} + ln -s ${faytheConfigFile} $out + ''; + in + { + imports = optionalExtraModules "client"; + + environment.systemPackages = with pkgs; [ + dig + dnsutils + getent + lsof + ]; + + systemd.services.faythe = { + path = with pkgs; [ + dnsutils + dig + ]; + environment.RUST_BACKTRACE = "full"; + environment.RUST_LOG = "warn,acme_lib=debug"; + wantedBy = [ "multi-user.target" ]; + preStart = '' + # vault provisioning time was masking this, but we need to + # wait for system nameserver to be up before we can start faythe + while ! dig +short -t SOA ${domain}; do + echo "Waiting for nameserver to be up" + sleep 1 + done + ''; + serviceConfig = { + ExecStart = "${pkgs.faythe}/bin/faythe ${faytheConfigFileChecked}"; + }; + }; + }; + }; + testScript = '' + start_all() + + ns.wait_for_unit("network-online.target") + acme.wait_for_unit("network-online.target") + client.wait_for_unit("network-online.target") + + ns.wait_for_unit("bind.service") + + client.wait_until_succeeds("ping -c1 ${nodes.ns.networking.primaryIPAddress}") + client.fail("host doesnotexist.${domain}") + + client.wait_for_unit("faythe.service") + '' + args.testScript; + } + ); + } diff --git a/nixos/vault-test.nix b/nixos/vault-test.nix index cc1bd59..eb4d6ec 100644 --- a/nixos/vault-test.nix +++ b/nixos/vault-test.nix @@ -1,292 +1,169 @@ { lib, pkgs }: let - nixos-lib = import (pkgs.path + "/nixos/lib") { }; - acme-test-module = "${pkgs.path}/nixos/tests/common/acme/server"; + testLib = import ./lib.nix { + inherit lib pkgs; + }; role_id_path = "/tmp/vault-role-id"; secret_id_path = "/tmp/vault-secret-id"; - domain = "faythe.test"; + domain = testLib.domain; vault_host = "vault.${domain}"; - ns_host = "ns.${domain}"; + ns_host = testLib.ns_host; # dev server vault_addr = "http://localhost:8200"; - - # Pebble in version > 2.3.1 (NixOS 24.11 and up) is ramping up towards ACME - # profiles and not issuing CNs for tlsserver profile certs. We want to test - # against behaviour that matches the current letsencrypt behaviour, so stick - # to 2.3.1. - pebble-cn-overlay = self: super: { - pebble = super.pebble.overrideAttrs (oa: rec { - version = "2.3.1"; - src = self.fetchFromGitHub { - owner = "letsencrypt"; - repo = "pebble"; - rev = "v${version}"; - hash = "sha256-S9+iRaTSRt4F6yMKK0OJO6Zto9p0dZ3q/mULaipudVo="; - }; - }); - }; - in -nixos-lib.runTest ( - test@{ nodes, ... }: - { - hostPkgs = pkgs; - name = "faythe-vault-test"; - defaults = { - nixpkgs.overlays = [ pebble-cn-overlay ]; - nixpkgs.pkgs = pkgs; - networking.nameservers = lib.mkForce [ nodes.ns.networking.primaryIPAddress ]; - networking.dhcpcd.enable = false; - security.pki.certificateFiles = [ nodes.acme.test-support.acme.caCert ]; - networking.hosts."${nodes.acme.networking.primaryIPAddress}" = [ nodes.acme.test-support.acme.caDomain ]; - }; - nodes = { - acme = - { pkgs, ... }: - { - imports = [ acme-test-module ]; - }; - - ns = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - dig - dnsutils - ]; - - networking.firewall.allowedTCPPorts = [ 53 ]; - networking.firewall.allowedUDPPorts = [ 53 ]; - - services.bind.enable = true; - - services.bind.zones."${domain}" = { - master = true; - file = "/etc/bind/zones/${domain}.zone"; - # the bind zone module is very opinionated and this sets allow-transfer. - slaves = [ nodes.client.networking.primaryIPAddress ]; - extraConfig = '' - allow-update { ${nodes.client.networking.primaryIPAddress}; }; - ''; - }; - - # Hack to allow access to the directory copied from environment.etc - systemd.services.bind.serviceConfig.ExecStartPre = "+${pkgs.coreutils}/bin/chown named /etc/bind/zones"; - - environment.etc."bind/zones/${domain}.zone" = { - mode = "0644"; - user = "named"; - group = "named"; - text = '' - $TTL 60 - ${domain}. IN SOA ${ns_host}. admin.${domain}. ( 1 3h 1h 1w 1d ) - - @ IN NS ${ns_host}. - - ${ns_host}. IN A ${nodes.ns.networking.primaryIPAddress} +testLib.mkFaytheTest ({ nodes, ... }: { + name = "faythe-vault-test"; + extraModules.client = [ + ({ config, pkgs, ... }: { + environment = { + systemPackages = with pkgs; [ + vault + ]; + variables.VAULT_ADDR = vault_addr; + }; - ${vault_host}. IN A ${nodes.client.networking.primaryIPAddress} - ''; - }; - }; + services.vault = { + enable = true; + # start unsealed and with known root token + dev = true; + devRootTokenID = "vaultroot"; + }; - client = - { pkgs, config, ... }: - let + systemd.services.faythe = { + wants = [ "vault-provision.service" ]; + after = [ "vault-provision.service" ]; + }; - faytheConfig = { - vault_monitor_configs = [ - { - inherit role_id_path secret_id_path vault_addr; - key_prefix = "path1"; - specs = [ - { - name = "path1-test"; - cn = "path1.${domain}"; - } - ]; - } - { - inherit role_id_path secret_id_path vault_addr; - key_prefix = "path2"; - specs = [ - { - name = "path2-test"; - cn = "path2.${domain}"; - } - ]; - } - ]; - lets_encrypt_url = "https://${nodes.acme.test-support.acme.caDomain}/dir"; - lets_encrypt_email = "test_mail@${domain}"; - zones = { - "${domain}" = { - auth_dns_server = ns_host; - challenge_driver.nsupdate ={ - server = ns_host; - key = "test"; + # FIXME: upstream this, makes ordering nicer + systemd.services.vault.serviceConfig.Type = "notify"; + + systemd.services.vault-provision = { + path = with pkgs; [ + vault + getent + openssl + ]; + environment.VAULT_ADDR = vault_addr; + wants = [ "vault.service" ]; + after = [ "vault.service" ]; + serviceConfig.Type = "oneshot"; + + script = '' + set -x + set -euo pipefail + + vault login ${config.services.vault.devRootTokenID} + + vault policy write faythe-policy ${ + pkgs.writeText "faythe-policy.json" ( + builtins.toJSON { + path."kv/data/path1/*" = { + capabilities = [ + "list" + "read" + "create" + "update" + ]; + }; + path."kv/metadata/path1/*" = { + capabilities = [ + "list" + "read" + "create" + "update" + ]; + }; + path."kv/data/path2/*" = { + capabilities = [ + "list" + "read" + "create" + "update" + ]; + }; + path."kv/metadata/path2/*" = { + capabilities = [ + "list" + "read" + "create" + "update" + ]; }; - }; - }; - val_dns_servers = [ ns_host ]; - }; - - faytheConfigFile = pkgs.writeText "faythe.config.json" (builtins.toJSON faytheConfig); - - faytheConfigFileChecked = pkgs.runCommand "faythe.config.checked.json" { } '' - ${pkgs.faythe}/bin/faythe --config-check ${faytheConfigFile} - ln -s ${faytheConfigFile} $out - ''; - in - { - environment.systemPackages = with pkgs; [ - dig - dnsutils - vault - getent - lsof - ]; - - environment.variables.VAULT_ADDR = vault_addr; - - services.vault = { - enable = true; - # start unsealed and with known root token - dev = true; - devRootTokenID = "vaultroot"; - }; - - # FIXME: upstream this, makes ordering nicer - systemd.services.vault.serviceConfig.Type = "notify"; - - systemd.services.vault-provision = { - path = with pkgs; [ - vault - getent - openssl - ]; - environment.VAULT_ADDR = vault_addr; - wants = [ "vault.service" ]; - after = [ "vault.service" ]; - serviceConfig.Type = "oneshot"; - - script = '' - set -x - set -euo pipefail - - vault login ${config.services.vault.devRootTokenID} - - vault policy write faythe-policy ${ - pkgs.writeText "faythe-policy.json" ( - builtins.toJSON { - path."kv/data/path1/*" = { - capabilities = [ - "list" - "read" - "create" - "update" - ]; - }; - path."kv/metadata/path1/*" = { - capabilities = [ - "list" - "read" - "create" - "update" - ]; - }; - path."kv/data/path2/*" = { - capabilities = [ - "list" - "read" - "create" - "update" - ]; - }; - path."kv/metadata/path2/*" = { - capabilities = [ - "list" - "read" - "create" - "update" - ]; - }; - } - ) } + ) + } - vault auth enable approle - vault write auth/approle/role/faythe type=service policies=faythe-policy - - vault read -field=role_id auth/approle/role/faythe/role-id > ${role_id_path} - vault write -f -field=secret_id auth/approle/role/faythe/secret-id > ${secret_id_path} - - vault secrets enable -path=kv kv-v2 - ''; - }; - - systemd.services.faythe = { - path = with pkgs; [ - dnsutils - dig - ]; - environment.RUST_BACKTRACE = "full"; - environment.RUST_LOG = "warn,acme_lib=debug"; - wantedBy = [ "multi-user.target" ]; - wants = [ "vault-provision.service" ]; - after = [ "vault-provision.service" ]; - serviceConfig = { - ExecStart = "${pkgs.faythe}/bin/faythe ${faytheConfigFileChecked}"; - }; - }; - }; - }; - testScript = '' - start_all() + vault auth enable approle + vault write auth/approle/role/faythe type=service policies=faythe-policy - ns.wait_for_unit("network-online.target") - acme.wait_for_unit("network-online.target") - client.wait_for_unit("network-online.target") + vault read -field=role_id auth/approle/role/faythe/role-id > ${role_id_path} + vault write -f -field=secret_id auth/approle/role/faythe/secret-id > ${secret_id_path} - ns.wait_for_unit("bind.service") - - client.wait_until_succeeds("ping -c1 ${nodes.ns.networking.primaryIPAddress}") - client.wait_until_succeeds("host ${vault_host}") - client.fail("host doesnotexist.${domain}") - - client.wait_for_unit("faythe.service") - - with subtest("Can get certs"): - client.wait_until_succeeds(""" - vault kv get kv/path1/path1-test/cert && vault kv get kv/path2/path2-test/cert - """) - - with subtest("Wakes up on old meta timestamp"): - client.succeed(""" - date +%s > starttime - vault kv put kv/path1/path1-test/faythe value=2000-01-01T00:00:00.000+00:00 - """) - - client.wait_until_succeeds(""" - journalctl --since "@$(cat starttime)" -u faythe | grep "State for cert: path1.faythe.test" | grep -q "Valid" - journalctl --since "@$(cat starttime)" -u faythe | grep "path1-test" | grep -q "touched" - """) - - client.succeed(""" - date -d "$(vault kv get -field value kv/path1/path1-test/faythe)" +%s > refreshtime - """) - - client.succeed(""" - [ $(cat refreshtime) -gt $(cat starttime) ] - """) - - with subtest("No failed dispatch in vaultrs"): - client.fail(""" - journalctl -u faythe | grep -q "dispatch task is gone: runtime dropped the dispatch task" - """) - ''; - } -) + vault secrets enable -path=kv kv-v2 + ''; + }; + }) + ]; + extraBindZoneFileLines = '' + ${vault_host}. IN A ${nodes.client.networking.primaryIPAddress} + ''; + faytheExtraConfig = { + vault_monitor_configs = [ + { + inherit role_id_path secret_id_path vault_addr; + key_prefix = "path1"; + specs = [ + { + name = "path1-test"; + cn = "path1.${domain}"; + } + ]; + } + { + inherit role_id_path secret_id_path vault_addr; + key_prefix = "path2"; + specs = [ + { + name = "path2-test"; + cn = "path2.${domain}"; + } + ]; + } + ]; + }; + testScript = '' + client.wait_until_succeeds("host ${vault_host}") + + with subtest("Can get certs"): + client.wait_until_succeeds(""" + vault kv get kv/path1/path1-test/cert && vault kv get kv/path2/path2-test/cert + """) + + with subtest("Wakes up on old meta timestamp"): + client.succeed(""" + date +%s > starttime + vault kv put kv/path1/path1-test/faythe value=2000-01-01T00:00:00.000+00:00 + """) + + client.wait_until_succeeds(""" + journalctl --since "@$(cat starttime)" -u faythe | grep "State for cert: path1.faythe.test" | grep -q "Valid" + journalctl --since "@$(cat starttime)" -u faythe | grep "path1-test" | grep -q "touched" + """) + + client.succeed(""" + date -d "$(vault kv get -field value kv/path1/path1-test/faythe)" +%s > refreshtime + """) + + client.succeed(""" + [ $(cat refreshtime) -gt $(cat starttime) ] + """) + + with subtest("No failed dispatch in vaultrs"): + client.fail(""" + journalctl -u faythe | grep -q "dispatch task is gone: runtime dropped the dispatch task" + """) + ''; +}) From 7e5361f2f9d97068ed49ae85ef2510253dfb7f6c Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 9 May 2025 10:26:50 +0200 Subject: [PATCH 2/4] tests: raise number of vcpus for the nodes, currently the vms cannot multitask, and thats sloooow also: add .nixos-test-history to .gitignore --- .gitignore | 2 ++ nixos/lib.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4ab4196..7ae1fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ result* .direnv + +.nixos-test-history diff --git a/nixos/lib.nix b/nixos/lib.nix index dbce32f..e0b89ff 100644 --- a/nixos/lib.nix +++ b/nixos/lib.nix @@ -42,6 +42,7 @@ in networking.dhcpcd.enable = false; security.pki.certificateFiles = [ nodes.acme.test-support.acme.caCert ]; networking.hosts."${nodes.acme.networking.primaryIPAddress}" = [ nodes.acme.test-support.acme.caDomain ]; + virtualisation.cores = 2; }; nodes = { acme = From 7c38f0313730912f10536de0252393a8b2f80d9a Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 9 May 2025 10:57:44 +0200 Subject: [PATCH 3/4] tests: add very simple test of the file monitor --- flake.nix | 1 + nixos/file-test.nix | 49 +++++++++++++++++++++++++++++++++++++++++++++ nixos/lib.nix | 2 +- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 nixos/file-test.nix diff --git a/flake.nix b/flake.nix index 61cefcc..c1c9112 100644 --- a/flake.nix +++ b/flake.nix @@ -72,6 +72,7 @@ faythe $DIR/$FILE --config-check >>$out done ''; + file = pkgs.callPackage ./nixos/file-test.nix {}; vault = pkgs.callPackage ./nixos/vault-test.nix {}; clippy = pkgs."${pname}-clippy"; }; diff --git a/nixos/file-test.nix b/nixos/file-test.nix new file mode 100644 index 0000000..c72c994 --- /dev/null +++ b/nixos/file-test.nix @@ -0,0 +1,49 @@ +{ lib, pkgs }: +let + testLib = import ./lib.nix { + inherit lib pkgs; + }; + + domain = testLib.domain; + cert_path = "/tmp/faythe"; +in +testLib.mkFaytheTest ({ nodes, ... }: { + name = "faythe-file-test"; + extraModules.client = [ + ({ config, pkgs, ... }: { + environment.systemPackages = [pkgs.openssl]; + + systemd.services.faythe.preStart = '' + mkdir -p ${cert_path} + ''; + }) + ]; + faytheExtraConfig = { + file_monitor_configs = [ + { + directory = cert_path; + prune = true; + specs = [ + { + name = "path1-test"; + cn = "path1.${domain}"; + key_file_name = "key.pem"; + } + ]; + } + ]; + }; + testScript = '' + with subtest("Normal first time issue"): + client.wait_until_succeeds("stat ${cert_path}/path1-test") + + client.wait_until_succeeds(""" + journalctl -u faythe | grep "path1-test" | grep -q "touched" + journalctl -u faythe | grep -q "changing group for" + """) + + client.succeed(""" + openssl x509 -in ${cert_path}/path1-test/fullchain.pem -text -noout | grep -q "Issuer: CN=Pebble Intermediate" + """) + ''; +}) diff --git a/nixos/lib.nix b/nixos/lib.nix index e0b89ff..a11f936 100644 --- a/nixos/lib.nix +++ b/nixos/lib.nix @@ -72,7 +72,7 @@ in @ IN NS ${ns_host}. ${ns_host}. IN A ${nodes.ns.networking.primaryIPAddress} - '' + args.extraBindZoneFileLines; + '' + (args.extraBindZoneFileLines or ""); }; networking.firewall.allowedTCPPorts = [ 53 ]; From 0d2ddb272ce0a4dbedcbf693d0f3f8abac29a88a Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 9 May 2025 21:33:51 +0200 Subject: [PATCH 4/4] ci: run the file flake check along with the other checks --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ec5ae7..b35d361 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,3 +31,11 @@ jobs: with: nix_path: nixpkgs=channel:nixos-24.11 - run: nix build ./#checks.x86_64-linux.vault + flake-check-file: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v30 + with: + nix_path: nixpkgs=channel:nixos-24.11 + - run: nix build ./#checks.x86_64-linux.file