#!/bin/bash # Regression harness for the first-boot per-device secret regeneration script # (audit finding F-03, phase 10 / KEY-02). # # What this pins, and why it exists at all: the script used to `touch` its # completion marker unconditionally, outside both success branches, so one # transient failure at first boot left the node running the ISO-wide shared # SSH host key and TLS private key forever, silently. The property that must # never regress is therefore negative — "on failure the marker is NOT created" # — and a negative property is only assertable if the failure can be forced. # So the generators are stubbed and the script is driven against a temp root # through the FIRST_BOOT_SECRETS_ROOT seam. # # The script under test is not a file in this repo: it is a heredoc inside # image-recipe/_archived/build-auto-installer-iso.sh (which is LIVE — # image-recipe/build-debian-iso.sh execs it). The harness extracts the heredoc # body between the SECRETSSCRIPT delimiters so it is testing the bytes that # actually ship, not a copy that can drift. # # Usage: bash tests/first-boot-secrets/run-tests.sh # Exit 0 only if all three cases PASS. set -euo pipefail REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" BUILDER="$REPO/image-recipe/_archived/build-auto-installer-iso.sh" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT PASS_COUNT=0 FAIL_COUNT=0 ok() { echo "PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); } bad() { echo "FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); } # ── Step 0: extract the script under test and syntax-check it ──────────── [ -f "$BUILDER" ] || { echo "FAIL: builder not found at $BUILDER"; exit 1; } SCRIPT="$WORK/first-boot-secrets.sh" awk '/^cat > "\$WORK_DIR\/first-boot-secrets.sh" <<.SECRETSSCRIPT.$/ { f = 1; next } f && /^SECRETSSCRIPT$/ { f = 0 } f { print }' \ "$BUILDER" > "$SCRIPT" if [ ! -s "$SCRIPT" ]; then echo "FAIL: could not extract the first-boot-secrets.sh heredoc from the builder" echo " (did the SECRETSSCRIPT delimiter or the cat> line change?)" exit 1 fi chmod +x "$SCRIPT" if bash -n "$SCRIPT"; then echo "extracted $(wc -l < "$SCRIPT") lines from the builder; bash -n clean" else echo "FAIL: extracted script does not parse" exit 1 fi # ── Stubs ───────────────────────────────────────────────────────────────── # A stub dir is prepended to PATH so the script's openssl / ssh-keygen / # systemctl / logger calls hit these instead of the real tools. Behaviour is # driven by env vars the stubs read at call time. # # STUB_OPENSSL_MODE ok | fail (fail affects `req` only — # `pkey`/`x509` validation still # works, so a failed generation # cannot be mistaken for a failed # validation) # STUB_SSHKEYGEN_MODE ok | fail | fail-twice (uses a counter file) # STUB_COUNTER_DIR where the counter file lives # STUB_SYSTEMCTL_FAILED_UNITS units `systemctl is-failed` should report failed # STUB_SYSTEMCTL_LOG file the systemctl stub appends its args to make_stubs() { local dir="$1" mkdir -p "$dir" cat > "$dir/openssl" <<'STUB' #!/bin/bash # Stub openssl. Honours -keyout/-out so the script's staging-then-swap and its # non-empty checks are exercised for real, and implements the `pkey`/`x509` # parse-back validation the generator does before it swaps. sub="${1:-}" case "$sub" in pkey|x509) # Validation: succeed iff the file exists and is non-empty. This is what # lets the harness prove a truncated artefact is never swapped in. f="" while [ $# -gt 0 ]; do case "$1" in -in) f="$2"; shift 2 ;; *) shift ;; esac done [ -n "$f" ] && [ -s "$f" ] || exit 1 exit 0 ;; esac [ "${STUB_OPENSSL_MODE:-ok}" = "fail" ] && exit 1 keyout="" out="" while [ $# -gt 0 ]; do case "$1" in -keyout) keyout="$2"; shift 2 ;; -out) out="$2"; shift 2 ;; *) shift ;; esac done [ -n "$keyout" ] && printf -- '-----BEGIN PRIVATE KEY-----\nstub\n-----END PRIVATE KEY-----\n' > "$keyout" [ -n "$out" ] && printf -- '-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n' > "$out" exit 0 STUB cat > "$dir/ssh-keygen" <<'STUB' #!/bin/bash # Stub ssh-keygen -A: writes a host-key set into <-f dir>/etc/ssh, matching # the real tool's layout, which is what the script globs for. mode="${STUB_SSHKEYGEN_MODE:-ok}" counter="${STUB_COUNTER_DIR:-/tmp}/ssh-keygen.count" n=$(cat "$counter" 2>/dev/null || echo 0) n=$((n + 1)) echo "$n" > "$counter" case "$mode" in fail) exit 1 ;; fail-twice) [ "$n" -le 2 ] && exit 1 ;; esac root="" while [ $# -gt 0 ]; do case "$1" in -f) root="$2"; shift 2 ;; *) shift ;; esac done [ -n "$root" ] || exit 1 mkdir -p "$root/etc/ssh" for t in rsa ecdsa ed25519; do printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nstub-%s\n' "$t" > "$root/etc/ssh/ssh_host_${t}_key" printf -- 'ssh-%s AAAAstub stub@archipelago\n' "$t" > "$root/etc/ssh/ssh_host_${t}_key.pub" done exit 0 STUB cat > "$dir/systemctl" <<'STUB' #!/bin/bash # Stub systemctl. Records every invocation so the harness can prove the # self-heal path actually restarts a unit that failed for want of a key, and # reports is-failed honestly so first-boot and self-heal take different paths. [ -n "${STUB_SYSTEMCTL_LOG:-}" ] && echo "$*" >> "$STUB_SYSTEMCTL_LOG" if [ "${1:-}" = "is-failed" ]; then unit="${!#}" for u in ${STUB_SYSTEMCTL_FAILED_UNITS:-}; do [ "$u" = "$unit" ] && exit 0 done exit 1 fi exit 0 STUB # Must not be allowed to touch the host journal during a test run. printf '#!/bin/bash\nexit 0\n' > "$dir/logger" chmod +x "$dir"/openssl "$dir"/ssh-keygen "$dir"/systemctl "$dir"/logger } STUBS="$WORK/stubs" make_stubs "$STUBS" # ── Runner ──────────────────────────────────────────────────────────────── # run_case [prestage] [reuse] # # prestage baked — root already holds the shared keys, i.e. a pre-strip # rootfs. Assertions can then prove the swap replaced them. # stripped — root holds no key material at all, i.e. the rootfs this # build actually ships. This is the state in which "no key # may appear from anywhere but the generator" is testable. # reuse 1 — do not wipe the root or the counters; continue from the # previous run against the same node. Models a reboot or a # timer-triggered retry. CASE_ROOT="" CASE_RC=0 CASE_SYSTEMCTL_LOG="" run_case() { local name="$1" openssl_mode="$2" sshkeygen_mode="$3" local prestage="${4:-baked}" reuse="${5:-0}" CASE_ROOT="$WORK/root-$name" CASE_SYSTEMCTL_LOG="$WORK/$name.systemctl" if [ "$reuse" != "1" ]; then rm -rf "$CASE_ROOT" mkdir -p "$CASE_ROOT/var/lib/archipelago" "$CASE_ROOT/var/log" \ "$CASE_ROOT/etc/ssh" "$CASE_ROOT/etc/archipelago/ssl" if [ "$prestage" = "baked" ]; then echo "BAKED-SHARED-HOST-KEY" > "$CASE_ROOT/etc/ssh/ssh_host_rsa_key" echo "BAKED-SHARED-TLS-KEY" > "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" fi rm -f "$WORK/counters-$name/ssh-keygen.count" mkdir -p "$WORK/counters-$name" : > "$CASE_SYSTEMCTL_LOG" fi set +e env PATH="$STUBS:$PATH" \ FIRST_BOOT_SECRETS_ROOT="$CASE_ROOT" \ FIRST_BOOT_SECRETS_BACKOFF="0 0 0" \ STUB_OPENSSL_MODE="$openssl_mode" \ STUB_SSHKEYGEN_MODE="$sshkeygen_mode" \ STUB_COUNTER_DIR="$WORK/counters-$name" \ STUB_SYSTEMCTL_LOG="$CASE_SYSTEMCTL_LOG" \ STUB_SYSTEMCTL_FAILED_UNITS="${STUB_SYSTEMCTL_FAILED_UNITS:-}" \ bash "$SCRIPT" > "$WORK/$name.out" 2> "$WORK/$name.err" CASE_RC=$? set -e } fail_detail() { echo " exit=$CASE_RC root=$CASE_ROOT" echo " stderr: $(head -c 300 "$WORK/$1.err" 2>/dev/null)" } # ── Case 1: both generators succeed ────────────────────────────────────── run_case both-ok ok ok c1="" [ "$CASE_RC" -eq 0 ] || c1="$c1 exit-nonzero" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c1="$c1 marker-missing" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c1="$c1 stale-failure-record" [ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] || c1="$c1 tls-key-missing" [ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c1="$c1 tls-crt-missing" grep -q BAKED-SHARED-TLS-KEY "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" && c1="$c1 tls-key-not-replaced" [ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c1="$c1 ssh-host-key-missing" grep -q BAKED-SHARED-HOST-KEY "$CASE_ROOT/etc/ssh/ssh_host_rsa_key" && c1="$c1 ssh-key-not-replaced" ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c1="$c1 dotnew-leftover" if [ -z "$c1" ]; then ok "both generators succeed -> exit 0, marker set, keys swapped in" else bad "both generators succeed ->$c1"; fail_detail both-ok fi # ── Case 2: openssl fails every attempt -> FAIL CLOSED ─────────────────── # This is the case that would have passed against the old script and is the # whole reason this harness exists: the old code logged a warning and set the # marker anyway. run_case tls-fail fail ok c2="" [ "$CASE_RC" -ne 0 ] || c2="$c2 exit-zero-on-failure" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c2="$c2 MARKER-SET-ON-FAILURE" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c2="$c2 no-failure-record" grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \ || c2="$c2 failure-record-does-not-name-TLS" ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c2="$c2 dotnew-leftover" grep -qi 'FAILED' "$WORK/tls-fail.err" || c2="$c2 no-loud-stderr" if [ -z "$c2" ]; then ok "openssl fails every attempt -> exit non-zero, NO marker, failure record names TLS" else bad "openssl fails every attempt ->$c2"; fail_detail tls-fail fi # ── Case 3: ssh-keygen fails twice then succeeds -> backoff recovers ───── run_case ssh-flaky ok fail-twice c3="" [ "$CASE_RC" -eq 0 ] || c3="$c3 exit-nonzero" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c3="$c3 marker-missing" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c3="$c3 failure-record-present" [ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c3="$c3 ssh-host-key-missing" attempts=$(cat "$WORK/counters-ssh-flaky/ssh-keygen.count" 2>/dev/null || echo 0) [ "$attempts" -eq 3 ] || c3="$c3 expected-3-attempts-got-$attempts" if [ -z "$c3" ]; then ok "ssh-keygen fails twice then succeeds -> backoff recovers within one boot (3 attempts)" else bad "ssh-keygen fails twice then succeeds ->$c3"; fail_detail ssh-flaky fi # ── Case 4: TLS fails every attempt on a STRIPPED root -> no key at all ── # Case 2 proves the marker is not set. This proves the stronger property that # replaced the installer's TLS fallback: on the rootfs we actually ship, a # failed generation leaves NO key, from any source. If anything ever mints a # key outside gen_tls — an install-time fallback, a placeholder, a zero-length # touch to keep nginx happy — this is the case that goes red. run_case tls-fail-stripped fail ok stripped c4="" [ "$CASE_RC" -ne 0 ] || c4="$c4 exit-zero-on-failure" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c4="$c4 MARKER-SET-ON-FAILURE" [ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] && c4="$c4 TLS-KEY-EXISTS-AFTER-FAILURE" [ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] && c4="$c4 TLS-CRT-EXISTS-AFTER-FAILURE" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c4="$c4 no-failure-record" grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \ || c4="$c4 failure-record-does-not-name-TLS" ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c4="$c4 dotnew-leftover" if [ -z "$c4" ]; then ok "TLS fails every attempt on a stripped root -> NO key, NO marker, non-zero exit, record names TLS" else bad "TLS fails every attempt on a stripped root ->$c4"; fail_detail tls-fail-stripped fi # ── Case 5: self-heal — a failed run, then a later run that succeeds ───── # The case that proves a node is not permanently dead. Run 1 is a machine whose # generator fails every retry; run 2 is the same machine minutes later, once the # transient cause cleared, driven by archipelago-first-boot-secrets.timer. It # must end with the key present and the marker set, with no human at a console. # Run 2 also declares nginx/ssh already `failed` — they tried to start without a # key — so the run must actively restart them, not just reload. A "recovery" # that leaves the services down is not a recovery. # # Run 1 asserts only enough to establish the precondition (it really did fail, # and it left the node eligible to retry). Whether a key exists after a failure # is case 4's job — asserting it here too would make a single defect light up # two cases and blunt the signal. run_case self-heal fail ok stripped c5="" [ "$CASE_RC" -ne 0 ] || c5="$c5 run1-exit-zero" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c5="$c5 run1-marker-set" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c5="$c5 run1-no-failure-record" STUB_SYSTEMCTL_FAILED_UNITS="nginx ssh" run_case self-heal ok ok stripped 1 [ "$CASE_RC" -eq 0 ] || c5="$c5 run2-exit-nonzero" [ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c5="$c5 run2-marker-missing" [ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] || c5="$c5 run2-key-missing" [ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c5="$c5 run2-crt-missing" [ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c5="$c5 run2-ssh-key-missing" [ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c5="$c5 run2-stale-failure-record" grep -q 'restart nginx' "$CASE_SYSTEMCTL_LOG" 2>/dev/null || c5="$c5 run2-did-not-restart-failed-nginx" if [ -z "$c5" ]; then ok "self-heal: failed run then a later successful run -> key present, marker set, failed units restarted" else bad "self-heal ->$c5"; fail_detail self-heal fi # ── Case 6: single-producer invariant ──────────────────────────────────── # The regression that would silently recreate F-03 is not a broken assertion — # it is somebody adding a second, well-meaning place that mints a key. A second # producer brings its own idea of success, its own absent retry policy and its # own absent failure record, and that is what made F-03 silent. # # So: every executable key-creating invocation in the builder must live inside # the first-boot-secrets.sh heredoc, i.e. inside gen_tls/gen_ssh. Comments are # exempt (they discuss the history); binary-existence checks are not matched # because they do not carry a key-creating subcommand. c6="" SS_START=$(grep -n '^cat > "\$WORK_DIR/first-boot-secrets.sh" <<.SECRETSSCRIPT.$' "$BUILDER" | cut -d: -f1) SS_END=$(awk -v s="$SS_START" 'NR>s && /^SECRETSSCRIPT$/ { print NR; exit }' "$BUILDER") if [ -z "$SS_START" ] || [ -z "$SS_END" ]; then c6="$c6 could-not-locate-generator-heredoc" else PRODUCERS=$(grep -nE 'openssl[[:space:]]+req|ssh-keygen[[:space:]]+-A|ssh-keygen[[:space:]]+-t' "$BUILDER" \ | grep -vE '^[0-9]+:[[:space:]]*#' || true) while IFS= read -r line; do [ -z "$line" ] && continue ln=${line%%:*} if [ "$ln" -lt "$SS_START" ] || [ "$ln" -gt "$SS_END" ]; then c6="$c6 SECOND-PRODUCER-at-line-$ln" fi done <<< "$PRODUCERS" # Sanity: the one producer we expect must actually be in there, otherwise an # empty result would pass this case vacuously. echo "$PRODUCERS" | grep -q 'openssl[[:space:]]*req' || c6="$c6 no-tls-producer-found-at-all" echo "$PRODUCERS" | grep -q 'ssh-keygen' || c6="$c6 no-ssh-producer-found-at-all" fi if [ -z "$c6" ]; then ok "single-producer invariant: every key-creating invocation is inside gen_tls/gen_ssh" else bad "single-producer invariant ->$c6" echo " generator heredoc spans lines $SS_START-$SS_END of $BUILDER" fi # ── Summary ─────────────────────────────────────────────────────────────── echo echo "──────── first-boot-secrets summary ────────" echo "passed: $PASS_COUNT failed: $FAIL_COUNT" [ "$FAIL_COUNT" -eq 0 ] || exit 1 exit 0