archy/tests/first-boot-secrets/run-tests.sh

#!/bin/bash
# Regression harness for the first-boot per-device secret regeneration script
# (audit finding F-03, phase 10 / KEY-02).
#
# What this pins, and why it exists at all: the script used to `touch` its
# completion marker unconditionally, outside both success branches, so one
# transient failure at first boot left the node running the ISO-wide shared
# SSH host key and TLS private key forever, silently. The property that must
# never regress is therefore negative — "on failure the marker is NOT created"
# — and a negative property is only assertable if the failure can be forced.
# So the generators are stubbed and the script is driven against a temp root
# through the FIRST_BOOT_SECRETS_ROOT seam.
#
# The script under test is not a file in this repo: it is a heredoc inside
# image-recipe/_archived/build-auto-installer-iso.sh (which is LIVE —
# image-recipe/build-debian-iso.sh execs it). The harness extracts the heredoc
# body between the SECRETSSCRIPT delimiters so it is testing the bytes that
# actually ship, not a copy that can drift.
#
# Usage:  bash tests/first-boot-secrets/run-tests.sh
# Exit 0 only if all three cases PASS.

set -euo pipefail

REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
BUILDER="$REPO/image-recipe/_archived/build-auto-installer-iso.sh"

WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT

PASS_COUNT=0
FAIL_COUNT=0

ok()   { echo "PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); }
bad()  { echo "FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); }

# ── Step 0: extract the script under test and syntax-check it ────────────
[ -f "$BUILDER" ] || { echo "FAIL: builder not found at $BUILDER"; exit 1; }

SCRIPT="$WORK/first-boot-secrets.sh"
awk '/^cat > "\$WORK_DIR\/first-boot-secrets.sh" <<.SECRETSSCRIPT.$/ { f = 1; next }
     f && /^SECRETSSCRIPT$/                                          { f = 0 }
     f                                                               { print }' \
    "$BUILDER" > "$SCRIPT"

if [ ! -s "$SCRIPT" ]; then
    echo "FAIL: could not extract the first-boot-secrets.sh heredoc from the builder"
    echo "      (did the SECRETSSCRIPT delimiter or the cat> line change?)"
    exit 1
fi
chmod +x "$SCRIPT"

if bash -n "$SCRIPT"; then
    echo "extracted $(wc -l < "$SCRIPT") lines from the builder; bash -n clean"
else
    echo "FAIL: extracted script does not parse"
    exit 1
fi

# ── Stubs ─────────────────────────────────────────────────────────────────
# A stub dir is prepended to PATH so the script's openssl / ssh-keygen /
# systemctl / logger calls hit these instead of the real tools. Behaviour is
# driven by env vars the stubs read at call time.
#
#   STUB_OPENSSL_MODE     ok | fail
#   STUB_SSHKEYGEN_MODE   ok | fail | fail-twice   (fail-twice uses a counter file)
#   STUB_COUNTER_DIR      where the counter file lives
make_stubs() {
    local dir="$1"
    mkdir -p "$dir"

    cat > "$dir/openssl" <<'STUB'
#!/bin/bash
# Stub openssl: honours -keyout/-out so the script's staging-then-swap and its
# non-empty checks are exercised for real.
[ "${STUB_OPENSSL_MODE:-ok}" = "fail" ] && exit 1
keyout="" out=""
while [ $# -gt 0 ]; do
    case "$1" in
        -keyout) keyout="$2"; shift 2 ;;
        -out)    out="$2";    shift 2 ;;
        *)       shift ;;
    esac
done
[ -n "$keyout" ] && printf -- '-----BEGIN PRIVATE KEY-----\nstub\n-----END PRIVATE KEY-----\n' > "$keyout"
[ -n "$out" ]    && printf -- '-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n' > "$out"
exit 0
STUB

    cat > "$dir/ssh-keygen" <<'STUB'
#!/bin/bash
# Stub ssh-keygen -A: writes a host-key set into <-f dir>/etc/ssh, matching
# the real tool's layout, which is what the script globs for.
mode="${STUB_SSHKEYGEN_MODE:-ok}"
counter="${STUB_COUNTER_DIR:-/tmp}/ssh-keygen.count"

n=$(cat "$counter" 2>/dev/null || echo 0)
n=$((n + 1))
echo "$n" > "$counter"

case "$mode" in
    fail)       exit 1 ;;
    fail-twice) [ "$n" -le 2 ] && exit 1 ;;
esac

root=""
while [ $# -gt 0 ]; do
    case "$1" in
        -f) root="$2"; shift 2 ;;
        *)  shift ;;
    esac
done
[ -n "$root" ] || exit 1
mkdir -p "$root/etc/ssh"
for t in rsa ecdsa ed25519; do
    printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nstub-%s\n' "$t" > "$root/etc/ssh/ssh_host_${t}_key"
    printf -- 'ssh-%s AAAAstub stub@archipelago\n' "$t"            > "$root/etc/ssh/ssh_host_${t}_key.pub"
done
exit 0
STUB

    # Neither of these must be allowed to touch the host during a test run.
    printf '#!/bin/bash\nexit 0\n' > "$dir/systemctl"
    printf '#!/bin/bash\nexit 0\n' > "$dir/logger"

    chmod +x "$dir"/openssl "$dir"/ssh-keygen "$dir"/systemctl "$dir"/logger
}

STUBS="$WORK/stubs"
make_stubs "$STUBS"

# ── Runner ────────────────────────────────────────────────────────────────
# Runs the script against a fresh temp root. Echoes the exit status; the
# caller asserts on it plus the resulting filesystem state.
CASE_ROOT=""
CASE_RC=0
run_case() {
    local name="$1" openssl_mode="$2" sshkeygen_mode="$3"
    CASE_ROOT="$WORK/root-$name"
    rm -rf "$CASE_ROOT"
    mkdir -p "$CASE_ROOT/var/lib/archipelago" "$CASE_ROOT/var/log" \
             "$CASE_ROOT/etc/ssh" "$CASE_ROOT/etc/archipelago/ssl"
    # A pre-existing baked host key + TLS key, i.e. the pre-strip rootfs state:
    # the assertions below then also show the swap actually replaced them.
    echo "BAKED-SHARED-HOST-KEY" > "$CASE_ROOT/etc/ssh/ssh_host_rsa_key"
    echo "BAKED-SHARED-TLS-KEY"  > "$CASE_ROOT/etc/archipelago/ssl/archipelago.key"

    rm -f "$WORK/counters-$name/ssh-keygen.count"
    mkdir -p "$WORK/counters-$name"

    set +e
    env PATH="$STUBS:$PATH" \
        FIRST_BOOT_SECRETS_ROOT="$CASE_ROOT" \
        FIRST_BOOT_SECRETS_BACKOFF="0 0 0" \
        STUB_OPENSSL_MODE="$openssl_mode" \
        STUB_SSHKEYGEN_MODE="$sshkeygen_mode" \
        STUB_COUNTER_DIR="$WORK/counters-$name" \
        bash "$SCRIPT" > "$WORK/$name.out" 2> "$WORK/$name.err"
    CASE_RC=$?
    set -e
}

fail_detail() {
    echo "    exit=$CASE_RC root=$CASE_ROOT"
    echo "    stderr: $(head -c 300 "$WORK/$1.err" 2>/dev/null)"
}

# ── Case 1: both generators succeed ──────────────────────────────────────
run_case both-ok ok ok
c1=""
[ "$CASE_RC" -eq 0 ] || c1="$c1 exit-nonzero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c1="$c1 marker-missing"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c1="$c1 stale-failure-record"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] || c1="$c1 tls-key-missing"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c1="$c1 tls-crt-missing"
grep -q BAKED-SHARED-TLS-KEY "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" && c1="$c1 tls-key-not-replaced"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c1="$c1 ssh-host-key-missing"
grep -q BAKED-SHARED-HOST-KEY "$CASE_ROOT/etc/ssh/ssh_host_rsa_key" && c1="$c1 ssh-key-not-replaced"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c1="$c1 dotnew-leftover"
if [ -z "$c1" ]; then
    ok "both generators succeed -> exit 0, marker set, keys swapped in"
else
    bad "both generators succeed ->$c1"; fail_detail both-ok
fi

# ── Case 2: openssl fails every attempt -> FAIL CLOSED ───────────────────
# This is the case that would have passed against the old script and is the
# whole reason this harness exists: the old code logged a warning and set the
# marker anyway.
run_case tls-fail fail ok
c2=""
[ "$CASE_RC" -ne 0 ] || c2="$c2 exit-zero-on-failure"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c2="$c2 MARKER-SET-ON-FAILURE"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c2="$c2 no-failure-record"
grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \
    || c2="$c2 failure-record-does-not-name-TLS"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c2="$c2 dotnew-leftover"
grep -qi 'FAILED' "$WORK/tls-fail.err" || c2="$c2 no-loud-stderr"
if [ -z "$c2" ]; then
    ok "openssl fails every attempt -> exit non-zero, NO marker, failure record names TLS"
else
    bad "openssl fails every attempt ->$c2"; fail_detail tls-fail
fi

# ── Case 3: ssh-keygen fails twice then succeeds -> backoff recovers ─────
run_case ssh-flaky ok fail-twice
c3=""
[ "$CASE_RC" -eq 0 ] || c3="$c3 exit-nonzero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c3="$c3 marker-missing"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c3="$c3 failure-record-present"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c3="$c3 ssh-host-key-missing"
attempts=$(cat "$WORK/counters-ssh-flaky/ssh-keygen.count" 2>/dev/null || echo 0)
[ "$attempts" -eq 3 ] || c3="$c3 expected-3-attempts-got-$attempts"
if [ -z "$c3" ]; then
    ok "ssh-keygen fails twice then succeeds -> backoff recovers within one boot (3 attempts)"
else
    bad "ssh-keygen fails twice then succeeds ->$c3"; fail_detail ssh-flaky
fi

# ── Summary ───────────────────────────────────────────────────────────────
echo
echo "──────── first-boot-secrets summary ────────"
echo "passed: $PASS_COUNT   failed: $FAIL_COUNT"
[ "$FAIL_COUNT" -eq 0 ] || exit 1
exit 0