From 210430967d32ed822aa021316b15155e647125ff Mon Sep 17 00:00:00 2001 From: archipelago Date: Sun, 2 Aug 2026 08:49:57 -0400 Subject: [PATCH] fix(10-03): fail closed on first-boot secret regeneration failure (F-03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first-boot per-device secret regeneration was fail-open: both branches logged a warning and continued, and `touch "$MARKER"` ran unconditionally outside both `if` blocks. Combined with the unit's ConditionPathExists=! and the script's own marker fast-path, one transient failure left that node on the image-wide shared SSH host key and TLS private key permanently and silently — and the ISO is a published artefact, so every downloader holds those keys. - Retry each generator 3 times with backoff (D-05), so a transient first-boot condition recovers inside the same boot instead of being terminal. - Write the completion marker ONLY when both TLS and SSH succeeded, so a failed boot leaves the unit eligible to run again on the next boot. - On terminal failure: durable record at /var/lib/archipelago/first-boot-secrets.failed naming which generator failed, plus console + logger + stderr, and exit 1 so the unit lands in `failed` rather than `active`. The record is cleared on a later success. - Add FIRST_BOOT_SECRETS_ROOT / FIRST_BOOT_SECRETS_BACKOFF seams. Unset in production the behaviour is byte-identical; set, they let the fail-closed property be asserted rather than claimed. - Order the unit After=systemd-random-seed.service (no-op today, correct if a seed file is ever baked). - State the operational trade in the script header: after the rootfs strip, a terminal failure means no SSH and no TLS and needs the physical console. That was chosen deliberately over running on fleet-shared keys. tests/first-boot-secrets/run-tests.sh extracts the shipped heredoc body from the builder and drives it against a temp root with stubbed generators: both succeed, openssl fails every attempt, ssh-keygen fails twice then succeeds. Moving the marker touch back outside the success branch makes case 2 fail with MARKER-SET-ON-FAILURE, which is the regression this pins. Co-Authored-By: Claude Opus 5 (1M context) --- .../_archived/build-auto-installer-iso.sh | 202 +++++++++++++--- tests/first-boot-secrets/run-tests.sh | 225 ++++++++++++++++++ 2 files changed, 393 insertions(+), 34 deletions(-) create mode 100755 tests/first-boot-secrets/run-tests.sh diff --git a/image-recipe/_archived/build-auto-installer-iso.sh b/image-recipe/_archived/build-auto-installer-iso.sh index 59d89f52..d7a9a44c 100755 --- a/image-recipe/_archived/build-auto-installer-iso.sh +++ b/image-recipe/_archived/build-auto-installer-iso.sh @@ -1601,6 +1601,11 @@ cat > "$WORK_DIR/archipelago-first-boot-secrets.service" <<'SECRETSSERVICE' Description=Regenerate per-device secrets (TLS key, SSH host keys) DefaultDependencies=no After=local-fs.target +# No random-seed file is baked into the rootfs today (verified by the entropy +# audit's C-4 tar listing), so this ordering is a no-op right now. It is here +# so that if one is ever introduced, the pool is credited before this unit — +# the first consumer of entropy on a freshly-flashed machine — draws from it. +After=systemd-random-seed.service Before=ssh.service nginx.service archipelago.service ConditionPathExists=!/var/lib/archipelago/.secrets-regenerated @@ -1615,53 +1620,182 @@ SECRETSSERVICE cat > "$WORK_DIR/first-boot-secrets.sh" <<'SECRETSSCRIPT' #!/bin/bash -# Replace image-baked secrets with device-unique ones on first boot. -# Never leaves the device without working keys: new material is generated -# to a staging path first and only swapped in on success. +# Create this device's own TLS keypair and SSH host keys on first boot. +# +# ── FAIL CLOSED — read this before changing anything below (audit F-03) ── +# +# The rootfs tar is byte-identical on every node flashed from one ISO, and the +# ISO is a published artefact. It therefore no longer carries any identity +# material: the rootfs Dockerfile in STEP 1 of this builder strips the SSH +# host keys, the TLS keypair and machine-id out of the shared image. THIS +# SCRIPT IS THE ONLY THING THAT CREATES THEM. That is deliberate. +# +# The operational consequence, in plain words: if regeneration fails every +# retry, this node has no SSH host key and no TLS key. sshd will not start and +# the nginx TLS listener will not start, so the node cannot be reached over +# the network and recovery requires the physical console. +# +# That cost was accepted on purpose. The behaviour it replaces was worse: log +# a warning, set the completion marker anyway, and run forever on the SSH host +# key and TLS private key that every downloader of the ISO also holds — which +# is undetectable host impersonation and transparent MITM of the web UI, on a +# node whose operator has no idea. +# +# So: the completion marker is written ONLY when both generators succeeded. A +# failed boot leaves the marker absent, which leaves the unit's +# ConditionPathExists=! satisfied, so the whole thing runs again on the next +# boot. Each generator is retried with backoff first, so a transient first-boot +# condition (slow entropy pool, momentarily full disk) recovers without needing +# a reboot at all. +# +# Testability seam: FIRST_BOOT_SECRETS_ROOT prefixes every absolute path. It is +# unset in production — the expansion is empty and behaviour is identical to a +# script with the paths hard-coded — and set to a temp dir by +# tests/first-boot-secrets/run-tests.sh, which is what makes the fail-closed +# property assertable instead of merely claimed. set -u -LOG=/var/log/archipelago-first-boot-secrets.log -MARKER=/var/lib/archipelago/.secrets-regenerated +ROOT="${FIRST_BOOT_SECRETS_ROOT:-}" + +# Waits between attempts at one generator. The attempt count is the number of +# entries; the wait after the final attempt is skipped, because a failed last +# attempt is terminal and there is nothing left to wait for. With the default +# 3-entry list that means 3 attempts at t=0s, t=2s and t=10s, and the trailing +# 20 is the ceiling that applies if the list is ever lengthened. Tests override +# this with zeros so the suite does not sleep. +BACKOFF="${FIRST_BOOT_SECRETS_BACKOFF:-2 8 20}" + +LOG="$ROOT/var/log/archipelago-first-boot-secrets.log" +MARKER="$ROOT/var/lib/archipelago/.secrets-regenerated" +FAILED="$ROOT/var/lib/archipelago/first-boot-secrets.failed" +CONSOLE="$ROOT/dev/console" +SSL_DIR="$ROOT/etc/archipelago/ssl" +SSH_DIR="$ROOT/etc/ssh" + [ -f "$MARKER" ] && exit 0 -mkdir -p /var/lib/archipelago +mkdir -p "$ROOT/var/lib/archipelago" "$ROOT/var/log" NODE_NAME=$(hostname 2>/dev/null || echo archipelago) -echo "$(date): regenerating per-device secrets" >> "$LOG" +log() { echo "$(date): $*" >> "$LOG"; } + +# A terminal failure must be impossible to miss: journal, console and stderr, +# on top of the durable on-disk record. Every channel is guarded so that a +# missing /dev/console (test root, or an early boot without one) cannot itself +# make the failure path fail. +shout() { + log "$*" + if command -v logger >/dev/null 2>&1; then + logger -t archipelago-first-boot-secrets "$*" 2>/dev/null || true + fi + if [ -w "$CONSOLE" ]; then + printf '%s\n' "$*" | tee -a "$CONSOLE" >/dev/null 2>&1 || true + fi + printf '%s\n' "$*" >&2 +} + +# retry