diff --git a/core/archipelago/src/api/rpc/system/handlers.rs b/core/archipelago/src/api/rpc/system/handlers.rs index 45774355..efb4c22b 100644 --- a/core/archipelago/src/api/rpc/system/handlers.rs +++ b/core/archipelago/src/api/rpc/system/handlers.rs @@ -193,7 +193,14 @@ impl RpcHandler { }; let (disk_used, disk_total) = read_disk_usage_path(df_target).await.unwrap_or((0, 0)); + // F-03 verdict. Surfacing it here rather than only in a log file is the + // point: an operator must be able to see that a node is running + // fleet-shared host keys without having to SSH into it — and SSH into + // it is exactly the thing a shared host key compromises. + let host_secrets = host_secrets_status(Path::new(HOST_SECRETS_STATE_DIR)).await; + Ok(serde_json::json!({ + "host_secrets": host_secrets, "uptime_secs": uptime as u64, "load_avg_1": load.0, "load_avg_5": load.1, @@ -499,6 +506,74 @@ const RM_BIN: &str = "/usr/bin/rm"; const TLS_KEY_FALLBACK_MODE: &str = "600"; const TLS_CRT_FALLBACK_MODE: &str = "644"; +/// Where `scripts/security/host-secrets-audit.sh` leaves its verdict. Hardcoded +/// rather than derived from `config.data_dir` because the script is a systemd +/// unit with no view of the daemon's configuration — this is the path both +/// sides agree on. +const HOST_SECRETS_STATE_DIR: &str = "/var/lib/archipelago"; +const HOST_SECRETS_AUDIT_FILE: &str = "host-secrets-audit.json"; +const HOST_KEY_ROTATION_FILE: &str = "host-key-rotation.json"; + +/// The `host_secrets` object carried by `system.stats`: has this node been +/// judged to be running the fleet-shared, image-baked SSH host keys and TLS +/// private key (audit F-03), and has it been rotated since? +/// +/// Three properties this must hold, because `system.stats` is in +/// `CACHEABLE_METHODS` (`api/rpc/middleware.rs:41`) and the dashboard polls it: +/// +/// 1. **It never errors.** A missing, truncated or unparseable verdict file +/// yields `{"verdict":"unknown"}`. A node that has not run the audit yet — +/// every node, until the OTA carrying the unit lands — must not turn its own +/// stats call into a failure. +/// 2. **It is cheap.** Two small file reads, no process spawn, no key parsing. +/// The expensive work (ssh-keygen, openssl) happens in the boot unit, once. +/// 3. **It carries no fingerprints.** Those live in the on-disk record only. +/// They are public data, but there is no reason to put them in a payload +/// that is polled every few seconds — the dashboard needs the verdict, and +/// an operator who needs the digests is already on the node. +async fn host_secrets_status(dir: &Path) -> serde_json::Value { + let unknown = || serde_json::json!({ "verdict": "unknown" }); + + let audit: serde_json::Value = match tokio::fs::read_to_string(dir.join(HOST_SECRETS_AUDIT_FILE)) + .await + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + { + Some(v) => v, + None => return unknown(), + }; + + let verdict = audit + .get("verdict") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + let mut out = serde_json::json!({ "verdict": verdict }); + + if let Some(checked_at) = audit.get("checked_at").and_then(|v| v.as_str()) { + out["checked_at"] = serde_json::json!(checked_at); + } + if let Some(evidence) = audit.get("evidence").and_then(|v| v.as_array()) { + out["evidence"] = serde_json::json!(evidence); + } + + // A rotation record only exists on a node an operator has actually + // rotated, so its absence is the normal case and is not reported. + if let Some(rotated_at) = tokio::fs::read_to_string(dir.join(HOST_KEY_ROTATION_FILE)) + .await + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()) + .and_then(|v| { + v.get("rotated_at") + .and_then(|r| r.as_str()) + .map(|s| s.to_string()) + }) + { + out["rotated_at"] = serde_json::json!(rotated_at); + } + + out +} + /// Where this node's TLS material lives, and how we are allowed to touch it. /// /// This type exists for two reasons. The first is that every step of the @@ -1306,3 +1381,88 @@ exec {OPENSSL_BIN} "$@" assert_eq!(private_key_mode(Some("garbage")), "600"); } } + +/// The `host_secrets` object in `system.stats` (phase 10 KEY-02 / audit F-03). +/// +/// These pin the contract that matters for a *cacheable, polled* method: the +/// absent and malformed cases must be indistinguishable from a plain "not +/// judged yet", never an error and never a panic. Every fleet node is in the +/// absent case until the OTA carrying the audit unit reaches it, so the absent +/// case is the common one, not the edge one. +#[cfg(test)] +mod host_secrets_tests { + use super::*; + + fn write(dir: &Path, name: &str, body: &str) { + std::fs::write(dir.join(name), body).unwrap(); + } + + #[tokio::test] + async fn verdict_is_unknown_when_the_audit_file_is_absent() { + let dir = tempfile::tempdir().unwrap(); + let v = host_secrets_status(dir.path()).await; + assert_eq!(v["verdict"], "unknown"); + assert!(v.get("checked_at").is_none()); + assert!(v.get("rotated_at").is_none()); + } + + #[tokio::test] + async fn verdict_is_unknown_when_the_audit_file_is_unparseable() { + let dir = tempfile::tempdir().unwrap(); + // A half-written file: the script writes to .tmp and renames, so this + // should not happen — which is exactly why the daemon must survive it + // if it ever does. + write(dir.path(), HOST_SECRETS_AUDIT_FILE, "{\"verdict\": \"shar"); + let v = host_secrets_status(dir.path()).await; + assert_eq!(v["verdict"], "unknown"); + } + + #[tokio::test] + async fn recorded_verdict_and_evidence_are_surfaced() { + let dir = tempfile::tempdir().unwrap(); + write( + dir.path(), + HOST_SECRETS_AUDIT_FILE, + r#"{ + "verdict": "shared", + "checked_at": "2026-08-02T12:00:00Z", + "evidence": ["shared: /etc/ssh/ssh_host_rsa_key mtime is older than the anchor"], + "ssh_host_key_fingerprints": ["/etc/ssh/ssh_host_rsa_key.pub: 3072 SHA256:abc"], + "tls_cert_sha256": "AA:BB:CC" + }"#, + ); + let v = host_secrets_status(dir.path()).await; + assert_eq!(v["verdict"], "shared"); + assert_eq!(v["checked_at"], "2026-08-02T12:00:00Z"); + assert_eq!(v["evidence"].as_array().unwrap().len(), 1); + + // The digests stay on disk. A payload polled every few seconds does not + // carry them, and a future edit that "helpfully" forwards the whole + // file should fail here. + assert!(v.get("ssh_host_key_fingerprints").is_none()); + assert!(v.get("tls_cert_sha256").is_none()); + } + + #[tokio::test] + async fn rotated_at_is_surfaced_only_when_a_rotation_was_recorded() { + let dir = tempfile::tempdir().unwrap(); + write( + dir.path(), + HOST_SECRETS_AUDIT_FILE, + r#"{"verdict":"per-node","checked_at":"2026-08-02T12:05:00Z","evidence":[]}"#, + ); + assert!(host_secrets_status(dir.path()) + .await + .get("rotated_at") + .is_none()); + + write( + dir.path(), + HOST_KEY_ROTATION_FILE, + r#"{"rotated_at":"2026-08-02T12:04:00Z","old_ssh_fingerprints":[],"old_tls_sha256":"AA"}"#, + ); + let v = host_secrets_status(dir.path()).await; + assert_eq!(v["verdict"], "per-node"); + assert_eq!(v["rotated_at"], "2026-08-02T12:04:00Z"); + } +} diff --git a/core/archipelago/src/bootstrap.rs b/core/archipelago/src/bootstrap.rs index 2739ff1c..14b9a637 100644 --- a/core/archipelago/src/bootstrap.rs +++ b/core/archipelago/src/bootstrap.rs @@ -358,7 +358,20 @@ async fn run_runtime_assets() -> Result { changed = true; } - for unit in ["archipelago-doctor.service", "archipelago-doctor.timer"] { + // archipelago-host-secrets-audit.service rides this same path (phase 10 + // KEY-02 / audit F-03). Nodes already in the field never received 10-03's + // ISO-build fix — the first-boot script is installed by the installer, not + // shipped by OTA — so a node that hit the old fail-open path is still on + // the SSH host key and TLS private key baked into its published ISO, and + // will never try again. This unit is how such a node reports itself. It is + // DETECT ONLY (D-06: detect-report-then-apply); rotation is operator-driven + // via `--apply --yes` and never fires from a unit. + let mut host_secrets_unit_installed = false; + for unit in [ + "archipelago-doctor.service", + "archipelago-doctor.timer", + "archipelago-host-secrets-audit.service", + ] { let src = configs.join(unit); if src.exists() { let src_s = src.to_string_lossy().to_string(); @@ -369,6 +382,9 @@ async fn run_runtime_assets() -> Result { if !status.success() { anyhow::bail!("install {} exited with {}", unit, status); } + if unit == "archipelago-host-secrets-audit.service" { + host_secrets_unit_installed = true; + } changed = true; } } @@ -411,6 +427,34 @@ async fn run_runtime_assets() -> Result { if changed { let _ = host_sudo(&["systemctl", "daemon-reload"]).await; + if host_secrets_unit_installed { + // `--now` on purpose: the verdict is the whole deliverable, and + // waiting for the next reboot to learn whether a node is running + // fleet-shared key material wastes the OTA that just delivered the + // means to find out. The unit is Type=oneshot, read-only and exits + // in milliseconds on a healthy node. Best-effort: a node that + // cannot enable it still boots, and the next OTA retries. + match host_sudo(&[ + "systemctl", + "enable", + "--now", + "archipelago-host-secrets-audit.service", + ]) + .await + { + Ok(status) if status.success() => { + info!("Enabled archipelago-host-secrets-audit.service from OTA runtime payload") + } + Ok(status) => tracing::warn!( + "enabling archipelago-host-secrets-audit.service exited with {}", + status + ), + Err(e) => tracing::warn!( + "failed to enable archipelago-host-secrets-audit.service: {}", + e + ), + } + } if nginx_src.exists() { match host_sudo(&["nginx", "-t"]).await { Ok(status) if status.success() => { diff --git a/image-recipe/configs/archipelago-host-secrets-audit.service b/image-recipe/configs/archipelago-host-secrets-audit.service new file mode 100644 index 00000000..a4a87ab6 --- /dev/null +++ b/image-recipe/configs/archipelago-host-secrets-audit.service @@ -0,0 +1,28 @@ +[Unit] +Description=Archipelago host-secret audit (are this node's SSH/TLS keys per-node?) +Documentation=file:///opt/archipelago/scripts/security/host-secrets-audit.sh +# Ordered after first-boot regeneration so a fresh node is judged on the keys +# it ends up with, not the ones it booted with. network.target because the +# fingerprints are only meaningful once the node has an identity to report as. +After=archipelago-first-boot-secrets.service network.target +ConditionPathExists=/opt/archipelago/scripts/security/host-secrets-audit.sh + +[Service] +Type=oneshot +# Reads /etc/ssh and /etc/archipelago/ssl and writes +# /var/lib/archipelago/host-secrets-audit.json, all of which are root-owned. +User=root +# DETECT ONLY. D-06 chose detect-report-then-apply +# (docs/security/KEY-02-FLEET-ROTATION.md): rotation is one-way and must never +# fire unattended across the fleet during an OTA. There is deliberately NO +# --apply here. Adding one is a decision, not a configuration change. +ExecStart=-/opt/archipelago/scripts/security/host-secrets-audit.sh --detect +# The leading `-` above: a failed audit must never fail a boot. The verdict is +# informational; a node that cannot be judged is still a node that must come up. +TimeoutStartSec=60 +RemainAfterExit=yes +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/scripts/security/host-secrets-audit.sh b/scripts/security/host-secrets-audit.sh new file mode 100755 index 00000000..f98be6e3 --- /dev/null +++ b/scripts/security/host-secrets-audit.sh @@ -0,0 +1,567 @@ +#!/bin/bash +# host-secrets-audit.sh — does THIS node run the fleet-shared, image-baked SSH +# host keys and TLS private key, or its own? +# +# Audit finding F-03 / phase 10 KEY-02, deployed half (decision D-06). +# +# 10-03 fixed the ISO builder: the rootfs no longer carries identity material +# and first-boot regeneration fails closed. Nodes already in the field never +# receive any of that — the first-boot script is installed by the installer, +# not shipped by OTA — and a node that hit the old fail-open path +# (`WARNING: TLS regeneration failed, keeping baked key` plus an unconditional +# `touch $MARKER`) is running key material that every downloader of that ISO +# also holds, and will never try again. This script is how such a node is +# found, and how it is fixed. +# +# ── SAFETY MODEL (D-06: detect-report-then-apply) ──────────────────────────── +# --detect (default) read-only. Writes only its own verdict file. Always +# exits 0: detection is informational and must never fail +# a boot. +# --apply prints what it WOULD do and exits 0 having touched +# nothing. A mistyped invocation is inert. +# --apply --yes rotates — and only if the detect pass returned `shared`. +# A node whose verdict is `per-node` cannot have its keys +# rotated by this script even by explicit command. +# +# The boot unit (image-recipe/configs/archipelago-host-secrets-audit.service) +# runs --detect only and contains no apply path. +# +# ── THIS IS A SANCTIONED KEY PRODUCER. THERE ARE NOW THREE. ───────────────── +# Do not unify them, and do not let their parameters drift apart: +# 1. gen_tls()/gen_ssh() in image-recipe/_archived/build-auto-installer-iso.sh +# — first boot, on the node, from the ISO. +# 2. TlsMaterial::regenerate() in core/archipelago/src/api/rpc/system/handlers.rs +# — TLS only, re-minted after `server.set-name` so the SAN matches. +# 3. rotate_tls()/rotate_ssh() below — deployed nodes, operator-driven, once. +# All three: rsa:2048, 3650 days, the same subject and the same SAN set, stage +# to `.new` siblings of the destination (same directory, so the final mv is a +# rename(2) and therefore atomic), parse both halves back AND prove they are a +# matching pair, then swap. A key from one generation beside a cert from +# another passes both individual parse checks and still breaks nginx. +# +# Producer 3 has to exist separately: producer 1 lives inside an ISO build +# script that is not present on a deployed node, and producer 2 does TLS only — +# nothing in the daemon has ever rotated an SSH host key. +# +# ── TEST SEAM ─────────────────────────────────────────────────────────────── +# HOST_SECRETS_ROOT prefixes every absolute path, exactly as +# FIRST_BOOT_SECRETS_ROOT does for 10-03's first-boot script. Unset in +# production the expansion is empty and behaviour is byte-identical; set, it is +# what makes tests/first-boot-secrets/rotation-tests.sh able to force a +# `shared` node into existence and drive a real rotation against it. +# +# Usage: +# host-secrets-audit.sh [--detect] [--json] [--quiet] +# host-secrets-audit.sh --apply [--yes] + +set -euo pipefail + +ROOT="${HOST_SECRETS_ROOT:-}" + +MARKER="$ROOT/var/lib/archipelago/.secrets-regenerated" +FAILED_RECORD="$ROOT/var/lib/archipelago/first-boot-secrets.failed" +FIRST_BOOT_LOG="$ROOT/var/log/archipelago-first-boot-secrets.log" +STRIPPED_MARKER="$ROOT/opt/archipelago/rootfs-identity-stripped" +LUKS_KEY="$ROOT/root/.luks-archipelago.key" +MACHINE_ID="$ROOT/etc/machine-id" +SSH_DIR="$ROOT/etc/ssh" +SSL_DIR="$ROOT/etc/archipelago/ssl" +TLS_KEY="$SSL_DIR/archipelago.key" +TLS_CRT="$SSL_DIR/archipelago.crt" +STATE_DIR="$ROOT/var/lib/archipelago" +AUDIT_JSON="$STATE_DIR/host-secrets-audit.json" +ROTATION_JSON="$STATE_DIR/host-key-rotation.json" +CONSOLE="$ROOT/dev/console" + +# A key regenerated at first boot carries an mtime within seconds of the +# anchor. A key baked into the image carries the image build time — days or +# weeks earlier. 300s absorbs the spread between the anchor being touched and +# the last key being written, without being wide enough to hide a build-time +# key. +ANCHOR_SKEW_SECONDS=300 + +MODE="detect" +CONFIRMED=0 +QUIET=0 +EMIT_JSON=0 + +while [ $# -gt 0 ]; do + case "$1" in + --detect) MODE="detect" ;; + --apply) MODE="apply" ;; + --yes) CONFIRMED=1 ;; + --json) EMIT_JSON=1 ;; + --quiet) QUIET=1 ;; + -h|--help) + sed -n '2,50p' "$0" + exit 0 + ;; + *) + echo "host-secrets-audit: unknown argument: $1" >&2 + exit 2 + ;; + esac + shift +done + +say() { [ "$QUIET" = 1 ] || echo "$*"; } + +# Evidence must name production paths, not the harness's temp root. +disp() { printf '%s' "${1#"$ROOT"}"; } + +json_escape() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'; } + +json_array() { + local first=1 item + printf '[' + for item in "$@"; do + [ "$first" = 1 ] || printf ', ' + first=0 + printf '"%s"' "$(json_escape "$item")" + done + printf ']' +} + +mtime_of() { stat -c %Y "$1" 2>/dev/null || true; } + +now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# ── Fingerprints ──────────────────────────────────────────────────────────── +# Fingerprints of PUBLIC keys are public data (T-10-35: accept). The private +# keys are never read by this script except by the generators that replace +# them. + +ssh_fingerprints() { + local f + for f in "$SSH_DIR"/ssh_host_*_key.pub; do + [ -e "$f" ] || continue + ssh-keygen -lf "$f" 2>/dev/null | sed "s|^|$(disp "$f"): |" || true + done +} + +tls_fingerprint() { + [ -s "$TLS_CRT" ] || return 0 + openssl x509 -in "$TLS_CRT" -noout -fingerprint -sha256 2>/dev/null \ + | sed 's/^.*=//' || true +} + +# ── Detection ─────────────────────────────────────────────────────────────── +# Outputs (globals, so --apply can reuse the pass without re-running it): +# VERDICT per-node | shared | fail-closed-missing | unknown +# EVIDENCE[] one string per signal that fired, each naming its file +# SSH_SHARED 1 when this node's SSH host keys are believed image-baked +# TLS_SHARED 1 when this node's TLS key is believed image-baked + +VERDICT="unknown" +EVIDENCE=() +SSH_SHARED=0 +TLS_SHARED=0 + +detect() { + VERDICT="unknown" + EVIDENCE=() + SSH_SHARED=0 + TLS_SHARED=0 + + local ssh_keys=() f + for f in "$SSH_DIR"/ssh_host_*_key; do + [ -e "$f" ] || continue + ssh_keys+=("$f") + done + + local have_ssh=0 have_tls=0 + [ "${#ssh_keys[@]}" -gt 0 ] && have_ssh=1 + [ -s "$TLS_KEY" ] && have_tls=1 + + # Signal 4 — rootfs provenance. Recorded on every run because it changes + # what missing material MEANS, and a reader of the JSON needs that context + # regardless of the verdict. + local stripped=0 + if [ -e "$STRIPPED_MARKER" ]; then + stripped=1 + EVIDENCE+=("provenance: $(disp "$STRIPPED_MARKER") present — this rootfs shipped identity-free (10-03 or later ISO)") + else + EVIDENCE+=("provenance: $(disp "$STRIPPED_MARKER") absent — this rootfs predates the 10-03 identity strip, so baked material is possible") + fi + + # Signal 3 — 10-03's durable failure record. + local failed_record=0 + if [ -e "$FAILED_RECORD" ]; then + failed_record=1 + EVIDENCE+=("failure record: $(disp "$FAILED_RECORD") present — first-boot generation reported failure and did not silently continue") + fi + + # ── Precedence step 1: is the material even there? ────────────────────── + # Missing material can never be SHARED material. On a stripped rootfs this + # is fail-closed working as designed; without the provenance marker it is + # still missing, and saying so is more honest than guessing. + if [ "$have_ssh" = 0 ] || [ "$have_tls" = 0 ]; then + [ "$have_ssh" = 0 ] && EVIDENCE+=("missing: no $(disp "$SSH_DIR")/ssh_host_*_key on this node") + [ "$have_tls" = 0 ] && EVIDENCE+=("missing: $(disp "$TLS_KEY") is absent or empty") + if [ "$stripped" = 0 ]; then + EVIDENCE+=("note: provenance marker absent, so 'fail-closed' is inferred from the absence itself, not from a build-time guarantee") + fi + VERDICT="fail-closed-missing" + return 0 + fi + + # ── Precedence step 2: the fail-open fingerprint ──────────────────────── + # `.secrets-regenerated` present AND a WARNING: line in the first-boot log + # is precisely what the pre-10-03 fail-open path produced (builder :1647, + # :1659, :1663). This is direct evidence, not an inference from timestamps, + # so it outranks the mtime signal — and the two WARNING strings name which + # class survived, so the rotation can be narrowed to it. + if [ -e "$MARKER" ] && [ -f "$FIRST_BOOT_LOG" ] && grep -q 'WARNING:' "$FIRST_BOOT_LOG" 2>/dev/null; then + local tls_warn=0 ssh_warn=0 + grep -q 'WARNING: TLS regeneration failed' "$FIRST_BOOT_LOG" 2>/dev/null && tls_warn=1 + grep -q 'WARNING: ssh-keygen -A failed' "$FIRST_BOOT_LOG" 2>/dev/null && ssh_warn=1 + if [ "$tls_warn" = 0 ] && [ "$ssh_warn" = 0 ]; then + # An unrecognised WARNING. Do not narrow on a guess. + tls_warn=1 + ssh_warn=1 + EVIDENCE+=("fail-open fingerprint: $(disp "$MARKER") present and $(disp "$FIRST_BOOT_LOG") carries an unrecognised WARNING: line — both key classes treated as shared") + else + EVIDENCE+=("fail-open fingerprint: $(disp "$MARKER") present and $(disp "$FIRST_BOOT_LOG") records the first-boot generator giving up and keeping the baked key") + fi + [ "$tls_warn" = 1 ] && { TLS_SHARED=1; EVIDENCE+=("shared: $(disp "$TLS_KEY") — the first-boot log says TLS regeneration failed and the baked key was kept"); } + [ "$ssh_warn" = 1 ] && { SSH_SHARED=1; EVIDENCE+=("shared: $(disp "$SSH_DIR")/ssh_host_*_key — the first-boot log says ssh-keygen -A failed and the baked host keys were kept"); } + VERDICT="shared" + return 0 + fi + + # ── Precedence step 3: the mtime anchor ───────────────────────────────── + local anchor="" anchor_kind="" + if [ -e "$MARKER" ]; then + anchor="$MARKER"; anchor_kind="first-boot regeneration marker" + elif [ -e "$LUKS_KEY" ]; then + anchor="$LUKS_KEY"; anchor_kind="LUKS key written by the installer with dd if=/dev/urandom" + elif [ -s "$MACHINE_ID" ]; then + anchor="$MACHINE_ID"; anchor_kind="machine-id, populated on this node's first boot" + fi + + if [ -z "$anchor" ]; then + EVIDENCE+=("no anchor: none of $(disp "$MARKER"), $(disp "$LUKS_KEY"), $(disp "$MACHINE_ID") is usable, so this node's first boot cannot be dated") + VERDICT="unknown" + return 0 + fi + + local anchor_mtime + anchor_mtime=$(mtime_of "$anchor") + if [ -z "$anchor_mtime" ]; then + EVIDENCE+=("no anchor: $(disp "$anchor") exists but could not be stat'd") + VERDICT="unknown" + return 0 + fi + EVIDENCE+=("anchor: $(disp "$anchor") ($anchor_kind), mtime $(date -u -d "@$anchor_mtime" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "$anchor_mtime")") + + older_than_anchor() { + local file="$1" m age + m=$(mtime_of "$file") + [ -n "$m" ] || return 1 + age=$((anchor_mtime - m)) + [ "$age" -gt "$ANCHOR_SKEW_SECONDS" ] + } + + for f in "${ssh_keys[@]}"; do + if older_than_anchor "$f"; then + SSH_SHARED=1 + EVIDENCE+=("shared: $(disp "$f") mtime is $(( anchor_mtime - $(mtime_of "$f") ))s older than the anchor (threshold ${ANCHOR_SKEW_SECONDS}s) — it came from the image, not from this node's first boot") + fi + done + if older_than_anchor "$TLS_KEY"; then + TLS_SHARED=1 + EVIDENCE+=("shared: $(disp "$TLS_KEY") mtime is $(( anchor_mtime - $(mtime_of "$TLS_KEY") ))s older than the anchor (threshold ${ANCHOR_SKEW_SECONDS}s) — it came from the image, not from this node's first boot") + fi + + if [ "$SSH_SHARED" = 1 ] || [ "$TLS_SHARED" = 1 ]; then + VERDICT="shared" + return 0 + fi + + # Never claim per-node while the node's own generator's last word was + # failure. A clean-looking mtime is not evidence that generation succeeded. + if [ "$failed_record" = 1 ]; then + EVIDENCE+=("withholding per-node: every key is newer than the anchor, but $(disp "$FAILED_RECORD") stands, so success is not established") + VERDICT="unknown" + return 0 + fi + + EVIDENCE+=("per-node: every SSH host key and the TLS key is newer than the anchor, so all of it was generated on this node") + VERDICT="per-node" + return 0 +} + +write_audit_json() { + local fps=() fp tls_fp + while IFS= read -r fp; do [ -n "$fp" ] && fps+=("$fp"); done < <(ssh_fingerprints) + tls_fp=$(tls_fingerprint) + + mkdir -p "$STATE_DIR" 2>/dev/null || true + local tmp="$AUDIT_JSON.tmp.$$" + { + printf '{\n' + printf ' "verdict": "%s",\n' "$(json_escape "$VERDICT")" + printf ' "checked_at": "%s",\n' "$(now_iso)" + printf ' "evidence": %s,\n' "$(json_array "${EVIDENCE[@]}")" + printf ' "ssh_host_key_fingerprints": %s,\n' "$(json_array "${fps[@]+"${fps[@]}"}")" + printf ' "tls_cert_sha256": "%s"\n' "$(json_escape "$tls_fp")" + printf '}\n' + } > "$tmp" + chmod 0644 "$tmp" + mv -f "$tmp" "$AUDIT_JSON" +} + +human_line() { + case "$VERDICT" in + per-node) + say "host-secrets: per-node — this node's SSH host keys and TLS key were generated here." ;; + shared) + say "host-secrets: SHARED — this node is running image-baked key material that every downloader of its ISO also holds. Rotate it: host-secrets-audit.sh --apply --yes" ;; + fail-closed-missing) + say "host-secrets: fail-closed-missing — key material is absent. Generation never succeeded; this node is not serving on a shared key, it is not serving." ;; + *) + say "host-secrets: unknown — not enough on-disk evidence to date this node's first boot." ;; + esac +} + +# ── Rotation ──────────────────────────────────────────────────────────────── + +# Same pair check as both other producers. Parsing each half back proves each +# is well-formed; it does NOT prove they belong together, and a key from one +# generation beside a cert from another passes both individual checks and then +# breaks nginx. +tls_pair_matches() { + local key="$1" crt="$2" kp cp + kp=$(openssl pkey -in "$key" -pubout 2>/dev/null) || return 1 + cp=$(openssl x509 -in "$crt" -noout -pubkey 2>/dev/null) || return 1 + [ -n "$kp" ] || return 1 + [ "$kp" = "$cp" ] +} + +TLS_STAGE_KEY="$SSL_DIR/archipelago.key.rotnew" +TLS_STAGE_CRT="$SSL_DIR/archipelago.crt.rotnew" +SSH_STAGE_DIR="" + +cleanup_staging() { + rm -f "$TLS_STAGE_KEY" "$TLS_STAGE_CRT" 2>/dev/null || true + [ -n "$SSH_STAGE_DIR" ] && rm -rf "$SSH_STAGE_DIR" 2>/dev/null || true +} + +# STAGE ONLY. Touches nothing live. Parameters kept identical to the other two +# producers — see the header. Do not let rsa:2048/3650 drift here alone. +stage_tls() { + local node_name + node_name=$(hostname 2>/dev/null || echo archipelago) + mkdir -p "$SSL_DIR" || return 1 + rm -f "$TLS_STAGE_KEY" "$TLS_STAGE_CRT" + openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ + -keyout "$TLS_STAGE_KEY" -out "$TLS_STAGE_CRT" \ + -subj "/C=XX/ST=Bitcoin/L=Node/O=Archipelago/CN=${node_name}" \ + -addext "subjectAltName=DNS:${node_name},DNS:${node_name}.local,DNS:archipelago,DNS:archipelago.local,DNS:localhost,IP:127.0.0.1" \ + >/dev/null 2>&1 || return 1 + [ -s "$TLS_STAGE_KEY" ] && [ -s "$TLS_STAGE_CRT" ] || return 1 + tls_pair_matches "$TLS_STAGE_KEY" "$TLS_STAGE_CRT" || return 1 + chmod 600 "$TLS_STAGE_KEY" + return 0 +} + +stage_ssh() { + SSH_STAGE_DIR=$(mktemp -d) || return 1 + mkdir -p "$SSH_STAGE_DIR/etc/ssh" + ssh-keygen -A -f "$SSH_STAGE_DIR" >/dev/null 2>&1 || return 1 + ls "$SSH_STAGE_DIR"/etc/ssh/ssh_host_*_key >/dev/null 2>&1 || return 1 + return 0 +} + +swap_tls() { + mv -f "$TLS_STAGE_KEY" "$TLS_KEY" || return 1 + mv -f "$TLS_STAGE_CRT" "$TLS_CRT" || return 1 + chmod 600 "$TLS_KEY" + return 0 +} + +# Overwrite in place rather than rm-then-mv. rm-then-mv opens a window — small, +# but real — in which the node has ZERO host keys on disk; sshd restarting into +# that window is unrecoverable on a remote machine. mv onto the existing path +# is a rename(2), so each key is replaced atomically and the directory is never +# empty. Only after every staged key has landed are leftovers of key types the +# new set does not include removed — leaving a stale ssh_host_dsa_key behind +# would leave shared material behind, which is the whole point of rotating. +swap_ssh() { + local f base staged=() + for f in "$SSH_STAGE_DIR"/etc/ssh/ssh_host_*; do + [ -e "$f" ] || continue + base=$(basename "$f") + mv -f "$f" "$SSH_DIR/$base" || return 1 + staged+=("$base") + done + [ "${#staged[@]}" -gt 0 ] || return 1 + for f in "$SSH_DIR"/ssh_host_*; do + [ -e "$f" ] || continue + base=$(basename "$f") + local keep=0 s + for s in "${staged[@]}"; do [ "$s" = "$base" ] && keep=1; done + [ "$keep" = 0 ] && rm -f "$f" + done + return 0 +} + +# reload, NEVER restart. THIS IS THE SINGLE MOST IMPORTANT LINE IN THIS FILE: +# a reload re-execs the sshd listener while already-forked session children +# keep running, so the operator's own SSH session survives its own rotation. A +# restart kills every session, and on a remote node reached only over SSH that +# is unrecoverable without physical console access. +reload_sshd() { + systemctl reload ssh >/dev/null 2>&1 || systemctl reload sshd >/dev/null 2>&1 || true +} + +reload_nginx() { + systemctl reload nginx >/dev/null 2>&1 || true +} + +shout() { + echo "$*" + [ -w "$CONSOLE" ] && printf '%s\n' "$*" > "$CONSOLE" 2>/dev/null || true +} + +write_rotation_json() { + # $1 = "pre" (old only) or "post" (old + new) + local phase="$1" + mkdir -p "$STATE_DIR" 2>/dev/null || true + local tmp="$ROTATION_JSON.tmp.$$" + { + printf '{\n' + printf ' "rotated_at": "%s",\n' "$(json_escape "$ROTATED_AT")" + printf ' "old_ssh_fingerprints": %s,\n' "$(json_array "${OLD_SSH_FPS[@]+"${OLD_SSH_FPS[@]}"}")" + if [ "$phase" = "pre" ]; then + printf ' "old_tls_sha256": "%s"\n' "$(json_escape "$OLD_TLS_FP")" + else + printf ' "old_tls_sha256": "%s",\n' "$(json_escape "$OLD_TLS_FP")" + printf ' "new_ssh_fingerprints": %s,\n' "$(json_array "${NEW_SSH_FPS[@]+"${NEW_SSH_FPS[@]}"}")" + printf ' "new_tls_sha256": "%s"\n' "$(json_escape "$NEW_TLS_FP")" + fi + printf '}\n' + } > "$tmp" + chmod 0644 "$tmp" + mv -f "$tmp" "$ROTATION_JSON" +} + +ROTATED_AT="" +OLD_SSH_FPS=() +OLD_TLS_FP="" +NEW_SSH_FPS=() +NEW_TLS_FP="" + +apply_rotation() { + trap cleanup_staging EXIT + + # Step 1 — stage EVERYTHING first. If any generation fails we abort before + # touching anything live and exit non-zero. A partial rotation is the + # failure mode that loses access, so there is no path here in which one + # class is swapped and the other has not been generated yet. + if [ "$TLS_SHARED" = 1 ]; then + if ! stage_tls; then + echo "host-secrets: ABORTED — could not generate a replacement TLS keypair. Nothing was changed." >&2 + cleanup_staging + return 1 + fi + say "staged: replacement TLS keypair" + fi + if [ "$SSH_SHARED" = 1 ]; then + if ! stage_ssh; then + echo "host-secrets: ABORTED — could not generate a replacement SSH host-key set. Nothing was changed." >&2 + cleanup_staging + return 1 + fi + say "staged: replacement SSH host-key set" + fi + + # Step 2 — record the OLD fingerprints BEFORE the swap. An operator who + # loses access anyway can still identify what changed; after the swap the + # old material is gone and unrecoverable. + ROTATED_AT=$(now_iso) + OLD_SSH_FPS=() + while IFS= read -r line; do [ -n "$line" ] && OLD_SSH_FPS+=("$line"); done < <(ssh_fingerprints) + OLD_TLS_FP=$(tls_fingerprint) + write_rotation_json pre + say "recorded old fingerprints to $(disp "$ROTATION_JSON") before touching anything" + + # Step 3 — TLS first. The web UI going down is recoverable over SSH; SSH + # going down on a remote node is not. Do the recoverable one first. + if [ "$TLS_SHARED" = 1 ]; then + if ! swap_tls; then + echo "host-secrets: TLS swap failed. SSH host keys were NOT touched." >&2 + cleanup_staging + return 1 + fi + reload_nginx + say "rotated: TLS keypair, nginx reloaded" + fi + + # Step 4 — SSH, then reload (never restart; see reload_sshd). + if [ "$SSH_SHARED" = 1 ]; then + if ! swap_ssh; then + echo "host-secrets: SSH swap failed partway. Check $(disp "$SSH_DIR") before disconnecting." >&2 + cleanup_staging + return 1 + fi + reload_sshd + say "rotated: SSH host keys, sshd reloaded (your current session is intentionally unaffected)" + fi + + # Step 5 — new fingerprints on the record, on stdout and on the console, + # then re-run detect so the verdict file reflects the post-rotation state. + NEW_SSH_FPS=() + while IFS= read -r line; do [ -n "$line" ] && NEW_SSH_FPS+=("$line"); done < <(ssh_fingerprints) + NEW_TLS_FP=$(tls_fingerprint) + write_rotation_json post + + shout "host-secrets: ROTATED $ROTATED_AT — new host key fingerprints for this node:" + for line in "${NEW_SSH_FPS[@]+"${NEW_SSH_FPS[@]}"}"; do shout " $line"; done + [ -n "$NEW_TLS_FP" ] && shout " TLS cert sha256: $NEW_TLS_FP" + shout "host-secrets: every known_hosts entry for this node is now stale. Update it against the fingerprints above, never by blindly accepting whatever is offered." + + detect + write_audit_json + human_line + + cleanup_staging + trap - EXIT + return 0 +} + +# ── Main ──────────────────────────────────────────────────────────────────── + +detect + +if [ "$MODE" = "detect" ]; then + write_audit_json + human_line + [ "$EMIT_JSON" = 1 ] && cat "$AUDIT_JSON" + exit 0 +fi + +# --apply. Deliberately writes NOTHING — not even its own verdict file — until +# --yes is given and a rotation actually starts. "Touches nothing" is a +# property worth being able to state without a footnote, and a footnote is what +# "except for one file it rewrites" would be. +if [ "$VERDICT" != "shared" ]; then + human_line + say "host-secrets: nothing to rotate (verdict is '$VERDICT', not 'shared'). No changes made." + exit 0 +fi + +if [ "$CONFIRMED" != 1 ]; then + say "host-secrets: DRY RUN — this node's verdict is 'shared'. Nothing has been changed." + say "" + say "Would rotate:" + [ "$TLS_SHARED" = 1 ] && say " - TLS keypair at $(disp "$TLS_KEY") (+ cert), then reload nginx" + [ "$SSH_SHARED" = 1 ] && say " - every $(disp "$SSH_DIR")/ssh_host_*_key, then reload (not restart) sshd" + say "" + say "Old fingerprints would be written to $(disp "$ROTATION_JSON") before the swap." + say "This is ONE-WAY: every known_hosts entry for this node breaks and the old key is destroyed." + say "Re-run with --yes from a session you are willing to lose." + exit 0 +fi + +apply_rotation diff --git a/tests/first-boot-secrets/rotation-tests.sh b/tests/first-boot-secrets/rotation-tests.sh new file mode 100755 index 00000000..31d2b971 --- /dev/null +++ b/tests/first-boot-secrets/rotation-tests.sh @@ -0,0 +1,458 @@ +#!/bin/bash +# Regression harness for scripts/security/host-secrets-audit.sh +# (audit finding F-03, phase 10 / KEY-02, deployed half — decision D-06). +# +# Sibling of run-tests.sh, which covers the ISO-build half. That one proves a +# node never SERVES on a key it did not generate. This one proves a node can +# TELL you whether it is already doing so, and can be fixed without losing the +# operator's session in the middle. +# +# The properties under test are mostly negative or ordering properties, and +# neither kind is assertable against real key material on a real node: +# +# - "--apply without --yes touches nothing" needs a tree to diff +# - "old fingerprints are recorded BEFORE the swap" needs the swap observed +# - "a failed generation leaves the live keys byte-identical" needs failure +# to be forcible +# - "a node with no anchor is reported unknown, never per-node" needs a node +# with no anchor to exist +# +# So the generators are stubbed and the script is driven against temp roots +# through its HOST_SECRETS_ROOT seam — the same move 10-03's harness makes with +# FIRST_BOOT_SECRETS_ROOT, and the same reason. +# +# Usage: bash tests/first-boot-secrets/rotation-tests.sh +# Exit 0 only if all seven cases PASS. + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT="$REPO/scripts/security/host-secrets-audit.sh" + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +PASS_COUNT=0 +FAIL_COUNT=0 +ok() { echo "PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); } +bad() { echo "FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); } + +[ -f "$SCRIPT" ] || { echo "FAIL: script not found at $SCRIPT"; exit 1; } +[ -x "$SCRIPT" ] || { echo "FAIL: $SCRIPT is not executable"; exit 1; } +if bash -n "$SCRIPT"; then + echo "host-secrets-audit.sh: $(wc -l < "$SCRIPT") lines; bash -n clean" +else + echo "FAIL: host-secrets-audit.sh does not parse"; exit 1 +fi + +# A rotation script that restarts sshd instead of reloading it disconnects the +# operator on a remote node with no console. Checked here rather than left to +# review, because it is a one-word edit away at all times. +if grep -qn 'systemctl restart ssh' "$SCRIPT"; then + echo "FAIL: host-secrets-audit.sh contains 'systemctl restart ssh' — a restart kills the operator's own session" + exit 1 +fi +grep -q 'systemctl reload ssh' "$SCRIPT" || { echo "FAIL: no 'systemctl reload ssh' in the script"; exit 1; } +echo "sshd handling: reload present, restart absent" + +# ── Stubs ───────────────────────────────────────────────────────────────── +# Prepended to PATH so openssl / ssh-keygen / systemctl calls land here. +# STUB_OPENSSL_MODE ok | fail (affects `req` only, so a failed +# generation is never mistaken for a +# failed validation) +# STUB_SSHKEYGEN_MODE ok | fail (affects `-A` only, so `-lf` keeps +# working and old fingerprints can still +# be read on the abort path) +# STUB_COUNTER_DIR where generation counters live +# STUB_SYSTEMCTL_LOG file the systemctl stub appends to +make_stubs() { + local dir="$1" + mkdir -p "$dir" + + cat > "$dir/openssl" <<'STUB' +#!/bin/bash +sub="${1:-}" +case "$sub" in + pkey) + f=""; pubout=0 + while [ $# -gt 0 ]; do + case "$1" in + -in) f="$2"; shift 2 ;; + -pubout) pubout=1; shift ;; + *) shift ;; + esac + done + [ -n "$f" ] && [ -s "$f" ] || exit 1 + p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1 + [ "$pubout" = 1 ] && printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p" + exit 0 + ;; + x509) + f=""; want_pub=0; want_fp=0 + while [ $# -gt 0 ]; do + case "$1" in + -in) f="$2"; shift 2 ;; + -pubkey) want_pub=1; shift ;; + -fingerprint) want_fp=1; shift ;; + *) shift ;; + esac + done + [ -n "$f" ] && [ -s "$f" ] || exit 1 + if [ "$want_pub" = 1 ]; then + p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1 + printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p" + fi + if [ "$want_fp" = 1 ]; then + # Content-derived, so a rotated cert has a different digest and the + # old/new comparison in case 7 means something. + printf 'sha256 Fingerprint=%s\n' "$(sha256sum "$f" | cut -c1-32)" + fi + exit 0 + ;; +esac + +[ "$sub" = "req" ] || exit 0 +c="${STUB_COUNTER_DIR:-/tmp}/openssl-req.count" +n=$(cat "$c" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$c" +[ "${STUB_OPENSSL_MODE:-ok}" = "fail" ] && exit 1 +keyout=""; out="" +while [ $# -gt 0 ]; do + case "$1" in + -keyout) keyout="$2"; shift 2 ;; + -out) out="$2"; shift 2 ;; + *) shift ;; + esac +done +# Both halves carry the same generation id, so the script's pair check passes +# for a real generation and would fail for a mismatched pair. +[ -n "$keyout" ] && printf -- '-----BEGIN PRIVATE KEY-----\nrotated\nSTUB_PUB=%s\n-----END PRIVATE KEY-----\n' "$n" > "$keyout" +[ -n "$out" ] && printf -- '-----BEGIN CERTIFICATE-----\nrotated\nSTUB_PUB=%s\n-----END CERTIFICATE-----\n' "$n" > "$out" +exit 0 +STUB + + cat > "$dir/ssh-keygen" <<'STUB' +#!/bin/bash +# -lf -> a fingerprint derived from the file's contents, so a rotated +# key necessarily fingerprints differently. +# -A -f -> a fresh host-key set, each generation distinct. +if [ "${1:-}" = "-lf" ]; then + f="${2:-}" + [ -s "$f" ] || exit 1 + printf '256 SHA256:%s %s (ED25519)\n' "$(sha256sum "$f" | cut -c1-24)" "stub@archipelago" + exit 0 +fi + +c="${STUB_COUNTER_DIR:-/tmp}/ssh-keygen.count" +n=$(cat "$c" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$c" +[ "${STUB_SSHKEYGEN_MODE:-ok}" = "fail" ] && exit 1 + +root="" +while [ $# -gt 0 ]; do + case "$1" in + -f) root="$2"; shift 2 ;; + *) shift ;; + esac +done +[ -n "$root" ] || exit 1 +mkdir -p "$root/etc/ssh" +for t in rsa ecdsa ed25519; do + printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nrotated-gen%s-%s\n' "$n" "$t" > "$root/etc/ssh/ssh_host_${t}_key" + printf -- 'ssh-%s AAAArotated-gen%s stub@archipelago\n' "$t" "$n" > "$root/etc/ssh/ssh_host_${t}_key.pub" +done +exit 0 +STUB + + cat > "$dir/systemctl" <<'STUB' +#!/bin/bash +# Records each call ALONGSIDE whether the rotation record already exists at +# that moment. That is how "old fingerprints were written BEFORE the swap" +# becomes an observable ordering fact rather than an inference from content: +# the first reload happens after the first swap, so the record must already be +# on disk by then. +if [ -n "${STUB_SYSTEMCTL_LOG:-}" ]; then + rj="no" + [ -f "${HOST_SECRETS_ROOT:-}/var/lib/archipelago/host-key-rotation.json" ] && rj="yes" + echo "$* rotjson=$rj" >> "$STUB_SYSTEMCTL_LOG" +fi +exit 0 +STUB + + chmod +x "$dir"/openssl "$dir"/ssh-keygen "$dir"/systemctl +} + +STUBS="$WORK/stubs" +make_stubs "$STUBS" + +# ── Tree builders ───────────────────────────────────────────────────────── +# T0 is a fixed "this node's first boot" instant. Everything is dated relative +# to it so the cases read as timelines rather than as magic numbers. +T0=$(date -u -d '2026-06-01 12:00:00' +%s) + +at() { date -u -d "@$1" '+%Y-%m-%d %H:%M:%S'; } + +new_root() { + local name="$1" + local r="$WORK/root-$name" + rm -rf "$r" + mkdir -p "$r/var/lib/archipelago" "$r/var/log" "$r/etc/ssh" \ + "$r/etc/archipelago/ssl" "$r/opt/archipelago" "$r/root" "$r/dev" + printf '%s' "$r" +} + +# Host keys + TLS material dated at . +put_material() { + local r="$1" when="$2" tag="${3:-baked}" + local t + for t in rsa ecdsa ed25519; do + printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\n%s-%s\n' "$tag" "$t" > "$r/etc/ssh/ssh_host_${t}_key" + printf -- 'ssh-%s AAAA%s stub@archipelago\n' "$t" "$tag" > "$r/etc/ssh/ssh_host_${t}_key.pub" + done + printf -- '-----BEGIN PRIVATE KEY-----\n%s\nSTUB_PUB=0\n-----END PRIVATE KEY-----\n' "$tag" > "$r/etc/archipelago/ssl/archipelago.key" + printf -- '-----BEGIN CERTIFICATE-----\n%s\nSTUB_PUB=0\n-----END CERTIFICATE-----\n' "$tag" > "$r/etc/archipelago/ssl/archipelago.crt" + touch -d "$(at "$when")" "$r"/etc/ssh/ssh_host_* \ + "$r/etc/archipelago/ssl/archipelago.key" "$r/etc/archipelago/ssl/archipelago.crt" +} + +put_anchor() { + local r="$1" when="$2" + : > "$r/var/lib/archipelago/.secrets-regenerated" + touch -d "$(at "$when")" "$r/var/lib/archipelago/.secrets-regenerated" +} + +CASE_RC=0 +CASE_OUT="" +CASE_ERR="" +run_script() { + local root="$1" name="$2"; shift 2 + CASE_OUT="$WORK/$name.out"; CASE_ERR="$WORK/$name.err" + mkdir -p "$WORK/counters-$name" + set +e + env PATH="$STUBS:$PATH" \ + HOST_SECRETS_ROOT="$root" \ + STUB_OPENSSL_MODE="${STUB_OPENSSL_MODE:-ok}" \ + STUB_SSHKEYGEN_MODE="${STUB_SSHKEYGEN_MODE:-ok}" \ + STUB_COUNTER_DIR="$WORK/counters-$name" \ + STUB_SYSTEMCTL_LOG="$WORK/$name.systemctl" \ + bash "$SCRIPT" "$@" > "$CASE_OUT" 2> "$CASE_ERR" + CASE_RC=$? + set -e +} + +verdict_of() { sed -n 's/.*"verdict": "\([^"]*\)".*/\1/p' "$1" | head -1; } + +# A content+mtime+mode snapshot of everything except the script's own outputs, +# so "touched nothing" can be asserted as a whole-tree fact. +snapshot_tree() { + local r="$1" + ( cd "$r" && find . -path ./var/lib/archipelago -prune -o \( -type f -o -type l \) -print0 \ + | sort -z | xargs -0 -r stat -c '%n %s %Y %a' ) 2>/dev/null + ( cd "$r" && find . -path ./var/lib/archipelago -prune -o -type f -print0 \ + | sort -z | xargs -0 -r sha256sum ) 2>/dev/null +} + +# ── Case 1: keys newer than the anchor -> per-node ──────────────────────── +R=$(new_root per-node) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 + 5))" fresh +BEFORE=$(snapshot_tree "$R") +run_script "$R" per-node --detect +c="" +[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero" +J="$R/var/lib/archipelago/host-secrets-audit.json" +[ -f "$J" ] || c="$c no-json" +[ "$(verdict_of "$J" 2>/dev/null)" = "per-node" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)" +grep -q '"checked_at"' "$J" 2>/dev/null || c="$c no-checked-at" +grep -q '"ssh_host_key_fingerprints"' "$J" 2>/dev/null || c="$c no-fingerprints" +[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c detect-modified-the-tree" +if [ -z "$c" ]; then + ok "host keys newer than the anchor -> per-node, JSON written, nothing else changed" +else + bad "host keys newer than the anchor ->$c"; echo " root=$R rc=$CASE_RC" +fi + +# ── Case 2: keys 30 days older than the anchor -> shared ───────────────── +R=$(new_root shared-mtime) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 - 30 * 86400))" baked +run_script "$R" shared-mtime --detect +c="" +J="$R/var/lib/archipelago/host-secrets-audit.json" +[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero" +[ "$(verdict_of "$J" 2>/dev/null)" = "shared" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)" +grep -q 'ssh_host_rsa_key mtime is' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-ssh-key" +grep -q 'archipelago.key mtime is' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-tls-key" +grep -qi 'SHARED' "$WORK/shared-mtime.out" || c="$c no-human-verdict-on-stdout" +if [ -z "$c" ]; then + ok "host keys 30 days older than the anchor -> shared, evidence names both key classes" +else + bad "host keys 30 days older than the anchor ->$c"; echo " root=$R rc=$CASE_RC" +fi + +# ── Case 3: the fail-open fingerprint -> shared on direct evidence ─────── +# Deliberately dated so the mtime signal says per-node. If this case passes it +# is because signal 2 fired, not because the timestamps happened to agree. +R=$(new_root fail-open) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 + 5))" kept-baked +{ + echo "Mon Jun 1 12:00:00 UTC 2026: regenerating per-device secrets" + echo "Mon Jun 1 12:00:01 UTC 2026: WARNING: TLS regeneration failed, keeping baked key" + echo "Mon Jun 1 12:00:02 UTC 2026: WARNING: ssh-keygen -A failed, keeping baked host keys" +} > "$R/var/log/archipelago-first-boot-secrets.log" +run_script "$R" fail-open --detect +c="" +J="$R/var/lib/archipelago/host-secrets-audit.json" +[ "$(verdict_of "$J" 2>/dev/null)" = "shared" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)" +grep -q '/var/lib/archipelago/.secrets-regenerated' "$J" 2>/dev/null || c="$c evidence-missing-marker-signal" +grep -q '/var/log/archipelago-first-boot-secrets.log' "$J" 2>/dev/null || c="$c evidence-missing-log-signal" +grep -q 'fail-open fingerprint' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-combination" +if [ -z "$c" ]; then + ok "marker plus a WARNING: line -> shared, with both signals in evidence, despite per-node mtimes" +else + bad "marker plus a WARNING: line ->$c"; echo " root=$R rc=$CASE_RC" +fi + +# ── Case 4: stripped rootfs, no host keys -> fail-closed-missing ───────── +# Not `shared`. The distinction is the whole reason signal 4 exists: on a +# 10-03-or-later node an absent key means generation never succeeded, which is +# fail-closed working, and rotating is not the remedy. +R=$(new_root stripped) +put_anchor "$R" "$T0" +printf 'F-03 identity strip\n' > "$R/opt/archipelago/rootfs-identity-stripped" +run_script "$R" stripped --detect +c="" +J="$R/var/lib/archipelago/host-secrets-audit.json" +[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero" +v=$(verdict_of "$J" 2>/dev/null) +[ "$v" = "fail-closed-missing" ] || c="$c verdict=$v" +[ "$v" = "shared" ] && c="$c CALLED-MISSING-MATERIAL-SHARED" +grep -q 'rootfs-identity-stripped' "$J" 2>/dev/null || c="$c evidence-missing-provenance" +if [ -z "$c" ]; then + ok "identity-stripped rootfs with no host keys -> fail-closed-missing, not shared" +else + bad "identity-stripped rootfs with no host keys ->$c"; echo " root=$R rc=$CASE_RC" +fi + +# ── Case 5: no anchor at all -> unknown ────────────────────────────────── +# The signal that must never be guessed. An absent anchor is absence of +# evidence, and reporting per-node here would leave an exposed node looking +# clean (T-10-37). +R=$(new_root no-anchor) +put_material "$R" "$T0" whatever +: > "$R/etc/machine-id" # present but empty, as on a stripped rootfs +run_script "$R" no-anchor --detect +c="" +J="$R/var/lib/archipelago/host-secrets-audit.json" +v=$(verdict_of "$J" 2>/dev/null) +[ "$v" = "unknown" ] || c="$c verdict=$v" +[ "$v" = "per-node" ] && c="$c CLAIMED-PER-NODE-WITHOUT-EVIDENCE" +grep -q 'no anchor' "$J" 2>/dev/null || c="$c evidence-does-not-explain-why" +if [ -z "$c" ]; then + ok "no first-boot anchor -> unknown, never per-node" +else + bad "no first-boot anchor ->$c"; echo " root=$R rc=$CASE_RC" +fi + +# ── Case 6: --apply without --yes is inert ─────────────────────────────── +R=$(new_root dry-run) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 - 30 * 86400))" baked +BEFORE=$(snapshot_tree "$R") +BEFORE_STATE=$(ls -A "$R/var/lib/archipelago") +run_script "$R" dry-run --apply +c="" +[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero" +[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c TREE-CHANGED" +[ "$(ls -A "$R/var/lib/archipelago")" = "$BEFORE_STATE" ] || c="$c STATE-DIR-CHANGED" +[ -f "$R/var/lib/archipelago/host-key-rotation.json" ] && c="$c rotation-record-written" +ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover" +grep -qi 'DRY RUN' "$WORK/dry-run.out" || c="$c no-dry-run-notice" +grep -qi 'one-way' "$WORK/dry-run.out" || c="$c does-not-warn-that-it-is-one-way" +if [ -z "$c" ]; then + ok "--apply without --yes -> exits 0 and not one byte of the tree changes" +else + bad "--apply without --yes ->$c"; echo " root=$R rc=$CASE_RC" + # `diff` exits 1 when it finds differences, which under `set -o pipefail` + # would abort the run before the summary — i.e. a failing case would hide the + # other cases. Report and carry on. + diff <(echo "$BEFORE") <(snapshot_tree "$R") | head -10 || true +fi + +# ── Case 7a: --apply --yes rotates, recording old fingerprints first ───── +R=$(new_root rotate) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 - 30 * 86400))" baked +OLD_SSH_SHA=$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key" | cut -d' ' -f1) +OLD_TLS_SHA=$(sha256sum "$R/etc/archipelago/ssl/archipelago.key" | cut -d' ' -f1) +# The fingerprint the old key WOULD produce, computed independently of the +# script, so the "old" half of the record is checked against an outside source. +OLD_FP_EXPECT=$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key.pub" | cut -c1-24) +run_script "$R" rotate --apply --yes +c="" +ROT="$R/var/lib/archipelago/host-key-rotation.json" +J="$R/var/lib/archipelago/host-secrets-audit.json" +[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero" +[ -f "$ROT" ] || c="$c no-rotation-record" +grep -q '"old_ssh_fingerprints"' "$ROT" 2>/dev/null || c="$c no-old-ssh-fingerprints" +grep -q '"new_ssh_fingerprints"' "$ROT" 2>/dev/null || c="$c no-new-ssh-fingerprints" +grep -q '"old_tls_sha256"' "$ROT" 2>/dev/null || c="$c no-old-tls" +grep -q '"new_tls_sha256"' "$ROT" 2>/dev/null || c="$c no-new-tls" +grep -q "$OLD_FP_EXPECT" "$ROT" 2>/dev/null || c="$c old-fingerprint-does-not-match-the-pre-rotation-key" +# ORDERING: the first systemctl call happens after the first swap, so the +# record must already exist by then. +FIRST_SYSTEMCTL=$(head -1 "$WORK/rotate.systemctl" 2>/dev/null || echo "") +case "$FIRST_SYSTEMCTL" in + *rotjson=yes) ;; + "") c="$c no-service-reload-happened" ;; + *) c="$c OLD-FINGERPRINTS-NOT-RECORDED-BEFORE-THE-SWAP[$FIRST_SYSTEMCTL]" ;; +esac +grep -q 'reload ssh' "$WORK/rotate.systemctl" 2>/dev/null || c="$c sshd-not-reloaded" +grep -q 'restart ssh' "$WORK/rotate.systemctl" 2>/dev/null && c="$c SSHD-RESTARTED" +grep -q 'reload nginx' "$WORK/rotate.systemctl" 2>/dev/null || c="$c nginx-not-reloaded" +# Material actually replaced. +[ "$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key" | cut -d' ' -f1)" = "$OLD_SSH_SHA" ] && c="$c ssh-key-not-replaced" +[ "$(sha256sum "$R/etc/archipelago/ssl/archipelago.key" | cut -d' ' -f1)" = "$OLD_TLS_SHA" ] && c="$c tls-key-not-replaced" +ls "$R"/etc/ssh/ssh_host_*_key >/dev/null 2>&1 || c="$c NO-HOST-KEYS-LEFT" +ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover" +# The verdict file must reflect the post-rotation state, not the pre-rotation one. +[ "$(verdict_of "$J" 2>/dev/null)" = "per-node" ] || c="$c post-rotation-verdict=$(verdict_of "$J" 2>/dev/null)" +if [ -z "$c" ]; then + ok "--apply --yes -> old fingerprints recorded BEFORE the swap, keys replaced, sshd reloaded not restarted, verdict re-derived" +else + bad "--apply --yes ->$c"; echo " root=$R rc=$CASE_RC" + echo " stderr: $(head -c 300 "$WORK/rotate.err" 2>/dev/null)" +fi + +# ── Case 7b: a failed generation aborts before touching anything live ──── +# The failure mode that loses a remote node forever is a rotation that gets +# halfway. Force the SSH generator to fail after the TLS generator succeeded — +# the exact interleaving in which a naive implementation has already swapped +# the TLS pair — and require the live material to be byte-identical. +R=$(new_root abort) +put_anchor "$R" "$T0" +put_material "$R" "$((T0 - 30 * 86400))" baked +BEFORE=$(snapshot_tree "$R") +STUB_SSHKEYGEN_MODE=fail run_script "$R" abort --apply --yes +c="" +[ "$CASE_RC" -ne 0 ] || c="$c exit-zero-on-aborted-rotation" +[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c LIVE-MATERIAL-CHANGED-ON-AN-ABORTED-ROTATION" +[ -f "$R/var/lib/archipelago/host-key-rotation.json" ] && c="$c rotation-record-written-for-a-rotation-that-never-happened" +ls "$R"/etc/ssh/ssh_host_*_key >/dev/null 2>&1 || c="$c NO-HOST-KEYS-LEFT" +ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover" +grep -qi 'ABORTED' "$WORK/abort.err" || c="$c no-loud-abort-on-stderr" +[ -s "$WORK/abort.systemctl" ] && c="$c reloaded-a-service-during-an-aborted-rotation" +if [ -z "$c" ]; then + ok "generation failure -> aborts before any swap; live keys byte-identical, no service reloaded" +else + bad "generation failure ->$c"; echo " root=$R rc=$CASE_RC" + # `diff` exits 1 when it finds differences, which under `set -o pipefail` + # would abort the run before the summary — i.e. a failing case would hide the + # other cases. Report and carry on. + diff <(echo "$BEFORE") <(snapshot_tree "$R") | head -10 || true + echo " stderr: $(head -c 300 "$WORK/abort.err" 2>/dev/null)" +fi + +echo "" +echo "──────── host-secrets-audit summary ────────" +echo "passed: $PASS_COUNT failed: $FAIL_COUNT" +[ "$FAIL_COUNT" -eq 0 ]