fix: preserve apps and diagnostics when first-boot setup retries

docs/release-1.9.0-acceptance.md

@@ -784,3 +784,34 @@ by explicitly mounting the disposable resolver fixture.
 Qualified web image169e59da is built from157c9ec0; backend2722fa29 frome6e46a14.
 Public demo deployment remains scheduled after release with rollback. RC2 raw
 ISO assembly is running; no boot/install or final OTA pass is claimed yet.
+
+### RC2 actual installation and first-boot retry correction
+
+RC2 passed mounted payload checks and completed a full UEFI installation to an
+80GiB disposable NVMe disk with encrypted data. Installed backend560, both
+security helpers, dashboardc18, AIUI415 and APK54 match qualified bytes. After
+boot from the installed disk, SSH, dashboard200 and backend health/version pass.
+Actual installed nginx passes32 IPv4/IPv6 public-source denial requests including
+unknown Host/SNI and forged forwarding headers (`/tmp/archy-190-vm-guard-test.log`).
+The VM's EDAC hardware initialization service reports unsupported virtual
+hardware; this is not claimed as physical ECC/EDAC acceptance.
+
+Actual first boot exposed `log: command not found` in the unbundled setup: the
+logger was declared below that path's early exit. Moving it before first use
+restores diagnostics. Retry testing also demonstrated that unconditional
+`podman system migrate` stops existing apps and races manager reconciliation.
+The unbundled path changes no ID mappings and no longer migrates; bundled setup
+only migrates when it actually adds mappings. Podman documents this stop behavior
+in its [migration reference](https://docs.podman.io/en/latest/markdown/podman-system-migrate.1.html).
+
+Corrected actual-VM retry preserves container IDs/start times and produces clean
+logs: `/tmp/archy-190-vm-firstboot-logging-fix-2.log`. Executable isolated retry
+regression passes twice with stored-secret preservation and fails against the
+prior script. Added to release harness. RC2 must not be published: rebuild the
+ISO with this script and qualify its boot/install path before signing. The OTA
+backend/frontend payloads are unchanged by this installer-only correction.
+
+Demo archive33ec4111…6fae8 matches after private server transfer; both tested image
+IDs loaded successfully without changing running demo containers. Clearing only
+unused build cache recovered1.743GB; no additional historical ISO, image, volume
+or application data was deleted. Final publication capacity must be rechecked.

scripts/first-boot-containers.sh

@@ -16,6 +16,8 @@
 # DO NOT split until tested on the build server — this is critical infrastructure.
 #
 LOG="/var/log/archipelago-first-boot.log"
+# The unbundled path exits early; diagnostics must exist before either path.
+log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG"; }
 
 # Source pinned image versions (single source of truth)
 # ISO copies to scripts/ subdir; also check the direct path for manual installs
@@ -111,7 +113,8 @@ if [ -f "$UNBUNDLED_MARKER" ]; then
     # Podman prerequisites
     loginctl enable-linger archipelago 2>/dev/null || true
     DOCKER="runuser -u archipelago -- env XDG_RUNTIME_DIR=/run/user/$(id -u archipelago) podman"
-    $DOCKER system migrate 2>/dev/null || true
+    # No ID mappings changed here. system migrate would stop existing apps
+    # on a retry and race the backend's reconciliation.
 
     # Ensure archy-net exists
     $DOCKER network create archy-net 2>/dev/null || true
@@ -245,8 +248,6 @@ fi
 # trying to reach bitcoin-core's RPC on the host (LND, ElectrumX, etc).
 ADD_HOST_FLAG="--add-host=host.containers.internal:host-gateway"
 
-log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG"; }
-
 # Ensure Tor is running for hidden services (LND connect, Electrumx, etc.)
 if ! systemctl is-active tor >/dev/null 2>&1; then
   log "Starting Tor..."
@@ -491,9 +492,9 @@ grep -q "^archipelago:" /etc/subuid 2>/dev/null || {
     echo "archipelago:100000:65536" >> /etc/subuid
     echo "archipelago:100000:65536" >> /etc/subgid
     log "  subuid/subgid configured"
+    # Only refresh the namespace when mappings actually changed.
+    $DOCKER system migrate 2>/dev/null || true
 }
-# Apply podman migrations after subuid/subgid changes (per official tutorial)
-$DOCKER system migrate 2>/dev/null || true
 # Ensure /etc/hosts is readable (rootless podman needs it)
 chmod 644 /etc/hosts 2>/dev/null
 

tests/regression/first-boot-retry.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Execute unbundled first-boot retry with temporary paths and fake system commands."""
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+
+root = Path(__file__).resolve().parents[2]
+source = (root / 'scripts/first-boot-containers.sh').read_text()
+# Exercise the real early-exit path, excluding the unrelated bundled installer.
+source = source.split('TARGET_IP=$(hostname -I', 1)[0]
+with tempfile.TemporaryDirectory(prefix='archy-firstboot-retry-') as directory:
+    tmp = Path(directory)
+    for original, replacement in [('/var/lib/archipelago', tmp / 'data'),
+                                   ('/opt/archipelago', tmp / 'opt'),
+                                   ('/home/archipelago', tmp / 'home'),
+                                   ('/var/log', tmp / 'logs')]:
+        source = source.replace(original, str(replacement))
+    for folder in ['data/secrets', 'data/wireguard', 'opt', 'logs']:
+        (tmp / folder).mkdir(parents=True, exist_ok=True)
+    (tmp / 'opt/.unbundled').touch()
+    for name in ['bitcoin-rpc-password', 'mempool-db-password',
+                 'btcpay-db-password', 'mysql-root-db-password']:
+        (tmp / 'data/secrets' / name).write_text('fixture-preserved')
+    (tmp / 'data/wireguard/private.key').write_text('fixture-preserved')
+    candidate = tmp / 'firstboot.sh'
+    candidate.write_text(source)
+    # Functions take precedence over host commands. Unexpected privileged work
+    # fails, and no real Podman/systemd command can run through these stubs.
+    harness = r'''
+id() { if [ "$*" = '-u' ]; then echo 0; else echo 1000; fi; }
+chown() { :; }
+loginctl() { :; }
+modprobe() { :; }
+systemctl() { :; }
+ufw() { echo 'Status: inactive'; }
+openssl() { echo 'unexpected credential rotation' >&2; return 99; }
+runuser() {
+  printf '%s\n' "$*" >> "$TRACE"
+  case "$*" in
+    *'podman system migrate'*) return 99 ;;
+    *'podman container exists filebrowser'*) return 0 ;;
+    *'podman ps -a'*) echo fedimint-clientd; return 0 ;;
+    *'podman network create archy-net'*) return 0 ;;
+    *) echo 'unexpected system command' >&2; return 99 ;;
+  esac
+}
+export -f id chown loginctl modprobe systemctl ufw openssl runuser
+bash "$CANDIDATE"
+'''
+    before = {str(p): p.read_bytes() for p in (tmp / 'data').rglob('*') if p.is_file()}
+    for attempt in range(2):
+        trace = tmp / f'trace-{attempt}'
+        result = subprocess.run(['bash', '-c', harness], capture_output=True,
+                                text=True, timeout=15, env=os.environ | {
+                                    'CANDIDATE': str(candidate), 'TRACE': str(trace),
+                                    'ARCHY_REGISTRY': 'fixture.invalid',
+                                    'BITCOIN_KNOTS_IMAGE': 'fixture.invalid/bitcoin',
+                                })
+        assert result.returncode == 0, result.stderr
+        assert not result.stderr, result.stderr
+        assert 'Unbundled first-boot complete' in result.stdout, result.stdout
+        assert 'system migrate' not in trace.read_text(), trace.read_text()
+        assert all(Path(p).read_bytes() == content for p, content in before.items())
+    print('PASS repeated unbundled setup logs correctly without stopping apps or rotating stored secrets')

tests/release/run.sh

@@ -73,6 +73,7 @@ stage "git-diff-check"   git diff --check
 stage "mirror-gate-regression" python3 scripts/tests/test_git_mirrors.py
 stage "iso-boot-runner-regression" python3 scripts/tests/test_iso_qemu_runner.py
 stage "iso-qualified-web-ui" python3 tests/regression/iso-qualified-web-ui.py
+stage "first-boot-retry" python3 tests/regression/first-boot-retry.py
 stage "demo-rpc-parity" timeout 120 node neode-ui/scripts/mock-rpc-parity.mjs
 stage "demo-resumable-uploads" timeout 120 node --test neode-ui/scripts/demo-upload-test.mjs
 stage "companion-signature-regression" python3 scripts/tests/test_companion_apk_verification.py