From 25162ee8469a051c59607ca5d7802c2ebf218cba Mon Sep 17 00:00:00 2001 From: archipelago Date: Sat, 18 Jul 2026 19:42:54 -0400 Subject: [PATCH] feat(update): auto-rollback OTA binaries that crash-loop before they can self-verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .198 crash-looped 236x on a truncated OTA binary with a good backup sitting in update-backup/ — verify_pending_update() can't help when the new binary never runs. New scripts/ota-crash-guard.sh runs as root from ExecStartPre: while the post-OTA pending-verify marker exists it counts start attempts, and after 5 failures restores the backup binary (atomic rename), replaces the marker with an update-rolled-back.json tombstone, and lets the service come back on the previous version. Wiring: fresh ISOs get the ExecStartPre line in the unit file; existing nodes get a drop-in installed by apply_update's runtime component step on their next OTA. '+-' prefix so a missing or failing guard can never block the service. Co-Authored-By: Claude Fable 5 --- core/archipelago/src/update.rs | 24 ++++++++ image-recipe/configs/archipelago.service | 5 ++ scripts/ota-crash-guard.sh | 72 ++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100755 scripts/ota-crash-guard.sh diff --git a/core/archipelago/src/update.rs b/core/archipelago/src/update.rs index ad99aef6..29e85be5 100644 --- a/core/archipelago/src/update.rs +++ b/core/archipelago/src/update.rs @@ -1785,6 +1785,30 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> { .await; } + // Install the OTA crash-loop guard as a drop-in on existing + // nodes (fresh ISOs carry it in the unit file itself). The + // guard restores the update-backup binary when a freshly + // applied binary SEGVs before it can run its own post-OTA + // verification — the .198 v1.7.103 truncated-binary loop. + // Best-effort: `+-` in the drop-in means a missing script can + // never block the service, and a failed install here must not + // abort the apply. + if Path::new("/opt/archipelago/scripts/ota-crash-guard.sh").exists() { + let dropin_dir = "/etc/systemd/system/archipelago.service.d"; + let _ = host_sudo(&["mkdir", "-p", dropin_dir]).await; + let _ = host_sudo(&[ + "bash", + "-c", + &format!( + "printf '%s\\n' '[Service]' \ + 'ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh' \ + > {}/ota-crash-guard.conf", + dropin_dir + ), + ]) + .await; + } + let _ = host_sudo(&["systemctl", "daemon-reload"]).await; let _ = host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await; diff --git a/image-recipe/configs/archipelago.service b/image-recipe/configs/archipelago.service index fc3ec8d9..bc605b5a 100644 --- a/image-recipe/configs/archipelago.service +++ b/image-recipe/configs/archipelago.service @@ -25,6 +25,11 @@ ExecStartPre=+/bin/bash -c 'mkdir -p /run/user/1000 /var/lib/containers && chown # once a VPN/bridge interface exists (netbird's wg tunnel sorted first and # poisoned every host_ip consumer). Falls back to hostname -I when routeless. ExecStartPre=+/bin/bash -c 'mkdir -p /var/lib/archipelago && chown archipelago:archipelago /var/lib/archipelago && IP=$(ip -4 route show default 2>/dev/null | sed -n "s/.* src \([0-9.]*\).*/\1/p" | head -1); [ -n "$$IP" ] || IP=$(hostname -I 2>/dev/null | awk "{print $$1}"); echo "ARCHIPELAGO_HOST_IP=$$IP" > /var/lib/archipelago/host-ip.env && chown archipelago:archipelago /var/lib/archipelago/host-ip.env' +# OTA crash-loop guard: if a just-applied binary can't start (SEGV loop), the +# in-binary post-OTA probe never runs — this restores the update-backup binary +# after 5 failed start attempts while the pending-verify marker exists. +# "-" so a missing/failed guard can never block the service itself. +ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh ExecStart=/usr/local/bin/archipelago Restart=on-failure RestartSec=5 diff --git a/scripts/ota-crash-guard.sh b/scripts/ota-crash-guard.sh new file mode 100755 index 00000000..8e980c09 --- /dev/null +++ b/scripts/ota-crash-guard.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# OTA crash-loop guard — runs as root from ExecStartPre=+- on archipelago.service. +# +# Covers the failure mode verify_pending_update() cannot: a freshly-applied +# binary that can't even start (SEGV/ENOEXEC — e.g. the truncated 17MB binary +# .198 installed on the v1.7.103 OTA, which crash-looped 236 times with a +# perfectly good backup sitting in update-backup/). The in-binary probe never +# runs because the binary never runs, so this guard counts start attempts from +# outside and restores the backup binary once the new one has clearly failed. +# +# Scope is deliberately narrow: it acts ONLY while the post-OTA pending-verify +# marker exists (written by apply_update just before the restart, deleted by +# the new binary once it boots and passes its probes). A crash loop with no +# marker is not an OTA gone wrong, and this script stays out of it. +# +# Always exits 0 — a guard must never be the reason the service can't start. + +set -u + +DATA_DIR=/var/lib/archipelago +MARKER="$DATA_DIR/update-pending-verify.json" +COUNT_FILE="$DATA_DIR/ota-crash-guard.count" +BACKUP="$DATA_DIR/update-backup/archipelago" +BINARY=/usr/local/bin/archipelago +MAX_ATTEMPTS=5 + +log() { + echo "$*" | systemd-cat -t ota-crash-guard -p warning 2>/dev/null || true +} + +# No pending OTA verification -> nothing to guard; clear any stale counter. +if [ ! -f "$MARKER" ]; then + rm -f "$COUNT_FILE" + exit 0 +fi + +# Count this start attempt. The counter only accumulates while the marker +# exists; a healthy new binary deletes the marker on its first successful +# boot, and the next start clears the counter above. +count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0) +case "$count" in ''|*[!0-9]*) count=0 ;; esac +count=$((count + 1)) +echo "$count" > "$COUNT_FILE" 2>/dev/null || true + +if [ "$count" -lt "$MAX_ATTEMPTS" ]; then + exit 0 +fi + +if [ ! -f "$BACKUP" ]; then + log "OTA crash guard: $count failed start attempts but no backup binary at $BACKUP — cannot roll back" + exit 0 +fi + +# Already restored (or the OTA never replaced the binary)? Don't loop. +if cmp -s "$BACKUP" "$BINARY"; then + exit 0 +fi + +# Restore via copy-to-temp + atomic rename; never truncate the live path. +tmp="$BINARY.rollback.$$" +if cp "$BACKUP" "$tmp" && chown root:root "$tmp" && chmod 755 "$tmp" && mv "$tmp" "$BINARY"; then + # Leave a tombstone for the UI/logs instead of the marker so the restored + # binary doesn't run the post-OTA probe against the rolled-back version. + mv "$MARKER" "$DATA_DIR/update-rolled-back.json" 2>/dev/null || rm -f "$MARKER" + rm -f "$COUNT_FILE" + log "OTA crash guard: restored previous binary after $count failed start attempts of the updated one" +else + rm -f "$tmp" 2>/dev/null + log "OTA crash guard: failed to restore backup binary (cp/mv error)" +fi + +exit 0