feat(update): auto-rollback OTA binaries that crash-loop before they can self-verify

.198 crash-looped 236x on a truncated OTA binary with a good backup
sitting in update-backup/ — verify_pending_update() can't help when
the new binary never runs. New scripts/ota-crash-guard.sh runs as
root from ExecStartPre: while the post-OTA pending-verify marker
exists it counts start attempts, and after 5 failures restores the
backup binary (atomic rename), replaces the marker with an
update-rolled-back.json tombstone, and lets the service come back
on the previous version.

Wiring: fresh ISOs get the ExecStartPre line in the unit file;
existing nodes get a drop-in installed by apply_update's runtime
component step on their next OTA. '+-' prefix so a missing or
failing guard can never block the service.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/archipelago/src/update.rs

@@ -1785,6 +1785,30 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> {
                     .await;
                 }
 
+                // Install the OTA crash-loop guard as a drop-in on existing
+                // nodes (fresh ISOs carry it in the unit file itself). The
+                // guard restores the update-backup binary when a freshly
+                // applied binary SEGVs before it can run its own post-OTA
+                // verification — the .198 v1.7.103 truncated-binary loop.
+                // Best-effort: `+-` in the drop-in means a missing script can
+                // never block the service, and a failed install here must not
+                // abort the apply.
+                if Path::new("/opt/archipelago/scripts/ota-crash-guard.sh").exists() {
+                    let dropin_dir = "/etc/systemd/system/archipelago.service.d";
+                    let _ = host_sudo(&["mkdir", "-p", dropin_dir]).await;
+                    let _ = host_sudo(&[
+                        "bash",
+                        "-c",
+                        &format!(
+                            "printf '%s\\n' '[Service]' \
+                             'ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh' \
+                             > {}/ota-crash-guard.conf",
+                            dropin_dir
+                        ),
+                    ])
+                    .await;
+                }
+
                 let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
                 let _ =
                     host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await;

image-recipe/configs/archipelago.service

@@ -25,6 +25,11 @@ ExecStartPre=+/bin/bash -c 'mkdir -p /run/user/1000 /var/lib/containers && chown
 # once a VPN/bridge interface exists (netbird's wg tunnel sorted first and
 # poisoned every host_ip consumer). Falls back to hostname -I when routeless.
 ExecStartPre=+/bin/bash -c 'mkdir -p /var/lib/archipelago && chown archipelago:archipelago /var/lib/archipelago && IP=$(ip -4 route show default 2>/dev/null | sed -n "s/.* src \([0-9.]*\).*/\1/p" | head -1); [ -n "$$IP" ] || IP=$(hostname -I 2>/dev/null | awk "{print $$1}"); echo "ARCHIPELAGO_HOST_IP=$$IP" > /var/lib/archipelago/host-ip.env && chown archipelago:archipelago /var/lib/archipelago/host-ip.env'
+# OTA crash-loop guard: if a just-applied binary can't start (SEGV loop), the
+# in-binary post-OTA probe never runs — this restores the update-backup binary
+# after 5 failed start attempts while the pending-verify marker exists.
+# "-" so a missing/failed guard can never block the service itself.
+ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh
 ExecStart=/usr/local/bin/archipelago
 Restart=on-failure
 RestartSec=5

scripts/ota-crash-guard.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# OTA crash-loop guard — runs as root from ExecStartPre=+- on archipelago.service.
+#
+# Covers the failure mode verify_pending_update() cannot: a freshly-applied
+# binary that can't even start (SEGV/ENOEXEC — e.g. the truncated 17MB binary
+# .198 installed on the v1.7.103 OTA, which crash-looped 236 times with a
+# perfectly good backup sitting in update-backup/). The in-binary probe never
+# runs because the binary never runs, so this guard counts start attempts from
+# outside and restores the backup binary once the new one has clearly failed.
+#
+# Scope is deliberately narrow: it acts ONLY while the post-OTA pending-verify
+# marker exists (written by apply_update just before the restart, deleted by
+# the new binary once it boots and passes its probes). A crash loop with no
+# marker is not an OTA gone wrong, and this script stays out of it.
+#
+# Always exits 0 — a guard must never be the reason the service can't start.
+
+set -u
+
+DATA_DIR=/var/lib/archipelago
+MARKER="$DATA_DIR/update-pending-verify.json"
+COUNT_FILE="$DATA_DIR/ota-crash-guard.count"
+BACKUP="$DATA_DIR/update-backup/archipelago"
+BINARY=/usr/local/bin/archipelago
+MAX_ATTEMPTS=5
+
+log() {
+    echo "$*" | systemd-cat -t ota-crash-guard -p warning 2>/dev/null || true
+}
+
+# No pending OTA verification -> nothing to guard; clear any stale counter.
+if [ ! -f "$MARKER" ]; then
+    rm -f "$COUNT_FILE"
+    exit 0
+fi
+
+# Count this start attempt. The counter only accumulates while the marker
+# exists; a healthy new binary deletes the marker on its first successful
+# boot, and the next start clears the counter above.
+count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0)
+case "$count" in ''|*[!0-9]*) count=0 ;; esac
+count=$((count + 1))
+echo "$count" > "$COUNT_FILE" 2>/dev/null || true
+
+if [ "$count" -lt "$MAX_ATTEMPTS" ]; then
+    exit 0
+fi
+
+if [ ! -f "$BACKUP" ]; then
+    log "OTA crash guard: $count failed start attempts but no backup binary at $BACKUP — cannot roll back"
+    exit 0
+fi
+
+# Already restored (or the OTA never replaced the binary)? Don't loop.
+if cmp -s "$BACKUP" "$BINARY"; then
+    exit 0
+fi
+
+# Restore via copy-to-temp + atomic rename; never truncate the live path.
+tmp="$BINARY.rollback.$$"
+if cp "$BACKUP" "$tmp" && chown root:root "$tmp" && chmod 755 "$tmp" && mv "$tmp" "$BINARY"; then
+    # Leave a tombstone for the UI/logs instead of the marker so the restored
+    # binary doesn't run the post-OTA probe against the rolled-back version.
+    mv "$MARKER" "$DATA_DIR/update-rolled-back.json" 2>/dev/null || rm -f "$MARKER"
+    rm -f "$COUNT_FILE"
+    log "OTA crash guard: restored previous binary after $count failed start attempts of the updated one"
+else
+    rm -f "$tmp" 2>/dev/null
+    log "OTA crash guard: failed to restore backup binary (cp/mv error)"
+fi
+
+exit 0