From f13fdc6451c5704d7c42e95116cf112e9fce5c45 Mon Sep 17 00:00:00 2001
From: archipelago <archipelago@localhost>
Date: Mon, 20 Jul 2026 01:32:42 -0400
Subject: [PATCH] fix(recovery): don't brick-loop startup on stale
 crash-snapshot containers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

After an unclean shutdown, crash recovery walked the running-containers
snapshot and retried 'no such container' failures (2 attempts, 10s
backoff) for containers that no longer exist. Recovery runs before the
server binds :5678 and notifies systemd ready, so a stale snapshot
pushed startup past TimeoutStartSec=5min — systemd killed the daemon
mid-recovery, the next boot saw a crash again, and the node looped
forever (151 restarts on .116 after a hard poweroff).

- pre-filter the snapshot against 'podman ps -a' and skip vanished
  containers outright (fail-open if the query fails)
- never retry a 'no such container' failure
- extend the systemd start timeout ahead of each container start so a
  heavy node recovering dozens of real containers isn't killed while
  making progress

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 core/archipelago/src/crash_recovery.rs | 53 ++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/core/archipelago/src/crash_recovery.rs b/core/archipelago/src/crash_recovery.rs
index 959f3997..ed06738b 100644
--- a/core/archipelago/src/crash_recovery.rs
+++ b/core/archipelago/src/crash_recovery.rs
@@ -372,6 +372,28 @@ pub async fn save_container_snapshot(data_dir: &Path) -> Result<()> {
 /// Recover containers that were running before a crash.
 /// Attempts to start each container, logging success/failure.
 pub async fn recover_containers(containers: &[RunningContainerRecord]) -> RecoveryReport {
+    // Snapshot entries can outlive their containers (removed while we were
+    // down, or podman storage partially reset by an unclean poweroff).
+    // `podman start` on those fails permanently, and recovery runs BEFORE the
+    // server binds its port and notifies systemd ready — burning retries on
+    // them pushed recovery past TimeoutStartSec and brick-looped the node
+    // (killed mid-recovery → next boot sees a crash again, forever).
+    let containers: Vec<&RunningContainerRecord> = match existing_container_names().await {
+        Some(existing) => {
+            let (present, missing): (Vec<_>, Vec<_>) =
+                containers.iter().partition(|r| existing.contains(&r.name));
+            if !missing.is_empty() {
+                warn!(
+                    "Skipping {} snapshot container(s) that no longer exist: {:?}",
+                    missing.len(),
+                    missing.iter().map(|r| r.name.as_str()).collect::<Vec<_>>()
+                );
+            }
+            present
+        }
+        None => containers.iter().collect(),
+    };
+
     let mut report = RecoveryReport {
         total: containers.len(),
         recovered: 0,
@@ -386,6 +408,15 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
             record.name, record.image
         );
 
+        // Recovery counts against systemd's start timeout; a heavy node
+        // legitimately needs several minutes for dozens of containers. Push
+        // the deadline out ahead of each container so systemd only kills us
+        // if we stop making progress (360s covers one full attempt chain).
+        let _ = sd_notify::notify(
+            false,
+            &[sd_notify::NotifyState::ExtendTimeoutUsec(360_000_000)],
+        );
+
         // Rate-limit container starts to avoid overwhelming podman on low-resource systems
         if i > 0 {
             tokio::time::sleep(std::time::Duration::from_secs(3)).await;
@@ -427,6 +458,11 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
                         attempt + 1,
                         stderr.trim()
                     );
+                    // The container is gone (raced past the pre-filter, or the
+                    // filter query failed) — retrying can never succeed.
+                    if stderr.contains("no such container") {
+                        break;
+                    }
                 }
                 Err(e) => {
                     warn!(
@@ -448,6 +484,23 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
     report
 }
 
+/// All container names podman knows about (running or not). `None` if the
+/// query fails — callers fail open and attempt every snapshot entry.
+async fn existing_container_names() -> Option<std::collections::HashSet<String>> {
+    let output = podman_output(&["ps", "-a", "--format", "{{.Names}}"], Duration::from_secs(30))
+        .await
+        .ok()?;
+    if !output.status.success() {
+        return None;
+    }
+    Some(
+        String::from_utf8_lossy(&output.stdout)
+            .split_whitespace()
+            .map(|s| s.to_string())
+            .collect(),
+    )
+}
+
 #[derive(Debug)]
 pub struct RecoveryReport {
     pub total: usize,