fix(install): stall-aware, retried image pulls in the orchestrator runtime

PodmanClient::pull_image had a single attempt with a hard 600s wall
clock — on a fresh node, any image that legitimately takes >10 min to
download got killed mid-pull and failed the install. Podman keeps the
completed layers, so the user's retry started half-done and succeeded:
the exact "first install fails, second works" pattern reported for
btcpay and indeedhub (and earlier for LND, whose fix in 72d7fa07 only
covered the legacy install path — this is the same cure ported to the
orchestrator path both fresh installs and upgrades use).

A pull is now only killed when nothing is observably happening — no
stderr output AND no staged-byte growth in TMPDIR for 180s — or at a
1800s absolute ceiling, with 3 attempts and backoff on top. Dead
registries still fail fast (no bytes ever land). Failures now surface
podman's actual stderr tail instead of a bare exit status, and the
pull stages via the user's containers tmp like every other pull path
(rootless /var/tmp is read-only).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/container/src/podman_client.rs

@@ -263,28 +263,38 @@ impl PodmanClient {
         )
         .await?;
 
-        // Image pull uses CLI — it's a streaming operation that the API handles differently
+        // Stall-aware + retried, mirroring the legacy installer's
-        let mut cmd = tokio::process::Command::new("podman");
+        // pull_one_url_with_progress. The old single-attempt hard 600s wall
-        cmd.arg("pull");
+        // clock killed slow-but-progressing pulls, which every orchestrator
-        if image_uses_insecure_registry(image) {
+        // stack (btcpay, indeedhub, …) surfaced as "first install fails,
-            cmd.arg("--tls-verify=false");
+        // second succeeds" once the layer cache was warm. Podman keeps
+        // completed layers between attempts, so retries resume cheaply.
+        const MAX_ATTEMPTS: u32 = 3;
+        const BACKOFF_SECS: [u64; 2] = [5, 15];
+
+        let mut last_err = anyhow::anyhow!("podman pull {image}: no attempt ran");
+        for attempt in 1..=MAX_ATTEMPTS {
+            match pull_image_stall_aware(image).await {
+                Ok(()) => return Ok(()),
+                Err(e) => {
+                    tracing::warn!(
+                        "Image pull failed for {} (attempt {}/{}): {:#}",
+                        image,
+                        attempt,
+                        MAX_ATTEMPTS,
+                        e
+                    );
+                    last_err = e;
+                    if attempt < MAX_ATTEMPTS {
+                        tokio::time::sleep(std::time::Duration::from_secs(
+                            BACKOFF_SECS[(attempt - 1) as usize],
+                        ))
+                        .await;
                     }
-        cmd.arg(image);
-
-        let output = tokio::time::timeout(
-            std::time::Duration::from_secs(600), // 10 min for large images
-            cmd.output(),
-        )
-        .await
-        .map_err(|_| anyhow::anyhow!("Image pull timed out after 10 minutes"))?
-        .context("Failed to execute podman pull")?;
-
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            return Err(anyhow::anyhow!("Failed to pull image: {}", stderr));
                 }
-
+            }
-        Ok(())
+        }
+        Err(last_err)
     }
 
     pub async fn create_container(&self, manifest: &AppManifest, name: &str) -> Result<String> {
@@ -710,6 +720,155 @@ fn podman_network_settings(
 
 // ─── Helpers ─────────────────────────────────────────────────────
 
+/// One `podman pull` attempt with a stall-aware budget instead of a hard
+/// wall clock. A pull is only killed when NOTHING is observably happening —
+/// no stderr output AND no byte growth in podman's TMPDIR staging dir — for
+/// PULL_STALL_TIMEOUT_SECS, or at a generous absolute ceiling. Dead
+/// registries still fail fast (no bytes ever land, so the 3-minute stall
+/// window is the effective bound), while a slow-but-moving multi-GB pull is
+/// left alone. Same design as the legacy installer's
+/// pull_one_url_with_progress (LND "first install fails" fix).
+async fn pull_image_stall_aware(image: &str) -> Result<()> {
+    const PULL_STALL_TIMEOUT_SECS: u64 = 180;
+    const PULL_MAX_SECS: u64 = 1800;
+    const PULL_POLL_INTERVAL_SECS: u64 = 5;
+
+    // Rootless podman's user namespace makes /var/tmp read-only; stage into
+    // the user's containers tmp (same dir every other pull path uses) — it
+    // doubles as the "bytes are moving" signal for stall detection.
+    let user_tmp = format!(
+        "{}/.local/share/containers/tmp",
+        std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string())
+    );
+    let _ = std::fs::create_dir_all(&user_tmp);
+
+    let mut cmd = tokio::process::Command::new("podman");
+    cmd.arg("pull");
+    if image_uses_insecure_registry(image) {
+        cmd.arg("--tls-verify=false");
+    }
+    cmd.arg(image);
+    cmd.env("TMPDIR", &user_tmp);
+    cmd.stdout(std::process::Stdio::piped());
+    cmd.stderr(std::process::Stdio::piped());
+    cmd.kill_on_drop(true);
+    let mut child = cmd.spawn().context("Failed to start podman pull")?;
+
+    let started = std::time::Instant::now();
+    // Seconds-since-start of the last stderr line, updated by the reader
+    // task. Starts at 0 so the stall window opens at spawn.
+    let last_line_at = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
+    // Ring of recent stderr lines so a failed pull reports podman's actual
+    // error, not just "exit status 125".
+    let recent_stderr = std::sync::Arc::new(std::sync::Mutex::new(std::collections::VecDeque::<
+        String,
+    >::with_capacity(8)));
+    if let Some(stderr) = child.stderr.take() {
+        use tokio::io::AsyncBufReadExt;
+        let mut lines = tokio::io::BufReader::new(stderr).lines();
+        let line_clock = std::sync::Arc::clone(&last_line_at);
+        let stderr_ring = std::sync::Arc::clone(&recent_stderr);
+        let started_reader = started;
+        tokio::spawn(async move {
+            while let Ok(Some(line)) = lines.next_line().await {
+                line_clock.store(
+                    started_reader.elapsed().as_secs(),
+                    std::sync::atomic::Ordering::Relaxed,
+                );
+                if let Ok(mut ring) = stderr_ring.lock() {
+                    if ring.len() >= 8 {
+                        ring.pop_front();
+                    }
+                    ring.push_back(line);
+                }
+            }
+        });
+    }
+
+    let stderr_tail = |ring: &std::sync::Mutex<std::collections::VecDeque<String>>| {
+        ring.lock()
+            .map(|r| r.iter().cloned().collect::<Vec<_>>().join(" | "))
+            .unwrap_or_default()
+    };
+
+    let mut last_staged_bytes = dir_size_bytes(&user_tmp);
+    let mut last_staged_change = std::time::Instant::now();
+    loop {
+        match child.try_wait() {
+            Ok(Some(status)) if status.success() => return Ok(()),
+            Ok(Some(status)) => {
+                anyhow::bail!(
+                    "podman pull {image} failed ({status}): {}",
+                    stderr_tail(&recent_stderr)
+                );
+            }
+            Ok(None) => {}
+            Err(e) => {
+                let _ = child.kill().await;
+                let _ = child.wait().await;
+                return Err(anyhow::anyhow!("podman pull {image} process error: {e}"));
+            }
+        }
+
+        tokio::time::sleep(std::time::Duration::from_secs(PULL_POLL_INTERVAL_SECS)).await;
+
+        if started.elapsed() > std::time::Duration::from_secs(PULL_MAX_SECS) {
+            let _ = child.kill().await;
+            let _ = child.wait().await; // reap zombie
+            anyhow::bail!("podman pull {image} exceeded absolute {PULL_MAX_SECS}s ceiling");
+        }
+
+        // Activity signal 1: stderr output from podman.
+        let line_age = started
+            .elapsed()
+            .as_secs()
+            .saturating_sub(last_line_at.load(std::sync::atomic::Ordering::Relaxed));
+        // Activity signal 2: staged layer bytes growing in TMPDIR.
+        let staged = dir_size_bytes(&user_tmp);
+        if staged != last_staged_bytes {
+            last_staged_bytes = staged;
+            last_staged_change = std::time::Instant::now();
+        }
+
+        if line_age > PULL_STALL_TIMEOUT_SECS
+            && last_staged_change.elapsed()
+                > std::time::Duration::from_secs(PULL_STALL_TIMEOUT_SECS)
+        {
+            let _ = child.kill().await;
+            let _ = child.wait().await; // reap zombie
+            anyhow::bail!(
+                "podman pull {image} stalled ({PULL_STALL_TIMEOUT_SECS}s with no output and no staged bytes): {}",
+                stderr_tail(&recent_stderr)
+            );
+        }
+    }
+}
+
+/// Total bytes under `path` (bounded recursive walk). Used as a coarse
+/// "bytes are moving" signal for pull stall detection — exact size doesn't
+/// matter, only whether it CHANGES between polls. Errors count as 0.
+fn dir_size_bytes(path: &str) -> u64 {
+    fn walk(dir: &std::path::Path, depth: u32) -> u64 {
+        if depth > 8 {
+            return 0;
+        }
+        let Ok(entries) = std::fs::read_dir(dir) else {
+            return 0;
+        };
+        let mut total = 0u64;
+        for entry in entries.flatten() {
+            let Ok(meta) = entry.metadata() else { continue };
+            if meta.is_dir() {
+                total = total.saturating_add(walk(&entry.path(), depth + 1));
+            } else {
+                total = total.saturating_add(meta.len());
+            }
+        }
+        total
+    }
+    walk(std::path::Path::new(path), 0)
+}
+
 fn parse_port_bindings(bindings: &serde_json::Value) -> Vec<String> {
     let mut ports = Vec::new();
     if let Some(obj) = bindings.as_object() {