diff --git a/core/container/src/podman_client.rs b/core/container/src/podman_client.rs index d69f5b0f..f4133543 100644 --- a/core/container/src/podman_client.rs +++ b/core/container/src/podman_client.rs @@ -263,28 +263,38 @@ impl PodmanClient { ) .await?; - // Image pull uses CLI — it's a streaming operation that the API handles differently - let mut cmd = tokio::process::Command::new("podman"); - cmd.arg("pull"); - if image_uses_insecure_registry(image) { - cmd.arg("--tls-verify=false"); + // Stall-aware + retried, mirroring the legacy installer's + // pull_one_url_with_progress. The old single-attempt hard 600s wall + // clock killed slow-but-progressing pulls, which every orchestrator + // stack (btcpay, indeedhub, …) surfaced as "first install fails, + // second succeeds" once the layer cache was warm. Podman keeps + // completed layers between attempts, so retries resume cheaply. + const MAX_ATTEMPTS: u32 = 3; + const BACKOFF_SECS: [u64; 2] = [5, 15]; + + let mut last_err = anyhow::anyhow!("podman pull {image}: no attempt ran"); + for attempt in 1..=MAX_ATTEMPTS { + match pull_image_stall_aware(image).await { + Ok(()) => return Ok(()), + Err(e) => { + tracing::warn!( + "Image pull failed for {} (attempt {}/{}): {:#}", + image, + attempt, + MAX_ATTEMPTS, + e + ); + last_err = e; + if attempt < MAX_ATTEMPTS { + tokio::time::sleep(std::time::Duration::from_secs( + BACKOFF_SECS[(attempt - 1) as usize], + )) + .await; + } + } + } } - cmd.arg(image); - - let output = tokio::time::timeout( - std::time::Duration::from_secs(600), // 10 min for large images - cmd.output(), - ) - .await - .map_err(|_| anyhow::anyhow!("Image pull timed out after 10 minutes"))? - .context("Failed to execute podman pull")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow::anyhow!("Failed to pull image: {}", stderr)); - } - - Ok(()) + Err(last_err) } pub async fn create_container(&self, manifest: &AppManifest, name: &str) -> Result { @@ -710,6 +720,155 @@ fn podman_network_settings( // ─── Helpers ───────────────────────────────────────────────────── +/// One `podman pull` attempt with a stall-aware budget instead of a hard +/// wall clock. A pull is only killed when NOTHING is observably happening — +/// no stderr output AND no byte growth in podman's TMPDIR staging dir — for +/// PULL_STALL_TIMEOUT_SECS, or at a generous absolute ceiling. Dead +/// registries still fail fast (no bytes ever land, so the 3-minute stall +/// window is the effective bound), while a slow-but-moving multi-GB pull is +/// left alone. Same design as the legacy installer's +/// pull_one_url_with_progress (LND "first install fails" fix). +async fn pull_image_stall_aware(image: &str) -> Result<()> { + const PULL_STALL_TIMEOUT_SECS: u64 = 180; + const PULL_MAX_SECS: u64 = 1800; + const PULL_POLL_INTERVAL_SECS: u64 = 5; + + // Rootless podman's user namespace makes /var/tmp read-only; stage into + // the user's containers tmp (same dir every other pull path uses) — it + // doubles as the "bytes are moving" signal for stall detection. + let user_tmp = format!( + "{}/.local/share/containers/tmp", + std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string()) + ); + let _ = std::fs::create_dir_all(&user_tmp); + + let mut cmd = tokio::process::Command::new("podman"); + cmd.arg("pull"); + if image_uses_insecure_registry(image) { + cmd.arg("--tls-verify=false"); + } + cmd.arg(image); + cmd.env("TMPDIR", &user_tmp); + cmd.stdout(std::process::Stdio::piped()); + cmd.stderr(std::process::Stdio::piped()); + cmd.kill_on_drop(true); + let mut child = cmd.spawn().context("Failed to start podman pull")?; + + let started = std::time::Instant::now(); + // Seconds-since-start of the last stderr line, updated by the reader + // task. Starts at 0 so the stall window opens at spawn. + let last_line_at = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + // Ring of recent stderr lines so a failed pull reports podman's actual + // error, not just "exit status 125". + let recent_stderr = std::sync::Arc::new(std::sync::Mutex::new(std::collections::VecDeque::< + String, + >::with_capacity(8))); + if let Some(stderr) = child.stderr.take() { + use tokio::io::AsyncBufReadExt; + let mut lines = tokio::io::BufReader::new(stderr).lines(); + let line_clock = std::sync::Arc::clone(&last_line_at); + let stderr_ring = std::sync::Arc::clone(&recent_stderr); + let started_reader = started; + tokio::spawn(async move { + while let Ok(Some(line)) = lines.next_line().await { + line_clock.store( + started_reader.elapsed().as_secs(), + std::sync::atomic::Ordering::Relaxed, + ); + if let Ok(mut ring) = stderr_ring.lock() { + if ring.len() >= 8 { + ring.pop_front(); + } + ring.push_back(line); + } + } + }); + } + + let stderr_tail = |ring: &std::sync::Mutex>| { + ring.lock() + .map(|r| r.iter().cloned().collect::>().join(" | ")) + .unwrap_or_default() + }; + + let mut last_staged_bytes = dir_size_bytes(&user_tmp); + let mut last_staged_change = std::time::Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) if status.success() => return Ok(()), + Ok(Some(status)) => { + anyhow::bail!( + "podman pull {image} failed ({status}): {}", + stderr_tail(&recent_stderr) + ); + } + Ok(None) => {} + Err(e) => { + let _ = child.kill().await; + let _ = child.wait().await; + return Err(anyhow::anyhow!("podman pull {image} process error: {e}")); + } + } + + tokio::time::sleep(std::time::Duration::from_secs(PULL_POLL_INTERVAL_SECS)).await; + + if started.elapsed() > std::time::Duration::from_secs(PULL_MAX_SECS) { + let _ = child.kill().await; + let _ = child.wait().await; // reap zombie + anyhow::bail!("podman pull {image} exceeded absolute {PULL_MAX_SECS}s ceiling"); + } + + // Activity signal 1: stderr output from podman. + let line_age = started + .elapsed() + .as_secs() + .saturating_sub(last_line_at.load(std::sync::atomic::Ordering::Relaxed)); + // Activity signal 2: staged layer bytes growing in TMPDIR. + let staged = dir_size_bytes(&user_tmp); + if staged != last_staged_bytes { + last_staged_bytes = staged; + last_staged_change = std::time::Instant::now(); + } + + if line_age > PULL_STALL_TIMEOUT_SECS + && last_staged_change.elapsed() + > std::time::Duration::from_secs(PULL_STALL_TIMEOUT_SECS) + { + let _ = child.kill().await; + let _ = child.wait().await; // reap zombie + anyhow::bail!( + "podman pull {image} stalled ({PULL_STALL_TIMEOUT_SECS}s with no output and no staged bytes): {}", + stderr_tail(&recent_stderr) + ); + } + } +} + +/// Total bytes under `path` (bounded recursive walk). Used as a coarse +/// "bytes are moving" signal for pull stall detection — exact size doesn't +/// matter, only whether it CHANGES between polls. Errors count as 0. +fn dir_size_bytes(path: &str) -> u64 { + fn walk(dir: &std::path::Path, depth: u32) -> u64 { + if depth > 8 { + return 0; + } + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + let mut total = 0u64; + for entry in entries.flatten() { + let Ok(meta) = entry.metadata() else { continue }; + if meta.is_dir() { + total = total.saturating_add(walk(&entry.path(), depth + 1)); + } else { + total = total.saturating_add(meta.len()); + } + } + total + } + walk(std::path::Path::new(path), 0) +} + fn parse_port_bindings(bindings: &serde_json::Value) -> Vec { let mut ports = Vec::new(); if let Some(obj) = bindings.as_object() {