lfg2025/archy
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects Releases 61 Wiki Activity

fix(install): stall-aware, retried image pulls in the orchestrator runtime

Browse Source 
PodmanClient::pull_image had a single attempt with a hard 600s wall
clock — on a fresh node, any image that legitimately takes >10 min to
download got killed mid-pull and failed the install. Podman keeps the
completed layers, so the user's retry started half-done and succeeded:
the exact "first install fails, second works" pattern reported for
btcpay and indeedhub (and earlier for LND, whose fix in 72d7fa07 only
covered the legacy install path — this is the same cure ported to the
orchestrator path both fresh installs and upgrades use).

A pull is now only killed when nothing is observably happening — no
stderr output AND no staged-byte growth in TMPDIR for 180s — or at a
1800s absolute ceiling, with 3 attempts and backoff on top. Dead
registries still fail fast (no bytes ever land). Failures now surface
podman's actual stderr tail instead of a bare exit status, and the
pull stages via the user's containers tmp like every other pull path
(rootless /var/tmp is read-only).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-16 13:02:53 -04:00
parent 500472944c
commit e5c7210663
1 changed files with 180 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
core/container/src/podman_client.rs
View File

@ -263,28 +263,38 @@ impl PodmanClient {
)
.await?;
// Image pull uses CLI — it's a streaming operation that the API handles differently
let mut cmd = tokio::process::Command::new("podman");
cmd.arg("pull");
if image_uses_insecure_registry(image) {
cmd.arg("--tls-verify=false");
// Stall-aware + retried, mirroring the legacy installer's
// pull_one_url_with_progress. The old single-attempt hard 600s wall
// clock killed slow-but-progressing pulls, which every orchestrator
// stack (btcpay, indeedhub, …) surfaced as "first install fails,
// second succeeds" once the layer cache was warm. Podman keeps
// completed layers between attempts, so retries resume cheaply.
const MAX_ATTEMPTS: u32 = 3;
const BACKOFF_SECS: [u64; 2] = [5, 15];
let mut last_err = anyhow::anyhow!("podman pull {image}: no attempt ran");
for attempt in 1..=MAX_ATTEMPTS {
match pull_image_stall_aware(image).await {
Ok(()) => return Ok(()),
Err(e) => {
tracing::warn!(
"Image pull failed for {} (attempt {}/{}): {:#}",
image,
attempt,
MAX_ATTEMPTS,
e
);
last_err = e;
if attempt < MAX_ATTEMPTS {
tokio::time::sleep(std::time::Duration::from_secs(
BACKOFF_SECS[(attempt - 1) as usize],
))
.await;
}
cmd.arg(image);
let output = tokio::time::timeout(
std::time::Duration::from_secs(600), // 10 min for large images
cmd.output(),
)
.await
.map_err(|_| anyhow::anyhow!("Image pull timed out after 10 minutes"))?
.context("Failed to execute podman pull")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow::anyhow!("Failed to pull image: {}", stderr));
}
Ok(())
}
}
Err(last_err)
}
pub async fn create_container(&self, manifest: &AppManifest, name: &str) -> Result<String> {
@ -710,6 +720,155 @@ fn podman_network_settings(
// ─── Helpers ─────────────────────────────────────────────────────
/// One `podman pull` attempt with a stall-aware budget instead of a hard
/// wall clock. A pull is only killed when NOTHING is observably happening —
/// no stderr output AND no byte growth in podman's TMPDIR staging dir — for
/// PULL_STALL_TIMEOUT_SECS, or at a generous absolute ceiling. Dead
/// registries still fail fast (no bytes ever land, so the 3-minute stall
/// window is the effective bound), while a slow-but-moving multi-GB pull is
/// left alone. Same design as the legacy installer's
/// pull_one_url_with_progress (LND "first install fails" fix).
async fn pull_image_stall_aware(image: &str) -> Result<()> {
const PULL_STALL_TIMEOUT_SECS: u64 = 180;
const PULL_MAX_SECS: u64 = 1800;
const PULL_POLL_INTERVAL_SECS: u64 = 5;
// Rootless podman's user namespace makes /var/tmp read-only; stage into
// the user's containers tmp (same dir every other pull path uses) — it
// doubles as the "bytes are moving" signal for stall detection.
let user_tmp = format!(
"{}/.local/share/containers/tmp",
std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string())
);
let _ = std::fs::create_dir_all(&user_tmp);
let mut cmd = tokio::process::Command::new("podman");
cmd.arg("pull");
if image_uses_insecure_registry(image) {
cmd.arg("--tls-verify=false");
}
cmd.arg(image);
cmd.env("TMPDIR", &user_tmp);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
cmd.kill_on_drop(true);
let mut child = cmd.spawn().context("Failed to start podman pull")?;
let started = std::time::Instant::now();
// Seconds-since-start of the last stderr line, updated by the reader
// task. Starts at 0 so the stall window opens at spawn.
let last_line_at = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
// Ring of recent stderr lines so a failed pull reports podman's actual
// error, not just "exit status 125".
let recent_stderr = std::sync::Arc::new(std::sync::Mutex::new(std::collections::VecDeque::<
String,
>::with_capacity(8)));
if let Some(stderr) = child.stderr.take() {
use tokio::io::AsyncBufReadExt;
let mut lines = tokio::io::BufReader::new(stderr).lines();
let line_clock = std::sync::Arc::clone(&last_line_at);
let stderr_ring = std::sync::Arc::clone(&recent_stderr);
let started_reader = started;
tokio::spawn(async move {
while let Ok(Some(line)) = lines.next_line().await {
line_clock.store(
started_reader.elapsed().as_secs(),
std::sync::atomic::Ordering::Relaxed,
);
if let Ok(mut ring) = stderr_ring.lock() {
if ring.len() >= 8 {
ring.pop_front();
}
ring.push_back(line);
}
}
});
}
let stderr_tail = |ring: &std::sync::Mutex<std::collections::VecDeque<String>>| {
ring.lock()
.map(|r| r.iter().cloned().collect::<Vec<_>>().join(" | "))
.unwrap_or_default()
};
let mut last_staged_bytes = dir_size_bytes(&user_tmp);
let mut last_staged_change = std::time::Instant::now();
loop {
match child.try_wait() {
Ok(Some(status)) if status.success() => return Ok(()),
Ok(Some(status)) => {
anyhow::bail!(
"podman pull {image} failed ({status}): {}",
stderr_tail(&recent_stderr)
);
}
Ok(None) => {}
Err(e) => {
let _ = child.kill().await;
let _ = child.wait().await;
return Err(anyhow::anyhow!("podman pull {image} process error: {e}"));
}
}
tokio::time::sleep(std::time::Duration::from_secs(PULL_POLL_INTERVAL_SECS)).await;
if started.elapsed() > std::time::Duration::from_secs(PULL_MAX_SECS) {
let _ = child.kill().await;
let _ = child.wait().await; // reap zombie
anyhow::bail!("podman pull {image} exceeded absolute {PULL_MAX_SECS}s ceiling");
}
// Activity signal 1: stderr output from podman.
let line_age = started
.elapsed()
.as_secs()
.saturating_sub(last_line_at.load(std::sync::atomic::Ordering::Relaxed));
// Activity signal 2: staged layer bytes growing in TMPDIR.
let staged = dir_size_bytes(&user_tmp);
if staged != last_staged_bytes {
last_staged_bytes = staged;
last_staged_change = std::time::Instant::now();
}
if line_age > PULL_STALL_TIMEOUT_SECS
&& last_staged_change.elapsed()
> std::time::Duration::from_secs(PULL_STALL_TIMEOUT_SECS)
{
let _ = child.kill().await;
let _ = child.wait().await; // reap zombie
anyhow::bail!(
"podman pull {image} stalled ({PULL_STALL_TIMEOUT_SECS}s with no output and no staged bytes): {}",
stderr_tail(&recent_stderr)
);
}
}
}
/// Total bytes under `path` (bounded recursive walk). Used as a coarse
/// "bytes are moving" signal for pull stall detection — exact size doesn't
/// matter, only whether it CHANGES between polls. Errors count as 0.
fn dir_size_bytes(path: &str) -> u64 {
fn walk(dir: &std::path::Path, depth: u32) -> u64 {
if depth > 8 {
return 0;
}
let Ok(entries) = std::fs::read_dir(dir) else {
return 0;
};
let mut total = 0u64;
for entry in entries.flatten() {
let Ok(meta) = entry.metadata() else { continue };
if meta.is_dir() {
total = total.saturating_add(walk(&entry.path(), depth + 1));
} else {
total = total.saturating_add(meta.len());
}
}
total
}
walk(std::path::Path::new(path), 0)
}
fn parse_port_bindings(bindings: &serde_json::Value) -> Vec<String> {
let mut ports = Vec::new();
if let Some(obj) = bindings.as_object() {