diff --git a/core/archipelago/src/container/boot_reconciler.rs b/core/archipelago/src/container/boot_reconciler.rs index cb417fda..346fa02e 100644 --- a/core/archipelago/src/container/boot_reconciler.rs +++ b/core/archipelago/src/container/boot_reconciler.rs @@ -108,17 +108,33 @@ impl BootReconciler { let orchestrator = self.orchestrator.clone(); let interval = self.interval; Some(tokio::spawn(async move { + let mut failure_rounds: u32 = 0; loop { let installed = orchestrator.manifest_ids().await; - for (companion, err) in crate::container::companion::reconcile(&installed).await - { + let failures = + crate::container::companion::reconcile(&installed).await; + for (companion, err) in &failures { tracing::warn!( companion = %companion, error = %err, "companion reconcile failed" ); } - time::sleep(interval).await; + // A failed repair can involve registry pulls and full + // image builds; retrying every 30s hammered unreachable + // registries ~174×/image/day on an offline node + // (archy-x250-dev log sweep, 2026-07-22). Back off + // exponentially while rounds keep failing — 30s doubling + // to a 1h cap — and reset the moment a round is clean. + failure_rounds = if failures.is_empty() { + 0 + } else { + failure_rounds.saturating_add(1) + }; + let backoff = interval + .saturating_mul(2u32.saturating_pow(failure_rounds.min(7))) + .min(Duration::from_secs(3600)); + time::sleep(backoff).await; } })) } else { diff --git a/core/archipelago/src/crash_recovery.rs b/core/archipelago/src/crash_recovery.rs index 0aaa9e7e..a41e78f5 100644 --- a/core/archipelago/src/crash_recovery.rs +++ b/core/archipelago/src/crash_recovery.rs @@ -403,6 +403,14 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove pending_boot_starts_add(containers.iter().map(|r| r.name.clone())); for (i, record) in containers.iter().enumerate() { + // Skip containers that are already up — `podman start` on a running + // container produces the noisy benign conmon "Failed to create + // container" + cgroup Permission-denied journal pair (fleet log + // sweep 2026-07-22). + if container_state(&record.name).await.as_deref() == Some("running") { + report.recovered += 1; + continue; + } info!( "Recovering container: {} (image: {})", record.name, record.image @@ -936,14 +944,21 @@ async fn container_state(container: &str) -> Option { } async fn start_existing_container(container: &str) -> bool { - info!("Recovering stack container: {}", container); let timeout = match container { "immich_server" | "netbird-server" => Duration::from_secs(120), _ => Duration::from_secs(90), }; - if container_state(container).await.as_deref() == Some("initialized") { - cleanup_container_runtime_state(container).await; + match container_state(container).await.as_deref() { + // Already up — `podman start` on a running container makes conmon + // try to join the live cgroup and fail with a scary "Failed to + // create container: exit status 1" + cgroup.procs Permission denied + // pair in the journal. Fleet log sweep 2026-07-22 found hundreds of + // these per day per node, all benign. Skip instead. + Some("running") => return true, + Some("initialized") => cleanup_container_runtime_state(container).await, + _ => {} } + info!("Recovering stack container: {}", container); match podman_output(&["start", container], timeout).await { Ok(output) if output.status.success() => { tokio::time::sleep(Duration::from_secs(3)).await;