lfg2025/archy
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects Releases 61 Wiki Activity

fix(recovery): quiet the fleet logs — skip running containers, back off failed companion repairs
Some checks failed
Demo images / Build & push demo images (push) Failing after 32s
Details

Browse Source 
Fleet log sweep (2026-07-22): (1) recovery paths podman-start'ed
containers that were already running, producing hundreds of benign but
scary conmon 'Failed to create container' + cgroup Permission-denied
pairs per node per day — both paths now check state first; (2) the 30s
companion-reconcile loop retried registry pulls/builds forever against
unreachable registries (~174×/image/day on an offline node) — failing
rounds now back off exponentially to a 1h cap.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-22 19:44:32 -04:00
parent b193e64ffb
commit 6347d16a2f
2 changed files with 37 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
core/archipelago/src/container/boot_reconciler.rs
View File

@ -108,17 +108,33 @@ impl BootReconciler {
let orchestrator = self.orchestrator.clone();
let interval = self.interval;
Some(tokio::spawn(async move {
let mut failure_rounds: u32 = 0;
loop {
let installed = orchestrator.manifest_ids().await;
for (companion, err) in crate::container::companion::reconcile(&installed).await
{
let failures =
crate::container::companion::reconcile(&installed).await;
for (companion, err) in &failures {
tracing::warn!(
companion = %companion,
error = %err,
"companion reconcile failed"
);
}
time::sleep(interval).await;
// A failed repair can involve registry pulls and full
// image builds; retrying every 30s hammered unreachable
// registries ~174×/image/day on an offline node
// (archy-x250-dev log sweep, 2026-07-22). Back off
// exponentially while rounds keep failing — 30s doubling
// to a 1h cap — and reset the moment a round is clean.
failure_rounds = if failures.is_empty() {
0
} else {
failure_rounds.saturating_add(1)
};
let backoff = interval
.saturating_mul(2u32.saturating_pow(failure_rounds.min(7)))
.min(Duration::from_secs(3600));
time::sleep(backoff).await;
}
}))
} else {

21
core/archipelago/src/crash_recovery.rs
View File

@ -403,6 +403,14 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
pending_boot_starts_add(containers.iter().map(|r| r.name.clone()));
for (i, record) in containers.iter().enumerate() {
// Skip containers that are already up — `podman start` on a running
// container produces the noisy benign conmon "Failed to create
// container" + cgroup Permission-denied journal pair (fleet log
// sweep 2026-07-22).
if container_state(&record.name).await.as_deref() == Some("running") {
report.recovered += 1;
continue;
}
info!(
"Recovering container: {} (image: {})",
record.name, record.image
@ -936,14 +944,21 @@ async fn container_state(container: &str) -> Option<String> {
}
async fn start_existing_container(container: &str) -> bool {
info!("Recovering stack container: {}", container);
let timeout = match container {
"immich_server" | "netbird-server" => Duration::from_secs(120),
_ => Duration::from_secs(90),
};
if container_state(container).await.as_deref() == Some("initialized") {
cleanup_container_runtime_state(container).await;
match container_state(container).await.as_deref() {
// Already up — `podman start` on a running container makes conmon
// try to join the live cgroup and fail with a scary "Failed to
// create container: exit status 1" + cgroup.procs Permission denied
// pair in the journal. Fleet log sweep 2026-07-22 found hundreds of
// these per day per node, all benign. Skip instead.
Some("running") => return true,
Some("initialized") => cleanup_container_runtime_state(container).await,
_ => {}
}
info!("Recovering stack container: {}", container);
match podman_output(&["start", container], timeout).await {
Ok(output) if output.status.success() => {
tokio::time::sleep(Duration::from_secs(3)).await;