From 92cffe9d46c0fe56cec99b8f6623d4dc39d08386 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 5 Aug 2026 22:50:42 -0400 Subject: [PATCH] fix(reconciler): recreate an absent stack member when its siblings are live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The periodic reconcile runs ExistingOnly — merely listing a catalog manifest must never install an app — and its only absent-container recovery keyed on the last running-names snapshot, which ages out after a few daemon restarts. An absent member of an installed stack then stays absent forever: .38 ran indeedhub with no minio/postgres for days, nginx down on 'host not found in upstream "minio"', and nothing ever put the members back. A live sibling container is proof the stack is installed on this node, so an absent member is now treated as a hole to repair, not a choice to respect: the recovery guard also fires when another member of the same stack (app_ops::stack_member_app_ids) has a container in any state. A stack with no containers at all is left untouched, and sibling app ids resolve through the loaded-manifest container names (immich-postgres runs as immich_postgres). Co-Authored-By: Claude Fable 5 --- core/archipelago/src/app_ops.rs | 3 +- .../src/container/prod_orchestrator.rs | 123 +++++++++++++++++- 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/core/archipelago/src/app_ops.rs b/core/archipelago/src/app_ops.rs index c036d863..7311ade7 100644 --- a/core/archipelago/src/app_ops.rs +++ b/core/archipelago/src/app_ops.rs @@ -75,7 +75,8 @@ pub fn address_caching_dependents(package_id: &str) -> &'static [&'static str] { /// The package whose lifecycle lock covers `app_id`: the stack package when /// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while /// they drive archy-mempool-web), otherwise the app itself. -fn owning_package(app_id: &str) -> &str { +/// Also consulted by the reconciler's absent-stack-member recovery. +pub fn owning_package(app_id: &str) -> &str { const STACKS: &[&str] = &[ "immich", "indeedhub", diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 2eb4dc12..0d66db21 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -104,6 +104,32 @@ fn dependency_manifests_required_by_active_apps<'a>( required } +/// Whether `app_id` is a member of a known multi-container stack that has at +/// least one OTHER member with a live container (any state). A live sibling +/// proves the stack is installed on this node, so an absent member is a hole +/// to repair — while a stack with no containers at all stays untouched +/// (uninstalled, or never installed here). Sibling app ids resolve to +/// container names through the loaded-manifest map when available (immich's +/// `immich-postgres` app id runs as container `immich_postgres`), falling +/// back to the id itself. +fn absent_stack_member_with_live_sibling( + app_id: &str, + present_containers: &HashSet, + container_name_by_app_id: &std::collections::HashMap, +) -> bool { + let stack = crate::app_ops::owning_package(app_id); + let members = crate::app_ops::stack_member_app_ids(stack); + members.iter().any(|member| { + *member != app_id + && present_containers.contains( + container_name_by_app_id + .get(*member) + .map(String::as_str) + .unwrap_or(member), + ) + }) +} + fn manifest_dependency_app_ids(manifest: &AppManifest) -> Vec { manifest .app @@ -1654,13 +1680,16 @@ impl ProdContainerOrchestrator { // app whose container vanished (e.g. a wedged teardown cleared by a // reboot) instead of leaving it down. See the immich .198 incident. let was_running = crate::crash_recovery::load_last_running_names(&self.data_dir).await; - let manifests: Vec = { + let (manifests, container_name_by_app_id): ( + Vec, + std::collections::HashMap, + ) = { let state = self.state.read().await; let dependency_required = dependency_manifests_required_by_active_apps( state.manifests.values().map(|lm| &lm.manifest), &user_stopped, ); - state + let filtered = state .manifests .iter() .filter(|(app_id, _)| !state.disabled.contains(*app_id)) @@ -1670,8 +1699,25 @@ impl ProdContainerOrchestrator { && !user_stopped.contains(&compute_container_name(&lm.manifest))) }) .map(|(_, lm)| lm.clone()) - .collect() + .collect(); + // Unfiltered id→container-name map for the absent-stack-member + // recovery below: a sibling may be excluded from this pass (e.g. + // user-stopped) yet its live container still proves the stack is + // installed. + let names = state + .manifests + .iter() + .map(|(id, lm)| (id.clone(), compute_container_name(&lm.manifest))) + .collect(); + (filtered, names) }; + // Live container names (any state), for the same recovery check. + let present_containers: std::collections::HashSet = self + .runtime + .list_containers() + .await + .map(|cs| cs.into_iter().map(|c| c.name).collect()) + .unwrap_or_default(); let mut report = ReconcileReport::default(); let disk_gb = self.disk_gb().await; // Register every candidate before the (sequential, possibly slow) @@ -1738,7 +1784,20 @@ impl ProdContainerOrchestrator { Ok(ReconcileAction::Left(reason)) if mode == ReconcileMode::ExistingOnly && reason == "absent" - && was_running.contains(&compute_container_name(&lm.manifest)) => + && (was_running.contains(&compute_container_name(&lm.manifest)) + // Absent STACK MEMBER whose siblings have live + // containers: the stack is installed, so the + // missing member is a hole, not a choice. The + // was_running snapshot ages out after a few daemon + // restarts, which left indeedhub-minio/-postgres + // permanently absent on .38 (2026-08-06) — nginx + // down on `host not found in upstream "minio"` + // with nothing ever recreating the members. + || absent_stack_member_with_live_sibling( + &app_id, + &present_containers, + &container_name_by_app_id, + )) => { tracing::warn!( app_id = %app_id, @@ -4451,6 +4510,62 @@ mod tests { items.iter().map(|s| s.to_string()).collect() } + /// The .38 indeedhub incident class: an absent stack member must be + /// recovered when its siblings have live containers (the stack is + /// installed), and left alone when the whole stack is gone or the app + /// is not a stack member at all. + #[test] + fn absent_stack_member_recovery_requires_a_live_sibling() { + let present: HashSet = ["indeedhub-redis", "indeedhub-relay", "indeedhub"] + .iter() + .map(|s| s.to_string()) + .collect(); + let names = std::collections::HashMap::new(); + // Missing members of a stack with live siblings → recover. + assert!(absent_stack_member_with_live_sibling( + "indeedhub-minio", + &present, + &names + )); + assert!(absent_stack_member_with_live_sibling( + "indeedhub-postgres", + &present, + &names + )); + // Whole stack absent → NOT recovered (uninstalled stays uninstalled). + let empty = HashSet::new(); + assert!(!absent_stack_member_with_live_sibling( + "indeedhub-minio", + &empty, + &names + )); + // Non-stack app → never. + assert!(!absent_stack_member_with_live_sibling( + "vaultwarden", + &present, + &names + )); + // An app's OWN container being present proves nothing about siblings. + let only_self: HashSet = + std::iter::once("indeedhub-minio".to_string()).collect(); + assert!(!absent_stack_member_with_live_sibling( + "indeedhub-minio", + &only_self, + &names + )); + // App-id → container-name mapping is honoured (immich_postgres runs + // under an underscore name while its app id is hyphenated). + let mut mapped = std::collections::HashMap::new(); + mapped.insert("immich-postgres".to_string(), "immich_postgres".to_string()); + let immich_present: HashSet = + std::iter::once("immich_postgres".to_string()).collect(); + assert!(absent_stack_member_with_live_sibling( + "immich-redis", + &immich_present, + &mapped + )); + } + #[test] fn command_drift_tolerates_quadlet_entrypoint_split() { // Quadlet writes Entrypoint=sh + Exec=-lc "