fix(reconciler): recreate an absent stack member when its siblings are live

The periodic reconcile runs ExistingOnly — merely listing a catalog
manifest must never install an app — and its only absent-container
recovery keyed on the last running-names snapshot, which ages out after
a few daemon restarts. An absent member of an installed stack then stays
absent forever: .38 ran indeedhub with no minio/postgres for days, nginx
down on 'host not found in upstream "minio"', and nothing ever put the
members back.

A live sibling container is proof the stack is installed on this node,
so an absent member is now treated as a hole to repair, not a choice to
respect: the recovery guard also fires when another member of the same
stack (app_ops::stack_member_app_ids) has a container in any state.
A stack with no containers at all is left untouched, and sibling app ids
resolve through the loaded-manifest container names (immich-postgres
runs as immich_postgres).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/archipelago/src/app_ops.rs

@@ -75,7 +75,8 @@ pub fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
 /// The package whose lifecycle lock covers `app_id`: the stack package when
 /// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while
 /// they drive archy-mempool-web), otherwise the app itself.
-fn owning_package(app_id: &str) -> &str {
+/// Also consulted by the reconciler's absent-stack-member recovery.
+pub fn owning_package(app_id: &str) -> &str {
     const STACKS: &[&str] = &[
         "immich",
         "indeedhub",

core/archipelago/src/container/prod_orchestrator.rs

@@ -104,6 +104,32 @@ fn dependency_manifests_required_by_active_apps<'a>(
     required
 }
 
+/// Whether `app_id` is a member of a known multi-container stack that has at
+/// least one OTHER member with a live container (any state). A live sibling
+/// proves the stack is installed on this node, so an absent member is a hole
+/// to repair — while a stack with no containers at all stays untouched
+/// (uninstalled, or never installed here). Sibling app ids resolve to
+/// container names through the loaded-manifest map when available (immich's
+/// `immich-postgres` app id runs as container `immich_postgres`), falling
+/// back to the id itself.
+fn absent_stack_member_with_live_sibling(
+    app_id: &str,
+    present_containers: &HashSet<String>,
+    container_name_by_app_id: &std::collections::HashMap<String, String>,
+) -> bool {
+    let stack = crate::app_ops::owning_package(app_id);
+    let members = crate::app_ops::stack_member_app_ids(stack);
+    members.iter().any(|member| {
+        *member != app_id
+            && present_containers.contains(
+                container_name_by_app_id
+                    .get(*member)
+                    .map(String::as_str)
+                    .unwrap_or(member),
+            )
+    })
+}
+
 fn manifest_dependency_app_ids(manifest: &AppManifest) -> Vec<String> {
     manifest
         .app
@@ -1654,13 +1680,16 @@ impl ProdContainerOrchestrator {
         // app whose container vanished (e.g. a wedged teardown cleared by a
         // reboot) instead of leaving it down. See the immich .198 incident.
         let was_running = crate::crash_recovery::load_last_running_names(&self.data_dir).await;
-        let manifests: Vec<LoadedManifest> = {
+        let (manifests, container_name_by_app_id): (
+            Vec<LoadedManifest>,
+            std::collections::HashMap<String, String>,
+        ) = {
             let state = self.state.read().await;
             let dependency_required = dependency_manifests_required_by_active_apps(
                 state.manifests.values().map(|lm| &lm.manifest),
                 &user_stopped,
             );
-            state
+            let filtered = state
                 .manifests
                 .iter()
                 .filter(|(app_id, _)| !state.disabled.contains(*app_id))
@@ -1670,8 +1699,25 @@ impl ProdContainerOrchestrator {
                             && !user_stopped.contains(&compute_container_name(&lm.manifest)))
                 })
                 .map(|(_, lm)| lm.clone())
-                .collect()
+                .collect();
+            // Unfiltered id→container-name map for the absent-stack-member
+            // recovery below: a sibling may be excluded from this pass (e.g.
+            // user-stopped) yet its live container still proves the stack is
+            // installed.
+            let names = state
+                .manifests
+                .iter()
+                .map(|(id, lm)| (id.clone(), compute_container_name(&lm.manifest)))
+                .collect();
+            (filtered, names)
         };
+        // Live container names (any state), for the same recovery check.
+        let present_containers: std::collections::HashSet<String> = self
+            .runtime
+            .list_containers()
+            .await
+            .map(|cs| cs.into_iter().map(|c| c.name).collect())
+            .unwrap_or_default();
         let mut report = ReconcileReport::default();
         let disk_gb = self.disk_gb().await;
         // Register every candidate before the (sequential, possibly slow)
@@ -1738,7 +1784,20 @@ impl ProdContainerOrchestrator {
                 Ok(ReconcileAction::Left(reason))
                     if mode == ReconcileMode::ExistingOnly
                         && reason == "absent"
-                        && was_running.contains(&compute_container_name(&lm.manifest)) =>
+                        && (was_running.contains(&compute_container_name(&lm.manifest))
+                            // Absent STACK MEMBER whose siblings have live
+                            // containers: the stack is installed, so the
+                            // missing member is a hole, not a choice. The
+                            // was_running snapshot ages out after a few daemon
+                            // restarts, which left indeedhub-minio/-postgres
+                            // permanently absent on .38 (2026-08-06) — nginx
+                            // down on `host not found in upstream "minio"`
+                            // with nothing ever recreating the members.
+                            || absent_stack_member_with_live_sibling(
+                                &app_id,
+                                &present_containers,
+                                &container_name_by_app_id,
+                            )) =>
                 {
                     tracing::warn!(
                         app_id = %app_id,
@@ -4451,6 +4510,62 @@ mod tests {
         items.iter().map(|s| s.to_string()).collect()
     }
 
+    /// The .38 indeedhub incident class: an absent stack member must be
+    /// recovered when its siblings have live containers (the stack is
+    /// installed), and left alone when the whole stack is gone or the app
+    /// is not a stack member at all.
+    #[test]
+    fn absent_stack_member_recovery_requires_a_live_sibling() {
+        let present: HashSet<String> = ["indeedhub-redis", "indeedhub-relay", "indeedhub"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+        let names = std::collections::HashMap::new();
+        // Missing members of a stack with live siblings → recover.
+        assert!(absent_stack_member_with_live_sibling(
+            "indeedhub-minio",
+            &present,
+            &names
+        ));
+        assert!(absent_stack_member_with_live_sibling(
+            "indeedhub-postgres",
+            &present,
+            &names
+        ));
+        // Whole stack absent → NOT recovered (uninstalled stays uninstalled).
+        let empty = HashSet::new();
+        assert!(!absent_stack_member_with_live_sibling(
+            "indeedhub-minio",
+            &empty,
+            &names
+        ));
+        // Non-stack app → never.
+        assert!(!absent_stack_member_with_live_sibling(
+            "vaultwarden",
+            &present,
+            &names
+        ));
+        // An app's OWN container being present proves nothing about siblings.
+        let only_self: HashSet<String> =
+            std::iter::once("indeedhub-minio".to_string()).collect();
+        assert!(!absent_stack_member_with_live_sibling(
+            "indeedhub-minio",
+            &only_self,
+            &names
+        ));
+        // App-id → container-name mapping is honoured (immich_postgres runs
+        // under an underscore name while its app id is hyphenated).
+        let mut mapped = std::collections::HashMap::new();
+        mapped.insert("immich-postgres".to_string(), "immich_postgres".to_string());
+        let immich_present: HashSet<String> =
+            std::iter::once("immich_postgres".to_string()).collect();
+        assert!(absent_stack_member_with_live_sibling(
+            "immich-redis",
+            &immich_present,
+            &mapped
+        ));
+    }
+
     #[test]
     fn command_drift_tolerates_quadlet_entrypoint_split() {
         // Quadlet writes Entrypoint=sh + Exec=-lc "<script>", so podman