From 9471d3727f262a88280ef911af9ba09df40942ae Mon Sep 17 00:00:00 2001 From: archipelago Date: Tue, 14 Jul 2026 09:26:09 -0400 Subject: [PATCH] fix(orchestrator): boot reconciler no longer installs apps on manifest presence alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 936b4cca made ANY loaded manifest with an absent container self-heal via install_fresh — but the signed catalog and the ISO ship manifests for EVERY app, installed or not, so 'manifest loaded' is not installation evidence. On a fresh node this mass-installed the catalog: the framework node grew portainer, vaultwarden, searxng, strfry, mempool, immich and fedimint uninvited within an hour (fleet nodes showed the same loop as reconcile-failed noise for unpullable images). ExistingOnly (boot) reconcile now creates containers from nothing only for required-baseline apps (filebrowser, fedimint-clientd); every other absent app is Left('absent') and recreated ONLY via desired-state recovery when it was running at the last snapshot — which still covers the vanished-container regressions (indeedhub/immich) that motivated 936b4cca. Reconcile suite 16/16. Co-Authored-By: Claude Fable 5 --- .../src/container/prod_orchestrator.rs | 59 ++++++++++++------- core/archipelago/src/crash_recovery.rs | 19 ++++++ 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 7e3ca2e5..e9494028 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -50,6 +50,18 @@ use crate::update::host_sudo; const UI_APP_IDS: &[&str] = &["bitcoin-ui", "electrs-ui", "lnd-ui"]; const ARCHIVAL_BITCOIN_DISK_GB: u64 = 1000; +/// Apps expected to exist from first boot on every node — the ONLY apps the +/// boot reconciler may install from nothing. Every other app needs +/// installation evidence (an existing container, or the was-running snapshot +/// handled by the caller's desired-state recovery). Without this gate, +/// "manifest loaded" counted as "installed" — and since the catalog + ISO ship +/// manifests for EVERY app, a fresh node mass-installed the entire catalog +/// (framework node 2026-07-14: portainer/vaultwarden/searxng/strfry/mempool/ +/// fedimint appeared uninvited within an hour of first boot). +fn is_required_baseline_app(app_id: &str) -> bool { + matches!(app_id, "filebrowser" | "fedimint-clientd") +} + fn is_restart_sensitive_app(app_id: &str) -> bool { matches!( app_id, @@ -2197,20 +2209,25 @@ impl ProdContainerOrchestrator { return Ok(ReconcileAction::Started); } - // By this point `app_id` is neither user-stopped nor - // user-uninstalled (both checked earlier in this fn) and its - // manifest is still loaded — i.e. it's a genuinely-installed - // app whose container is simply gone (crash, lost record, - // wedged teardown cleared by reboot). It must self-heal - // regardless of whether it happens to be one of the hardcoded - // "required baseline" apps: an app the user installed and - // never removed should come back on its own, the same as - // baseline services always have. `is_required_baseline_app` - // used to gate this and left every other installed-but-absent - // app (e.g. a stack's backend containers) stuck forever. + // Container absent. A loaded manifest is NOT installation + // evidence — the catalog and the ISO ship manifests for every + // app, installed or not. In ExistingOnly (boot) mode only two + // things may create a container from nothing: + // 1. required-baseline apps (first-boot bootstrap), here; + // 2. desired-state recovery for apps whose container was + // running at the last snapshot — the caller matches + // Left("absent") against the snapshot and recreates + // (this is what heals a stack backend that vanished: + // the indeedhub/immich cases). + // Everything else stays absent. Installing on manifest + // presence alone mass-installed the whole catalog on a fresh + // node (framework, 2026-07-14). if mode == ReconcileMode::ExistingOnly { - self.install_fresh(lm).await?; - return Ok(ReconcileAction::Installed); + if is_required_baseline_app(&app_id) { + self.install_fresh(lm).await?; + return Ok(ReconcileAction::Installed); + } + return Ok(ReconcileAction::Left("absent".to_string())); } self.install_fresh(lm).await?; Ok(ReconcileAction::Installed) @@ -5773,13 +5790,13 @@ app: #[tokio::test] async fn reconcile_existing_self_heals_missing_optional_installed_app() { - // A non-baseline app (gitea) whose manifest is still loaded (i.e. - // genuinely installed, not user-uninstalled — see the - // durable-user-uninstalled-marker test above for that case) must - // self-heal the same as a required baseline app when its container - // is fully gone. Leaving any installed-but-absent app stuck forever - // regressed a real node (indeedhub's backend containers never came - // back after going absent) — self-heal is no longer baseline-only. + // A non-baseline app (gitea) self-heals ONLY with installation + // evidence: its container was running at the last periodic snapshot + // (desired-state recovery). Manifest presence alone must NOT install + // — the catalog ships manifests for every app, and treating them as + // installed mass-installed the catalog on a fresh node (2026-07-14). + // The indeedhub/immich vanished-container cases are exactly the + // snapshot-covered scenario exercised here. let rt = Arc::new(MockRuntime::default()); let mut orch = orch_with(rt.clone()).await; orch.set_disk_gb_for_test(500); @@ -5788,6 +5805,8 @@ app: PathBuf::from("/tmp/gitea"), ) .await; + // Installation evidence: gitea was running at the last snapshot. + crate::crash_recovery::save_container_snapshot_for_test(&orch.data_dir, &["gitea"]).await; let report = orch.reconcile_existing().await; diff --git a/core/archipelago/src/crash_recovery.rs b/core/archipelago/src/crash_recovery.rs index e65cce43..9831ab78 100644 --- a/core/archipelago/src/crash_recovery.rs +++ b/core/archipelago/src/crash_recovery.rs @@ -118,6 +118,25 @@ pub async fn load_last_running_names(data_dir: &Path) -> std::collections::HashS } } +/// Test helper: write a running-containers snapshot with the given names — +/// the "installation evidence" the boot reconciler's desired-state recovery +/// reads via `load_last_running_names`. +#[cfg(test)] +pub async fn save_container_snapshot_for_test(data_dir: &Path, names: &[&str]) { + let snapshot = ContainerSnapshot { + timestamp: 0, + containers: names + .iter() + .map(|n| RunningContainerRecord { + name: n.to_string(), + image: format!("test/{n}:latest"), + }) + .collect(), + }; + let path = data_dir.join(CONTAINER_STATE_FILE); + let _ = fs::write(&path, serde_json::to_string_pretty(&snapshot).unwrap()).await; +} + /// Save the set of user-stopped containers to disk. pub async fn save_user_stopped(data_dir: &Path, stopped: &std::collections::HashSet) { let path = data_dir.join(USER_STOPPED_FILE);