fix(orchestrator): durable uninstall marker for baseline apps + archival-bitcoin/version-report gaps

- mempool-api now declares dependencies:[bitcoin:archival] directly, closing a
  gap where installing it standalone (a legitimate direct orchestrator-install
  target) bypassed the mempool umbrella's pruning gate entirely.
- New durable user-uninstalled marker (crash_recovery.rs, mirrors user_stopped)
  fixes required-baseline-app self-heal (bitcoin-knots/electrumx/lnd/mempool/
  etc.) resurrecting itself after an explicit uninstall survives a restart or
  reboot, since the in-memory disabled set is wiped by every load_manifests().
- installed_version() (set_config.rs) no longer trusts a floating image tag
  ("latest") as the reported running version -- a stale local :latest cache
  reported "latest" forever regardless of what latest had moved on to. Now
  falls back to asking the Bitcoin backend directly via `bitcoind --version`
  when the tag is floating.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

core/archipelago/src/api/rpc/package/dependencies.rs

@@ -624,4 +624,20 @@ mod tests {
         // An id with no manifest on disk at all.
         assert!(!manifest_declares_archival_bitcoin("does-not-exist"));
     }
+
+    #[test]
+    fn mempool_api_is_directly_installable_and_covered_by_the_archival_gate() {
+        // `mempool-api` is a legitimate direct `package.install` target
+        // (`uses_orchestrator_install_flow` in install.rs), reachable without
+        // going through the `mempool`/`mempool-web` umbrella id that the old
+        // hardcoded fallback list only recognized. It was missing from that
+        // list, so installing/repairing it directly skipped the archival
+        // Bitcoin gate entirely. Its manifest now declares `bitcoin:archival`
+        // directly, closing the gap the manifest-driven path exists for.
+        assert!(requires_unpruned_bitcoin("mempool-api"));
+        assert!(manifest_declares_archival_bitcoin("mempool-api"));
+        // `archy-mempool-web` has no direct Bitcoin RPC access
+        // (bitcoin_integration.rpc_access: none) and correctly stays excluded.
+        assert!(!requires_unpruned_bitcoin("archy-mempool-web"));
+    }
 }

core/archipelago/src/api/rpc/package/set_config.rs

@@ -59,7 +59,54 @@ async fn installed_version(app_id: &str) -> Option<String> {
         return None;
     }
     let image = String::from_utf8_lossy(&out.stdout).trim().to_string();
-    image_tag(&image)
+    let tag = image_tag(&image)?;
+    // A floating tag (latest/stable/...) names the reference used to CREATE the
+    // container, not what's actually running — podman never re-resolves it once
+    // cached, so a stale local `:latest` reports "latest" even when the real
+    // `latest` moved on months ago (.228, 2026-07-01: ran a 4-month-old cached
+    // image while a newer one already sat locally, unused). Ask the Bitcoin
+    // backends directly instead of trusting the tag literal in that case.
+    if is_floating_tag(&tag) {
+        if let Some(real) = bitcoind_reported_version(app_id, name).await {
+            return Some(real);
+        }
+    }
+    Some(tag)
+}
+
+fn is_floating_tag(tag: &str) -> bool {
+    matches!(tag, "latest" | "stable" | "release" | "main")
+}
+
+/// Best-effort: ask the running bitcoind binary for its own version, trimmed to
+/// the catalog's version-tag format (e.g. `29.3.knots20260210`, `29.2`). `None`
+/// for apps other than the Bitcoin backends (no generic way to introspect a
+/// third-party image's content version this way) or if the exec fails.
+async fn bitcoind_reported_version(app_id: &str, container_name: &str) -> Option<String> {
+    if !matches!(app_id, "bitcoin-core" | "bitcoin-knots") {
+        return None;
+    }
+    let out = tokio::process::Command::new("podman")
+        .args(["exec", container_name, "bitcoind", "--version"])
+        .output()
+        .await
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    parse_bitcoind_version_output(&String::from_utf8_lossy(&out.stdout))
+}
+
+/// Parses e.g. "Bitcoin Knots daemon version v29.3.knots20260210\n..." or
+/// "Bitcoin Core version v29.2.0\n..." down to the version tag after `version v`.
+fn parse_bitcoind_version_output(output: &str) -> Option<String> {
+    let first_line = output.lines().next()?;
+    let (_, version) = first_line.rsplit_once("version v")?;
+    let version = version.trim();
+    if version.is_empty() {
+        return None;
+    }
+    Some(version.to_string())
 }
 
 impl RpcHandler {
@@ -248,7 +295,42 @@ impl RpcHandler {
 
 #[cfg(test)]
 mod tests {
-    use super::image_tag;
+    use super::{image_tag, is_floating_tag, parse_bitcoind_version_output};
+
+    #[test]
+    fn floating_tag_detects_generic_channel_names() {
+        for tag in ["latest", "stable", "release", "main"] {
+            assert!(is_floating_tag(tag), "{tag}");
+        }
+        for tag in ["29.3.knots20260508", "28.4", "v29.2.0"] {
+            assert!(!is_floating_tag(tag), "{tag}");
+        }
+    }
+
+    #[test]
+    fn parses_knots_version_line() {
+        assert_eq!(
+            parse_bitcoind_version_output(
+                "Bitcoin Knots daemon version v29.3.knots20260210\nCopyright...\n"
+            )
+            .as_deref(),
+            Some("29.3.knots20260210")
+        );
+    }
+
+    #[test]
+    fn parses_core_version_line() {
+        assert_eq!(
+            parse_bitcoind_version_output("Bitcoin Core version v29.2.0\n").as_deref(),
+            Some("29.2.0")
+        );
+    }
+
+    #[test]
+    fn parse_returns_none_when_output_has_no_version_marker() {
+        assert_eq!(parse_bitcoind_version_output("garbage output\n"), None);
+        assert_eq!(parse_bitcoind_version_output(""), None);
+    }
 
     #[test]
     fn image_tag_keeps_registry_port_colon() {

core/archipelago/src/container/prod_orchestrator.rs

@@ -1567,6 +1567,25 @@ impl ProdContainerOrchestrator {
             }
         }
 
+        // Same durability problem as user-stopped above, but for uninstall:
+        // `is_required_baseline_app` below otherwise self-heals bitcoin-knots,
+        // electrumx, lnd, mempool, etc. the moment their container is missing —
+        // including right after an explicit uninstall, since the in-memory
+        // `disabled` set doesn't survive a `load_manifests()` reload (every
+        // archipelago restart/reboot runs one before the first reconcile).
+        {
+            let user_uninstalled =
+                crate::crash_recovery::load_user_uninstalled(&self.data_dir).await;
+            if user_uninstalled.contains(&app_id) || user_uninstalled.contains(&name) {
+                tracing::debug!(
+                    app_id = %app_id,
+                    container = %name,
+                    "reconcile skipped — app is user-uninstalled (must stay removed)"
+                );
+                return Ok(ReconcileAction::Left("user-uninstalled".into()));
+            }
+        }
+
         match self.runtime.get_container_status(&name).await {
             Ok(status) => {
                 // Phase 3.3: migrate pre-Phase-3 containers in place, but only
@@ -3317,6 +3336,10 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
         // `ensure_running_with_mode` doesn't skip the very container we're
         // installing. (start/restart RPC handlers clear it on their side too.)
         crate::crash_recovery::clear_user_stopped(&self.data_dir, app_id).await;
+        // Same for the user-uninstalled marker — otherwise re-installing a
+        // baseline app the user had previously uninstalled would hit the
+        // same reconcile guard and silently no-op.
+        crate::crash_recovery::clear_user_uninstalled(&self.data_dir, app_id).await;
         // Idempotent: if the container is already up and healthy, just
         // refresh hooks and return. If it's stopped, start it. If it's
         // missing or in a wedged state, install fresh.
@@ -3364,6 +3387,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
         // `ensure_running_with_mode` doesn't skip this container (symmetric with
         // install; the start/restart RPC handlers also clear it).
         crate::crash_recovery::clear_user_stopped(&self.data_dir, app_id).await;
+        crate::crash_recovery::clear_user_uninstalled(&self.data_dir, app_id).await;
         let lm = self.loaded(app_id).await?;
         let action = self.ensure_running(&lm).await?;
         match action {
@@ -3532,11 +3556,20 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
             }
         }
 
-        let mut state = self.state.write().await;
-        state.disabled.insert(app_id.to_string());
+        {
+            let mut state = self.state.write().await;
+            state.disabled.insert(app_id.to_string());
+        }
         if let Some(e) = remove_err {
             return Err(e);
         }
+        // Durable, unlike `state.disabled` above (wiped by every
+        // `load_manifests()`, which runs on every archipelago restart/reboot
+        // before the first reconcile) — without this, `is_required_baseline_app`
+        // self-heals bitcoin-knots/electrumx/lnd/mempool/etc. right back after
+        // an explicit uninstall survives to the next restart. Only mark on the
+        // success path above — a failed removal means the app isn't actually gone.
+        crate::crash_recovery::mark_user_uninstalled(&self.data_dir, app_id).await;
         Ok(())
     }
 
@@ -4611,6 +4644,38 @@ app:
         assert!(calls.iter().any(|c| c == "start_container:filebrowser"));
     }
 
+    #[tokio::test]
+    async fn reconcile_existing_respects_durable_user_uninstalled_marker_for_baseline_apps() {
+        let rt = Arc::new(MockRuntime::default());
+        let mut orch = orch_with(rt.clone()).await;
+        orch.set_disk_gb_for_test(500);
+        orch.insert_manifest_for_test(
+            pull_manifest("filebrowser", "docker.io/filebrowser/filebrowser:latest"),
+            PathBuf::from("/tmp/filebrowser"),
+        )
+        .await;
+        // Simulates the real-world sequence: user uninstalls a required
+        // baseline app, then archipelago restarts (which wipes the in-memory
+        // `disabled` set via `load_manifests()` before this harness even
+        // gets involved) — the durable marker is all that's left standing
+        // between that and the baseline self-heal below reinstalling it.
+        crate::crash_recovery::mark_user_uninstalled(&orch.data_dir, "filebrowser").await;
+
+        let report = orch.reconcile_existing().await;
+
+        assert_eq!(
+            report.actions,
+            vec![(
+                "filebrowser".to_string(),
+                ReconcileAction::Left("user-uninstalled".to_string())
+            )]
+        );
+        assert!(report.failures.is_empty());
+        let calls = rt.calls();
+        assert!(!calls.iter().any(|c| c.starts_with("pull_image:")));
+        assert!(!calls.iter().any(|c| c.starts_with("create_container:")));
+    }
+
     #[tokio::test]
     async fn reconcile_existing_skips_archival_baseline_apps_on_pruned_hosts() {
         let rt = Arc::new(MockRuntime::default());

core/archipelago/src/crash_recovery.rs

@@ -20,6 +20,7 @@ use tracing::{info, warn};
 const PID_FILE: &str = "archipelago.pid";
 const CONTAINER_STATE_FILE: &str = "running-containers.json";
 const USER_STOPPED_FILE: &str = "user-stopped.json";
+const USER_UNINSTALLED_FILE: &str = "user-uninstalled.json";
 
 /// Shared flag: true once boot recovery is complete. Health monitor should wait for this.
 pub static RECOVERY_COMPLETE: AtomicBool = AtomicBool::new(false);
@@ -100,6 +101,51 @@ pub async fn clear_user_stopped(data_dir: &Path, name: &str) {
     }
 }
 
+// ── User-uninstalled tracking ───────────────────────────────────────────
+// Baseline apps (bitcoin-knots, electrumx, lnd, mempool, ...) self-heal when
+// their container is missing — see `is_required_baseline_app` in
+// prod_orchestrator.rs — because they're expected to exist from first boot.
+// That self-heal has no way to distinguish "container vanished after a
+// crash" from "user explicitly uninstalled this," and the in-memory
+// `disabled` set the orchestrator otherwise uses is wiped by every
+// `load_manifests()` call (once per archipelago startup). Without a durable
+// marker, uninstalling a baseline app only "sticks" until the next reboot or
+// archipelago restart, at which point the boot reconciler resurrects it.
+// This mirrors `user_stopped` exactly, just for uninstall instead of stop.
+
+/// Load the set of explicitly user-uninstalled app/container names from disk.
+pub async fn load_user_uninstalled(data_dir: &Path) -> std::collections::HashSet<String> {
+    let path = data_dir.join(USER_UNINSTALLED_FILE);
+    match fs::read_to_string(&path).await {
+        Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
+        Err(_) => std::collections::HashSet::new(),
+    }
+}
+
+/// Save the set of user-uninstalled app/container names to disk.
+pub async fn save_user_uninstalled(data_dir: &Path, uninstalled: &std::collections::HashSet<String>) {
+    let path = data_dir.join(USER_UNINSTALLED_FILE);
+    if let Ok(json) = serde_json::to_string_pretty(uninstalled) {
+        let _ = fs::write(&path, json).await;
+    }
+}
+
+/// Mark a name as user-uninstalled (won't be self-healed by the baseline-app
+/// reconciler across restarts/reboots).
+pub async fn mark_user_uninstalled(data_dir: &Path, name: &str) {
+    let mut uninstalled = load_user_uninstalled(data_dir).await;
+    uninstalled.insert(name.to_string());
+    save_user_uninstalled(data_dir, &uninstalled).await;
+}
+
+/// Clear the user-uninstalled flag (app was explicitly (re)installed/started).
+pub async fn clear_user_uninstalled(data_dir: &Path, name: &str) {
+    let mut uninstalled = load_user_uninstalled(data_dir).await;
+    if uninstalled.remove(name) {
+        save_user_uninstalled(data_dir, &uninstalled).await;
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RunningContainerRecord {
     pub name: String,