fix: release v1.7.50-alpha OTA runtime repair

Hotfix: archipelago.service ExecStartPre now mkdirs /run/containers and
/var/lib/containers before the unit's mount-namespace setup tries to bind
them. Without this, fresh nodes that don't have /run/containers (e.g.
nodes provisioned without a prior podman session) fail at the namespace
step with:

  Failed to set up mount namespacing: /run/containers: No such file or directory
  Failed at step NAMESPACE spawning /bin/bash: No such file or directory

Existing nodes don't pick up systemd unit changes via OTA — they need a
one-time `systemctl edit archipelago` adding the same mkdir. ISO installs
from this version forward have the fix baked in.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v1.7.49-alpha (2026-04-30)
+
+- Bitcoin Knots/Core UI now reports connection, reconnecting, syncing, and error states from a backend status bridge instead of showing a stale "Unable to connect" message while the node is warming up.
+- ElectrumX UI now exposes indexed height, local Bitcoin height, known headers, status, and progress source so indexing/waiting states are readable during long initial sync.
+- Added container doctor timer and smoke/lifecycle test coverage for Bitcoin Knots/Core, ElectrumX, Mempool, BTCPay/NBXplorer, and UI surface availability.
+- Bitcoin Core and Bitcoin Knots are mutually exclusive variants, with a real Bitcoin Core manifest and corrected install conflict handling.
+- IndeeHub now launches only on direct web UI port `7778`; the broken `/app/indeedhub/` path proxy was removed, and port `7777` remains the Nostr relay.
+- BTCPay/NBXplorer Postgres environment formatting fixed so installs do not carry malformed connection strings.
+
+## v1.7.48-alpha (2026-04-29)
+
+- archipelago.service no longer fails to start with "Failed to set up mount namespacing: /run/containers: No such file or directory" on nodes where /run/containers wasn't pre-created. ExecStartPre now creates it. Existing nodes need a one-time `systemctl edit archipelago` to add the mkdir; ISO installs from this version forward have the fix baked in.
+
 ## v1.7.47-alpha (2026-04-29)
 
 - Bitcoin Knots/Core sync is now significantly faster. The container now uses every available core for script verification (was capped at 2) and has 8GB of memory instead of 4GB so its 4GB UTXO cache has headroom for the mempool and peer connections. Existing nodes pick up the new limits on next install/update; freshly-installed nodes start at full speed.

app-catalog/catalog.json

@@ -26,7 +26,7 @@
       "id": "bitcoin-core",
       "title": "Bitcoin Core",
       "version": "28.4",
-      "description": "Reference implementation of the Bitcoin protocol. Run a full node validating and relaying blocks.",
+      "description": "Reference Bitcoin node implementation. Alternative to Bitcoin Knots; uninstall Knots before switching.",
       "icon": "/assets/img/app-icons/bitcoin-core.svg",
       "author": "Bitcoin Core contributors",
       "category": "money",

apps/archy-nbxplorer/manifest.yml

@@ -47,7 +47,7 @@ app:
     - NBXPLORER_BIND=0.0.0.0:32838
     - NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332
     - NBXPLORER_BTCRPCUSER=archipelago
-    - NBXPLORER_POSTGRES=User ID=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true
+    - NBXPLORER_POSTGRES=Username=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=nbxplorer
 
   health_check:
     type: http

apps/bitcoin-core/manifest.yml

@@ -1,13 +1,13 @@
 app:
   id: bitcoin-core
-  name: Bitcoin Knots
+  name: Bitcoin Core
   version: 28.4.0
-  description: Full Bitcoin Knots node with dynamic prune/full-mode startup based on host disk.
+  description: Reference Bitcoin Core node with dynamic prune/full-mode startup based on host disk.
 
-  container_name: bitcoin-knots
+  container_name: bitcoin-core
 
   container:
-    image: 146.59.87.168:3000/lfg2025/bitcoin-knots:latest
+    image: 146.59.87.168:3000/lfg2025/bitcoin:28.4
     pull_policy: if-not-present
     network: archy-net
     entrypoint: ["sh", "-lc"]

apps/bitcoin-knots/manifest.yml

@@ -0,0 +1,75 @@
+app:
+  id: bitcoin-knots
+  name: Bitcoin Knots
+  version: 28.1.0
+  description: Full Bitcoin Knots node with dynamic prune/full-mode startup based on host disk.
+
+  container_name: bitcoin-knots
+
+  container:
+    image: 146.59.87.168:3000/lfg2025/bitcoin-knots:latest
+    pull_policy: if-not-present
+    network: archy-net
+    entrypoint: ["sh", "-lc"]
+    custom_args:
+      # Sync-speed flags: -par=0 uses every core (was capped at 2 by
+      # --cpus=2, now removed for bitcoin/electrumx). -dbcache sized to
+      # the IBD sweet spot — 4GB on full nodes, 1GB on pruned. Container
+      # --memory=8g (config.rs::get_memory_limit) leaves headroom for
+      # mempool + connections.
+      - >-
+        if [ "${DISK_GB:-0}" -lt 1000 ]; then
+          exec bitcoind -server=1 -prune=550 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 -listen=1 -bind=0.0.0.0:8333 -dbcache=1024 -par=0 -maxconnections=125 -rpcuser="${BITCOIN_RPC_USER}" -rpcpassword="${BITCOIN_RPC_PASS}";
+        else
+          exec bitcoind -server=1 -txindex=1 -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 -listen=1 -bind=0.0.0.0:8333 -dbcache=4096 -par=0 -maxconnections=125 -rpcuser="${BITCOIN_RPC_USER}" -rpcpassword="${BITCOIN_RPC_PASS}";
+        fi
+    derived_env:
+      - key: DISK_GB
+        template: "{{DISK_GB}}"
+    secret_env:
+      - key: BITCOIN_RPC_PASS
+        secret_file: bitcoin-rpc-password
+    data_uid: "100101:100101"
+
+  dependencies:
+    - storage: 500Gi
+
+  resources:
+    cpu_limit: 0
+    memory_limit: 4Gi
+    disk_limit: 500Gi
+
+  security:
+    capabilities: [CHOWN, FOWNER, SETUID, SETGID, DAC_OVERRIDE]
+    readonly_root: false
+    network_policy: isolated
+
+  ports:
+    - host: 8332
+      container: 8332
+      protocol: tcp
+    - host: 8333
+      container: 8333
+      protocol: tcp
+
+  volumes:
+    - type: bind
+      source: /var/lib/archipelago/bitcoin
+      target: /home/bitcoin/.bitcoin
+      options: [rw]
+
+  environment:
+    - BITCOIN_RPC_USER=archipelago
+
+  health_check:
+    type: tcp
+    endpoint: localhost:8332
+    interval: 30s
+    timeout: 5s
+    retries: 3
+
+  bitcoin_integration:
+    rpc_access: admin
+    sync_required: true
+    testnet_support: false
+    pruning_support: true

apps/btcpay-server/manifest.yml

@@ -51,7 +51,7 @@ app:
     - BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838
     - BTCPAY_BTCRPCURL=http://bitcoin-knots:8332
     - BTCPAY_BTCRPCUSER=archipelago
-    - BTCPAY_POSTGRES=User ID=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true
+    - BTCPAY_POSTGRES=Username=btcpay;Password=${BTCPAY_DB_PASS};Host=archy-btcpay-db;Port=5432;Database=btcpay
 
   health_check:
     type: http

apps/indeedhub/manifest.yml

@@ -27,9 +27,9 @@ app:
     apparmor_profile: default
 
   ports:
-    - host: 7777
-      container: 3000
-      protocol: tcp  # Web UI (Next.js)
+    - host: 7778
+      container: 7777
+      protocol: tcp  # Web UI. Port 7777 on the host is reserved for Nostr relay.
 
   volumes:
     - type: tmpfs
@@ -57,7 +57,7 @@ app:
       name: Web UI
       description: Stream Bitcoin documentaries with Nostr identity
       type: ui
-      port: 7777
+      port: 7778
       protocol: http
       path: /
 

core/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
 
 [[package]]
 name = "archipelago"
-version = "1.7.47-alpha"
+version = "1.7.50-alpha"
 dependencies = [
  "anyhow",
  "archipelago-container",

core/archipelago/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "archipelago"
-version = "1.7.47-alpha"
+version = "1.7.50-alpha"
 edition = "2021"
 description = "Archipelago Bitcoin Node OS - Native backend"
 authors = ["Archipelago Team"]

core/archipelago/src/api/handler/mod.rs

@@ -429,6 +429,7 @@ impl ApiHandler {
 
             // Electrs status — unauthenticated (read-only sync status)
             (Method::GET, "/electrs-status") => Self::handle_electrs_status().await,
+            (Method::GET, "/bitcoin-status") => Self::handle_bitcoin_status().await,
 
             // App-catalog proxy — fetches catalog.json from the configured
             // upstream URLs server-side so the browser doesn't hit CORS

core/archipelago/src/api/handler/proxy.rs

@@ -1,5 +1,6 @@
 use super::build_response;
 use crate::api::rpc::RpcHandler;
+use crate::bitcoin_status;
 use crate::electrs_status;
 use anyhow::Result;
 use hyper::{Response, StatusCode};
@@ -76,11 +77,23 @@ impl ApiHandler {
     pub(super) async fn handle_electrs_status() -> Result<Response<hyper::Body>> {
         let status = electrs_status::get_electrs_sync_status().await;
         let body = serde_json::to_vec(&status).unwrap_or_default();
-        Ok(build_response(
-            StatusCode::OK,
-            "application/json",
-            hyper::Body::from(body),
-        ))
+        Ok(Response::builder()
+            .status(StatusCode::OK)
+            .header("Content-Type", "application/json")
+            .header("Cache-Control", "no-store")
+            .body(hyper::Body::from(body))
+            .unwrap_or_else(|_| Response::new(hyper::Body::from("{}"))))
+    }
+
+    pub(super) async fn handle_bitcoin_status() -> Result<Response<hyper::Body>> {
+        let status = bitcoin_status::get_bitcoin_status().await;
+        let body = serde_json::to_vec(&status).unwrap_or_default();
+        Ok(Response::builder()
+            .status(StatusCode::OK)
+            .header("Content-Type", "application/json")
+            .header("Cache-Control", "no-store")
+            .body(hyper::Body::from(body))
+            .unwrap_or_else(|_| Response::new(hyper::Body::from("{}"))))
     }
 
     pub(super) async fn handle_lnd_connect_info(

core/archipelago/src/api/rpc/package/install.rs

@@ -229,6 +229,7 @@ impl RpcHandler {
         let deps = detect_running_deps().await?;
         check_install_deps(package_id, &deps)?;
         log_optional_dep_info(package_id, &deps);
+        check_bitcoin_implementation_conflict(package_id).await?;
 
         // Check if container already exists
         let check_output = tokio::process::Command::new("podman")
@@ -1961,9 +1962,51 @@ fn should_try_orchestrator_install(package_id: &str, orchestrator_available: boo
     orchestrator_available && uses_orchestrator_install_flow(package_id)
 }
 
+async fn check_bitcoin_implementation_conflict(package_id: &str) -> Result<()> {
+    let other = match package_id {
+        "bitcoin-core" => "bitcoin-knots",
+        "bitcoin-knots" => "bitcoin-core",
+        _ => return Ok(()),
+    };
+
+    let output = tokio::process::Command::new("podman")
+        .args([
+            "ps",
+            "-a",
+            "--format",
+            "{{.Names}}",
+            "--filter",
+            &format!("name=^{}$", other),
+        ])
+        .output()
+        .await
+        .context("Failed to check existing Bitcoin node containers")?;
+
+    if String::from_utf8_lossy(&output.stdout).trim().is_empty() {
+        return Ok(());
+    }
+
+    let current = match other {
+        "bitcoin-core" => "Bitcoin Core",
+        "bitcoin-knots" => "Bitcoin Knots",
+        _ => "another Bitcoin node",
+    };
+    let requested = match package_id {
+        "bitcoin-core" => "Bitcoin Core",
+        "bitcoin-knots" => "Bitcoin Knots",
+        _ => "the requested Bitcoin node",
+    };
+
+    Err(anyhow::anyhow!(
+        "{} is already installed. Stop and uninstall {} before installing {}; both implementations use the same Bitcoin data directory and ports.",
+        current,
+        current,
+        requested
+    ))
+}
+
 fn orchestrator_install_app_id(package_id: &str) -> &str {
     match package_id {
-        "bitcoin-knots" => "bitcoin-core",
         "electrs" | "mempool-electrs" => "electrumx",
         _ => package_id,
     }
@@ -2049,7 +2092,8 @@ mod tests {
 
     #[test]
     fn install_aliases_map_to_manifest_app_ids() {
-        assert_eq!(orchestrator_install_app_id("bitcoin-knots"), "bitcoin-core");
+        assert_eq!(orchestrator_install_app_id("bitcoin-knots"), "bitcoin-knots");
+        assert_eq!(orchestrator_install_app_id("bitcoin-core"), "bitcoin-core");
         assert_eq!(orchestrator_install_app_id("electrs"), "electrumx");
         assert_eq!(orchestrator_install_app_id("mempool-electrs"), "electrumx");
         assert_eq!(orchestrator_install_app_id("lnd"), "lnd");

core/archipelago/src/api/rpc/tor/mod.rs

@@ -355,7 +355,7 @@ pub(in crate::api::rpc) fn known_service_port(name: &str) -> u16 {
         "penpot" => 9001,
         "nginx-proxy-manager" => 81,
         "vaultwarden" => 8343,
-        "indeedhub" => 7777,
+        "indeedhub" => 7778,
         _ => 0,
     }
 }

core/archipelago/src/bitcoin_status.rs

@@ -0,0 +1,186 @@
+//! Cached Bitcoin node status for browser UIs.
+//!
+//! The bitcoin-ui should not poll Bitcoin RPC directly for display state.
+//! During container restarts, reindexing, and IBD, direct browser RPC polling
+//! turns short RPC gaps into visible UI failures. This module owns the RPC
+//! polling loop, caches the last successful snapshot, and serves stale-but-known
+//! state while the node is reconnecting.
+
+use anyhow::{Context, Result};
+use serde::Serialize;
+use std::sync::OnceLock;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::sync::RwLock;
+use tracing::{debug, warn};
+
+const CACHE_REFRESH_SECS: u64 = 5;
+
+#[derive(Debug, Clone, Serialize)]
+pub struct BitcoinNodeStatus {
+    pub ok: bool,
+    pub stale: bool,
+    pub updated_at_ms: u64,
+    pub error: Option<String>,
+    pub blockchain_info: Option<serde_json::Value>,
+    pub network_info: Option<serde_json::Value>,
+    pub index_info: Option<serde_json::Value>,
+    pub zmq_notifications: Option<serde_json::Value>,
+}
+
+impl Default for BitcoinNodeStatus {
+    fn default() -> Self {
+        Self {
+            ok: false,
+            stale: false,
+            updated_at_ms: 0,
+            error: Some("Connecting to Bitcoin node...".to_string()),
+            blockchain_info: None,
+            network_info: None,
+            index_info: None,
+            zmq_notifications: None,
+        }
+    }
+}
+
+static STATUS_CACHE: OnceLock<RwLock<BitcoinNodeStatus>> = OnceLock::new();
+
+fn cache() -> &'static RwLock<BitcoinNodeStatus> {
+    STATUS_CACHE.get_or_init(|| RwLock::new(BitcoinNodeStatus::default()))
+}
+
+fn now_ms() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+fn transient_error(err_msg: &str) -> bool {
+    let lower = err_msg.to_lowercase();
+    lower.contains("connect")
+        || lower.contains("reset")
+        || lower.contains("refused")
+        || lower.contains("timed out")
+        || lower.contains("timeout")
+        || lower.contains("broken pipe")
+        || lower.contains("eof")
+        || lower.contains("500 internal server error")
+}
+
+pub fn spawn_status_cache() {
+    tokio::spawn(async {
+        loop {
+            let fresh = fetch_bitcoin_status().await;
+            let mut cached = cache().write().await;
+            match fresh {
+                Ok(mut status) => {
+                    status.ok = true;
+                    status.stale = false;
+                    status.error = None;
+                    *cached = status;
+                }
+                Err(e) => {
+                    let err_msg = e.to_string();
+                    if transient_error(&err_msg) {
+                        debug!("Bitcoin status: transient RPC failure: {}", err_msg);
+                    } else {
+                        warn!("Bitcoin status: RPC failure: {}", err_msg);
+                    }
+
+                    if cached.blockchain_info.is_some() {
+                        cached.ok = false;
+                        cached.stale = true;
+                        cached.error = Some(format!(
+                            "Bitcoin node is reconnecting; showing last known state: {}",
+                            err_msg
+                        ));
+                    } else {
+                        *cached = BitcoinNodeStatus {
+                            ok: false,
+                            stale: false,
+                            updated_at_ms: now_ms(),
+                            error: Some(format!("Connecting to Bitcoin node: {}", err_msg)),
+                            ..BitcoinNodeStatus::default()
+                        };
+                    }
+                }
+            }
+            drop(cached);
+            tokio::time::sleep(Duration::from_secs(CACHE_REFRESH_SECS)).await;
+        }
+    });
+}
+
+pub async fn get_bitcoin_status() -> BitcoinNodeStatus {
+    cache().read().await.clone()
+}
+
+async fn fetch_bitcoin_status() -> Result<BitcoinNodeStatus> {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(8))
+        .build()
+        .context("build Bitcoin status HTTP client")?;
+
+    let blockchain_info = bitcoin_rpc_call(&client, "getblockchaininfo", serde_json::json!([]))
+        .await
+        .context("getblockchaininfo")?;
+    let network_info = bitcoin_rpc_call(&client, "getnetworkinfo", serde_json::json!([]))
+        .await
+        .context("getnetworkinfo")
+        .ok();
+    let index_info = bitcoin_rpc_call(&client, "getindexinfo", serde_json::json!([]))
+        .await
+        .context("getindexinfo")
+        .ok();
+    let zmq_notifications =
+        bitcoin_rpc_call(&client, "getzmqnotifications", serde_json::json!([]))
+            .await
+            .context("getzmqnotifications")
+            .ok();
+
+    Ok(BitcoinNodeStatus {
+        ok: true,
+        stale: false,
+        updated_at_ms: now_ms(),
+        error: None,
+        blockchain_info: Some(blockchain_info),
+        network_info,
+        index_info,
+        zmq_notifications,
+    })
+}
+
+async fn bitcoin_rpc_call(
+    client: &reqwest::Client,
+    method: &str,
+    params: serde_json::Value,
+) -> Result<serde_json::Value> {
+    let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
+    let body = serde_json::json!({
+        "jsonrpc": "1.0",
+        "id": "bitcoin-status",
+        "method": method,
+        "params": params,
+    });
+
+    let resp = client
+        .post(crate::constants::BITCOIN_RPC_URL)
+        .basic_auth(rpc_user, Some(rpc_pass))
+        .header("Content-Type", "application/json")
+        .json(&body)
+        .send()
+        .await
+        .context("Bitcoin RPC request failed")?;
+
+    let status = resp.status();
+    let json: serde_json::Value = resp.json().await.context("decode Bitcoin RPC JSON")?;
+    if !status.is_success() {
+        anyhow::bail!("Bitcoin RPC returned {}: {}", status, json);
+    }
+    if let Some(error) = json.get("error").filter(|e| !e.is_null()) {
+        anyhow::bail!("Bitcoin RPC {} error: {}", method, error);
+    }
+    json.get("result")
+        .cloned()
+        .context("missing Bitcoin RPC result")
+}

core/archipelago/src/bootstrap.rs

@@ -15,7 +15,7 @@
 //! best-effort — failures are logged but never abort the backend.
 
 use anyhow::{Context, Result};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use tokio::fs;
 use tracing::{debug, info, warn};
 
@@ -31,6 +31,7 @@ const DOCTOR_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-doctor.servic
 const DOCTOR_TIMER_PATH: &str = "/etc/systemd/system/archipelago-doctor.timer";
 
 const NGINX_CONF_PATH: &str = "/etc/nginx/sites-available/archipelago";
+const RUNTIME_ASSETS_DIR: &str = "/opt/archipelago/web-ui/archipelago-runtime";
 
 /// Inserted into every server block of the nginx config that lacks the
 /// `/api/app-catalog` proxy. Kept in sync with the canonical block in
@@ -40,6 +41,11 @@ const NGINX_APP_CATALOG_BLOCK: &str = "\n    # App Store catalog proxy — backe
 /// Entry point called from main startup. Never returns an error to the caller —
 /// failing to bootstrap host artifacts must not prevent the backend from serving.
 pub async fn ensure_doctor_installed() {
+    match run_runtime_assets().await {
+        Ok(changed) if changed => info!("Runtime assets synchronized from OTA payload"),
+        Ok(_) => debug!("No OTA runtime payload to synchronize"),
+        Err(e) => warn!("Runtime asset bootstrap failed (non-fatal): {:#}", e),
+    }
     match run().await {
         Ok(changed) if changed => info!("Doctor artifacts synchronized with binary"),
         Ok(_) => debug!("Doctor artifacts already in sync"),
@@ -52,6 +58,117 @@ pub async fn ensure_doctor_installed() {
     }
 }
 
+async fn run_runtime_assets() -> Result<bool> {
+    // The v1.7.50 OTA bridge puts scripts/apps/docker assets inside the
+    // frontend tarball because older binaries only know how to apply the
+    // backend binary and frontend archive. Once the new backend starts, it
+    // promotes that payload into /opt so app installs use the matching specs.
+    let runtime_dir = Path::new(RUNTIME_ASSETS_DIR);
+    if !runtime_dir.exists() {
+        return Ok(false);
+    }
+
+    let mut changed = false;
+    for (relative, dest) in [
+        ("apps", "/opt/archipelago/apps"),
+        ("scripts", "/opt/archipelago/scripts"),
+        ("docker", "/opt/archipelago/docker"),
+    ] {
+        let src = runtime_dir.join(relative);
+        if src.exists() {
+            replace_dir_from_runtime(&src, dest).await?;
+            if relative == "scripts" {
+                let _ = host_sudo(&[
+                    "find", dest, "-type", "f", "-name", "*.sh", "-exec", "chmod", "755", "{}", "+",
+                ])
+                .await;
+                let image_versions = format!("{}/image-versions.sh", dest);
+                if Path::new(&image_versions).exists() {
+                    let _ =
+                        host_sudo(&["cp", &image_versions, "/opt/archipelago/image-versions.sh"])
+                            .await;
+                }
+            }
+            changed = true;
+        }
+    }
+
+    let configs = runtime_dir.join("image-recipe/configs");
+    for unit in ["archipelago-doctor.service", "archipelago-doctor.timer"] {
+        let src = configs.join(unit);
+        if src.exists() {
+            let src_s = src.to_string_lossy().to_string();
+            let dest = format!("/etc/systemd/system/{}", unit);
+            let status = host_sudo(&["install", "-m", "644", &src_s, &dest])
+                .await
+                .with_context(|| format!("install {}", unit))?;
+            if !status.success() {
+                anyhow::bail!("install {} exited with {}", unit, status);
+            }
+            changed = true;
+        }
+    }
+
+    if changed {
+        let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
+        let _ = host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await;
+    }
+    Ok(changed)
+}
+
+async fn replace_dir_from_runtime(src: &Path, dest: &str) -> Result<()> {
+    let tmp = format!("{}.new.{}", dest, chrono::Utc::now().timestamp_millis());
+    let src_dot = path_dot(src);
+    let mkdir = host_sudo(&["mkdir", "-p", &tmp])
+        .await
+        .with_context(|| format!("mkdir {}", tmp))?;
+    if !mkdir.success() {
+        anyhow::bail!("mkdir {} exited with {}", tmp, mkdir);
+    }
+    let copy = host_sudo(&["cp", "-a", &src_dot, &tmp])
+        .await
+        .with_context(|| format!("copy runtime {} -> {}", src.display(), tmp))?;
+    if !copy.success() {
+        let _ = host_sudo(&["rm", "-rf", &tmp]).await;
+        anyhow::bail!("copy runtime {} exited with {}", src.display(), copy);
+    }
+    let _ = host_sudo(&["mkdir", "-p", dest]).await;
+    let cleanup = host_sudo(&[
+        "find",
+        dest,
+        "-mindepth",
+        "1",
+        "-maxdepth",
+        "1",
+        "-exec",
+        "rm",
+        "-rf",
+        "{}",
+        "+",
+    ])
+    .await
+    .with_context(|| format!("clean {}", dest))?;
+    if !cleanup.success() {
+        let _ = host_sudo(&["rm", "-rf", &tmp]).await;
+        anyhow::bail!("clean {} exited with {}", dest, cleanup);
+    }
+    let tmp_dot = format!("{}/.", tmp);
+    let promote = host_sudo(&["cp", "-a", &tmp_dot, dest])
+        .await
+        .with_context(|| format!("promote {} -> {}", tmp, dest))?;
+    let _ = host_sudo(&["rm", "-rf", &tmp]).await;
+    if !promote.success() {
+        anyhow::bail!("promote {} exited with {}", dest, promote);
+    }
+    Ok(())
+}
+
+fn path_dot(path: &Path) -> String {
+    let mut p = PathBuf::from(path);
+    p.push(".");
+    p.to_string_lossy().to_string()
+}
+
 async fn run() -> Result<bool> {
     // Dev-box guard: on contributors' laptops `/home/archipelago/archy` is
     // typically a symlink into the git checkout, and writing through it

core/archipelago/src/container/bitcoin_ui_nginx.conf.template

@@ -15,5 +15,13 @@ server {
         add_header Access-Control-Allow-Headers "Content-Type, Authorization";
         if ($request_method = OPTIONS) { return 204; }
     }
+    location /bitcoin-status {
+        proxy_pass http://127.0.0.1:5678/bitcoin-status;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        add_header Cache-Control "no-store";
+    }
     location / { try_files $uri $uri/ /index.html; }
 }

core/archipelago/src/electrs_status.rs

@@ -30,9 +30,11 @@ async fn bitcoin_rpc_auth() -> String {
 #[derive(Debug, Clone, Serialize)]
 pub struct ElectrsSyncStatus {
     pub indexed_height: u64,
+    pub bitcoin_height: u64,
     pub network_height: u64,
     pub progress_pct: f64,
     pub status: String,
+    pub stale: bool,
     pub error: Option<String>,
     /// Index data size in human-readable format (e.g. "11.2 GB")
     pub index_size: Option<String>,
@@ -44,9 +46,11 @@ impl Default for ElectrsSyncStatus {
     fn default() -> Self {
         Self {
             indexed_height: 0,
+            bitcoin_height: 0,
             network_height: 0,
             progress_pct: 0.0,
             status: "starting".to_string(),
+            stale: false,
             error: None,
             index_size: None,
             tor_onion: None,
@@ -64,15 +68,33 @@ fn cache() -> &'static RwLock<ElectrsSyncStatus> {
 /// Spawn background task that refreshes ElectrumX status every CACHE_REFRESH_SECS.
 pub fn spawn_status_cache() {
     tokio::spawn(async {
-        // Initial delay — let services start up before first query
-        tokio::time::sleep(Duration::from_secs(5)).await;
-
-        let mut interval = tokio::time::interval(Duration::from_secs(CACHE_REFRESH_SECS));
         loop {
-            interval.tick().await;
-            let fresh = fetch_electrs_sync_status().await;
+            let mut fresh = fetch_electrs_sync_status().await;
             let mut cached = cache().write().await;
+            if fresh.indexed_height == 0
+                && cached.indexed_height > 0
+                && matches!(fresh.status.as_str(), "indexing" | "waiting")
+            {
+                fresh.indexed_height = cached.indexed_height;
+                if fresh.network_height == 0 {
+                    fresh.network_height = cached.network_height;
+                }
+                if fresh.bitcoin_height == 0 {
+                    fresh.bitcoin_height = cached.bitcoin_height;
+                }
+                if fresh.progress_pct <= 0.0 {
+                    fresh.progress_pct = cached.progress_pct;
+                }
+                fresh.stale = true;
+                fresh.error = Some(
+                    fresh
+                        .error
+                        .unwrap_or_else(|| "ElectrumX is reconnecting; showing last known indexed height.".to_string()),
+                );
+            }
             *cached = fresh;
+            drop(cached);
+            tokio::time::sleep(Duration::from_secs(CACHE_REFRESH_SECS)).await;
         }
     });
 }
@@ -187,13 +209,69 @@ async fn electrumx_indexed_height() -> Result<u64> {
     Ok(height)
 }
 
-/// Fetch Bitcoin network height via JSON-RPC.
-async fn bitcoin_network_height() -> Result<u64> {
+fn parse_electrumx_height_from_logs(logs: &str) -> Option<u64> {
+    let mut height = None;
+
+    for line in logs.lines() {
+        if let Some(idx) = line.find("BlockProcessor:our height:") {
+            let rest = &line[idx + "BlockProcessor:our height:".len()..];
+            if let Some(parsed) = parse_first_u64_token(rest) {
+                height = Some(parsed);
+            }
+            continue;
+        }
+
+        if let Some(idx) = line.find("DB:height:") {
+            let rest = &line[idx + "DB:height:".len()..];
+            if let Some(parsed) = parse_first_u64_token(rest) {
+                height = Some(parsed);
+            }
+        }
+    }
+
+    height
+}
+
+fn parse_first_u64_token(input: &str) -> Option<u64> {
+    let token: String = input
+        .trim_start()
+        .chars()
+        .take_while(|c| c.is_ascii_digit() || *c == ',')
+        .filter(|c| *c != ',')
+        .collect();
+
+    if token.is_empty() {
+        None
+    } else {
+        token.parse().ok()
+    }
+}
+
+async fn electrumx_log_indexed_height() -> Result<u64> {
+    let output = tokio::process::Command::new("podman")
+        .args(["logs", "--tail", "500", "electrumx"])
+        .output()
+        .await
+        .context("Failed to read ElectrumX logs")?;
+
+    if !output.status.success() {
+        anyhow::bail!(
+            "podman logs electrumx failed: {}",
+            String::from_utf8_lossy(&output.stderr).trim()
+        );
+    }
+
+    let logs = String::from_utf8_lossy(&output.stdout);
+    parse_electrumx_height_from_logs(&logs).context("No ElectrumX indexed height in logs")
+}
+
+/// Fetch Bitcoin local block height and best-known network header height via JSON-RPC.
+async fn bitcoin_chain_heights() -> Result<(u64, u64)> {
     let client = reqwest::Client::new();
     let body = serde_json::json!({
         "jsonrpc": "1.0",
         "id": "electrs-status",
-        "method": "getblockcount",
+        "method": "getblockchaininfo",
         "params": []
     });
     let resp = client
@@ -211,11 +289,18 @@ async fn bitcoin_network_height() -> Result<u64> {
     }
 
     let json: serde_json::Value = resp.json().await?;
-    let height = json
+    let result = json
         .get("result")
-        .and_then(|r| r.as_u64())
         .context("Missing result in Bitcoin RPC")?;
-    Ok(height)
+    let blocks = result
+        .get("blocks")
+        .and_then(|h| h.as_u64())
+        .context("Missing blocks in Bitcoin RPC")?;
+    let headers = result
+        .get("headers")
+        .and_then(|h| h.as_u64())
+        .unwrap_or(blocks);
+    Ok((blocks, headers.max(blocks)))
 }
 
 /// Fetch fresh ElectrumX sync status (called by background cache task).
@@ -260,8 +345,8 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus {
         onion
     };
 
-    let network_height = match bitcoin_network_height().await {
-        Ok(h) => h,
+    let (bitcoin_blocks, network_height) = match bitcoin_chain_heights().await {
+        Ok(heights) => heights,
         Err(e) => {
             let err_msg = e.to_string();
             if is_transient_error(&err_msg) {
@@ -271,9 +356,11 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus {
             }
             return ElectrsSyncStatus {
                 indexed_height: 0,
+                bitcoin_height: 0,
                 network_height: 0,
                 progress_pct: 0.0,
                 status: "waiting".to_string(),
+                stale: false,
                 error: Some("Waiting for Bitcoin node...".to_string()),
                 index_size,
                 tor_onion,
@@ -283,7 +370,9 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus {
 
     let indexed_height = match electrumx_indexed_height().await {
         Ok(h) => h,
-        Err(e) => {
+        Err(e) => match electrumx_log_indexed_height().await {
+            Ok(h) if h > 0 => h,
+            _ => {
             let err_msg = e.to_string();
             if is_transient_error(&err_msg) {
                 // ElectrumX is starting up or busy — estimate from data size
@@ -295,9 +384,11 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus {
                 let size_str = index_size.clone().unwrap_or_else(|| "0 MB".to_string());
                 return ElectrsSyncStatus {
                     indexed_height: 0,
+                    bitcoin_height: bitcoin_blocks,
                     network_height,
                     progress_pct,
                     status: "indexing".to_string(),
+                    stale: false,
                     error: Some(format!(
                         "Building index ({} / ~130 GB estimated). Electrum RPC will be available when complete.",
                         size_str
@@ -310,35 +401,85 @@ async fn fetch_electrs_sync_status() -> ElectrsSyncStatus {
             warn!("ElectrumX status: unexpected error: {}", err_msg);
             return ElectrsSyncStatus {
                 indexed_height: 0,
+                bitcoin_height: bitcoin_blocks,
                 network_height,
                 progress_pct: 0.0,
                 status: "error".to_string(),
+                stale: false,
                 error: Some(format!("ElectrumX: {}", err_msg)),
                 index_size,
                 tor_onion,
             };
         }
+        },
     };
 
-    let progress_pct = if network_height > 0 {
-        (indexed_height as f64 / network_height as f64) * 100.0
+    let observed_header_height = network_height.max(indexed_height);
+    let bitcoin_catching_up = bitcoin_blocks > 0 && bitcoin_blocks < observed_header_height;
+    let electrum_waiting_on_bitcoin =
+        bitcoin_catching_up && indexed_height >= bitcoin_blocks.saturating_sub(1);
+    let sync_target_height = if bitcoin_blocks > 0 {
+        bitcoin_blocks
+    } else {
+        observed_header_height
+    };
+
+    let progress_pct = if electrum_waiting_on_bitcoin && observed_header_height > 0 {
+        ((bitcoin_blocks as f64 / observed_header_height as f64) * 100.0).min(99.9)
+    } else if sync_target_height > 0 {
+        ((indexed_height as f64 / sync_target_height as f64) * 100.0).min(100.0)
     } else {
         0.0
     };
 
-    let status = if indexed_height >= network_height.saturating_sub(1) {
+    let status = if sync_target_height == 0 {
+        "waiting"
+    } else if electrum_waiting_on_bitcoin {
+        "waiting"
+    } else if indexed_height >= sync_target_height.saturating_sub(1) {
         "synced"
     } else {
         "syncing"
     };
 
+    let error = if electrum_waiting_on_bitcoin {
+        Some(format!(
+            "ElectrumX is indexed to {:}; waiting for the local Bitcoin node to catch up from {:} to known header {:}.",
+            indexed_height, bitcoin_blocks, observed_header_height
+        ))
+    } else if status == "syncing" && bitcoin_blocks < observed_header_height {
+        Some(format!(
+            "Indexing local Bitcoin node height {:} of {:}. Bitcoin node is still catching up to known header {:}.",
+            indexed_height, bitcoin_blocks, observed_header_height
+        ))
+    } else {
+        None
+    };
+
     ElectrsSyncStatus {
         indexed_height,
-        network_height,
+        bitcoin_height: bitcoin_blocks,
+        network_height: observed_header_height,
         progress_pct,
         status: status.to_string(),
-        error: None,
+        stale: false,
+        error,
         index_size,
         tor_onion,
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::parse_electrumx_height_from_logs;
+
+    #[test]
+    fn parses_latest_electrumx_progress_height_from_logs() {
+        let logs = r#"
+INFO:DB:height: 228,238
+INFO:BlockProcessor:our height: 228,248 daemon: 731,568 UTXOs 1MB hist 1MB
+INFO:BlockProcessor:our height: 232,117 daemon: 732,108 UTXOs 281MB hist 83MB
+"#;
+        assert_eq!(parse_electrumx_height_from_logs(logs), Some(232_117));
+    }
+}

core/archipelago/src/main.rs

@@ -29,6 +29,7 @@ mod auth;
 mod avatar;
 mod backup;
 mod bitcoin_rpc;
+mod bitcoin_status;
 mod blobs;
 mod bootstrap;
 mod config;
@@ -289,6 +290,7 @@ async fn main() -> Result<()> {
 
     // Spawn ElectrumX status cache (refreshes every 15s, serves cached data to avoid race conditions)
     electrs_status::spawn_status_cache();
+    bitcoin_status::spawn_status_cache();
 
     let startup_ms = startup_start.elapsed().as_millis();
     info!(

core/archipelago/src/update.rs

@@ -571,15 +571,10 @@ pub async fn check_for_updates(data_dir: &Path) -> Result<UpdateState> {
                                 mirror = %manifest_url,
                                 "No newer version in manifest"
                             );
-                            if state.available_update.is_some() {
-                                // A later mirror might still have a
-                                // newer version — don't clobber what an
-                                // earlier mirror told us. But also don't
-                                // break: another mirror could be ahead.
-                                continue 'mirrors;
-                            }
                             state.manifest_mirror = None;
                             state.available_update = None;
+                            handled = true;
+                            continue 'mirrors;
                         }
                         handled = true;
                         break 'mirrors;
@@ -1173,6 +1168,125 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> {
                 }
                 info!(name = %name, "Frontend archive extracted to /opt/archipelago/web-ui");
             }
+            _ if name.contains("runtime") && name.ends_with(".tar.gz") => {
+                let ts = chrono::Utc::now().timestamp_millis();
+                let staging_new = format!("/opt/archipelago/runtime.new.{}", ts);
+                let archive = src.to_string_lossy().to_string();
+
+                let mk = host_sudo(&["mkdir", "-p", &staging_new])
+                    .await
+                    .context("Failed to create runtime staging dir")?;
+                if !mk.success() {
+                    anyhow::bail!("mkdir {} failed", staging_new);
+                }
+
+                let extract = host_sudo(&["tar", "-xzf", &archive, "-C", &staging_new])
+                    .await
+                    .with_context(|| format!("Failed to extract {}", name))?;
+                if !extract.success() {
+                    let _ = host_sudo(&["rm", "-rf", &staging_new]).await;
+                    anyhow::bail!("tar extraction failed for {}", name);
+                }
+
+                let runtime_paths = [
+                    ("apps", "apps"),
+                    ("scripts", "scripts"),
+                    ("docker", "docker"),
+                    (
+                        "image-recipe/configs/archipelago-doctor.service",
+                        "archipelago-doctor.service",
+                    ),
+                    (
+                        "image-recipe/configs/archipelago-doctor.timer",
+                        "archipelago-doctor.timer",
+                    ),
+                ];
+
+                for (relative, label) in runtime_paths {
+                    let staged_path = format!("{}/{}", staging_new, relative);
+                    if !Path::new(&staged_path).exists() {
+                        tracing::debug!(path = %relative, "Runtime artifact path absent, skipping");
+                        continue;
+                    }
+
+                    match label {
+                        "apps" | "scripts" | "docker" => {
+                            let dest = format!("/opt/archipelago/{}", label);
+                            let tmp_dest =
+                                format!("{}.new.{}", dest, chrono::Utc::now().timestamp_millis());
+                            let _ = host_sudo(&["mkdir", "-p", &tmp_dest]).await;
+                            let staged_dot = format!("{}/.", staged_path);
+                            let copy = host_sudo(&["cp", "-a", &staged_dot, &tmp_dest])
+                                .await
+                                .with_context(|| format!("Failed to copy runtime {}", label))?;
+                            if !copy.success() {
+                                let _ = host_sudo(&["rm", "-rf", &tmp_dest]).await;
+                                anyhow::bail!("runtime copy failed for {}", label);
+                            }
+                            let _ = host_sudo(&["mkdir", "-p", &dest]).await;
+                            let clean = host_sudo(&[
+                                "find",
+                                &dest,
+                                "-mindepth",
+                                "1",
+                                "-maxdepth",
+                                "1",
+                                "-exec",
+                                "rm",
+                                "-rf",
+                                "{}",
+                                "+",
+                            ])
+                            .await
+                            .with_context(|| format!("Failed to clean runtime {}", label))?;
+                            if !clean.success() {
+                                let _ = host_sudo(&["rm", "-rf", &tmp_dest]).await;
+                                anyhow::bail!("runtime clean failed for {}", label);
+                            }
+                            let tmp_dot = format!("{}/.", tmp_dest);
+                            let promote = host_sudo(&["cp", "-a", &tmp_dot, &dest])
+                                .await
+                                .with_context(|| format!("Failed to promote runtime {}", label))?;
+                            let _ = host_sudo(&["rm", "-rf", &tmp_dest]).await;
+                            if !promote.success() {
+                                anyhow::bail!("runtime promote failed for {}", label);
+                            }
+                            if label == "scripts" {
+                                let _ = host_sudo(&[
+                                    "find", &dest, "-type", "f", "-name", "*.sh", "-exec", "chmod",
+                                    "755", "{}", "+",
+                                ])
+                                .await;
+                            }
+                        }
+                        "archipelago-doctor.service" | "archipelago-doctor.timer" => {
+                            let dest = format!("/etc/systemd/system/{}", label);
+                            let install = host_sudo(&["install", "-m", "644", &staged_path, &dest])
+                                .await
+                                .with_context(|| format!("Failed to install {}", label))?;
+                            if !install.success() {
+                                anyhow::bail!("runtime unit install failed for {}", label);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+
+                if Path::new(&format!("{}/scripts/image-versions.sh", staging_new)).exists() {
+                    let _ = host_sudo(&[
+                        "cp",
+                        &format!("{}/scripts/image-versions.sh", staging_new),
+                        "/opt/archipelago/image-versions.sh",
+                    ])
+                    .await;
+                }
+
+                let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
+                let _ =
+                    host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await;
+                let _ = host_sudo(&["rm", "-rf", &staging_new]).await;
+                info!(name = %name, "Runtime assets applied to /opt/archipelago");
+            }
             _ => {
                 debug!(name = %name, "Unknown component, skipping");
             }

docker/bitcoin-ui/index.html

@@ -606,7 +606,8 @@
         console.log('[Bitcoin UI] Script loaded, initializing...');
         
         // RPC Configuration - Use local Nginx proxy within container
-        const RPC_ENDPOINT = '/bitcoin-rpc/';
+        const RPC_ENDPOINT = 'bitcoin-rpc/';
+        const STATUS_ENDPOINT = 'bitcoin-status';
         console.log('[Bitcoin UI] RPC Endpoint:', RPC_ENDPOINT);
 
         // Make RPC call to Bitcoin node via local proxy
@@ -645,6 +646,14 @@
             }
         }
 
+        async function fetchBitcoinStatus() {
+            const response = await fetch(STATUS_ENDPOINT, { cache: 'no-store' });
+            if (!response.ok) {
+                throw new Error(`status HTTP ${response.status}`);
+            }
+            return response.json();
+        }
+
         // Implementation branding — detected from getnetworkinfo.subversion.
         // Bitcoin Knots identifies as "/Satoshi:<ver>/Knots:<date>/", Bitcoin Core as "/Satoshi:<ver>/".
         let brandingApplied = false;
@@ -672,22 +681,62 @@
 
         // Track last block count for animations
         let lastBlockCount = 0;
+        let consecutiveRpcFailures = 0;
+        let lastSuccessfulUpdateAt = 0;
+
+        function formatPercent(value) {
+            if (!Number.isFinite(value) || value <= 0) return '0.00';
+            if (value < 0.01) return '<0.01';
+            return value.toFixed(2);
+        }
+
+        function formatBytes(bytes) {
+            if (!Number.isFinite(bytes) || bytes <= 0) return null;
+            const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+            let value = bytes;
+            let unit = 0;
+            while (value >= 1000 && unit < units.length - 1) {
+                value /= 1000;
+                unit += 1;
+            }
+            return `${value.toFixed(unit >= 3 ? 1 : 0)} ${units[unit]}`;
+        }
 
         // Update blockchain info
         async function updateBlockchainInfo() {
             console.log('[Bitcoin UI] updateBlockchainInfo() called');
             try {
-                const blockchainInfo = await callRPC('getblockchaininfo');
+                const status = await fetchBitcoinStatus();
+                const blockchainInfo = status.blockchain_info;
                 console.log('[Bitcoin UI] blockchainInfo:', blockchainInfo);
                 
                 if (!blockchainInfo) {
                     console.error('[Bitcoin UI] No blockchain info received');
-                    document.getElementById('syncStatusText').textContent = 'Unable to connect to Bitcoin node';
-                    document.getElementById('syncStatusText').className = 'text-red-400 text-sm';
+                    consecutiveRpcFailures += 1;
+                    const syncStatusText = document.getElementById('syncStatusText');
+                    const syncIcon = document.getElementById('syncIcon');
+                    if (syncStatusText) {
+                        if (status.stale) {
+                            syncStatusText.textContent = status.error || 'Bitcoin node is reconnecting... showing last known values';
+                            syncStatusText.className = 'text-yellow-300 text-sm font-medium';
+                        } else if (consecutiveRpcFailures < 6) {
+                            syncStatusText.textContent = status.error || 'Connecting to Bitcoin node...';
+                            syncStatusText.className = 'text-yellow-300 text-sm font-medium';
+                        } else {
+                            syncStatusText.textContent = status.error || 'Bitcoin node is not responding yet';
+                            syncStatusText.className = 'text-red-400 text-sm font-medium';
+                        }
+                    }
+                    if (syncIcon) {
+                        syncIcon.classList.add('animate-spin-slow');
+                        syncIcon.classList.remove('text-green-500');
+                    }
                     return;
                 }
+                consecutiveRpcFailures = 0;
+                lastSuccessfulUpdateAt = Date.now();
 
-                const networkInfo = await callRPC('getnetworkinfo');
+                const networkInfo = status.network_info;
 
                 applyImplBranding(networkInfo && networkInfo.subversion);
 
@@ -743,44 +792,51 @@
                 }
 
                 // Populate Settings — Transaction Index, ZMQ, RPC (fire-and-forget)
-                (async () => {
-                    const txIndexEl = document.getElementById('settingsTxIndex');
-                    if (txIndexEl) {
-                        const idx = await callRPC('getindexinfo');
-                        if (idx && typeof idx === 'object') {
-                            const names = Object.keys(idx);
-                            txIndexEl.textContent = names.length
-                                ? `Enabled: ${names.join(', ')}`
-                                : 'Disabled';
-                        } else {
-                            txIndexEl.textContent = 'Disabled';
-                        }
+                const txIndexEl = document.getElementById('settingsTxIndex');
+                if (txIndexEl) {
+                    const idx = status.index_info;
+                    if (idx && typeof idx === 'object') {
+                        const names = Object.keys(idx);
+                        txIndexEl.textContent = names.length
+                            ? `Enabled: ${names.join(', ')}`
+                            : 'Disabled';
+                    } else {
+                        txIndexEl.textContent = 'Unavailable while node starts';
                     }
-                    const zmqEl = document.getElementById('settingsZmq');
-                    if (zmqEl) {
-                        const zmq = await callRPC('getzmqnotifications');
-                        if (Array.isArray(zmq) && zmq.length) {
-                            zmqEl.textContent = zmq.map(z => `${z.type}@${z.address}`).join('; ');
-                        } else {
-                            zmqEl.textContent = 'Not enabled';
-                        }
+                }
+                const zmqEl = document.getElementById('settingsZmq');
+                if (zmqEl) {
+                    const zmq = status.zmq_notifications;
+                    if (Array.isArray(zmq) && zmq.length) {
+                        zmqEl.textContent = zmq.map(z => `${z.type}@${z.address}`).join('; ');
+                    } else if (Array.isArray(zmq)) {
+                        zmqEl.textContent = 'Not enabled';
+                    } else {
+                        zmqEl.textContent = 'Unavailable while node starts';
                     }
-                    const rpcEl = document.getElementById('settingsRpc');
-                    if (rpcEl && networkInfo) {
-                        const port = chain === 'main' ? 8332 : (chain === 'test' ? 18332 : (chain === 'signet' ? 38332 : 18443));
-                        rpcEl.textContent = `Reachable on port ${port}`;
-                    }
-                })();
+                }
+                const rpcEl = document.getElementById('settingsRpc');
+                if (rpcEl) {
+                    const port = chain === 'main' ? 8332 : (chain === 'test' ? 18332 : (chain === 'signet' ? 38332 : 18443));
+                    rpcEl.textContent = status.stale
+                        ? `Reconnecting on port ${port}`
+                        : `Reachable on port ${port}`;
+                }
 
                 // Update sync status
                 const blocks = blockchainInfo.blocks || 0;
                 const headers = blockchainInfo.headers || 0;
                 const verificationProgress = blockchainInfo.verificationprogress || 0;
-                const isSynced = blocks >= headers - 1;
+                const initialBlockDownload = blockchainInfo.initialblockdownload === true;
+                const isSynced = headers > 0 && blocks >= headers - 1 && !initialBlockDownload;
+                const diskSize = formatBytes(blockchainInfo.size_on_disk || 0);
+                const appearsToBeReindexing = initialBlockDownload && blocks === 0 && headers > 0 && (blockchainInfo.size_on_disk || 0) > 1024 * 1024 * 1024;
                 
                 // Calculate actual sync percentage based on blocks/headers
-                const actualSyncPercentage = headers > 0 ? ((blocks / headers) * 100).toFixed(2) : '0.00';
-                const verificationPercentage = (verificationProgress * 100).toFixed(2);
+                const actualSyncValue = headers > 0 ? (blocks / headers) * 100 : 0;
+                const actualSyncPercentage = formatPercent(actualSyncValue);
+                const progressWidth = Math.max(0, Math.min(100, actualSyncValue));
+                const verificationPercentage = formatPercent(verificationProgress * 100);
 
                 // Animate block count if it changed
                 const currentHeightElem = document.getElementById('currentHeight');
@@ -795,16 +851,27 @@
                 document.getElementById('headers').textContent = headers.toLocaleString();
                 document.getElementById('verificationProgress').textContent = `${verificationPercentage}%`;
                 document.getElementById('syncPercentage').textContent = `${actualSyncPercentage}%`;
-                document.getElementById('currentBlock').textContent = `Block ${blocks.toLocaleString()}`;
-                document.getElementById('syncProgressBar').style.width = `${actualSyncPercentage}%`;
+                document.getElementById('currentBlock').textContent = appearsToBeReindexing
+                    ? 'Reindexing from disk'
+                    : `Block ${blocks.toLocaleString()}`;
+                document.getElementById('syncProgressBar').style.width = `${progressWidth}%`;
 
                 // Update sync status text and icon
                 const syncStatusText = document.getElementById('syncStatusText');
                 const syncIcon = document.getElementById('syncIcon');
                 
-                if (isSynced) {
-                    syncStatusText.textContent = '✓ Fully synchronized with the network';
-                    syncStatusText.className = 'text-green-400 text-sm font-medium';
+                if (appearsToBeReindexing) {
+                    syncStatusText.textContent = `Reindexing local block files${diskSize ? ` (${diskSize} on disk)` : ''}`;
+                    syncStatusText.className = 'text-orange-400 text-sm font-medium';
+                    if (syncIcon) {
+                        syncIcon.classList.add('animate-spin-slow');
+                        syncIcon.classList.remove('text-green-500');
+                    }
+                } else if (isSynced) {
+                    syncStatusText.textContent = status.stale
+                        ? 'Bitcoin node is reconnecting... showing last known synchronized state'
+                        : '✓ Fully synchronized with the network';
+                    syncStatusText.className = status.stale ? 'text-yellow-300 text-sm font-medium' : 'text-green-400 text-sm font-medium';
                     // Stop spinning when synced
                     if (syncIcon) {
                         syncIcon.classList.remove('animate-spin-slow');
@@ -812,8 +879,12 @@
                     }
                 } else {
                     const remaining = headers - blocks;
-                    syncStatusText.textContent = `Syncing... ${remaining.toLocaleString()} blocks remaining`;
-                    syncStatusText.className = 'text-orange-400 text-sm font-medium';
+                    syncStatusText.textContent = status.stale
+                        ? 'Bitcoin node is reconnecting... showing last known sync state'
+                        : initialBlockDownload
+                        ? `Initial block download... ${remaining.toLocaleString()} blocks remaining`
+                        : `Syncing... ${remaining.toLocaleString()} blocks remaining`;
+                    syncStatusText.className = status.stale ? 'text-yellow-300 text-sm font-medium' : 'text-orange-400 text-sm font-medium';
                     // Keep spinning while syncing
                     if (syncIcon) {
                         syncIcon.classList.add('animate-spin-slow');
@@ -834,8 +905,15 @@
 
             } catch (error) {
                 console.error('Failed to update blockchain info:', error);
-                document.getElementById('syncStatusText').textContent = 'Unable to fetch blockchain data';
-                document.getElementById('syncStatusText').className = 'text-red-400 text-sm';
+                consecutiveRpcFailures += 1;
+                const syncStatusText = document.getElementById('syncStatusText');
+                if (syncStatusText) {
+                    const hasRecentData = lastSuccessfulUpdateAt > 0 && Date.now() - lastSuccessfulUpdateAt < 120000;
+                    syncStatusText.textContent = hasRecentData
+                        ? 'Bitcoin status bridge is reconnecting... keeping last known values'
+                        : 'Connecting to Bitcoin status bridge...';
+                    syncStatusText.className = 'text-yellow-300 text-sm font-medium';
+                }
             }
         }
 

docker/electrs-ui/index.html

@@ -68,6 +68,7 @@
         @media (min-width: 768px) {
             .md-flex-row { flex-direction: row; }
             .md-grid-cols-4 { grid-template-columns: repeat(4, 1fr); }
+            .md-grid-cols-5 { grid-template-columns: repeat(5, 1fr); }
         }
 
         /* Connection details */
@@ -147,13 +148,17 @@
                 </div>
             </div>
 
-            <div class="grid grid-cols-2 md-grid-cols-4 gap-3">
+            <div class="grid grid-cols-2 md-grid-cols-5 gap-3">
                 <div class="info-card">
-                    <p class="text-xs text-white-60 mb-1">Indexed Height</p>
+                    <p class="text-xs text-white-60 mb-1">Electrum Indexed</p>
                     <p class="text-lg font-semibold text-white" id="indexedHeight">-</p>
                 </div>
                 <div class="info-card">
-                    <p class="text-xs text-white-60 mb-1">Network Height</p>
+                    <p class="text-xs text-white-60 mb-1">Bitcoin Node</p>
+                    <p class="text-lg font-semibold text-white" id="bitcoinHeight">-</p>
+                </div>
+                <div class="info-card">
+                    <p class="text-xs text-white-60 mb-1">Known Headers</p>
                     <p class="text-lg font-semibold text-white" id="networkHeight">-</p>
                 </div>
                 <div class="info-card">
@@ -370,15 +375,36 @@
             }
 
             var indexedH = data.indexed_height || 0;
-            var networkH = data.network_height || 0;
+            var bitcoinH = data.bitcoin_height || 0;
+            var reportedNetworkH = data.network_height || 0;
+            var knownHeaderH = Math.max(reportedNetworkH, indexedH, bitcoinH);
+            var targetH = bitcoinH > 0 ? bitcoinH : knownHeaderH;
             var pct = data.progress_pct || 0;
+            var hasIndexedHeight = indexedH > 0 || data.stale;
+            var indexedLabel = hasIndexedHeight
+                ? indexedH.toLocaleString()
+                : (data.status === 'indexing' ? 'Pending' : '-');
+            var currentBlockLabel;
+            if (hasIndexedHeight && bitcoinH > 0 && indexedH > bitcoinH) {
+                currentBlockLabel = 'Bitcoin node ' + bitcoinH.toLocaleString()
+                    + (knownHeaderH > 0 ? ' of known headers ' + knownHeaderH.toLocaleString() : '')
+                    + '; Electrum index ' + indexedH.toLocaleString();
+            } else if (hasIndexedHeight) {
+                currentBlockLabel = 'Indexed ' + indexedH.toLocaleString() + ' of '
+                    + (targetH > 0 ? targetH.toLocaleString() : 'Bitcoin node height');
+            } else {
+                currentBlockLabel = data.index_size
+                    ? 'Index building from disk (' + data.index_size + ')'
+                    : 'Waiting for Electrum index height';
+            }
 
-            document.getElementById('indexedHeight').textContent = indexedH > 0 ? indexedH.toLocaleString() : (data.status === 'indexing' ? 'Building...' : '-');
-            document.getElementById('networkHeight').textContent = networkH > 0 ? networkH.toLocaleString() : '-';
+            document.getElementById('indexedHeight').textContent = indexedLabel;
+            document.getElementById('bitcoinHeight').textContent = bitcoinH > 0 ? bitcoinH.toLocaleString() : 'Checking...';
+            document.getElementById('networkHeight').textContent = knownHeaderH > 0 ? knownHeaderH.toLocaleString() : 'Checking...';
             document.getElementById('indexSize').textContent = data.index_size || '-';
-            document.getElementById('progressPct').textContent = pct > 0 ? pct.toFixed(1) + '%' : '-';
-            document.getElementById('currentBlock').textContent = indexedH > 0 ? 'Block ' + indexedH.toLocaleString() : (data.index_size ? 'Index: ' + data.index_size : 'Block 0');
-            document.getElementById('syncPercentage').textContent = pct > 0 ? pct.toFixed(1) + '%' : '0%';
+            document.getElementById('progressPct').textContent = (knownHeaderH > 0 || pct > 0) ? pct.toFixed(1) + '%' : '-';
+            document.getElementById('currentBlock').textContent = currentBlockLabel;
+            document.getElementById('syncPercentage').textContent = (knownHeaderH > 0 || pct > 0) ? pct.toFixed(1) + '%' : '0%';
             document.getElementById('syncProgressBar').style.width = Math.max(pct, 0.5) + '%';
 
             var statusTextEl = document.getElementById('syncStatusText');
@@ -389,14 +415,16 @@
                 statusTextEl.textContent = data.error || 'Starting up...';
                 statusTextEl.style.color = '#fbbf24';
                 statusDot.className = 'status-dot bg-yellow animate-pulse';
-                document.getElementById('statusText').textContent = 'Starting';
+                document.getElementById('statusText').textContent = data.status === 'waiting' ? 'Waiting' : 'Starting';
                 syncIcon.classList.add('animate-spin-slow');
                 document.getElementById('connSubtitle').textContent = 'Connections will be available once ElectrumX has completed syncing.';
             } else if (data.status === 'indexing') {
-                statusTextEl.textContent = data.error || 'Building index...';
+                statusTextEl.textContent = data.stale
+                    ? (data.error || 'ElectrumX is reconnecting; showing last known indexed height.')
+                    : (data.error || 'Building index. Indexed height will appear when Electrum RPC is ready.');
                 statusTextEl.style.color = '#fbbf24';
                 statusDot.className = 'status-dot bg-amber animate-pulse';
-                document.getElementById('statusText').textContent = 'Indexing';
+                document.getElementById('statusText').textContent = data.stale ? 'Reconnecting' : 'Indexing';
                 syncIcon.classList.add('animate-spin-slow');
                 document.getElementById('connSubtitle').textContent = 'Connections will be available once ElectrumX has completed syncing.';
             } else if (data.status === 'error') {
@@ -414,8 +442,10 @@
                 syncIcon.style.color = '#4ade80';
                 document.getElementById('connSubtitle').textContent = 'Use the following details to connect your wallet or application to ElectrumX.';
             } else {
-                var remaining = networkH - indexedH;
-                statusTextEl.textContent = 'Syncing... ' + remaining.toLocaleString() + ' blocks remaining';
+                var remaining = Math.max(targetH - indexedH, 0);
+                statusTextEl.textContent = data.error || (targetH > 0
+                    ? 'Syncing... ' + remaining.toLocaleString() + ' blocks remaining'
+                    : 'Waiting for Bitcoin network height...');
                 statusTextEl.style.color = '#fb923c';
                 statusDot.className = 'status-dot bg-yellow';
                 document.getElementById('statusText').textContent = 'Syncing';

image-recipe/configs/archipelago-doctor.service

@@ -7,6 +7,6 @@ Type=oneshot
 # Runs as root: needs to kill orphaned conmon processes, fix permissions
 User=root
 ExecStart=/home/archipelago/archy/scripts/container-doctor.sh --local
-TimeoutStartSec=120
+TimeoutStartSec=300
 StandardOutput=journal
 StandardError=journal

image-recipe/configs/archipelago-doctor.timer

@@ -2,9 +2,11 @@
 Description=Archipelago container doctor (periodic)
 
 [Timer]
-# First run 5 minutes after boot, then every 30 minutes
-OnBootSec=5min
-OnUnitActiveSec=30min
+# First run 2 minutes after boot, then every 5 minutes. The doctor is
+# idempotent and exits quickly when no drift exists; this keeps vanished
+# rootless port listeners and stopped containers from remaining broken.
+OnBootSec=2min
+OnUnitActiveSec=5min
 # Jitter to avoid load spikes
 RandomizedDelaySec=60
 

image-recipe/configs/archipelago.service

@@ -10,7 +10,7 @@ Environment="ARCHIPELAGO_BIND=127.0.0.1:5678"
 # DEV_MODE disabled in production — enabled via override.conf on dev servers
 Environment="XDG_RUNTIME_DIR=/run/user/1000"
 # + prefix runs these as root (needed for chown/mkdir outside ReadWritePaths)
-ExecStartPre=+/bin/bash -c 'mkdir -p /run/user/1000 && chown archipelago:archipelago /run/user/1000 && chmod 700 /run/user/1000'
+ExecStartPre=+/bin/bash -c 'mkdir -p /run/user/1000 /run/containers /var/lib/containers && chown archipelago:archipelago /run/user/1000 && chmod 700 /run/user/1000'
 ExecStartPre=+/bin/bash -c 'mkdir -p /var/lib/archipelago && chown archipelago:archipelago /var/lib/archipelago && echo "ARCHIPELAGO_HOST_IP=$(hostname -I 2>/dev/null | awk "{print $$1}")" > /var/lib/archipelago/host-ip.env && chown archipelago:archipelago /var/lib/archipelago/host-ip.env'
 ExecStart=/usr/local/bin/archipelago
 Restart=on-failure

neode-ui/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "neode-ui",
-  "version": "1.7.44-alpha",
+  "version": "1.7.50-alpha",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "neode-ui",
-      "version": "1.7.44-alpha",
+      "version": "1.7.50-alpha",
       "dependencies": {
         "@types/dompurify": "^3.0.5",
         "@vue-leaflet/vue-leaflet": "^0.10.1",

neode-ui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "neode-ui",
   "private": true,
-  "version": "1.7.47-alpha",
+  "version": "1.7.50-alpha",
   "type": "module",
   "scripts": {
     "start": "./start-dev.sh",

neode-ui/public/catalog.json

@@ -26,7 +26,7 @@
       "id": "bitcoin-core",
       "title": "Bitcoin Core",
       "version": "28.4",
-      "description": "Reference implementation of the Bitcoin protocol. Run a full node validating and relaying blocks.",
+      "description": "Reference Bitcoin node implementation. Alternative to Bitcoin Knots; uninstall Knots before switching.",
       "icon": "/assets/img/app-icons/bitcoin-core.svg",
       "author": "Bitcoin Core contributors",
       "category": "money",

neode-ui/src/stores/appLauncher.ts

@@ -13,7 +13,7 @@ const NEW_TAB_PORTS = new Set([
   '8085',  // Nextcloud — X-Frame-Options: SAMEORIGIN
   '3002',  // Uptime Kuma — X-Frame-Options: SAMEORIGIN
   '9001',  // Penpot — not reachable
-  // IndeedHub (7777) uses proxy path for NIP-07 nostr-provider.js — NOT new tab
+  // Port 7777 is the Nostr relay; IndeeHub's web UI is exposed on 7778.
 ])
 
 const NEW_TAB_APP_IDS = new Set([
@@ -34,6 +34,7 @@ function mustOpenInNewTab(url: string): boolean {
 function inferAppIdFromTitle(title?: string): string | null {
   const t = (title || '').toLowerCase()
   if (!t) return null
+  if (t.includes('indeehub') || t.includes('indeedhub')) return 'indeedhub'
   if ((t.includes('uptime') && t.includes('kuma')) || t.includes('uptime-kuma')) return 'uptime-kuma'
   if ((t.includes('nginx') && t.includes('proxy') && t.includes('manager')) || t.includes('nginx-proxy-manager')) return 'nginx-proxy-manager'
   if (t.includes('gitea')) return 'gitea'
@@ -47,6 +48,10 @@ function normalizeLaunchUrl(urlStr: string, appIdHint?: string | null): string {
     const normalizedPath = u.pathname === '/' ? '' : u.pathname
     const rebuilt = (port: string) => `${u.protocol}//${u.hostname}:${port}${normalizedPath}${u.search}${u.hash}`
 
+    if (sameHost && appIdHint === 'indeedhub' && u.port === '7777') {
+      return rebuilt('7778')
+    }
+
     if (sameHost && appIdHint === 'uptime-kuma' && u.port === '3001') {
       return rebuilt('3002')
     }
@@ -87,7 +92,7 @@ const PORT_TO_APP_ID: Record<string, string> = {
   '8175': 'fedimint',
   '8176': 'fedimint-gateway',
   '3100': 'dwn',
-  '7777': 'indeedhub',
+  '7778': 'indeedhub',
   '50002': 'electrumx',
   '3010': 'thunderhub',
 }

neode-ui/src/utils/dummyApps.ts

@@ -504,7 +504,7 @@ export const dummyApps: Record<string, PackageDataEntry> = {
       'interface-addresses': {
         main: {
           'tor-address': '',
-          'lan-address': 'http://localhost:8190'
+          'lan-address': 'http://localhost:7778'
         }
       },
       status: ServiceStatus.Running
@@ -749,4 +749,3 @@ export const dummyApps: Record<string, PackageDataEntry> = {
     }
   }
 }
-

neode-ui/src/views/Kiosk.vue

@@ -107,7 +107,7 @@ const launchableApps = computed<KioskApp[]>(() => {
     'fedimint': '/app/fedimint/',
     'fedimint-gateway': '/app/fedimint-gateway/',
     'dwn': '/app/dwn/',
-    'indeedhub': 'http://localhost:8190',
+    'indeedhub': 'http://localhost:7778',
     'botfights': 'http://localhost:9100',
     'nwnn': 'https://nwnn.l484.com',
     '484-kitchen': 'https://484.kitchen',

neode-ui/src/views/appSession/appSessionConfig.ts

@@ -38,15 +38,14 @@ export const APP_PORTS: Record<string, number> = {
   'fedimint':      8175,
   'fedimintd':     8175,
   'fedimint-gateway': 8176,
-  'indeedhub':     7777,
+  'indeedhub':     7778,
   'botfights':     9100,
   'dwn':           3100,
   'endurain':      8080,
 }
 
 /** Apps that need nginx proxy for iframe embedding.
- *  IndeedHub loads via /app/indeedhub/ proxy for nostr-provider.js injection
- *  from the container's internal nginx so iframe works on all servers. */
+ *  IndeeHub web UI is on 7778. Port 7777 is the Nostr relay. */
 export const PROXY_APPS: Record<string, string> = {
   'gitea': '/app/gitea/',
   'nginx-proxy-manager': '/app/nginx-proxy-manager/',
@@ -60,9 +59,10 @@ export const HTTPS_PROXY_PATHS: Record<string, string> = {
   'bitcoin-core': '/app/bitcoin-ui/',
   'bitcoin-ui': '/app/bitcoin-ui/',
   'lnd': '/app/lnd/',
-  'electrumx': '/app/electrs/',
-  'electrs': '/app/electrs/',
-  'mempool-electrs': '/app/electrs/',
+  'electrumx': '/app/electrumx/',
+  'electrs': '/app/electrumx/',
+  'archy-electrs-ui': '/app/electrumx/',
+  'mempool-electrs': '/app/electrumx/',
   'mempool': '/app/mempool/',
   'mempool-web': '/app/mempool/',
   'archy-mempool-web': '/app/mempool/',
@@ -87,7 +87,6 @@ export const HTTPS_PROXY_PATHS: Record<string, string> = {
   'btcpay-server': '/app/btcpay/',
   'nextcloud': '/app/nextcloud/',
   'grafana': '/app/grafana/',
-  'indeedhub': '/app/indeedhub/',
   'botfights': '/app/botfights/',
   'gitea': '/app/gitea/',
 }

release-manifest.json

@@ -1,31 +1,27 @@
 {
-  "version": "1.7.44-alpha",
-  "release_date": "2026-04-28",
+  "version": "1.7.50-alpha",
+  "release_date": "2026-05-01",
   "changelog": [
-    "43de3b73 feat(orchestrator): complete container migration and release hardening",
-    "ce39430b feat(self-update): sync and rebuild UI containers on OTA",
-    "72dec5aa fix(lnd-ui): align container port across all specs",
-    "83aacdf2 chore(release): archive ISO build recipes, tarball-only releases",
-    "All notable changes to Archipelago will be documented in this file.",
-    "The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),",
-    "and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)."
+    "OTA now carries app manifests, scripts, Docker contexts, and doctor units inside the frontend payload so older nodes restore /opt/archipelago runtime assets after updating.",
+    "Startup bootstrap promotes the embedded runtime payload into /opt/archipelago and reloads the doctor timer without requiring rsync.",
+    "Update checks now continue past stale mirrors so nodes can discover a newer manifest from the next configured mirror."
   ],
   "components": [
     {
       "name": "archipelago",
-      "current_version": "1.7.44-alpha",
-      "new_version": "1.7.44-alpha",
-      "download_url": "https://github.com/archipelago-os/releases/releases/download/v1.7.44-alpha/archipelago",
-      "sha256": "ea9167d376210bdc416dd0dcf3c1737e2fa2b1bdb9f192e473f2edaaf0d046a7",
-      "size_bytes": 41373032
+      "current_version": "1.7.49-alpha",
+      "new_version": "1.7.50-alpha",
+      "download_url": "https://git.tx1138.com/lfg2025/archy/raw/branch/main/releases/v1.7.50-alpha/archipelago",
+      "sha256": "494f71d58608787a4f485ec6e295057e70045e905d745837d1cd6089e484197d",
+      "size_bytes": 41779280
     },
     {
-      "name": "archipelago-frontend-1.7.44-alpha.tar.gz",
-      "current_version": "1.7.44-alpha",
-      "new_version": "1.7.44-alpha",
-      "download_url": "https://github.com/archipelago-os/releases/releases/download/v1.7.44-alpha/archipelago-frontend-1.7.44-alpha.tar.gz",
-      "sha256": "28bdbccd8151bfa7834761adb5363862ef54b76c6010d1446fb8cca4ed4fe18c",
-      "size_bytes": 162089899
+      "name": "archipelago-frontend-1.7.50-alpha.tar.gz",
+      "current_version": "1.7.49-alpha",
+      "new_version": "1.7.50-alpha",
+      "download_url": "https://git.tx1138.com/lfg2025/archy/raw/branch/main/releases/v1.7.50-alpha/archipelago-frontend-1.7.50-alpha.tar.gz",
+      "sha256": "6684d5f1083fd2d1db236fcc4a94efee7986ab54602b3231b031f3e9f7dec361",
+      "size_bytes": 165153241
     }
   ]
 }

scripts/app-surface-smoke-test.sh

@@ -0,0 +1,161 @@
+#!/bin/bash
+#
+# App surface smoke test.
+#
+# Verifies that installed containers have their published host ports listening
+# and that known nginx app proxy paths return a non-5xx response. This catches
+# the common "container is running but UI disappeared" failure mode.
+#
+# Usage:
+#   scripts/app-surface-smoke-test.sh --target archipelago@192.168.1.228 --ssh-key /path/key
+
+set -euo pipefail
+
+TARGET=""
+SSH_KEY="${ARCHIPELAGO_SSH_KEY:-}"
+SSH_EXTRA=()
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --target) TARGET="${2:-}"; shift 2 ;;
+    --ssh-key) SSH_KEY="${2:-}"; shift 2 ;;
+    --ssh-option) SSH_EXTRA+=("-o" "${2:-}"); shift 2 ;;
+    -h|--help) sed -n '1,12p' "$0"; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+
+[ -n "$TARGET" ] || { echo "--target is required" >&2; exit 2; }
+
+SSH_OPTS=(-F /dev/null -o BatchMode=yes -o PreferredAuthentications=publickey -o PasswordAuthentication=no)
+[ -n "$SSH_KEY" ] && SSH_OPTS+=(-i "$SSH_KEY")
+SSH_OPTS+=("${SSH_EXTRA[@]}")
+
+ssh_run() {
+  ssh "${SSH_OPTS[@]}" "$TARGET" "$@"
+}
+
+ssh_run 'bash -s' <<'REMOTE'
+set -u
+
+pass=0
+fail=0
+
+ok() { echo "  PASS $*"; pass=$((pass + 1)); }
+bad() { echo "  FAIL $*"; fail=$((fail + 1)); }
+
+container_exists() {
+  podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$1"
+}
+
+port_listening() {
+  ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$1$"
+}
+
+http_code() {
+  local url="$1" code
+  for _ in 1 2 3; do
+    code=$(curl -ksS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)
+    [ -n "$code" ] || code=000
+    [ "$code" != "000" ] && { echo "$code"; return; }
+    sleep 2
+  done
+  echo "$code"
+}
+
+http_post_code() {
+  local url="$1" code
+  for _ in 1 2 3; do
+    code=$(curl -ksS -o /dev/null -w '%{http_code}' --max-time 25 \
+      -H 'Content-Type: application/json' \
+      -d '{"jsonrpc":"2.0","id":1,"method":"getblockchaininfo","params":[]}' \
+      "$url" 2>/dev/null || true)
+    [ -n "$code" ] || code=000
+    [ "$code" != "000" ] && { echo "$code"; return; }
+    sleep 2
+  done
+  echo "$code"
+}
+
+assert_http() {
+  local label="$1" url="$2" code
+  code=$(http_code "$url")
+  case "$code" in
+    200|204|301|302|307|308|401|403) ok "$label HTTP $code" ;;
+    *) bad "$label HTTP $code ($url)" ;;
+  esac
+}
+
+assert_http_post() {
+  local label="$1" url="$2" code
+  code=$(http_post_code "$url")
+  case "$code" in
+    200|204|401|403) ok "$label HTTP POST $code" ;;
+    *) bad "$label HTTP POST $code ($url)" ;;
+  esac
+}
+
+assert_container_ports() {
+  local name="$1" ports port missing=0
+  container_exists "$name" || return 0
+  ports=$(podman inspect "$name" --format '{{range $p,$bindings := .NetworkSettings.Ports}}{{if $bindings}}{{range $bindings}}{{.HostPort}}{{"\n"}}{{end}}{{end}}{{end}}' 2>/dev/null | sort -u)
+  [ -n "$ports" ] || return 0
+  while IFS= read -r port; do
+    [ -n "$port" ] || continue
+    if port_listening "$port"; then
+      ok "$name port $port listening"
+    else
+      bad "$name port $port missing listener"
+      missing=1
+    fi
+  done <<< "$ports"
+  return "$missing"
+}
+
+assert_env_contains() {
+  local name="$1" key="$2" needle="$3" val
+  container_exists "$name" || return 0
+  val=$(podman inspect "$name" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | sed -n "s/^${key}=//p" | head -n 1)
+  if [ -n "$val" ] && printf '%s' "$val" | grep -qF "$needle"; then
+    ok "$name env $key"
+  else
+    bad "$name env $key missing $needle"
+  fi
+}
+
+echo "[surface] host=$(hostname) ip=$(hostname -I 2>/dev/null | awk '{print $1}')"
+
+for c in $(podman ps -a --format '{{.Names}}' 2>/dev/null | sort); do
+  assert_container_ports "$c" || true
+done
+
+container_exists archy-bitcoin-ui && {
+  assert_http "bitcoin-ui" "http://127.0.0.1/app/bitcoin-ui/"
+  assert_http "bitcoin status" "http://127.0.0.1/app/bitcoin-ui/bitcoin-status"
+  assert_http_post "bitcoin rpc proxy" "http://127.0.0.1/app/bitcoin-ui/bitcoin-rpc/"
+}
+
+container_exists archy-electrs-ui && {
+  assert_http "electrumx ui" "http://127.0.0.1/app/electrumx/"
+  assert_http "electrumx status" "http://127.0.0.1/app/electrumx/electrs-status"
+  assert_http "electrs legacy status" "http://127.0.0.1/app/electrs/electrs-status"
+}
+
+container_exists mempool && assert_http "mempool ui" "http://127.0.0.1/app/mempool/"
+container_exists indeedhub && assert_http "indeedhub ui" "http://127.0.0.1:7778/"
+container_exists uptime-kuma && assert_http "uptime-kuma" "http://127.0.0.1/app/uptime-kuma/"
+container_exists filebrowser && assert_http "filebrowser" "http://127.0.0.1/app/filebrowser/"
+container_exists searxng && assert_http "searxng" "http://127.0.0.1/app/searxng/"
+container_exists grafana && assert_http "grafana" "http://127.0.0.1/app/grafana/"
+container_exists portainer && assert_http "portainer" "http://127.0.0.1/app/portainer/"
+container_exists vaultwarden && assert_http "vaultwarden" "http://127.0.0.1/app/vaultwarden/"
+container_exists nextcloud && assert_http "nextcloud" "http://127.0.0.1/app/nextcloud/"
+container_exists archy-nbxplorer && assert_env_contains "archy-nbxplorer" "NBXPLORER_POSTGRES" "Database=nbxplorer"
+container_exists btcpay-server && {
+  assert_env_contains "btcpay-server" "BTCPAY_POSTGRES" "Database=btcpay"
+  assert_http "btcpay" "http://127.0.0.1/app/btcpay/"
+}
+
+echo "[surface] summary: pass=$pass fail=$fail"
+[ "$fail" -eq 0 ]
+REMOTE

scripts/bitcoin-stack-lifecycle-test.sh

@@ -0,0 +1,249 @@
+#!/bin/bash
+#
+# Bitcoin stack lifecycle test.
+#
+# Exercises the production Bitcoin stack under repeated stop/start and
+# remove/recreate cycles while asserting the actual user-facing surfaces:
+# Bitcoin RPC, bitcoin-ui /bitcoin-rpc, ElectrumX status, and electrs-ui.
+#
+# This intentionally removes containers but not data volumes. It is safe for
+# installed nodes, but it will briefly interrupt Bitcoin/ElectrumX service.
+#
+# Usage:
+#   scripts/bitcoin-stack-lifecycle-test.sh --target archipelago@192.168.1.228
+#   scripts/bitcoin-stack-lifecycle-test.sh --target archipelago@192.168.1.116 --cycles 5
+
+set -euo pipefail
+
+TARGET=""
+SSH_KEY="${ARCHIPELAGO_SSH_KEY:-}"
+CYCLES=3
+SSH_EXTRA=()
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --target)
+      TARGET="${2:-}"
+      shift 2
+      ;;
+    --ssh-key)
+      SSH_KEY="${2:-}"
+      shift 2
+      ;;
+    --cycles)
+      CYCLES="${2:-}"
+      shift 2
+      ;;
+    --ssh-option)
+      SSH_EXTRA+=("-o" "${2:-}")
+      shift 2
+      ;;
+    -h|--help)
+      sed -n '1,22p' "$0"
+      exit 0
+      ;;
+    *)
+      echo "unknown argument: $1" >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [ -z "$TARGET" ]; then
+  echo "--target is required, for example archipelago@192.168.1.228" >&2
+  exit 2
+fi
+
+SSH=(ssh -F /dev/null -o BatchMode=yes -o PreferredAuthentications=publickey -o PasswordAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null)
+if [ -n "$SSH_KEY" ]; then
+  SSH+=("-i" "$SSH_KEY")
+fi
+SSH+=("${SSH_EXTRA[@]}")
+
+"${SSH[@]}" "$TARGET" "CYCLES='$CYCLES' bash -s" <<'REMOTE'
+set -euo pipefail
+
+PODMAN="${PODMAN:-podman}"
+SCRIPTS_DIR="/opt/archipelago/scripts"
+if [ ! -x "$SCRIPTS_DIR/reconcile-containers.sh" ]; then
+  SCRIPTS_DIR="$HOME/archy/scripts"
+fi
+RECONCILE="$SCRIPTS_DIR/reconcile-containers.sh"
+
+pass_count=0
+fail_count=0
+
+log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
+pass() { pass_count=$((pass_count + 1)); printf '  PASS %s\n' "$*"; }
+fail() { fail_count=$((fail_count + 1)); printf '  FAIL %s\n' "$*" >&2; }
+
+retry() {
+  local timeout="$1" label="$2"
+  shift 2
+  local end=$((SECONDS + timeout))
+  local out rc
+  while [ "$SECONDS" -lt "$end" ]; do
+    set +e
+    out=$("$@" 2>&1)
+    rc=$?
+    set -e
+    if [ "$rc" -eq 0 ]; then
+      pass "$label"
+      return 0
+    fi
+    sleep 2
+  done
+  fail "$label: $out"
+  return 1
+}
+
+rpc_pass() {
+  cat /var/lib/archipelago/secrets/bitcoin-rpc-password
+}
+
+json_rpc_reachable_or_warming() {
+  local url="$1" auth_arg=() body rc
+  if [ "${2:-}" = "auth" ]; then
+    auth_arg=(--user "archipelago:$(rpc_pass)")
+  fi
+  set +e
+  body=$(curl --connect-timeout 3 --max-time 20 -sS "${auth_arg[@]}" \
+    -H "Content-Type: application/json" \
+    --data-binary '{"jsonrpc":"1.0","id":"lifecycle-test","method":"getblockchaininfo","params":[]}' \
+    "$url" 2>&1)
+  rc=$?
+  set -e
+  [ "$rc" -eq 0 ] || {
+    echo "$body"
+    return 1
+  }
+  echo "$body" | grep -q '"result"' && return 0
+  echo "$body" | grep -q '"code":-28' && return 0
+  echo "$body"
+  return 1
+}
+
+bitcoin_status_usable() {
+  local url="$1"
+  local body
+  body=$(curl --connect-timeout 3 --max-time 20 -fsS "$url")
+  echo "$body" | grep -q '"ok":\(true\|false\)' || {
+    echo "$body"
+    return 1
+  }
+  echo "$body" | grep -q '"blockchain_info"' || echo "$body" | grep -q '"error"'
+}
+
+http_ok() {
+  local url="$1"
+  curl --connect-timeout 3 --max-time 20 -fsS -o /dev/null "$url"
+}
+
+electrs_status_ok() {
+  local url="${1:-http://127.0.0.1:50002/electrs-status}"
+  local body
+  body=$(curl --connect-timeout 3 --max-time 20 -fsS "$url")
+  echo "$body" | grep -q '"network_height":[1-9]' || {
+    echo "$body"
+    return 1
+  }
+  echo "$body" | grep -q '"status":"\(indexing\|syncing\|synced\|waiting\)"'
+}
+
+container_running() {
+  local name="$1"
+  [ "$($PODMAN inspect "$name" --format '{{.State.Status}}' 2>/dev/null || true)" = "running" ]
+}
+
+container_healthy_or_starting() {
+  local name="$1"
+  local health
+  health=$($PODMAN inspect "$name" --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' 2>/dev/null || true)
+  [ "$health" = "healthy" ] || [ "$health" = "starting" ] || [ -z "$health" ]
+}
+
+assert_bitcoin_stack() {
+  retry 90 "bitcoin-knots running" container_running bitcoin-knots
+  retry 90 "bitcoin-knots healthy/starting" container_healthy_or_starting bitcoin-knots
+  retry 90 "host Bitcoin RPC reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1:8332/ auth
+  retry 90 "backend Bitcoin status bridge usable" bitcoin_status_usable http://127.0.0.1:5678/bitcoin-status
+  retry 90 "bitcoin-ui page" http_ok http://127.0.0.1:8334/
+  retry 90 "bitcoin-ui status bridge usable" bitcoin_status_usable http://127.0.0.1:8334/bitcoin-status
+  retry 90 "bitcoin-ui app-session status bridge usable" bitcoin_status_usable http://127.0.0.1/app/bitcoin-ui/bitcoin-status
+  retry 90 "bitcoin-ui RPC proxy reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1:8334/bitcoin-rpc/
+  retry 90 "bitcoin-ui app-session RPC proxy reachable/ready" json_rpc_reachable_or_warming http://127.0.0.1/app/bitcoin-ui/bitcoin-rpc/
+}
+
+assert_electrum_stack() {
+  retry 120 "electrumx running" container_running electrumx
+  retry 120 "electrumx healthy/starting" container_healthy_or_starting electrumx
+  retry 90 "electrs-ui page" http_ok http://127.0.0.1:50002/
+  retry 120 "electrs status has network height" electrs_status_ok
+  retry 120 "electrs app-session status has network height" electrs_status_ok http://127.0.0.1/app/electrumx/electrs-status
+  retry 120 "electrs legacy app-session status has network height" electrs_status_ok http://127.0.0.1/app/electrs/electrs-status
+}
+
+reconcile_one() {
+  local name="$1"
+  "$RECONCILE" --container="$name" --force --force-recreate --create-missing
+}
+
+restart_container() {
+  local name="$1"
+  log "restart $name"
+  $PODMAN restart "$name" >/dev/null || {
+    log "podman restart failed for $name; using stop/start"
+    $PODMAN stop "$name" >/dev/null 2>&1 || true
+    sleep 3
+    $PODMAN start "$name" >/dev/null
+  }
+}
+
+remove_and_reconcile() {
+  local name="$1"
+  log "remove/recreate $name"
+  $PODMAN rm -f "$name" >/dev/null 2>&1 || true
+  reconcile_one "$name"
+}
+
+log "target $(hostname) cycles=$CYCLES"
+log "using reconciler: $RECONCILE"
+
+assert_bitcoin_stack
+assert_electrum_stack
+
+for i in $(seq 1 "$CYCLES"); do
+  log "cycle $i/$CYCLES: bitcoin restart"
+  restart_container bitcoin-knots
+  assert_bitcoin_stack
+  assert_electrum_stack
+
+  log "cycle $i/$CYCLES: bitcoin remove/reconcile"
+  remove_and_reconcile bitcoin-knots
+  assert_bitcoin_stack
+  assert_electrum_stack
+
+  log "cycle $i/$CYCLES: bitcoin UI remove/reconcile"
+  remove_and_reconcile archy-bitcoin-ui
+  assert_bitcoin_stack
+
+  log "cycle $i/$CYCLES: electrumx restart"
+  restart_container electrumx
+  assert_electrum_stack
+
+  log "cycle $i/$CYCLES: electrumx remove/reconcile"
+  remove_and_reconcile electrumx
+  assert_electrum_stack
+
+  log "cycle $i/$CYCLES: electrs UI remove/reconcile"
+  remove_and_reconcile archy-electrs-ui
+  assert_electrum_stack
+done
+
+log "final container state"
+$PODMAN ps -a --format 'table {{.Names}}\t{{.State}}\t{{.Status}}' \
+  | grep -E 'bitcoin-knots|electrumx|archy-bitcoin-ui|archy-electrs-ui' || true
+
+log "summary: pass=$pass_count fail=$fail_count"
+[ "$fail_count" -eq 0 ]
+REMOTE

scripts/container-doctor.sh

@@ -15,6 +15,7 @@
 #   6. Bitcoin Knots prune+txindex conflict
 #   7. Containers stuck with exit code 127 (binary not found)
 #   8. Stopped core containers (rootless restart policy workaround)
+#   9. Missing rootless port listeners while Podman still shows published ports
 #
 # Safe to run multiple times (idempotent). Never blocks deploy (exit 0 always).
 #
@@ -31,6 +32,21 @@ FIX_NAMES=()
 
 log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
 
+podman_rootless() {
+    if [ "$(id -u)" = "0" ] && id archipelago >/dev/null 2>&1; then
+        local archi_uid
+        archi_uid=$(id -u archipelago)
+        sudo -u archipelago env XDG_RUNTIME_DIR="/run/user/$archi_uid" podman "$@"
+    else
+        podman "$@"
+    fi
+}
+
+port_is_listening() {
+    local port="$1"
+    ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
+}
+
 run_fix() {
     local name="$1"
     shift
@@ -374,6 +390,11 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
 # at 0 peers; package pulls fail. The only reliable repair is a stop-all/
 # start-all cycle so pasta + aardvark-dns rebuild the netns from scratch.
 fix_rootless_netns_egress() {
+    # Needs root for nsenter. When doctor runs as the rootless container owner,
+    # a failed nsenter probe is a permissions artifact, not evidence of broken
+    # egress; do not cycle the fleet from that context.
+    [ "$(id -u)" = "0" ] || return 1
+
     local archi_uid
     archi_uid=$(id -u archipelago 2>/dev/null) || return 1
 
@@ -453,6 +474,44 @@ fix_stopped_core_containers() {
     [ ${#restarted[@]} -gt 0 ] && return 0 || return 1
 }
 
+# ── Fix 10: Missing rootless port listeners ─────────────────
+# Rootless Podman can leave a container running with PortBindings still present
+# while the host-side rootlessport process has disappeared. Nginx then returns
+# 502 and direct app ports refuse connections even though `podman ps` looks OK.
+fix_missing_rootless_ports() {
+    local containers
+    containers=$(podman_rootless ps --format '{{.Names}}' 2>/dev/null || true)
+    [ -n "$containers" ] || return 1
+
+    local fixed=false
+    local name
+    for name in $containers; do
+        local ports
+        ports=$(podman_rootless inspect "$name" --format '{{range $p,$bindings := .NetworkSettings.Ports}}{{if $bindings}}{{range $bindings}}{{.HostPort}}{{"\n"}}{{end}}{{end}}{{end}}' 2>/dev/null | sort -u)
+        [ -n "$ports" ] || continue
+
+        local missing=()
+        local port
+        for port in $ports; do
+            [ -n "$port" ] || continue
+            if ! port_is_listening "$port"; then
+                missing+=("$port")
+            fi
+        done
+
+        if [ ${#missing[@]} -gt 0 ]; then
+            log "Restarting $name: missing rootlessport listener(s): ${missing[*]}"
+            if podman_rootless restart "$name" >/dev/null 2>&1; then
+                fixed=true
+            else
+                log "WARN: failed to restart $name for missing rootlessport listener(s)"
+            fi
+        fi
+    done
+
+    $fixed && return 0 || return 1
+}
+
 # ── Main ─────────────────────────────────────────────────────
 
 # If remote host provided, run via SSH
@@ -481,6 +540,7 @@ run_fix "bitcoin-txindex" fix_bitcoin_txindex
 run_fix "exit-127"        fix_exit_127
 run_fix "netns-egress"    fix_rootless_netns_egress
 run_fix "stopped-core"    fix_stopped_core_containers
+run_fix "rootless-ports" fix_missing_rootless_ports
 
 echo ""
 if [ $FIXES_APPLIED -gt 0 ]; then

scripts/container-specs.sh

@@ -252,7 +252,7 @@ load_spec_archy-nbxplorer() {
   SPEC_VOLUMES="/var/lib/archipelago/nbxplorer:/data"
   SPEC_MEMORY="$(mem_limit archy-nbxplorer)"
   SPEC_HEALTH_CMD="curl -sf http://localhost:32838/ || exit 1"
-  SPEC_ENV="NBXPLORER_DATADIR=/data NBXPLORER_NETWORK=mainnet NBXPLORER_CHAINS=btc NBXPLORER_BIND=0.0.0.0:32838 NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS NBXPLORER_POSTGRES=User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true"
+  SPEC_ENV="NBXPLORER_DATADIR=/data NBXPLORER_NETWORK=mainnet NBXPLORER_CHAINS=btc NBXPLORER_BIND=0.0.0.0:32838 NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS NBXPLORER_POSTGRES=Username=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer"
   SPEC_TIER="2"
   SPEC_DATA_DIR="/var/lib/archipelago/nbxplorer"
   SPEC_DEPENDS="bitcoin-knots archy-btcpay-db"
@@ -268,7 +268,7 @@ load_spec_btcpay-server() {
   SPEC_VOLUMES="/var/lib/archipelago/btcpay:/datadir"
   SPEC_MEMORY="$(mem_limit btcpay-server)"
   SPEC_HEALTH_CMD="curl -sf http://localhost:49392/ || exit 1"
-  SPEC_ENV="ASPNETCORE_URLS=http://0.0.0.0:49392 BTCPAY_PROTOCOL=http BTCPAY_HOST=$HOST_IP:23000 BTCPAY_CHAINS=btc BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS BTCPAY_POSTGRES=User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true"
+  SPEC_ENV="ASPNETCORE_URLS=http://0.0.0.0:49392 BTCPAY_PROTOCOL=http BTCPAY_HOST=$HOST_IP:23000 BTCPAY_CHAINS=btc BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 BTCPAY_BTCRPCUSER=$BITCOIN_RPC_USER BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS BTCPAY_POSTGRES=Username=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay"
   SPEC_TIER="2"
   SPEC_DATA_DIR="/var/lib/archipelago/btcpay"
   SPEC_DEPENDS="archy-nbxplorer archy-btcpay-db"
@@ -344,7 +344,7 @@ load_spec_homeassistant() {
   SPEC_ENV="TZ=UTC"
   SPEC_TIER="3"
   SPEC_DATA_DIR="/var/lib/archipelago/home-assistant"
-  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE"
+  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE"
   SPEC_OPTIONAL="true"
 }
 
@@ -362,7 +362,7 @@ load_spec_grafana() {
   SPEC_TIER="3"
   SPEC_DATA_DIR="/var/lib/archipelago/grafana"
   SPEC_DATA_UID="100472:100472"
-  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE"
+  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE"
   SPEC_OPTIONAL="true"
 }
 
@@ -370,7 +370,7 @@ load_spec_uptime-kuma() {
   reset_spec
   SPEC_NAME="uptime-kuma"
   SPEC_IMAGE="${UPTIME_KUMA_IMAGE}"
-  SPEC_PORTS="3001:3001"
+  SPEC_PORTS="3002:3001"
   SPEC_VOLUMES="/var/lib/archipelago/uptime-kuma:/app/data"
   SPEC_MEMORY="$(mem_limit uptime-kuma)"
   SPEC_HEALTH_CMD="curl -sf http://localhost:3001/ || exit 1"
@@ -434,7 +434,7 @@ load_spec_nextcloud() {
   SPEC_HEALTH_CMD="curl -sf http://localhost:80/ || exit 1"
   SPEC_TIER="3"
   SPEC_DATA_DIR="/var/lib/archipelago/nextcloud"
-  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE"
+  SPEC_CAPS="CHOWN SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE"
   SPEC_OPTIONAL="true"
 }
 
@@ -539,6 +539,7 @@ load_spec_archy-bitcoin-ui() {
   SPEC_NAME="archy-bitcoin-ui"
   SPEC_IMAGE="localhost/bitcoin-ui:local"
   SPEC_NETWORK="host"
+  SPEC_VOLUMES="/var/lib/archipelago/bitcoin-ui/nginx.conf:/etc/nginx/conf.d/default.conf:ro"
   SPEC_MEMORY="$(mem_limit archy-bitcoin-ui)"
   SPEC_TIER="4"
   SPEC_LOCAL_IMAGE="true"

scripts/create-release-manifest.sh

@@ -87,6 +87,30 @@ if [ -z "$FRONTEND_ARCHIVE" ]; then
             echo "  Including AIUI from demo/aiui/"
             cp -r "$PROJECT_ROOT/demo/aiui" "$STAGING_DIR/aiui"
         fi
+        # OTA bridge for nodes running older updaters: they only know how to
+        # apply the backend binary and frontend archive. Carry host runtime
+        # assets inside the frontend tarball; the new backend promotes them
+        # from /opt/archipelago/web-ui/archipelago-runtime on first startup.
+        RUNTIME_DIR="$STAGING_DIR/archipelago-runtime"
+        mkdir -p "$RUNTIME_DIR"
+        for runtime_path in apps scripts docker; do
+            if [ -d "$PROJECT_ROOT/$runtime_path" ]; then
+                echo "  Including runtime $runtime_path/"
+                cp -r "$PROJECT_ROOT/$runtime_path" "$RUNTIME_DIR/$runtime_path"
+            fi
+        done
+        if [ -f "$PROJECT_ROOT/image-recipe/configs/archipelago-doctor.service" ] || \
+           [ -f "$PROJECT_ROOT/image-recipe/configs/archipelago-doctor.timer" ]; then
+            mkdir -p "$RUNTIME_DIR/image-recipe/configs"
+            for unit in archipelago-doctor.service archipelago-doctor.timer; do
+                if [ -f "$PROJECT_ROOT/image-recipe/configs/$unit" ]; then
+                    echo "  Including runtime unit $unit"
+                    cp "$PROJECT_ROOT/image-recipe/configs/$unit" "$RUNTIME_DIR/image-recipe/configs/$unit"
+                fi
+            done
+        fi
+        rm -rf "$RUNTIME_DIR/scripts/resilience/reports"
+        find "$RUNTIME_DIR" -type f \( -name '*.bak' -o -name '._*' -o -name '*.log' \) -delete
         # Force world-readable perms on every entry BEFORE tar, so the
         # archive's internal mode bits are 755/644 regardless of what
         # the staging dir's umask gave us. Without this, mktemp -d

scripts/nginx-https-app-proxies.conf

@@ -183,6 +183,26 @@ location /app/electrs/ {
     proxy_hide_header X-Frame-Options;
     proxy_hide_header Content-Security-Policy;
 }
+location /app/electrumx/ {
+    proxy_pass http://127.0.0.1:50002/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_hide_header X-Frame-Options;
+    proxy_hide_header Content-Security-Policy;
+}
+location /app/electrs-ui/ {
+    proxy_pass http://127.0.0.1:50002/;
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_hide_header X-Frame-Options;
+    proxy_hide_header Content-Security-Policy;
+}
 location /app/nginx-proxy-manager/ {
     proxy_pass http://127.0.0.1:81/;
     proxy_http_version 1.1;

scripts/reconcile-containers.sh

@@ -8,6 +8,7 @@
 #   sudo ./reconcile-containers.sh                 # Fix everything
 #   sudo ./reconcile-containers.sh --check-only    # Audit only, no changes
 #   sudo ./reconcile-containers.sh --force          # Override user-stopped
+#   sudo ./reconcile-containers.sh --force-recreate # Recreate matched containers
 #   sudo ./reconcile-containers.sh --tier=2         # Only reconcile tier 2
 #   sudo ./reconcile-containers.sh --container=lnd  # Only reconcile lnd
 #
@@ -18,6 +19,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 # ── Parse arguments ──────────────────────────────────────────────────
 CHECK_ONLY=false
 FORCE=false
+FORCE_RECREATE=false
 CREATE_MISSING=false
 FILTER_TIER=""
 FILTER_CONTAINER=""
@@ -25,14 +27,18 @@ for arg in "$@"; do
   case "$arg" in
     --check-only)         CHECK_ONLY=true ;;
     --force)              FORCE=true ;;
+    --force-recreate)     FORCE_RECREATE=true ;;
     --create-missing)     CREATE_MISSING=true ;;
     --tier=*)             FILTER_TIER="${arg#*=}" ;;
     --container=*)        FILTER_CONTAINER="${arg#*=}" ;;
     -h|--help)
-      echo "Usage: $0 [--check-only] [--force] [--create-missing] [--tier=N] [--container=NAME]"
+      echo "Usage: $0 [--check-only] [--force] [--force-recreate] [--create-missing] [--tier=N] [--container=NAME]"
       echo ""
       echo "  --check-only      Audit only, no changes."
       echo "  --force           Override user-stopped state."
+      echo "  --force-recreate  Recreate matched existing containers even if they"
+      echo "                    otherwise match the spec. Use with --container or"
+      echo "                    --tier for scoped image/config refreshes."
       echo "  --create-missing  Override SPEC_OPTIONAL for containers that have on-disk"
       echo "                    data but no live container (recovery from failed updates)."
       echo "  --tier=N          Only reconcile containers in tier N."
@@ -110,6 +116,14 @@ container_image() {
   $PODMAN inspect "$1" --format '{{.ImageName}}' 2>/dev/null
 }
 
+container_image_id() {
+  $PODMAN inspect "$1" --format '{{.Image}}' 2>/dev/null
+}
+
+spec_image_id() {
+  $PODMAN image inspect "$SPEC_IMAGE" --format '{{.Id}}' 2>/dev/null
+}
+
 container_network() {
   # Use actual Networks map — NetworkMode is unreliable (always shows 'bridge' in rootless)
   local nets
@@ -122,6 +136,34 @@ container_memory() {
   $PODMAN inspect "$1" --format '{{.HostConfig.Memory}}' 2>/dev/null
 }
 
+container_health_cmd() {
+  $PODMAN inspect "$1" --format '{{with .Config.Healthcheck}}{{range .Test}}{{println .}}{{end}}{{end}}' 2>/dev/null \
+    | awk 'NR > 1 { print }' \
+    | paste -sd ' ' -
+}
+
+normalize_health_cmd() {
+  printf '%s' "$1" | sed 's/\\"/"/g; s/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//'
+}
+
+host_port_listening() {
+  local port="$1"
+  ss -ltn 2>/dev/null | awk -v p=":$port" '
+    $4 == p || $4 ~ p "$" { found=1 }
+    END { exit found ? 0 : 1 }
+  '
+}
+
+container_has_mount() {
+  local name="$1" source="$2" target="$3"
+  $PODMAN inspect "$name" --format '{{range .Mounts}}{{println .Source "|" .Destination}}{{end}}' 2>/dev/null \
+    | awk -F'|' -v src="$source" -v dst="$target" '
+        { gsub(/[[:space:]]+$/, "", $1); gsub(/^[[:space:]]+/, "", $2); }
+        $1 == src && $2 == dst { found=1 }
+        END { exit found ? 0 : 1 }
+      '
+}
+
 # Read one environment variable's current value from a running/stopped container.
 # Returns empty string if the var is not set.
 container_env_val() {
@@ -153,6 +195,36 @@ image_exists() {
   echo "$images" | grep -qF "$1"
 }
 
+resolve_spec_image() {
+  image_exists "$SPEC_IMAGE" && return
+
+  local image_path image_name image_tag candidate repo
+  image_path="${SPEC_IMAGE#*/}"
+  image_name="${SPEC_IMAGE##*/}"
+  image_tag="${image_name#*:}"
+  image_name="${image_name%%:*}"
+
+  for candidate in \
+    "${ARCHY_REGISTRY_FALLBACK:-}/${image_path}" \
+    "80.71.235.15:3000/archipelago/${image_name}:${image_tag}" \
+    "80.71.235.15:3000/lfg2025/${image_name}:${image_tag}"; do
+    [ "$candidate" = "/" ] && continue
+    if image_exists "$candidate"; then
+      info "$SPEC_NAME — using local image alias $candidate"
+      SPEC_IMAGE="$candidate"
+      return
+    fi
+  done
+
+  repo=$($PODMAN images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null \
+    | grep -E "/${image_name}:${image_tag}$" \
+    | head -1 || true)
+  if [ -n "$repo" ]; then
+    info "$SPEC_NAME — using local image alias $repo"
+    SPEC_IMAGE="$repo"
+  fi
+}
+
 # Convert memory string to bytes for comparison
 mem_to_bytes() {
   local m="$1"
@@ -262,6 +334,10 @@ reconcile() {
     return
   fi
 
+  # Resolve registry aliases before create/recreate. ISOs and older installers
+  # may seed the same image under a fallback registry tag.
+  resolve_spec_image
+
   # Local images: skip if image doesn't exist and container doesn't exist
   if [ "$SPEC_LOCAL_IMAGE" = "true" ]; then
     if ! image_exists "$SPEC_IMAGE" && ! container_exists "$name"; then
@@ -284,14 +360,28 @@ reconcile() {
   local reasons=""
 
   if container_exists "$name"; then
-    local cur_image cur_network cur_memory
+    local cur_image cur_image_id want_image_id cur_network cur_memory
     cur_image=$(container_image "$name")
+    cur_image_id=$(container_image_id "$name")
+    want_image_id=$(spec_image_id)
     cur_network=$(container_network "$name")
     cur_memory=$(container_memory "$name")
     local spec_memory_bytes expected_network
 
     spec_memory_bytes=$(mem_to_bytes "$SPEC_MEMORY")
 
+    if [ "$FORCE_RECREATE" = "true" ]; then
+      action="RECREATE"
+      reasons+="force-recreate "
+    fi
+
+    # Same-tag local rebuilds leave running containers on the old image ID.
+    # Recreate when the currently tagged spec image points at a different ID.
+    if [ "$action" = "OK" ] && [ -n "$want_image_id" ] && [ -n "$cur_image_id" ] && [ "$cur_image_id" != "$want_image_id" ]; then
+      action="RECREATE"
+      reasons+="image-id "
+    fi
+
     # Check network mismatch
     # For archy-net and host: exact match required
     # For bridge/default: accept any non-archy-net, non-host network
@@ -319,6 +409,19 @@ reconcile() {
       reasons+="memory(none→$SPEC_MEMORY) "
     fi
 
+    # Healthcheck drift matters: a stale check can leave an otherwise working
+    # service permanently unhealthy (for example ElectrumX images do not ship
+    # curl, so the healthcheck must use python's socket module).
+    if [ "$action" = "OK" ] && [ -n "$SPEC_HEALTH_CMD" ]; then
+      local cur_health spec_health
+      cur_health=$(normalize_health_cmd "$(container_health_cmd "$name")")
+      spec_health=$(normalize_health_cmd "$SPEC_HEALTH_CMD")
+      if [ "$cur_health" != "$spec_health" ]; then
+        action="RECREATE"
+        reasons+="healthcheck "
+      fi
+    fi
+
     # Check URL/HOST env drift — catches stale network topology baked into
     # container env (fedimint April-11 bug: FM_P2P_URL pointed at old IP).
     # Only checks URL-shaped keys; other env drift (passwords rotated, etc.)
@@ -342,6 +445,40 @@ reconcile() {
       done
     fi
 
+    # Check bind mounts. This catches companion UIs recreated from older specs,
+    # especially bitcoin-ui: its image intentionally does not bake nginx.conf,
+    # so the rendered RPC proxy config must be mounted from the host.
+    if [ "$action" = "OK" ] && [ -n "$SPEC_VOLUMES" ]; then
+      for v in $SPEC_VOLUMES; do
+        local mount_source mount_rest mount_target
+        mount_source="${v%%:*}"
+        mount_rest="${v#*:}"
+        mount_target="${mount_rest%%:*}"
+        [ -n "$mount_source" ] && [ -n "$mount_target" ] || continue
+        if ! container_has_mount "$name" "$mount_source" "$mount_target"; then
+          action="RECREATE"
+          reasons+="mount($mount_target) "
+          break
+        fi
+      done
+    fi
+
+    # Rootless Podman can occasionally leave a container running while its
+    # rootlessport listener is gone. The container still looks healthy in
+    # `podman ps`, but host-network UIs and backend status probes fail against
+    # 127.0.0.1. Treat missing host listeners as spec drift.
+    if [ "$action" = "OK" ] && [ -n "$SPEC_PORTS" ]; then
+      for p in $SPEC_PORTS; do
+        local host_port="${p%%:*}"
+        [ -n "$host_port" ] || continue
+        if ! host_port_listening "$host_port"; then
+          action="RECREATE"
+          reasons+="port($host_port-not-listening) "
+          break
+        fi
+      done
+    fi
+
     # Check if running
     if ! container_running "$name" && [ "$action" = "OK" ]; then
       action="START"
@@ -476,7 +613,7 @@ ensure_secrets() {
 ensure_bitcoin_conf() {
   local BITCOIN_CONF="/var/lib/archipelago/bitcoin/bitcoin.conf"
   sudo mkdir -p /var/lib/archipelago/bitcoin 2>/dev/null
-  if [ ! -f "$BITCOIN_CONF" ] || ! grep -q "^rpcauth=" "$BITCOIN_CONF" 2>/dev/null; then
+  if [ ! -f "$BITCOIN_CONF" ] || ! sudo grep -q "^rpcauth=" "$BITCOIN_CONF" 2>/dev/null; then
     if ! $CHECK_ONLY && [ -n "$BITCOIN_RPC_PASS" ]; then
       local salt hash rpcauth
       salt=$(openssl rand -hex 16)
@@ -491,10 +628,14 @@ BTCEOF
       info "Generated bitcoin.conf"
     fi
   fi
-  # Strip duplicate server/rpc/listen lines from existing conf to avoid conflicts with custom args
-  if [ -f "$BITCOIN_CONF" ]; then
-    sudo sed -i '/^server=/d; /^rpcbind=/d; /^rpcallowip=/d; /^rpcport=/d; /^listen=/d' "$BITCOIN_CONF" 2>/dev/null
-  fi
+  # Strip duplicate server/rpc/listen lines from existing conf files to avoid
+  # conflicts with custom args. Knots can persist runtime args in
+  # bitcoin_rw.conf, so clean both files.
+  for conf in "$BITCOIN_CONF" "/var/lib/archipelago/bitcoin/bitcoin_rw.conf"; do
+    if [ -f "$conf" ]; then
+      sudo sed -i '/^server=/d; /^txindex=/d; /^rpcbind=/d; /^rpcallowip=/d; /^rpcport=/d; /^listen=/d; /^bind=/d; /^dbcache=/d' "$conf" 2>/dev/null
+    fi
+  done
   sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
 }
 
@@ -531,6 +672,63 @@ LNDEOF
   fi
 }
 
+# ── Ensure bitcoin-ui nginx.conf ────────────────────────────────────
+ensure_bitcoin_ui_nginx_conf() {
+  local CONF_DIR="/var/lib/archipelago/bitcoin-ui"
+  local CONF_PATH="$CONF_DIR/nginx.conf"
+  [ -n "$BITCOIN_RPC_PASS" ] || return
+  if $CHECK_ONLY; then
+    [ -f "$CONF_PATH" ] || info "Would generate bitcoin-ui nginx.conf"
+    return
+  fi
+
+  local auth_b64 tmp
+  auth_b64=$(printf '%s' "${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASS}" | base64 | tr -d '\n')
+  sudo mkdir -p "$CONF_DIR" 2>/dev/null
+  tmp="${CONF_PATH}.tmp.$$"
+  sudo tee "$tmp" >/dev/null << EOF
+server {
+    listen 8334;
+    server_name _;
+    root /usr/share/nginx/html;
+    index index.html;
+
+    location /bitcoin-rpc/ {
+        proxy_pass http://127.0.0.1:8332/;
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header Authorization "Basic ${auth_b64}";
+        add_header Access-Control-Allow-Origin *;
+        add_header Access-Control-Allow-Methods "POST, GET, OPTIONS";
+        add_header Access-Control-Allow-Headers "Content-Type, Authorization";
+        if (\$request_method = OPTIONS) { return 204; }
+    }
+
+    location /bitcoin-status {
+        proxy_pass http://127.0.0.1:5678/bitcoin-status;
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        add_header Cache-Control "no-store";
+    }
+
+    location / {
+        try_files \$uri \$uri/ /index.html;
+    }
+}
+EOF
+  if ! sudo cmp -s "$tmp" "$CONF_PATH" 2>/dev/null; then
+    sudo mv "$tmp" "$CONF_PATH"
+    sudo chmod 644 "$CONF_PATH"
+    info "Generated bitcoin-ui nginx.conf"
+  else
+    sudo rm -f "$tmp"
+  fi
+}
+
 # ── Ensure BTCPay databases ─────────────────────────────────────────
 ensure_btcpay_db() {
   if container_running "archy-btcpay-db"; then
@@ -548,8 +746,10 @@ START_TIME=$(date +%s)
 
 header "Phase 0: Prerequisites"
 ensure_secrets
+detect_environment
 ensure_bitcoin_conf
 ensure_lnd_conf
+ensure_bitcoin_ui_nginx_conf
 
 TIER_NAMES=("Databases" "Core Infrastructure" "Services" "Applications" "Frontend UIs")
 

scripts/resilience/lib.sh

@@ -136,7 +136,7 @@ expected_containers_for() {
 ui_proxy_path_for() {
     case "$1" in
         bitcoin-knots|bitcoin-core)  echo "/app/bitcoin-ui/" ;;
-        electrumx|electrs)           echo "/app/electrs-ui/" ;;
+        electrumx|electrs)           echo "/app/electrumx/" ;;
         lnd)                         echo "/app/lnd-ui/" ;;
         btcpay-server)               echo "/app/btcpay/" ;;
         *)                           echo "/app/$1/" ;;

scripts/self-update.sh

@@ -186,7 +186,7 @@ fi
 # for backward compatibility with older binaries that still look there.
 SCRIPTS_DEST="/opt/archipelago/scripts"
 sudo mkdir -p "$SCRIPTS_DEST"
-for script in image-versions.sh reconcile-containers.sh container-specs.sh; do
+for script in image-versions.sh reconcile-containers.sh container-specs.sh container-doctor.sh app-surface-smoke-test.sh bitcoin-stack-lifecycle-test.sh; do
     src="$REPO_DIR/scripts/$script"
     if [ -f "$src" ]; then
         sudo install -m 755 "$src" "$SCRIPTS_DEST/$script"
@@ -200,6 +200,19 @@ if [ -f "$REPO_DIR/scripts/image-versions.sh" ]; then
     sudo cp "$REPO_DIR/scripts/image-versions.sh" /opt/archipelago/image-versions.sh
 fi
 
+# Sync app manifests and app-local build contexts into the canonical
+# production manifest root. The backend orchestrator loads install specs from
+# /opt/archipelago/apps; updating only the binary/frontend can leave a node
+# with new installer logic but stale or missing app manifests.
+APPS_DEST="/opt/archipelago/apps"
+if [ -d "$REPO_DIR/apps" ]; then
+    sudo mkdir -p "$APPS_DEST"
+    sudo rsync -a --delete "$REPO_DIR/apps/" "$APPS_DEST/"
+    ok "App manifests synced"
+else
+    warn "Apps directory not found at $REPO_DIR/apps — install manifests may be stale"
+fi
+
 # Update first-boot-containers.sh too (the canonical first-boot orchestrator).
 # Nodes run it once on install, but keeping a fresh copy on disk means any
 # future boot or reconciler invocation uses current port specs and caps.
@@ -299,6 +312,25 @@ if [ -f "$REPO_DIR/image-recipe/configs/archipelago.service" ]; then
     fi
 fi
 
+# Keep the doctor timer/service current too. Container uptime fixes rely on
+# these units as much as on the helper scripts themselves.
+DOCTOR_UNITS_CHANGED=false
+for unit in archipelago-doctor.service archipelago-doctor.timer; do
+    src="$REPO_DIR/image-recipe/configs/$unit"
+    dst="/etc/systemd/system/$unit"
+    [ -f "$src" ] || continue
+    if [ ! -f "$dst" ] || ! diff -q "$src" "$dst" &>/dev/null; then
+        sudo install -m 644 "$src" "$dst"
+        DOCTOR_UNITS_CHANGED=true
+        ok "Updated $unit"
+    fi
+done
+if [ "$DOCTOR_UNITS_CHANGED" = "true" ]; then
+    sudo systemctl daemon-reload
+    sudo systemctl enable --now archipelago-doctor.timer 2>>"$LOG_FILE" || \
+        warn "Failed to enable archipelago-doctor.timer"
+fi
+
 # Install/refresh tmpfiles.d rules. The logs rule creates
 # /var/log/archipelago/ + container-installs.log with archipelago:archipelago
 # ownership so the non-root backend can append install audit lines.