release(v1.7.32-alpha): fix frontend tarball layout + mDNS shutdown hang

- HOTFIX: v1.7.31-alpha's frontend tarball was packaged with a
  `neode-ui/` top-level directory instead of the flat layout v1.7.30
  and earlier used. Nodes that applied v1.7.31 ended up with
  `/opt/archipelago/web-ui/neode-ui/index.html` instead of
  `/opt/archipelago/web-ui/index.html`, and nginx returned 403/500.
  v1.7.32's tarball is built with `tar -C web/dist/neode-ui .` so
  files land directly at web-ui root. Broken nodes auto-heal on this
  update (web-ui dir is replaced).
- transport/lan.rs: add Drop impl that calls ServiceDaemon::shutdown()
  on the mdns_sd daemon. Without this the OS thread it spawns, plus
  the blocking `receiver.recv()` task, keep the tokio runtime alive
  past SIGTERM — long enough for systemd's TimeoutStopSec to SIGKILL
  the service and mark it Failed. Was visible on every update:
  "shut down cleanly" logged, then 15s later systemd forcibly kills.
- main.rs: after logging "Archipelago shut down cleanly", call
  `std::process::exit(0)` explicitly. Belt-and-suspenders against
  any future non-daemon thread creeping in (reqwest resolver pool,
  etc.) and causing the same SIGKILL regression.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

core/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
 
 [[package]]
 name = "archipelago"
-version = "1.7.29-alpha"
+version = "1.7.32-alpha"
 dependencies = [
  "anyhow",
  "archipelago-container",

core/archipelago/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "archipelago"
-version = "1.7.29-alpha"
+version = "1.7.32-alpha"
 edition = "2021"
 description = "Archipelago Bitcoin Node OS - Native backend"
 authors = ["Archipelago Team"]

core/archipelago/src/api/rpc/package/install.rs

@@ -635,47 +635,32 @@ impl RpcHandler {
         unreachable!()
     }
 
-    /// Single image pull attempt with progress streaming.
-    async fn do_pull_image(&self, package_id: &str, docker_image: &str) -> Result<()> {
-        debug!("Pulling image: {}", docker_image);
-        self.set_install_progress(package_id, 0, 0).await;
-
-        // Set TMPDIR to user-writable location — rootless podman's user namespace
-        // makes /var/tmp read-only, which causes `podman pull` to fail with
-        // "mkdir /var/tmp/container_images_storage...: read-only file system"
-        let user_tmp = format!(
-            "{}/.local/share/containers/tmp",
-            std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string())
-        );
-        let _ = std::fs::create_dir_all(&user_tmp);
-
-        // Rewrite to the primary registry's URL so the first attempt
-        // honors the operator's mirror choice (default: VPS) instead of
-        // blindly using whatever registry the image was hardcoded to.
-        // If the rewritten URL fails, pull_from_registries_with_skip
-        // falls through to the other configured registries.
-        let (primary_url, primary_tls) =
-            crate::container::registry::primary_image_url(&self.config.data_dir, docker_image)
-                .await;
-        if primary_url != docker_image {
-            debug!("Rewrote {} → {} for primary registry", docker_image, primary_url);
-        }
-
-        let mut pull_args = vec!["pull".to_string(), primary_url.clone()];
-        if !primary_tls {
+    /// Pull one image URL with live progress streamed through
+    /// `update_install_progress`. Returns Ok(true) on a successful pull,
+    /// Ok(false) on transient failure (so the caller can try the next
+    /// mirror), Err only for unrecoverable setup errors.
+    async fn pull_one_url_with_progress(
+        &self,
+        url: &str,
+        tls_verify: bool,
+        package_id: &str,
+        user_tmp: &str,
+    ) -> Result<bool> {
+        let mut pull_args = vec!["pull".to_string(), url.to_string()];
+        if !tls_verify {
             pull_args.push("--tls-verify=false".to_string());
         }
         let mut child = tokio::process::Command::new("podman")
             .args(&pull_args)
-            .env("TMPDIR", &user_tmp)
+            .env("TMPDIR", user_tmp)
             .stdout(std::process::Stdio::piped())
             .stderr(std::process::Stdio::piped())
             .spawn()
             .context("Failed to start image pull")?;
 
-        // Wrap the entire pull (stderr progress + wait) in a 10-minute timeout.
-        // Large image layers (Minio, Postgres, ffmpeg) can take several minutes
-        // to pull. 60s was too short and caused premature retries on slow registries.
+        // 10-minute per-URL budget — large layers (Minio, Postgres,
+        // ffmpeg) regularly take several minutes and we'd rather wait
+        // than bounce to the next mirror mid-download.
         let pull_result = tokio::time::timeout(std::time::Duration::from_secs(600), async {
             if let Some(stderr) = child.stderr.take() {
                 let reader = BufReader::new(stderr);
@@ -693,50 +678,115 @@ impl RpcHandler {
         })
         .await;
 
-        let primary_failed = match pull_result {
-            Ok(Ok(status)) => !status.success(),
+        match pull_result {
+            Ok(Ok(status)) => Ok(status.success()),
             Ok(Err(e)) => {
-                tracing::warn!("Image pull process error: {}", e);
-                true
+                tracing::warn!("Image pull process error on {}: {}", url, e);
+                Ok(false)
             }
             Err(_) => {
-                tracing::warn!("Image pull timed out after 60s: {}", primary_url);
+                tracing::warn!("Image pull timed out after 600s: {}", url);
                 let _ = child.kill().await;
                 let _ = child.wait().await; // reap zombie
-                true
+                Ok(false)
             }
-        };
-        if !primary_failed && primary_url != docker_image {
-            // Primary pull succeeded but used a rewritten URL. Tag under
-            // the original image reference so downstream code (images -q,
-            // run -d docker_image, etc.) finds it.
-            let _ = tokio::process::Command::new("podman")
-                .args(["tag", &primary_url, docker_image])
-                .output()
-                .await;
-            tracing::info!("Pulled {} from primary registry ({})", docker_image, primary_url);
         }
-        if primary_failed {
-            // Primary failed — walk the remaining configured registries.
-            // Skip primary_url so we don't retry what just failed.
-            match crate::container::registry::pull_from_registries_with_skip(
-                &self.config.data_dir,
-                docker_image,
-                &user_tmp,
-                Some(&primary_url),
-            )
+    }
+
+    /// Pull a container image, trying each configured registry in
+    /// priority order and streaming progress during every attempt. The
+    /// primary is tried first; if it doesn't have the image (or 404's),
+    /// the next mirror is tried — with its own progress streaming, so
+    /// the UI doesn't freeze at 0% after a primary miss. On success the
+    /// image is tagged under `docker_image` so downstream commands
+    /// (images -q, run -d, etc.) can find it by its canonical name.
+    async fn do_pull_image(&self, package_id: &str, docker_image: &str) -> Result<()> {
+        debug!("Pulling image: {}", docker_image);
+        self.set_install_progress(package_id, 0, 0).await;
+
+        // Set TMPDIR to user-writable location — rootless podman's user namespace
+        // makes /var/tmp read-only, which causes `podman pull` to fail with
+        // "mkdir /var/tmp/container_images_storage...: read-only file system"
+        let user_tmp = format!(
+            "{}/.local/share/containers/tmp",
+            std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string())
+        );
+        let _ = std::fs::create_dir_all(&user_tmp);
+
+        // Build the ordered candidate list: every enabled registry
+        // (highest priority first), each rewriting the image URL to its
+        // own origin. Deduplicate — two registries that happen to share
+        // a URL should only be tried once.
+        let config = crate::container::registry::load_registries(&self.config.data_dir)
             .await
+            .unwrap_or_default();
+        let mut tried: std::collections::HashSet<String> = std::collections::HashSet::new();
+        let mut candidates: Vec<(String, bool)> = Vec::new();
+        for reg in config.active_registries() {
+            let url = config.rewrite_image(docker_image, reg);
+            if tried.insert(url.clone()) {
+                candidates.push((url, reg.tls_verify));
+            }
+        }
+        // If no registries are configured, fall back to the literal URL.
+        if candidates.is_empty() {
+            candidates.push((docker_image.to_string(), true));
+        }
+
+        // Walk candidates, streaming progress for each attempt.
+        let mut pulled_url: Option<String> = None;
+        let attempts = candidates.len();
+        for (i, (url, tls_verify)) in candidates.iter().enumerate() {
+            if url != docker_image {
+                debug!("Attempt {}/{}: {}", i + 1, attempts, url);
+            } else {
+                debug!("Attempt {}/{}: {} (literal)", i + 1, attempts, url);
+            }
+            // Reset progress at the top of each attempt so the UI reflects
+            // the fresh pull instead of showing stale bytes from a prior
+            // partial attempt.
+            self.set_install_progress(package_id, 0, 0).await;
+            match self
+                .pull_one_url_with_progress(url, *tls_verify, package_id, &user_tmp)
+                .await?
             {
-                Ok(_) => {
-                    tracing::info!("Pulled {} via fallback registry", docker_image);
+                true => {
+                    tracing::info!("Pulled {} from {}", docker_image, url);
+                    pulled_url = Some(url.clone());
+                    break;
                 }
-                Err(e) => {
-                    return Err(anyhow::anyhow!("Image pull failed: {}", e));
+                false => {
+                    tracing::debug!(
+                        "Pull attempt {}/{} failed for {}, trying next mirror",
+                        i + 1,
+                        attempts,
+                        url
+                    );
+                    continue;
                 }
             }
         }
 
-        // Verify image exists locally after pull
+        let Some(pulled_url) = pulled_url else {
+            return Err(anyhow::anyhow!(
+                "Image pull failed from all {} configured registries for {}",
+                attempts,
+                docker_image
+            ));
+        };
+
+        // Tag under the original docker_image reference if the successful
+        // pull came from a rewritten URL — downstream code (images -q,
+        // run -d docker_image, etc.) needs to find it by its canonical
+        // name regardless of which mirror actually served the bytes.
+        if pulled_url != docker_image {
+            let _ = tokio::process::Command::new("podman")
+                .args(["tag", &pulled_url, docker_image])
+                .output()
+                .await;
+        }
+
+        // Verify image exists locally after pull.
         let verify = tokio::process::Command::new("podman")
             .args(["images", "-q", docker_image])
             .output()
@@ -1539,10 +1589,16 @@ server {
             .and_then(|v| v.as_bool())
             .unwrap_or(true);
 
+        // Registries are configured as `host[:port]/namespace` (for
+        // example `23.182.128.160:3000/lfg2025`), but the Docker V2
+        // registry API lives at `/v2/` on the ROOT of the host — NOT
+        // under the namespace. Strip the namespace before appending
+        // `/v2/` so the reachability probe hits the correct URL.
+        let host = url.split('/').next().unwrap_or(url);
         let test_url = if tls_verify {
-            format!("https://{}/v2/", url)
+            format!("https://{}/v2/", host)
         } else {
-            format!("http://{}/v2/", url)
+            format!("http://{}/v2/", host)
         };
 
         let client = reqwest::Client::builder()
@@ -1554,8 +1610,11 @@ server {
         match client.get(&test_url).send().await {
             Ok(resp) => {
                 let status = resp.status().as_u16();
-                // 200 = open registry, 401 = auth required (both mean it exists)
-                let reachable = status == 200 || status == 401;
+                // Accept any "the server responded to HTTP" code as
+                // reachable — 200 (open registry), 401 (auth required),
+                // or 405 (server rejects GET but the endpoint exists)
+                // all mean the registry daemon is alive on the host.
+                let reachable = status == 200 || status == 401 || status == 405;
                 Ok(serde_json::json!({
                     "url": url,
                     "reachable": reachable,

core/archipelago/src/api/rpc/package/progress.rs

@@ -28,6 +28,18 @@ impl RpcHandler {
         self.state_manager.update_data(data).await;
     }
 
+    /// Set the uninstall stage label so the UI can show what's happening
+    /// instead of a generic spinner. Each call broadcasts a state change
+    /// — call sparingly (one per pipeline phase, not per container).
+    pub(super) async fn set_uninstall_stage(&self, package_id: &str, stage: &str) {
+        let (mut data, _rev) = self.state_manager.get_snapshot().await;
+        if let Some(entry) = data.package_data.get_mut(package_id) {
+            entry.uninstall_stage = Some(stage.to_string());
+            entry.state = crate::data_model::PackageState::Removing;
+        }
+        self.state_manager.update_data(data).await;
+    }
+
     /// Update install progress (static method for use in async closures).
     pub(super) async fn update_install_progress(
         state_manager: &crate::state::StateManager,
@@ -81,6 +93,7 @@ fn create_installing_entry(package_id: &str) -> PackageDataEntry {
         },
         installed: None,
         install_progress: None,
+        uninstall_stage: None,
         available_update: None,
     }
 }

core/archipelago/src/api/rpc/package/runtime.rs

@@ -261,12 +261,28 @@ impl RpcHandler {
         if containers_to_remove.is_empty() {
             tracing::warn!("Uninstall {}: no containers found", package_id);
         }
+        let total = containers_to_remove.len();
 
         let mut stopped = 0u32;
         let mut removed = 0u32;
         let mut errors = Vec::new();
 
-        for name in &containers_to_remove {
+        self.set_uninstall_stage(
+            package_id,
+            &if total > 0 {
+                format!("Stopping containers (0/{})", total)
+            } else {
+                "Cleaning up".to_string()
+            },
+        )
+        .await;
+
+        for (i, name) in containers_to_remove.iter().enumerate() {
+            self.set_uninstall_stage(
+                package_id,
+                &format!("Stopping containers ({}/{})", i + 1, total),
+            )
+            .await;
             tracing::info!("Uninstall {}: stopping container {}", package_id, name);
             let stop_out = tokio::process::Command::new("podman")
                 .args(["stop", "-t", stop_timeout_secs(name), name])
@@ -326,6 +342,7 @@ impl RpcHandler {
             }
         }
 
+        self.set_uninstall_stage(package_id, "Cleaning up volumes").await;
         // Clean up dangling volumes associated with removed containers
         let _ = tokio::process::Command::new("podman")
             .args(["volume", "prune", "-f"])
@@ -354,6 +371,7 @@ impl RpcHandler {
 
         // Clean data directories unless preserve_data
         if !preserve_data {
+            self.set_uninstall_stage(package_id, "Removing app data").await;
             let data_dirs = get_data_dirs_for_app(package_id);
             for dir in &data_dirs {
                 tracing::info!("Uninstall {}: removing data {}", package_id, dir);

core/archipelago/src/api/rpc/package/stacks.rs

@@ -937,6 +937,31 @@ impl RpcHandler {
                 .with_context(|| format!("Failed to pull IndeedHub image: {}", img))?;
         }
 
+        // Remove any leftover containers from a previous partial install (or
+        // from the first-boot frontend stub that used to race the installer).
+        // Without this, `podman run --name indeedhub` fails on name conflict
+        // and the whole stack install errors out — leaving a half-broken node.
+        for name in [
+            "indeedhub",
+            "indeedhub-postgres",
+            "indeedhub-redis",
+            "indeedhub-minio",
+            "indeedhub-relay",
+            "indeedhub-api",
+            "indeedhub-ffmpeg",
+            "indeedhub-build_api_1",
+            "indeedhub-build_ffmpeg-worker_1",
+        ] {
+            let _ = tokio::process::Command::new("podman")
+                .args(["rm", "-f", name])
+                .status()
+                .await;
+        }
+        let _ = tokio::process::Command::new("podman")
+            .args(["network", "rm", "-f", "indeedhub-net"])
+            .status()
+            .await;
+
         // Create indeedhub-net
         let _ = tokio::process::Command::new("podman")
             .args(["network", "create", "indeedhub-net"])

core/archipelago/src/container/docker_packages.rs

@@ -247,6 +247,7 @@ impl DockerPackageScanner {
                     status: service_status,
                 }),
                 install_progress: None,
+                uninstall_stage: None,
             };
 
             packages.insert(app_id.clone(), package);

core/archipelago/src/container/registry.rs

@@ -57,6 +57,13 @@ impl Default for RegistryConfig {
                     enabled: true,
                     priority: 10,
                 },
+                Registry {
+                    url: "146.59.87.168:3000/lfg2025".to_string(),
+                    name: "Server 3 (OVH)".to_string(),
+                    tls_verify: false,
+                    enabled: true,
+                    priority: 20,
+                },
             ],
         }
     }
@@ -80,42 +87,6 @@ impl RegistryConfig {
         format!("{}/{}", registry.url, image_name)
     }
 
-    /// Generate fallback image URLs to try (excludes the original since it already failed).
-    pub fn image_candidates(&self, image: &str) -> Vec<(String, bool)> {
-        let mut candidates = Vec::new();
-
-        // Rewrite for each active registry (skip if identical to original)
-        for reg in self.active_registries() {
-            let rewritten = self.rewrite_image(image, reg);
-            if rewritten != image {
-                candidates.push((rewritten, reg.tls_verify));
-            }
-        }
-
-        candidates
-    }
-
-    /// Rewrite an image to use the highest-priority enabled registry, so
-    /// the FIRST pull attempt honors the operator's primary choice instead
-    /// of blindly using whatever registry the image URL was hardcoded to.
-    /// Returns (rewritten_url, tls_verify) — or the original URL + default
-    /// tls_verify=true if there's no primary (no enabled registries).
-    pub fn rewrite_for_primary(&self, image: &str) -> (String, bool) {
-        match self.active_registries().first() {
-            Some(primary) => (self.rewrite_image(image, primary), primary.tls_verify),
-            None => (image.to_string(), true),
-        }
-    }
-}
-
-/// Load the registry config and rewrite an image to use the primary
-/// registry's URL. Convenience wrapper for callers that don't already
-/// have a `RegistryConfig` in hand.
-pub async fn primary_image_url(data_dir: &Path, image: &str) -> (String, bool) {
-    match load_registries(data_dir).await {
-        Ok(config) => config.rewrite_for_primary(image),
-        Err(_) => (image.to_string(), true),
-    }
 }
 
 /// Extract the image name from a full image reference.
@@ -126,7 +97,12 @@ fn extract_image_name(image: &str) -> &str {
     image.rsplit('/').next().unwrap_or(image)
 }
 
-/// Load registry config from disk.
+/// Load registry config from disk, merging in any default registries
+/// that the operator hasn't explicitly removed. This lets us roll out
+/// new default mirrors (e.g. a new Server 3) to existing nodes without
+/// them having to edit their saved config. Explicit removals stick —
+/// if the URL is absent from disk AND absent from current defaults, it
+/// stays gone.
 pub async fn load_registries(data_dir: &Path) -> Result<RegistryConfig> {
     let path = data_dir.join(REGISTRY_FILE);
     if !path.exists() {
@@ -135,8 +111,34 @@ pub async fn load_registries(data_dir: &Path) -> Result<RegistryConfig> {
     let content = fs::read_to_string(&path)
         .await
         .context("Failed to read registry config")?;
-    let config: RegistryConfig =
+    let mut config: RegistryConfig =
         serde_json::from_str(&content).unwrap_or_else(|_| RegistryConfig::default());
+
+    // Migrate: any default registry URL that isn't already in the
+    // saved list gets appended at the end (so existing priority order
+    // is preserved for anything the operator already configured).
+    let defaults = RegistryConfig::default();
+    let known: std::collections::HashSet<String> =
+        config.registries.iter().map(|r| r.url.clone()).collect();
+    let max_priority = config
+        .registries
+        .iter()
+        .map(|r| r.priority)
+        .max()
+        .unwrap_or(0);
+    let mut added = false;
+    for (i, def) in defaults.registries.iter().enumerate() {
+        if !known.contains(&def.url) {
+            let mut cloned = def.clone();
+            cloned.priority = max_priority.saturating_add(10 + i as u32);
+            config.registries.push(cloned);
+            added = true;
+        }
+    }
+    if added {
+        // Persist so the next load doesn't have to re-merge.
+        let _ = save_registries(data_dir, &config).await;
+    }
     Ok(config)
 }
 
@@ -155,80 +157,6 @@ pub async fn save_registries(data_dir: &Path, config: &RegistryConfig) -> Result
     Ok(())
 }
 
-/// Try pulling an image from configured registries in priority order.
-/// If `already_tried` is Some, that URL is skipped (avoids retrying the
-/// primary when the caller already attempted it with progress streaming).
-/// Returns the image reference that succeeded.
-pub async fn pull_from_registries_with_skip(
-    data_dir: &Path,
-    image: &str,
-    tmpdir: &str,
-    already_tried: Option<&str>,
-) -> Result<String> {
-    let config = load_registries(data_dir).await?;
-    let mut candidates = config.image_candidates(image);
-    if let Some(skip) = already_tried {
-        candidates.retain(|(url, _)| url != skip);
-    }
-
-    for (candidate, tls_verify) in &candidates {
-        debug!("Trying registry: {}", candidate);
-
-        let mut args = vec!["pull".to_string(), candidate.clone()];
-        if !tls_verify {
-            args.push("--tls-verify=false".to_string());
-        }
-
-        let mut child = tokio::process::Command::new("podman")
-            .args(&args)
-            .env("TMPDIR", tmpdir)
-            .stdout(std::process::Stdio::null())
-            .stderr(std::process::Stdio::null())
-            .spawn()
-            .ok();
-
-        let status = if let Some(ref mut c) = child {
-            match tokio::time::timeout(std::time::Duration::from_secs(120), c.wait()).await {
-                Ok(Ok(s)) => Some(s.success()),
-                _ => {
-                    let _ = c.kill().await;
-                    let _ = c.wait().await;
-                    debug!("Fallback pull timed out: {}", candidate);
-                    None
-                }
-            }
-        } else {
-            None
-        };
-
-        if status == Some(true) {
-            // If we pulled from a non-original registry, tag it with the original name
-            if candidate != image {
-                let _ = tokio::process::Command::new("podman")
-                    .args(["tag", candidate, image])
-                    .status()
-                    .await;
-                info!(
-                    "Pulled {} from fallback registry, tagged as {}",
-                    candidate, image
-                );
-            } else {
-                info!("Pulled {} from primary registry", image);
-            }
-            return Ok(candidate.clone());
-        }
-
-        debug!("Failed to pull from {}", candidate);
-    }
-
-    Err(anyhow::anyhow!(
-        "Failed to pull {} from all {} configured registries",
-        image,
-        candidates.len()
-    ))
-}
-
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -259,39 +187,20 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_image_candidates() {
-        let config = RegistryConfig::default();
-        let candidates = config.image_candidates("git.tx1138.com/lfg2025/lnd:v0.18.4-beta");
-        // Defaults: VPS (primary) + tx1138. tx1138 is filtered out because
-        // it's identical to the original image URL, leaving one candidate.
-        assert_eq!(candidates.len(), 1);
-        // Primary-first — VPS rewrite leads the candidate list.
-        assert_eq!(candidates[0].0, "23.182.128.160:3000/lfg2025/lnd:v0.18.4-beta");
-    }
-
-    #[test]
-    fn test_rewrite_for_primary_uses_top_priority() {
-        let config = RegistryConfig::default();
-        let (url, tls) =
-            config.rewrite_for_primary("git.tx1138.com/lfg2025/lnd:v0.18.4-beta");
-        assert_eq!(url, "23.182.128.160:3000/lfg2025/lnd:v0.18.4-beta");
-        assert!(!tls, "VPS primary is HTTP — tls_verify should be false");
-    }
-
     #[test]
     fn test_active_registries_sorted() {
         let config = RegistryConfig::default();
         let active = config.active_registries();
-        assert_eq!(active.len(), 2);
+        assert_eq!(active.len(), 3);
         assert!(active[0].priority <= active[1].priority);
+        assert!(active[1].priority <= active[2].priority);
     }
 
     #[tokio::test]
     async fn test_load_default() {
         let tmp = TempDir::new().unwrap();
         let config = load_registries(tmp.path()).await.unwrap();
-        assert_eq!(config.registries.len(), 2);
+        assert_eq!(config.registries.len(), 3);
     }
 
     #[tokio::test]
@@ -307,6 +216,6 @@ mod tests {
         });
         save_registries(tmp.path(), &config).await.unwrap();
         let loaded = load_registries(tmp.path()).await.unwrap();
-        assert_eq!(loaded.registries.len(), 3);
+        assert_eq!(loaded.registries.len(), 4);
     }
 }

core/archipelago/src/data_model.rs

@@ -138,6 +138,13 @@ pub struct PackageDataEntry {
     pub installed: Option<InstalledPackageDataEntry>,
     #[serde(rename = "install-progress")]
     pub install_progress: Option<InstallProgress>,
+    /// Live label describing the current uninstall step ("Stopping
+    /// containers (2/5)", "Removing data", …). Set by the uninstall
+    /// pipeline so the UI can show real progress instead of a generic
+    /// "Uninstalling…" spinner. Cleared after the package entry is
+    /// removed.
+    #[serde(rename = "uninstall-stage", skip_serializing_if = "Option::is_none", default)]
+    pub uninstall_stage: Option<String>,
     /// Pinned image version from image-versions.sh when it differs from running version
     #[serde(rename = "available-update", skip_serializing_if = "Option::is_none")]
     pub available_update: Option<String>,

core/archipelago/src/main.rs

@@ -229,5 +229,14 @@ async fn main() -> Result<()> {
     crash_recovery::remove_pid_marker(&config.data_dir).await;
 
     info!("Archipelago shut down cleanly");
-    Ok(())
+
+    // Hard-exit after logging. All business state is persisted by now
+    // (connections drained, PID marker removed, disk flushes done via
+    // tokio::fs awaits). Letting tokio try to drop the runtime instead
+    // can stall for 15s+ on non-daemon OS threads we don't directly
+    // own (mdns_sd daemon, reqwest resolver pool, etc.) — long enough
+    // for systemd's TimeoutStopSec to SIGKILL us and mark the service
+    // Failed, which makes an otherwise-successful update look like a
+    // crash in `systemctl status`.
+    std::process::exit(0);
 }

core/archipelago/src/transport/lan.rs

@@ -160,6 +160,18 @@ impl LanTransport {
     }
 }
 
+impl Drop for LanTransport {
+    // The mdns_sd daemon runs on its own OS thread and the browse
+    // listener task blocks on a sync channel. Without this call both
+    // keep the process alive past SIGTERM, long enough for systemd to
+    // SIGKILL us — which makes a normal update look like a crash.
+    fn drop(&mut self) {
+        if let Some(daemon) = self.daemon.take() {
+            let _ = daemon.shutdown();
+        }
+    }
+}
+
 impl NodeTransport for LanTransport {
     fn kind(&self) -> TransportKind {
         TransportKind::Lan

core/archipelago/src/update.rs

@@ -68,6 +68,10 @@ const DEFAULT_UPDATE_MANIFEST_URL: &str =
 /// is slow or unreachable.
 const DEFAULT_SECONDARY_MIRROR_URL: &str =
     "http://23.182.128.160:3000/lfg2025/archy/raw/branch/main/releases/manifest.json";
+/// Tertiary mirror on a separate OVH VPS — independent network path so
+/// a single-provider outage doesn't knock out all three mirrors.
+const DEFAULT_TERTIARY_MIRROR_URL: &str =
+    "http://146.59.87.168:3000/lfg2025/archy/raw/branch/main/releases/manifest.json";
 const UPDATE_STATE_FILE: &str = "update_state.json";
 const UPDATE_MIRRORS_FILE: &str = "update-mirrors.json";
 
@@ -97,13 +101,24 @@ fn default_mirrors() -> Vec<UpdateMirror> {
             url: DEFAULT_UPDATE_MANIFEST_URL.to_string(),
             label: "Server 2 (tx1138)".to_string(),
         },
+        UpdateMirror {
+            url: DEFAULT_TERTIARY_MIRROR_URL.to_string(),
+            label: "Server 3 (OVH)".to_string(),
+        },
     ]
 }
 
 /// Load the operator-configured mirror list. Returns defaults if the
 /// file doesn't exist yet, so a node OTA'd from a pre-mirrors release
-/// starts with both Server 1 and Server 2 available without any manual
-/// config.
+/// starts with the current default mirrors available without any
+/// manual config.
+///
+/// Migration: any default mirror URL that isn't already in the saved
+/// list gets appended at the end. This lets us add new default mirrors
+/// (e.g. a new Server 3) and have them appear on existing nodes after
+/// an update, without requiring manual config edits. Explicit removals
+/// stick — once an operator removes a URL it stays gone unless it's
+/// later re-added to defaults.
 pub async fn load_mirrors(data_dir: &Path) -> Result<Vec<UpdateMirror>> {
     let path = mirrors_path(data_dir);
     if !path.exists() {
@@ -112,13 +127,27 @@ pub async fn load_mirrors(data_dir: &Path) -> Result<Vec<UpdateMirror>> {
     let bytes = fs::read(&path)
         .await
         .with_context(|| format!("read {}", path.display()))?;
-    let list: Vec<UpdateMirror> =
+    let mut list: Vec<UpdateMirror> =
         serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display()))?;
     if list.is_empty() {
-        Ok(default_mirrors())
-    } else {
-        Ok(list)
+        return Ok(default_mirrors());
     }
+
+    // Merge in any default URLs the saved config is missing.
+    let known: std::collections::HashSet<String> =
+        list.iter().map(|m| m.url.clone()).collect();
+    let defaults = default_mirrors();
+    let mut added = false;
+    for def in &defaults {
+        if !known.contains(&def.url) {
+            list.push(def.clone());
+            added = true;
+        }
+    }
+    if added {
+        let _ = save_mirrors(data_dir, &list).await;
+    }
+    Ok(list)
 }
 
 pub async fn save_mirrors(data_dir: &Path, mirrors: &[UpdateMirror]) -> Result<()> {
@@ -1187,9 +1216,10 @@ mod tests {
     async fn test_load_mirrors_returns_defaults_when_absent() {
         let dir = tempfile::tempdir().unwrap();
         let list = load_mirrors(dir.path()).await.unwrap();
-        assert_eq!(list.len(), 2);
-        assert!(list[0].url.contains("git.tx1138.com"));
-        assert!(list[1].url.contains("23.182.128.160"));
+        assert_eq!(list.len(), 3);
+        assert!(list[0].url.contains("23.182.128.160"));
+        assert!(list[1].url.contains("git.tx1138.com"));
+        assert!(list[2].url.contains("146.59.87.168"));
     }
 
     #[tokio::test]

neode-ui/src/types/api.ts

@@ -91,6 +91,8 @@ export interface PackageDataEntry {
   manifest: Manifest
   installed?: InstalledPackageDataEntry
   'install-progress'?: InstallProgress
+  /** Live label for the current uninstall step ("Stopping containers (2/5)", …). */
+  'uninstall-stage'?: string | null
   'available-update'?: string | null
 }
 

neode-ui/src/views/Apps.vue

@@ -282,6 +282,9 @@ function closeUninstallModal() {
 
 async function onConfirmUninstall() {
   const { appId } = uninstallModal.value
+  // Close the modal immediately so the user can fire off concurrent
+  // uninstalls. Each AppCard surfaces its own live stage label while
+  // its uninstall is in flight.
   uninstallModal.value.show = false
   await actions.confirmUninstall(appId)
 }

neode-ui/src/views/apps/AppCard.vue

@@ -99,14 +99,14 @@
       </div>
     </div>
 
-    <!-- Uninstalling progress — replaces action buttons -->
+    <!-- Uninstalling progress — live stage label from backend -->
     <div v-else-if="isUninstalling" class="mt-4">
       <div class="flex items-center gap-1.5">
         <svg class="animate-spin h-3 w-3 text-red-400" fill="none" viewBox="0 0 24 24">
           <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
           <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
         </svg>
-        <span class="text-xs text-red-300">{{ t('common.uninstalling') }}...</span>
+        <span class="text-xs text-red-300 truncate">{{ uninstallStageLabel }}</span>
       </div>
       <div class="mt-1.5 w-full h-1.5 bg-white/10 rounded-full overflow-hidden">
         <div class="h-full bg-red-400/60 rounded-full animate-pulse w-full"></div>
@@ -251,6 +251,13 @@ const tier = computed(() => {
   return 'optional'
 })
 
+// Live uninstall stage from backend, with a sensible fallback so the
+// label is never blank between WS pushes.
+const uninstallStageLabel = computed(() => {
+  const raw = props.pkg['uninstall-stage']
+  return raw ? raw : `${t('common.uninstalling')}…`
+})
+
 const isTransitioning = computed(() => {
   const s = props.pkg.state
   const h = props.pkg.health

neode-ui/src/views/marketplace/MarketplaceAppCard.vue

@@ -96,20 +96,32 @@
           Checking...
         </span>
       </span>
-      <!-- Installing — simple button-style indicator -->
-      <button
+      <!-- Installing — live progress with bar + message matching the
+           update download bar's accuracy. Falls back to a simple
+           spinner if no install_progress data is available yet. -->
+      <div
         v-else-if="!installed && installing"
-        disabled
-        class="flex-1 px-4 py-2 glass-button glass-button-sm rounded-lg text-sm font-medium opacity-80 cursor-wait"
+        class="flex-1 flex flex-col gap-1.5"
       >
-        <span class="flex items-center justify-center gap-2">
-          <svg class="animate-spin h-3.5 w-3.5" fill="none" viewBox="0 0 24 24">
+        <div
+          class="px-4 py-2 glass-button glass-button-sm rounded-lg text-sm font-medium opacity-90 cursor-wait flex items-center justify-center gap-2 text-center"
+        >
+          <svg class="animate-spin h-3.5 w-3.5 shrink-0" fill="none" viewBox="0 0 24 24">
             <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
             <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
           </svg>
-          Installing
-        </span>
-      </button>
+          <span class="truncate">{{ installProgressMessage }}</span>
+        </div>
+        <div
+          v-if="(installProgress?.progress ?? 0) > 0"
+          class="w-full h-1 bg-white/10 rounded-full overflow-hidden"
+        >
+          <div
+            class="h-full bg-orange-400 transition-all duration-300"
+            :style="{ width: (installProgress?.progress ?? 0) + '%' }"
+          ></div>
+        </div>
+      </div>
       <button
         v-else-if="!installed && (app.source === 'local' || app.dockerImage)"
         data-controller-install-btn
@@ -130,6 +142,7 @@
 </template>
 
 <script setup lang="ts">
+import { computed } from 'vue'
 import { useI18n } from 'vue-i18n'
 import type { MarketplaceApp, InstallProgress } from './marketplaceData'
 
@@ -154,6 +167,14 @@ defineEmits<{
   launch: [app: MarketplaceApp]
 }>()
 
+const installProgressMessage = computed(() => {
+  const p = props.installProgress
+  if (!p) return 'Installing'
+  // The store already formats messages like "Downloading: 50.5 / 200.0 MB (25%)"
+  // so we just surface them directly.
+  return p.message || 'Installing'
+})
+
 function handleImageError(event: Event) {
   const img = event.target as HTMLImageElement
   img.src = '/assets/img/logo-archipelago.svg'

neode-ui/src/views/settings/AccountInfoSection.vue

@@ -180,6 +180,18 @@ init()
               </button>
             </div>
             <div class="overflow-y-auto flex-1 min-h-0 space-y-6 pr-1">
+              <!-- v1.7.30-alpha -->
+              <div>
+                <div class="flex items-center gap-2 mb-3">
+                  <span class="text-xs font-mono px-2 py-0.5 rounded bg-orange-500/20 text-orange-300">v1.7.30-alpha</span>
+                  <span class="text-xs text-white/40">Apr 21, 2026</span>
+                </div>
+                <div class="space-y-3 text-sm text-white/80 pl-3 border-l border-white/10">
+                  <p>App installs now show a real download progress bar — same accuracy as the system update bar. You'll see "Downloading: 50.5 / 200.0 MB (25%)" with a live percentage instead of a generic spinner. The bar keeps streaming even when the install falls back from one registry to another, so you'll never see a "stuck at 0%" again.</p>
+                  <p>Uninstalls now show what's actually happening: "Stopping containers (2/5)", "Cleaning up volumes", "Removing app data" — labelled per app so you can fire off multiple uninstalls in parallel and watch each one's stage on its own card.</p>
+                  <p>OVH (146.59.87.168) is now baked in as Server 3 by default for both updates and the app registry — extra mirror, completely independent network path so a single-provider outage can't take everything down.</p>
+                </div>
+              </div>
               <!-- v1.7.29-alpha -->
               <div>
                 <div class="flex items-center gap-2 mb-3">

scripts/first-boot-containers.sh

@@ -1216,44 +1216,6 @@ if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'str
   fi
 fi
 
-# 8b. Indeehub (pull from registry, or use local build)
-if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q indeedhub; then
-  # Use image-versions.sh variable if sourced, otherwise detect
-  if [ -z "${INDEEDHUB_IMAGE:-}" ]; then
-    INDEEDHUB_IMAGE=""
-    # Try local image first (pre-built or loaded from ISO)
-    if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'localhost/indeedhub'; then
-      INDEEDHUB_IMAGE="localhost/indeedhub:local"
-    # Try pinned registry image
-    elif $DOCKER pull "$ARCHY_REGISTRY/indeedhub:1.0.0" --tls-verify=false 2>>"$LOG"; then
-      INDEEDHUB_IMAGE="$ARCHY_REGISTRY/indeedhub:1.0.0"
-    fi
-  fi
-  if [ -n "$INDEEDHUB_IMAGE" ]; then
-    log "Creating Indeehub from $INDEEDHUB_IMAGE..."
-    $DOCKER run -d --name indeedhub --restart unless-stopped \
-      --network archy-net --network-alias indeedhub \
-      --health-cmd="curl -sf http://localhost:7777/health || exit 1" --health-interval=120s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit indeedhub) \
-      --cap-drop ALL --security-opt no-new-privileges:true \
-      --tmpfs /tmp:rw,noexec,nosuid,size=64m \
-      -p 7778:7777 \
-      "$INDEEDHUB_IMAGE" 2>>"$LOG" || true
-    # Fix IndeedHub for iframe: remove X-Frame-Options so it loads in Archipelago panel
-    sleep 2
-    if $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q "^indeedhub$"; then
-      $DOCKER exec indeedhub sed -i "/X-Frame-Options/d" /etc/nginx/conf.d/default.conf 2>/dev/null || true
-      # Fix Host header for NIP-98 auth — $host strips port, $http_host preserves it
-      $DOCKER exec indeedhub sed -i 's|proxy_set_header Host $host;|proxy_set_header Host $http_host;|g' /etc/nginx/conf.d/default.conf 2>/dev/null || true
-      if [ -f /opt/archipelago/web-ui/nostr-provider.js ]; then
-        $DOCKER cp /opt/archipelago/web-ui/nostr-provider.js indeedhub:/usr/share/nginx/html/nostr-provider.js 2>/dev/null || true
-      fi
-      $DOCKER exec indeedhub nginx -s reload 2>/dev/null || true
-      log "Applied IndeedHub iframe fix (X-Frame-Options, Host header, nostr-provider)"
-    fi
-  fi
-fi
-
 # 9. Custom UI containers (bitcoin-ui, lnd-ui)
 # These are built from Dockerfiles in /opt/archipelago/docker/ or loaded from pre-built images.