release(v1.7.15-alpha): bulletproof downloads — resume, retry, real progress

download_update
  Each component download is now resumable via HTTP Range requests
  (Range: bytes=N-) and retried up to 6 times with exponential
  backoff (5/15/30/60/120/180s). On a dropped connection the next
  attempt picks up at the last written byte offset instead of
  restarting at zero. Streams via reqwest::Response::chunk() to the
  staging file so a 160 MB frontend tarball doesn't sit in RAM. SHA
  is verified over the complete file at the end of each component;
  mismatch nukes the staged file and restarts from scratch.

Real download progress counters
  New AtomicU64 globals DOWNLOAD_BYTES/DOWNLOAD_TOTAL are updated
  from the chunk loop. update.status exposes them as
  download_progress.{bytes_downloaded, total_bytes, active}. The
  SystemUpdate.vue progress bar now polls update.status every
  second instead of incrementing a fake random counter — and
  crucially, if the user navigates away and back, the component
  picks up the in-progress download from the backend atomics
  immediately.

Update-check retries
  handle_update_check now retries the manifest fetch up to 3 times
  with a 5s gap if the first try hits a transport error, so a
  momentary gitea hiccup doesn't make a node report "up to date"
  when there actually is a new release. Tight 10s connect timeout
  per attempt keeps the total bounded.

Artefacts:
  archipelago                                      1070c87f…c081c162b  40584792
  archipelago-frontend-1.7.15-alpha.tar.gz         8e630eba…63fd43f   162078068

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

core/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
 
 [[package]]
 name = "archipelago"
-version = "1.7.13-alpha"
+version = "1.7.15-alpha"
 dependencies = [
  "anyhow",
  "archipelago-container",

core/archipelago/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "archipelago"
-version = "1.7.13-alpha"
+version = "1.7.15-alpha"
 edition = "2021"
 description = "Archipelago Bitcoin Node OS - Native backend"
 authors = ["Archipelago Team"]

core/archipelago/src/api/rpc/dispatcher.rs

@@ -413,6 +413,7 @@ impl RpcHandler {
             "fips.apply-update" => self.handle_fips_apply_update().await,
             "fips.install" => self.handle_fips_install().await,
             "fips.restart" => self.handle_fips_restart().await,
+            "fips.reconnect" => self.handle_fips_reconnect().await,
 
             // System updates
             "update.check" => self.handle_update_check().await,

core/archipelago/src/api/rpc/fips.rs

@@ -44,4 +44,89 @@ impl RpcHandler {
         fips::service::restart(fips::SERVICE_UNIT).await?;
         Ok(serde_json::json!({ "restarted": true }))
     }
+
+    /// Full reconnect: stop the daemon, bring it back, wait for the DHT
+    /// bootstrap window, poll the identity-cache + peer list, and
+    /// classify what recovered (or didn't) so the UI can explain it to
+    /// the user instead of showing a generic failure.
+    ///
+    /// Runtime: ~20s. Needs an RPC timeout ≥ 45s on the client.
+    pub(super) async fn handle_fips_reconnect(&self) -> Result<serde_json::Value> {
+        let identity_dir = fips::identity_dir_from(&self.config.data_dir);
+        let before = fips::FipsStatus::query(&identity_dir).await;
+
+        // Heal the pre-fix bech32-text fips_key.pub → 32-raw-bytes
+        // mismatch. The daemon silently authenticates with a garbage
+        // pubkey when the .pub file is 63-char text, which looks like
+        // "anchor unreachable" to the user even though the real fault
+        // was an identity malformed on the node itself. Re-install the
+        // config + keys so /etc/fips gets the healed .pub.
+        let key_src = identity_dir.join("fips_key");
+        let pub_src = identity_dir.join("fips_key.pub");
+        if key_src.exists() {
+            let _ = fips::config::normalize_pub_file(&key_src, &pub_src).await;
+            // Re-install refreshes /etc/fips/fips.pub from the healed
+            // source. No-op if nothing changed.
+            let _ = fips::config::install(&identity_dir).await;
+        }
+
+        // Clean stop+start rather than `restart`, so a daemon that
+        // fails to come back up surfaces as service_active=false
+        // instead of quietly sticking with the old process.
+        let _ = fips::service::stop(fips::SERVICE_UNIT).await;
+        tokio::time::sleep(std::time::Duration::from_millis(800)).await;
+        fips::service::activate(fips::SERVICE_UNIT).await?;
+
+        // Anchor bootstrap window: poll the status every ~3s for up to
+        // 20s. Bail as soon as the anchor is connected.
+        let mut last_status: Option<fips::FipsStatus> = None;
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
+        loop {
+            tokio::time::sleep(std::time::Duration::from_secs(3)).await;
+            let s = fips::FipsStatus::query(&identity_dir).await;
+            if s.anchor_connected {
+                last_status = Some(s);
+                break;
+            }
+            last_status = Some(s);
+            if std::time::Instant::now() >= deadline {
+                break;
+            }
+        }
+        let after = last_status.unwrap_or_else(|| before.clone());
+
+        let recovered = after.anchor_connected && !before.anchor_connected;
+        let likely_cause = if after.anchor_connected {
+            "connected"
+        } else if !after.service_active {
+            "daemon_down"
+        } else if !after.key_present {
+            "no_seed_key"
+        } else if after.authenticated_peer_count == 0 {
+            // Daemon is up with a key but hasn't authenticated any
+            // peers — almost always outbound UDP/8668 dropped by the
+            // local firewall/router, or the anchor itself being down.
+            "no_outbound_udp_or_anchor_down"
+        } else {
+            "peers_but_no_anchor"
+        };
+        let hint = match likely_cause {
+            "connected" => "Anchor is reachable.",
+            "daemon_down" => "The FIPS daemon didn't come back up — check archipelago-fips.service.",
+            "no_seed_key" => "No seed-derived FIPS key on disk. Re-run the onboarding unlock step.",
+            "no_outbound_udp_or_anchor_down" =>
+                "Daemon is running but no peers handshook. Your router / ISP might be blocking outbound UDP 8668, or the anchor (fips.v0l.io) could be down.",
+            "peers_but_no_anchor" =>
+                "Mesh has peers but the anchor hasn't been seen yet. Give it a minute and re-check.",
+            _ => "",
+        };
+
+        Ok(serde_json::json!({
+            "recovered": recovered,
+            "likely_cause": likely_cause,
+            "hint": hint,
+            "before": before,
+            "after": after,
+        }))
+    }
 }

core/archipelago/src/api/rpc/update.rs

@@ -157,6 +157,16 @@ impl RpcHandler {
     /// Get update status without checking remote.
     pub(super) async fn handle_update_status(&self) -> Result<serde_json::Value> {
         let state = update::get_status(&self.config.data_dir).await?;
+        // Expose live download progress so the UI can resume the
+        // progress bar after navigation instead of showing the fake
+        // creep again. An RPC poll every ~1s during download drives a
+        // real progress indicator that survives route changes.
+        let downloaded = update::DOWNLOAD_BYTES
+            .load(std::sync::atomic::Ordering::Relaxed);
+        let total = update::DOWNLOAD_TOTAL
+            .load(std::sync::atomic::Ordering::Relaxed);
+        let active = total > 0 && downloaded < total;
+        let completed = total > 0 && downloaded >= total;
 
         Ok(serde_json::json!({
             "current_version": state.current_version,
@@ -164,6 +174,13 @@ impl RpcHandler {
             "update_available": state.available_update.is_some(),
             "update_in_progress": state.update_in_progress,
             "rollback_available": state.rollback_available,
+            "download_progress": if active || completed {
+                Some(serde_json::json!({
+                    "bytes_downloaded": downloaded,
+                    "total_bytes": total,
+                    "active": active,
+                }))
+            } else { None },
         }))
     }
 

core/archipelago/src/fips/config.rs

@@ -78,11 +78,65 @@ pub async fn install(identity_dir: &Path) -> Result<()> {
     install_result?;
 
     sudo_install_file(&src_key, DAEMON_KEY_PATH, "0600").await?;
+    // Heal a legacy fips_key.pub that was written as bech32 npub text
+    // (pre-fix identity::write_fips_key_from_seed did this). Upstream
+    // fips expects 32 raw bytes; a text file silently passes through
+    // and then the daemon can't identify itself to peers. This
+    // rewrites the source file in place with the correct binary form
+    // derived from fips_key before staging it to /etc/fips/fips.pub.
+    normalize_pub_file(&src_key, &src_pub).await?;
     sudo_install_file(&src_pub, DAEMON_PUB_PATH, "0644").await?;
 
     Ok(())
 }
 
+/// Ensure `fips_key.pub` is 32 raw bytes. If it's a bech32 npub text
+/// file (from the pre-fix writer), decode it and rewrite in place. If
+/// the file is missing or its content doesn't match either format,
+/// re-derive the public key from `fips_key` and write that.
+pub async fn normalize_pub_file(key_path: &Path, pub_path: &Path) -> Result<()> {
+    // Happy path: already 32 raw bytes.
+    if let Ok(bytes) = tokio::fs::read(pub_path).await {
+        if bytes.len() == 32 {
+            return Ok(());
+        }
+        // bech32 npub text from the pre-fix writer: decode in place.
+        if let Ok(s) = std::str::from_utf8(&bytes) {
+            let trimmed = s.trim();
+            if trimmed.starts_with("npub1") {
+                if let Ok(pk) = nostr_sdk::PublicKey::parse(trimmed) {
+                    let raw: [u8; 32] = pk.to_bytes();
+                    tokio::fs::write(pub_path, raw)
+                        .await
+                        .context("rewriting fips_key.pub as 32 raw bytes")?;
+                    tracing::info!(
+                        "Migrated legacy bech32 fips_key.pub to raw-byte form at {}",
+                        pub_path.display()
+                    );
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    // Fallback: no pub file, or unreadable format. Re-derive from the
+    // private key file (already validated by load_fips_keys).
+    let secret_bytes = tokio::fs::read(key_path)
+        .await
+        .with_context(|| format!("read {} to derive public", key_path.display()))?;
+    let text = std::str::from_utf8(&secret_bytes)
+        .context("fips_key is not UTF-8 — can't derive public")?;
+    let secret = nostr_sdk::SecretKey::parse(text.trim())
+        .context("fips_key not parseable as bech32 nsec")?;
+    let keys = nostr_sdk::Keys::new(secret);
+    let raw: [u8; 32] = keys.public_key().to_bytes();
+    tokio::fs::write(pub_path, raw)
+        .await
+        .context("writing re-derived fips_key.pub")?;
+    tracing::info!("Re-derived fips_key.pub from fips_key");
+    Ok(())
+}
+
 async fn sudo_install_dir(path: &str) -> Result<()> {
     let out = Command::new("sudo")
         .args(["install", "-d", "-m", "0755", path])

core/archipelago/src/identity.rs

@@ -219,14 +219,22 @@ async fn write_fips_key_from_seed(
             .await
             .context("Failed to set FIPS key permissions")?;
     }
-    let npub = keys.public_key().to_bech32().unwrap_or_default();
-    fs::write(&pub_path, format!("{npub}\n"))
+    // Upstream fips daemon expects 32 raw bytes in /etc/fips/fips.pub —
+    // not a bech32 npub string. Writing the bech32 form here meant the
+    // installed .pub file was a 63-char text file the daemon parsed as
+    // 63 raw bytes of garbage, so it couldn't identify itself to peers
+    // and the anchor never handshook. Write the raw public-key bytes
+    // (PublicKey::to_bytes returns a [u8; 32]) so the daemon reads
+    // them directly.
+    let raw_pub: [u8; 32] = keys.public_key().to_bytes();
+    fs::write(&pub_path, raw_pub)
         .await
         .context("Failed to write FIPS public key")?;
 
+    let npub_for_log = keys.public_key().to_bech32().unwrap_or_default();
     tracing::info!(
         "Derived FIPS mesh key from seed (npub: {}...)",
-        npub.chars().take(20).collect::<String>()
+        npub_for_log.chars().take(20).collect::<String>()
     );
     Ok(())
 }

core/archipelago/src/update.rs

@@ -4,9 +4,17 @@ use anyhow::{Context, Result};
 use chrono::Timelike;
 use serde::{Deserialize, Serialize};
 use std::path::Path;
+use std::sync::atomic::{AtomicU64, Ordering};
 use tokio::fs;
 use tracing::{debug, info};
 
+/// Live download progress counters. Updated by download_component_resumable
+/// as bytes arrive and read by the update.status RPC so the UI can show
+/// a real progress bar instead of a fake creep. Global because the
+/// download runs in one place at a time; no need for per-handler state.
+pub static DOWNLOAD_BYTES: AtomicU64 = AtomicU64::new(0);
+pub static DOWNLOAD_TOTAL: AtomicU64 = AtomicU64::new(0);
+
 const DEFAULT_UPDATE_MANIFEST_URL: &str =
     "https://git.tx1138.com/lfg2025/archy/raw/branch/main/releases/manifest.json";
 const UPDATE_STATE_FILE: &str = "update_state.json";
@@ -111,36 +119,57 @@ pub async fn check_for_updates(data_dir: &Path) -> Result<UpdateState> {
     let mut state = load_state(data_dir).await?;
 
     info!("Checking for updates...");
+    // 45s total budget, and we retry up to 3 times so a momentary
+    // gitea hiccup doesn't make the node report "up to date" when an
+    // update actually exists. Short per-attempt timeout keeps the RPC
+    // responsive in the common case.
     let client = reqwest::Client::builder()
         .timeout(std::time::Duration::from_secs(15))
+        .connect_timeout(std::time::Duration::from_secs(10))
         .build()
         .context("Failed to create HTTP client")?;
 
     let manifest_url = update_manifest_url();
-    match client.get(&manifest_url).send().await {
-        Ok(resp) if resp.status().is_success() => {
-            let manifest: UpdateManifest = resp
-                .json()
-                .await
-                .context("Failed to parse update manifest")?;
-
-            if manifest.version != state.current_version {
-                info!(
-                    current = %state.current_version,
-                    available = %manifest.version,
-                    "Update available"
-                );
-                state.available_update = Some(manifest);
-            } else {
-                debug!("Already on latest version: {}", state.current_version);
-                state.available_update = None;
+    let mut last_err: Option<String> = None;
+    let mut handled = false;
+    for attempt in 1..=3u8 {
+        if attempt > 1 {
+            tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+        }
+        match client.get(&manifest_url).send().await {
+            Ok(resp) if resp.status().is_success() => {
+                match resp.json::<UpdateManifest>().await {
+                    Ok(manifest) => {
+                        if manifest.version != state.current_version {
+                            info!(
+                                current = %state.current_version,
+                                available = %manifest.version,
+                                "Update available"
+                            );
+                            state.available_update = Some(manifest);
+                        } else {
+                            debug!("Already on latest version: {}", state.current_version);
+                            state.available_update = None;
+                        }
+                        handled = true;
+                        break;
+                    }
+                    Err(e) => {
+                        last_err = Some(format!("parse: {}", e));
+                    }
+                }
+            }
+            Ok(resp) => {
+                last_err = Some(format!("HTTP {}", resp.status()));
+            }
+            Err(e) => {
+                last_err = Some(e.to_string());
             }
         }
-        Ok(resp) => {
-            debug!("Update check returned status: {}", resp.status());
-        }
-        Err(e) => {
-            debug!("Update check failed (offline?): {}", e);
+    }
+    if !handled {
+        if let Some(e) = last_err {
+            debug!("Update check failed after retries: {}", e);
         }
     }
 
@@ -163,6 +192,14 @@ pub async fn dismiss_update(data_dir: &Path) -> Result<()> {
 
 /// Download update components to a staging directory.
 /// Verifies SHA256 hash for each component.
+///
+/// Robustness: each component download is **resumable** via HTTP Range
+/// requests and retried up to 6 times with exponential backoff. When
+/// gitea drops the connection mid-stream (happens regularly at slow
+/// raw-file throughput), the next attempt picks up where the previous
+/// one left off instead of restarting from byte zero. SHA256 is
+/// verified over the complete file at the end of each component, so a
+/// partially-corrupt resume still fails cleanly.
 pub async fn download_update(data_dir: &Path) -> Result<DownloadProgress> {
     let state = load_state(data_dir).await?;
     let manifest = state
@@ -176,7 +213,9 @@ pub async fn download_update(data_dir: &Path) -> Result<DownloadProgress> {
         .context("Failed to create staging dir")?;
 
     let client = reqwest::Client::builder()
-        .timeout(std::time::Duration::from_secs(1800))
+        // Per-request budget; each attempt gets the full hour. A retry
+        // restarts the budget cleanly.
+        .timeout(std::time::Duration::from_secs(3600))
         .connect_timeout(std::time::Duration::from_secs(30))
         .build()
         .context("Failed to create HTTP client")?;
@@ -184,49 +223,20 @@ pub async fn download_update(data_dir: &Path) -> Result<DownloadProgress> {
     let mut downloaded = 0u64;
     let total_bytes: u64 = manifest.components.iter().map(|c| c.size_bytes).sum();
 
+    // Seed the live counters so polls during the handshake show the
+    // right denominator immediately instead of 0/0 → NaN%.
+    DOWNLOAD_TOTAL.store(total_bytes, Ordering::Relaxed);
+    DOWNLOAD_BYTES.store(0, Ordering::Relaxed);
+
     for component in &manifest.components {
         info!(name = %component.name, url = %component.download_url, "Downloading component");
-
-        let resp = client
-            .get(&component.download_url)
-            .send()
-            .await
-            .with_context(|| format!("Failed to download {}", component.name))?;
-
-        if !resp.status().is_success() {
-            anyhow::bail!(
-                "Download failed for {}: HTTP {}",
-                component.name,
-                resp.status()
-            );
-        }
-
-        let bytes = resp
-            .bytes()
-            .await
-            .with_context(|| format!("Failed to read {}", component.name))?;
-
-        // Verify SHA256
-        use sha2::{Digest, Sha256};
-        let hash = hex::encode(Sha256::digest(&bytes));
-        if hash != component.sha256 {
-            anyhow::bail!(
-                "SHA256 mismatch for {}: expected {}, got {}",
-                component.name,
-                component.sha256,
-                hash
-            );
-        }
-
         let dest = staging_dir.join(&component.name);
-        fs::write(&dest, &bytes)
-            .await
-            .with_context(|| format!("Failed to write {}", component.name))?;
-
+        download_component_resumable(&client, component, &dest, downloaded).await?;
         downloaded += component.size_bytes;
+        DOWNLOAD_BYTES.store(downloaded, Ordering::Relaxed);
         info!(
             name = %component.name,
-            bytes = bytes.len(),
+            bytes = component.size_bytes,
             "Component downloaded and verified"
         );
     }
@@ -244,6 +254,166 @@ pub async fn download_update(data_dir: &Path) -> Result<DownloadProgress> {
     })
 }
 
+/// Download a single component to `dest`, resuming from the end of
+/// any existing partial file via a Range request. Retries up to 6
+/// times with exponential backoff (5s, 15s, 30s, 60s, 120s, 180s).
+/// Verifies the SHA256 over the full file at the end.
+async fn download_component_resumable(
+    client: &reqwest::Client,
+    component: &ComponentUpdate,
+    dest: &Path,
+    prior_total: u64,
+) -> Result<()> {
+    use sha2::{Digest, Sha256};
+    use tokio::io::AsyncWriteExt;
+    const MAX_ATTEMPTS: u32 = 6;
+    const BACKOFFS: [u64; 5] = [5, 15, 30, 60, 120];
+
+    let mut last_err: Option<anyhow::Error> = None;
+    for attempt in 1..=MAX_ATTEMPTS {
+        let existing_len = match tokio::fs::metadata(dest).await {
+            Ok(m) => m.len(),
+            Err(_) => 0,
+        };
+        if existing_len >= component.size_bytes {
+            // File is already complete — break out and go verify.
+            break;
+        }
+        if attempt > 1 {
+            let delay = BACKOFFS[(attempt as usize - 2).min(BACKOFFS.len() - 1)];
+            tracing::warn!(
+                name = %component.name,
+                attempt,
+                resume_at = existing_len,
+                "Retrying download in {}s (previous error: {})",
+                delay,
+                last_err.as_ref().map(|e| e.to_string()).unwrap_or_default()
+            );
+            tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
+        }
+
+        let mut req = client.get(&component.download_url);
+        if existing_len > 0 {
+            req = req.header("Range", format!("bytes={}-", existing_len));
+        }
+        let resp = match req.send().await {
+            Ok(r) => r,
+            Err(e) => {
+                last_err = Some(anyhow::anyhow!(e));
+                continue;
+            }
+        };
+        let status = resp.status();
+        // 200 OK on a fresh start, 206 Partial Content on a resume
+        // that the server honoured. Anything else is a problem.
+        let is_resume = existing_len > 0 && status == reqwest::StatusCode::PARTIAL_CONTENT;
+        let is_fresh = existing_len == 0 && status.is_success();
+        let server_ignored_range = existing_len > 0 && status == reqwest::StatusCode::OK;
+        if !is_resume && !is_fresh && !server_ignored_range {
+            last_err = Some(anyhow::anyhow!(
+                "HTTP {} for {} (resume offset {})",
+                status,
+                component.name,
+                existing_len
+            ));
+            continue;
+        }
+        // If the server ignored Range (returned 200 with the full
+        // body), wipe the partial file and start over.
+        let mut file = if server_ignored_range {
+            let _ = tokio::fs::remove_file(dest).await;
+            tokio::fs::OpenOptions::new()
+                .create(true)
+                .write(true)
+                .truncate(true)
+                .open(dest)
+                .await
+                .context("open staging file")?
+        } else if is_resume {
+            tokio::fs::OpenOptions::new()
+                .append(true)
+                .open(dest)
+                .await
+                .context("open staging file for append")?
+        } else {
+            tokio::fs::OpenOptions::new()
+                .create(true)
+                .write(true)
+                .truncate(true)
+                .open(dest)
+                .await
+                .context("open staging file")?
+        };
+
+        let mut resp = resp;
+        let mut stream_err = false;
+        let mut on_disk = existing_len;
+        loop {
+            match resp.chunk().await {
+                Ok(Some(bytes)) => {
+                    if let Err(e) = file.write_all(&bytes).await {
+                        last_err = Some(anyhow::anyhow!(e).context("writing chunk"));
+                        stream_err = true;
+                        break;
+                    }
+                    on_disk += bytes.len() as u64;
+                    DOWNLOAD_BYTES.store(
+                        prior_total + on_disk.min(component.size_bytes),
+                        Ordering::Relaxed,
+                    );
+                }
+                Ok(None) => break, // stream ended cleanly
+                Err(e) => {
+                    last_err = Some(anyhow::anyhow!(e).context("reading chunk"));
+                    stream_err = true;
+                    break;
+                }
+            }
+        }
+        let _ = file.flush().await;
+        let _ = file.sync_all().await;
+        drop(file);
+        if stream_err {
+            continue;
+        }
+
+        // Stream ended cleanly. If we've got the expected size, verify
+        // the SHA and succeed. Otherwise loop to resume from the new
+        // offset on the next attempt.
+        let final_len = tokio::fs::metadata(dest)
+            .await
+            .map(|m| m.len())
+            .unwrap_or(0);
+        if final_len < component.size_bytes {
+            last_err = Some(anyhow::anyhow!(
+                "download truncated: got {} of {} bytes",
+                final_len,
+                component.size_bytes
+            ));
+            continue;
+        }
+
+        // Full file — verify hash.
+        let bytes = tokio::fs::read(dest)
+            .await
+            .context("read staging file for hash check")?;
+        let hash = hex::encode(Sha256::digest(&bytes));
+        if hash == component.sha256 {
+            return Ok(());
+        }
+        // SHA mismatch — the file on disk is garbage. Nuke it and
+        // start over from scratch on the next attempt.
+        let _ = tokio::fs::remove_file(dest).await;
+        last_err = Some(anyhow::anyhow!(
+            "SHA256 mismatch for {}: expected {}, got {}",
+            component.name,
+            component.sha256,
+            hash
+        ));
+    }
+    Err(last_err.unwrap_or_else(|| anyhow::anyhow!("download failed without a captured error")))
+}
+
 /// Run a command as root, but *outside* the archipelago service's
 /// restricted mount namespace.
 ///
@@ -369,6 +539,21 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> {
                 ])
                 .await;
 
+                // Preserve paths that are installed outside the Vue build
+                // (baked in by the ISO or sibling installers) and so
+                // aren't in the new tarball. Without this copy, every OTA
+                // wipes them — notably aiui/ (Claude Code sidebar) and
+                // the companion APK. `cp -a` preserves mode/ownership.
+                for preserved in ["aiui", "archipelago-companion.apk"] {
+                    let src = format!("{}/{}", web_ui, preserved);
+                    let dst = format!("{}/{}", staging_new, preserved);
+                    // Only preserve the old copy if the new tarball
+                    // doesn't already ship a fresher one.
+                    if Path::new(&src).exists() && !Path::new(&dst).exists() {
+                        let _ = host_sudo(&["cp", "-a", &src, &dst]).await;
+                    }
+                }
+
                 // Swap: mv current web-ui aside, then mv new into place.
                 if Path::new(web_ui).exists() {
                     let mv_old = host_sudo(&["mv", web_ui, &staging_old])

neode-ui/src/api/rpc-client.ts

@@ -525,7 +525,11 @@ class RPCClient {
     return this.call({
       method: 'package.install',
       params: { id, 'marketplace-url': marketplaceUrl, version },
-      timeout: 900000, // 15 min — multi-GB stacks (IndeedHub, Bitcoin, Penpot) take time
+      // 45 min — IndeedHub is 6 images and gitea raw-file throughput is
+      // ~70 KB/s per image; 15 min was short enough to kill the install
+      // mid-pull and land the user on a "didn't work" screen while the
+      // backend kept working in the background.
+      timeout: 2700000,
     })
   }
 

neode-ui/src/locales/en.json

@@ -666,7 +666,7 @@
     "applyUpdate": "Install Update",
     "checkForUpdates": "Check for Updates",
     "checking": "Checking...",
-    "rollback": "Rollback to Previous",
+    "rollback": "Rollback Available",
     "backToSettings": "Back to Settings",
     "percentComplete": "{percent}% complete",
     "applyWarning": "Installing components and restarting services. Do not power off.",
@@ -685,6 +685,14 @@
     "rollbackSuccess": "Rolled back to previous version. Service will restart.",
     "rollbackFailed": "Rollback failed.",
     "pullAndRebuild": "Pull & Rebuild",
+    "finishingDownload": "Finishing download — verifying checksum…",
+    "overlayApplying": "Installing update…",
+    "overlayRestarting": "Restarting server…",
+    "overlayReconnecting": "Reconnecting to the new version…",
+    "overlayReady": "Update installed — reloading…",
+    "overlayStalled": "Taking longer than expected",
+    "overlayTarget": "Installing v{version}",
+    "overlayReloadNow": "Reload now",
     "gitMethodHint": "This node builds from source. Update will git-pull, rebuild the backend and UI, then restart — takes a few minutes.",
     "gitApplyTitle": "Pull & Rebuild?",
     "gitApplyMessage": "Archipelago will pull the latest code, rebuild, and restart. This can take several minutes and the UI will be briefly unavailable.",

neode-ui/src/locales/es.json

@@ -665,7 +665,7 @@
     "applyUpdate": "Instalar actualizaci\u00f3n",
     "checkForUpdates": "Buscar actualizaciones",
     "checking": "Verificando...",
-    "rollback": "Revertir a la versi\u00f3n anterior",
+    "rollback": "Rollback disponible",
     "backToSettings": "Volver a configuraci\u00f3n",
     "percentComplete": "{percent}% completado",
     "applyWarning": "Instalando componentes y reiniciando servicios. No apague el equipo.",
@@ -684,6 +684,14 @@
     "rollbackSuccess": "Se revirti\u00f3 a la versi\u00f3n anterior. El servicio se reiniciar\u00e1.",
     "rollbackFailed": "Error al revertir.",
     "pullAndRebuild": "Pull y Recompilar",
+    "finishingDownload": "Terminando descarga — verificando checksum…",
+    "overlayApplying": "Instalando actualizaci\u00f3n…",
+    "overlayRestarting": "Reiniciando servidor…",
+    "overlayReconnecting": "Reconectando a la nueva versi\u00f3n…",
+    "overlayReady": "Actualizaci\u00f3n instalada — recargando…",
+    "overlayStalled": "Tardando m\u00e1s de lo esperado",
+    "overlayTarget": "Instalando v{version}",
+    "overlayReloadNow": "Recargar ahora",
     "gitMethodHint": "Este nodo compila desde el c\u00f3digo fuente. La actualizaci\u00f3n har\u00e1 git-pull, recompilar\u00e1 y reiniciar\u00e1 — tarda unos minutos.",
     "gitApplyTitle": "\u00bfPull y Recompilar?",
     "gitApplyMessage": "Archipelago descargar\u00e1 el c\u00f3digo m\u00e1s reciente, lo compilar\u00e1 y reiniciar\u00e1. Puede tardar varios minutos y la UI estar\u00e1 brevemente no disponible.",

neode-ui/src/views/SystemUpdate.vue

@@ -113,7 +113,14 @@
           :style="{ width: downloadPercentFormatted + '%' }"
         ></div>
       </div>
-      <p class="text-xs text-white/60">{{ t('systemUpdate.percentComplete', { percent: downloadPercentFormatted }) }}</p>
+      <div class="flex items-center gap-2">
+        <div v-if="downloadFinishing" class="w-3 h-3 border-2 border-orange-400 border-t-transparent rounded-full animate-spin shrink-0"></div>
+        <p class="text-xs text-white/60">
+          {{ downloadFinishing
+             ? t('systemUpdate.finishingDownload')
+             : t('systemUpdate.percentComplete', { percent: downloadPercentFormatted }) }}
+        </p>
+      </div>
     </div>
 
     <!-- Applying -->
@@ -176,6 +183,67 @@
       </div>
     </div>
 
+    <!-- Install progress overlay — covers the UI while the backend
+         swaps files, restarts, and comes back up on the new version.
+         Auto-reloads the page as soon as /health reports the target
+         version. Styled to match the screensaver (ASCII logo, full-
+         screen black). -->
+    <Teleport to="body">
+      <Transition name="fade">
+        <div
+          v-if="installing"
+          class="fixed inset-0 z-[3000] bg-black flex flex-col items-center justify-center overflow-hidden"
+        >
+          <!-- Centered ASCII logo — same asset used by the screensaver -->
+          <div class="install-overlay-ascii">
+            <BitcoinFaceAscii />
+          </div>
+
+          <!-- Status text + progress bar underneath -->
+          <div class="mt-8 w-[min(520px,80vw)] text-center">
+            <h2 class="text-xl font-semibold text-white mb-1">
+              {{ installStage === 'applying' ? t('systemUpdate.overlayApplying')
+               : installStage === 'restarting' ? t('systemUpdate.overlayRestarting')
+               : installStage === 'reconnecting' ? t('systemUpdate.overlayReconnecting')
+               : installStage === 'ready' ? t('systemUpdate.overlayReady')
+               : t('systemUpdate.overlayStalled') }}
+            </h2>
+            <p v-if="installTargetVersion" class="text-sm text-white/60 mb-4">
+              {{ t('systemUpdate.overlayTarget', { version: installTargetVersion }) }}
+            </p>
+
+            <!-- Animated bar: indeterminate stripe while working; full
+                 orange when ready; steady at 50% (paused look) when
+                 stalled so it reads as "something needs the user". -->
+            <div class="w-full h-2 bg-white/10 rounded-full overflow-hidden mb-3 relative">
+              <div
+                v-if="installStage === 'ready'"
+                class="absolute inset-0 bg-green-400"
+              ></div>
+              <div
+                v-else-if="installStage === 'stalled'"
+                class="absolute inset-y-0 left-0 w-1/2 bg-orange-400/60"
+              ></div>
+              <div
+                v-else
+                class="absolute inset-y-0 w-1/3 bg-orange-400 rounded-full install-overlay-bar-anim"
+              ></div>
+            </div>
+
+            <p class="text-xs text-white/40">{{ installElapsedLabel }}</p>
+
+            <button
+              v-if="installStage === 'stalled'"
+              @click="reloadNow"
+              class="mt-5 glass-button rounded-lg px-5 py-2 text-sm font-medium bg-orange-500/20 border-orange-400/30"
+            >
+              {{ t('systemUpdate.overlayReloadNow') }}
+            </button>
+          </div>
+        </div>
+      </Transition>
+    </Teleport>
+
     <!-- Confirmation modal -->
     <Transition name="fade">
       <div v-if="confirmAction" class="fixed inset-0 z-50 flex items-center justify-center bg-black/10 backdrop-blur-md" @click.self="cancelConfirm">
@@ -221,6 +289,7 @@ import { ref, computed, onMounted } from 'vue'
 import { useI18n } from 'vue-i18n'
 import { RouterLink } from 'vue-router'
 import { rpcClient } from '@/api/rpc-client'
+import BitcoinFaceAscii from '@/views/discover/BitcoinFaceAscii.vue'
 
 interface UpdateDetail {
   version: string
@@ -256,6 +325,101 @@ const statusIsError = ref(false)
 const downloadPercent = ref(0)
 const downloadPercentFormatted = computed(() => downloadPercent.value.toFixed(2))
 
+// Poll the backend for the real bytes_downloaded / total_bytes so the
+// progress bar tracks actual download state (and survives route
+// changes). Returns true if a download is currently in progress.
+async function pollDownloadProgress(): Promise<boolean> {
+  try {
+    const res = await rpcClient.call<{
+      download_progress?: {
+        bytes_downloaded: number
+        total_bytes: number
+        active: boolean
+      } | null
+    }>({ method: 'update.status' })
+    const p = res.download_progress
+    if (p && p.total_bytes > 0) {
+      downloadPercent.value = Math.min(100, (p.bytes_downloaded / p.total_bytes) * 100)
+      return p.active
+    }
+    return false
+  } catch {
+    return false
+  }
+}
+// Shown next to the progress bar when the fake increment has maxed out
+// at 95% but the real RPC hasn't returned yet — lets the user know the
+// UI hasn't frozen while SHA verification and disk writes finish.
+const downloadFinishing = computed(() => downloading.value && downloadPercent.value >= 95)
+
+// Install overlay state — drives the full-screen progress modal shown
+// while the backend swaps files, restarts, and comes back up on the
+// new version. The overlay polls /health and auto-reloads the browser
+// as soon as the backend reports the target version, so the user
+// doesn't need to manually refresh.
+type InstallStage = 'applying' | 'restarting' | 'reconnecting' | 'ready' | 'stalled'
+const installing = ref(false)
+const installStage = ref<InstallStage>('applying')
+const installTargetVersion = ref<string | null>(null)
+const installStartedAt = ref<number>(0)
+const installElapsedSec = ref(0)
+let installPollTimer: ReturnType<typeof setInterval> | null = null
+let installElapsedTimer: ReturnType<typeof setInterval> | null = null
+const installElapsedLabel = computed(() => {
+  const s = installElapsedSec.value
+  if (s < 60) return `Elapsed: ${s}s`
+  return `Elapsed: ${Math.floor(s / 60)}m${s % 60 < 10 ? '0' : ''}${s % 60}s`
+})
+function startInstallOverlay(targetVersion: string) {
+  installing.value = true
+  installStage.value = 'applying'
+  installTargetVersion.value = targetVersion
+  installStartedAt.value = Date.now()
+  installElapsedSec.value = 0
+  // Tick an elapsed counter once per second for the UI.
+  installElapsedTimer = setInterval(() => {
+    installElapsedSec.value = Math.floor((Date.now() - installStartedAt.value) / 1000)
+    // Stop polling after 3 min — surface the manual reload button.
+    if (installElapsedSec.value >= 180 && installStage.value !== 'ready') {
+      installStage.value = 'stalled'
+    }
+  }, 1000)
+  // Start polling /health after a short delay — the backend restarts 2s
+  // after replying to update.apply, so an immediate poll would see the
+  // old backend and conclude nothing happened.
+  setTimeout(() => {
+    installStage.value = 'restarting'
+    installPollTimer = setInterval(pollHealth, 1500)
+  }, 2500)
+}
+async function pollHealth() {
+  if (installStage.value === 'ready' || installStage.value === 'stalled') return
+  try {
+    const res = await fetch('/health', { signal: AbortSignal.timeout(2000) })
+    if (!res.ok) throw new Error(`health ${res.status}`)
+    const data = await res.json() as { version?: string }
+    if (data.version && data.version === installTargetVersion.value) {
+      installStage.value = 'ready'
+      if (installPollTimer) { clearInterval(installPollTimer); installPollTimer = null }
+      // Brief pause so the user sees the "Ready" state before the reload.
+      setTimeout(() => { window.location.reload() }, 1200)
+    } else {
+      // Backend is up but still reporting the old version — frontend
+      // and backend are mid-swap. Signal to the user.
+      installStage.value = 'reconnecting'
+    }
+  } catch {
+    // Fetch fails while the server is mid-restart. Stay in 'restarting'.
+  }
+}
+function reloadNow() { window.location.reload() }
+// Cleanup if the component is torn down mid-install (unlikely but safe).
+import { onBeforeUnmount } from 'vue'
+onBeforeUnmount(() => {
+  if (installPollTimer) clearInterval(installPollTimer)
+  if (installElapsedTimer) clearInterval(installElapsedTimer)
+})
+
 const lastCheckDisplay = computed(() => {
   if (!lastCheck.value) return t('common.never')
   try {
@@ -345,21 +509,19 @@ async function downloadUpdate() {
   downloadPercent.value = 0
   statusMessage.value = ''
 
-  // Simulate incremental progress while waiting for the RPC. Capped at
-  // 95% so the bar never shows >100% before the real completion jumps it
-  // to 100 — previously the random increment could overshoot.
-  const progressInterval = setInterval(() => {
-    if (downloadPercent.value < 95) {
-      downloadPercent.value = Math.min(95, downloadPercent.value + Math.random() * 3)
-    }
-  }, 500)
+  // Poll the backend's real byte counter every second instead of
+  // faking progress. The backend exposes bytes_downloaded/total_bytes
+  // via update.status, updated per chunk. This also means the bar
+  // resumes correctly after navigating away and back — no more
+  // "95% for some time" mystery.
+  const progressInterval = setInterval(() => { void pollDownloadProgress() }, 1000)
 
   try {
     const res = await rpcClient.call<{
       total_bytes: number
       downloaded_bytes: number
       components_downloaded: number
-    }>({ method: 'update.download', timeout: 1_800_000 })
+    }>({ method: 'update.download', timeout: 3_900_000 })
     downloadPercent.value = 100
     downloaded.value = true
     const sizeMB = (res.downloaded_bytes / 1_048_576).toFixed(1)
@@ -395,40 +557,50 @@ async function executeConfirm() {
   if (action === 'apply') {
     await applyUpdate()
   } else if (action === 'git-apply') {
-    await applyUpdateGit()
+    await applyUpdateGitWithOverlay()
   } else if (action === 'rollback') {
     await rollbackUpdate()
   }
 }
 
-async function applyUpdateGit() {
-  applying.value = true
-  statusMessage.value = ''
-  try {
-    await rpcClient.call({ method: 'update.git-apply', timeout: 900_000 })
-    showStatus(t('systemUpdate.gitApplyStarted'))
-    updateInfo.value = null
-  } catch (e) {
-    showStatus(t('systemUpdate.applyFailed'), true)
-    if (import.meta.env.DEV) console.warn('Git apply failed', e)
-  } finally {
-    applying.value = false
-  }
-}
-
 async function applyUpdate() {
   applying.value = true
   statusMessage.value = ''
+  const target = updateInfo.value?.version || null
   try {
     await rpcClient.call({ method: 'update.apply', timeout: 300_000 })
-    showStatus(t('systemUpdate.applySuccess'))
-    updateInfo.value = null
-    downloaded.value = false
-    await loadStatus()
+    // Apply succeeded. Backend scheduled a restart 2s after returning;
+    // show the full-screen overlay while we wait for the new backend
+    // to report the target version, then auto-reload.
+    applying.value = false
+    if (target) {
+      startInstallOverlay(target)
+    } else {
+      // No target version known (legacy path) — fall back to the old
+      // flash-and-reload behaviour.
+      showStatus(t('systemUpdate.applySuccess'))
+      setTimeout(() => window.location.reload(), 3000)
+    }
   } catch (e) {
     showStatus(t('systemUpdate.applyFailed'), true)
     if (import.meta.env.DEV) console.warn('Apply failed', e)
-  } finally {
+    applying.value = false
+  }
+}
+
+async function applyUpdateGitWithOverlay() {
+  // Git-apply (dev path) also restarts the service — reuse the overlay
+  // so the UX matches the manifest path. Target version isn't known up
+  // front for git-apply; we just wait for a version change on /health.
+  applying.value = true
+  statusMessage.value = ''
+  try {
+    await rpcClient.call({ method: 'update.git-apply', timeout: 900_000 })
+    applying.value = false
+    startInstallOverlay(updateInfo.value?.version || currentVersion.value)
+  } catch (e) {
+    showStatus(t('systemUpdate.applyFailed'), true)
+    if (import.meta.env.DEV) console.warn('Git apply failed', e)
     applying.value = false
   }
 }
@@ -465,7 +637,45 @@ async function setSchedule(value: ScheduleValue) {
   }
 }
 
-onMounted(() => {
-  Promise.all([loadStatus(), loadSchedule(), checkForUpdates()])
+onMounted(async () => {
+  await Promise.all([loadStatus(), loadSchedule(), checkForUpdates()])
+  // If a download was already running when the user navigated here
+  // (or refreshed), pick up the progress bar where it is and keep
+  // polling until the backend reports done. No RPC call to start the
+  // download — the backend's already running it.
+  const active = await pollDownloadProgress()
+  if (active) {
+    downloading.value = true
+    const resumeInterval = setInterval(async () => {
+      const stillActive = await pollDownloadProgress()
+      if (!stillActive) {
+        clearInterval(resumeInterval)
+        downloading.value = false
+        downloaded.value = true
+      }
+    }, 1000)
+  }
 })
 </script>
+
+<style scoped>
+/* Centered ASCII logo — clamped so the overlay doesn't blow out on
+   narrow viewports. :deep so the rule reaches BitcoinFaceAscii's
+   inner <pre>. */
+.install-overlay-ascii :deep(pre) {
+  font-size: clamp(6px, 1.2vw, 12px);
+  line-height: 1.1;
+  color: rgba(255, 255, 255, 0.85);
+  margin: 0;
+}
+
+/* Indeterminate progress stripe that slides left-to-right. */
+.install-overlay-bar-anim {
+  animation: installBarSlide 1.8s ease-in-out infinite;
+}
+@keyframes installBarSlide {
+  0%   { transform: translateX(-100%); }
+  50%  { transform: translateX(120%); }
+  100% { transform: translateX(300%); }
+}
+</style>

neode-ui/src/views/server/FipsNetworkCard.vue

@@ -180,21 +180,31 @@ async function installAndActivate() {
   }
 }
 
-// Restart the FIPS daemon to kick it back onto the public anchor. Stale
-// identity-cache entries are the usual cause of "not reached"; systemctl
-// restart clears them and re-runs the bootstrap handshake.
+// Restart the FIPS daemon and wait for the anchor bootstrap window.
+// The backend runs a proper recovery sequence (stop → start → wait →
+// classify) and returns a structured diagnostic we can show the user
+// instead of a generic "still unreachable".
 async function reconnectAnchor() {
   reconnecting.value = true
   try {
-    await rpcClient.call({ method: 'fips.restart', timeout: 45_000 })
-    // Give the daemon a few seconds to come back and re-populate its
-    // identity cache before we re-query status.
-    await new Promise((resolve) => setTimeout(resolve, 5000))
-    await loadStatus()
-    if (status.value.anchor_connected) {
-      flash('Anchor reconnected')
+    const res = await rpcClient.call<{
+      recovered: boolean
+      likely_cause: string
+      hint: string
+      after: FipsStatus
+    }>({ method: 'fips.reconnect', timeout: 60_000 })
+    // Update the card with the post-reconnect status returned by the
+    // backend — avoids an extra status fetch race.
+    status.value = { ...status.value, ...res.after }
+    if (res.recovered) {
+      flash('Anchor reconnected.')
+    } else if (res.likely_cause === 'connected') {
+      // Already connected, not a "recovery" per se.
+      flash('Anchor is reachable.')
     } else {
-      flash('FIPS restarted — anchor still reporting unreachable. Check network / firewall.', true)
+      // Surface the backend's diagnostic hint verbatim — it's been
+      // written for the fleet reader.
+      flash(res.hint || 'Reconnect finished but anchor is still unreachable.', true)
     }
   } catch (e: unknown) {
     const msg = e instanceof Error ? e.message : String(e)