fix(boot): signal systemd READY before heavy boot recovery + Restart=always

Root cause of 'server starting up' forever / crash-on-install
(framework-pt, v1.7.114->115, 2026-07-26): on a node with many stacks,
the synchronous boot recovery (recover + start_stopped_containers) runs
BEFORE sd_notify(Ready), so the unit sits in 'activating' for minutes.
Anything touching the service in that window — a superseding
start/restart, an install-time reconcile churn — killed a half-started
instance; it exits 0 on SIGTERM and Restart=on-failure then never
restarts it. Node dead behind 'server starting up'.

Fixes:
- signal READY (+ start the watchdog keepalive) BEFORE boot recovery, so
  the unit reaches 'active' in seconds; recovery/reconcile/listener
  continue after. No more minutes-long activating window.
- Restart=always (was on-failure): a clean-exit SIGTERM must still bring
  the daemon back. Manual  is still honored.
- OTA restart via a PID1-owned transient timer (systemd-run --on-active=2)
  instead of a tokio-sleep child of the process being stopped, whose
  start-half was being lost.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/archipelago/src/main.rs

@@ -199,6 +199,26 @@ async fn main() -> Result<()> {
     // Now mark this instance as running so the next startup can detect a crash.
     crash_recovery::write_pid_marker(&config.data_dir).await?;
 
+    // Signal READY *before* the heavy synchronous boot recovery below. On a
+    // node with many stacks that recovery takes minutes, and the unit sat in
+    // `activating` the whole time — so anything that touched the service in
+    // that window (a superseding start/restart, a start-timeout) killed a
+    // half-started instance, which then exited 0 and (under the old
+    // Restart=on-failure) never came back: "server starting up" forever,
+    // reproduced on framework-pt installing apps on 2026-07-26. The daemon's
+    // real work (recovery, reconcile, listener) continues after READY; being
+    // "active" early is honest — the process is up and doing its job.
+    let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
+    // Watchdog pings must run DURING the long recovery too, or a slow boot
+    // trips WatchdogSec. Spawn the keepalive here rather than after serve().
+    tokio::spawn(async {
+        let mut interval = tokio::time::interval(std::time::Duration::from_secs(120));
+        loop {
+            interval.tick().await;
+            let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Watchdog]);
+        }
+    });
+
     // Run crash recovery before starting the manifest reconciler. Both paths
     // mutate Podman; running them concurrently can corrupt transient runtime
     // state and leave netavark/conmon unable to start containers.
@@ -479,16 +499,9 @@ async fn main() -> Result<()> {
 
     // Notify systemd that we're ready (Type=notify)
     // Note: first param `false` keeps NOTIFY_SOCKET so watchdog pings work
-    let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]);
-
-    // Spawn systemd watchdog ping (WatchdogSec=300, ping every 120s)
-    tokio::spawn(async {
-        let mut interval = tokio::time::interval(std::time::Duration::from_secs(120));
-        loop {
-            interval.tick().await;
-            let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Watchdog]);
-        }
-    });
+    // READY + watchdog keepalive were already signalled/spawned earlier
+    // (before boot recovery) so the unit reaches `active` in seconds instead
+    // of sitting in `activating` through a minutes-long recovery.
 
     // Graceful shutdown: wait for SIGTERM or SIGINT
     let mut sigterm = signal::unix::signal(signal::unix::SignalKind::terminate())

core/archipelago/src/update.rs

@@ -1868,13 +1868,38 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> {
     // UI before systemd kills us. --no-block makes sure systemctl doesn't
     // try to wait for the current service (us) to exit cleanly before
     // starting the new process — it would deadlock otherwise.
-    tokio::spawn(async {
-        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
-        // systemctl talks to PID 1 over D-Bus — doesn't need the host
-        // mount namespace, but routing through host_sudo keeps the
-        // apply flow's sudo calls uniform.
-        let _ = host_sudo(&["systemctl", "--no-block", "restart", "archipelago"]).await;
-    });
+    // PID1-owned timer transient: submit NOW (synchronously, while we are
+    // definitely alive), fire in 2s from systemd itself. The old approach —
+    // tokio sleep + `systemd-run --wait -- systemctl --no-block restart` —
+    // ran as a child of the process being stopped; on v1.7.114->115 the
+    // stop landed but the start never fired and the node sat dead all
+    // night. A timer unit owned by PID1 cannot be killed by our own death,
+    // and Restart=always on the unit is the second net.
+    let submitted = tokio::process::Command::new("sudo")
+        .args([
+            "systemd-run",
+            "--collect",
+            "--on-active=2",
+            "--timer-property=AccuracySec=100ms",
+            "--",
+            "systemctl",
+            "restart",
+            "archipelago",
+        ])
+        .status()
+        .await;
+    match submitted {
+        Ok(st) if st.success() => {}
+        other => {
+            tracing::warn!(
+                "detached restart submission failed ({other:?}) — falling back to in-process restart"
+            );
+            tokio::spawn(async {
+                tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+                let _ = host_sudo(&["systemctl", "--no-block", "restart", "archipelago"]).await;
+            });
+        }
+    }
 
     Ok(())
 }

image-recipe/configs/archipelago.service

@@ -31,7 +31,13 @@ ExecStartPre=+/bin/bash -c 'mkdir -p /var/lib/archipelago && chown archipelago:a
 # "-" so a missing/failed guard can never block the service itself.
 ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh
 ExecStart=/usr/local/bin/archipelago
-Restart=on-failure
+# always (not on-failure): the OTA restart path once stopped the daemon
+# cleanly and the queued start never fired (framework-pt, v1.7.114->115,
+# 2026-07-26) — the node sat dead all night behind "server starting up".
+# Restart=always self-heals any lost start job; an explicit
+# `systemctl stop` is still honored (systemd never auto-restarts after
+# a manual stop).
+Restart=always
 RestartSec=5
 WatchdogSec=300
 TimeoutStartSec=300