From 70bcbc4acc52ececc1f5658fcae9e72071c8ad94 Mon Sep 17 00:00:00 2001 From: Dorian Date: Mon, 27 Jul 2026 00:31:54 +0100 Subject: [PATCH] fix(boot): signal systemd READY before heavy boot recovery + Restart=always MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of 'server starting up' forever / crash-on-install (framework-pt, v1.7.114->115, 2026-07-26): on a node with many stacks, the synchronous boot recovery (recover + start_stopped_containers) runs BEFORE sd_notify(Ready), so the unit sits in 'activating' for minutes. Anything touching the service in that window — a superseding start/restart, an install-time reconcile churn — killed a half-started instance; it exits 0 on SIGTERM and Restart=on-failure then never restarts it. Node dead behind 'server starting up'. Fixes: - signal READY (+ start the watchdog keepalive) BEFORE boot recovery, so the unit reaches 'active' in seconds; recovery/reconcile/listener continue after. No more minutes-long activating window. - Restart=always (was on-failure): a clean-exit SIGTERM must still bring the daemon back. Manual is still honored. - OTA restart via a PID1-owned transient timer (systemd-run --on-active=2) instead of a tokio-sleep child of the process being stopped, whose start-half was being lost. Co-Authored-By: Claude Fable 5 --- core/archipelago/src/main.rs | 33 ++++++++++++++------ core/archipelago/src/update.rs | 39 +++++++++++++++++++----- image-recipe/configs/archipelago.service | 8 ++++- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index 9d67c409..3a732200 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -199,6 +199,26 @@ async fn main() -> Result<()> { // Now mark this instance as running so the next startup can detect a crash. crash_recovery::write_pid_marker(&config.data_dir).await?; + // Signal READY *before* the heavy synchronous boot recovery below. On a + // node with many stacks that recovery takes minutes, and the unit sat in + // `activating` the whole time — so anything that touched the service in + // that window (a superseding start/restart, a start-timeout) killed a + // half-started instance, which then exited 0 and (under the old + // Restart=on-failure) never came back: "server starting up" forever, + // reproduced on framework-pt installing apps on 2026-07-26. The daemon's + // real work (recovery, reconcile, listener) continues after READY; being + // "active" early is honest — the process is up and doing its job. + let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]); + // Watchdog pings must run DURING the long recovery too, or a slow boot + // trips WatchdogSec. Spawn the keepalive here rather than after serve(). + tokio::spawn(async { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(120)); + loop { + interval.tick().await; + let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Watchdog]); + } + }); + // Run crash recovery before starting the manifest reconciler. Both paths // mutate Podman; running them concurrently can corrupt transient runtime // state and leave netavark/conmon unable to start containers. @@ -479,16 +499,9 @@ async fn main() -> Result<()> { // Notify systemd that we're ready (Type=notify) // Note: first param `false` keeps NOTIFY_SOCKET so watchdog pings work - let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]); - - // Spawn systemd watchdog ping (WatchdogSec=300, ping every 120s) - tokio::spawn(async { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(120)); - loop { - interval.tick().await; - let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Watchdog]); - } - }); + // READY + watchdog keepalive were already signalled/spawned earlier + // (before boot recovery) so the unit reaches `active` in seconds instead + // of sitting in `activating` through a minutes-long recovery. // Graceful shutdown: wait for SIGTERM or SIGINT let mut sigterm = signal::unix::signal(signal::unix::SignalKind::terminate()) diff --git a/core/archipelago/src/update.rs b/core/archipelago/src/update.rs index 3e6e22f3..a4c3fa25 100644 --- a/core/archipelago/src/update.rs +++ b/core/archipelago/src/update.rs @@ -1868,13 +1868,38 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> { // UI before systemd kills us. --no-block makes sure systemctl doesn't // try to wait for the current service (us) to exit cleanly before // starting the new process — it would deadlock otherwise. - tokio::spawn(async { - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - // systemctl talks to PID 1 over D-Bus — doesn't need the host - // mount namespace, but routing through host_sudo keeps the - // apply flow's sudo calls uniform. - let _ = host_sudo(&["systemctl", "--no-block", "restart", "archipelago"]).await; - }); + // PID1-owned timer transient: submit NOW (synchronously, while we are + // definitely alive), fire in 2s from systemd itself. The old approach — + // tokio sleep + `systemd-run --wait -- systemctl --no-block restart` — + // ran as a child of the process being stopped; on v1.7.114->115 the + // stop landed but the start never fired and the node sat dead all + // night. A timer unit owned by PID1 cannot be killed by our own death, + // and Restart=always on the unit is the second net. + let submitted = tokio::process::Command::new("sudo") + .args([ + "systemd-run", + "--collect", + "--on-active=2", + "--timer-property=AccuracySec=100ms", + "--", + "systemctl", + "restart", + "archipelago", + ]) + .status() + .await; + match submitted { + Ok(st) if st.success() => {} + other => { + tracing::warn!( + "detached restart submission failed ({other:?}) — falling back to in-process restart" + ); + tokio::spawn(async { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let _ = host_sudo(&["systemctl", "--no-block", "restart", "archipelago"]).await; + }); + } + } Ok(()) } diff --git a/image-recipe/configs/archipelago.service b/image-recipe/configs/archipelago.service index bc605b5a..e3dd33b7 100644 --- a/image-recipe/configs/archipelago.service +++ b/image-recipe/configs/archipelago.service @@ -31,7 +31,13 @@ ExecStartPre=+/bin/bash -c 'mkdir -p /var/lib/archipelago && chown archipelago:a # "-" so a missing/failed guard can never block the service itself. ExecStartPre=+-/opt/archipelago/scripts/ota-crash-guard.sh ExecStart=/usr/local/bin/archipelago -Restart=on-failure +# always (not on-failure): the OTA restart path once stopped the daemon +# cleanly and the queued start never fired (framework-pt, v1.7.114->115, +# 2026-07-26) — the node sat dead all night behind "server starting up". +# Restart=always self-heals any lost start job; an explicit +# `systemctl stop` is still honored (systemd never auto-restarts after +# a manual stop). +Restart=always RestartSec=5 WatchdogSec=300 TimeoutStartSec=300