lfg2025/archy
1
0
Fork 0
Code Issues 24 Pull Requests 1 Actions Packages Projects Releases 64 Wiki Activity

chore(release): stage v1.7.52-alpha

Browse Source
This commit is contained in:
archipelago
2026-05-05 11:29:18 -04:00
parent 10fbb8f87c
commit 745cb1c626
86 changed files with 4084 additions and 966 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/bootstrap.rs
+22 -105
View File
@@ -71,88 +71,35 @@ pub async fn ensure_doctor_installed() {
Ok(_) => debug!("Secrets directory already at expected mode"),
Err(e) => warn!("Secrets dir tightening failed (non-fatal): {:#}", e),
}
// Podman self-heal MUST be the last bootstrap stage. If podman's
// runtime state is wedged, the orchestrator's first reconcile tick
// (which fires seconds after bootstrap returns) will hang or error
// on every container. Cleaning the runroot here gives the rest of
// the process a healthy podman to talk to.
// Podman probing MUST be the last bootstrap stage. We used to delete
// transient runroot state here when `podman info` failed, but live nodes
// can still have rootlessport/conmon processes holding that state. Removing
// it automatically makes failures worse: containers lose `.containerenv`,
// ports stay bound, and later starts fail. Report the fault instead; repair
// must be deliberate/operator-driven.
match heal_podman_state().await {
Ok(PodmanHealOutcome::Healthy) => debug!("podman runtime state healthy"),
Ok(PodmanHealOutcome::Cleaned) => warn!(
"podman runtime state was wedged at startup — cleaned runroot and re-probed (CRITICAL)"
Ok(PodmanHealOutcome::Unhealthy) => warn!(
"podman runtime state is unhealthy at startup — skipping automatic runroot cleanup"
),
Err(e) => warn!(
"podman self-heal failed (non-fatal, will retry next boot): {:#}",
e
),
Err(e) => warn!("podman self-heal failed (non-fatal, will retry next boot): {:#}", e),
}
}
#[derive(Debug, PartialEq, Eq)]
enum PodmanHealOutcome {
Healthy,
Cleaned,
Unhealthy,
}
/// Probe `podman info`. If it succeeds the daemon's runtime state is
/// fine and we return `Healthy` immediately. If it times out, fails to
/// spawn, or returns an "invalid internal status" / "database state"
/// error, the runtime state under `$XDG_RUNTIME_DIR/{containers,libpod}`
/// is likely wedged. We delete those two dirs and re-probe — podman
/// rebuilds runtime state from persistent storage under
/// `$HOME/.local/share/containers/storage/`.
///
/// `$XDG_RUNTIME_DIR/podman/` is **deliberately not touched**: that's
/// where systemd's socket-activated `podman.sock` listener lives. If we
/// removed it, every libpod HTTP call from the orchestrator would fail
/// with "connection refused" until `systemctl --user restart
/// podman.socket` ran — far worse than the wedge we'd be trying to fix.
///
/// Why this is safe at startup:
/// - We run BEFORE the orchestrator starts its reconcile loop, so no
/// archipelago code is currently calling podman.
/// - Persistent container metadata lives under
/// `~/.local/share/containers/`, which we never touch.
/// - `unless-stopped` containers and Quadlet-supervised services are
/// parented under user.slice, not archipelago.service, so they keep
/// running even while we clean podman's runtime view of them. After
/// the cleanup + re-probe podman re-discovers them.
///
/// What this does NOT cover:
/// - Storage corruption under `~/.local/share/containers/storage/`.
/// That requires a destructive `podman system reset`, which we will
/// never do automatically — operator must intervene.
/// - Networking corruption (netavark cache). Currently `podman info`
/// doesn't diagnose that; if cleanup doesn't fix it, the operator
/// will see the warning in the journal.
/// Subdirectories of `$XDG_RUNTIME_DIR` that hold podman's transient
/// state and are safe to remove when `podman info` is wedged. The
/// `podman/` subdir is **deliberately absent** — that's where systemd's
/// socket-activated `podman.sock` listener lives. Removing it would
/// silently break every libpod HTTP call from the orchestrator until
/// `systemctl --user restart podman.socket`. See
/// `heal_podman_state` docstring for the full rationale and the
/// `heal_podman_state_does_not_clean_socket_dir` regression test.
const HEAL_RUNTIME_SUBDIRS: &[&str] = &["containers", "libpod"];
async fn heal_podman_state() -> Result<PodmanHealOutcome> {
if probe_podman_ok().await {
return Ok(PodmanHealOutcome::Healthy);
}
let xdg = std::env::var("XDG_RUNTIME_DIR")
.context("XDG_RUNTIME_DIR not set; can't locate podman runtime state to clean")?;
for sub in HEAL_RUNTIME_SUBDIRS {
let path = PathBuf::from(&xdg).join(sub);
match fs::remove_dir_all(&path).await {
Ok(()) => debug!(path = %path.display(), "removed podman runtime state dir"),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => warn!(path = %path.display(), "remove failed: {}", e),
}
}
if probe_podman_ok().await {
Ok(PodmanHealOutcome::Cleaned)
} else {
Err(anyhow::anyhow!(
"podman info still failing after runtime cleanup; storage may be corrupt — operator must intervene"
))
}
Ok(PodmanHealOutcome::Unhealthy)
}
/// True iff `podman info` returns 0 within 5s. Any timeout, spawn
@@ -298,7 +245,6 @@ async fn run_runtime_assets() -> Result<bool> {
if changed {
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
let _ = host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await;
}
Ok(changed)
}
@@ -453,21 +399,14 @@ async fn run() -> Result<bool> {
let timer_changed = write_root_if_needed(DOCTOR_TIMER_PATH, DOCTOR_TIMER).await?;
changed = changed || service_changed || timer_changed;
// 3. Reload + enable. Only when we actually touched units, or when the
// timer isn't enabled yet (catches fresh upgrades of boxes that predate
// the doctor entirely).
let timer_enabled = is_timer_enabled().await;
if service_changed || timer_changed || !timer_enabled {
// 3. Reload if units changed. Do not enable/start the timer here: lifecycle
// qualification and explicit app operations need deterministic Podman
// ownership, and the doctor can race those flows. Operators can enable it
// separately when they want periodic host repair.
if service_changed || timer_changed {
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
warn!("daemon-reload failed: {:#}", e);
}
if let Err(e) =
host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await
{
warn!("enable archipelago-doctor.timer failed: {:#}", e);
} else if !timer_enabled {
info!("Enabled archipelago-doctor.timer");
}
}
Ok(changed)
@@ -508,15 +447,6 @@ async fn write_root_if_needed(path: &str, content: &str) -> Result<bool> {
Ok(true)
}
async fn is_timer_enabled() -> bool {
tokio::process::Command::new("systemctl")
.args(["is-enabled", "--quiet", "archipelago-doctor.timer"])
.status()
.await
.map(|s| s.success())
.unwrap_or(false)
}
/// Patch the nginx site config to add a `/api/app-catalog` proxy block if
/// it's missing. The original ISO shipped individual per-endpoint `location`
/// blocks and no catch-all `/api/`, so `/api/app-catalog` silently fell
@@ -615,22 +545,9 @@ async fn run_nginx() -> Result<bool> {
mod tests {
use super::*;
/// Regression gate for the 2026-05-01 bootstrap bug: heal_podman_state
/// was removing $XDG_RUNTIME_DIR/podman/ alongside containers/ and
/// libpod/, which silently broke the systemd-bound podman.sock and
/// every libpod HTTP call from the orchestrator. If anyone re-adds
/// "podman" to HEAL_RUNTIME_SUBDIRS this test fires before we ship.
#[test]
fn heal_podman_state_does_not_clean_socket_dir() {
assert!(
!HEAL_RUNTIME_SUBDIRS.contains(&"podman"),
"HEAL_RUNTIME_SUBDIRS must not include 'podman' — that dir holds \
systemd's podman.sock listener; removing it breaks every libpod \
HTTP call from the orchestrator. See bootstrap.rs commit bb421803."
);
// Sanity: the actually-runtime-state dirs are still in the list so
// we don't accidentally turn the heal into a no-op.
assert!(HEAL_RUNTIME_SUBDIRS.contains(&"containers"));
assert!(HEAL_RUNTIME_SUBDIRS.contains(&"libpod"));
fn podman_heal_outcome_no_longer_has_cleanup_variant() {
let outcome = PodmanHealOutcome::Unhealthy;
assert_ne!(outcome, PodmanHealOutcome::Healthy);
}
}