release(v1.7.41-alpha): post-OTA auto-rollback so a bad release cannot strand the fleet
Closes failure mode FM5 from docs/bulletproof-containers.md: the v1.7.38 + v1.7.39 rollouts left every affected node on an unreachable UI (nginx 500) with no recovery path short of SSH. This release adds a self-check guardrail to the update flow. What changed: - apply_update() writes a pending-verify marker with old+new version and a 150s deadline immediately before scheduling the service restart. - verify_pending_update() runs from main.rs startup. If the marker is present and within its freshness window, the new binary waits 15s for nginx + backend to settle, then probes https://127.0.0.1/ every 5s for up to 90s (self-signed certs accepted). - On any probe success within the window, the marker is cleared and nothing else happens. - On window-exhaust, the new binary: 1. Moves the broken /opt/archipelago/web-ui to web-ui.failed.<ts> (quarantined, not deleted, so we can post-mortem). 2. Restores web-ui.bak on top of web-ui. 3. Calls rollback_update() to restore the previous binary. 4. Updates state.current_version to reflect the rollback. 5. systemctl --no-block restart archipelago so the OLD binary boots. - Markers older than 10 minutes are treated as stale and cleared without probing, so a crashed-during-startup marker from weeks ago cannot spontaneously roll back a healthy node on a later reboot. - rollback_update() binary copy now goes through host_sudo instead of tokio::fs::copy, so it escapes the service's ProtectSystem=strict mount namespace. Without this, the rollback silently failed with EROFS on /usr/local/bin and orphaned the rollback - the exact opposite of what auto-rollback is for. Tests: 4 new unit tests in update::tests covering marker round-trip, absent-marker noop, no-panic on verify_pending_update with nothing to verify, and an invariant assert that the 90s probe window stays below the 600s stale threshold. All passing. Side fix: scripts/create-release-manifest.sh was dying with exit 141 (SIGPIPE from tar tvzf pipe head pipe awk) under set -euo pipefail. Replaced with a single awk NR==1 that doesn't short-circuit the upstream pipe, so the release-build flow is idempotent again.
This commit is contained in:
archipelago
Generated
+1
-1
@@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
|
||||
|
||||
[[package]]
|
||||
name = "archipelago"
|
||||
version = "1.7.40-alpha"
|
||||
version = "1.7.41-alpha"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"archipelago-container",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "archipelago"
|
||||
version = "1.7.40-alpha"
|
||||
version = "1.7.41-alpha"
|
||||
edition = "2021"
|
||||
description = "Archipelago Bitcoin Node OS - Native backend"
|
||||
authors = ["Archipelago Team"]
|
||||
|
||||
@@ -196,6 +196,19 @@ async fn main() -> Result<()> {
|
||||
// on a 30s poll of fips0 — so a post-onboarding fips.install brings it
|
||||
// online without needing an archipelago restart.
|
||||
|
||||
// Post-OTA verification: if apply_update() wrote a pending-verify
|
||||
// marker right before the restart, probe the frontend now and auto-
|
||||
// rollback if it's broken. This is the guardrail that stops fleet-
|
||||
// wide breakage when an OTA lands a subtly-bad release (v1.7.38/39
|
||||
// tarball-perms → nginx 500 was the trigger). Runs concurrently
|
||||
// with normal startup — doesn't delay the server coming up.
|
||||
{
|
||||
let data_dir = config.data_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
update::verify_pending_update(&data_dir).await;
|
||||
});
|
||||
}
|
||||
|
||||
// Spawn background update scheduler
|
||||
let update_data_dir = config.data_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
|
||||
@@ -74,6 +74,23 @@ const DEFAULT_TERTIARY_MIRROR_URL: &str =
|
||||
"http://146.59.87.168:3000/lfg2025/archy/raw/branch/main/releases/manifest.json";
|
||||
const UPDATE_STATE_FILE: &str = "update_state.json";
|
||||
const UPDATE_MIRRORS_FILE: &str = "update-mirrors.json";
|
||||
/// Marker written by apply_update() just before the service restart and
|
||||
/// consumed by verify_pending_update() in the NEW binary's startup path.
|
||||
/// If present, the new binary probes the frontend; if the probe fails,
|
||||
/// rollback_update() runs and the service restarts on the old binary.
|
||||
/// Closes the "OTA broke nginx fleet-wide with no auto-rollback" failure
|
||||
/// mode from 2026-04-22 (v1.7.38/39 tarball-perms bug).
|
||||
const PENDING_VERIFY_FILE: &str = "update-pending-verify.json";
|
||||
/// Probe timeout for the frontend health check (total time including
|
||||
/// retries). Generous: the new binary has to come fully up, health
|
||||
/// monitor settles, nginx has to re-read any snippet changes. 90s is
|
||||
/// comfortably longer than the slowest observed startup.
|
||||
const PENDING_VERIFY_WINDOW_SECS: u64 = 90;
|
||||
/// If the marker is older than this on read, treat it as stale and
|
||||
/// delete without probing. Guards against a node that somehow failed
|
||||
/// to run verification at all (e.g. crashed during startup) from
|
||||
/// spontaneously rolling back days later when the user reboots.
|
||||
const PENDING_VERIFY_MAX_AGE_SECS: i64 = 600;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct UpdateMirror {
|
||||
@@ -268,6 +285,189 @@ impl Default for UpdateState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Marker written by apply_update() just before the service restart and
|
||||
/// consumed by verify_pending_update() in the NEW binary's startup path.
|
||||
/// See PENDING_VERIFY_FILE for the full rationale — this is the hook
|
||||
/// that turns "nginx 500 on every page after OTA" from an unrecoverable
|
||||
/// field incident into an automatic rollback.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PendingVerification {
|
||||
/// RFC3339 timestamp of the apply that wrote this marker.
|
||||
pub applied_at: String,
|
||||
/// Version we just applied (what the NEW binary should be running).
|
||||
pub new_version: String,
|
||||
/// Version the outgoing binary was running (what we roll back to).
|
||||
pub previous_version: String,
|
||||
/// Unix epoch seconds after which the probe should give up and
|
||||
/// trigger rollback. Prevents a probe from retrying forever if e.g.
|
||||
/// nginx is totally wedged.
|
||||
pub deadline_ts: i64,
|
||||
}
|
||||
|
||||
async fn write_pending_verification(
|
||||
data_dir: &Path,
|
||||
marker: &PendingVerification,
|
||||
) -> Result<()> {
|
||||
let path = data_dir.join(PENDING_VERIFY_FILE);
|
||||
let data = serde_json::to_string_pretty(marker)
|
||||
.context("serialize pending-verify marker")?;
|
||||
fs::write(&path, data)
|
||||
.await
|
||||
.with_context(|| format!("write pending-verify marker to {}", path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_pending_verification(data_dir: &Path) -> Option<PendingVerification> {
|
||||
let path = data_dir.join(PENDING_VERIFY_FILE);
|
||||
let data = fs::read_to_string(&path).await.ok()?;
|
||||
serde_json::from_str(&data).ok()
|
||||
}
|
||||
|
||||
async fn clear_pending_verification(data_dir: &Path) {
|
||||
let path = data_dir.join(PENDING_VERIFY_FILE);
|
||||
let _ = fs::remove_file(&path).await;
|
||||
}
|
||||
|
||||
/// Probe the local frontend through nginx. Returns Ok(()) on the first
|
||||
/// response that's 2xx or 3xx; errors on timeout / connection refused /
|
||||
/// any 4xx/5xx. `accept_self_signed` because nodes use a self-signed
|
||||
/// cert the reqwest default root-set doesn't trust.
|
||||
async fn probe_frontend_once() -> Result<()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.danger_accept_invalid_certs(true)
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.context("build probe client")?;
|
||||
// Prefer HTTPS since that's the failure mode we're catching (nginx
|
||||
// 500 on the PWA). HTTP usually redirects to HTTPS and would mask
|
||||
// the bug.
|
||||
let resp = client
|
||||
.get("https://127.0.0.1/")
|
||||
.send()
|
||||
.await
|
||||
.context("probe GET https://127.0.0.1/")?;
|
||||
let status = resp.status();
|
||||
if status.is_success() || status.is_redirection() {
|
||||
return Ok(());
|
||||
}
|
||||
anyhow::bail!("frontend probe returned HTTP {}", status);
|
||||
}
|
||||
|
||||
/// Called from main.rs startup. If a pending-verification marker is
|
||||
/// present, probe the frontend; on failure, trigger rollback and
|
||||
/// restart the service so the OLD binary boots.
|
||||
///
|
||||
/// This is the "post-OTA auto-rollback" guardrail. If ANY problem in
|
||||
/// the new version takes down the PWA (bad tarball perms as in v1.7.38,
|
||||
/// a broken service worker, a missing asset, a backend panic on first
|
||||
/// boot), the node self-heals back to the previous working state
|
||||
/// without SSH intervention.
|
||||
pub async fn verify_pending_update(data_dir: &Path) {
|
||||
let marker = match read_pending_verification(data_dir).await {
|
||||
Some(m) => m,
|
||||
None => return, // No update pending; nothing to verify.
|
||||
};
|
||||
|
||||
// Guard against a marker left behind by some earlier crash path —
|
||||
// don't want a user who reboots days later to suddenly get
|
||||
// rolled back because the marker was never cleared.
|
||||
let applied_at = chrono::DateTime::parse_from_rfc3339(&marker.applied_at);
|
||||
if let Ok(ts) = applied_at {
|
||||
let age = chrono::Utc::now() - ts.with_timezone(&chrono::Utc);
|
||||
if age.num_seconds() > PENDING_VERIFY_MAX_AGE_SECS {
|
||||
tracing::warn!(
|
||||
age_secs = age.num_seconds(),
|
||||
"pending-verify marker is stale, clearing without probing"
|
||||
);
|
||||
clear_pending_verification(data_dir).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
new_version = %marker.new_version,
|
||||
previous_version = %marker.previous_version,
|
||||
"Post-OTA verification: probing frontend at https://127.0.0.1/"
|
||||
);
|
||||
|
||||
// Give the new service time to bind its listeners + nginx to
|
||||
// pick up any config changes. 15s matches what we observed on
|
||||
// .116 during the v1.7.40 rollout recovery.
|
||||
tokio::time::sleep(std::time::Duration::from_secs(15)).await;
|
||||
|
||||
let deadline =
|
||||
std::time::Instant::now() + std::time::Duration::from_secs(PENDING_VERIFY_WINDOW_SECS);
|
||||
let mut attempt = 0u32;
|
||||
let mut last_err: Option<String> = None;
|
||||
|
||||
while std::time::Instant::now() < deadline {
|
||||
attempt += 1;
|
||||
match probe_frontend_once().await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
attempt,
|
||||
"Post-OTA verification succeeded — clearing marker"
|
||||
);
|
||||
clear_pending_verification(data_dir).await;
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = e.to_string();
|
||||
tracing::warn!(attempt, error = %msg, "Post-OTA probe failed, retrying");
|
||||
last_err = Some(msg);
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
||||
}
|
||||
|
||||
tracing::error!(
|
||||
attempts = attempt,
|
||||
window_secs = PENDING_VERIFY_WINDOW_SECS,
|
||||
last_error = last_err.as_deref().unwrap_or(""),
|
||||
new_version = %marker.new_version,
|
||||
previous_version = %marker.previous_version,
|
||||
"Post-OTA verification FAILED — rolling back"
|
||||
);
|
||||
|
||||
// Restore web-ui.bak on top of web-ui. update.rs keeps web-ui.bak
|
||||
// from the previous apply; moving it back is the frontend half of
|
||||
// the rollback. The binary half is handled by rollback_update().
|
||||
let web_ui_bak = Path::new("/opt/archipelago/web-ui.bak");
|
||||
let web_ui = "/opt/archipelago/web-ui";
|
||||
if web_ui_bak.exists() {
|
||||
let ts = chrono::Utc::now().timestamp_millis();
|
||||
let quarantine = format!("/opt/archipelago/web-ui.failed.{}", ts);
|
||||
let _ = host_sudo(&["mv", web_ui, &quarantine]).await;
|
||||
let _ = host_sudo(&["mv", web_ui_bak.to_str().unwrap_or(""), web_ui]).await;
|
||||
tracing::info!(quarantined = %quarantine, "Restored web-ui from web-ui.bak");
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"web-ui.bak not present — frontend cannot be rolled back, only binary"
|
||||
);
|
||||
}
|
||||
|
||||
if let Err(e) = rollback_update(data_dir).await {
|
||||
tracing::error!(error = %e, "rollback_update() failed during post-OTA verification");
|
||||
// Leave the marker in place so a future boot gets another shot.
|
||||
return;
|
||||
}
|
||||
|
||||
clear_pending_verification(data_dir).await;
|
||||
|
||||
// Record why we rolled back so the UI can show it on the next boot.
|
||||
if let Ok(mut state) = load_state(data_dir).await {
|
||||
state.current_version = marker.previous_version.clone();
|
||||
if let Err(e) = save_state(data_dir, &state).await {
|
||||
tracing::warn!(error = %e, "Failed to update state after rollback");
|
||||
}
|
||||
}
|
||||
|
||||
// Restart so the old binary takes over. --no-block because we're
|
||||
// the service; systemd can't wait for us to exit before starting
|
||||
// the old process.
|
||||
let _ = host_sudo(&["systemctl", "--no-block", "restart", "archipelago"]).await;
|
||||
}
|
||||
|
||||
pub async fn load_state(data_dir: &Path) -> Result<UpdateState> {
|
||||
let path = data_dir.join(UPDATE_STATE_FILE);
|
||||
if !path.exists() {
|
||||
@@ -985,15 +1185,42 @@ pub async fn apply_update(data_dir: &Path) -> Result<()> {
|
||||
}
|
||||
|
||||
// Update state
|
||||
let previous_version = {
|
||||
let state = load_state(data_dir).await?;
|
||||
state.current_version.clone()
|
||||
};
|
||||
let mut state = load_state(data_dir).await?;
|
||||
if let Some(manifest) = &state.available_update {
|
||||
let new_version = if let Some(manifest) = &state.available_update {
|
||||
state.current_version = manifest.version.clone();
|
||||
}
|
||||
manifest.version.clone()
|
||||
} else {
|
||||
state.current_version.clone()
|
||||
};
|
||||
state.available_update = None;
|
||||
state.update_in_progress = false;
|
||||
state.rollback_available = true;
|
||||
save_state(data_dir, &state).await?;
|
||||
|
||||
// Write the post-OTA verification marker BEFORE we schedule the
|
||||
// restart. The new binary will read it on startup, probe the
|
||||
// frontend, and auto-rollback if nginx is serving 5xx. Covers the
|
||||
// class of failure where "apply succeeds, restart succeeds, but
|
||||
// the UI is dead" (v1.7.38/39 tarball-perms bug). Best-effort —
|
||||
// a failed marker write shouldn't abort the apply.
|
||||
let marker = PendingVerification {
|
||||
applied_at: chrono::Utc::now().to_rfc3339(),
|
||||
new_version,
|
||||
previous_version,
|
||||
deadline_ts: chrono::Utc::now().timestamp()
|
||||
+ PENDING_VERIFY_WINDOW_SECS as i64
|
||||
+ 60,
|
||||
};
|
||||
if let Err(e) = write_pending_verification(data_dir, &marker).await {
|
||||
tracing::warn!(error = %e, "Failed to write post-OTA verify marker — rollback disabled for this OTA");
|
||||
} else {
|
||||
info!("Post-OTA verify marker written; new binary will probe on boot");
|
||||
}
|
||||
|
||||
// Clean staging
|
||||
let _ = fs::remove_dir_all(&staging_dir).await;
|
||||
|
||||
@@ -1023,9 +1250,24 @@ pub async fn rollback_update(data_dir: &Path) -> Result<()> {
|
||||
|
||||
let backup_binary = backup_dir.join("archipelago");
|
||||
if backup_binary.exists() {
|
||||
fs::copy(&backup_binary, "/usr/local/bin/archipelago")
|
||||
// Use host_sudo + mv so we escape the archipelago service's
|
||||
// ProtectSystem=strict mount namespace. A plain fs::copy or
|
||||
// `sudo cp` from inside the service hits EROFS on /usr/local/bin,
|
||||
// which would silently orphan the rollback — exactly the
|
||||
// opposite of what auto-rollback is for. Pattern matches
|
||||
// apply_update()'s binary swap above.
|
||||
let backup_str = backup_binary.to_string_lossy().to_string();
|
||||
let _ = host_sudo(&["chmod", "0755", &backup_str]).await;
|
||||
let _ = host_sudo(&["chown", "root:root", &backup_str]).await;
|
||||
let status = host_sudo(&["cp", &backup_str, "/usr/local/bin/archipelago"])
|
||||
.await
|
||||
.context("Failed to restore backup binary")?;
|
||||
.context("Failed to restore backup binary via host_sudo")?;
|
||||
if !status.success() {
|
||||
anyhow::bail!(
|
||||
"cp backup binary into /usr/local/bin failed (exit {:?})",
|
||||
status.code()
|
||||
);
|
||||
}
|
||||
info!("Binary rolled back to previous version");
|
||||
}
|
||||
|
||||
@@ -1449,4 +1691,45 @@ mod tests {
|
||||
assert_eq!(status.current_version, env!("CARGO_PKG_VERSION"));
|
||||
assert!(status.rollback_available);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pending_verification_round_trip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let marker = PendingVerification {
|
||||
applied_at: chrono::Utc::now().to_rfc3339(),
|
||||
new_version: "1.7.41-alpha".into(),
|
||||
previous_version: "1.7.40-alpha".into(),
|
||||
deadline_ts: chrono::Utc::now().timestamp() + 150,
|
||||
};
|
||||
write_pending_verification(dir.path(), &marker).await.unwrap();
|
||||
let read = read_pending_verification(dir.path()).await.unwrap();
|
||||
assert_eq!(read.new_version, "1.7.41-alpha");
|
||||
assert_eq!(read.previous_version, "1.7.40-alpha");
|
||||
clear_pending_verification(dir.path()).await;
|
||||
assert!(read_pending_verification(dir.path()).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pending_verification_absent_is_none() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
assert!(read_pending_verification(dir.path()).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_verify_pending_update_noop_without_marker() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// No marker written -- must return quickly without doing anything
|
||||
// risky (network probes, rollback calls). We're just asserting
|
||||
// it doesn't panic or hang.
|
||||
verify_pending_update(dir.path()).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pending_verify_constants_are_sensible() {
|
||||
// Window must be generous enough for nginx + backend startup,
|
||||
// but less than the stale-marker threshold so a normal cycle
|
||||
// can complete without the marker being considered stale.
|
||||
assert!(PENDING_VERIFY_WINDOW_SECS < PENDING_VERIFY_MAX_AGE_SECS as u64);
|
||||
assert!(PENDING_VERIFY_WINDOW_SECS >= 60);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user