From 1bbf85e0d438efaf15ba194626f9f452ac517e98 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 01:14:49 -0400 Subject: [PATCH 01/13] Recover stopped update staging draft on current private-image preflight --- .../src/api/rpc/package/async_lifecycle.rs | 6 +- .../archipelago/src/api/rpc/package/config.rs | 39 +- .../src/api/rpc/package/runtime.rs | 14 +- .../archipelago/src/api/rpc/package/update.rs | 177 ++++- .../src/container/docker_packages.rs | 17 +- core/archipelago/src/container/hooks.rs | 12 + core/archipelago/src/container/mod.rs | 6 +- .../src/container/prod_orchestrator.rs | 637 +++++++++++++++++- .../src/container/staged_update.rs | 314 +++++++++ core/archipelago/src/container/traits.rs | 5 + 10 files changed, 1206 insertions(+), 21 deletions(-) create mode 100644 core/archipelago/src/container/staged_update.rs diff --git a/core/archipelago/src/api/rpc/package/async_lifecycle.rs b/core/archipelago/src/api/rpc/package/async_lifecycle.rs index 80678e92..63ad037b 100644 --- a/core/archipelago/src/api/rpc/package/async_lifecycle.rs +++ b/core/archipelago/src/api/rpc/package/async_lifecycle.rs @@ -362,7 +362,11 @@ impl RpcHandler { set_package_state( &handler.state_manager, &package_id_spawn, - if result.get("status").and_then(|v| v.as_str()) == Some("up-to-date") { + if result.get("status").and_then(|v| v.as_str()) == Some("staged") { + PackageState::Stopped + } else if result.get("status").and_then(|v| v.as_str()) + == Some("up-to-date") + { pre_state.clone().unwrap_or(PackageState::Running) } else { PackageState::Running diff --git a/core/archipelago/src/api/rpc/package/config.rs b/core/archipelago/src/api/rpc/package/config.rs index 20cf592d..28586bfd 100644 --- a/core/archipelago/src/api/rpc/package/config.rs +++ b/core/archipelago/src/api/rpc/package/config.rs @@ -526,7 +526,18 @@ pub(in crate::api::rpc) async fn get_containers_for_app(package_id: &str) -> Res .await .context("podman ps timed out while listing containers")? .context("Failed to list containers")?; - let stdout = String::from_utf8_lossy(&output.stdout); + containers_from_list_output(package_id, &output) +} + +fn containers_from_list_output( + package_id: &str, + output: &std::process::Output, +) -> Result> { + anyhow::ensure!( + output.status.success(), + "podman ps failed while listing containers" + ); + let stdout = std::str::from_utf8(&output.stdout).context("Invalid container list response")?; let all: Vec<&str> = stdout.lines().filter(|s| !s.is_empty()).collect(); let patterns = all_container_names(package_id); @@ -543,6 +554,32 @@ pub(in crate::api::rpc) async fn get_containers_for_app(package_id: &str) -> Res mod tests { use super::{all_container_names, get_data_dirs_for_app, get_health_check_args}; + #[test] + fn failed_container_listing_is_not_an_absent_app() { + use std::os::unix::process::ExitStatusExt; + let mut output = std::process::Output { + status: std::process::ExitStatus::from_raw(1 << 8), + stdout: vec![], + stderr: b"store unavailable".to_vec(), + }; + assert!(super::containers_from_list_output("node-demo-music", &output).is_err()); + output.stdout = b"node-demo-music\n".to_vec(); + assert!(super::containers_from_list_output("node-demo-music", &output).is_err()); + output.status = std::process::ExitStatus::from_raw(0); + assert_eq!( + super::containers_from_list_output("node-demo-music", &output).unwrap(), + vec!["node-demo-music"] + ); + output.stdout.clear(); + assert!( + super::containers_from_list_output("node-demo-music", &output) + .unwrap() + .is_empty() + ); + output.stdout = vec![0xff]; + assert!(super::containers_from_list_output("node-demo-music", &output).is_err()); + } + #[test] fn bitcoin_variant_container_names_are_precise() { let core = all_container_names("bitcoin-core"); diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index ca081db9..9a46c547 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -126,7 +126,19 @@ impl RpcHandler { Err(e) => { tracing::error!("package.start {} failed: {:#}", package_id_owned, e); install_log(&format!("START FAIL: {} — {:#}", package_id_owned, e)).await; - if let Some(prev) = pre_state { + if e.downcast_ref::() + .is_some() + { + // Installed is neutral and scanner-owned. Restoring the + // prior Stopped state would conceal a failed cleanup; + // keeping Starting would prevent scanner convergence. + set_package_state( + &state_manager, + &package_id_owned, + PackageState::Installed, + ) + .await; + } else if let Some(prev) = pre_state { set_package_state(&state_manager, &package_id_owned, prev).await; } else { warn!( diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index f6daf30e..19957959 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -4,7 +4,7 @@ //! remove old container(s) → recreate (orchestrator-first, legacy fallback) → verify running. //! Data volumes are preserved (bind mounts, not stored in container). -use super::config::get_containers_for_app; +use super::config::{all_container_names, get_containers_for_app}; use super::install::install_log; use super::progress::parse_pull_progress; use super::runtime::stop_timeout_secs; @@ -13,6 +13,7 @@ use crate::api::rpc::RpcHandler; use crate::container::image_versions; use crate::data_model::{InstallPhase, PackageState}; use anyhow::{Context, Result}; +use std::{collections::HashSet, path::Path}; use tokio::io::{AsyncBufReadExt, BufReader}; use tracing::{error, info, warn}; @@ -60,9 +61,24 @@ impl RpcHandler { let targets = pinned .as_ref() .map(|target| self.resolve_images_to_pull(package_id, target)); + // A stopped Quadlet normally removes its --rm container. Absence is + // not an install decision: only a known single managed app with durable + // installed AND stopped evidence may enter the recreate path. + let known_managed = + if should_try_orchestrator_update(package_id, self.orchestrator.is_some()) { + self.orchestrator + .as_ref() + .expect("orchestrator presence checked") + .knows_app(orchestrator_update_app_id(package_id)) + .await + } else { + false + }; + let markers = UpdateMarkers::load(&self.config.data_dir, package_id).await?; + let installed = inspect_update_images(package_id).await?; + validate_update_presence(package_id, known_managed, !installed.is_empty(), &markers)?; if let Some(targets) = &targets { - let installed = inspect_update_images(package_id).await?; - if !update_targets_need_change(targets, &installed)? { + if !update_targets_need_change(targets, &installed)? && !markers.stopped { install_log(&format!( "UPDATE SKIP: {} — target versions already installed", package_id @@ -111,6 +127,19 @@ impl RpcHandler { if let Some(orchestrator) = self.orchestrator.as_ref() { match orchestrator.upgrade(orchestrator_app_id).await { Ok(()) => { + if let Some(image) = orchestrator + .staged_upgrade_image(orchestrator_app_id) + .await? + { + // The orchestrator proved the pinned image exists and + // persisted its reviewed manifest. No running container + // or healthy service is claimed for a stopped update. + self.clear_install_progress(package_id).await; + return Ok(serde_json::json!({ + "status": "staged", "state": "stopped", + "package_id": package_id, "image": image, + })); + } if let Some(targets) = &targets { verify_update_targets( targets, @@ -574,13 +603,69 @@ impl RpcHandler { } } -async fn inspect_update_images(package_id: &str) -> Result> { - let containers = get_containers_for_app(package_id).await?; +#[derive(Default)] +struct UpdateMarkers { + installed: bool, + stopped: bool, + uninstalled: bool, +} + +impl UpdateMarkers { + async fn load(data_dir: &Path, package_id: &str) -> Result { + let mut names = all_container_names(package_id); + names.push(package_id.to_string()); + names.push(orchestrator_update_app_id(package_id).to_string()); + let installed = read_update_markers(data_dir, "installed-apps.json").await?; + let stopped = read_update_markers(data_dir, "user-stopped.json").await?; + let uninstalled = read_update_markers(data_dir, "user-uninstalled.json").await?; + Ok(Self { + installed: names.iter().any(|name| installed.contains(name)), + stopped: names.iter().any(|name| stopped.contains(name)), + uninstalled: names.iter().any(|name| uninstalled.contains(name)), + }) + } +} + +// The recovery loaders intentionally return empty on malformed state. An update +// must instead distinguish unavailable evidence from a proven absence of an +// uninstall decision before it can recreate a removed Quadlet container. +async fn read_update_markers(data_dir: &Path, filename: &str) -> Result> { + match tokio::fs::read(data_dir.join(filename)).await { + Ok(bytes) => serde_json::from_slice(&bytes) + .with_context(|| format!("Cannot read {filename}; update cancelled")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(HashSet::new()), + Err(error) => { + Err(error).with_context(|| format!("Cannot read {filename}; update cancelled")) + } + } +} + +fn validate_update_presence( + package_id: &str, + known_managed: bool, + has_containers: bool, + markers: &UpdateMarkers, +) -> Result<()> { anyhow::ensure!( - !containers.is_empty(), - "No containers found for {}", + !markers.uninstalled, + "{} was uninstalled; use an explicit install instead of update", package_id ); + anyhow::ensure!( + has_containers || (known_managed && markers.installed && markers.stopped), + "No containers or durable stopped installation found for {}", + package_id + ); + Ok(()) +} + +async fn inspect_update_images(package_id: &str) -> Result> { + let containers = get_containers_for_app(package_id).await?; + if containers.is_empty() { + // The caller decides whether durable installation evidence authorizes + // a missing runtime. Never invoke bare `podman inspect` here. + return Ok(Vec::new()); + } let mut command = tokio::process::Command::new("podman"); command.arg("inspect").args(&containers).kill_on_drop(true); let output = tokio::time::timeout(std::time::Duration::from_secs(30), command.output()) @@ -626,6 +711,7 @@ fn verify_update_targets( targets: &[(String, String)], installed: &[(String, String)], ) -> Result<()> { + anyhow::ensure!(!targets.is_empty(), "No update targets resolved"); for (app_id, target) in targets { let running = installed_image_for_target(app_id, installed).ok_or_else(|| { anyhow::anyhow!("Update {}: target container missing after recreate", app_id) @@ -651,6 +737,7 @@ fn update_targets_need_change( installed: &[(String, String)], ) -> Result { use std::cmp::Ordering; + anyhow::ensure!(!targets.is_empty(), "No update targets resolved"); let mut changed = false; for (app_id, target) in targets { let running = installed_image_for_target(app_id, installed); @@ -771,9 +858,83 @@ mod tests { use super::{ candidate_app_ids_for_container, immutable_update_image, orchestrator_update_app_id, should_try_orchestrator_update, update_targets_need_change, uses_legacy_update_flow, - verify_update_targets, + validate_update_presence, verify_update_targets, UpdateMarkers, }; + #[tokio::test] + async fn stopped_installed_managed_app_updates_after_quadlet_container_disappears() { + let root = tempfile::tempdir().unwrap(); + crate::crash_recovery::mark_installed(root.path(), "node-demo-music").await; + crate::crash_recovery::mark_user_stopped(root.path(), "node-demo-music").await; + let markers = UpdateMarkers::load(root.path(), "node-demo-music") + .await + .unwrap(); + validate_update_presence("node-demo-music", true, false, &markers).unwrap(); + let target = vec![("node-demo-music".into(), "localhost/music:2".into())]; + assert!(super::update_targets_need_change(&target, &[]).unwrap()); + // Success must still prove that upgrade actually created the target. + assert!(verify_update_targets(&target, &[]).is_err()); + assert!(verify_update_targets(&target, &target).is_ok()); + assert!(crate::crash_recovery::load_user_stopped(root.path()) + .await + .contains("node-demo-music")); + } + + #[test] + fn absent_catalog_app_or_unmanaged_runtime_is_not_an_update_installation() { + for (managed, installed, stopped) in [ + (true, false, false), + (true, false, true), + (true, true, false), + (false, true, true), + ] { + let markers = UpdateMarkers { + installed, + stopped, + uninstalled: false, + }; + assert!(validate_update_presence("optional", managed, false, &markers).is_err()); + } + // Existing legacy containers remain updateable without modern markers. + assert!(validate_update_presence("legacy", false, true, &UpdateMarkers::default()).is_ok()); + assert!(super::update_targets_need_change(&[], &[]).is_err()); + assert!(verify_update_targets(&[], &[]).is_err()); + } + + #[tokio::test] + async fn uninstall_tombstone_beats_stale_installed_and_stopped_markers() { + let root = tempfile::tempdir().unwrap(); + crate::crash_recovery::mark_installed(root.path(), "node-demo-music").await; + crate::crash_recovery::mark_user_stopped(root.path(), "node-demo-music").await; + crate::crash_recovery::mark_user_uninstalled(root.path(), "archy-node-demo-music").await; + let markers = UpdateMarkers::load(root.path(), "node-demo-music") + .await + .unwrap(); + for present in [false, true] { + assert!(validate_update_presence("node-demo-music", true, present, &markers).is_err()); + } + } + + #[tokio::test] + async fn damaged_lifecycle_markers_cannot_authorize_recreation() { + let root = tempfile::tempdir().unwrap(); + let absent = UpdateMarkers::load(root.path(), "optional").await.unwrap(); + assert!(validate_update_presence("optional", true, false, &absent).is_err()); + for name in [ + "installed-apps.json", + "user-stopped.json", + "user-uninstalled.json", + ] { + tokio::fs::write(root.path().join(name), b"not-json") + .await + .unwrap(); + assert!(UpdateMarkers::load(root.path(), "optional").await.is_err()); + tokio::fs::remove_file(root.path().join(name)) + .await + .unwrap(); + } + } + #[tokio::test] async fn stack_image_failure_precedes_every_lifecycle_action() { use std::sync::{Arc, Mutex}; diff --git a/core/archipelago/src/container/docker_packages.rs b/core/archipelago/src/container/docker_packages.rs index 2b341b80..1fe2f1d0 100644 --- a/core/archipelago/src/container/docker_packages.rs +++ b/core/archipelago/src/container/docker_packages.rs @@ -298,7 +298,22 @@ impl DockerPackageScanner { uninstall_stage: None, }; - apply_manifest_presentation(&app_id, &mut package); + match super::staged_update::installed_manifest(data_dir, &app_id, &container.image) + .await + { + Ok(Some(manifest)) => { + apply_manifest_value(&serde_json::to_value(manifest)?, &mut package) + } + Ok(None) => apply_manifest_presentation(&app_id, &mut package), + Err(error) => { + tracing::warn!(app_id, error = %error, "Cannot verify retained installed manifest entry point"); + // Do not invent a launch path from a later catalog when its + // binding to the running version cannot be established. + if let Some(installed) = package.installed.as_mut() { + installed.interface_addresses.clear(); + } + } + } packages.insert(app_id.clone(), package); info!( "Detected container: {} ({})", diff --git a/core/archipelago/src/container/hooks.rs b/core/archipelago/src/container/hooks.rs index 4541792c..0ca32aa1 100644 --- a/core/archipelago/src/container/hooks.rs +++ b/core/archipelago/src/container/hooks.rs @@ -85,6 +85,18 @@ pub async fn run_post_install(manifest: &AppManifest, container_name: &str, data } } +/// Strict completion for an explicitly staged update. Keep the durable pin on failure. +pub(super) async fn run_post_install_strict( + manifest: &AppManifest, + container: &str, + data_dir: &Path, +) -> Result<()> { + for step in &manifest.app.hooks.post_install { + run_step(step, container, &manifest.app.id, data_dir).await?; + } + Ok(()) +} + async fn run_step(step: &HookStep, container: &str, app_id: &str, data_dir: &Path) -> Result<()> { match step { HookStep::Exec { exec } => { diff --git a/core/archipelago/src/container/mod.rs b/core/archipelago/src/container/mod.rs index 60a71a1a..0153fd0d 100644 --- a/core/archipelago/src/container/mod.rs +++ b/core/archipelago/src/container/mod.rs @@ -1,5 +1,4 @@ pub mod app_catalog; -pub mod node_catalog; pub mod app_gate_config; pub mod bitcoin_ui; pub mod boot_reconciler; @@ -14,11 +13,12 @@ pub mod image_policy; pub mod image_versions; pub mod lnd; pub mod migration_backup; +pub mod node_catalog; pub mod npm; pub mod prod_orchestrator; pub mod quadlet; -pub mod registry; pub mod registration_pin; +pub mod registry; pub mod secrets; pub mod traits; pub mod ui_detection; @@ -29,3 +29,5 @@ pub use dev_orchestrator::DevContainerOrchestrator; pub use docker_packages::DockerPackageScanner; pub use prod_orchestrator::ProdContainerOrchestrator; pub use traits::ContainerOrchestrator; + +mod staged_update; diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 2dfcbbbc..2c547f16 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -1422,6 +1422,17 @@ fn host_port_collisions<'m>( out } +/// A staged start failed and its runtime could not be proven stopped. Callers +/// must not restore a cached Stopped state; the scanner/reconciler owns recovery. +#[derive(Debug)] +pub(crate) struct StagedCleanupFailure(String); +impl std::fmt::Display for StagedCleanupFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} +impl std::error::Error for StagedCleanupFailure {} + /// Internal: track a manifest together with the absolute directory it was loaded /// from, so Build sources can resolve relative `context:` paths. #[derive(Debug, Clone)] @@ -2043,7 +2054,9 @@ impl ProdContainerOrchestrator { } } } - let mut report = ReconcileReport::default(); + // Stopped/disabled apps are excluded from ordinary start reconciliation. + // Pending cleanup must still run for them, including after a restart. + let mut report = self.reconcile_staged_updates().await; let disk_gb = self.disk_gb().await; let bitcoin_pruned = disk_gb < ARCHIVAL_BITCOIN_DISK_GB || crate::settings::bitcoin_storage::load(&self.data_dir) @@ -2336,6 +2349,12 @@ impl ProdContainerOrchestrator { let lock = self.app_lock(&app_id).await; let _guard = lock.lock().await; + if let Some(record) = super::staged_update::load(&self.data_dir, &app_id).await? { + let name = compute_container_name(&record.manifest); + self.stop_staged_runtime(&name).await?; + return Ok(ReconcileAction::Left("staged-update-awaiting-start".into())); + } + self.ensure_app_secrets(&app_id).await?; // Don't fight the Bitcoin-implementation switch: bitcoin-core and @@ -2849,11 +2868,171 @@ impl ProdContainerOrchestrator { } /// Build-or-pull, create, start. Assumes the per-app mutex is already held. + async fn reconcile_staged_updates(&self) -> ReconcileReport { + let mut report = ReconcileReport::default(); + let ids = match super::staged_update::pending_ids(&self.data_dir).await { + Ok(ids) => ids, + Err(error) => { + report + .failures + .push(("staged-updates".into(), error.to_string())); + return report; + } + }; + for app_id in ids { + if crate::app_ops::lifecycle_op_in_flight(&app_id) { + continue; + } + let lock = self.app_lock(&app_id).await; + let _guard = lock.lock().await; + // Re-read after acquiring the lock: a successful explicit start + // may have completed and cleared the stage while this pass waited. + let result = async { + if let Some(record) = super::staged_update::load(&self.data_dir, &app_id).await? { + self.stop_staged_runtime(&compute_container_name(&record.manifest)) + .await?; + report.record( + &app_id, + ReconcileAction::Left("staged-update-awaiting-start".into()), + ); + } + Ok::<_, anyhow::Error>(()) + } + .await; + if let Err(error) = result { + report.failures.push((app_id, error.to_string())); + } + } + report + } + + async fn stop_staged_runtime(&self, name: &str) -> Result<()> { + let mut errors = Vec::new(); + if let Err(e) = self.remove_quadlet_unit_if_present(name).await { + errors.push(format!("could not disable staged unit: {e:#}")); + } + let inventory = self + .runtime + .list_containers() + .await + .context("Cannot verify staged runtime inventory")?; + if inventory.iter().any(|c| { + c.name.trim_start_matches('/') == name + && !matches!( + c.state, + ContainerState::Stopped | ContainerState::Exited | ContainerState::Created + ) + }) { + if let Err(e) = self.runtime.stop_container(name).await { + errors.push(format!("could not stop staged container: {e:#}")); + } + } + let inventory = self + .runtime + .list_containers() + .await + .context("Cannot verify staged runtime stopped")?; + if inventory.iter().any(|c| { + c.name.trim_start_matches('/') == name + && !matches!( + c.state, + ContainerState::Stopped | ContainerState::Exited | ContainerState::Created + ) + }) { + errors.push("staged container is still active".into()); + } + anyhow::ensure!(errors.is_empty(), "{}", errors.join("; ")); + Ok(()) + } + + async fn start_staged_update(&self, app_id: &str) -> Result { + use super::staged_update; + if staged_update::load(&self.data_dir, app_id).await?.is_none() { + return Ok(false); + } + let lock = self.app_lock(app_id).await; + let _guard = lock.lock().await; + let record = staged_update::load(&self.data_dir, app_id) + .await? + .context("Staged update changed; retry start")?; + let name = compute_container_name(&record.manifest); + anyhow::ensure!( + !staged_update::marked(&self.data_dir, "user-uninstalled.json", app_id, &name).await?, + "App was uninstalled; staged update cannot reinstall it" + ); + anyhow::ensure!( + record.ready, + "Update preparation was interrupted; retry update before start" + ); + anyhow::ensure!( + self.runtime.image_exists(record.image()?).await?, + "Staged image is unavailable; refusing to start another version" + ); + // Clear only this explicit start's stopped intent. The pending manifest + // remains durable until all creation, hooks and readiness checks succeed. + crate::crash_recovery::clear_user_stopped(&self.data_dir, app_id).await; + crate::crash_recovery::clear_user_stopped(&self.data_dir, &name).await; + crate::crash_recovery::clear_user_stopped(&self.data_dir, &format!("archy-{app_id}")).await; + self.state.write().await.disabled.remove(app_id); + let lm = LoadedManifest { + manifest: record.manifest.clone(), + manifest_dir: record.manifest_dir.clone(), + }; + let result = async { + self.remove_quadlet_unit_if_present(&name).await?; + if self + .runtime + .list_containers() + .await? + .iter() + .any(|c| c.name.trim_start_matches('/') == name) + { + self.runtime.stop_container(&name).await?; + self.runtime.remove_container(&name).await?; + } + self.install_fresh_with_pin(&lm, true).await?; + let observed = self.runtime.get_container_status(&name).await?; + anyhow::ensure!( + observed.state == ContainerState::Running, + "Pinned start did not become running" + ); + anyhow::ensure!( + staged_update::matches_image(record.image()?, &observed.image), + "Pinned start produced a different image" + ); + // Installed presentation is durable before pending intent disappears. + // A crash on either side of clear cannot fall back to a later catalog. + staged_update::save_installed(&self.data_dir, &record).await?; + staged_update::clear(&self.data_dir, app_id).await + } + .await; + if let Err(error) = result { + crate::crash_recovery::mark_user_stopped(&self.data_dir, app_id).await; + self.state.write().await.disabled.insert(app_id.into()); + let retained = staged_update::save(&self.data_dir, &record).await; + let cleanup = self.stop_staged_runtime(&name).await; + if let Err(cleanup) = cleanup { + return Err(StagedCleanupFailure(format!( + "Staged start failed: {error:#}; cleanup failed: {cleanup:#}; pin persistence: {:?}", retained.err() + )).into()); + } + retained.context("Could not retain failed staged start pin")?; + return Err(error); + } + Ok(true) + } + async fn install_fresh(&self, lm: &LoadedManifest) -> Result<()> { + self.install_fresh_with_pin(lm, false).await + } + + async fn install_fresh_with_pin(&self, lm: &LoadedManifest, pinned: bool) -> Result<()> { self.ensure_app_secrets(&lm.manifest.app.id).await?; let mut resolved_manifest = lm.manifest.clone(); self.resolve_dynamic_env(&mut resolved_manifest).await?; - resolve_catalog_image(&mut resolved_manifest); + if !pinned { + resolve_catalog_image(&mut resolved_manifest); + } let resolved = resolved_manifest.app.container.resolve().ok_or_else(|| { anyhow::anyhow!( @@ -3008,7 +3187,17 @@ impl ProdContainerOrchestrator { // freshly created container — exactly when container mutations (e.g. // indeedhub's nginx X-Frame-Options strip + nostr-provider injection) must // be re-applied. Best-effort + idempotent: never fails the install. - crate::container::hooks::run_post_install(&resolved_manifest, &name, &self.data_dir).await; + if pinned { + crate::container::hooks::run_post_install_strict( + &resolved_manifest, + &name, + &self.data_dir, + ) + .await?; + } else { + crate::container::hooks::run_post_install(&resolved_manifest, &name, &self.data_dir) + .await; + } if uses_pasta_network(&resolved_manifest) { if let Err(err) = wait_for_manifest_host_ports( &resolved_manifest, @@ -4825,6 +5014,8 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { } async fn install(&self, app_id: &str) -> Result { + anyhow::ensure!(super::staged_update::load(&self.data_dir, app_id).await?.is_none(), + "An update is staged; use explicit start, or uninstall before selecting another installation"); let lm = self.loaded(app_id).await?; // Optional shared-service preconditions are checked before recording // installation or creating anything. A headless adapter must not claim @@ -4896,7 +5087,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { // point). Just delegate. self.prepare_for_start(&lm.manifest).await?; let action = self.ensure_running(&lm).await?; - match action { + let result = match action { ReconcileAction::NoOp | ReconcileAction::Started | ReconcileAction::Installed => { Ok(name) } @@ -4915,10 +5106,17 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { self.install_fresh(&lm).await?; Ok(name) } + }; + if result.is_ok() { + super::staged_update::clear_installed(&self.data_dir, app_id).await?; } + result } async fn start(&self, app_id: &str) -> Result<()> { + if self.start_staged_update(app_id).await? { + return Ok(()); + } if let Some(members) = self.mempool_umbrella_members(app_id).await { tracing::info!( app_id, @@ -5136,6 +5334,8 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { self.remove_quadlet_unit_if_present(&name).await?; } self.state.write().await.disabled.insert(app_id.to_string()); + super::staged_update::clear(&self.data_dir, app_id).await?; + super::staged_update::clear_installed(&self.data_dir, app_id).await?; return Ok(()); } let lm = self.loaded(app_id).await?; @@ -5189,15 +5389,83 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { // stale claim behind would let desired-state recovery recreate the very // app that was just uninstalled. crate::crash_recovery::clear_installed(&self.data_dir, app_id).await; + super::staged_update::clear(&self.data_dir, app_id).await?; + super::staged_update::clear_installed(&self.data_dir, app_id).await?; Ok(()) } /// Upgrade: stop-remove-reinstall (re-pulls or rebuilds as required). async fn upgrade(&self, app_id: &str) -> Result<()> { - let lm = self.loaded(app_id).await?; + use super::staged_update; let lock = self.app_lock(app_id).await; let _guard = lock.lock().await; + let existing = staged_update::load(&self.data_dir, app_id).await?; + let lm = if let Some(record) = &existing { + LoadedManifest { + manifest: record.manifest.clone(), + manifest_dir: record.manifest_dir.clone(), + } + } else { + self.loaded(app_id).await? + }; let name = compute_container_name(&lm.manifest); + anyhow::ensure!( + !staged_update::marked(&self.data_dir, "user-uninstalled.json", app_id, &name).await?, + "App was uninstalled; update cannot reinstall it" + ); + let stopped = + staged_update::marked(&self.data_dir, "user-stopped.json", app_id, &name).await?; + if stopped || existing.is_some() { + anyhow::ensure!( + staged_update::marked(&self.data_dir, "installed-apps.json", app_id, &name).await?, + "Stopped update needs durable installation evidence" + ); + let mut record = match existing { + Some(record) => record, + None => { + let mut manifest = lm.manifest.clone(); + resolve_catalog_image(&mut manifest); + staged_update::StagedUpdate { + manifest, + manifest_dir: lm.manifest_dir.clone(), + ready: false, + } + } + }; + record.image()?; + crate::crash_recovery::mark_user_stopped(&self.data_dir, app_id).await; + anyhow::ensure!( + staged_update::marked(&self.data_dir, "user-stopped.json", app_id, &name).await?, + "Could not persist stopped update intent" + ); + // Freeze the approved manifest before any pull. Retries cannot + // silently switch to another catalog revision after an interruption. + record.ready = false; + staged_update::save(&self.data_dir, &record).await?; + self.remove_quadlet_unit_if_present(&name).await?; + if self + .runtime + .list_containers() + .await? + .iter() + .any(|c| c.name.trim_start_matches('/') == name) + { + self.runtime.stop_container(&name).await?; + self.runtime.remove_container(&name).await?; + } + let pinned = LoadedManifest { + manifest: record.manifest.clone(), + manifest_dir: record.manifest_dir.clone(), + }; + self.ensure_resolved_source_available(&pinned).await?; + anyhow::ensure!( + self.runtime.image_exists(record.image()?).await?, + "Staged image unavailable after preparation" + ); + record.ready = true; + staged_update::save(&self.data_dir, &record).await?; + return Ok(()); + } let mut resolved = lm.manifest.clone(); resolve_catalog_image(&mut resolved); if resolved.app.container.build.is_none() { @@ -5213,7 +5481,10 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { running.image, target ), - Some(std::cmp::Ordering::Equal) => return Ok(()), + Some(std::cmp::Ordering::Equal) => { + staged_update::clear_installed(&self.data_dir, app_id).await?; + return Ok(()); + } _ => {} } } @@ -5221,7 +5492,22 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { } let _ = self.runtime.stop_container(&name).await; let _ = self.runtime.remove_container(&name).await; - self.install_fresh(&lm).await + self.install_fresh(&lm).await?; + staged_update::clear_installed(&self.data_dir, app_id).await + } + + async fn staged_upgrade_image(&self, app_id: &str) -> Result> { + match super::staged_update::load(&self.data_dir, app_id).await? { + Some(record) => { + anyhow::ensure!(record.ready, "Staged update is not ready"); + anyhow::ensure!( + self.runtime.image_exists(record.image()?).await?, + "Staged image unavailable" + ); + Ok(Some(record.image()?.to_string())) + } + None => Ok(None), + } } async fn status(&self, app_id: &str) -> Result { @@ -5778,6 +6064,7 @@ mod tests { fail_image_exists: StdMutex>, /// If set, `start_container` for this container fails with this message. fail_start: StdMutex>, + fail_stop: StdMutex>, } impl MockRuntime { @@ -5841,6 +6128,12 @@ mod tests { ) -> Result { self.record(format!("create_container:{name}:offset={port_offset}")); self.set_state(name, ContainerState::Created); + if let Some(image) = manifest.app.container.image_ref() { + self.running_images + .lock() + .unwrap() + .insert(name.into(), image); + } self.created_env .lock() .unwrap() @@ -5862,6 +6155,9 @@ mod tests { } async fn stop_container(&self, name: &str) -> Result<()> { self.record(format!("stop_container:{name}")); + if let Some(error) = self.fail_stop.lock().unwrap().remove(name) { + return Err(anyhow::anyhow!(error)); + } self.set_state(name, ContainerState::Stopped); Ok(()) } @@ -6310,6 +6606,333 @@ app: } } + async fn stopped_update_fixture() -> (Arc, ProdContainerOrchestrator, String) { + let rt = Arc::new(MockRuntime::default()); + let orch = orch_with(rt.clone()).await; + let image = format!("docker.io/library/alpine@sha256:{}", "a".repeat(64)); + let mut manifest = pull_manifest("staged-music", &image); + manifest + .app + .environment + .push("STAGED_VERSION=original".into()); + orch.insert_manifest_for_test(manifest, PathBuf::from("/tmp")) + .await; + crate::crash_recovery::mark_installed(&orch.data_dir, "staged-music").await; + crate::crash_recovery::mark_user_stopped(&orch.data_dir, "staged-music").await; + (rt, orch, image) + } + + #[tokio::test] + async fn stopped_upgrade_stages_without_start_then_starts_pinned_manifest() { + let (rt, orch, image) = stopped_update_fixture().await; + let sentinel = orch.data_dir.join("persistent-song-and-secret-fixture"); + tokio::fs::write(&sentinel, b"preserved").await.unwrap(); + orch.upgrade("staged-music").await.unwrap(); + assert_eq!( + orch.staged_upgrade_image("staged-music").await.unwrap(), + Some(image) + ); + assert!(!rt + .calls() + .iter() + .any(|c| c.starts_with("create_container:") || c.starts_with("start_container:"))); + let replacement = pull_manifest( + "staged-music", + &format!("docker.io/library/alpine@sha256:{}", "b".repeat(64)), + ); + orch.insert_manifest_for_test(replacement, PathBuf::from("/tmp/new-catalog")) + .await; + let lm = orch.loaded("staged-music").await.unwrap(); + assert_eq!( + orch.ensure_running(&lm).await.unwrap(), + ReconcileAction::Left("staged-update-awaiting-start".into()) + ); + orch.start("staged-music").await.unwrap(); + assert!(rt + .created_env_for("staged-music") + .contains(&"STAGED_VERSION=original".into())); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Running + ); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_none()); + assert!(!crate::crash_recovery::load_user_stopped(&orch.data_dir) + .await + .contains("staged-music")); + assert_eq!(tokio::fs::read(sentinel).await.unwrap(), b"preserved"); + } + + #[tokio::test] + async fn interrupted_stopped_pull_retains_original_pin_and_requires_retry() { + let (rt, orch, image) = stopped_update_fixture().await; + *rt.fail_pull.lock().unwrap() = Some("interrupted pull".into()); + assert!(orch.upgrade("staged-music").await.is_err()); + assert!(orch + .start("staged-music") + .await + .unwrap_err() + .to_string() + .contains("interrupted")); + let record = super::super::staged_update::load(&orch.data_dir, "staged-music") + .await + .unwrap() + .unwrap(); + assert!(!record.ready); + assert_eq!(record.image().unwrap(), image); + *rt.fail_pull.lock().unwrap() = None; + orch.insert_manifest_for_test( + pull_manifest("staged-music", "docker.io/library/alpine:latest"), + PathBuf::from("/tmp"), + ) + .await; + orch.upgrade("staged-music").await.unwrap(); + assert_eq!( + orch.staged_upgrade_image("staged-music").await.unwrap(), + Some(image) + ); + assert!(!rt.calls().iter().any(|c| c.starts_with("start_container:"))); + } + + #[tokio::test] + async fn failed_staged_start_preserves_pin_and_stop_until_retry_succeeds() { + let (rt, orch, _) = stopped_update_fixture().await; + orch.upgrade("staged-music").await.unwrap(); + rt.fail_start + .lock() + .unwrap() + .insert("staged-music".into(), "injected start failure".into()); + assert!(orch.start("staged-music").await.is_err()); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_some()); + assert!(crate::crash_recovery::load_user_stopped(&orch.data_dir) + .await + .contains("staged-music")); + assert!(matches!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Stopped | ContainerState::Created + )); + orch.start("staged-music").await.unwrap(); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn staged_missing_image_uninstall_and_damaged_record_refuse_start() { + let (rt, orch, image) = stopped_update_fixture().await; + orch.upgrade("staged-music").await.unwrap(); + rt.images.lock().unwrap().remove(&image); + assert!(orch + .start("staged-music") + .await + .unwrap_err() + .to_string() + .contains("unavailable")); + rt.mark_image_present(&image); + crate::crash_recovery::mark_user_uninstalled(&orch.data_dir, "staged-music").await; + assert!(orch + .start("staged-music") + .await + .unwrap_err() + .to_string() + .contains("uninstalled")); + assert!(orch.upgrade("staged-music").await.is_err()); + tokio::fs::write( + orch.data_dir.join("staged-updates/staged-music.json"), + b"damaged", + ) + .await + .unwrap(); + assert!(orch.start("staged-music").await.is_err()); + assert!(!rt.calls().iter().any(|c| c.starts_with("start_container:"))); + } + + #[tokio::test] + async fn staged_failed_post_install_hook_keeps_pin_and_stops_container() { + let (rt, orch, _) = stopped_update_fixture().await; + let mut lm = orch.loaded("staged-music").await.unwrap(); + lm.manifest.app.hooks.post_install = serde_yaml::from_str( + "- copy_from_host:\n src: nonexistent-staged-hook-file\n dest: /tmp/test\n", + ) + .unwrap(); + orch.insert_manifest_for_test(lm.manifest, lm.manifest_dir) + .await; + orch.upgrade("staged-music").await.unwrap(); + assert!(orch.start("staged-music").await.is_err()); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_some()); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Stopped + ); + assert!(crate::crash_recovery::load_user_stopped(&orch.data_dir) + .await + .contains("staged-music")); + } + + #[tokio::test] + async fn failed_staged_cleanup_reports_active_runtime_and_reconcile_retries_stop() { + let (rt, orch, _) = stopped_update_fixture().await; + let mut lm = orch.loaded("staged-music").await.unwrap(); + lm.manifest.app.hooks.post_install = serde_yaml::from_str( + "- copy_from_host:\n src: nonexistent-staged-hook-file\n dest: /tmp/test\n", + ) + .unwrap(); + orch.insert_manifest_for_test(lm.manifest, lm.manifest_dir) + .await; + orch.upgrade("staged-music").await.unwrap(); + rt.fail_stop + .lock() + .unwrap() + .insert("staged-music".into(), "stop refused".into()); + let error = orch.start("staged-music").await.unwrap_err(); + assert!(error.downcast_ref::().is_some()); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Running + ); + let report = orch.reconcile_all().await; + assert!(report.failures.is_empty()); + assert!(report + .actions + .iter() + .any(|(app, action)| app == "staged-music" + && action == &ReconcileAction::Left("staged-update-awaiting-start".into()))); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Stopped + ); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_some()); + } + + #[tokio::test] + async fn pending_stage_enforces_stop_after_restart_when_stop_marker_cannot_be_saved() { + let (rt, orch, _) = stopped_update_fixture().await; + let mut lm = orch.loaded("staged-music").await.unwrap(); + lm.manifest.app.hooks.post_install = serde_yaml::from_str( + "- copy_from_host:\n src: nonexistent-staged-hook-file\n dest: /tmp/test\n", + ) + .unwrap(); + orch.insert_manifest_for_test(lm.manifest.clone(), lm.manifest_dir.clone()) + .await; + orch.upgrade("staged-music").await.unwrap(); + let marker = orch.data_dir.join("user-stopped.json"); + tokio::fs::remove_file(&marker).await.unwrap(); + tokio::fs::create_dir(&marker).await.unwrap(); + rt.fail_stop + .lock() + .unwrap() + .insert("staged-music".into(), "stop interrupted".into()); + let failure = orch.start("staged-music").await.unwrap_err(); + assert!(failure.downcast_ref::().is_some()); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Running + ); + let starts = rt + .calls() + .iter() + .filter(|c| c.starts_with("start_container:")) + .count(); + let mut resumed = orch_with(rt.clone()).await; + resumed.set_data_dir(orch.data_dir.clone()); + resumed + .insert_manifest_for_test(lm.manifest, lm.manifest_dir) + .await; + let report = resumed.reconcile_all().await; + assert!(report.failures.is_empty()); + assert_eq!( + rt.get_container_status("staged-music").await.unwrap().state, + ContainerState::Stopped + ); + assert_eq!( + rt.calls() + .iter() + .filter(|c| c.starts_with("start_container:")) + .count(), + starts + ); + assert!(resumed + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_some()); + } + + #[tokio::test] + async fn staged_configuration_failure_keeps_stop_and_pin_before_create() { + let (rt, mut orch, _) = stopped_update_fixture().await; + orch.secrets_dir = orch.data_dir.join("fixture-secrets"); + let mut lm = orch.loaded("staged-music").await.unwrap(); + lm.manifest.app.container.secret_env = + serde_yaml::from_str("- key: REQUIRED_SECRET\n secret_file: missing-staged-secret\n") + .unwrap(); + orch.insert_manifest_for_test(lm.manifest, lm.manifest_dir) + .await; + orch.upgrade("staged-music").await.unwrap(); + assert!(orch.start("staged-music").await.is_err()); + assert!(orch + .staged_upgrade_image("staged-music") + .await + .unwrap() + .is_some()); + assert!(crate::crash_recovery::load_user_stopped(&orch.data_dir) + .await + .contains("staged-music")); + assert!(!rt + .calls() + .iter() + .any(|c| c.starts_with("create_container:"))); + } + + #[tokio::test] + async fn staged_record_persistence_failure_precedes_runtime_changes() { + let (rt, orch, _) = stopped_update_fixture().await; + tokio::fs::write(orch.data_dir.join("staged-updates"), b"blocked directory") + .await + .unwrap(); + assert!(orch.upgrade("staged-music").await.is_err()); + assert!(!rt.calls().iter().any(|c| c.starts_with("pull_image:") + || c.starts_with("stop_container:") + || c.starts_with("remove_container:") + || c.starts_with("create_container:"))); + } + + #[tokio::test] + async fn stopped_unpinned_update_refuses_without_starting() { + let (rt, orch, _) = stopped_update_fixture().await; + orch.insert_manifest_for_test( + pull_manifest("staged-music", "docker.io/library/alpine:3.20"), + PathBuf::from("/tmp"), + ) + .await; + assert!(orch + .upgrade("staged-music") + .await + .unwrap_err() + .to_string() + .contains("digest-pinned")); + assert!(!rt.calls().iter().any(|c| c.starts_with("pull_image:") + || c.starts_with("create_container:") + || c.starts_with("start_container:"))); + } + #[tokio::test] async fn install_fresh_pull() { let rt = Arc::new(MockRuntime::default()); diff --git a/core/archipelago/src/container/staged_update.rs b/core/archipelago/src/container/staged_update.rs new file mode 100644 index 00000000..f0e022ff --- /dev/null +++ b/core/archipelago/src/container/staged_update.rs @@ -0,0 +1,314 @@ +//! Durable, immutable preparation for updates that must remain stopped. +use anyhow::{Context, Result}; +use archipelago_container::AppManifest; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct StagedUpdate { + pub manifest: AppManifest, + pub manifest_dir: PathBuf, + pub ready: bool, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: u32, + payload: String, + checksum: String, +} + +impl StagedUpdate { + pub fn image(&self) -> Result<&str> { + anyhow::ensure!( + self.manifest.app.container.build.is_none(), + "Stopped updates of build-based apps require immutable image resolution" + ); + let image = self + .manifest + .app + .container + .image + .as_deref() + .context("Missing staged image")?; + let digest = image.rsplit_once("@sha256:").map(|(_, d)| d); + anyhow::ensure!( + digest.is_some_and(|d| d.len() == 64 && d.bytes().all(|b| b.is_ascii_hexdigit())), + "Stopped update requires a digest-pinned image; app remains stopped" + ); + Ok(image) + } +} + +fn record_path(root: &Path, directory: &str, app: &str) -> Result { + anyhow::ensure!( + !app.is_empty() + && app.len() <= 128 + && app + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')), + "Invalid staged app id" + ); + Ok(root.join(directory).join(format!("{app}.json"))) +} + +fn path(root: &Path, app: &str) -> Result { + record_path(root, "staged-updates", app) +} + +pub(super) async fn load(root: &Path, app: &str) -> Result> { + load_at(root, "staged-updates", app).await +} + +async fn load_at(root: &Path, directory: &str, app: &str) -> Result> { + let bytes = match tokio::fs::read(record_path(root, directory, app)?).await { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + let envelope: Envelope = serde_json::from_slice(&bytes) + .context("Damaged staged update; refusing another version")?; + anyhow::ensure!(envelope.schema == 1, "Unsupported staged update schema"); + anyhow::ensure!( + envelope.checksum == hex::encode(Sha256::digest(envelope.payload.as_bytes())), + "Staged update checksum mismatch" + ); + let record: StagedUpdate = + serde_json::from_str(&envelope.payload).context("Damaged staged manifest")?; + record + .manifest + .validate() + .context("Invalid staged manifest")?; + anyhow::ensure!( + record.manifest.app.id == app, + "Staged update belongs to another app" + ); + record.image()?; + Ok(Some(record)) +} + +pub(super) async fn save(root: &Path, record: &StagedUpdate) -> Result<()> { + save_at(root, "staged-updates", record).await +} + +pub(super) async fn save_installed(root: &Path, record: &StagedUpdate) -> Result<()> { + anyhow::ensure!(record.ready, "Cannot publish incomplete installed manifest"); + save_at(root, "installed-manifests", record).await +} + +pub(super) async fn installed_manifest( + root: &Path, + app: &str, + observed_image: &str, +) -> Result> { + let Some(record) = load_at(root, "installed-manifests", app).await? else { + return Ok(None); + }; + let expected = record.image()?; + let matches = matches_image(expected, observed_image); + Ok((record.ready && matches).then_some(record.manifest)) +} + +pub(super) fn matches_image(expected: &str, observed: &str) -> bool { + expected == observed + || match ( + expected.rsplit_once("@sha256:"), + observed.rsplit_once("@sha256:"), + ) { + (Some((_, a)), Some((_, b))) => a.len() == 64 && a.eq_ignore_ascii_case(b), + _ => false, + } +} + +async fn save_at(root: &Path, directory: &str, record: &StagedUpdate) -> Result<()> { + use std::{ + io::Write, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + }; + record.image()?; + let target = record_path(root, directory, &record.manifest.app.id)?; + record + .manifest + .validate() + .context("Invalid staged manifest")?; + let payload = serde_json::to_string(record)?; + let checksum = hex::encode(Sha256::digest(payload.as_bytes())); + let bytes = serde_json::to_vec(&Envelope { + schema: 1, + payload, + checksum, + })?; + // Bounded metadata commit stays in this poll while the caller owns its + // lifecycle lock; cancellation cannot leave a late detached writer. + (|| -> Result<()> { + let parent = target.parent().context("Missing staged update directory")?; + match std::fs::DirBuilder::new().mode(0o700).create(parent) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))?; + // Persist the directory entry itself before an acknowledged record can + // depend on it surviving a crash (syncing the child alone is insufficient). + std::fs::File::open(parent.parent().context("Missing staged update parent")?)? + .sync_all()?; + let tmp = parent.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let result = (|| -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&tmp)?; + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&tmp, &target)?; + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(tmp); + } + result + })() +} + +pub(super) async fn clear(root: &Path, app: &str) -> Result<()> { + clear_at(root, "staged-updates", app).await +} + +pub(super) async fn clear_installed(root: &Path, app: &str) -> Result<()> { + clear_at(root, "installed-manifests", app).await +} + +async fn clear_at(root: &Path, directory: &str, app: &str) -> Result<()> { + let target = record_path(root, directory, app)?; + // Bounded metadata commit stays in this poll while the caller owns its + // lifecycle lock; cancellation cannot leave a late detached writer. + (|| -> Result<()> { + match std::fs::remove_file(&target) { + Ok(()) => std::fs::File::open(target.parent().unwrap())?.sync_all()?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + Ok(()) + })() +} + +pub(super) async fn marked(root: &Path, file: &str, app: &str, container: &str) -> Result { + let bytes = match tokio::fs::read(root.join(file)).await { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e).with_context(|| format!("Cannot read {file}")), + }; + let set: HashSet = + serde_json::from_slice(&bytes).with_context(|| format!("Damaged {file}"))?; + Ok(set.contains(app) || set.contains(container) || set.contains(&format!("archy-{app}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::FutureExt; + use std::os::unix::fs::PermissionsExt; + + #[tokio::test] + async fn envelope_rejects_corruption_schema_and_invalid_manifest_and_is_private() { + let root = tempfile::tempdir().unwrap(); + let manifest = AppManifest::parse(&format!("app:\n id: music\n name: Music\n version: 1.0.0\n container:\n image: docker.io/library/alpine@sha256:{}\n", "a".repeat(64))).unwrap(); + let record = StagedUpdate { + manifest, + manifest_dir: root.path().into(), + ready: true, + }; + save(root.path(), &record) + .now_or_never() + .expect("Journal commit must finish before lifecycle lock cancellation is possible") + .unwrap(); + let target = path(root.path(), "music").unwrap(); + assert_eq!( + std::fs::metadata(target.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + std::fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert!(load(root.path(), "music").await.unwrap().unwrap().ready); + save_installed(root.path(), &record).await.unwrap(); + clear(root.path(), "music") + .now_or_never() + .expect("Journal clear must not leave a detached late deletion") + .unwrap(); + assert!(load(root.path(), "music").await.unwrap().is_none()); + assert!( + installed_manifest(root.path(), "music", record.image().unwrap()) + .await + .unwrap() + .is_some() + ); + assert!(installed_manifest( + root.path(), + "music", + &format!("docker.io/library/alpine@sha256:{}", "b".repeat(64)) + ) + .await + .unwrap() + .is_none()); + save(root.path(), &record).await.unwrap(); + let original = tokio::fs::read(&target).await.unwrap(); + let mut envelope: Envelope = serde_json::from_slice(&original).unwrap(); + envelope.payload = envelope.payload.replace("Music", "Changed Music"); + tokio::fs::write(&target, serde_json::to_vec(&envelope).unwrap()) + .await + .unwrap(); + assert!(load(root.path(), "music") + .await + .unwrap_err() + .to_string() + .contains("checksum")); + envelope = serde_json::from_slice(&original).unwrap(); + envelope.schema = 2; + tokio::fs::write(&target, serde_json::to_vec(&envelope).unwrap()) + .await + .unwrap(); + assert!(load(root.path(), "music").await.is_err()); + envelope = serde_json::from_slice(&original).unwrap(); + let mut value: serde_json::Value = serde_json::from_str(&envelope.payload).unwrap(); + value["manifest"]["app"]["container"]["image"] = serde_json::Value::Null; + envelope.payload = serde_json::to_string(&value).unwrap(); + envelope.checksum = hex::encode(Sha256::digest(envelope.payload.as_bytes())); + tokio::fs::write(&target, serde_json::to_vec(&envelope).unwrap()) + .await + .unwrap(); + assert!(load(root.path(), "music").await.is_err()); + } +} + +pub(super) async fn pending_ids(root: &Path) -> Result> { + let mut dir = match tokio::fs::read_dir(root.join("staged-updates")).await { + Ok(dir) => dir, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]), + Err(e) => return Err(e.into()), + }; + let mut ids = Vec::new(); + while let Some(entry) = dir.next_entry().await? { + if let Some(name) = entry + .file_name() + .to_str() + .and_then(|name| name.strip_suffix(".json")) + { + ids.push(name.to_string()); + } + } + Ok(ids) +} diff --git a/core/archipelago/src/container/traits.rs b/core/archipelago/src/container/traits.rs index 1a71a0a0..a681f0c1 100644 --- a/core/archipelago/src/container/traits.rs +++ b/core/archipelago/src/container/traits.rs @@ -62,6 +62,11 @@ pub trait ContainerOrchestrator: Send + Sync { /// Pull/rebuild the image and recreate the container from scratch. async fn upgrade(&self, app_id: &str) -> Result<()>; + /// Exact image prepared by a stopped upgrade; no running container is claimed. + async fn staged_upgrade_image(&self, _app_id: &str) -> Result> { + Ok(None) + } + /// Current state of a single container. async fn status(&self, app_id: &str) -> Result; From 45579e53c7e78d19f61927f89a0e255dcb9bb79a Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 01:24:50 -0400 Subject: [PATCH 02/13] Draft retained-container update journal and original-runtime recovery --- .../src/api/rpc/package/install.rs | 8 +- .../src/api/rpc/package/runtime.rs | 15 + .../archipelago/src/api/rpc/package/update.rs | 163 +--- core/archipelago/src/container/mod.rs | 2 + .../src/container/prod_orchestrator.rs | 17 + .../src/container/update_transaction.rs | 879 ++++++++++++++++++ 6 files changed, 944 insertions(+), 140 deletions(-) create mode 100644 core/archipelago/src/container/update_transaction.rs diff --git a/core/archipelago/src/api/rpc/package/install.rs b/core/archipelago/src/api/rpc/package/install.rs index 419c92c0..05d8c92b 100644 --- a/core/archipelago/src/api/rpc/package/install.rs +++ b/core/archipelago/src/api/rpc/package/install.rs @@ -280,6 +280,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; let docker_image = params .get("dockerImage") @@ -1901,7 +1904,10 @@ autopilot.active=false\n", .await .context("DATUM credentials are not available yet; wait for installation to finish")?; let password = password.trim(); - anyhow::ensure!(!password.is_empty(), "DATUM administrator password is empty"); + anyhow::ensure!( + !password.is_empty(), + "DATUM administrator password is empty" + ); return Ok(serde_json::json!({ "title": "DATUM Gateway login", "description": "Use this password when DATUM asks you to unlock configuration. In Config, set your Bitcoin payout address. Point miners at this node's IP address on Stratum port 23334 (stratum+tcp://NODE-IP:23334). Gashboard connects automatically.", diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index 9a46c547..88b16795 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -60,6 +60,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; // A cuprate node that starts on a too-small disk fills it and takes // Archipelago down with it (no upstream pruning — see // dependencies::check_cuprate_disk_compatibility). Fail the start @@ -104,6 +107,7 @@ impl RpcHandler { let op_lock = app_op_lock(package_id); let data_dir = self.config.data_dir.clone(); tokio::spawn(async move { + let _lifecycle_guard = lifecycle_guard; let _op_guard = op_lock.lock().await; let result = if let Some(orchestrator) = orchestrator.as_ref() { do_orchestrator_package_start(orchestrator.as_ref(), &to_start).await @@ -167,6 +171,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; let single_orchestrator_app = self.orchestrator.is_some() && uses_single_orchestrator_app(package_id); @@ -231,6 +238,7 @@ impl RpcHandler { let op_lock = app_op_lock(package_id); tokio::spawn(async move { + let _lifecycle_guard = lifecycle_guard; let _op_guard = op_lock.lock().await; let result = if let Some(orchestrator) = orchestrator.as_ref() { do_orchestrator_package_stop(orchestrator.as_ref(), &to_stop).await @@ -269,6 +277,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; // Restart is stop + recreate, so on a disk that shrank below the cuprate // minimum after install it resumes the doomed unprunable sync just like // start would — same gate, same "fail before clearing user-stopped / @@ -331,6 +342,7 @@ impl RpcHandler { let op_lock = app_op_lock(package_id); let data_dir = self.config.data_dir.clone(); tokio::spawn(async move { + let _lifecycle_guard = lifecycle_guard; let _op_guard = op_lock.lock().await; let result = if let Some(orchestrator) = orchestrator.as_ref() { do_orchestrator_package_restart(orchestrator.as_ref(), &to_restart).await @@ -374,6 +386,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; let preserve_data = params .get("preserve_data") .and_then(|v| v.as_bool()) diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index 19957959..4d73e58a 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -7,7 +7,6 @@ use super::config::{all_container_names, get_containers_for_app}; use super::install::install_log; use super::progress::parse_pull_progress; -use super::runtime::stop_timeout_secs; use super::validation::validate_app_id; use crate::api::rpc::RpcHandler; use crate::container::image_versions; @@ -32,6 +31,9 @@ impl RpcHandler { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("Missing package id"))?; validate_app_id(package_id)?; + let lifecycle_guard = + crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; + lifecycle_guard.require_clear()?; // An Update click must not act on an hourly cache that predates the // button. Fetch and verify first; failure leaves running containers alone. @@ -217,7 +219,7 @@ impl RpcHandler { match preflighted_stack_update( &images_to_pull, |image| async move { self.pull_update_image(package_id, &image).await }, - || self.execute_update(package_id, &containers, &images_to_pull), + || self.execute_update(package_id, &containers, &images_to_pull, &lifecycle_guard), ) .await { @@ -241,10 +243,14 @@ impl RpcHandler { package_id, e )) .await; - self.rollback_update(package_id, &containers).await; + // Transaction executor already recovered original identities or + // returned an explicit unresolved state. Never guess/reinstall. self.clear_install_progress(package_id).await; self.clear_update_state(package_id).await; - Err(e.context(format!("Update {} failed, rolled back", package_id))) + Err(e.context(format!( + "Update {} failed; see retained-runtime recovery result", + package_id + ))) } } } @@ -289,114 +295,28 @@ impl RpcHandler { } } - /// Images are prepared first; then stop → remove → recreate → verify. + /// Images are prepared first. The transaction preserves original identities, + /// creates stopped replacements, and restores the exact old states on failure. async fn execute_update( &self, package_id: &str, containers: &[String], images_to_pull: &[(String, String)], + guard: &crate::container::update_transaction::Guard, ) -> Result<()> { - // Phase: Preparing — about to stop the running container(s) so - // we can swap images. Fast. + use crate::container::update_transaction::{self, Podman}; + let targets = Podman::targets(images_to_pull).await?; + let names: HashSet<_> = containers.iter().map(String::as_str).collect(); + anyhow::ensure!( + targets.len() == names.len() + && targets + .iter() + .all(|target| names.contains(target.name.as_str())), + "Update target membership differs from installed stack; no containers changed" + ); self.set_install_phase(package_id, InstallPhase::Preparing) .await; - - // 1. Graceful stop all containers (reverse order for dependencies) - info!( - "Update {}: stopping {} containers", - package_id, - containers.len() - ); - for name in containers.iter().rev() { - let timeout = stop_timeout_secs(name); - info!( - "Update {}: stopping {} (timeout: {}s)", - package_id, name, timeout - ); - let out = tokio::process::Command::new("podman") - .args(["stop", "-t", timeout, name]) - .output() - .await - .context(format!("Failed to stop {}", name))?; - if !out.status.success() { - let stderr = String::from_utf8_lossy(&out.stderr); - warn!( - "Update {}: stop {} failed: {}", - package_id, - name, - stderr.trim() - ); - // Continue — container might already be stopped - } - } - - // 3. Remove old containers - info!("Update {}: removing old containers", package_id); - for name in containers { - let out = tokio::process::Command::new("podman") - .args(["rm", name]) - .output() - .await - .context(format!("Failed to remove {}", name))?; - if !out.status.success() { - let stderr = String::from_utf8_lossy(&out.stderr); - // Force remove as fallback - warn!( - "Update {}: rm {} failed ({}), forcing", - package_id, - name, - stderr.trim() - ); - let _ = tokio::process::Command::new("podman") - .args(["rm", "-f", name]) - .output() - .await; - } - } - - // Phase: CreatingContainer — about to recreate each container. - self.set_install_phase(package_id, InstallPhase::CreatingContainer) - .await; - - // 4. Recreate containers (orchestrator-first, reconcile fallback) - info!("Update {}: recreating containers", package_id); - for name in containers { - self.recreate_container_for_update(package_id, name).await?; - // Brief delay between containers for dependency initialization - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - - // Phase: WaitingHealthy — reconcile has started every container, - // now verifying each reached running state. - self.set_install_phase(package_id, InstallPhase::WaitingHealthy) - .await; - - // 5. Verify containers reached running state - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - for name in containers { - let status = tokio::process::Command::new("podman") - .args(["inspect", name, "--format", "{{.State.Status}}"]) - .output() - .await; - if let Ok(o) = status { - let state = String::from_utf8_lossy(&o.stdout).trim().to_string(); - anyhow::ensure!( - o.status.success() && state == "running", - "Update {}: container {} is not running after recreate", - package_id, - name - ); - } else { - anyhow::bail!( - "Update {}: cannot inspect recreated container {}", - package_id, - name - ); - } - } - - verify_update_targets(images_to_pull, &inspect_update_images(package_id).await?)?; - Ok(()) + update_transaction::execute(guard, package_id, &targets, &Podman).await } async fn recreate_container_for_update( @@ -555,41 +475,6 @@ impl RpcHandler { stack_images } - /// Rollback: restart old containers if they still exist. - /// Called when update fails partway through. - async fn rollback_update(&self, package_id: &str, containers: &[String]) { - warn!("Rolling back update for {}", package_id); - for name in containers { - // Try to start — works if container still exists (wasn't removed yet) - let out = tokio::process::Command::new("podman") - .args(["start", name]) - .output() - .await; - match out { - Ok(o) if o.status.success() => { - info!("Rollback: restarted {}", name); - } - Ok(o) => { - let stderr = String::from_utf8_lossy(&o.stderr); - warn!("Rollback: could not restart {}: {}", name, stderr.trim()); - // Container was already removed (forward path ran `podman rm`). - // Recreate via orchestrator-first path with legacy fallback. - if let Err(recreate_err) = - self.recreate_container_for_update(package_id, name).await - { - error!( - "Rollback: failed to recreate {} during rollback of {}: {}", - name, package_id, recreate_err - ); - } - } - Err(e) => { - error!("Rollback: failed to restart {}: {}", name, e); - } - } - } - } - /// Clear the Updating state (used on failure/rollback). async fn clear_update_state(&self, package_id: &str) { let (mut data, _) = self.state_manager.get_snapshot().await; diff --git a/core/archipelago/src/container/mod.rs b/core/archipelago/src/container/mod.rs index 0153fd0d..056e21da 100644 --- a/core/archipelago/src/container/mod.rs +++ b/core/archipelago/src/container/mod.rs @@ -31,3 +31,5 @@ pub use prod_orchestrator::ProdContainerOrchestrator; pub use traits::ContainerOrchestrator; mod staged_update; + +pub(crate) mod update_transaction; diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 2c547f16..a5c0d696 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -1977,6 +1977,23 @@ impl ProdContainerOrchestrator { } async fn reconcile_all_with_mode(&self, mode: ReconcileMode) -> ReconcileReport { + // Resolve exact retained update identities before normal desired-state + // reconciliation can recreate or start a member. Keep ownership for pass. + let _update_guard = match super::update_transaction::recover( + &self.data_dir, + &super::update_transaction::Podman, + ) + .await + { + Ok(guard) => guard, + Err(error) => { + let mut report = ReconcileReport::default(); + report + .failures + .push(("update-recovery".into(), format!("{error:#}"))); + return report; + } + }; let user_stopped = crate::crash_recovery::load_user_stopped(&self.data_dir).await; // Durable desired-state signal: the container names that were running at // the last periodic snapshot. Used below to recreate a previously-running diff --git a/core/archipelago/src/container/update_transaction.rs b/core/archipelago/src/container/update_transaction.rs new file mode 100644 index 00000000..c632e9a2 --- /dev/null +++ b/core/archipelago/src/container/update_transaction.rs @@ -0,0 +1,879 @@ +//! Retained-container updates. This is runtime recovery, never database rollback. +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashSet, + future::Future, + io::Write, + os::fd::AsRawFd, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct Observed { + pub id: String, + pub name: String, + pub image: String, + pub running: bool, + pub config_sha256: String, + /// False for auto-remove, external supervision, pods, paused/unknown states. + pub retainable: bool, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct Target { + pub name: String, + pub reference: String, + pub image: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Member { + original: Observed, + target: Target, + backup: String, + replacement_name: String, + replacement_id: Option, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +enum Phase { + Prepared, + Replacing, + Verified, + Committed, + Restored, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + schema: u8, + operation: String, + package: String, + phase: Phase, + members: Vec, +} + +/// An adapter must address destructive commands by immutable ID. `create` must +/// not start the new member, including when the original was stopped. +pub(crate) trait Runtime: Sync { + fn inspect(&self, id_or_name: &str) -> impl Future>> + Send; + fn stop(&self, id: &str) -> impl Future> + Send; + fn start(&self, id: &str) -> impl Future> + Send; + fn rename(&self, id: &str, name: &str) -> impl Future> + Send; + fn create( + &self, + original: &str, + name: &str, + target: &str, + ) -> impl Future> + Send; + fn remove(&self, id: &str) -> impl Future> + Send; + fn healthy(&self, id: &str) -> impl Future> + Send; +} + +pub(crate) struct Guard { + _file: std::fs::File, + root: PathBuf, +} +impl Guard { + pub(crate) fn acquire(data: &Path) -> Result { + let root = data.join("update-transactions"); + match std::fs::DirBuilder::new().mode(0o700).create(&root) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + anyhow::ensure!( + !std::fs::symlink_metadata(&root)?.file_type().is_symlink(), + "Update journal directory is a symlink" + ); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(root.join("lock"))?; + anyhow::ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0, + "An app lifecycle operation is active; retry when it completes" + ); + Ok(Self { _file: file, root }) + } + fn path(&self, operation: &str) -> Result { + anyhow::ensure!( + uuid::Uuid::parse_str(operation)?.to_string() == operation, + "Invalid update operation" + ); + Ok(self.root.join(format!("{operation}.json"))) + } + fn save(&self, record: &Record) -> Result<()> { + record.validate()?; + let target = self.path(&record.operation)?; + let temp = self.root.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let bytes = serde_json::to_vec(record)?; + anyhow::ensure!(bytes.len() <= 1024 * 1024, "Update journal too large"); + // No await/detached writer after the ownership lock can be released. + let result = (|| -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temp)?; + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&temp, &target)?; + std::fs::File::open(&self.root)?.sync_all()?; + std::fs::File::open(self.root.parent().unwrap())?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(temp); + } + result + } + fn records(&self) -> Result> { + let mut records = Vec::new(); + for entry in std::fs::read_dir(&self.root)? { + let entry = entry?; + if entry.path().extension().and_then(|v| v.to_str()) != Some("json") { + continue; + } + anyhow::ensure!(records.len() < 256, "Too many retained update journals"); + let meta = entry.metadata()?; + anyhow::ensure!( + entry.file_type()?.is_file() && meta.len() <= 1024 * 1024, + "Invalid update journal file" + ); + let record: Record = serde_json::from_slice(&std::fs::read(entry.path())?)?; + record.validate()?; + anyhow::ensure!( + self.path(&record.operation)? == entry.path(), + "Update journal name changed" + ); + records.push(record); + } + Ok(records) + } + pub(crate) fn require_clear(&self) -> Result<()> { + anyhow::ensure!( + self.records()? + .iter() + .all(|r| matches!(r.phase, Phase::Committed | Phase::Restored)), + "An interrupted app update needs recovery before another lifecycle action" + ); + Ok(()) + } +} +fn name_ok(value: &str) -> bool { + !value.is_empty() + && value.len() <= 200 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) +} +fn id_ok(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) +} +impl Record { + fn validate(&self) -> Result<()> { + anyhow::ensure!( + self.schema == 1 + && name_ok(&self.package) + && !self.members.is_empty() + && self.members.len() <= 32, + "Invalid update journal" + ); + let operation = uuid::Uuid::parse_str(&self.operation)?; + anyhow::ensure!( + operation.to_string() == self.operation, + "Invalid update UUID" + ); + let mut names = HashSet::new(); + let mut ids = HashSet::new(); + for (i, m) in self.members.iter().enumerate() { + anyhow::ensure!( + name_ok(&m.original.name) + && m.target.name == m.original.name + && id_ok(&m.original.id) + && id_ok(&m.original.config_sha256) + && ids.insert(&m.original.id) + && names.insert(&m.original.name) + && m.original.retainable + && m.backup == format!("archy-update-{}-{i}-old", operation.simple()) + && m.replacement_name == format!("archy-update-{}-{i}-new", operation.simple()) + && m.replacement_id.as_deref().is_none_or(id_ok), + "Invalid retained update member" + ); + } + Ok(()) + } +} +async fn exact(runtime: &impl Runtime, id: &str) -> Result { + let found = runtime + .inspect(id) + .await? + .context("Retained container is missing; runtime recovery unresolved")?; + anyhow::ensure!( + found.id == id, + "Container identity changed; recovery unresolved" + ); + Ok(found) +} + +/// All original inventory is proven before the first stop/rename. Unsupported +/// runtimes stay untouched. The caller has already prepared every target image. +pub(crate) async fn execute( + guard: &Guard, + package: &str, + targets: &[Target], + runtime: &impl Runtime, +) -> Result<()> { + guard.require_clear()?; + anyhow::ensure!( + !targets.is_empty() && targets.len() <= 32, + "Invalid update members" + ); + let operation = uuid::Uuid::new_v4(); + let mut members = Vec::new(); + for (i, target) in targets.iter().enumerate() { + let original = runtime + .inspect(&target.name) + .await? + .context("Original update member is missing")?; + anyhow::ensure!( + original.retainable, + "{} cannot retain its original runtime safely; no containers changed", + target.name + ); + members.push(Member { + original, + target: target.clone(), + backup: format!("archy-update-{}-{i}-old", operation.simple()), + replacement_name: format!("archy-update-{}-{i}-new", operation.simple()), + replacement_id: None, + }); + } + let mut record = Record { + schema: 1, + operation: operation.to_string(), + package: package.into(), + phase: Phase::Prepared, + members, + }; + guard.save(&record)?; + match replace(&guard,&mut record,runtime).await { + Ok(())=>Ok(()), + Err(error)=>match restore(&guard,&mut record,runtime).await { + Ok(())=>Err(error.context("Update failed; original runtime states restored (persistent data was not rolled back)")), + Err(recovery)=>Err(error.context(format!("Update failed; runtime recovery remains unresolved: {recovery:#}"))), + } + } +} +async fn replace(guard: &Guard, record: &mut Record, runtime: &impl Runtime) -> Result<()> { + for member in record.members.iter().rev() { + let current = exact(runtime, &member.original.id).await?; + anyhow::ensure!( + current.name == member.original.name + && current.image == member.original.image + && current.config_sha256 == member.original.config_sha256 + && current.retainable, + "Original changed before update" + ); + if current.running { + runtime.stop(¤t.id).await?; + } + anyhow::ensure!( + !exact(runtime, ¤t.id).await?.running, + "Original did not stop" + ); + } + record.phase = Phase::Replacing; + guard.save(record)?; + for index in 0..record.members.len() { + let member = &record.members[index]; + anyhow::ensure!( + runtime.inspect(&member.backup).await?.is_none() + && runtime.inspect(&member.replacement_name).await?.is_none(), + "Update backup name collision" + ); + runtime.rename(&member.original.id, &member.backup).await?; + let id = runtime + .create( + &member.original.id, + &member.replacement_name, + &member.target.reference, + ) + .await?; + anyhow::ensure!(id_ok(&id), "Invalid replacement identity"); + record.members[index].replacement_id = Some(id.clone()); + guard.save(record)?; + let member = &record.members[index]; + let created = exact(runtime, &id).await?; + anyhow::ensure!( + created.name == member.replacement_name + && !created.running + && created.image == member.target.image + && created.config_sha256 == member.original.config_sha256, + "Replacement configuration/state mismatch" + ); + runtime.rename(&id, &member.original.name).await?; + } + for member in &record.members { + let id = member + .replacement_id + .as_deref() + .context("Missing replacement")?; + if member.original.running { + runtime.start(id).await?; + } + let observed = exact(runtime, id).await?; + anyhow::ensure!( + observed.running == member.original.running && observed.image == member.target.image, + "Replacement state mismatch" + ); + if member.original.running { + anyhow::ensure!( + runtime.healthy(id).await?, + "Updated member failed health verification" + ); + } + } + record.phase = Phase::Verified; + guard.save(record)?; + // Original backups are retained. Committing does not remove volumes or + // silently discard the only recoverable original after a schema migration. + record.phase = Phase::Committed; + guard.save(record) +} +async fn restore(guard: &Guard, record: &mut Record, runtime: &impl Runtime) -> Result<()> { + // Preflight every original before mutating any replacement. Never infer + // ownership of a new container from a reused public name. + for member in &record.members { + let original = exact(runtime, &member.original.id).await?; + anyhow::ensure!( + original.image == member.original.image + && original.config_sha256 == member.original.config_sha256 + && (original.name == member.original.name || original.name == member.backup), + "Original rollback identity/config changed" + ); + if member.replacement_id.is_none() { + anyhow::ensure!( + runtime.inspect(&member.replacement_name).await?.is_none(), + "Unacknowledged replacement exists; preserve it for explicit recovery" + ); + } + if let Some(current) = runtime.inspect(&member.original.name).await? { + anyhow::ensure!( + current.id == member.original.id + || Some(¤t.id) == member.replacement_id.as_ref(), + "Foreign container occupies original name" + ); + } + } + for member in record.members.iter().rev() { + if let Some(id) = &member.replacement_id { + if let Some(current) = runtime.inspect(id).await? { + anyhow::ensure!( + current.id == *id && current.image == member.target.image, + "Replacement identity changed" + ); + if current.running { + runtime.stop(id).await?; + } + runtime.remove(id).await?; + } + } + } + for member in &record.members { + let current = exact(runtime, &member.original.id).await?; + if current.name != member.original.name { + runtime.rename(¤t.id, &member.original.name).await?; + } + if member.original.running && !current.running { + runtime.start(¤t.id).await?; + } else if !member.original.running && current.running { + runtime.stop(¤t.id).await?; + } + let observed = exact(runtime, ¤t.id).await?; + anyhow::ensure!( + observed.name == member.original.name + && observed.image == member.original.image + && observed.running == member.original.running, + "Original state was not restored" + ); + } + record.phase = Phase::Restored; + guard.save(record) +} +/// Called before ordinary reconciliation. An unresolved recovery must prevent +/// reconciliation from guessing a replacement or starting stopped originals. +pub(crate) async fn recover(data: &Path, runtime: &impl Runtime) -> Result { + let guard = Guard::acquire(data)?; + for mut record in guard.records()? { + if !matches!(record.phase, Phase::Committed | Phase::Restored) { + restore(&guard, &mut record, runtime).await?; + } + } + Ok(guard) +} + +pub(crate) struct Podman; +impl Podman { + async fn output(args: &[&str]) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(120), + tokio::process::Command::new("podman") + .args(args) + .kill_on_drop(true) + .output(), + ) + .await + .context("Container operation timed out; inspect original operation before retry")? + .context("Container runtime unavailable") + } + async fn command(args: &[&str]) -> Result { + let output = Self::output(args).await?; + anyhow::ensure!( + output.status.success(), + "Container runtime rejected {} (exit {:?})", + args.first().unwrap_or(&"operation"), + output.status.code() + ); + Ok(std::str::from_utf8(&output.stdout)?.trim().to_string()) + } + pub(crate) async fn targets(images: &[(String, String)]) -> Result> { + // Confirm clone support before any stop. Never silently switch to a + // latest-catalog install if this runtime cannot create without running. + Self::command(&["container", "clone", "--help"]).await?; + let mut targets = Vec::new(); + for (name, reference) in images { + let raw = Self::command(&["image", "inspect", reference]).await?; + let rows: Vec = serde_json::from_str(&raw)?; + let image = rows + .first() + .and_then(|v| v.get("Id").or_else(|| v.get("ID"))) + .and_then(|v| v.as_str()) + .context("Prepared image identity unavailable")?; + targets.push(Target { + name: name.clone(), + reference: reference.clone(), + image: image.strip_prefix("sha256:").unwrap_or(image).into(), + }); + } + Ok(targets) + } +} +impl Runtime for Podman { + async fn inspect(&self, value: &str) -> Result> { + let exists = Self::output(&["container", "exists", value]).await?; + match exists.status.code() { + Some(0) => {} + Some(1) => return Ok(None), + _ => anyhow::bail!("Container inventory unavailable"), + } + let raw = Self::command(&["inspect", value]).await?; + let rows: Vec = serde_json::from_str(&raw)?; + let row = rows.first().context("Empty container inspection")?; + let string = |key: &str| { + row.get(key) + .and_then(|v| v.as_str()) + .context("Incomplete container identity") + }; + let state = row + .pointer("/State/Status") + .and_then(|v| v.as_str()) + .context("Missing runtime state")?; + let labels = row.pointer("/Config/Labels"); + let supervised = labels.and_then(|v| v.get("PODMAN_SYSTEMD_UNIT")).is_some() + || labels + .and_then(|v| v.get("io.containers.systemd.unit")) + .is_some(); + let supervised = supervised + || row + .pointer("/Config/Env") + .and_then(|v| v.as_array()) + .is_some_and(|env| { + env.iter().any(|v| { + v.as_str() + .is_some_and(|v| v.starts_with("PODMAN_SYSTEMD_UNIT=")) + }) + }); + let restart = row + .pointer("/HostConfig/RestartPolicy/Name") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let retainable = row + .pointer("/HostConfig/AutoRemove") + .and_then(|v| v.as_bool()) + == Some(false) + && !supervised + && matches!(restart, "" | "no") + && row + .get("Pod") + .and_then(|v| v.as_str()) + .is_none_or(str::is_empty) + && row + .get("Dependencies") + .and_then(|v| v.as_array()) + .is_none_or(Vec::is_empty) + && matches!(state, "running" | "stopped" | "exited" | "created"); + let config = serde_json::json!({ + "mounts":row.get("Mounts"),"env":row.pointer("/Config/Env"), + "command":row.pointer("/Config/Cmd"),"entrypoint":row.pointer("/Config/Entrypoint"), + "user":row.pointer("/Config/User"),"directory":row.pointer("/Config/WorkingDir"), + "ports":row.pointer("/HostConfig/PortBindings"),"network":row.pointer("/HostConfig/NetworkMode"), + "privileged":row.pointer("/HostConfig/Privileged"),"devices":row.pointer("/HostConfig/Devices"), + "security":row.pointer("/HostConfig/SecurityOpt"),"cap_add":row.pointer("/HostConfig/CapAdd"),"cap_drop":row.pointer("/HostConfig/CapDrop") + }); + use sha2::Digest; + Ok(Some(Observed { + id: string("Id")?.into(), + name: string("Name")?.trim_start_matches('/').into(), + image: string("Image")? + .strip_prefix("sha256:") + .unwrap_or(string("Image")?) + .into(), + running: state == "running", + retainable, + config_sha256: hex::encode(sha2::Sha256::digest(serde_json::to_vec(&config)?)), + })) + } + async fn stop(&self, id: &str) -> Result<()> { + let original = exact(self, id).await?; + let grace = archipelago_container::runtime::stop_grace_secs_for(&original.name); + let timeout = grace.to_string(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(grace + 30), + tokio::process::Command::new("podman") + .args(["stop", "--time", &timeout, id]) + .kill_on_drop(true) + .output(), + ) + .await + .context("Graceful stop is unresolved; original runtime retained")??; + anyhow::ensure!(result.status.success(), "Graceful container stop failed"); + Ok(()) + } + async fn start(&self, id: &str) -> Result<()> { + Self::command(&["start", id]).await?; + Ok(()) + } + async fn rename(&self, id: &str, name: &str) -> Result<()> { + Self::command(&["rename", id, name]).await?; + Ok(()) + } + async fn create(&self, id: &str, name: &str, target: &str) -> Result { + Self::command(&["container", "clone", id, name, target]).await + } + async fn remove(&self, id: &str) -> Result<()> { + Self::command(&["rm", id]).await?; + Ok(()) + } + async fn healthy(&self, id: &str) -> Result { + // Missing healthcheck is not fabricated health. Running-only containers + // can pass runtime readiness; healthchecked members must report healthy. + for _ in 0..30 { + let raw = Self::command(&["inspect", id]).await?; + let rows: Vec = serde_json::from_str(&raw)?; + let row = rows.first().context("Missing updated container")?; + if row.pointer("/State/Status").and_then(|v| v.as_str()) != Some("running") { + return Ok(false); + } + match row.pointer("/State/Health/Status").and_then(|v| v.as_str()) { + None | Some("") | Some("healthy") => return Ok(true), + Some("unhealthy") => return Ok(false), + _ => {} + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + Ok(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Mutex, + }; + struct Mock { + rows: Mutex>, + calls: Mutex>, + fail_at: AtomicUsize, + sequence: AtomicUsize, + lost_create: AtomicBool, + fail_health: AtomicBool, + } + impl Mock { + fn new() -> Self { + Self { + rows: Mutex::new(vec![ + Observed { + id: format!("{:064x}", 1), + name: "db".into(), + image: "old-db".into(), + running: true, + retainable: true, + config_sha256: "a".repeat(64), + }, + Observed { + id: format!("{:064x}", 2), + name: "web".into(), + image: "old-web".into(), + running: false, + retainable: true, + config_sha256: "b".repeat(64), + }, + ]), + calls: Default::default(), + fail_at: AtomicUsize::new(0), + sequence: AtomicUsize::new(0), + lost_create: AtomicBool::new(false), + fail_health: AtomicBool::new(false), + } + } + fn event(&self, action: String) -> Result<()> { + self.calls.lock().unwrap().push(action); + let step = self.sequence.fetch_add(1, Ordering::SeqCst) + 1; + anyhow::ensure!( + step != self.fail_at.load(Ordering::SeqCst), + "Injected lifecycle failure" + ); + Ok(()) + } + fn targets() -> Vec { + vec![ + Target { + name: "db".into(), + reference: "new-db".into(), + image: "new-db".into(), + }, + Target { + name: "web".into(), + reference: "new-web".into(), + image: "new-web".into(), + }, + ] + } + fn originals_restored(&self) { + let rows = self.rows.lock().unwrap(); + assert_eq!(rows.len(), 2); + for (i, name, running) in [(1, "db", true), (2, "web", false)] { + let row = rows.iter().find(|r| r.id == format!("{i:064x}")).unwrap(); + assert_eq!(row.name, name); + assert_eq!(row.running, running); + assert_eq!(row.image, format!("old-{name}")); + } + } + } + impl Runtime for Mock { + async fn inspect(&self, value: &str) -> Result> { + Ok(self + .rows + .lock() + .unwrap() + .iter() + .find(|r| r.id == value || r.name == value) + .cloned()) + } + async fn stop(&self, id: &str) -> Result<()> { + self.event(format!("stop:{id}"))?; + self.rows + .lock() + .unwrap() + .iter_mut() + .find(|r| r.id == id) + .unwrap() + .running = false; + Ok(()) + } + async fn start(&self, id: &str) -> Result<()> { + self.event(format!("start:{id}"))?; + self.rows + .lock() + .unwrap() + .iter_mut() + .find(|r| r.id == id) + .unwrap() + .running = true; + Ok(()) + } + async fn rename(&self, id: &str, name: &str) -> Result<()> { + self.event(format!("rename:{id}:{name}"))?; + let mut rows = self.rows.lock().unwrap(); + anyhow::ensure!(!rows.iter().any(|r| r.name == name), "Name occupied"); + rows.iter_mut().find(|r| r.id == id).unwrap().name = name.into(); + Ok(()) + } + async fn create(&self, original: &str, name: &str, target: &str) -> Result { + self.event(format!("create-stopped:{name}"))?; + let mut rows = self.rows.lock().unwrap(); + let mut row = rows.iter().find(|r| r.id == original).unwrap().clone(); + row.id = format!("{:064x}", 100 + rows.len()); + row.name = name.into(); + row.image = target.into(); + row.running = false; + let id = row.id.clone(); + rows.push(row); + anyhow::ensure!( + !self.lost_create.swap(false, Ordering::SeqCst), + "Lost create acknowledgement" + ); + Ok(id) + } + async fn remove(&self, id: &str) -> Result<()> { + self.event(format!("remove:{id}"))?; + self.rows.lock().unwrap().retain(|r| r.id != id); + Ok(()) + } + async fn healthy(&self, _id: &str) -> Result { + Ok(!self.fail_health.load(Ordering::SeqCst)) + } + } + #[tokio::test] + async fn mixed_stack_update_never_starts_stopped_member_and_retains_original_ids() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + let guard = Guard::acquire(root.path()).unwrap(); + execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap(); + let record = guard.records().unwrap().pop().unwrap(); + assert_eq!(record.phase, Phase::Committed); + let rows = runtime.rows.lock().unwrap(); + assert_eq!(rows.len(), 4); + for member in &record.members { + let original = rows.iter().find(|r| r.id == member.original.id).unwrap(); + assert_eq!(original.name, member.backup); + assert!(!original.running); + let new = rows + .iter() + .find(|r| Some(&r.id) == member.replacement_id.as_ref()) + .unwrap(); + assert_eq!(new.running, member.original.running); + } + assert!(!runtime.calls.lock().unwrap().iter().any(|call| call + == &format!( + "start:{}", + record.members[1].replacement_id.as_ref().unwrap() + ))); + assert!(!runtime + .calls + .lock() + .unwrap() + .iter() + .any(|call| call.starts_with("remove:"))); + } + #[tokio::test] + async fn every_mutation_failure_restores_exact_originals_without_catalog_recreate() { + // Eight mutating calls in this two-member update, including starting db. + for fail in 1..=8 { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + runtime.fail_at.store(fail, Ordering::SeqCst); + let guard = Guard::acquire(root.path()).unwrap(); + let error = execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("original runtime states restored"), + "failure {fail}: {error:#}" + ); + runtime.originals_restored(); + assert_eq!(guard.records().unwrap()[0].phase, Phase::Restored); + } + } + #[tokio::test] + async fn unsupported_member_refuses_before_any_runtime_mutation() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + runtime.rows.lock().unwrap()[1].retainable = false; + let guard = Guard::acquire(root.path()).unwrap(); + assert!(execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .is_err()); + assert!(runtime.calls.lock().unwrap().is_empty()); + assert!(guard.records().unwrap().is_empty()); + runtime.originals_restored(); + } + #[tokio::test] + async fn failed_health_restores_running_and_stopped_states() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + runtime.fail_health.store(true, Ordering::SeqCst); + let guard = Guard::acquire(root.path()).unwrap(); + assert!(execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .is_err()); + runtime.originals_restored(); + } + #[tokio::test] + async fn lost_create_reply_is_unresolved_and_never_deletes_unowned_container() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + runtime.lost_create.store(true, Ordering::SeqCst); + let guard = Guard::acquire(root.path()).unwrap(); + let error = execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap_err(); + assert!(error.to_string().contains("unresolved")); + assert!(guard.require_clear().is_err()); + assert!(!runtime + .calls + .lock() + .unwrap() + .iter() + .any(|call| call.starts_with("remove:"))); + drop(guard); + assert!(recover(root.path(), &runtime).await.is_err()); + assert_eq!(runtime.rows.lock().unwrap().len(), 3); + } + #[tokio::test] + async fn daemon_restart_restores_interrupted_rename_before_normal_reconciliation() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + let guard = Guard::acquire(root.path()).unwrap(); + execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap(); + let mut record = guard.records().unwrap().pop().unwrap(); + // Simulate interruption after replacements existed but before durable commit. + record.phase = Phase::Replacing; + guard.save(&record).unwrap(); + drop(guard); + let recovered = recover(root.path(), &runtime).await.unwrap(); + runtime.originals_restored(); + assert_eq!(recovered.records().unwrap()[0].phase, Phase::Restored); + } + #[tokio::test] + async fn missing_original_does_not_fabricate_rollback_or_start_other_members() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + let guard = Guard::acquire(root.path()).unwrap(); + execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap(); + let mut record = guard.records().unwrap().pop().unwrap(); + record.phase = Phase::Replacing; + guard.save(&record).unwrap(); + runtime + .rows + .lock() + .unwrap() + .retain(|r| r.id != format!("{:064x}", 1)); + runtime.calls.lock().unwrap().clear(); + drop(guard); + assert!(recover(root.path(), &runtime).await.is_err()); + assert!(runtime.calls.lock().unwrap().is_empty()); + } + #[test] + fn lifecycle_lock_excludes_competing_commands() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + assert!(Guard::acquire(root.path()).is_err()); + drop(guard); + assert!(Guard::acquire(root.path()).is_ok()); + } +} From 68eeb6396dc161335c0c42964dc9422efc3de807 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 01:49:01 -0400 Subject: [PATCH 03/13] Retain update recovery holds and journal supervised runtime restoration --- .../src/api/rpc/package/async_lifecycle.rs | 65 +- .../src/api/rpc/package/install.rs | 1 + .../src/api/rpc/package/runtime.rs | 4 + .../archipelago/src/api/rpc/package/update.rs | 4 +- core/archipelago/src/container/mod.rs | 2 + .../src/container/prod_orchestrator.rs | 14 + .../src/container/supervised_update.rs | 809 ++++++++++++++++++ .../src/container/update_transaction.rs | 129 ++- 8 files changed, 1019 insertions(+), 9 deletions(-) create mode 100644 core/archipelago/src/container/supervised_update.rs diff --git a/core/archipelago/src/api/rpc/package/async_lifecycle.rs b/core/archipelago/src/api/rpc/package/async_lifecycle.rs index 63ad037b..fe67b145 100644 --- a/core/archipelago/src/api/rpc/package/async_lifecycle.rs +++ b/core/archipelago/src/api/rpc/package/async_lifecycle.rs @@ -377,12 +377,34 @@ impl RpcHandler { Err(e) => { error!("package.update {} failed: {:#}", package_id_spawn, e); install_log(&format!("UPDATE FAIL: {} — {:#}", package_id_spawn, e)).await; - // Inner handler already ran rollback_update + cleared - // update state, but be defensive: revert to pre-state - // in case the inner flow died before its cleanup. - if let Some(prev) = pre_state { - set_package_state(&handler.state_manager, &package_id_spawn, prev).await; - } + // Release the transitional overlay before asking the scanner + // for real state. Prior Running is not proof of successful + // rollback, and a failed preflight is not proof of Stopped. + handler + .state_manager + .mutate_data(|data| { + if let Some(entry) = data.package_data.get_mut(&package_id_spawn) { + finish_failed_update(entry); + } + data.notifications.retain(|item| { + item.id != format!("update-failed-{package_id_spawn}") + }); + data.notifications.push(crate::data_model::Notification { + id: format!("update-failed-{package_id_spawn}"), + level: crate::data_model::NotificationLevel::Error, + title: format!("Could not update {package_id_spawn}"), + message: format!( + "{e}. Runtime recovery does not roll back database changes." + ), + timestamp: chrono::Utc::now().to_rfc3339(), + app_id: Some(package_id_spawn.clone()), + }); + while data.notifications.len() > 20 { + data.notifications.remove(0); + } + }) + .await; + kick_scanner_and_wait(&handler).await; } } }); @@ -585,3 +607,34 @@ async fn kick_scanner_and_wait(handler: &RpcHandler) { }) .await; } + +fn finish_failed_update(entry: &mut crate::data_model::PackageDataEntry) { + if entry.state == PackageState::Updating { + entry.state = PackageState::Installed; + } + entry.install_progress = None; +} +#[cfg(test)] +mod update_completion_tests { + use super::*; + #[test] + fn failure_releases_spinner_without_inventing_stopped_or_restored_runtime() { + let mut entry = super::super::progress::create_installing_entry("movie"); + entry.state = PackageState::Updating; + finish_failed_update(&mut entry); + assert_eq!(entry.state, PackageState::Installed); + assert!(entry.install_progress.is_none()); + for actual in [ + PackageState::Running, + PackageState::Stopped, + PackageState::Exited, + ] { + entry.state = actual.clone(); + finish_failed_update(&mut entry); + assert_eq!( + entry.state, actual, + "Fresh scanner evidence must win over old pre-update intent" + ); + } + } +} diff --git a/core/archipelago/src/api/rpc/package/install.rs b/core/archipelago/src/api/rpc/package/install.rs index 05d8c92b..1609c037 100644 --- a/core/archipelago/src/api/rpc/package/install.rs +++ b/core/archipelago/src/api/rpc/package/install.rs @@ -283,6 +283,7 @@ impl RpcHandler { let lifecycle_guard = crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; lifecycle_guard.require_clear()?; + lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?; let docker_image = params .get("dockerImage") diff --git a/core/archipelago/src/api/rpc/package/runtime.rs b/core/archipelago/src/api/rpc/package/runtime.rs index 88b16795..edba9f64 100644 --- a/core/archipelago/src/api/rpc/package/runtime.rs +++ b/core/archipelago/src/api/rpc/package/runtime.rs @@ -63,6 +63,7 @@ impl RpcHandler { let lifecycle_guard = crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; lifecycle_guard.require_clear()?; + lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?; // A cuprate node that starts on a too-small disk fills it and takes // Archipelago down with it (no upstream pruning — see // dependencies::check_cuprate_disk_compatibility). Fail the start @@ -174,6 +175,7 @@ impl RpcHandler { let lifecycle_guard = crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; lifecycle_guard.require_clear()?; + lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?; let single_orchestrator_app = self.orchestrator.is_some() && uses_single_orchestrator_app(package_id); @@ -280,6 +282,7 @@ impl RpcHandler { let lifecycle_guard = crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; lifecycle_guard.require_clear()?; + lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?; // Restart is stop + recreate, so on a disk that shrank below the cuprate // minimum after install it resumes the doomed unprunable sync just like // start would — same gate, same "fail before clearing user-stopped / @@ -389,6 +392,7 @@ impl RpcHandler { let lifecycle_guard = crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?; lifecycle_guard.require_clear()?; + lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?; let preserve_data = params .get("preserve_data") .and_then(|v| v.as_bool()) diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index 4d73e58a..037f0f93 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -481,7 +481,9 @@ impl RpcHandler { if let Some(entry) = data.package_data.get_mut(package_id) { // Don't overwrite state from scanner — just clear if still Updating if entry.state == PackageState::Updating { - entry.state = PackageState::Stopped; + // Unknown is not stopped: the authoritative scanner will + // refresh actual retained runtime immediately in the wrapper. + entry.state = PackageState::Installed; } } self.state_manager.update_data(data).await; diff --git a/core/archipelago/src/container/mod.rs b/core/archipelago/src/container/mod.rs index 056e21da..98db75e1 100644 --- a/core/archipelago/src/container/mod.rs +++ b/core/archipelago/src/container/mod.rs @@ -33,3 +33,5 @@ pub use traits::ContainerOrchestrator; mod staged_update; pub(crate) mod update_transaction; + +pub(crate) mod supervised_update; diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index a5c0d696..fbb9478d 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -1994,6 +1994,16 @@ impl ProdContainerOrchestrator { return report; } }; + let held_names = match _update_guard.held_names() { + Ok(names) => names, + Err(error) => { + let mut report = ReconcileReport::default(); + report + .failures + .push(("update-recovery".into(), format!("{error:#}"))); + return report; + } + }; let user_stopped = crate::crash_recovery::load_user_stopped(&self.data_dir).await; // Durable desired-state signal: the container names that were running at // the last periodic snapshot. Used below to recreate a previously-running @@ -2015,6 +2025,7 @@ impl ProdContainerOrchestrator { let filtered = state .manifests .iter() + .filter(|(_, lm)| !held_names.contains(&compute_container_name(&lm.manifest))) .filter(|(app_id, _)| !state.disabled.contains(*app_id)) .filter(|(app_id, lm)| { dependency_required.contains(*app_id) @@ -3442,6 +3453,9 @@ impl ProdContainerOrchestrator { /// changes restart the service and retain a durable pending marker until /// that succeeds, including across daemon restarts and failed reloads. async fn sync_quadlet_unit(&self, lm: &LoadedManifest, name: &str) -> Result<()> { + if super::update_transaction::is_held(&self.data_dir, name)? { + return Ok(()); // Preserve the recovered unit instead of current catalog drift. + } // Companions: same reasoning as migrate_to_quadlet_if_needed — // companion.rs renders these units with a different shape, syncing // here would clobber them. diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs new file mode 100644 index 00000000..37fe6a94 --- /dev/null +++ b/core/archipelago/src/container/supervised_update.rs @@ -0,0 +1,809 @@ +//! Quadlet recovery preserves exact original launch configuration and image, not +//! ephemeral --rm container IDs. Persistent application data is never rolled back. +use super::update_transaction::{Guard, Observed, Target}; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + future::Future, + io::Write, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct Unit { + pub name: String, + pub body: String, + pub image: String, + pub container_id: String, + pub running: bool, + pub config_sha256: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PreparedTarget { + pub body: String, + pub manifest: archipelago_container::AppManifest, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RecoveryImage { + pub image: String, + pub source_container_id: String, + pub operation_id: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Member { + original: Unit, + target: Target, + target_body: String, + target_manifest: archipelago_container::AppManifest, + pinned_original_body: String, + original_tag: String, + recovery_image: Option, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +enum Phase { + Prepared, + Aborted, + Editing, + Starting, + Committed, + Restored, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Journal { + schema: u8, + id: String, + package: String, + phase: Phase, + members: Vec, +} + +pub(crate) trait Supervisor: Sync { + /// Only an internal reviewed signed-manifest planner may supply this value; + /// browser parameters must never become a unit body or hook recipe. + fn prepare_target( + &self, + target: &Target, + original: &Unit, + ) -> impl Future> + Send; + fn target_hooks( + &self, + name: &str, + manifest: &archipelago_container::AppManifest, + ) -> impl Future> + Send; + + /// Commit the exact original writable layer to a local-only operation-owned + /// image, explicitly pausing and excluding mounted volumes. A retry must + /// recover its matching image rather than overwrite an unrelated tag. + /// This does not establish application-level write quiescence or DB backup. + fn snapshot( + &self, + original: &Unit, + operation_id: &str, + tag: &str, + ) -> impl Future> + Send; + fn capture(&self, name: &str) -> impl Future> + Send; + fn read(&self, name: &str) -> impl Future> + Send; + fn write( + &self, + name: &str, + expected: &[String], + body: &str, + ) -> impl Future> + Send; + fn pin(&self, image: &str, tag: &str) -> impl Future> + Send; + fn stop(&self, name: &str) -> impl Future> + Send; + fn reload(&self) -> impl Future> + Send; + fn start(&self, name: &str) -> impl Future> + Send; + fn observed(&self, name: &str) -> impl Future>> + Send; + fn healthy(&self, name: &str) -> impl Future> + Send; +} +fn simple(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) +} +fn digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) +} +/// Parse only the renderer-owned unit shape. Ambiguous images, includes and +/// external environment files cannot become a guessed recovery recipe. +pub(crate) fn pin_body(body: &str, name: &str, image: &str) -> Result { + anyhow::ensure!( + body.len() <= 1024 * 1024 && simple(name), + "Invalid original unit" + ); + let mut section = ""; + let mut images = 0; + let mut names = 0; + let mut output = String::new(); + for line in body.lines() { + let trimmed = line.trim(); + anyhow::ensure!( + !trimmed.ends_with('\\') + && !trimmed.starts_with(".include") + && !trimmed.starts_with("EnvironmentFile=") + && !trimmed.starts_with("EnvFile="), + "Unit has external/continued configuration; exact recovery is not supported yet" + ); + if trimmed.starts_with('[') { + section = trimmed; + } + if section == "[Container]" && trimmed.starts_with("Image=") { + images += 1; + output.push_str(&format!("Image={image}\n")); + } else { + if section == "[Container]" && trimmed.starts_with("ContainerName=") { + names += 1; + anyhow::ensure!( + trimmed == format!("ContainerName={name}"), + "Unit container ownership mismatch" + ); + } + output.push_str(line); + output.push('\n'); + } + } + anyhow::ensure!( + images == 1 && names == 1, + "Original Quadlet must bind one container and image" + ); + Ok(output) +} +fn root(guard: &Guard) -> Result { + let path = guard.directory().join("supervised"); + match std::fs::DirBuilder::new().mode(0o700).create(&path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + anyhow::ensure!( + !std::fs::symlink_metadata(&path)?.file_type().is_symlink(), + "Invalid supervised journal directory" + ); + Ok(path) +} +fn validate(record: &Journal) -> Result<()> { + anyhow::ensure!( + record.schema == 1 + && simple(&record.package) + && uuid::Uuid::parse_str(&record.id)?.to_string() == record.id + && !record.members.is_empty() + && record.members.len() <= 32, + "Invalid supervised update journal" + ); + let mut names = std::collections::HashSet::new(); + for (index, member) in record.members.iter().enumerate() { + anyhow::ensure!( + simple(&member.original.name) + && names.insert(&member.original.name) + && member.target.name == member.original.name + && digest(&member.target.image) + && digest(&member.original.image) + && digest(&member.original.container_id) + && digest(&member.original.config_sha256), + "Invalid original supervised identity" + ); + anyhow::ensure!( + member.original_tag == format!("localhost/archy-update-recovery:{}-{index}", record.id), + "Recovery image pin changed" + ); + if let Some(image) = &member.recovery_image { + anyhow::ensure!( + digest(&image.image) + && image.source_container_id == member.original.container_id + && image.operation_id == record.id, + "Recovery image ownership changed" + ); + } + anyhow::ensure!( + matches!(record.phase, Phase::Prepared | Phase::Aborted) + || member.recovery_image.is_some(), + "Destructive update lacks a durable writable-layer recovery image" + ); + let restore_image = member + .recovery_image + .as_ref() + .map(|value| value.image.as_str()) + .unwrap_or(&member.original.image); + anyhow::ensure!( + member.pinned_original_body + == pin_body( + &member.original.body, + &member.original.name, + &format!("sha256:{restore_image}") + )? + && member.target_body + == pin_body( + &member.target_body, + &member.original.name, + &member.target.reference + )? + && member.target_manifest.app.container.image.as_deref() + == Some(member.target.reference.as_str()), + "Saved unit recipe changed" + ); + } + Ok(()) +} +fn save(guard: &Guard, record: &Journal) -> Result<()> { + validate(record)?; + let dir = root(guard)?; + let bytes = serde_json::to_vec(record)?; + anyhow::ensure!( + bytes.len() <= 4 * 1024 * 1024, + "Supervised recovery journal too large" + ); + let temporary = dir.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let result = (|| -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&temporary, dir.join(format!("{}.json", record.id)))?; + std::fs::File::open(&dir)?.sync_all()?; + std::fs::File::open(guard.directory())?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(temporary); + } + result +} +fn records(guard: &Guard) -> Result> { + let dir = root(guard)?; + let mut records = Vec::new(); + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + if entry.path().extension().and_then(|v| v.to_str()) != Some("json") { + continue; + } + anyhow::ensure!( + records.len() < 128 + && entry.file_type()?.is_file() + && entry.metadata()?.len() <= 4 * 1024 * 1024, + "Invalid supervised recovery inventory" + ); + let record: Journal = serde_json::from_slice(&std::fs::read(entry.path())?)?; + validate(&record)?; + anyhow::ensure!( + entry.file_name() == format!("{}.json", record.id).as_str(), + "Supervised journal name changed" + ); + records.push(record); + } + Ok(records) +} +pub(crate) fn require_clear(guard: &Guard) -> Result<()> { + anyhow::ensure!( + records(guard)? + .iter() + .all(|r| matches!(r.phase, Phase::Committed | Phase::Restored | Phase::Aborted)), + "A supervised update needs recovery first" + ); + Ok(()) +} +pub(crate) async fn execute( + guard: &Guard, + package: &str, + targets: &[Target], + supervisor: &impl Supervisor, +) -> Result<()> { + guard.require_clear()?; + anyhow::ensure!( + !targets.is_empty() && targets.len() <= 32, + "Invalid supervised stack" + ); + let id = uuid::Uuid::new_v4().to_string(); + let mut members = Vec::new(); + for (index, target) in targets.iter().enumerate() { + let original = supervisor.capture(&target.name).await?; + // Inactive units require a durable explicit-start staging path; never + // implement this by starting and then stopping a user's stopped member. + anyhow::ensure!(original.running,"Stopped supervised member requires staged update; all original services remain unchanged"); + let prepared = supervisor.prepare_target(target, &original).await?; + anyhow::ensure!( + prepared.manifest.app.container.image.as_deref() == Some(target.reference.as_str()), + "Reviewed target manifest image changed" + ); + let target_body = pin_body(&prepared.body, &original.name, &target.reference)?; + anyhow::ensure!( + target_body == prepared.body, + "Reviewed target unit image changed" + ); + let pinned_original_body = pin_body( + &original.body, + &original.name, + &format!("sha256:{}", original.image), + )?; + members.push(Member { + original, + target: target.clone(), + target_body, + target_manifest: prepared.manifest, + pinned_original_body, + original_tag: format!("localhost/archy-update-recovery:{id}-{index}"), + recovery_image: None, + }); + } + let mut record = Journal { + schema: 1, + id, + package: package.into(), + phase: Phase::Prepared, + members, + }; + save(guard, &record)?; + for member in &record.members { + guard.hold(&member.original.name, &record.id)?; + } + let result = apply(guard, &mut record, supervisor).await; + if let Err(error) = result { + return match restore(guard,&mut record,supervisor).await { + Ok(())=>Err(error.context("Original supervised image/configuration and running intent restored; container IDs may change and data was not rolled back")), + Err(recovery)=>Err(error.context(format!("Supervised runtime recovery remains unresolved: {recovery:#}"))), + }; + } + Ok(()) +} +async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> { + for index in 0..record.members.len() { + let member = &record.members[index]; + anyhow::ensure!( + supervisor.read(&member.original.name).await? == member.original.body, + "Unit edited before update; originals retained" + ); + let image = supervisor + .snapshot(&member.original, &record.id, &member.original_tag) + .await?; + anyhow::ensure!( + digest(&image.image) + && image.source_container_id == member.original.container_id + && image.operation_id == record.id, + "Writable-layer recovery ownership mismatch" + ); + let member = &mut record.members[index]; + member.pinned_original_body = pin_body( + &member.original.body, + &member.original.name, + &format!("sha256:{}", image.image), + )?; + member.recovery_image = Some(image); + // The image acknowledgement becomes durable before any original stop. + save(guard, record)?; + } + record.phase = Phase::Editing; + save(guard, record)?; + for member in record.members.iter().rev() { + supervisor.stop(&member.original.name).await?; + } + for member in &record.members { + supervisor + .write( + &member.original.name, + &[member.original.body.clone()], + &member.target_body, + ) + .await?; + } + supervisor.reload().await?; + record.phase = Phase::Starting; + save(guard, record)?; + for member in &record.members { + supervisor.start(&member.original.name).await?; + supervisor + .target_hooks(&member.original.name, &member.target_manifest) + .await?; + let observed = supervisor + .observed(&member.original.name) + .await? + .context("Updated supervised member missing")?; + anyhow::ensure!( + observed.running + && observed.image == member.target.image + && supervisor.healthy(&member.original.name).await?, + "Updated supervised member failed verification" + ); + } + record.phase = Phase::Committed; + save(guard, record)?; + for member in &record.members { + guard.release_hold(&member.original.name, &record.id)?; + } + Ok(()) +} +async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> { + if record.phase == Phase::Prepared { + // Only image snapshots may have happened. Never stop/recreate an intact + // app just because preflight or snapshotting failed on another member. + for member in &record.members { + let original = supervisor + .observed(&member.original.name) + .await? + .context("Original service disappeared during preparation")?; + anyhow::ensure!( + supervisor.read(&member.original.name).await? == member.original.body + && original.id == member.original.container_id + && original.image == member.original.image + && original.running == member.original.running + && original.config_sha256 == member.original.config_sha256, + "Original service changed during preparation; recovery requires inspection" + ); + } + record.phase = Phase::Aborted; + save(guard, record)?; + for member in &record.members { + guard.release_hold(&member.original.name, &record.id)?; + } + return Ok(()); + } + // Refuse to overwrite a foreign edit before stopping any surviving member. + for member in &record.members { + let body = supervisor.read(&member.original.name).await?; + anyhow::ensure!( + [ + &member.original.body, + &member.target_body, + &member.pinned_original_body + ] + .contains(&&body), + "Foreign unit edit requires explicit recovery" + ); + supervisor + .pin( + &member + .recovery_image + .as_ref() + .context("Missing recovery image")? + .image, + &member.original_tag, + ) + .await?; + } + for member in record.members.iter().rev() { + supervisor.stop(&member.original.name).await?; + } + for member in &record.members { + supervisor + .write( + &member.original.name, + &[ + member.original.body.clone(), + member.target_body.clone(), + member.pinned_original_body.clone(), + ], + &member.pinned_original_body, + ) + .await?; + } + supervisor.reload().await?; + for member in &record.members { + if member.original.running { + supervisor.start(&member.original.name).await?; + } + let observed = supervisor.observed(&member.original.name).await?; + if member.original.running { + let current = observed.context("Original supervised service did not return")?; + anyhow::ensure!( + current.running + && current.image + == member + .recovery_image + .as_ref() + .context("Missing recovery image")? + .image + && current.config_sha256 == member.original.config_sha256, + "Original launch configuration did not recover" + ); + } else { + anyhow::ensure!( + observed.is_none_or(|v| !v.running), + "Originally stopped service unexpectedly running" + ); + } + } + for member in &record.members { + guard.hold(&member.original.name, &record.id)?; + } + record.phase = Phase::Restored; + save(guard, record) +} +pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Result<()> { + for mut record in records(guard)? { + match record.phase { + Phase::Committed | Phase::Aborted => { + for member in &record.members { + guard.release_hold(&member.original.name, &record.id)?; + } + } + Phase::Restored => {} // A later retry may own the current hold. + _ => restore(guard, &mut record, supervisor).await?, + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Mutex, + }; + struct Mock { + original: Unit, + body: Mutex, + running: AtomicBool, + calls: Mutex>, + fail_new_hooks: AtomicBool, + fail_snapshot: AtomicBool, + generation: AtomicUsize, + } + impl Mock { + fn new() -> Self { + let body=format!("[Container]\nContainerName=movie\nImage=sha256:{}\nEnvironment=OPERATOR_VALUE=retained\nPull=never\n[Service]\nRestart=always\n", "a".repeat(64)); + Self { + original: Unit { + name: "movie".into(), + body: body.clone(), + image: "a".repeat(64), + container_id: format!("{:064x}", 1), + running: true, + config_sha256: "c".repeat(64), + }, + body: Mutex::new(body), + running: AtomicBool::new(true), + calls: Default::default(), + fail_new_hooks: AtomicBool::new(false), + fail_snapshot: AtomicBool::new(false), + generation: AtomicUsize::new(1), + } + } + fn target() -> Target { + Target { + name: "movie".into(), + reference: format!("localhost/new@sha256:{}", "b".repeat(64)), + image: "b".repeat(64), + } + } + } + impl Supervisor for Mock { + async fn prepare_target( + &self, + target: &Target, + _original: &Unit, + ) -> Result { + let manifest = archipelago_container::AppManifest::parse(&format!( + "app:\n id: movie\n name: Movie\n version: 2.0.0\n container:\n image: {}\n", + target.reference + ))?; + let body = pin_body(&self.original.body, "movie", &target.reference)?.replace( + "Pull=never", + "Environment=NEW_FEATURE=enabled\nVolume=/identity:/run/identity:ro\nPull=never", + ); + Ok(PreparedTarget { body, manifest }) + } + async fn target_hooks( + &self, + _name: &str, + _manifest: &archipelago_container::AppManifest, + ) -> Result<()> { + self.calls.lock().unwrap().push("new-hooks".into()); + anyhow::ensure!( + !self.fail_new_hooks.load(Ordering::SeqCst), + "New provider hook failed" + ); + Ok(()) + } + async fn snapshot( + &self, + original: &Unit, + operation_id: &str, + _tag: &str, + ) -> Result { + self.calls.lock().unwrap().push("snapshot-original".into()); + anyhow::ensure!( + !self.fail_snapshot.load(Ordering::SeqCst), + "Snapshot failed" + ); + Ok(RecoveryImage { + image: "e".repeat(64), + source_container_id: original.container_id.clone(), + operation_id: operation_id.into(), + }) + } + async fn capture(&self, _name: &str) -> Result { + Ok(self.original.clone()) + } + async fn read(&self, _name: &str) -> Result { + Ok(self.body.lock().unwrap().clone()) + } + async fn write(&self, _name: &str, expected: &[String], body: &str) -> Result<()> { + let mut current = self.body.lock().unwrap(); + anyhow::ensure!(expected.contains(&*current), "Foreign unit edit"); + *current = body.into(); + self.calls.lock().unwrap().push("write".into()); + Ok(()) + } + async fn pin(&self, _image: &str, _tag: &str) -> Result<()> { + self.calls.lock().unwrap().push("pin-original".into()); + Ok(()) + } + async fn stop(&self, _name: &str) -> Result<()> { + self.calls.lock().unwrap().push("stop-unit".into()); + self.running.store(false, Ordering::SeqCst); + Ok(()) + } + async fn reload(&self) -> Result<()> { + self.calls.lock().unwrap().push("reload".into()); + Ok(()) + } + async fn start(&self, _name: &str) -> Result<()> { + self.calls.lock().unwrap().push("start-unit".into()); + self.running.store(true, Ordering::SeqCst); + self.generation.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + async fn observed(&self, _name: &str) -> Result> { + if !self.running.load(Ordering::SeqCst) { + return Ok(None); + } + let new = self.body.lock().unwrap().contains("NEW_FEATURE=enabled"); + Ok(Some(Observed { + id: format!("{:064x}", self.generation.load(Ordering::SeqCst)), + name: "movie".into(), + image: if new { + "b".repeat(64) + } else if self.body.lock().unwrap().contains(&"e".repeat(64)) { + "e".repeat(64) + } else { + "a".repeat(64) + }, + running: true, + retainable: false, + config_sha256: if new { "d".repeat(64) } else { "c".repeat(64) }, + })) + } + async fn healthy(&self, _name: &str) -> Result { + Ok(true) + } + } + #[tokio::test] + async fn snapshot_failure_never_stops_or_recreates_original_runtime() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + runtime.fail_snapshot.store(true, Ordering::SeqCst); + assert!(execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .is_err()); + assert_eq!(*runtime.calls.lock().unwrap(), ["snapshot-original"]); + let original = runtime.observed("movie").await.unwrap().unwrap(); + assert_eq!(original.id, runtime.original.container_id); + assert_eq!(original.image, runtime.original.image); + assert_eq!(*runtime.body.lock().unwrap(), runtime.original.body); + assert_eq!(records(&guard).unwrap()[0].phase, Phase::Aborted); + assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + recover(&guard, &runtime).await.unwrap(); + assert_eq!(*runtime.calls.lock().unwrap(), ["snapshot-original"]); + } + #[tokio::test] + async fn committed_restart_releases_only_its_own_hold_without_runtime_mutation() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap(); + let record = records(&guard).unwrap().pop().unwrap(); + guard.hold("movie", &record.id).unwrap(); + runtime.calls.lock().unwrap().clear(); + recover(&guard, &runtime).await.unwrap(); + assert!(runtime.calls.lock().unwrap().is_empty()); + assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + let next = uuid::Uuid::new_v4().to_string(); + guard.hold("movie", &next).unwrap(); + recover(&guard, &runtime).await.unwrap(); + assert!(super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + } + #[tokio::test] + async fn forward_applies_reviewed_new_configuration_and_hooks_not_only_image() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap(); + let body = runtime.body.lock().unwrap(); + assert!(body.contains("OPERATOR_VALUE=retained")); + assert!(body.contains("NEW_FEATURE=enabled")); + assert!(body.contains("Volume=/identity:/run/identity:ro")); + assert!(runtime.calls.lock().unwrap().contains(&"new-hooks".into())); + assert_eq!(records(&guard).unwrap()[0].phase, Phase::Committed); + assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + } + #[tokio::test] + async fn auto_remove_recovery_restores_old_configuration_without_claiming_original_id() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + runtime.fail_new_hooks.store(true, Ordering::SeqCst); + let error = execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap_err(); + assert!(error.to_string().contains("container IDs may change")); + assert_eq!( + *runtime.body.lock().unwrap(), + pin_body( + &runtime.original.body, + "movie", + &format!("sha256:{}", "e".repeat(64)) + ) + .unwrap() + ); + let restored = runtime.observed("movie").await.unwrap().unwrap(); + assert_eq!(restored.image, "e".repeat(64)); + assert_eq!(restored.config_sha256, runtime.original.config_sha256); + assert_ne!(restored.id, format!("{:064x}", 1)); + assert!(super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + assert_eq!(records(&guard).unwrap()[0].phase, Phase::Restored); + } + #[tokio::test] + async fn interrupted_restart_uses_saved_old_unit_even_after_catalog_plan_changes() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap(); + let mut record = records(&guard).unwrap().pop().unwrap(); + record.phase = Phase::Starting; + save(&guard, &record).unwrap(); + runtime.calls.lock().unwrap().clear(); + recover(&guard, &runtime).await.unwrap(); + assert_eq!( + *runtime.body.lock().unwrap(), + pin_body( + &runtime.original.body, + "movie", + &format!("sha256:{}", "e".repeat(64)) + ) + .unwrap() + ); + assert!(!runtime.calls.lock().unwrap().contains(&"new-hooks".into())); + } + #[tokio::test] + async fn foreign_unit_edit_blocks_recovery_before_stopping_other_services() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap(); + let mut record = records(&guard).unwrap().pop().unwrap(); + record.phase = Phase::Starting; + save(&guard, &record).unwrap(); + *runtime.body.lock().unwrap() = "operator replaced unit".into(); + runtime.calls.lock().unwrap().clear(); + assert!(recover(&guard, &runtime).await.is_err()); + assert!(runtime.calls.lock().unwrap().is_empty()); + assert_eq!(*runtime.body.lock().unwrap(), "operator replaced unit"); + } + #[tokio::test] + async fn unsupported_stopped_supervised_member_is_never_started_as_a_workaround() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let mut runtime = Mock::new(); + runtime.original.running = false; + runtime.running.store(false, Ordering::SeqCst); + assert!(execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .is_err()); + assert!(runtime.calls.lock().unwrap().is_empty()); + assert!(records(&guard).unwrap().is_empty()); + } +} diff --git a/core/archipelago/src/container/update_transaction.rs b/core/archipelago/src/container/update_transaction.rs index c632e9a2..429b5f86 100644 --- a/core/archipelago/src/container/update_transaction.rs +++ b/core/archipelago/src/container/update_transaction.rs @@ -74,6 +74,9 @@ pub(crate) struct Guard { root: PathBuf, } impl Guard { + pub(crate) fn directory(&self) -> &Path { + &self.root + } pub(crate) fn acquire(data: &Path) -> Result { let root = data.join("update-transactions"); match std::fs::DirBuilder::new().mode(0o700).create(&root) { @@ -153,7 +156,76 @@ impl Guard { } Ok(records) } + pub(crate) fn hold(&self, name: &str, operation: &str) -> Result<()> { + anyhow::ensure!(name_ok(name), "Invalid held container name"); + uuid::Uuid::parse_str(operation)?; + let directory = self.root.join("holds"); + match std::fs::DirBuilder::new().mode(0o700).create(&directory) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + let temporary = directory.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(operation.as_bytes())?; + file.sync_all()?; + std::fs::rename(temporary, directory.join(name))?; + std::fs::File::open(&directory)?.sync_all()?; + std::fs::File::open(&self.root)?.sync_all()?; + Ok(()) + } + pub(crate) fn release_hold(&self, name: &str, operation: &str) -> Result<()> { + anyhow::ensure!(name_ok(name), "Invalid held container name"); + let path = self.root.join("holds").join(name); + match std::fs::read_to_string(&path) { + Ok(owner) if owner == operation => { + std::fs::remove_file(&path)?; + std::fs::File::open(path.parent().unwrap())?.sync_all()?; + } + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + Ok(()) + } + pub(crate) fn held_names(&self) -> Result> { + let directory = self.root.join("holds"); + let mut held = HashSet::new(); + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(held), + Err(e) => return Err(e.into()), + }; + for entry in entries { + let entry = entry?; + let name = entry + .file_name() + .into_string() + .map_err(|_| anyhow::anyhow!("Invalid recovery hold name"))?; + if name.starts_with('.') { + continue; + } + anyhow::ensure!( + name_ok(&name) && entry.file_type()?.is_file() && entry.metadata()?.len() == 36, + "Damaged recovery hold" + ); + uuid::Uuid::parse_str(&std::fs::read_to_string(entry.path())?)?; + held.insert(name); + } + Ok(held) + } + pub(crate) fn require_unheld(&self, names: &[String]) -> Result<()> { + let held = self.held_names()?; + anyhow::ensure!(!names.iter().any(|name|held.contains(name)), + "Original app runtime is retained after rollback; retry its update before changing lifecycle"); + Ok(()) + } pub(crate) fn require_clear(&self) -> Result<()> { + super::supervised_update::require_clear(self)?; anyhow::ensure!( self.records()? .iter() @@ -260,6 +332,9 @@ pub(crate) async fn execute( members, }; guard.save(&record)?; + for member in &record.members { + guard.hold(&member.original.name, &record.operation)?; + } match replace(&guard,&mut record,runtime).await { Ok(())=>Ok(()), Err(error)=>match restore(&guard,&mut record,runtime).await { @@ -342,7 +417,11 @@ async fn replace(guard: &Guard, record: &mut Record, runtime: &impl Runtime) -> // Original backups are retained. Committing does not remove volumes or // silently discard the only recoverable original after a schema migration. record.phase = Phase::Committed; - guard.save(record) + guard.save(record)?; + for member in &record.members { + guard.release_hold(&member.original.name, &record.operation)?; + } + Ok(()) } async fn restore(guard: &Guard, record: &mut Record, runtime: &impl Runtime) -> Result<()> { // Preflight every original before mutating any replacement. Never infer @@ -401,8 +480,12 @@ async fn restore(guard: &Guard, record: &mut Record, runtime: &impl Runtime) -> "Original state was not restored" ); } + for member in &record.members { + guard.hold(&member.original.name, &record.operation)?; + } record.phase = Phase::Restored; - guard.save(record) + guard.save(record)?; + Ok(()) } /// Called before ordinary reconciliation. An unresolved recovery must prevent /// reconciliation from guessing a replacement or starting stopped originals. @@ -411,11 +494,32 @@ pub(crate) async fn recover(data: &Path, runtime: &impl Runtime) -> Result Result { + anyhow::ensure!(name_ok(name), "Invalid recovery name"); + let path = data.join("update-transactions/holds").join(name); + match std::fs::symlink_metadata(&path) { + Ok(meta) => { + anyhow::ensure!( + meta.is_file() && meta.len() == 36, + "Damaged update recovery hold" + ); + uuid::Uuid::parse_str(&std::fs::read_to_string(path)?)?; + Ok(true) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e.into()), + } +} + pub(crate) struct Podman; impl Podman { async fn output(args: &[&str]) -> Result { @@ -868,6 +972,27 @@ mod tests { assert!(recover(root.path(), &runtime).await.is_err()); assert!(runtime.calls.lock().unwrap().is_empty()); } + #[tokio::test] + async fn restored_original_remains_held_until_explicit_successful_update() { + let root = tempfile::tempdir().unwrap(); + let runtime = Mock::new(); + runtime.fail_health.store(true, Ordering::SeqCst); + let guard = Guard::acquire(root.path()).unwrap(); + assert!(execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .is_err()); + assert!(is_held(root.path(), "db").unwrap()); + assert!(guard.require_unheld(&["db".into()]).is_err()); + drop(guard); + let guard = recover(root.path(), &runtime).await.unwrap(); + assert!(is_held(root.path(), "db").unwrap()); + runtime.fail_health.store(false, Ordering::SeqCst); + execute(&guard, "stack", &Mock::targets(), &runtime) + .await + .unwrap(); + assert!(!is_held(root.path(), "db").unwrap()); + guard.require_unheld(&["db".into(), "web".into()]).unwrap(); + } #[test] fn lifecycle_lock_excludes_competing_commands() { let root = tempfile::tempdir().unwrap(); From 6c030a8109fa4fe79e27287d9992002b4037912a Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 01:52:36 -0400 Subject: [PATCH 04/13] Plan reviewed Quadlet migrations and preserve private writable-layer snapshots --- .../src/container/supervised_update.rs | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs index 37fe6a94..0f4d736b 100644 --- a/core/archipelago/src/container/supervised_update.rs +++ b/core/archipelago/src/container/supervised_update.rs @@ -97,6 +97,114 @@ pub(crate) trait Supervisor: Sync { fn observed(&self, name: &str) -> impl Future>> + Send; fn healthy(&self, name: &str) -> impl Future> + Send; } +/// Local recovery images are never pushed or exported. A matching tag may be +/// reused after a lost commit response only when image labels bind the exact +/// original container and transaction. Mounted data is deliberately excluded. +pub(crate) async fn capture_local_recovery_image( + original: &Unit, + operation: &str, + tag: &str, +) -> Result { + use super::update_transaction::{Podman, Runtime}; + anyhow::ensure!( + digest(&original.container_id) + && uuid::Uuid::parse_str(operation)?.to_string() == operation + && tag.starts_with(&format!("localhost/archy-update-recovery:{operation}-")) + && tag + .rsplit('-') + .next() + .is_some_and(|v| v.parse::().is_ok()), + "Invalid local recovery image ownership" + ); + async fn command(args: &[&str]) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(300), + tokio::process::Command::new("podman") + .args(args) + .kill_on_drop(true) + .output(), + ) + .await + .context("Recovery image operation timed out; original runtime retained")? + .context("Recovery image runtime unavailable") + } + async fn inspect(tag: &str, original: &Unit, operation: &str) -> Result { + let result = command(&["image", "inspect", tag]).await?; + anyhow::ensure!( + result.status.success(), + "Local recovery image inspection failed" + ); + let rows: Vec = serde_json::from_slice(&result.stdout)?; + anyhow::ensure!(rows.len() == 1, "Ambiguous recovery image"); + let row = &rows[0]; + let labels = row + .pointer("/Config/Labels") + .context("Recovery image has no ownership labels")?; + anyhow::ensure!( + labels + .get("io.archipelago.recovery.operation") + .and_then(|v| v.as_str()) + == Some(operation) + && labels + .get("io.archipelago.recovery.container") + .and_then(|v| v.as_str()) + == Some(original.container_id.as_str()), + "Existing recovery image belongs to another operation" + ); + let image = row + .get("Id") + .or_else(|| row.get("ID")) + .and_then(|v| v.as_str()) + .context("Recovery image identity missing")? + .trim_start_matches("sha256:") + .to_string(); + anyhow::ensure!(digest(&image), "Invalid recovery image digest"); + Ok(RecoveryImage { + image, + source_container_id: original.container_id.clone(), + operation_id: operation.into(), + }) + } + let current = Podman + .inspect(&original.name) + .await? + .context("Original container missing before snapshot")?; + anyhow::ensure!( + current.id == original.container_id + && current.image == original.image + && current.running == original.running + && current.config_sha256 == original.config_sha256, + "Original runtime changed before writable-layer snapshot" + ); + let exists = command(&["image", "exists", tag]).await?; + match exists.status.code() { + Some(0) => return inspect(tag, original, operation).await, + Some(1) => {} + _ => anyhow::bail!("Recovery image inventory unavailable"), + } + let operation_label = format!("LABEL io.archipelago.recovery.operation={operation}"); + let container_label = format!( + "LABEL io.archipelago.recovery.container={}", + original.container_id + ); + let result = command(&[ + "commit", + "--pause=true", + "--include-volumes=false", + "--change", + &operation_label, + "--change", + &container_label, + &original.container_id, + tag, + ]) + .await?; + anyhow::ensure!( + result.status.success(), + "Writable-layer snapshot failed; original runtime retained" + ); + inspect(tag, original, operation).await +} fn simple(value: &str) -> bool { !value.is_empty() && value.len() <= 128 @@ -151,6 +259,122 @@ pub(crate) fn pin_body(body: &str, name: &str, image: &str) -> Result { ); Ok(output) } +/// Three-way configuration migration: the previous renderer output must come +/// from the installed immutable manifest, never today's mutable catalog. Keep +/// unrelated operator overrides; a conflicting required change is a preflight +/// error rather than an overwrite. Values are never included in errors. +pub(crate) fn merge_reviewed_unit(original: &str, previous: &str, next: &str) -> Result { + type Key = (String, String); + fn parse(body: &str) -> Result<(Vec, std::collections::BTreeMap>)> { + anyhow::ensure!(body.len() <= 1024 * 1024, "Unit exceeds migration limit"); + let mut section = String::new(); + let mut sections = std::collections::HashSet::new(); + let mut order = Vec::new(); + let mut values = std::collections::BTreeMap::>::new(); + for raw in body.lines() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + anyhow::ensure!( + !line.ends_with('\\') && !line.starts_with(".include"), + "Continued or included unit cannot be migrated automatically" + ); + if line.starts_with('[') { + anyhow::ensure!( + line.ends_with(']') && sections.insert(line.to_string()), + "Repeated or malformed unit section" + ); + section = line.into(); + continue; + } + let (directive, value) = line.split_once('=').context("Malformed unit directive")?; + anyhow::ensure!( + !section.is_empty() + && !value.is_empty() + && directive.bytes().all(|b| b.is_ascii_alphanumeric()) + && !matches!(directive, "EnvironmentFile" | "EnvFile"), + "Unsupported unit reset or external configuration" + ); + let key = if directive == "Environment" { + let env = value.strip_prefix('"').unwrap_or(value); + let (name, _) = env + .split_once('=') + .context("Unsupported environment directive")?; + anyhow::ensure!( + !name.is_empty() + && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') + && (value.starts_with('"') && value.ends_with('"') + || !value.contains(char::is_whitespace)), + "Ambiguous environment directive" + ); + format!("Environment:{name}") + } else { + directive.into() + }; + let key = (section.clone(), key); + if !values.contains_key(&key) { + order.push(key.clone()); + } + values.entry(key).or_default().push(raw.to_string()); + } + Ok((order, values)) + } + let (mut order, original) = parse(original)?; + let (_, previous) = parse(previous)?; + let (next_order, next) = parse(next)?; + let mut merged = original.clone(); + let keys: std::collections::BTreeSet<_> = previous.keys().chain(next.keys()).cloned().collect(); + for key in keys { + let old = previous.get(&key); + let wanted = next.get(&key); + if old == wanted { + continue; + } + let actual = original.get(&key); + anyhow::ensure!( + actual == old || actual == wanted, + "Required unit configuration conflicts with an operator override in {} {}", + key.0, + key.1 + ); + match wanted { + Some(lines) => { + merged.insert(key, lines.clone()); + } + None => { + merged.remove(&key); + } + } + } + for key in next_order { + if !order.contains(&key) { + order.push(key); + } + } + // Group sections once; preserve directive and repeated-value order within + // each section, including operator-only directives absent from both plans. + let mut sections = Vec::::new(); + for (section, _) in &order { + if !sections.contains(section) { + sections.push(section.clone()); + } + } + let mut output = String::new(); + for section in sections { + output.push_str(§ion); + output.push('\n'); + for key in order.iter().filter(|key| key.0 == section) { + if let Some(lines) = merged.get(key) { + for line in lines { + output.push_str(line); + output.push('\n'); + } + } + } + } + Ok(output) +} fn root(guard: &Guard) -> Result { let path = guard.directory().join("supervised"); match std::fs::DirBuilder::new().mode(0o700).create(&path) { @@ -672,6 +896,42 @@ mod tests { Ok(true) } } + #[test] + fn reviewed_migration_keeps_unrelated_operator_values_and_applies_required_new_fields() { + let old = "[Container]\nImage=old\nEnvironment=FEATURE=old\nEnvironment=PORT=1\n[Service]\nRestart=always\n"; + let actual = old + .replace("PORT=1", "PORT=42") + .replace("[Service]", "Environment=OPERATOR=mine\n[Service]"); + let next = old + .replace("Image=old", "Image=new") + .replace("FEATURE=old", "FEATURE=new") + .replace("[Service]", "Volume=/identity:/run/identity:ro\n[Service]"); + let merged = merge_reviewed_unit(&actual, old, &next).unwrap(); + assert!(merged.contains("Image=new")); + assert!(merged.contains("FEATURE=new")); + assert!(merged.contains("PORT=42")); + assert!(merged.contains("OPERATOR=mine")); + assert!(merged.contains("Volume=/identity:/run/identity:ro")); + assert_eq!(merge_reviewed_unit(&merged, old, &next).unwrap(), merged); + } + #[test] + fn conflicting_required_environment_or_mount_migration_is_rejected_before_lifecycle() { + let old = "[Container]\nEnvironment=FEATURE=old\nVolume=/old:/data\n"; + assert!(merge_reviewed_unit( + &old.replace("FEATURE=old", "FEATURE=operator"), + old, + &old.replace("FEATURE=old", "FEATURE=new") + ) + .is_err()); + assert!(merge_reviewed_unit( + &old.replace("/old:/data", "/operator:/data"), + old, + &old.replace("/old:/data", "/required:/data") + ) + .is_err()); + assert!(merge_reviewed_unit("[Container]\nEnvironment=\n", old, old).is_err()); + assert!(merge_reviewed_unit("[Container]\nEnvironment=A=1 B=2\n", old, old).is_err()); + } #[tokio::test] async fn snapshot_failure_never_stops_or_recreates_original_runtime() { let root = tempfile::tempdir().unwrap(); From 6e7ea8b9d6679b30e9e5dfa8f1252ec7cc03649b Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:05:52 -0400 Subject: [PATCH 05/13] Add managed runtime adapter and require operation-owned write drain through update --- core/archipelago/src/container/mod.rs | 1 + .../src/container/supervised_runtime.rs | 427 ++++++++++++++++++ .../src/container/supervised_update.rs | 117 ++++- 3 files changed, 539 insertions(+), 6 deletions(-) create mode 100644 core/archipelago/src/container/supervised_runtime.rs diff --git a/core/archipelago/src/container/mod.rs b/core/archipelago/src/container/mod.rs index 98db75e1..465dc48d 100644 --- a/core/archipelago/src/container/mod.rs +++ b/core/archipelago/src/container/mod.rs @@ -34,4 +34,5 @@ mod staged_update; pub(crate) mod update_transaction; +pub(crate) mod supervised_runtime; pub(crate) mod supervised_update; diff --git a/core/archipelago/src/container/supervised_runtime.rs b/core/archipelago/src/container/supervised_runtime.rs new file mode 100644 index 00000000..21f51705 --- /dev/null +++ b/core/archipelago/src/container/supervised_runtime.rs @@ -0,0 +1,427 @@ +//! Production systemd/Podman adapter. Application write admission/drain is an +//! explicit dependency: neither process pause nor a filesystem receipt is drain. +use super::{ + supervised_update::{self, PreparedTarget, RecoveryImage, Supervisor, Unit}, + update_transaction::{Observed, Podman, Runtime, Target}, +}; +use anyhow::{Context, Result}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + future::Future, + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, + time::Duration, +}; + +pub(crate) trait DrainBarrier: Sync { + /// Persist ownership before blocking admissions. Return only after API, + /// direct uploads and workers have drained and the coherent backup finished. + fn acquire( + &self, + operation: &str, + originals: &[Unit], + ) -> impl Future> + Send; + /// Must inspect the live admission fence and same-operation ownership, not + /// merely trust an old 'backup complete' file. Keep it through snapshot/stop. + fn verify(&self, operation: &str) -> impl Future> + Send; + /// Idempotent; an old recovery must never clear a newer operation's fence. + fn release(&self, operation: &str) -> impl Future> + Send; +} + +#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MigrationPlan { + original_sha256: String, + prepared: PreparedTarget, +} +impl MigrationPlan { + /// The old render must be reproducible from an image-bound retained original + /// manifest. A current target catalog is not an installed-version receipt. + pub(crate) fn from_installed_render( + original: &str, + old_render: &str, + target_render: &str, + manifest: archipelago_container::AppManifest, + ) -> Result { + manifest.validate()?; + let body = supervised_update::merge_reviewed_unit(original, old_render, target_render)?; + Ok(Self { + original_sha256: hex::encode(Sha256::digest(original.as_bytes())), + prepared: PreparedTarget { body, manifest }, + }) + } + /// Legacy overrides whose provenance cannot be reconstructed need a reviewed + /// exact-original-hash-bound migration, prepared by node administration code. + /// No browser RPC accepts unit bodies or this type. + pub(crate) fn from_reviewed_legacy( + original_sha256: &str, + body: String, + manifest: archipelago_container::AppManifest, + ) -> Result { + anyhow::ensure!( + original_sha256.len() == 64 && original_sha256.bytes().all(|v| v.is_ascii_hexdigit()), + "Invalid original unit commitment" + ); + manifest.validate()?; + Ok(Self { + original_sha256: original_sha256.into(), + prepared: PreparedTarget { body, manifest }, + }) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct PlanFile { + schema: u8, + package: String, + plans: HashMap, +} +/// Administrative preparation is separate from the Update RPC. The RPC never +/// accepts unit bodies, hook commands, or a browser-chosen preparation path. +pub(crate) fn load_reviewed_plans( + data_dir: &Path, + package: &str, +) -> Result> { + anyhow::ensure!( + !package.is_empty() + && package.len() <= 128 + && package + .bytes() + .all(|v| v.is_ascii_alphanumeric() || matches!(v, b'-' | b'_')), + "Invalid managed package" + ); + let dir = data_dir.join("managed-update-plans"); + let directory = std::fs::symlink_metadata(&dir) + .context("No original-bound managed update plan; existing app remains unchanged")?; + anyhow::ensure!( + directory.is_dir() + && !directory.file_type().is_symlink() + && directory.uid() == unsafe { libc::geteuid() } + && directory.mode() & 0o077 == 0, + "Managed update plan directory must be private and node-owned" + ); + let path = dir.join(format!("{package}.json")); + let metadata = std::fs::symlink_metadata(&path)?; + anyhow::ensure!( + metadata.is_file() + && !metadata.file_type().is_symlink() + && metadata.uid() == unsafe { libc::geteuid() } + && metadata.mode() & 0o077 == 0 + && metadata.len() <= 4 * 1024 * 1024, + "Managed update plan must be private and node-owned" + ); + let record: PlanFile = serde_json::from_slice(&std::fs::read(path)?)?; + anyhow::ensure!( + record.schema == 1 + && record.package == package + && !record.plans.is_empty() + && record.plans.len() <= 32, + "Managed update plan does not match this package" + ); + for plan in record.plans.values() { + anyhow::ensure!( + plan.original_sha256.len() == 64 + && plan.original_sha256.bytes().all(|v| v.is_ascii_hexdigit()), + "Invalid original unit commitment" + ); + plan.prepared.manifest.validate()?; + } + Ok(record.plans) +} + +pub(crate) struct SystemdSupervisor { + data_dir: PathBuf, + unit_dir: PathBuf, + plans: HashMap, + barrier: B, +} +impl SystemdSupervisor { + pub(crate) async fn new( + data_dir: PathBuf, + plans: HashMap, + barrier: B, + ) -> Result { + let unit_dir = super::quadlet::unit_dir().await?; + let meta = std::fs::symlink_metadata(&unit_dir)?; + anyhow::ensure!( + meta.is_dir() + && !meta.file_type().is_symlink() + && meta.uid() == unsafe { libc::geteuid() } + && meta.mode() & 0o022 == 0, + "Quadlet directory ownership changed" + ); + Ok(Self { + data_dir, + unit_dir, + plans, + barrier, + }) + } + fn path(&self, name: &str) -> Result { + anyhow::ensure!( + !name.is_empty() + && name.len() <= 128 + && name + .bytes() + .all(|v| v.is_ascii_alphanumeric() || matches!(v, b'-' | b'_')), + "Invalid supervised unit name" + ); + Ok(self.unit_dir.join(format!("{name}.container"))) + } + async fn manager(args: &[&str]) -> Result { + let output = tokio::time::timeout( + Duration::from_secs(120), + tokio::process::Command::new("systemctl") + .arg("--user") + .args(args) + .kill_on_drop(true) + .output(), + ) + .await + .context("User service manager timed out")??; + anyhow::ensure!( + output.status.success(), + "User service manager rejected operation" + ); + Ok(String::from_utf8(output.stdout)?) + } + async fn owned_file(&self, name: &str) -> Result<(PathBuf, std::fs::Metadata)> { + let expected = self.path(name)?; + let service = format!("{name}.service"); + let output = Self::manager(&[ + "show", + &service, + "--property=SourcePath", + "--property=DropInPaths", + "--property=LoadState", + ]) + .await?; + let values: HashMap<_, _> = output + .lines() + .filter_map(|line| line.split_once('=')) + .collect(); + anyhow::ensure!( + values.get("LoadState") == Some(&"loaded") + && values.get("DropInPaths") == Some(&"") + && values + .get("SourcePath") + .is_some_and(|path| Path::new(path) == expected), + "Service is not owned by the exact original source Quadlet or has external overrides" + ); + let meta = std::fs::symlink_metadata(&expected)?; + anyhow::ensure!( + meta.is_file() + && !meta.file_type().is_symlink() + && meta.uid() == unsafe { libc::geteuid() } + && meta.mode() & 0o022 == 0 + && meta.len() <= 1024 * 1024, + "Original Quadlet ownership changed" + ); + Ok((expected, meta)) + } +} +impl Supervisor for SystemdSupervisor { + async fn begin_barrier(&self, operation: &str, originals: &[Unit]) -> Result<()> { + self.barrier.acquire(operation, originals).await + } + async fn verify_barrier(&self, operation: &str) -> Result<()> { + self.barrier.verify(operation).await + } + async fn release_barrier(&self, operation: &str) -> Result<()> { + self.barrier.release(operation).await + } + async fn prepare_target(&self, target: &Target, original: &Unit) -> Result { + let plan = self + .plans + .get(&target.name) + .context("No reviewed original-bound managed migration; app unchanged")?; + anyhow::ensure!( + hex::encode(Sha256::digest(original.body.as_bytes())) == plan.original_sha256 + && plan.prepared.manifest.app.container.image.as_deref() + == Some(target.reference.as_str()) + && super::prod_orchestrator::compute_container_name(&plan.prepared.manifest) + == target.name, + "Original unit or reviewed immutable target changed before update" + ); + Ok(plan.prepared.clone()) + } + async fn target_hooks( + &self, + name: &str, + manifest: &archipelago_container::AppManifest, + ) -> Result<()> { + super::hooks::run_post_install_strict(manifest, name, &self.data_dir).await + } + async fn capture(&self, name: &str) -> Result { + let (path, meta) = self.owned_file(name).await?; + let body = std::fs::read_to_string(path)?; + let observed = Podman + .inspect(name) + .await? + .context("Original supervised runtime missing")?; + Ok(Unit { + name: name.into(), + body, + image: observed.image, + container_id: observed.id, + file_mode: meta.mode() & 0o777, + running: observed.running, + config_sha256: observed.config_sha256, + }) + } + async fn validate_original_file(&self, original: &Unit) -> Result<()> { + let (_, meta) = self.owned_file(&original.name).await?; + anyhow::ensure!( + meta.mode() & 0o777 == original.file_mode, + "Original Quadlet mode changed" + ); + Ok(()) + } + async fn read(&self, name: &str) -> Result { + let (path, _) = self.owned_file(name).await?; + Ok(std::fs::read_to_string(path)?) + } + async fn write(&self, original: &Unit, expected: &[String], body: &str) -> Result<()> { + let (path, meta) = self.owned_file(&original.name).await?; + anyhow::ensure!( + meta.mode() & 0o777 == original.file_mode + && expected.contains(&std::fs::read_to_string(&path)?), + "Original Quadlet changed; refusing to replace a foreign edit" + ); + let temporary = self + .unit_dir + .join(format!(".archy-update-{}.tmp", uuid::Uuid::new_v4())); + // Commit point is synchronous while the transaction guard is held; + // cancelled futures cannot rename over a later recovery after unlock. + let result = (|| -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(original.file_mode) + .open(&temporary)?; + file.write_all(body.as_bytes())?; + file.sync_all()?; + std::fs::rename(&temporary, &path)?; + std::fs::File::open(&self.unit_dir)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result + } + async fn snapshot(&self, original: &Unit, operation: &str, tag: &str) -> Result { + self.barrier.verify(operation).await?; + supervised_update::capture_local_recovery_image(original, operation, tag).await + } + async fn pin(&self, image: &str, tag: &str) -> Result<()> { + anyhow::ensure!( + image.len() == 64 + && image.bytes().all(|v| v.is_ascii_hexdigit()) + && tag.starts_with("localhost/archy-update-recovery:"), + "Invalid recovery image pin" + ); + let output = tokio::time::timeout( + Duration::from_secs(30), + tokio::process::Command::new("podman") + .args(["tag", &format!("sha256:{image}"), tag]) + .kill_on_drop(true) + .output(), + ) + .await??; + anyhow::ensure!( + output.status.success(), + "Original recovery image unavailable" + ); + Ok(()) + } + async fn stop(&self, name: &str) -> Result<()> { + self.owned_file(name).await?; + super::quadlet::stop_service_with_timeout( + &format!("{name}.service"), + Duration::from_secs(archipelago_container::runtime::stop_grace_secs_for(name) + 30), + ) + .await + } + async fn reload(&self) -> Result<()> { + super::quadlet::daemon_reload_user().await + } + async fn start(&self, name: &str) -> Result<()> { + self.owned_file(name).await?; + super::quadlet::enable_now(&format!("{name}.service")).await + } + async fn observed(&self, name: &str) -> Result> { + Podman.inspect(name).await + } + async fn healthy(&self, name: &str) -> Result { + Podman.healthy(name).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + struct UnusedBarrier; + impl DrainBarrier for UnusedBarrier { + async fn acquire(&self, _: &str, _: &[Unit]) -> Result<()> { + anyhow::bail!("No application barrier installed") + } + async fn verify(&self, _: &str) -> Result<()> { + anyhow::bail!("No application barrier installed") + } + async fn release(&self, _: &str) -> Result<()> { + Ok(()) + } + } + #[tokio::test] + async fn original_bound_plan_rejects_changed_unit_or_image_without_service_calls() { + let image = format!("localhost/movie@sha256:{}", "b".repeat(64)); + let manifest = archipelago_container::AppManifest::parse(&format!( + "app:\n id: movie\n name: Movie\n version: 2.0.0\n container:\n image: {image}\n")).unwrap(); + let original_body = "[Container]\nContainerName=movie\nImage=old\n"; + let next_body = format!("[Container]\nContainerName=movie\nImage={image}\n"); + let plan = MigrationPlan::from_reviewed_legacy( + &hex::encode(Sha256::digest(original_body)), + next_body.clone(), + manifest, + ) + .unwrap(); + let adapter = SystemdSupervisor { + data_dir: PathBuf::from("/unused"), + unit_dir: PathBuf::from("/unused"), + plans: HashMap::from([("movie".into(), plan)]), + barrier: UnusedBarrier, + }; + let mut original = Unit { + name: "movie".into(), + body: original_body.into(), + image: "a".repeat(64), + container_id: "c".repeat(64), + file_mode: 0o600, + running: true, + config_sha256: "d".repeat(64), + }; + let mut target = Target { + name: "movie".into(), + reference: image, + image: "b".repeat(64), + }; + assert_eq!( + adapter + .prepare_target(&target, &original) + .await + .unwrap() + .body, + next_body + ); + original.body.push_str("Environment=OPERATOR=changed\n"); + assert!(adapter.prepare_target(&target, &original).await.is_err()); + original.body = original_body.into(); + target.reference = format!("localhost/movie@sha256:{}", "e".repeat(64)); + assert!(adapter.prepare_target(&target, &original).await.is_err()); + assert!(adapter.begin_barrier("unused", &[original]).await.is_err()); + } +} diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs index 0f4d736b..3d96d3fe 100644 --- a/core/archipelago/src/container/supervised_update.rs +++ b/core/archipelago/src/container/supervised_update.rs @@ -15,6 +15,7 @@ pub(crate) struct Unit { pub body: String, pub image: String, pub container_id: String, + pub file_mode: u32, pub running: bool, pub config_sha256: String, } @@ -59,6 +60,16 @@ struct Journal { } pub(crate) trait Supervisor: Sync { + /// Admission must be fenced and in-flight application work drained under + /// this durable operation before snapshots. A paused process is not proof. + /// Acquisition/release are idempotent and operation-owned across restart. + fn begin_barrier( + &self, + operation: &str, + originals: &[Unit], + ) -> impl Future> + Send; + fn verify_barrier(&self, operation: &str) -> impl Future> + Send; + fn release_barrier(&self, operation: &str) -> impl Future> + Send; /// Only an internal reviewed signed-manifest planner may supply this value; /// browser parameters must never become a unit body or hook recipe. fn prepare_target( @@ -83,10 +94,11 @@ pub(crate) trait Supervisor: Sync { tag: &str, ) -> impl Future> + Send; fn capture(&self, name: &str) -> impl Future> + Send; + fn validate_original_file(&self, original: &Unit) -> impl Future> + Send; fn read(&self, name: &str) -> impl Future> + Send; fn write( &self, - name: &str, + original: &Unit, expected: &[String], body: &str, ) -> impl Future> + Send; @@ -406,6 +418,8 @@ fn validate(record: &Journal) -> Result<()> { && digest(&member.target.image) && digest(&member.original.image) && digest(&member.original.container_id) + && member.original.file_mode & !0o777 == 0 + && member.original.file_mode & 0o022 == 0 && digest(&member.original.config_sha256), "Invalid original supervised identity" ); @@ -567,6 +581,10 @@ pub(crate) async fn execute( } let result = apply(guard, &mut record, supervisor).await; if let Err(error) = result { + if record.phase == Phase::Committed { + return Err(error + .context("Update committed; completion cleanup must resume without rolling back")); + } return match restore(guard,&mut record,supervisor).await { Ok(())=>Err(error.context("Original supervised image/configuration and running intent restored; container IDs may change and data was not rolled back")), Err(recovery)=>Err(error.context(format!("Supervised runtime recovery remains unresolved: {recovery:#}"))), @@ -575,12 +593,20 @@ pub(crate) async fn execute( Ok(()) } async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> { + let originals: Vec<_> = record + .members + .iter() + .map(|member| member.original.clone()) + .collect(); + supervisor.begin_barrier(&record.id, &originals).await?; for index in 0..record.members.len() { let member = &record.members[index]; + supervisor.validate_original_file(&member.original).await?; anyhow::ensure!( supervisor.read(&member.original.name).await? == member.original.body, "Unit edited before update; originals retained" ); + supervisor.verify_barrier(&record.id).await?; let image = supervisor .snapshot(&member.original, &record.id, &member.original_tag) .await?; @@ -600,15 +626,17 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor // The image acknowledgement becomes durable before any original stop. save(guard, record)?; } + supervisor.verify_barrier(&record.id).await?; record.phase = Phase::Editing; save(guard, record)?; for member in record.members.iter().rev() { + supervisor.verify_barrier(&record.id).await?; supervisor.stop(&member.original.name).await?; } for member in &record.members { supervisor .write( - &member.original.name, + &member.original, &[member.original.body.clone()], &member.target_body, ) @@ -635,6 +663,7 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor } record.phase = Phase::Committed; save(guard, record)?; + supervisor.release_barrier(&record.id).await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } @@ -660,13 +689,16 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis } record.phase = Phase::Aborted; save(guard, record)?; + supervisor.release_barrier(&record.id).await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } return Ok(()); } + supervisor.verify_barrier(&record.id).await?; // Refuse to overwrite a foreign edit before stopping any surviving member. for member in &record.members { + supervisor.validate_original_file(&member.original).await?; let body = supervisor.read(&member.original.name).await?; anyhow::ensure!( [ @@ -694,7 +726,7 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis for member in &record.members { supervisor .write( - &member.original.name, + &member.original, &[ member.original.body.clone(), member.target_body.clone(), @@ -734,17 +766,21 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis guard.hold(&member.original.name, &record.id)?; } record.phase = Phase::Restored; - save(guard, record) + save(guard, record)?; + supervisor.release_barrier(&record.id).await } pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Result<()> { for mut record in records(guard)? { match record.phase { Phase::Committed | Phase::Aborted => { + supervisor.release_barrier(&record.id).await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } } - Phase::Restored => {} // A later retry may own the current hold. + Phase::Restored => { + supervisor.release_barrier(&record.id).await?; + } // Never release a newer owner. _ => restore(guard, &mut record, supervisor).await?, } } @@ -765,6 +801,8 @@ mod tests { calls: Mutex>, fail_new_hooks: AtomicBool, fail_snapshot: AtomicBool, + fail_barrier_release: AtomicBool, + barrier: Mutex>, generation: AtomicUsize, } impl Mock { @@ -776,6 +814,7 @@ mod tests { body: body.clone(), image: "a".repeat(64), container_id: format!("{:064x}", 1), + file_mode: 0o600, running: true, config_sha256: "c".repeat(64), }, @@ -784,6 +823,8 @@ mod tests { calls: Default::default(), fail_new_hooks: AtomicBool::new(false), fail_snapshot: AtomicBool::new(false), + fail_barrier_release: AtomicBool::new(false), + barrier: Mutex::new(None), generation: AtomicUsize::new(1), } } @@ -796,6 +837,33 @@ mod tests { } } impl Supervisor for Mock { + async fn begin_barrier(&self, operation: &str, _originals: &[Unit]) -> Result<()> { + let mut held = self.barrier.lock().unwrap(); + anyhow::ensure!( + held.as_deref().is_none_or(|id| id == operation), + "Another drain barrier owns admission" + ); + *held = Some(operation.into()); + Ok(()) + } + async fn verify_barrier(&self, operation: &str) -> Result<()> { + anyhow::ensure!( + self.barrier.lock().unwrap().as_deref() == Some(operation), + "Write barrier not held" + ); + Ok(()) + } + async fn release_barrier(&self, operation: &str) -> Result<()> { + anyhow::ensure!( + !self.fail_barrier_release.load(Ordering::SeqCst), + "Barrier release unavailable" + ); + let mut held = self.barrier.lock().unwrap(); + if held.as_deref() == Some(operation) { + *held = None; + } + Ok(()) + } async fn prepare_target( &self, target: &Target, @@ -843,10 +911,13 @@ mod tests { async fn capture(&self, _name: &str) -> Result { Ok(self.original.clone()) } + async fn validate_original_file(&self, _original: &Unit) -> Result<()> { + Ok(()) + } async fn read(&self, _name: &str) -> Result { Ok(self.body.lock().unwrap().clone()) } - async fn write(&self, _name: &str, expected: &[String], body: &str) -> Result<()> { + async fn write(&self, _original: &Unit, expected: &[String], body: &str) -> Result<()> { let mut current = self.body.lock().unwrap(); anyhow::ensure!(expected.contains(&*current), "Foreign unit edit"); *current = body.into(); @@ -933,6 +1004,38 @@ mod tests { assert!(merge_reviewed_unit("[Container]\nEnvironment=A=1 B=2\n", old, old).is_err()); } #[tokio::test] + async fn lost_barrier_release_after_commit_never_rolls_back_a_verified_update() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + runtime.fail_barrier_release.store(true, Ordering::SeqCst); + let error = execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap_err(); + assert!(error.to_string().contains("Update committed")); + assert_eq!(records(&guard).unwrap()[0].phase, Phase::Committed); + assert_eq!( + runtime.observed("movie").await.unwrap().unwrap().image, + "b".repeat(64) + ); + assert_eq!( + runtime + .calls + .lock() + .unwrap() + .iter() + .filter(|call| *call == "stop-unit") + .count(), + 1 + ); + runtime.calls.lock().unwrap().clear(); + runtime.fail_barrier_release.store(false, Ordering::SeqCst); + recover(&guard, &runtime).await.unwrap(); + assert!(runtime.calls.lock().unwrap().is_empty()); + assert!(runtime.barrier.lock().unwrap().is_none()); + assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + } + #[tokio::test] async fn snapshot_failure_never_stops_or_recreates_original_runtime() { let root = tempfile::tempdir().unwrap(); let guard = Guard::acquire(root.path()).unwrap(); @@ -1022,6 +1125,7 @@ mod tests { .unwrap(); let mut record = records(&guard).unwrap().pop().unwrap(); record.phase = Phase::Starting; + *runtime.barrier.lock().unwrap() = Some(record.id.clone()); save(&guard, &record).unwrap(); runtime.calls.lock().unwrap().clear(); recover(&guard, &runtime).await.unwrap(); @@ -1046,6 +1150,7 @@ mod tests { .unwrap(); let mut record = records(&guard).unwrap().pop().unwrap(); record.phase = Phase::Starting; + *runtime.barrier.lock().unwrap() = Some(record.id.clone()); save(&guard, &record).unwrap(); *runtime.body.lock().unwrap() = "operator replaced unit".into(); runtime.calls.lock().unwrap().clear(); From 6d450caebf7cec8de72e1431f179757ec124f601 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:24:50 -0400 Subject: [PATCH 06/13] Add operation-owned legacy IndeeHub maintenance controller draft --- .../indeehub-legacy-maintenance-controller.md | 73 ++++++ scripts/indeehub-maintenance-controller.py | 244 ++++++++++++++++++ .../test_indeehub_maintenance_controller.py | 85 ++++++ 3 files changed, 402 insertions(+) create mode 100644 docs/indeehub-legacy-maintenance-controller.md create mode 100644 scripts/indeehub-maintenance-controller.py create mode 100644 tests/regression/test_indeehub_maintenance_controller.py diff --git a/docs/indeehub-legacy-maintenance-controller.md b/docs/indeehub-legacy-maintenance-controller.md new file mode 100644 index 00000000..3351704a --- /dev/null +++ b/docs/indeehub-legacy-maintenance-controller.md @@ -0,0 +1,73 @@ +# Legacy IndeeHub maintenance controller + +Status: isolated source implementation. Ten pure Python fake-runtime regressions +pass; no live invocation or production qualification. The controller is not part +of the already signed private app candidate and needs no new app image/API. + +The supervised updater owns the lifecycle flock, seven durable holds, original +Quadlets and private writable-layer recovery images. It records its destructive +obligation before invoking the fixed controller with bounded JSON over stdin: + +``` +python3 /opt/archipelago/scripts/indeehub-maintenance-controller.py acquire +{ "operation_id": "", "original_members": [ + { "name": "indeedhub", "container_id": "<64hex>", "image_id": "<64hex>", + "unit_sha256": "<64hex>", "config_sha256": "<64hex>", "running": true } + // All seven exact members; JSON does not include this illustrative comment. +], "recovery": false } +``` + +Other actions are `verify` with operation_id, and `release` with operation_id and +outcome committed/restored/aborted. Replies are <=4KiB and report drained, held, +released, or recovering for explicit recovery acquire. Inherited +ARCHY_UPDATE_LOCK_FD stays open and is passed to child commands; the script never +unlocks it. Journal: data/update-transactions/indeehub-maintenance//journal.json. + +## Forward sequence + +- Validate exact original IDs/source-unit hashes and known port exposure. Only + frontend127.0.0.1:7778 is supported; direct backend/S3 ports refuse before stop. +- Require deployed native AppGate and legacy nginx maintenance guards. Inspect + every known legacy sublocation and any direct7778 proxy; unknown routes refuse. + Save an operation-owned readable sentinel, then verify local ingress returns503. +- Record prior BullMQ transcode pause state, globally pause future job admission, + retain queued/delayed/failed jobs. Gracefully stop frontend ingress; bounded + polling waits for active transcodes to finish before stopping worker and API. +- Require successful systemd shutdown plus an exact original Podman died event + with exit0. Forced exits and missing event evidence retain the hold and are + never labelled completed writes. +- While PostgreSQL remains running, capture a fresh custom dump. Cleanly stop + MinIO/Redis/relay/Postgres, then archive all four complete quiescent volumes + (including SQLite WAL and Redis persistence) with metadata. No volume deletion + or migration rollback. Archive hashes/size and per-step obligations are durable. +- Keep admission closed while the updater renders, starts and verifies targets. + +## Interrupted recovery + +The node first records phase Restoring with boolean target_startup_began, then +calls acquire with recovery:true. That path preserves the original failure and +fence; it does not retry a killed original into a fictitious successful drain or +claim missing backups exist. The node restores exact saved old runtime under the +same hold. Release before any target startup can state only that original runtime +was restored. If target startup/migration began, recorded data-compatibility +verification is required before restored release; an old image alone does not +prove compatibility with newly changed data. No automatic DB/media restore exists. + +## Qualification and remaining integration + +`python3 tests/regression/test_indeehub_maintenance_controller.py` passes ten +fake-runtime cases in temporary directories, without services/network/containers. +Source nginx template guard coverage also passes its parser check. Production +adapter compilation, actual Podman event format/systemd clean-exit behavior, +application writer shutdown, interrupted backup and supervised restart still need +isolated lifecycle fixtures and then coordinated node acceptance. A long-lived +WebSocket or active upload can exceed graceful-stop deadlines; the current code +refuses completion and preserves recovery obligations rather than silently +calling interrupted work finished. + +The deployment must install the exact qualified controller script and record its +hash alongside the backend artifact. Binary-only deployment does not install it. +The backend must refuse missing/mismatched prerequisites before snapshots/stops. +Native AppGate + nginx guards are separate node source changes owned by the +supervised updater agent. The signed app catalog/private image receipts remain +unchanged. Existing live stop/uninstall intent must not be rewritten as maintenance. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py new file mode 100644 index 00000000..a152086e --- /dev/null +++ b/scripts/indeehub-maintenance-controller.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Operation-owned legacy IndeeHub maintenance. Called only by supervised updater. +No live execution is part of source qualification. Original writable-layer images +must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another hold. +""" +import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid +NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay') +DATA = pathlib.Path('/var/lib/archipelago') +QUEUE_SCRIPT = r'''const {Queue}=require('bullmq'); +(async()=>{const q=new Queue('transcode',{connection:{host:process.env.QUEUE_HOST,port:Number(process.env.QUEUE_PORT||6379),password:process.env.QUEUE_PASSWORD,maxRetriesPerRequest:1}}); +try{const action=process.argv[1];if(action==='pause')await q.pause();else if(action==='resume')await q.resume();else if(action!=='status')throw Error('action'); +console.log(JSON.stringify({paused:await q.isPaused(),counts:await q.getJobCounts('active','waiting','paused','delayed','failed','completed')}));} +finally{await q.close()}})().catch(()=>process.exit(1));''' +def require(condition, message): + if not condition: raise RuntimeError(message) +def atomic(path, value): + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + require(not path.is_symlink(), 'Refuse symbolic journal path') + temporary=path.with_name('.'+path.name+'.'+str(uuid.uuid4())) + with temporary.open('x') as stream: + json.dump(value,stream,separators=(',',':'));stream.flush();os.fsync(stream.fileno()) + os.chmod(temporary,0o600);os.replace(temporary,path) + descriptor=os.open(path.parent,os.O_RDONLY);os.fsync(descriptor);os.close(descriptor) +def sha(path): + with path.open('rb') as stream:return hashlib.file_digest(stream,'sha256').hexdigest() +def validate_members(members): + require(isinstance(members,list) and len(members)==7,'Seven exact original members required') + require({m.get('name') for m in members}==set(NAMES),'IndeeHub member scope changed') + for m in members: + require(set(m)=={'name','container_id','image_id','unit_sha256','config_sha256','running'},'Unexpected member fields') + for key in ('container_id','image_id','unit_sha256','config_sha256'): + require(bool(re.fullmatch('[0-9a-f]{64}',m[key])),'Invalid original identity/hash') + require(m['running'] is True,'Legacy barrier currently supports an originally running complete stack only') + return sorted(members,key=lambda m:m['name']) +def validate_nginx_guards(config): + guard='if (-f /var/lib/archipelago/app-maintenance/indeedhub) { return 503; }' + lines=config.splitlines();matched=0;index=0 + while index=3,'Expected complete legacy IndeeHub route guards') + return matched +class Controller: + def __init__(self, data, operation, lock_fd, runner=None): + self.data=pathlib.Path(data);self.operation=operation;self.lock_fd=lock_fd;self.runner=runner + require(str(uuid.UUID(operation))==operation,'Invalid operation UUID') + self.root=self.data/'update-transactions'/'indeehub-maintenance'/operation + self.path=self.root/'journal.json';self.fence=self.data/'app-maintenance'/'indeedhub' + self.record=json.loads(self.path.read_text()) if self.path.exists() else None + if self.record:require(self.record['operation_id']==operation,'Maintenance journal changed') + def save(self): atomic(self.path,self.record) + def run(self, argv, timeout=30, output=None): + if self.runner:return self.runner(argv,timeout,output) + self.root.mkdir(mode=0o700,parents=True,exist_ok=True) + with (self.root/'commands.private.log').open('ab') as errors: + result=subprocess.run(argv,stdout=output or subprocess.PIPE,stderr=errors,timeout=timeout,check=True,pass_fds=(self.lock_fd,)) + if output:return b'' + require(len(result.stdout)<=2*1024*1024,'Command response exceeds bound') + return result.stdout + def inspect(self, name): + rows=json.loads(self.run(['podman','inspect',name]));require(len(rows)==1,'Unexpected container inspection');return rows[0] + def holds(self): + for name in NAMES: + path=self.data/'update-transactions'/'holds'/name + require(path.is_file() and not path.is_symlink() and path.read_text()==self.operation,'Matching durable lifecycle hold required') + def fence_matches(self): + require(self.fence.is_file() and not self.fence.is_symlink() and self.fence.read_text()==self.operation,'Admission fence ownership changed') + def close_ingress(self): + # The deployed native AppGate and legacy nginx guards consume this exact + # sentinel. This code never edits arbitrary nginx configuration. + config=self.run(['sudo','-n','nginx','-T']).decode() + validate_nginx_guards(config) + self.fence.parent.mkdir(mode=0o755,exist_ok=True) + self.fence.parent.chmod(0o755) + require(not self.fence.parent.is_symlink(),'Admission directory is a symlink') + if self.fence.exists():self.fence_matches() + else: + with self.fence.open('x') as stream:stream.write(self.operation);stream.flush();os.fsync(stream.fileno()) + self.fence.chmod(0o644) + fd=os.open(self.fence.parent,os.O_RDONLY);os.fsync(fd);os.close(fd) + # A local legacy probe must be rejected without entering the old app. + import urllib.request,urllib.error + try: + urllib.request.urlopen('http://127.0.0.1/app/indeedhub/__maintenance_probe',timeout=5) + raise RuntimeError('Legacy ingress was not fenced') + except urllib.error.HTTPError as error: + require(error.code==503,'Legacy ingress guard did not return maintenance status') + self.record['ingress_closed']=True;self.save() + def queue(self, action): + result=json.loads(self.run(['podman','exec','indeedhub-api','node','-e',QUEUE_SCRIPT,action])) + require(type(result.get('paused')) is bool and isinstance(result.get('counts'),dict),'Invalid queue observation') + for value in result['counts'].values():require(type(value) is int and value>=0,'Invalid job count') + return result + def pause_queue(self): + if 'queue_was_paused' not in self.record: + original=self.queue('status');self.record['queue_was_paused']=original['paused'];self.record['queue_original_counts']=original['counts'];self.save() + state=self.queue('pause');require(state['paused'],'Worker admission did not close') + self.record['queue_pause_confirmed']=True;self.save() + def graceful_stop(self, name): + # Save the obligation before systemd can remove an AutoRemove container. + stopped=self.record.setdefault('stopped',{}) + if stopped.get(name,{}).get('confirmed'):return + member=next(m for m in self.record['original_members'] if m['name']==name) + if name not in stopped: + actual=self.inspect(name);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'],'Original container changed before stop') + stopped[name]={'intent_at':time.time(),'container_id':actual['Id']};self.save() + self.run(['systemctl','--user','stop',name+'.service'],timeout=180) + properties=self.run(['systemctl','--user','show',name+'.service','--property=ActiveState,SubState,Result,ExecMainStatus']).decode() + require('ActiveState=inactive' in properties and 'Result=success' in properties,'Service did not stop successfully') + # --rm removes inspect state. Require a persisted Podman died event for + # this exact original ID; a forced SIGKILL is never called completed work. + events=self.run(['podman','events','--stream=false','--since',str(int(stopped[name]['intent_at'])-1),'--filter','container='+member['container_id'],'--filter','event=died','--format','json']).decode().splitlines() + matching=[json.loads(line) for line in events if line.strip()] + matching=[event for event in matching if event.get('ID',event.get('id'))==member['container_id']] + require(matching,'Original process exit evidence unavailable; hold retained') + code=matching[-1].get('ContainerExitCode',matching[-1].get('containerExitCode')) + require(str(code)=='0','Original process did not exit cleanly; active work is not claimed completed') + stopped[name].update(confirmed=True,exit_code=0,confirmed_at=time.time());self.save() + def volume_sources(self): + expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data'] + rows=json.loads(self.run(['podman','volume','inspect',*expected])) + require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed') + return {row['Name']:row['Mountpoint'] for row in rows} + def backup(self): + if self.record.get('backup_complete'):return + sources=self.volume_sources();self.record['volume_sources']=sources;self.save() + backup=self.root/'backup';backup.mkdir(mode=0o700,exist_ok=True) + if 'database.dump' not in self.record.setdefault('artifacts',{}): + path=backup/'database.dump.partial' + with path.open('wb') as output:self.run(['podman','exec','indeedhub-postgres','pg_dump','-U','indeedhub','-d','indeedhub','--format=custom','--no-owner','--no-acl'],timeout=300,output=output);output.flush();os.fsync(output.fileno()) + final=backup/'database.dump';os.replace(path,final);self.record['artifacts']['database.dump']={'bytes':final.stat().st_size,'sha256':sha(final)};self.save() + # Redis stop flushes persisted queue state; its clean process exit is + # checked exactly as every other service. SQLite WAL is archived with DB. + for name in ('indeedhub-minio','indeedhub-redis','indeedhub-relay','indeedhub-postgres'):self.graceful_stop(name) + for volume,source in sources.items(): + name=volume+'.tar' + if name in self.record['artifacts']:continue + require(pathlib.Path(source).is_absolute() and source.endswith('/_data'),'Invalid volume mountpoint') + available=shutil.disk_usage(backup).free + measured=int(self.run(['podman','unshare','du','-sb',source]).decode().split()[0]) + require(available>measured+512*1024*1024,'Insufficient durable backup space') + partial=backup/(name+'.partial') + with partial.open('wb') as output:self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner','-C',source,'-cpf','-','.'],timeout=1800,output=output);output.flush();os.fsync(output.fileno()) + final=backup/name;os.replace(partial,final);self.record['artifacts'][name]={'bytes':final.stat().st_size,'sha256':sha(final)};self.save() + self.record['backup_complete']=True;self.record['phase']='Drained';self.save() + def acquire(self, members, recovery=False): + members=validate_members(members);self.holds() + if self.record:require(self.record['original_members']==members,'Original operation terms changed') + else: + self.record={'operation_id':self.operation,'original_members':members,'phase':'Prepared','created_at':time.time()};self.save() + require(self.record['phase']!='Released','Completed maintenance must not be reacquired') + if not recovery and not self.record.get('originals_validated'): + for member in members: + actual=self.inspect(member['name']);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'],'Original member changed') + bindings=actual['HostConfig'].get('PortBindings') or {} + if member['name']=='indeedhub':require(bindings=={'7777/tcp':[{'HostIp':'127.0.0.1','HostPort':'7778'}]},'Unsupported direct frontend exposure') + else:require(not bindings,'Unsupported direct writer exposure') + source=pathlib.Path(self.run(['systemctl','--user','show',member['name']+'.service','--property=SourcePath','--value']).decode().strip()) + require(source.is_file() and not source.is_symlink() and source.suffix=='.container','Original unit source missing') + require(source.stat().st_uid==os.getuid() and sha(source)==member['unit_sha256'],'Original unit source changed') + self.record['originals_validated']=True;self.save() + self.close_ingress() + if recovery: + runtime=json.loads((self.data/'update-transactions'/'supervised'/(self.operation+'.json')).read_text()) + require(runtime.get('phase')=='Restoring' and type(runtime.get('target_startup_began')) is bool,'Durable explicit restoring obligation required') + self.record['phase']='Recovering';self.record['target_startup_began']=runtime['target_startup_began'];self.save() + return {'operation_id':self.operation,'state':'recovering'} + if not self.record.get('stopped',{}).get('indeedhub-api',{}).get('confirmed'): + self.pause_queue() + self.graceful_stop('indeedhub') + deadline=time.monotonic()+300 + while True: + state=self.queue('status');require(state['paused'],'Worker admission reopened') + self.record['last_queue_counts']=state['counts'];self.save() + if state['counts'].get('active',0)==0:break + require(time.monotonic() Date: Wed, 7 Oct 2026 02:26:51 -0400 Subject: [PATCH 07/13] Recover aborted maintenance without fabricated drain or foreign fence changes --- docs/indeehub-legacy-maintenance-controller.md | 9 +++++++-- scripts/indeehub-maintenance-controller.py | 6 ++++++ .../test_indeehub_maintenance_controller.py | 13 +++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/indeehub-legacy-maintenance-controller.md b/docs/indeehub-legacy-maintenance-controller.md index 3351704a..a357ca8c 100644 --- a/docs/indeehub-legacy-maintenance-controller.md +++ b/docs/indeehub-legacy-maintenance-controller.md @@ -1,6 +1,6 @@ # Legacy IndeeHub maintenance controller -Status: isolated source implementation. Ten pure Python fake-runtime regressions +Status: isolated source implementation. Twelve pure Python fake-runtime regressions pass; no live invocation or production qualification. The controller is not part of the already signed private app candidate and needs no new app image/API. @@ -55,7 +55,7 @@ prove compatibility with newly changed data. No automatic DB/media restore exist ## Qualification and remaining integration -`python3 tests/regression/test_indeehub_maintenance_controller.py` passes ten +`python3 tests/regression/test_indeehub_maintenance_controller.py` passes twelve fake-runtime cases in temporary directories, without services/network/containers. Source nginx template guard coverage also passes its parser check. Production adapter compilation, actual Podman event format/systemd clean-exit behavior, @@ -71,3 +71,8 @@ The backend must refuse missing/mismatched prerequisites before snapshots/stops. Native AppGate + nginx guards are separate node source changes owned by the supervised updater agent. The signed app catalog/private image receipts remain unchanged. Existing live stop/uninstall intent must not be rewritten as maintenance. + +A pre-acquire snapshot/preflight failure may leave no controller journal. An +Aborted node journal with target_startup_began=false then permits idempotent +no-op acknowledgement, without touching any other operation’s admission fence. +A matching fence without its controller journal requires recovery investigation. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index a152086e..6ce10ff9 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -204,6 +204,12 @@ class Controller: return {'operation_id':self.operation,'state':'held'} def release(self, outcome): require(outcome in ('committed','restored','aborted'),'Invalid release outcome') + if self.record is None and outcome=='aborted': + runtime=json.loads((self.data/'update-transactions'/'supervised'/(self.operation+'.json')).read_text()) + require(runtime.get('phase')=='Aborted' and runtime.get('target_startup_began') is False,'Untouched abort evidence required') + if self.fence.exists(): + require(not self.fence.is_symlink() and self.fence.read_text()!=self.operation,'Matching fence without journal requires recovery') + return {'operation_id':self.operation,'state':'released'} require(self.record is not None,'Unknown maintenance operation') if self.record['phase']=='Released': require(self.record.get('outcome')==outcome,'Maintenance outcome changed') diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 1c22becd..439f0daa 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -82,4 +82,17 @@ class MaintenanceTests(unittest.TestCase): self.assertEqual(module.validate_nginx_guards('\n'.join(blocks)),3) with self.assertRaisesRegex(RuntimeError,'missing its maintenance guard'):module.validate_nginx_guards('\n'.join(blocks).replace(guard,'',1)) with self.assertRaisesRegex(RuntimeError,'Unrecognized direct'):module.validate_nginx_guards('\n'.join(blocks)+'\nlocation /other/ {\n proxy_pass http://127.0.0.1:7778/;\n}') + def test_pre_acquire_abort_acknowledges_without_mutating_foreign_fence(self): + c=self.controller;runtime=c.data/'update-transactions'/'supervised'/(self.operation+'.json') + module.atomic(runtime,{'phase':'Aborted','target_startup_began':False}) + self.assertEqual(c.release('aborted')['state'],'released') + c.fence.parent.mkdir(parents=True);foreign=str(uuid.uuid4());c.fence.write_text(foreign) + self.assertEqual(c.release('aborted')['state'],'released');self.assertEqual(c.fence.read_text(),foreign) + module.atomic(runtime,{'phase':'Aborted','target_startup_began':True}) + with self.assertRaisesRegex(RuntimeError,'Untouched abort'):c.release('aborted') + def test_matching_fence_without_journal_is_not_an_untouched_abort(self): + c=self.controller;module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Aborted','target_startup_began':False}) + c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) + with self.assertRaisesRegex(RuntimeError,'without journal'):c.release('aborted') + self.assertTrue(c.fence.exists()) if __name__=='__main__':unittest.main() From f79ecd11aecff2811338a826af649459e02faeb8 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:34:51 -0400 Subject: [PATCH 08/13] Integrate legacy managed update maintenance and fenced recovery before reconciliation --- .../archipelago/src/api/rpc/package/update.rs | 34 +++- core/archipelago/src/appgate/mod.rs | 40 ++++ core/archipelago/src/bootstrap.rs | 50 ++++- .../src/container/supervised_runtime.rs | 183 ++++++++++++++++-- .../src/container/supervised_update.rs | 119 +++++++++--- .../src/container/update_transaction.rs | 13 +- image-recipe/configs/nginx-archipelago.conf | 3 + 7 files changed, 399 insertions(+), 43 deletions(-) diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index 037f0f93..61379e10 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -305,7 +305,22 @@ impl RpcHandler { guard: &crate::container::update_transaction::Guard, ) -> Result<()> { use crate::container::update_transaction::{self, Podman}; - let targets = Podman::targets(images_to_pull).await?; + let mut supervised = 0; + for name in containers { + if crate::container::quadlet::unit_exists(name).await { + supervised += 1; + } + } + anyhow::ensure!( + supervised == 0 || supervised == containers.len(), + "Mixed managed/unmanaged stack requires an explicit recovery plan; originals unchanged" + ); + let managed = supervised != 0; + anyhow::ensure!( + !managed || package_id == "indeedhub", + "This managed app has no qualified maintenance controller; originals unchanged" + ); + let targets = Podman::targets(images_to_pull, !managed).await?; let names: HashSet<_> = containers.iter().map(String::as_str).collect(); anyhow::ensure!( targets.len() == names.len() @@ -316,7 +331,22 @@ impl RpcHandler { ); self.set_install_phase(package_id, InstallPhase::Preparing) .await; - update_transaction::execute(guard, package_id, &targets, &Podman).await + if managed { + use crate::container::supervised_runtime::{ + load_reviewed_plans, LegacyIndeeMaintenance, SystemdSupervisor, + }; + let plans = load_reviewed_plans(&self.config.data_dir, package_id)?; + let adapter = SystemdSupervisor::new( + self.config.data_dir.clone(), + plans, + LegacyIndeeMaintenance::new(guard)?, + ) + .await?; + crate::container::supervised_update::execute(guard, package_id, &targets, &adapter) + .await + } else { + update_transaction::execute(guard, package_id, &targets, &Podman).await + } } async fn recreate_container_for_update( diff --git a/core/archipelago/src/appgate/mod.rs b/core/archipelago/src/appgate/mod.rs index d82fef6d..dbdd1eed 100644 --- a/core/archipelago/src/appgate/mod.rs +++ b/core/archipelago/src/appgate/mod.rs @@ -146,6 +146,16 @@ impl AppGate { // snapshot when the port momentarily leaves the map mid-refresh. let live = self.port_map.read().await.gated(app.port).cloned(); let app = live.as_ref().unwrap_or(app); + if maintenance_active(&self.data_dir, &app.app_id) { + return Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header(header::CACHE_CONTROL, "no-store") + .header(header::RETRY_AFTER, "60") + .body(Body::from( + "This app is temporarily unavailable while its update is recovered.", + )) + .unwrap(); + } let path = req.uri().path().to_string(); // A dashboard same-origin proxy strips `/app//` before this gate @@ -838,6 +848,16 @@ dashboard and check {name} under My Apps.

"#, resp } +fn maintenance_active(data_dir: &std::path::Path, app_id: &str) -> bool { + if app_id != "indeedhub" && !app_id.starts_with("indeedhub-") { + return false; + } + match std::fs::symlink_metadata(data_dir.join("app-maintenance/indeedhub")) { + Ok(_) => true, + Err(error) => error.kind() != std::io::ErrorKind::NotFound, + } +} + fn not_found() -> Response { Response::builder() .status(StatusCode::NOT_FOUND) @@ -1857,3 +1877,23 @@ mod tests { ) } } + +#[cfg(test)] +mod maintenance_tests { + #[test] + fn held_indee_ingress_never_reaches_upstream_or_another_app() { + let root = tempfile::tempdir().unwrap(); + assert!(!super::maintenance_active(root.path(), "indeedhub")); + std::fs::create_dir(root.path().join("app-maintenance")).unwrap(); + std::fs::write( + root.path().join("app-maintenance/indeedhub"), + uuid::Uuid::new_v4().to_string(), + ) + .unwrap(); + assert!(super::maintenance_active(root.path(), "indeedhub")); + assert!(super::maintenance_active(root.path(), "indeedhub-api")); + assert!(!super::maintenance_active(root.path(), "node-demo-v4v")); + std::fs::remove_file(root.path().join("app-maintenance/indeedhub")).unwrap(); + assert!(!super::maintenance_active(root.path(), "indeedhub")); + } +} diff --git a/core/archipelago/src/bootstrap.rs b/core/archipelago/src/bootstrap.rs index 34d068cb..0b521061 100644 --- a/core/archipelago/src/bootstrap.rs +++ b/core/archipelago/src/bootstrap.rs @@ -83,6 +83,32 @@ const RUNTIME_ASSETS_DIR: &str = "/opt/archipelago/web-ui/archipelago-runtime"; /// Inserted into every server block of the nginx config that lacks the /// `/api/app-catalog` proxy. Kept in sync with the canonical block in /// image-recipe/configs/nginx-archipelago.conf. +const INDEEHUB_MAINTENANCE_GUARD: &str = + "if (-f /var/lib/archipelago/app-maintenance/indeedhub) { return 503; }"; +/// Patch only recognized literal IndeeHub routes, including asset/WebSocket +/// sublocations; unknown operator routes are not guessed by this repair. +fn heal_indeehub_maintenance_guards(content: &str) -> String { + let route=regex::Regex::new(r"(?m)^([ \t]*)(location[ \t]+(?:\^~[ \t]+|=[ \t]+)?/app/indeedhub(?:/[^\s{]*)?[ \t]*\{)[ \t]*$").unwrap(); + let mut result = String::new(); + let mut previous = 0; + for capture in route.captures_iter(content) { + let full = capture.get(0).unwrap(); + result.push_str(&content[previous..full.end()]); + if !content[full.end()..] + .trim_start() + .starts_with(INDEEHUB_MAINTENANCE_GUARD) + { + result.push_str(&format!( + "\n{} {}", + &capture[1], INDEEHUB_MAINTENANCE_GUARD + )); + } + previous = full.end(); + } + result.push_str(&content[previous..]); + result +} + const NGINX_APP_CATALOG_BLOCK: &str = "\n # App Store catalog proxy — backend fetches from configured registries\n # so the browser doesn't hit CORS/CSP. Without this block nginx falls\n # through to the SPA index.html and the frontend gets HTML back instead\n # of JSON.\n location ~ ^/api/(?:app-catalog|node-app-catalog)$ {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header Cookie $http_cookie;\n proxy_connect_timeout 15s;\n proxy_read_timeout 30s;\n proxy_send_timeout 15s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n\n"; const NGINX_SOURCE_PROXY_BLOCK: &str = " # GitWorkshop follows the dashboard origin so LAN, Tailscale, FIPS, Tor,\n # hostnames and reverse proxies all use the connection that already works.\n location /app/archipelago-source/ {\n proxy_pass http://127.0.0.2:8337/;\n proxy_http_version 1.1;\n proxy_set_header Host $http_host;\n proxy_set_header Cookie $http_cookie;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n proxy_set_header X-Forwarded-Prefix /app/archipelago-source;\n proxy_hide_header X-Frame-Options;\n add_header X-Frame-Options \"SAMEORIGIN\" always;\n add_header X-Content-Type-Options \"nosniff\" always;\n proxy_read_timeout 300s;\n }\n"; @@ -2011,8 +2037,10 @@ async fn patch_nginx_conf(path: &str) -> Result { let missing_source_prefix = heal_source_forwarded_prefix(&content).is_some(); let missing_nostr_signer = heal_missing_nostr_signer(&content).is_some(); let missing_rental_playback = heal_rental_playback_route(&content) != content; + let missing_maintenance = heal_indeehub_maintenance_guards(&content) != content; let legacy_catalog_route = content.contains("location /api/app-catalog {"); - if !missing_rental_playback + if !missing_maintenance + && !missing_rental_playback && !missing_app_catalog && !legacy_catalog_route && !missing_bitcoin_status @@ -2031,7 +2059,9 @@ async fn patch_nginx_conf(path: &str) -> Result { return Ok(false); } - let mut patched = heal_rental_playback_route(&heal_node_catalog_route(&content)); + let mut patched = heal_indeehub_maintenance_guards(&heal_rental_playback_route( + &heal_node_catalog_route(&content), + )); if let Some(p) = heal_stale_web_search_block(&patched) { patched = p; @@ -2515,3 +2545,19 @@ pub async fn ensure_restart_policy() { Err(e) => tracing::warn!(error = %e, "could not repair archipelago.service restart policy"), } } + +#[cfg(test)] +mod indeehub_maintenance_tests { + #[test] + fn legacy_routes_are_fenced_independently_and_repair_is_idempotent() { + let source="server {\n location /app/indeedhub/ {\n proxy_pass http://127.0.0.1:7778;\n }\n location /app/indeedhub/ws/ {\n proxy_pass http://127.0.0.1:7778;\n }\n location /app/other/ {\n proxy_pass http://127.0.0.1:7777;\n }\n}\n"; + let repaired = super::heal_indeehub_maintenance_guards(source); + assert_eq!( + repaired.matches(super::INDEEHUB_MAINTENANCE_GUARD).count(), + 2 + ); + assert_eq!(super::heal_indeehub_maintenance_guards(&repaired), repaired); + assert!(repaired.contains("location /app/other/ {\n proxy_pass")); + assert_eq!(repaired.matches("proxy_pass").count(), 3); + } +} diff --git a/core/archipelago/src/container/supervised_runtime.rs b/core/archipelago/src/container/supervised_runtime.rs index 21f51705..68a79287 100644 --- a/core/archipelago/src/container/supervised_runtime.rs +++ b/core/archipelago/src/container/supervised_runtime.rs @@ -1,7 +1,7 @@ //! Production systemd/Podman adapter. Application write admission/drain is an //! explicit dependency: neither process pause nor a filesystem receipt is drain. use super::{ - supervised_update::{self, PreparedTarget, RecoveryImage, Supervisor, Unit}, + supervised_update::{self, Completion, PreparedTarget, RecoveryImage, Supervisor, Unit}, update_transaction::{Observed, Podman, Runtime, Target}, }; use anyhow::{Context, Result}; @@ -10,24 +10,155 @@ use std::{ collections::HashMap, future::Future, io::Write, - os::unix::fs::{MetadataExt, OpenOptionsExt}, + os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}, path::{Path, PathBuf}, time::Duration, }; pub(crate) trait DrainBarrier: Sync { - /// Persist ownership before blocking admissions. Return only after API, - /// direct uploads and workers have drained and the coherent backup finished. + /// Called after the node persisted original writable recovery images. + /// Persist ownership before blocking admissions or stopping writers. Return + /// only after API/direct uploads/workers drained and coherent backup finished. fn acquire( &self, operation: &str, originals: &[Unit], + recovery: bool, ) -> impl Future> + Send; /// Must inspect the live admission fence and same-operation ownership, not /// merely trust an old 'backup complete' file. Keep it through snapshot/stop. fn verify(&self, operation: &str) -> impl Future> + Send; /// Idempotent; an old recovery must never clear a newer operation's fence. - fn release(&self, operation: &str) -> impl Future> + Send; + fn release( + &self, + operation: &str, + outcome: Completion, + ) -> impl Future> + Send; +} + +/// The controller is deployed from reviewed platform source. Its durable +/// journal owns legacy ingress/drain/backup across API/container shutdown. +pub(crate) struct LegacyIndeeMaintenance { + lock: std::sync::Arc, +} +impl LegacyIndeeMaintenance { + pub(crate) fn new(guard: &super::update_transaction::Guard) -> Result { + Self::controller_path()?; + Ok(Self { + lock: std::sync::Arc::new(guard.clone_lifecycle_lock()?), + }) + } + fn controller_path() -> Result<&'static Path> { + let path = Path::new("/opt/archipelago/scripts/indeehub-maintenance-controller.py"); + let metadata = std::fs::symlink_metadata(path) + .context("Legacy maintenance controller is not installed; original runtime retained")?; + anyhow::ensure!( + metadata.is_file() + && !metadata.file_type().is_symlink() + && (metadata.uid() == 0 || metadata.uid() == unsafe { libc::geteuid() }), + "Maintenance controller ownership changed" + ); + anyhow::ensure!( + metadata.mode() & 0o022 == 0, + "Maintenance controller is writable by another user" + ); + anyhow::ensure!(metadata.len() <= 128*1024 + && Sha256::digest(std::fs::read(path)?) == Sha256::digest(include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../scripts/indeehub-maintenance-controller.py"))), + "Maintenance controller does not match this qualified backend; install the exact companion script first"); + Ok(path) + } + async fn invoke(&self, action: &'static str, request: serde_json::Value) -> Result<()> { + use std::os::{fd::AsRawFd, unix::process::CommandExt}; + use tokio::io::AsyncWriteExt; + let path = Self::controller_path()?; + let operation = request["operation_id"] + .as_str() + .context("Missing maintenance operation")? + .to_owned(); + let recovery = request.get("recovery").and_then(|v| v.as_bool()) == Some(true); + let bytes = serde_json::to_vec(&request)?; + let lock = self.lock.try_clone()?; + // Detaching the RPC future must not detach mutation from its lifecycle + // lock. This task and the child retain the same flock open description. + tokio::spawn(async move { + let fd = lock.as_raw_fd(); + let mut command = tokio::process::Command::new("/usr/bin/python3"); + command + .arg(path) + .arg(action) + .env("ARCHY_UPDATE_LOCK_FD", fd.to_string()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(false); + unsafe { + command.as_std_mut().pre_exec(move || { + if libc::fcntl(fd, libc::F_SETFD, 0) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut child = command + .spawn() + .context("Could not start legacy maintenance controller")?; + if let Some(mut stdin) = child.stdin.take() { + stdin.write_all(&bytes).await?; + stdin.shutdown().await?; + } + let result = child.wait_with_output().await?; + anyhow::ensure!( + result.status.success() && result.stdout.len() <= 4096, + "Legacy maintenance remains unresolved; inspect its original operation journal" + ); + let reply: serde_json::Value = serde_json::from_slice(&result.stdout)?; + let wanted = match action { + "acquire" if recovery => "recovering", + "acquire" => "drained", + "verify" => "held", + "release" => "released", + _ => unreachable!(), + }; + anyhow::ensure!( + reply["operation_id"].as_str() == Some(operation.as_str()) + && reply["state"].as_str() == Some(wanted), + "Maintenance controller did not acknowledge this operation" + ); + drop(lock); + Ok::<_, anyhow::Error>(()) + }) + .await + .context("Maintenance completion task interrupted")? + } +} +impl DrainBarrier for LegacyIndeeMaintenance { + async fn acquire(&self, operation: &str, originals: &[Unit], recovery: bool) -> Result<()> { + let members: Vec<_> = originals + .iter() + .map(|unit| { + serde_json::json!({ + "name":unit.name,"container_id":unit.container_id,"image_id":unit.image, + "unit_sha256":hex::encode(Sha256::digest(unit.body.as_bytes())), + "config_sha256":unit.config_sha256,"running":unit.running}) + }) + .collect(); + self.invoke( + "acquire", + serde_json::json!({"operation_id":operation,"original_members":members,"recovery":recovery}), + ) + .await + } + async fn verify(&self, operation: &str) -> Result<()> { + self.invoke("verify", serde_json::json!({"operation_id":operation})) + .await + } + async fn release(&self, operation: &str, outcome: Completion) -> Result<()> { + self.invoke( + "release", + serde_json::json!({"operation_id":operation,"outcome":outcome}), + ) + .await + } } #[derive(Clone, serde::Serialize, serde::Deserialize)] @@ -132,6 +263,22 @@ pub(crate) fn load_reviewed_plans( Ok(record.plans) } +pub(crate) async fn recover_before_reconcile( + data_dir: &Path, + guard: &super::update_transaction::Guard, +) -> Result<()> { + if !supervised_update::needs_recovery(guard)? { + return Ok(()); + } + let adapter = SystemdSupervisor::new( + data_dir.into(), + HashMap::new(), + LegacyIndeeMaintenance::new(guard)?, + ) + .await?; + supervised_update::recover(guard, &adapter).await +} + pub(crate) struct SystemdSupervisor { data_dir: PathBuf, unit_dir: PathBuf, @@ -224,14 +371,19 @@ impl SystemdSupervisor { } } impl Supervisor for SystemdSupervisor { - async fn begin_barrier(&self, operation: &str, originals: &[Unit]) -> Result<()> { - self.barrier.acquire(operation, originals).await + async fn begin_barrier( + &self, + operation: &str, + originals: &[Unit], + recovery: bool, + ) -> Result<()> { + self.barrier.acquire(operation, originals, recovery).await } async fn verify_barrier(&self, operation: &str) -> Result<()> { self.barrier.verify(operation).await } - async fn release_barrier(&self, operation: &str) -> Result<()> { - self.barrier.release(operation).await + async fn release_barrier(&self, operation: &str, outcome: Completion) -> Result<()> { + self.barrier.release(operation, outcome).await } async fn prepare_target(&self, target: &Target, original: &Unit) -> Result { let plan = self @@ -303,6 +455,7 @@ impl Supervisor for SystemdSupervisor { .mode(original.file_mode) .open(&temporary)?; file.write_all(body.as_bytes())?; + file.set_permissions(std::fs::Permissions::from_mode(original.file_mode))?; file.sync_all()?; std::fs::rename(&temporary, &path)?; std::fs::File::open(&self.unit_dir)?.sync_all()?; @@ -314,7 +467,8 @@ impl Supervisor for SystemdSupervisor { result } async fn snapshot(&self, original: &Unit, operation: &str, tag: &str) -> Result { - self.barrier.verify(operation).await?; + // Pre-maintenance image capture preserves code and hook mutations; + // mounted-data consistency is established separately by acquire(). supervised_update::capture_local_recovery_image(original, operation, tag).await } async fn pin(&self, image: &str, tag: &str) -> Result<()> { @@ -366,13 +520,13 @@ mod tests { use super::*; struct UnusedBarrier; impl DrainBarrier for UnusedBarrier { - async fn acquire(&self, _: &str, _: &[Unit]) -> Result<()> { + async fn acquire(&self, _: &str, _: &[Unit], _: bool) -> Result<()> { anyhow::bail!("No application barrier installed") } async fn verify(&self, _: &str) -> Result<()> { anyhow::bail!("No application barrier installed") } - async fn release(&self, _: &str) -> Result<()> { + async fn release(&self, _: &str, _: Completion) -> Result<()> { Ok(()) } } @@ -422,6 +576,9 @@ mod tests { original.body = original_body.into(); target.reference = format!("localhost/movie@sha256:{}", "e".repeat(64)); assert!(adapter.prepare_target(&target, &original).await.is_err()); - assert!(adapter.begin_barrier("unused", &[original]).await.is_err()); + assert!(adapter + .begin_barrier("unused", &[original], false) + .await + .is_err()); } } diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs index 3d96d3fe..05d62528 100644 --- a/core/archipelago/src/container/supervised_update.rs +++ b/core/archipelago/src/container/supervised_update.rs @@ -46,6 +46,7 @@ enum Phase { Aborted, Editing, Starting, + Restoring, Committed, Restored, } @@ -57,19 +58,39 @@ struct Journal { package: String, phase: Phase, members: Vec, + #[serde(default)] + cleanup_done: bool, + #[serde(default)] + target_startup_began: bool, +} + +#[derive(Clone, Copy, Debug, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Completion { + Committed, + Restored, + Aborted, } pub(crate) trait Supervisor: Sync { - /// Admission must be fenced and in-flight application work drained under - /// this durable operation before snapshots. A paused process is not proof. + /// Called only after every original writable recovery image is durable. + /// Legacy acquisition may gracefully stop AutoRemove writers, so its + /// destructive obligation is journaled before entering the controller. + /// It must fence/drain writers and finish coherent mounted-data backup; + /// image snapshots alone never establish application consistency. /// Acquisition/release are idempotent and operation-owned across restart. fn begin_barrier( &self, operation: &str, originals: &[Unit], + recovery: bool, ) -> impl Future> + Send; fn verify_barrier(&self, operation: &str) -> impl Future> + Send; - fn release_barrier(&self, operation: &str) -> impl Future> + Send; + fn release_barrier( + &self, + operation: &str, + outcome: Completion, + ) -> impl Future> + Send; /// Only an internal reviewed signed-manifest planner may supply this value; /// browser parameters must never become a unit body or hook recipe. fn prepare_target( @@ -518,9 +539,8 @@ fn records(guard: &Guard) -> Result> { } pub(crate) fn require_clear(guard: &Guard) -> Result<()> { anyhow::ensure!( - records(guard)? - .iter() - .all(|r| matches!(r.phase, Phase::Committed | Phase::Restored | Phase::Aborted)), + records(guard)?.iter().all(|r| r.cleanup_done + && matches!(r.phase, Phase::Committed | Phase::Restored | Phase::Aborted)), "A supervised update needs recovery first" ); Ok(()) @@ -574,6 +594,8 @@ pub(crate) async fn execute( package: package.into(), phase: Phase::Prepared, members, + cleanup_done: false, + target_startup_began: false, }; save(guard, &record)?; for member in &record.members { @@ -593,12 +615,6 @@ pub(crate) async fn execute( Ok(()) } async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> { - let originals: Vec<_> = record - .members - .iter() - .map(|member| member.original.clone()) - .collect(); - supervisor.begin_barrier(&record.id, &originals).await?; for index in 0..record.members.len() { let member = &record.members[index]; supervisor.validate_original_file(&member.original).await?; @@ -606,7 +622,6 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor supervisor.read(&member.original.name).await? == member.original.body, "Unit edited before update; originals retained" ); - supervisor.verify_barrier(&record.id).await?; let image = supervisor .snapshot(&member.original, &record.id, &member.original_tag) .await?; @@ -626,9 +641,19 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor // The image acknowledgement becomes durable before any original stop. save(guard, record)?; } - supervisor.verify_barrier(&record.id).await?; record.phase = Phase::Editing; save(guard, record)?; + // The legacy controller may stop original writers. Every writable layer + // is already recoverable and this obligation survives cancellation. + let originals: Vec<_> = record + .members + .iter() + .map(|member| member.original.clone()) + .collect(); + supervisor + .begin_barrier(&record.id, &originals, false) + .await?; + supervisor.verify_barrier(&record.id).await?; for member in record.members.iter().rev() { supervisor.verify_barrier(&record.id).await?; supervisor.stop(&member.original.name).await?; @@ -644,6 +669,8 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor } supervisor.reload().await?; record.phase = Phase::Starting; + record.target_startup_began = true; + record.cleanup_done = false; save(guard, record)?; for member in &record.members { supervisor.start(&member.original.name).await?; @@ -663,11 +690,14 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor } record.phase = Phase::Committed; save(guard, record)?; - supervisor.release_barrier(&record.id).await?; + supervisor + .release_barrier(&record.id, Completion::Committed) + .await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } - Ok(()) + record.cleanup_done = true; + save(guard, record) } async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> { if record.phase == Phase::Prepared { @@ -689,12 +719,25 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis } record.phase = Phase::Aborted; save(guard, record)?; - supervisor.release_barrier(&record.id).await?; + supervisor + .release_barrier(&record.id, Completion::Aborted) + .await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } - return Ok(()); + record.cleanup_done = true; + return save(guard, record); } + let originals: Vec<_> = record + .members + .iter() + .map(|member| member.original.clone()) + .collect(); + record.phase = Phase::Restoring; + save(guard, record)?; + supervisor + .begin_barrier(&record.id, &originals, true) + .await?; supervisor.verify_barrier(&record.id).await?; // Refuse to overwrite a foreign edit before stopping any surviving member. for member in &record.members { @@ -767,25 +810,44 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis } record.phase = Phase::Restored; save(guard, record)?; - supervisor.release_barrier(&record.id).await + supervisor + .release_barrier(&record.id, Completion::Restored) + .await?; + record.cleanup_done = true; + save(guard, record) } pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Result<()> { for mut record in records(guard)? { + if record.cleanup_done { + continue; + } match record.phase { Phase::Committed | Phase::Aborted => { - supervisor.release_barrier(&record.id).await?; + let outcome = if record.phase == Phase::Committed { + Completion::Committed + } else { + Completion::Aborted + }; + supervisor.release_barrier(&record.id, outcome).await?; for member in &record.members { guard.release_hold(&member.original.name, &record.id)?; } } Phase::Restored => { - supervisor.release_barrier(&record.id).await?; + supervisor + .release_barrier(&record.id, Completion::Restored) + .await?; } // Never release a newer owner. _ => restore(guard, &mut record, supervisor).await?, } + record.cleanup_done = true; + save(guard, &record)?; } Ok(()) } +pub(crate) fn needs_recovery(guard: &Guard) -> Result { + Ok(records(guard)?.iter().any(|record| !record.cleanup_done)) +} #[cfg(test)] mod tests { @@ -837,7 +899,12 @@ mod tests { } } impl Supervisor for Mock { - async fn begin_barrier(&self, operation: &str, _originals: &[Unit]) -> Result<()> { + async fn begin_barrier( + &self, + operation: &str, + _originals: &[Unit], + _recovery: bool, + ) -> Result<()> { let mut held = self.barrier.lock().unwrap(); anyhow::ensure!( held.as_deref().is_none_or(|id| id == operation), @@ -853,7 +920,7 @@ mod tests { ); Ok(()) } - async fn release_barrier(&self, operation: &str) -> Result<()> { + async fn release_barrier(&self, operation: &str, _outcome: Completion) -> Result<()> { anyhow::ensure!( !self.fail_barrier_release.load(Ordering::SeqCst), "Barrier release unavailable" @@ -1062,7 +1129,9 @@ mod tests { execute(&guard, "movie", &[Mock::target()], &runtime) .await .unwrap(); - let record = records(&guard).unwrap().pop().unwrap(); + let mut record = records(&guard).unwrap().pop().unwrap(); + record.cleanup_done = false; + save(&guard, &record).unwrap(); guard.hold("movie", &record.id).unwrap(); runtime.calls.lock().unwrap().clear(); recover(&guard, &runtime).await.unwrap(); @@ -1125,6 +1194,7 @@ mod tests { .unwrap(); let mut record = records(&guard).unwrap().pop().unwrap(); record.phase = Phase::Starting; + record.cleanup_done = false; *runtime.barrier.lock().unwrap() = Some(record.id.clone()); save(&guard, &record).unwrap(); runtime.calls.lock().unwrap().clear(); @@ -1150,6 +1220,7 @@ mod tests { .unwrap(); let mut record = records(&guard).unwrap().pop().unwrap(); record.phase = Phase::Starting; + record.cleanup_done = false; *runtime.barrier.lock().unwrap() = Some(record.id.clone()); save(&guard, &record).unwrap(); *runtime.body.lock().unwrap() = "operator replaced unit".into(); diff --git a/core/archipelago/src/container/update_transaction.rs b/core/archipelago/src/container/update_transaction.rs index 429b5f86..4db3742d 100644 --- a/core/archipelago/src/container/update_transaction.rs +++ b/core/archipelago/src/container/update_transaction.rs @@ -74,6 +74,9 @@ pub(crate) struct Guard { root: PathBuf, } impl Guard { + pub(crate) fn clone_lifecycle_lock(&self) -> Result { + Ok(self._file.try_clone()?) + } pub(crate) fn directory(&self) -> &Path { &self.root } @@ -500,6 +503,7 @@ pub(crate) async fn recover(data: &Path, runtime: &impl Runtime) -> Result Result> { + pub(crate) async fn targets( + images: &[(String, String)], + require_clone: bool, + ) -> Result> { // Confirm clone support before any stop. Never silently switch to a // latest-catalog install if this runtime cannot create without running. - Self::command(&["container", "clone", "--help"]).await?; + if require_clone { + Self::command(&["container", "clone", "--help"]).await?; + } let mut targets = Vec::new(); for (name, reference) in images { let raw = Self::command(&["image", "inspect", reference]).await?; diff --git a/image-recipe/configs/nginx-archipelago.conf b/image-recipe/configs/nginx-archipelago.conf index 8925f589..bcabd9e8 100644 --- a/image-recipe/configs/nginx-archipelago.conf +++ b/image-recipe/configs/nginx-archipelago.conf @@ -691,6 +691,7 @@ server { sub_filter '' ''; } location /app/indeedhub/_next/ { + if (-f /var/lib/archipelago/app-maintenance/indeedhub) { return 503; } proxy_pass http://127.0.0.1:7778/_next/; proxy_http_version 1.1; proxy_set_header Host $host; @@ -699,6 +700,7 @@ server { } # IndeeHub WebSocket proxy location /app/indeedhub/ws/ { + if (-f /var/lib/archipelago/app-maintenance/indeedhub) { return 503; } proxy_pass http://127.0.0.1:7778/ws/; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -708,6 +710,7 @@ server { proxy_read_timeout 86400s; } location /app/indeedhub/ { + if (-f /var/lib/archipelago/app-maintenance/indeedhub) { return 503; } proxy_pass http://127.0.0.1:7778/; proxy_http_version 1.1; proxy_set_header Host $host; From 5a9aac18ea0dbbbdabaa96ecafc96d34e9fc29c5 Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:35:21 -0400 Subject: [PATCH 09/13] Qualify idle legacy process termination without inventing completed work --- .../indeehub-legacy-maintenance-controller.md | 12 +++++++-- scripts/indeehub-maintenance-controller.py | 23 +++++++++++++--- .../test_indeehub_maintenance_controller.py | 26 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/docs/indeehub-legacy-maintenance-controller.md b/docs/indeehub-legacy-maintenance-controller.md index a357ca8c..ef294fb8 100644 --- a/docs/indeehub-legacy-maintenance-controller.md +++ b/docs/indeehub-legacy-maintenance-controller.md @@ -1,6 +1,6 @@ # Legacy IndeeHub maintenance controller -Status: isolated source implementation. Twelve pure Python fake-runtime regressions +Status: isolated source implementation. Fourteen pure Python fake-runtime regressions pass; no live invocation or production qualification. The controller is not part of the already signed private app candidate and needs no new app image/API. @@ -55,7 +55,7 @@ prove compatibility with newly changed data. No automatic DB/media restore exist ## Qualification and remaining integration -`python3 tests/regression/test_indeehub_maintenance_controller.py` passes twelve +`python3 tests/regression/test_indeehub_maintenance_controller.py` passes fourteen fake-runtime cases in temporary directories, without services/network/containers. Source nginx template guard coverage also passes its parser check. Production adapter compilation, actual Podman event format/systemd clean-exit behavior, @@ -76,3 +76,11 @@ A pre-acquire snapshot/preflight failure may leave no controller journal. An Aborted node journal with target_startup_began=false then permits idempotent no-op acknowledgement, without touching any other operation’s admission fence. A matching fence without its controller journal requires recovery investigation. + +Read-only source evidence from actual old API/ffmpeg shows neither has SIGTERM +shutdown hooks. The controller permits worker143 only after a paused queue has +zero active jobs. Legacy API143 additionally requires closed/stopped frontend, +stopped worker, and a fresh empty projects/contents/payments/shareholders/ +subscriptions/library_items store with no other active DB transaction. This is +a narrow first-upgrade compatibility path, not evidence populated work completed. +Populated or ambiguous legacy state remains a refused forward cutover. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index 6ce10ff9..79d04a12 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -107,6 +107,20 @@ class Controller: original=self.queue('status');self.record['queue_was_paused']=original['paused'];self.record['queue_original_counts']=original['counts'];self.save() state=self.queue('pause');require(state['paused'],'Worker admission did not close') self.record['queue_pause_confirmed']=True;self.save() + def legacy_api_idle(self): + # Narrow first-upgrade compatibility for the observed legacy API which + # has no SIGTERM hooks. Existing customer/business work is never inferred + # completed: this path requires a fresh empty store behind closed ingress. + require(self.record.get('stopped',{}).get('indeedhub',{}).get('confirmed'),'Frontend ingress must already be stopped') + require(self.record.get('stopped',{}).get('indeedhub-ffmpeg',{}).get('confirmed'),'Transcode worker must already be stopped') + require(self.record.get('queue_pause_confirmed') is True and self.record.get('last_queue_counts',{}).get('active')==0,'Worker queue is not proven idle') + tables=('projects','contents','payments','shareholders','subscriptions','library_items') + fields=','.join("'%s',(SELECT count(*) FROM public.%s)"%(name,name) for name in tables) + sql="SELECT json_build_object("+fields+",'other_active_transactions',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state<>'idle'))" + counts=json.loads(self.run(['podman','exec','indeedhub-postgres','psql','-XAt','-U','indeedhub','-d','indeedhub','-c',sql])) + require(set(counts)==set(tables)|{'other_active_transactions'},'Legacy API business-state observation incomplete') + require(all(type(value) is int and value==0 for value in counts.values()),'Legacy API has business work or active transactions; completion cannot be inferred') + self.record['legacy_api_empty_state']=counts;self.save() def graceful_stop(self, name): # Save the obligation before systemd can remove an AutoRemove container. stopped=self.record.setdefault('stopped',{}) @@ -125,8 +139,11 @@ class Controller: matching=[event for event in matching if event.get('ID',event.get('id'))==member['container_id']] require(matching,'Original process exit evidence unavailable; hold retained') code=matching[-1].get('ContainerExitCode',matching[-1].get('containerExitCode')) - require(str(code)=='0','Original process did not exit cleanly; active work is not claimed completed') - stopped[name].update(confirmed=True,exit_code=0,confirmed_at=time.time());self.save() + idle_worker = name=='indeedhub-ffmpeg' and self.record.get('queue_pause_confirmed') is True and self.record.get('last_queue_counts',{}).get('active')==0 + empty_api = name=='indeedhub-api' and self.record.get('legacy_api_empty_state') is not None + require(str(code)=='0' or (str(code)=='143' and (idle_worker or empty_api)),'Original process did not exit cleanly; active work is not claimed completed') + classification=('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit' + stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save() def volume_sources(self): expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data'] rows=json.loads(self.run(['podman','volume','inspect',*expected])) @@ -186,7 +203,7 @@ class Controller: if state['counts'].get('active',0)==0:break require(time.monotonic() Date: Wed, 7 Oct 2026 02:49:31 -0400 Subject: [PATCH 10/13] Preserve reviewed managed unit recipes after update completion --- .../archipelago/src/api/rpc/package/update.rs | 1 + .../src/container/prod_orchestrator.rs | 15 +++ .../src/container/supervised_runtime.rs | 36 ++++++ .../src/container/supervised_update.rs | 119 ++++++++++++++++++ 4 files changed, 171 insertions(+) diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index 61379e10..5680b165 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -342,6 +342,7 @@ impl RpcHandler { LegacyIndeeMaintenance::new(guard)?, ) .await?; + let targets = adapter.reviewed_targets(&targets).await?; crate::container::supervised_update::execute(guard, package_id, &targets, &adapter) .await } else { diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index fbb9478d..9396e306 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -3055,6 +3055,7 @@ impl ProdContainerOrchestrator { } async fn install_fresh_with_pin(&self, lm: &LoadedManifest, pinned: bool) -> Result<()> { + anyhow::ensure!(super::supervised_update::installed_unit(&self.data_dir, &compute_container_name(&lm.manifest))?.is_none(), "Reviewed managed runtime is missing; restore its saved unit/image explicitly instead of recreating from the current catalog"); self.ensure_app_secrets(&lm.manifest.app.id).await?; let mut resolved_manifest = lm.manifest.clone(); self.resolve_dynamic_env(&mut resolved_manifest).await?; @@ -3354,6 +3355,9 @@ impl ProdContainerOrchestrator { lm: &LoadedManifest, name: &str, ) -> Result> { + if super::supervised_update::installed_unit(&self.data_dir, name)?.is_some() { + return Ok(None); // Never remove an original runtime to migrate it from changed catalog data. + } // Skip companion apps — bitcoin-ui / electrs-ui / lnd-ui have shipped // via Quadlet since v1.7.41 (companion.rs renders the unit). Running // migration for them races companion rendering: when migration ran @@ -3456,6 +3460,15 @@ impl ProdContainerOrchestrator { if super::update_transaction::is_held(&self.data_dir, name)? { return Ok(()); // Preserve the recovered unit instead of current catalog drift. } + if let Some((body, mode)) = super::supervised_update::installed_unit(&self.data_dir, name)? + { + use std::os::unix::fs::PermissionsExt; + let path = quadlet::unit_dir().await?.join(format!("{name}.container")); + let meta = std::fs::symlink_metadata(&path) + .context("Saved managed unit is missing; recovery required")?; + anyhow::ensure!(meta.is_file() && meta.permissions().mode() & 0o777 == mode && std::fs::read_to_string(&path)? == body, "Reviewed managed unit changed; preserve it for explicit reconciliation instead of overwriting operator configuration"); + return Ok(()); + } // Companions: same reasoning as migrate_to_quadlet_if_needed — // companion.rs renders these units with a different shape, syncing // here would clobber them. @@ -5363,6 +5376,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { let _guard = lock.lock().await; for name in [app_id.to_string(), format!("archy-{app_id}")] { self.remove_quadlet_unit_if_present(&name).await?; + super::supervised_update::forget_installed(&self.data_dir, &name)?; } self.state.write().await.disabled.insert(app_id.to_string()); super::staged_update::clear(&self.data_dir, app_id).await?; @@ -5422,6 +5436,7 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { crate::crash_recovery::clear_installed(&self.data_dir, app_id).await; super::staged_update::clear(&self.data_dir, app_id).await?; super::staged_update::clear_installed(&self.data_dir, app_id).await?; + super::supervised_update::forget_installed(&self.data_dir, &name)?; Ok(()) } diff --git a/core/archipelago/src/container/supervised_runtime.rs b/core/archipelago/src/container/supervised_runtime.rs index 68a79287..d633f304 100644 --- a/core/archipelago/src/container/supervised_runtime.rs +++ b/core/archipelago/src/container/supervised_runtime.rs @@ -307,6 +307,42 @@ impl SystemdSupervisor { barrier, }) } + /// A reviewed plan may replace a mutable catalog spelling with its exact + /// locally verified digest, but never select different image bytes. + pub(crate) async fn reviewed_targets(&self, targets: &[Target]) -> Result> { + let mut refs = Vec::new(); + for target in targets { + let plan = self + .plans + .get(&target.name) + .context("Missing reviewed managed target")?; + let reference = plan + .prepared + .manifest + .app + .container + .image + .as_ref() + .context("Reviewed image missing")?; + anyhow::ensure!( + reference.rsplit_once("@sha256:").is_some_and( + |(_, hash)| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit()) + ), + "Managed target must use an immutable digest" + ); + refs.push((target.name.clone(), reference.clone())); + } + let resolved = Podman::targets(&refs, false).await?; + anyhow::ensure!( + resolved + .iter() + .zip(targets) + .all(|(reviewed, catalog)| reviewed.name == catalog.name + && reviewed.image == catalog.image), + "Reviewed managed image differs from prepared catalog image" + ); + Ok(resolved) + } fn path(&self, name: &str) -> Result { anyhow::ensure!( !name.is_empty() diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs index 05d62528..3c0770a6 100644 --- a/core/archipelago/src/container/supervised_update.rs +++ b/core/archipelago/src/container/supervised_update.rs @@ -513,6 +513,91 @@ fn save(guard: &Guard, record: &Journal) -> Result<()> { } result } +// The committed unit is installation evidence, not a cache of today's catalog. +// Keep it until explicit uninstall or the next reviewed managed transaction. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct InstalledUnit { + schema: u8, + operation: String, + name: String, + body: String, + mode: u32, +} +fn installed_path(data: &Path, name: &str) -> Result { + anyhow::ensure!(simple(name), "Invalid managed member name"); + Ok(data + .join("update-transactions/installed-units") + .join(format!("{name}.json"))) +} +fn publish_installed(guard: &Guard, record: &Journal) -> Result<()> { + anyhow::ensure!( + record.phase == Phase::Committed, + "Only committed units may be published" + ); + let dir = guard.directory().join("installed-units"); + match std::fs::DirBuilder::new().mode(0o700).create(&dir) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + anyhow::ensure!( + std::fs::symlink_metadata(&dir)?.is_dir(), + "Invalid installed unit directory" + ); + for member in &record.members { + let saved = InstalledUnit { + schema: 1, + operation: record.id.clone(), + name: member.original.name.clone(), + body: member.target_body.clone(), + mode: member.original.file_mode, + }; + let temporary = dir.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&serde_json::to_vec(&saved)?)?; + file.sync_all()?; + std::fs::rename(&temporary, dir.join(format!("{}.json", saved.name)))?; + } + std::fs::File::open(&dir)?.sync_all()?; + std::fs::File::open(guard.directory())?.sync_all()?; + Ok(()) +} +pub(crate) fn installed_unit(data: &Path, name: &str) -> Result> { + let path = installed_path(data, name)?; + let meta = match std::fs::symlink_metadata(&path) { + Ok(meta) => meta, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + anyhow::ensure!( + meta.is_file() && meta.len() <= 4 * 1024 * 1024, + "Invalid installed managed recipe" + ); + let saved: InstalledUnit = serde_json::from_slice(&std::fs::read(path)?)?; + anyhow::ensure!( + saved.schema == 1 + && saved.name == name + && uuid::Uuid::parse_str(&saved.operation)?.to_string() == saved.operation + && saved.mode & !0o777 == 0 + && saved.mode & 0o022 == 0, + "Invalid installed managed recipe binding" + ); + Ok(Some((saved.body, saved.mode))) +} +pub(crate) fn forget_installed(data: &Path, name: &str) -> Result<()> { + let path = installed_path(data, name)?; + match std::fs::remove_file(&path) { + Ok(()) => std::fs::File::open(path.parent().unwrap())?.sync_all()?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + }; + Ok(()) +} fn records(guard: &Guard) -> Result> { let dir = root(guard)?; let mut records = Vec::new(); @@ -673,6 +758,10 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor record.cleanup_done = false; save(guard, record)?; for member in &record.members { + anyhow::ensure!( + supervisor.read(&member.original.name).await? == member.target_body, + "Reviewed unit changed before target start; recovery required" + ); supervisor.start(&member.original.name).await?; supervisor .target_hooks(&member.original.name, &member.target_manifest) @@ -690,6 +779,7 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor } record.phase = Phase::Committed; save(guard, record)?; + publish_installed(guard, record)?; supervisor .release_barrier(&record.id, Completion::Committed) .await?; @@ -782,6 +872,10 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis supervisor.reload().await?; for member in &record.members { if member.original.running { + anyhow::ensure!( + supervisor.read(&member.original.name).await? == member.pinned_original_body, + "Original recovery unit changed before start" + ); supervisor.start(&member.original.name).await?; } let observed = supervisor.observed(&member.original.name).await?; @@ -823,6 +917,9 @@ pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Resu } match record.phase { Phase::Committed | Phase::Aborted => { + if record.phase == Phase::Committed { + publish_installed(guard, &record)?; + } let outcome = if record.phase == Phase::Committed { Completion::Committed } else { @@ -1122,6 +1219,28 @@ mod tests { assert_eq!(*runtime.calls.lock().unwrap(), ["snapshot-original"]); } #[tokio::test] + async fn committed_unit_survives_restart_until_explicit_uninstall() { + let root = tempfile::tempdir().unwrap(); + let guard = Guard::acquire(root.path()).unwrap(); + let runtime = Mock::new(); + execute(&guard, "movie", &[Mock::target()], &runtime) + .await + .unwrap(); + let saved = installed_unit(root.path(), "movie").unwrap().unwrap(); + assert_eq!(saved.0, *runtime.body.lock().unwrap()); + assert!(saved.0.contains("OPERATOR_VALUE=retained")); + runtime.calls.lock().unwrap().clear(); + recover(&guard, &runtime).await.unwrap(); + assert_eq!( + installed_unit(root.path(), "movie").unwrap().unwrap(), + saved + ); + assert!(runtime.calls.lock().unwrap().is_empty()); + forget_installed(root.path(), "movie").unwrap(); + recover(&guard, &runtime).await.unwrap(); + assert!(installed_unit(root.path(), "movie").unwrap().is_none()); + } + #[tokio::test] async fn committed_restart_releases_only_its_own_hold_without_runtime_mutation() { let root = tempfile::tempdir().unwrap(); let guard = Guard::acquire(root.path()).unwrap(); From 541f073a2e665fded7ebcf58d92b50e8aa544a4f Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:50:09 -0400 Subject: [PATCH 11/13] Verify original database commitments before releasing restored IndeeHub --- .../indeehub-legacy-maintenance-controller.md | 28 ++++++++ scripts/indeehub-maintenance-controller.py | 72 ++++++++++++++++++- .../test_indeehub_maintenance_controller.py | 23 ++++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/docs/indeehub-legacy-maintenance-controller.md b/docs/indeehub-legacy-maintenance-controller.md index ef294fb8..576feb7e 100644 --- a/docs/indeehub-legacy-maintenance-controller.md +++ b/docs/indeehub-legacy-maintenance-controller.md @@ -84,3 +84,31 @@ stopped worker, and a fresh empty projects/contents/payments/shareholders/ subscriptions/library_items store with no other active DB transaction. This is a narrow first-upgrade compatibility path, not evidence populated work completed. Populated or ambiguous legacy state remains a refused forward cutover. + +### Operation-bound rollback data verification + +Restored release after any target startup now performs its own PostgreSQL +compatibility check; it does not accept a manually asserted verification boolean. +Before the coherent backup, the controller captures a read-only, repeatable-read +transaction containing every original public table's columns, constraints, +indexes, triggers, row-security policies, row count and canonical row SHA-256, +plus the exact migration history. The private operation journal binds this +baseline to the original operation UUID. + +After original runtime recovery, while ingress and worker admission remain +closed, the controller captures the same observations again. It requires original +tables and definitions unchanged, original non-migration rows unchanged and the +original migration-history prefix intact. Additional migration records must be +the exact ordered three migrations qualified for API commit `3b09b81`, and only +their five named new tables may appear, all empty. Any unexpected data or schema +change keeps ingress closed. Successful proof records before/after commitment +hashes and the operation UUID before release. No down migration or automatic +volume restoration is performed. + +Sixteen pure Python regressions pass, including altered rows/schema/history, +foreign operation, nonempty added tables and durable proof before fence release. +The SQL transaction and Podman lifecycle still require isolated integration +qualification. These are table-level compatibility checks, not a claim that +arbitrary database extensions/functions, other writers, or changed application +code are safe. The candidate images, migration scope and admission barrier must +also match the reviewed operation. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index 79d04a12..d38810b8 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -11,6 +11,53 @@ QUEUE_SCRIPT = r'''const {Queue}=require('bullmq'); try{const action=process.argv[1];if(action==='pause')await q.pause();else if(action==='resume')await q.resume();else if(action!=='status')throw Error('action'); console.log(JSON.stringify({paused:await q.isPaused(),counts:await q.getJobCounts('active','waiting','paused','delayed','failed','completed')}));} finally{await q.close()}})().catch(()=>process.exit(1));''' +# The exact three migrations in the privately qualified API candidate. This is +# an allowlist of additive schema history, never permission to discard app data. +ADDITIVE_MIGRATIONS = { + 'AddArchipelagoPublicationsAndRentals1791288000000':1791288000000, + 'AddMediaRegistrationIntents1791374400000':1791374400000, + 'AddMediaRegistrationRetirements1791374401000':1791374401000, +} +ADDITIVE_TABLES = {'archipelago_media_registrations','archipelago_publications','archipelago_publication_outbox','archipelago_rental_entitlements','archipelago_registration_intents'} +DB_COMMITMENTS_SQL = r''' +BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY; +SELECT format($query$ + SELECT jsonb_build_object('table',%L,'schema', + jsonb_build_object( + 'columns',(SELECT coalesce(jsonb_agg(jsonb_build_array(a.attnum,a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,pg_get_expr(d.adbin,d.adrelid)) ORDER BY a.attnum),'[]'::jsonb) FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attrelid=%s AND a.attnum>0 AND NOT a.attisdropped), + 'constraints',(SELECT coalesce(jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid,true)) ORDER BY conname),'[]'::jsonb) FROM pg_constraint WHERE conrelid=%s), + 'indexes',(SELECT coalesce(jsonb_agg(pg_get_indexdef(indexrelid) ORDER BY indexrelid::regclass::text),'[]'::jsonb) FROM pg_index WHERE indrelid=%s), + 'triggers',(SELECT coalesce(jsonb_agg(pg_get_triggerdef(oid,true) ORDER BY tgname),'[]'::jsonb) FROM pg_trigger WHERE tgrelid=%s AND NOT tgisinternal), + 'rls',%L,'policies',(SELECT coalesce(jsonb_agg(to_jsonb(p) ORDER BY policyname),'[]'::jsonb) FROM pg_policies p WHERE schemaname='public' AND tablename=%L)), + 'rows',count(*),'rows_sha256',encode(sha256(convert_to(coalesce(string_agg(to_jsonb(t)::text,E'\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM public.%I t; + $query$,c.relname,c.oid,c.oid,c.oid,c.oid,c.relrowsecurity::text||':'||c.relforcerowsecurity::text,c.relname,c.relname) +FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname +\gexec +SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.migrations m; +COMMIT; +''' +def verify_database_compatibility(before, after): + require(before.get('operation_id')==after.get('operation_id'),'Data compatibility operation changed') + old=before['tables'];new=after['tables'];require(set(old)<=set(new),'Data compatibility lost original tables') + extra=set(new)-set(old);require(extra<=ADDITIVE_TABLES,'Data compatibility contains unreviewed tables') + for name in old: + require(old[name]['schema']==new[name]['schema'],'Data compatibility changed original table schema') + if name!='migrations': + require(old[name]['rows']==new[name]['rows'] and old[name]['rows_sha256']==new[name]['rows_sha256'],'Data compatibility changed original rows') + for name in extra:require(new[name]['rows']==0,'Data compatibility contains new application data') + previous=before['migrations'];current=after['migrations'] + require(current[:len(previous)]==previous,'Data compatibility changed original migration history') + added=current[len(previous):];seen=set() + for row in added: + require(set(row)=={'id','timestamp','name'} and type(row['id']) is int and row['name'] not in seen and ADDITIVE_MIGRATIONS.get(row['name'])==row['timestamp'],'Data compatibility has unreviewed migration history') + seen.add(row['name']) + ordered=list(ADDITIVE_MIGRATIONS) + require([row['name'] for row in added]==ordered[:len(added)],'Data compatibility migration order changed') + expected_extra=(ADDITIVE_TABLES-{'archipelago_registration_intents'}) if added else set() + if len(added)>=2:expected_extra=ADDITIVE_TABLES + require(extra==expected_extra-set(old),'Data compatibility additions do not match reviewed migration evidence') + return {'original_tables':len(old),'new_empty_tables':sorted(extra),'reviewed_migrations':[r['name'] for r in added]} + def require(condition, message): if not condition: raise RuntimeError(message) def atomic(path, value): @@ -60,11 +107,11 @@ class Controller: self.record=json.loads(self.path.read_text()) if self.path.exists() else None if self.record:require(self.record['operation_id']==operation,'Maintenance journal changed') def save(self): atomic(self.path,self.record) - def run(self, argv, timeout=30, output=None): + def run(self, argv, timeout=30, output=None, input_bytes=None): if self.runner:return self.runner(argv,timeout,output) self.root.mkdir(mode=0o700,parents=True,exist_ok=True) with (self.root/'commands.private.log').open('ab') as errors: - result=subprocess.run(argv,stdout=output or subprocess.PIPE,stderr=errors,timeout=timeout,check=True,pass_fds=(self.lock_fd,)) + result=subprocess.run(argv,stdout=output or subprocess.PIPE,stderr=errors,timeout=timeout,check=True,pass_fds=(self.lock_fd,),input=input_bytes) if output:return b'' require(len(result.stdout)<=2*1024*1024,'Command response exceeds bound') return result.stdout @@ -149,8 +196,27 @@ class Controller: rows=json.loads(self.run(['podman','volume','inspect',*expected])) require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed') return {row['Name']:row['Mountpoint'] for row in rows} + def database_commitments(self): + raw=self.run(['podman','exec','-i','indeedhub-postgres','psql','-XqAt','--set=ON_ERROR_STOP=1','-U','indeedhub','-d','indeedhub'],timeout=300,input_bytes=DB_COMMITMENTS_SQL.encode()) + rows=[json.loads(line) for line in raw.decode().splitlines() if line.strip()] + tables={};migrations=None + for row in rows: + if 'migration_rows' in row: + require(migrations is None,'Duplicate database migration observation');migrations=row['migration_rows'] + else: + name=row.pop('table');require(name not in tables and re.fullmatch('[a-zA-Z_][a-zA-Z0-9_]*',name),'Invalid database table observation');tables[name]=row + require(tables and 'migrations' in tables and isinstance(migrations,list),'Database compatibility observation incomplete') + return {'operation_id':self.operation,'tables':tables,'migrations':migrations} + def verify_restored_data(self): + baseline=self.record.get('database_before') + require(baseline and baseline.get('operation_id')==self.operation,'Data compatibility baseline missing') + current=self.database_commitments();proof=verify_database_compatibility(baseline,current) + self.record['recovery_data_verification']={'operation_id':self.operation,'checked_at':time.time(),'before_sha256':hashlib.sha256(json.dumps(baseline,sort_keys=True).encode()).hexdigest(),'after_sha256':hashlib.sha256(json.dumps(current,sort_keys=True).encode()).hexdigest(),**proof} + self.record['recovery_data_verified']=True;self.save() def backup(self): if self.record.get('backup_complete'):return + if 'database_before' not in self.record: + self.record['database_before']=self.database_commitments();self.save() sources=self.volume_sources();self.record['volume_sources']=sources;self.save() backup=self.root/'backup';backup.mkdir(mode=0o700,exist_ok=True) if 'database.dump' not in self.record.setdefault('artifacts',{}): @@ -241,7 +307,7 @@ class Controller: if outcome!='committed': require(type(runtime.get('target_startup_began')) is bool,'Target-start obligation unavailable') if runtime['target_startup_began']: - require(self.record.get('recovery_data_verified') is True,'Data compatibility after rollback needs recorded verification; ingress remains closed') + self.verify_restored_data() else: self.record['rollback_data_claim']='No target startup/migration began; only original runtime restored.' diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 71358f39..d2a12762 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -121,4 +121,27 @@ class MaintenanceTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError,'business work'):c.legacy_api_idle() counts['other_active_transactions']=0;c.legacy_api_idle() self.assertEqual(c.record['legacy_api_empty_state'],counts) + def test_rollback_compatibility_binds_operation_preserves_rows_and_allows_only_empty_additions(self): + import copy + table={'schema':{'columns':['original']},'rows':0,'rows_sha256':'a'*64} + before={'operation_id':self.operation,'tables':{'migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]} + after=copy.deepcopy(before) + self.assertEqual(module.verify_database_compatibility(before,after)['original_tables'],2) + names=list(module.ADDITIVE_MIGRATIONS) + after['migrations'] += [{'id':i+2,'timestamp':module.ADDITIVE_MIGRATIONS[name],'name':name} for i,name in enumerate(names)] + after['tables']['migrations']['rows']=4;after['tables']['migrations']['rows_sha256']='b'*64 + for name in module.ADDITIVE_TABLES:after['tables'][name]=copy.deepcopy(table) + self.assertEqual(len(module.verify_database_compatibility(before,after)['new_empty_tables']),5) + for mutate in [lambda d:d.update(operation_id=str(uuid.uuid4())),lambda d:d['tables']['contents'].update(rows_sha256='c'*64),lambda d:d['tables']['contents']['schema'].update(columns=['changed']),lambda d:d['tables']['archipelago_publications'].update(rows=1),lambda d:d['migrations'][0].update(name='Altered'),lambda d:d['migrations'][-1].update(name='Unreviewed'),lambda d:d['tables'].update(unreviewed=copy.deepcopy(table))]: + damaged=copy.deepcopy(after);mutate(damaged) + with self.assertRaisesRegex(RuntimeError,'Data compatibility'):module.verify_database_compatibility(before,damaged) + def test_verified_rollback_records_operation_proof_before_releasing_fence(self): + c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'migrations':table},'migrations':[]} + c.record={'operation_id':self.operation,'phase':'Recovering','database_before':baseline};c.save() + c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) + module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Restored','target_startup_began':True}) + c.database_commitments=lambda:baseline + self.assertEqual(c.release('restored')['state'],'released') + self.assertEqual(c.record['recovery_data_verification']['operation_id'],self.operation) + self.assertEqual(c.record['recovery_data_verification']['before_sha256'],c.record['recovery_data_verification']['after_sha256']) if __name__=='__main__':unittest.main() From 2512a9f62bc490a10fbf8d8ba7ee779b5dc34e6d Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 02:57:31 -0400 Subject: [PATCH 12/13] Retain verified restored units and validate original installer identity bindings --- .../src/container/prod_orchestrator.rs | 21 +++++++ .../src/container/supervised_runtime.rs | 52 +++++++++++++++++ .../src/container/supervised_update.rs | 24 ++++++-- .../managed-update-recovery-implementation.md | 56 +++++++++++++++++++ 4 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 docs/managed-update-recovery-implementation.md diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 9396e306..993993f1 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -3283,6 +3283,16 @@ impl ProdContainerOrchestrator { } async fn prepare_for_start(&self, manifest: &AppManifest) -> Result<()> { + if super::supervised_update::installed_unit( + &self.data_dir, + &compute_container_name(manifest), + )? + .is_some() + { + // Already-reviewed managed configuration is authoritative. Applying + // today's catalog files/mount preparation could mutate restored data. + return Ok(()); + } self.run_pre_start_hooks(&manifest.app.id).await?; self.ensure_bind_mount_sockets(manifest).await?; self.ensure_bind_mount_dirs(manifest).await?; @@ -3585,6 +3595,17 @@ impl ProdContainerOrchestrator { } async fn ensure_resolved_source_available(&self, lm: &LoadedManifest) -> Result<()> { + if let Some((body, _)) = super::supervised_update::installed_unit( + &self.data_dir, + &compute_container_name(&lm.manifest), + )? { + let image = body + .lines() + .find_map(|line| line.trim().strip_prefix("Image=")) + .context("Saved managed image missing")?; + anyhow::ensure!(self.runtime.image_exists(image).await?, "Saved managed image is unavailable; refusing to pull or recreate from a changed catalog"); + return Ok(()); + } let resolved = lm.manifest.app.container.resolve().ok_or_else(|| { anyhow::anyhow!( "manifest for {} has invalid container source (neither image nor build)", diff --git a/core/archipelago/src/container/supervised_runtime.rs b/core/archipelago/src/container/supervised_runtime.rs index d633f304..ab7d5d8c 100644 --- a/core/archipelago/src/container/supervised_runtime.rs +++ b/core/archipelago/src/container/supervised_runtime.rs @@ -434,6 +434,58 @@ impl Supervisor for SystemdSupervisor { == target.name, "Original unit or reviewed immutable target changed before update" ); + if plan + .prepared + .manifest + .app + .container + .media_registration_identity + { + let identity = + crate::identity::NodeIdentity::load_existing(&self.data_dir.join("identity")) + .await?; + let pin = super::registration_pin::load_existing( + &self.data_dir, + &plan.prepared.manifest.app.id, + &identity, + )?; + let mut expected = plan.prepared.manifest.clone(); + super::registration_pin::apply_environment(&mut expected, &pin)?; + for entry in expected + .app + .environment + .iter() + .filter(|entry| entry.starts_with("ARCHIPELAGO_REGISTRATION_")) + { + let key = entry + .split_once('=') + .context("Malformed registration binding")? + .0; + let in_manifest: Vec<_> = plan + .prepared + .manifest + .app + .environment + .iter() + .filter(|value| value.split_once('=').is_some_and(|(name, _)| name == key)) + .collect(); + let in_unit: Vec<_> = plan + .prepared + .body + .lines() + .filter_map(|line| line.trim().strip_prefix("Environment=")) + .map(|value| value.trim_matches('"')) + .filter(|value| value.split_once('=').is_some_and(|(name, _)| name == key)) + .collect(); + anyhow::ensure!( + in_manifest.len() == 1 + && in_manifest[0] == entry + && in_unit.len() == 1 + && in_unit[0] == entry, + "Reviewed registration identity does not match the existing installer pin" + ); + } + } Ok(plan.prepared.clone()) } async fn target_hooks( diff --git a/core/archipelago/src/container/supervised_update.rs b/core/archipelago/src/container/supervised_update.rs index 3c0770a6..a3f0ee59 100644 --- a/core/archipelago/src/container/supervised_update.rs +++ b/core/archipelago/src/container/supervised_update.rs @@ -532,8 +532,8 @@ fn installed_path(data: &Path, name: &str) -> Result { } fn publish_installed(guard: &Guard, record: &Journal) -> Result<()> { anyhow::ensure!( - record.phase == Phase::Committed, - "Only committed units may be published" + matches!(record.phase, Phase::Committed | Phase::Restored), + "Only verified terminal units may be published" ); let dir = guard.directory().join("installed-units"); match std::fs::DirBuilder::new().mode(0o700).create(&dir) { @@ -550,7 +550,11 @@ fn publish_installed(guard: &Guard, record: &Journal) -> Result<()> { schema: 1, operation: record.id.clone(), name: member.original.name.clone(), - body: member.target_body.clone(), + body: if record.phase == Phase::Restored { + member.pinned_original_body.clone() + } else { + member.target_body.clone() + }, mode: member.original.file_mode, }; let temporary = dir.join(format!(".{}.tmp", uuid::Uuid::new_v4())); @@ -904,9 +908,13 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis } record.phase = Phase::Restored; save(guard, record)?; + publish_installed(guard, record)?; supervisor .release_barrier(&record.id, Completion::Restored) .await?; + for member in &record.members { + guard.release_hold(&member.original.name, &record.id)?; + } record.cleanup_done = true; save(guard, record) } @@ -931,9 +939,13 @@ pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Resu } } Phase::Restored => { + publish_installed(guard, &record)?; supervisor .release_barrier(&record.id, Completion::Restored) .await?; + for member in &record.members { + guard.release_hold(&member.original.name, &record.id)?; + } } // Never release a newer owner. _ => restore(guard, &mut record, supervisor).await?, } @@ -1300,7 +1312,11 @@ mod tests { assert_eq!(restored.image, "e".repeat(64)); assert_eq!(restored.config_sha256, runtime.original.config_sha256); assert_ne!(restored.id, format!("{:064x}", 1)); - assert!(super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap()); + assert_eq!( + installed_unit(root.path(), "movie").unwrap().unwrap().0, + *runtime.body.lock().unwrap() + ); assert_eq!(records(&guard).unwrap()[0].phase, Phase::Restored); } #[tokio::test] diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md new file mode 100644 index 00000000..bb153fa4 --- /dev/null +++ b/docs/managed-update-recovery-implementation.md @@ -0,0 +1,56 @@ +# Managed update runtime recovery + +Status: isolated source implementation; Rust and real Podman/systemd qualification +pending. No live update, snapshot, stop, backup or rollback has been performed by +this work. Active deployed source is unchanged. + +The managed path captures original source Quadlet bytes, mode, immutable image, +container identity, launch configuration and running intent. It requires an +original-hash-bound reviewed forward plan and verifies every planned image against +the already prepared catalog image. New manifest configuration/hooks are applied +on the forward path; rollback uses the captured original recipe and a private, +local-only writable-layer image. AutoRemove rollback recreates containers and does +not claim to restore their original IDs. Stopped supervised stacks currently fail +before mutation; the retained-container and separate stopped-stage paths cover +only their respective supported cases. + +The legacy IndeeHub controller runs under the same inherited lifecycle lock and +operation-owned reconciliation holds. Original writable layers are captured +before any destructive stop. The controller fences ingress, drains supported +legacy work, takes coherent quiescent volume/database backups and retains the +fence through cutover or recovery. Its exact source hash must match the separately +installed script; a backend binary alone does not install the controller. + +Completed updates and verified runtime restorations publish exact unit recipes +before releasing holds. Routine drift reconciliation validates those recipes; +it does not regenerate them from a newer catalog. Catalog-driven pre-start file +and mount mutations are skipped for these pinned installations. Missing saved +units/images are explicit recovery failures, never permission to reconstruct a +different runtime. Explicit uninstall removes the installed recipe, and completed +old journals cannot recreate it. A new reviewed transaction replaces the recipe. + +Opted-in API registration environments must match an already provisioned pin +and the existing node identity in both manifest and exact Quadlet. Administrative +plan preparation must use the existing installer provisioning code. Execution +never invents a node identity or accepts a browser-supplied unit or hook. + +Runtime restoration does not establish database compatibility. The controller's +restored-release verifier compares original table schemas and row commitments, +permitting only the exact reviewed additive migration prefix and empty new +application tables. Any other data/schema change keeps ingress closed. This is +not automatic database rollback or a promise that arbitrary migrations are +reversible. + +Qualification required before integration/activation: + +- Isolated backend compile and injectable lifecycle/fault tests, including lost + replies, daemon interruption, foreign units/holds and preflight failures. +- Disposable real Podman/systemd and PostgreSQL execution of the controller and + adapter. Sixteen pure controller tests currently pass; SQL/runtime behavior is + not yet qualified. +- Final seven-member private plan with installer-resolved identity environment, + verified local images and source-unit provenance; review required changes and + retained operator configuration without exposing secret values. +- Disk-capacity and recovery-image retention checks, backup integrity and a + documented recovery path for missing runtime artifacts. +- Actual-node controlled deployment and acceptance, preserving persistent data. From 68082cec4957a815c9cf87701877f120fbb36a7f Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 13:50:11 -0400 Subject: [PATCH 13/13] Verify complete maintenance backup hashes and restored database commitments --- .../managed-update-recovery-implementation.md | 21 ++++ scripts/indeehub-maintenance-controller.py | 6 +- .../test_indeehub_maintenance_controller.py | 24 ++++ .../test_indeehub_maintenance_postgres.py | 114 ++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 tests/regression/test_indeehub_maintenance_postgres.py diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index bb153fa4..9f59b0c0 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -54,3 +54,24 @@ Qualification required before integration/activation: - Disk-capacity and recovery-image retention checks, backup integrity and a documented recovery path for missing runtime artifacts. - Actual-node controlled deployment and acceptance, preserving persistent data. + +## Resumed qualification — 2026-10-07 + +Backup verification now requires the full database/four-volume artifact set and +rechecks SHA256, including same-size corruption. Nineteen pure controller tests +pass. The owned, network-none PostgreSQL fixture passes unchanged/additive +commitments, rejects four data/schema/history mutations, restores a real custom +dump with matching original commitments, and rejects a truncated dump. It mounts +no live volume and removes only its own container. This does not qualify actual +application writer drain or the complete supervised systemd cutover. + +The updater compiled and its full isolated suite ran: 1,958 passed, one failed, +five ignored. The failure is the old snake_case rental receipt JSON fixture; +`c2c4d915` already corrects that exact test on the release candidate branch. +Do not duplicate or suppress it here. Integrate and rerun the complete candidate +suite before claiming a green backend gate. The earlier interrupted compile +and PostgreSQL timeout remain failed/incomplete attempts, not acceptance. + +Evidence: `/tmp/archy-resumed-20261007-updater-full-backend.log`, +`/tmp/archy-resumed-20261007-indeehub-controller-tests-final.log`, and +`/tmp/archy-resumed-20261007-indeehub-postgres-restore.log`. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index d38810b8..daee9d4d 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -5,6 +5,7 @@ must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another ho """ import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay') +VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data') DATA = pathlib.Path('/var/lib/archipelago') QUEUE_SCRIPT = r'''const {Queue}=require('bullmq'); (async()=>{const q=new Queue('transcode',{connection:{host:process.env.QUEUE_HOST,port:Number(process.env.QUEUE_PORT||6379),password:process.env.QUEUE_PASSWORD,maxRetriesPerRequest:1}}); @@ -192,7 +193,7 @@ class Controller: classification=('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit' stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save() def volume_sources(self): - expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data'] + expected=VOLUMES rows=json.loads(self.run(['podman','volume','inspect',*expected])) require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed') return {row['Name']:row['Mountpoint'] for row in rows} @@ -282,8 +283,11 @@ class Controller: # Verification remains possible when API/storage endpoints are stopped. # The native adapter separately validates target/original runtime identity. for name in NAMES:require(self.record.get('stopped',{}).get(name,{}).get('confirmed'),'Original writer stop evidence missing') + expected_artifacts={'database.dump',*(volume+'.tar' for volume in VOLUMES)} + require(set(self.record.get('artifacts',{}))==expected_artifacts,'Backup artifact inventory incomplete or unexpected') for name,record in self.record['artifacts'].items(): path=self.root/'backup'/name;require(path.is_file() and not path.is_symlink() and path.stat().st_size==record['bytes'],'Backup artifact missing or changed') + require(sha(path)==record['sha256'],'Backup artifact checksum changed') return {'operation_id':self.operation,'state':'held'} def release(self, outcome): require(outcome in ('committed','restored','aborted'),'Invalid release outcome') diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index d2a12762..96ef50bb 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -47,6 +47,30 @@ class MaintenanceTests(unittest.TestCase): c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) (c.root/'backup').mkdir();(c.root/'backup'/'database.dump').write_bytes(b'not evidence') with self.assertRaisesRegex(RuntimeError,'Drain not complete'):c.verify() + def completed_backup(self): + c=self.controller + c.record={'operation_id':self.operation,'phase':'Drained','backup_complete':True, + 'stopped':{name:{'confirmed':True} for name in module.NAMES},'artifacts':{}} + c.save();c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) + (c.root/'backup').mkdir() + for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]: + path=c.root/'backup'/name;path.write_bytes(b'original') + c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)} + c.save() + return c + def test_complete_backup_checksums_allow_verification(self): + self.assertEqual(self.completed_backup().verify()['state'],'held') + def test_same_size_corruption_keeps_admission_closed(self): + c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!') + with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation);self.assertEqual(self.calls,[]) + def test_incomplete_or_unexpected_artifact_inventory_cannot_pass(self): + c=self.completed_backup();original=dict(c.record['artifacts']) + for artifacts in [{}, {k:v for k,v in original.items() if k!='database.dump'}, + {**original,'../foreign':original['database.dump']}]: + c.record['artifacts']=artifacts + with self.assertRaisesRegex(RuntimeError,'inventory'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation) def test_forced_original_exit_never_marks_writer_completed(self): c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared','original_members':module.validate_members(members())};c.save() def command(argv,timeout,output): diff --git a/tests/regression/test_indeehub_maintenance_postgres.py b/tests/regression/test_indeehub_maintenance_postgres.py new file mode 100644 index 00000000..8410c35c --- /dev/null +++ b/tests/regression/test_indeehub_maintenance_postgres.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Exercise rollback commitments on an owned, network-isolated PostgreSQL. + +Requires an already imported image: --image IMAGE. Never mounts node volumes, +publishes ports, or invokes the maintenance entrypoint against installed apps. +""" +import argparse +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +MODULE = Path(__file__).resolve().parents[2] / 'scripts/indeehub-maintenance-controller.py' +spec = importlib.util.spec_from_file_location('maintenance', MODULE) +maintenance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(maintenance) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--image', required=True) + args = parser.parse_args() + image = subprocess.check_output( + ['podman', 'image', 'inspect', '--format', '{{.Id}}', args.image], text=True, + ).strip() + name = 'archy-maintenance-sql-' + uuid.uuid4().hex + container = None + try: + container = subprocess.check_output([ + 'podman', 'run', '-d', '--pull=never', '--network=none', '--name', name, + '--tmpfs', '/var/lib/postgresql/data:rw', + '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=indeedhub', + '-e', 'POSTGRES_DB=indeedhub', image, + ], text=True).strip() + deadline = time.monotonic() + 60 + while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + if time.monotonic() > deadline: + raise RuntimeError('Disposable PostgreSQL did not become ready') + time.sleep(0.5) + + def sql(statement, database='indeedhub'): + return subprocess.check_output([ + 'podman', 'exec', '-i', container, 'psql', '-XqAt', + '--set=ON_ERROR_STOP=1', '-U', 'indeedhub', '-d', database, + ], input=statement.encode(), timeout=60) + + sql('CREATE TABLE migrations(id serial PRIMARY KEY, timestamp bigint NOT NULL, name text NOT NULL);' + "INSERT INTO migrations(timestamp,name) VALUES(1,'Original1');" + 'CREATE TABLE contents(id int PRIMARY KEY, title text NOT NULL);' + "INSERT INTO contents VALUES(1,'retained original');") + with tempfile.TemporaryDirectory(prefix=name) as root: + controller = maintenance.Controller(root, str(uuid.uuid4()), 0) + + database = 'indeedhub' + + def fixture_run(argv, timeout=30, output=None, input_bytes=None): + assert argv[:4] == ['podman', 'exec', '-i', 'indeedhub-postgres'] + assert output is None and input_bytes is not None + return sql(input_bytes.decode(), database) + + controller.run = fixture_run + before = controller.database_commitments() + dump = subprocess.check_output([ + 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub', + '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl', + ], timeout=60) + sql('CREATE DATABASE restore_check') + restore_command = ['podman', 'exec', '-i', container, 'pg_restore', + '-U', 'indeedhub', '-d', 'restore_check', + '--exit-on-error', '--no-owner', '--no-acl'] + subprocess.run(restore_command, input=dump, check=True, timeout=60) + database = 'restore_check' + maintenance.verify_database_compatibility(before, controller.database_commitments()) + database = 'indeedhub' + rejected_dump = subprocess.run(restore_command, input=dump[:32], timeout=60, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert rejected_dump.returncode != 0, 'Truncated dump incorrectly accepted' + maintenance.verify_database_compatibility(before, controller.database_commitments()) + for table in sorted(maintenance.ADDITIVE_TABLES): + sql(f'CREATE TABLE {table}(id int PRIMARY KEY);') + for migration, timestamp in maintenance.ADDITIVE_MIGRATIONS.items(): + sql(f"INSERT INTO migrations(timestamp,name) VALUES({timestamp},'{migration}');") + proof = maintenance.verify_database_compatibility(before, controller.database_commitments()) + assert len(proof['new_empty_tables']) == 5 + rejected = 0 + for mutation, undo in [ + ("UPDATE contents SET title='changed'", "UPDATE contents SET title='retained original'"), + ('ALTER TABLE contents ADD COLUMN unexpected text', 'ALTER TABLE contents DROP COLUMN unexpected'), + ('INSERT INTO archipelago_publications VALUES(1)', 'DELETE FROM archipelago_publications'), + ("UPDATE migrations SET name='changed' WHERE id=1", "UPDATE migrations SET name='Original1' WHERE id=1"), + ]: + sql(mutation) + try: + maintenance.verify_database_compatibility(before, controller.database_commitments()) + except RuntimeError: + rejected += 1 + else: + raise AssertionError('Changed database incorrectly accepted') + sql(undo) + maintenance.verify_database_compatibility(before, controller.database_commitments()) + print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected, + 'network': 'none', 'live_volumes_mounted': False, + 'custom_dump_restored': True, 'truncated_dump_rejected': True})) + finally: + if container: + subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL) + + +if __name__ == '__main__': + main()