fix(lifecycle): reconciler skips apps with an in-flight lifecycle op
The reconciler doesn't take the RPC layer's per-app FIFO op lock (known
limit of 891cbba4): between a restart worker's stop and start halves it saw
the mempool frontend "missing", repair-recreated it behind systemd's back,
killed the worker's fresh container 11s after start, and left the unit down
for ~3.5 min until the next heal — gate test 123 measured exactly that
window (.228 iteration 3, 2026-07-09).
New crate::app_ops module owns the op-lock registry + stack member table
(runtime.rs and dependencies.rs now delegate) so the reconciler can probe
lifecycle_op_in_flight(app_id) — covering both the app's own key and its
owning stack package — and skip that app for the cycle. The ownership-sweep
podman restart gets the same guard. Health monitor is a follow-up.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
e21f3baf22
commit
e275494a68
@@ -1482,6 +1482,18 @@ impl ProdContainerOrchestrator {
|
||||
for lm in manifests {
|
||||
let app_id = lm.manifest.app.id.clone();
|
||||
let container_name = compute_container_name(&lm.manifest);
|
||||
// A package.start/stop/restart worker is mid-sequence on this app
|
||||
// (or its stack): between its stop and start halves the container
|
||||
// is legitimately absent, and repair-recreating it here races the
|
||||
// worker — the reconciler once killed a restart's fresh container
|
||||
// and left the unit down for minutes (.228 mempool frontend, gate
|
||||
// 2026-07-09). Skip this cycle; the worker owns the outcome.
|
||||
if crate::app_ops::lifecycle_op_in_flight(&app_id) {
|
||||
report.record(&app_id, ReconcileAction::Left("lifecycle-op-in-flight".into()));
|
||||
crate::crash_recovery::pending_boot_start_done(&app_id);
|
||||
crate::crash_recovery::pending_boot_start_done(&container_name);
|
||||
continue;
|
||||
}
|
||||
if mode == ReconcileMode::ExistingOnly
|
||||
&& requires_archival_bitcoin(&app_id)
|
||||
&& disk_gb < ARCHIVAL_BITCOIN_DISK_GB
|
||||
@@ -1544,6 +1556,11 @@ impl ProdContainerOrchestrator {
|
||||
.iter()
|
||||
.filter(|c| matches!(c.state, ContainerState::Running))
|
||||
{
|
||||
// Same in-flight guard as the main loop: don't podman-restart
|
||||
// a container a lifecycle worker is mid-way through cycling.
|
||||
if crate::app_ops::lifecycle_op_in_flight(&c.name) {
|
||||
continue;
|
||||
}
|
||||
if ensure_running_container_ownership(&c.name).await {
|
||||
tracing::info!(container = %c.name, "volume ownership repaired during reconcile — restarting to recover");
|
||||
let _ = tokio::process::Command::new("podman")
|
||||
|
||||
Reference in New Issue
Block a user