fix(lifecycle): reconciler skips apps with an in-flight lifecycle op
The reconciler doesn't take the RPC layer's per-app FIFO op lock (known
limit of 891cbba4): between a restart worker's stop and start halves it saw
the mempool frontend "missing", repair-recreated it behind systemd's back,
killed the worker's fresh container 11s after start, and left the unit down
for ~3.5 min until the next heal — gate test 123 measured exactly that
window (.228 iteration 3, 2026-07-09).
New crate::app_ops module owns the op-lock registry + stack member table
(runtime.rs and dependencies.rs now delegate) so the reconciler can probe
lifecycle_op_in_flight(app_id) — covering both the app's own key and its
owning stack package — and skip that app for the cycle. The ownership-sweep
podman restart gets the same guard. Health monitor is a follow-up.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
e21f3baf22
commit
e275494a68
@@ -623,29 +623,9 @@ pub(super) async fn ordered_containers_for_start(package_id: &str) -> Result<Vec
|
||||
/// other members absent AND user-stopped (.228 indeedhub, 2026-07-09 — the
|
||||
/// reconciler then correctly refused to revive them).
|
||||
pub(super) fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] {
|
||||
match package_id {
|
||||
"immich" => &["immich-postgres", "immich-redis", "immich"],
|
||||
"indeedhub" => &[
|
||||
"indeedhub-postgres",
|
||||
"indeedhub-redis",
|
||||
"indeedhub-minio",
|
||||
"indeedhub-relay",
|
||||
"indeedhub-api",
|
||||
"indeedhub-ffmpeg",
|
||||
"indeedhub",
|
||||
],
|
||||
"btcpay-server" | "btcpayserver" | "btcpay" => {
|
||||
&["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"]
|
||||
}
|
||||
"netbird" => &["netbird-server", "netbird-dashboard", "netbird"],
|
||||
// The legacy umbrella id maps to the split stack (the orchestrator's
|
||||
// umbrella alias handles this too; listing it here keeps the RPC
|
||||
// layer's fan-out explicit).
|
||||
"mempool" | "mempool-web" => {
|
||||
&["archy-mempool-db", "mempool-api", "archy-mempool-web"]
|
||||
}
|
||||
_ => &[],
|
||||
}
|
||||
// Canonical table moved to crate::app_ops (shared with the reconciler's
|
||||
// in-flight-op guard).
|
||||
crate::app_ops::stack_member_app_ids(package_id)
|
||||
}
|
||||
|
||||
fn order_present_containers(package_id: &str, containers: Vec<String>) -> Vec<String> {
|
||||
|
||||
@@ -1210,17 +1210,10 @@ async fn cascade_restart_address_caching_dependents(
|
||||
/// Workers take the app's lock as their first await; tokio's Mutex is fair
|
||||
/// (FIFO), so queued operations run in RPC arrival order and the final
|
||||
/// state matches the last request.
|
||||
static APP_OP_LOCKS: std::sync::LazyLock<
|
||||
std::sync::Mutex<std::collections::HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
|
||||
> = std::sync::LazyLock::new(Default::default);
|
||||
|
||||
/// Registry lives in crate::app_ops so background actors (reconciler) can
|
||||
/// probe in-flight ops without an api ↔ container dependency cycle.
|
||||
fn app_op_lock(package_id: &str) -> Arc<tokio::sync::Mutex<()>> {
|
||||
APP_OP_LOCKS
|
||||
.lock()
|
||||
.expect("APP_OP_LOCKS poisoned")
|
||||
.entry(orchestrator_app_id(package_id).to_string())
|
||||
.or_default()
|
||||
.clone()
|
||||
crate::app_ops::op_lock(orchestrator_app_id(package_id))
|
||||
}
|
||||
|
||||
fn uses_single_orchestrator_app(package_id: &str) -> bool {
|
||||
|
||||
Reference in New Issue
Block a user