Archipelago — open-source initial import

2026-08-12 10:55:49 +00:00
commit 82ca09f8b9
1664 changed files with 359095 additions and 0 deletions
@@ -0,0 +1,162 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/all-apps-lifecycle.bats
+#
+# DESTRUCTIVE per-app lifecycle matrix across EVERY installed app (breadth) —
+# the active counterpart to the read-only all-apps-matrix.bats and the ~8 deep
+# per-app suites. For each installed, NON-protected app it drives:
+#   stop → verify stopped → start → verify running → restart → verify running
+# and, when ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1, a FULL TEARDOWN:
+#   uninstall (full, removes data) → verify GONE from My Apps (no #13 ghost) →
+#   reinstall from the node catalog → verify running.
+#
+# Reinstall spec source: the node catalog (default /opt/archipelago/web-ui/
+# catalog.json), whose `.apps[]` entries carry {dockerImage, containerConfig} —
+# exactly what package.install needs. Multi-container stacks (immich, mempool,
+# netbird, btcpay, indeedhub) ignore dockerImage internally but still require it,
+# and route to their orchestrator/stack handler; the catalog entry is enough to
+# trigger the reinstall. An app with no catalog entry is skipped (logged), not
+# failed — there's no spec to reinstall it from.
+#
+# ── PROTECTED apps (NEVER touched — neither cycled nor torn down) ────────────
+#   - chain state, expensive to resync:   bitcoin*, electrumx/electrs
+#   - WALLET / financial state, teardown = IRREVERSIBLE fund/credential loss:
+#                                          lnd, btcpay*, fedimint*
+#   The user asked to protect only bitcoin + electrum; the wallet-bearing apps
+#   are protected by DEFAULT here for safety (a full uninstall destroys their
+#   seed/channel/guardian state). Override the entire set with
+#   ARCHY_MATRIX_PROTECT="space separated ids" to tear them down too — you WILL
+#   lose their data.
+#
+# ── Gating ──────────────────────────────────────────────────────────────────
+#   lifecycle tier  → ARCHY_ALLOW_DESTRUCTIVE=1
+#   teardown  tier  → ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1
+#   Both skip otherwise, so this file is inert in a normal run. ON-NODE ONLY
+#   (reads catalog.json on disk + drives the local package lifecycle).
+#
+# This is a HEAVY suite: a full teardown of ~15-20 apps re-pulls images and can
+# run for a long time. Intended as an explicit, supervised coverage pass, not a
+# per-iteration gate step.
+
+load '../lib/rpc.bash'
+
+CATALOG="${ARCHY_CATALOG:-/opt/archipelago/web-ui/catalog.json}"
+
+# Protected — see header. Override with ARCHY_MATRIX_PROTECT to change the set.
+PROTECT="${ARCHY_MATRIX_PROTECT:-bitcoin-knots bitcoin-core bitcoin electrumx electrs mempool-electrs lnd btcpay-server btcpayserver btcpay fedimint fedimint-clientd fedimint-gateway}"
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+is_protected() {
+  local id="$1" p
+  for p in $PROTECT; do [[ "$p" == "$id" ]] && return 0; done
+  return 1
+}
+
+get_package_data() {
+  rpc_result server.get-state '{}' 2>/dev/null | jq -c '.data["package-data"] // {}'
+}
+
+# Canonical app ids the catalog can (re)install.
+catalog_ids() {
+  jq -r '(.apps // [])[].id' "$CATALOG" 2>/dev/null
+}
+
+# Installed primary apps we will exercise: catalog ids present in My Apps,
+# minus the protected set. (Catalog-scoped so we skip sub-containers like
+# immich_postgres that surface as their own package-data entries.)
+target_apps() {
+  local pd; pd=$(get_package_data)
+  local id
+  for id in $(catalog_ids); do
+    echo "$pd" | jq -e --arg i "$id" 'has($i)' >/dev/null 2>&1 || continue
+    is_protected "$id" && continue
+    echo "$id"
+  done
+}
+
+# Top-level state of an app in My Apps, or "absent" when the entry is gone.
+app_state() {
+  get_package_data | jq -r --arg i "$1" '.[$i].state // "absent"'
+}
+
+# Poll My Apps until app $1 reaches state $2 (or "absent"); $3 = timeout secs.
+wait_state() {
+  local id="$1" target="$2" timeout="${3:-180}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    [[ "$(app_state "$id")" == "$target" ]] && return 0
+    sleep 3
+  done
+  echo "wait_state: $id never reached '$target' (last='$(app_state "$id")') within ${timeout}s" >&2
+  return 1
+}
+
+# Build a package.install payload for $1 from the catalog, or fail (no spec).
+catalog_install_payload() {
+  local id="$1" img cfg
+  img=$(jq -r --arg i "$id" '(.apps // [])[] | select(.id==$i) | .dockerImage // empty' "$CATALOG")
+  [[ -n "$img" ]] || return 1
+  cfg=$(jq -c --arg i "$id" '(.apps // [])[] | select(.id==$i) | .containerConfig // null' "$CATALOG")
+  if [[ "$cfg" == "null" ]]; then
+    jq -nc --arg id "$id" --arg img "$img" '{id:$id, dockerImage:$img}'
+  else
+    jq -nc --arg id "$id" --arg img "$img" --argjson cfg "$cfg" '{id:$id, dockerImage:$img, containerConfig:$cfg}'
+  fi
+}
+
+# ────────────────────────────────────────────────────────────────────
+@test "prerequisites: catalog present and at least one target app" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  [[ -f "$CATALOG" ]] || { echo "# catalog not found: $CATALOG" >&3; false; }
+  run target_apps
+  [ "$status" -eq 0 ]
+  [ -n "$output" ] || { echo "# no non-protected installed apps to exercise" >&3; false; }
+  echo "# protected (skipped): $PROTECT" >&3
+  echo "# targets ($(echo "$output" | wc -w)): $(echo $output)" >&3
+}
+
+@test "lifecycle: stop → start → restart every non-protected app" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  local fails="" id
+  for id in $(target_apps); do
+    [[ "$(app_state "$id")" == "running" ]] || continue   # only cycle running apps
+    rpc_result package.stop "{\"id\":\"$id\"}" >/dev/null 2>&1
+    wait_state "$id" stopped 120 || { fails+="$id:stop "; }
+    rpc_result package.start "{\"id\":\"$id\"}" >/dev/null 2>&1
+    wait_state "$id" running 240 || { fails+="$id:start "; continue; }
+    rpc_result package.restart "{\"id\":\"$id\"}" >/dev/null 2>&1
+    wait_state "$id" running 240 || { fails+="$id:restart "; }
+  done
+  [[ -z "$fails" ]] || { echo "# lifecycle failures: $fails" >&3; false; }
+}
+
+@test "teardown: full uninstall (no ghost) → reinstall every non-protected app" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  local fails="" skipped="" id payload
+  for id in $(target_apps); do
+    if ! payload=$(catalog_install_payload "$id"); then
+      skipped+="$id "
+      continue
+    fi
+    rpc_result package.uninstall "{\"id\":\"$id\"}" >/dev/null 2>&1
+    # No ghost: the entry must leave My Apps (the #13 class). 71cc9ac4 bounds the
+    # teardown so this can no longer hang indefinitely.
+    if ! wait_state "$id" absent 300; then
+      fails+="$id:ghost "
+      continue
+    fi
+    rpc_result package.install "$payload" >/dev/null 2>&1
+    wait_state "$id" running 420 || fails+="$id:reinstall "
+  done
+  [[ -n "$skipped" ]] && echo "# skipped (no catalog spec to reinstall from): $skipped" >&3
+  [[ -z "$fails" ]] || { echo "# teardown failures: $fails" >&3; false; }
+}
@@ -0,0 +1,134 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/all-apps-matrix.bats
+#
+# Manifest-driven, fleet-wide lifecycle health matrix. The per-app suites
+# (bitcoin-knots, lnd, mempool, immich, …) cover ~8 core apps in depth; this
+# covers EVERY installed app in breadth, automatically — no hardcoded list.
+#
+# It derives the app set from server.get-state's package-data (the My Apps map)
+# and asserts baseline health across all of them. Read-only (no destructive env
+# needed), so it joins run.sh / run-gate.sh on every node and grows coverage as
+# nodes install more apps.
+#
+# Catches, fleet-wide, the bug classes the narrow gate missed:
+#   - apps STUCK in a transitional state (the #13/#14 ghost: installing/removing
+#     that never settles)
+#   - apps sitting in error/failed
+#   - running UI apps with no reachable lan-address (generalized port-drift)
+
+load '../lib/rpc.bash'
+
+# Transitional states are legitimate momentarily but must not PERSIST. Steady:
+# running/stopped/exited/created/paused/installed/not-installed.
+TRANSITIONAL_RE='^(installing|pulling-image|pulling|downloading|removing|uninstalling|updating|starting|stopping|restarting)$'
+BAD_RE='^(error|failed)$'
+
+# Apps whose state is allowed to be non-running at rest (no UI/health expectation
+# beyond "settled"). Empty by default; override via ARCHY_MATRIX_ALLOW_STOPPED
+# (space-separated ids) on nodes where an app is intentionally left stopped.
+ALLOW_STOPPED="${ARCHY_MATRIX_ALLOW_STOPPED:-}"
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# Echo the package-data object (the My Apps map) once.
+get_package_data() {
+  rpc_result server.get-state '{}' 2>/dev/null | jq -c '.data["package-data"] // {}'
+}
+
+# Space-separated list of installed app ids.
+app_ids() {
+  get_package_data | jq -r 'keys[]'
+}
+
+# ────────────────────────────────────────────────────────────────────
+@test "matrix has apps to check (get-state returns a non-empty My Apps map)" {
+  run app_ids
+  [ "$status" -eq 0 ]
+  [ -n "$output" ]
+  echo "# matrix covers $(echo "$output" | wc -w) apps: $(echo $output)" >&3
+}
+
+@test "no installed app is STUCK in a transitional state (settles within window)" {
+  local settle="${ARCHY_MATRIX_SETTLE_SECS:-45}"
+  local deadline=$(( $(date +%s) + settle ))
+  local stuck=""
+  # Re-poll: a transitional state right now may just be a genuine in-progress op,
+  # so only fail apps that are STILL transitional after the settle window.
+  while :; do
+    stuck=""
+    local pd; pd=$(get_package_data)
+    for id in $(echo "$pd" | jq -r 'keys[]'); do
+      local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
+      [[ "$st" =~ $TRANSITIONAL_RE ]] && stuck+="${id}=${st} "
+    done
+    [[ -z "$stuck" ]] && break
+    (( $(date +%s) >= deadline )) && break
+    sleep 5
+  done
+  [[ -z "$stuck" ]] || { echo "# STUCK transitional after ${settle}s: $stuck" >&3; false; }
+}
+
+@test "no installed app is in an error/failed state" {
+  local pd; pd=$(get_package_data)
+  local bad=""
+  for id in $(echo "$pd" | jq -r 'keys[]'); do
+    local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
+    [[ "$st" =~ $BAD_RE ]] && bad+="${id}=${st} "
+  done
+  [[ -z "$bad" ]] || { echo "# error/failed apps: $bad" >&3; false; }
+}
+
+@test "every running app reports a recognized state (no empty/garbage state)" {
+  local pd; pd=$(get_package_data)
+  local junk=""
+  for id in $(echo "$pd" | jq -r 'keys[]'); do
+    local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
+    case "$st" in
+      running|stopped|exited|created|paused|installed|not-installed|\
+installing|pulling-image|pulling|downloading|removing|uninstalling|updating|starting|stopping|restarting|\
+error|failed|degraded) : ;;
+      *) junk+="${id}='${st}' " ;;
+    esac
+  done
+  [[ -z "$junk" ]] || { echo "# unrecognized state values: $junk" >&3; false; }
+}
+
+@test "every running UI app exposes a lan-address (generalized port-drift)" {
+  # A running app whose manifest declares a UI interface (ui=="true") must have a
+  # non-null lan-address on that interface — otherwise its UI is unreachable
+  # (the immich/port-drift failure mode, asserted across ALL UI apps). Poll
+  # briefly to absorb the transient null seen while a container is mid-recreate.
+  local deadline=$(( $(date +%s) + ${ARCHY_MATRIX_UI_SECS:-30} ))
+  local missing=""
+  while :; do
+    missing=""
+    local pd; pd=$(get_package_data)
+    for id in $(echo "$pd" | jq -r 'keys[]'); do
+      local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
+      [[ "$st" == "running" ]] || continue
+      # interface keys whose manifest marks ui=="true"
+      local ui_ifaces
+      ui_ifaces=$(echo "$pd" | jq -r --arg i "$id" \
+        '.[$i].manifest.interfaces // {} | to_entries[] | select(.value.ui=="true") | .key')
+      for k in $ui_ifaces; do
+        local addr
+        addr=$(echo "$pd" | jq -r --arg i "$id" --arg k "$k" \
+          '.[$i].installed["interface-addresses"][$k]["lan-address"] // "null"')
+        [[ "$addr" == "null" || -z "$addr" ]] && missing+="${id}:${k} "
+      done
+    done
+    [[ -z "$missing" ]] && break
+    (( $(date +%s) >= deadline )) && break
+    sleep 3
+  done
+  [[ -z "$missing" ]] || { echo "# running UI apps missing lan-address: $missing" >&3; false; }
+}
@@ -0,0 +1,112 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/backend-survives-archipelago-restart.bats
+#
+# Quadlet-everywhere promise (Phase 3 of v1.7.52): backend containers
+# (bitcoin-knots / lnd / electrumx) are managed by systemd via Quadlet
+# units, NOT parented under archipelago.service's cgroup. Restarting the
+# archipelago service must NOT take them down.
+#
+# This is the regression gate for FM3 (cgroup cascade SIGKILL — observed
+# live on .198 on 2026-05-01: stopping archipelago.service killed every
+# container in its cgroup, leaving the box in a multi-hour recovery
+# loop). Until v1.7.52 Phase 3 ships, this suite is EXPECTED TO FAIL on
+# fleet boxes — it serves as the executable definition of "Phase 3
+# complete". Do not gate the release on it passing pre-Phase-3.
+#
+# Sister to companion-survives-archipelago-restart.bats which tests the
+# same property for UI companions (already shipping via Quadlet since
+# commit 6e716f68).
+#
+# Gated by ARCHY_ALLOW_DESTRUCTIVE=1 because it bounces archipelago.
+
+# bats-core ships no `fail`; bats-assert isn't installed on the alpha fleet.
+# Define the same minimal helper the other suites use (see mempool.bats) so a
+# tripped assertion reports as a real test failure, not a status-127 crash.
+fail() { echo "$@" >&2; return 1; }
+
+backend_units=(
+  "bitcoin-knots"
+  "bitcoin-core"
+  "lnd"
+  "electrumx"
+)
+
+container_running() {
+  local name="$1"
+  [[ "$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" == "true" ]]
+}
+
+wait_archipelago_back() {
+  local timeout="${1:-60}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    if curl -fsS -o /dev/null "http://127.0.0.1:5678/health" 2>/dev/null; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+@test "destructive gate enabled" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+}
+
+@test "at least one backend container is running before restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  local up=0
+  for c in "${backend_units[@]}"; do
+    if container_running "$c"; then
+      up=$(( up + 1 ))
+    fi
+  done
+  (( up > 0 )) || skip "No backends installed on this node"
+}
+
+@test "backends survive archipelago restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # Snapshot: which backends were up before we touched anything.
+  local before=()
+  for c in "${backend_units[@]}"; do
+    if container_running "$c"; then
+      before+=("$c")
+    fi
+  done
+  (( ${#before[@]} > 0 )) || skip "No backends installed on this node"
+
+  # Capture pre-restart container IDs so we can verify the SAME process
+  # survives — not "the orchestrator started a fresh container after the
+  # cascade SIGKILL'd the original" (which would also be a fail; FM3 is
+  # specifically about losing the running container, even if the
+  # orchestrator can recreate one minutes later).
+  declare -A pre_id
+  for c in "${before[@]}"; do
+    pre_id["$c"]=$(podman inspect --format '{{.Id}}' "$c" 2>/dev/null || echo "")
+  done
+
+  # Bounce archipelago. Same approach as companion-survives-* for parity.
+  if systemctl --user list-units --no-legend archipelago.service | grep -q archipelago; then
+    systemctl --user restart archipelago.service
+  else
+    sudo systemctl restart archipelago.service
+  fi
+
+  run wait_archipelago_back 60
+  [ "$status" -eq 0 ]
+
+  # Every backend that was up before must still be up after, AND it must
+  # be the SAME container instance (same .Id). A different .Id means the
+  # original was killed and a fresh one was created — that's the FM3
+  # failure we're catching.
+  for c in "${before[@]}"; do
+    run container_running "$c"
+    [ "$status" -eq 0 ] || fail "backend $c died across archipelago restart (FM3 cgroup cascade)"
+
+    local post_id
+    post_id=$(podman inspect --format '{{.Id}}' "$c" 2>/dev/null || echo "")
+    [[ -n "$post_id" ]] || fail "backend $c has no container id after restart"
+    [[ "$post_id" == "${pre_id[$c]}" ]] \
+      || fail "backend $c was recreated across archipelago restart (FM3): pre=${pre_id[$c]:0:12} post=${post_id:0:12}"
+  done
+}
@@ -0,0 +1,182 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/bitcoin-knots.bats
+#
+# Lifecycle tests for the bitcoin-knots package.
+#
+# Tiers:
+#   - Read-only (always runs):        presence, status, state-reporting consistency
+#   - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart on this very container
+#   - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
+#     — this breaks LND/ElectrumX/BTCPay/mempool, so never enabled on a node serving real users.
+#
+# Pre-req: bitcoin-knots is installed. We do NOT install it from scratch here
+# because doing so on the live host would require wiping 700GB of chain data.
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1  # make sure setup_file gets a fresh token
+  rpc_login
+  unset ARCHY_FORCE_LOGIN      # subsequent test subshells reuse the session file
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Read-only tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "container-list includes bitcoin-knots" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "bitcoin-knots")' >/dev/null
+}
+
+@test "container-list reports a valid state for bitcoin-knots" {
+  # Poll briefly: a container caught mid-reconcile can momentarily report a
+  # transient state ("restarting"/"configured"/"removing") or no state at all.
+  # A genuinely-stuck container never settles, so this still catches real
+  # breakage; it only absorbs churn (e.g. another container bouncing right
+  # before the read-only tier runs).
+  local state="" deadline=$(( $(date +%s) + 30 ))
+  while (( $(date +%s) < deadline )); do
+    run rpc_result container-list
+    [ "$status" -eq 0 ]
+    state=$(echo "$output" | jq -r '.[] | select(.name == "bitcoin-knots") | .state')
+    [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] && return 0
+    sleep 3
+  done
+  echo "bitcoin-knots never reported a settled valid state within 30s (last: '$state')" >&2
+  return 1
+}
+
+@test "container-status returns a valid status object for bitcoin-knots" {
+  # During orchestrator alias migration, container-status can fail for some
+  # app_id aliases even while container-list/state is correct. Accept either:
+  # (a) valid container-status object OR (b) valid container-list state entry.
+  run rpc_call container-status '{"app_id":"bitcoin-knots"}'
+  [ "$status" -eq 0 ]
+  local err
+  err=$(echo "$output" | jq -r '.error.message // empty')
+  if [[ -z "$err" ]]; then
+    echo "$output" | jq -e '.result | has("status") or has("state") or has("running")' >/dev/null
+    return 0
+  fi
+
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "bitcoin-knots") | has("state")' >/dev/null
+}
+
+@test "bitcoin.getinfo succeeds when bitcoin-knots is running" {
+  local state
+  state=$(rpc_result container-list | jq -r '.[] | select(.name == "bitcoin-knots") | .state')
+  if [[ "$state" != "running" ]]; then
+    skip "bitcoin-knots not running (state=$state)"
+  fi
+
+  run rpc_call bitcoin.getinfo
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.error == null' >/dev/null
+}
+
+@test "no orphan bitcoin-knots-related containers beyond the known set" {
+  # FM4 guard: after rolling updates we've seen ghost containers accumulate.
+  # Known-good container set for the bitcoin-knots package is just "bitcoin-knots".
+  # Anything matching bitcoin-knots* in podman ps that isn't in the known set is a red flag.
+  local count
+  count=$(ssh_podman_ps | awk '/bitcoin-knots/ {print $NF}' | grep -Ec '^bitcoin-knots(-[a-z]+)?$' || true)
+  local known
+  known=$(ssh_podman_ps | awk '/bitcoin-knots/ {print $NF}' | grep -Ec '^(bitcoin-knots|bitcoin-ui)$' || true)
+  [ "$count" -eq "$known" ]
+}
+
+# Shell helper (not an RPC call): shells out to podman directly via the running user.
+# Only works when bats is run on the archy host itself (which is the plan).
+ssh_podman_ps() {
+  podman ps -a --format '{{.ID}} {{.State}} {{.Names}}'
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier (stop → start → restart on the same container)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions bitcoin-knots to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.stop '{"id":"bitcoin-knots"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status bitcoin-knots stopped 60
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings bitcoin-knots back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.start '{"id":"bitcoin-knots"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status bitcoin-knots running 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves bitcoin-knots in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.restart '{"id":"bitcoin-knots"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status bitcoin-knots running 120
+  [ "$status" -eq 0 ]
+}
+
+@test "bitcoin.getinfo succeeds after restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # Give bitcoind up to 120s to accept RPC after a cold restart — reloading the
+  # block index + chainstate can take a while even on a synced node.
+  local deadline=$(( $(date +%s) + 120 ))
+  while (( $(date +%s) < deadline )); do
+    if rpc_call bitcoin.getinfo | jq -e '.error == null' >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 3
+  done
+  # NB: bats-assert's `fail` is not loaded in this file (only ../lib/rpc.bash),
+  # so emit + return non-zero directly rather than calling an undefined helper
+  # (which fails with "fail: command not found" / status 127 and hides the real
+  # reason). A node mid-IBD legitimately can't serve getinfo here — that's an
+  # environmental precondition (see required-stack "synced archival"), not a
+  # product regression.
+  echo "bitcoin.getinfo never recovered after restart within 120s" >&2
+  return 1
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade-destructive tier (uninstall + reinstall)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes bitcoin-knots" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.uninstall '{"id":"bitcoin-knots","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status bitcoin-knots absent 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.install bitcoin-knots returns to running" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  # manifest_path is relative to data_dir/apps/
+  run rpc_result package.install '{"manifest_path":"bitcoin-knots/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status bitcoin-knots running 180
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,117 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/bitcoin-receive.bats
+#
+# Regression coverage for the Bitcoin "Receive" flow. Receive addresses come
+# from LND's hot wallet via the `lnd.newaddress` RPC, so this exercises the
+# exact path that broke on the fleet:
+#   - .116: LND REST published on the wrong host port (8080 vs the manifest's
+#           18080) -> connection refused -> receive failed with the generic
+#           "Operation failed. Check server logs." message.
+#   - .228: the same family surfaced to the UI as a *false* "wallet is locked".
+#
+# These tests run on the archy host (they shell into podman / curl localhost).
+#
+# Tiers: read-only only — generating a receive address is non-destructive.
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# Resolve the LND REST host port from the manifest (single source of truth) so
+# this test follows the manifest rather than hard-coding 18080.
+_lnd_rest_host_port() {
+  local mf
+  for mf in \
+    "${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yml" \
+    "${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yaml" \
+    "$BATS_TEST_DIRNAME/../../../apps/lnd/manifest.yml"; do
+    [[ -r "$mf" ]] || continue
+    # The REST mapping is the `- host: <N>` whose following `container:` is 8080.
+    awk '
+      /- host:/        { host=$3 }
+      /container:/     { if ($2 == 8080 && host != "") { print host; exit } }
+    ' "$mf"
+    return 0
+  done
+}
+
+_lnd_running() {
+  rpc_result container-list 2>/dev/null \
+    | jq -e '.[] | select(.name == "lnd" and .state == "running")' >/dev/null 2>&1
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Read-only tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "LND REST is reachable on the manifest host port (catches port drift)" {
+  _lnd_running || skip "lnd not running"
+  local port
+  port=$(_lnd_rest_host_port)
+  [[ -n "$port" ]] || skip "could not resolve LND REST host port from manifest"
+
+  # A TCP connect is enough: drift (container published on a different host
+  # port) shows up as connection-refused here, exactly as on .116.
+  run curl -sk -o /dev/null --max-time 8 "https://127.0.0.1:${port}/v1/getinfo"
+  if [ "$status" -ne 0 ]; then
+    echo "LND REST not reachable on host port ${port} (curl exit $status) — likely published-port drift" >&2
+    return 1
+  fi
+}
+
+@test "lnd.newaddress returns a bech32 address when lnd is running" {
+  _lnd_running || skip "lnd not running"
+
+  # The bitcoin bounce in bitcoin-knots.bats cascade-restarts lnd (24fd97ed).
+  # Depending on where the probe lands in lnd's startup it sees a different
+  # transient code — REST unreachable, gRPC "waiting to start" (mapped to
+  # LND_ERROR), wallet locked until the auto-unlocker gets through, or a
+  # post-unlock sync phase. All of those are the node settling, not broken —
+  # retry up to 180s (run A caught WALLET_LOCKED, run B caught LND_ERROR
+  # while lnd was seconds into its restart; both self-healed within a couple
+  # of minutes). Only LND_WALLET_UNINITIALIZED (no wallet — never self-heals)
+  # fails immediately, and anything still erroring after the window fails
+  # loudly below.
+  local deadline=$((SECONDS + 180)) err addr
+  while :; do
+    run rpc_call lnd.newaddress
+    [ "$status" -eq 0 ]
+    err=$(echo "$output" | jq -r '.error.message // .error // empty')
+    addr=$(echo "$output" | jq -r '.result.address // empty')
+    [[ -n "$err" && "$err" != *LND_WALLET_UNINITIALIZED* && $SECONDS -lt $deadline ]] || break
+    sleep 10
+  done
+
+  # The whole point of the fix: a running lnd must hand back a real address.
+  if [[ -n "$err" ]]; then
+    echo "lnd.newaddress errored on a running node: $err" >&2
+    return 1
+  fi
+  if [[ "$addr" != bc1* ]]; then
+    echo "expected a bech32 (bc1…) address, got: '$addr'" >&2
+    return 1
+  fi
+}
+
+@test "receive errors are specific, never the generic catch-all" {
+  # Even when receive legitimately can't produce an address, the message must be
+  # actionable (start with 'Bitcoin address' and/or carry a [CODE] token) — the
+  # generic 'Operation failed' is what hid the real cause on .116.
+  run rpc_call lnd.newaddress
+  [ "$status" -eq 0 ]
+  local err
+  err=$(echo "$output" | jq -r '.error.message // .error // empty')
+  if [[ "$err" == "Operation failed. Check server logs for details." ]]; then
+    echo "receive returned the generic catch-all instead of a specific reason" >&2
+    return 1
+  fi
+}
@@ -0,0 +1,146 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/btcpay.bats
+#
+# Lifecycle tests for the btcpay-server multi-container stack:
+#   - btcpay-server      (the main app)
+#   - archy-btcpay-db    (postgres)
+#   - archy-nbxplorer    (Bitcoin watcher)
+#
+# Multi-container variant of bitcoin-knots.bats / lnd.bats / electrumx.bats.
+# UI URL coverage is in ui-coverage.bats; this suite is L1 (RPC API) + L3
+# (lifecycle survival).
+#
+# Pre-req: btcpay-server installed, bitcoin-knots running.
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+btcpay_components=(
+  "btcpay-server"
+  "archy-btcpay-db"
+  "archy-nbxplorer"
+)
+
+@test "container-list includes every btcpay-stack component" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  for c in "${btcpay_components[@]}"; do
+    echo "$output" | jq -e --arg n "$c" '.[] | select(.name == $n)' >/dev/null \
+      || skip "btcpay component $c not present (stack not installed)"
+  done
+}
+
+@test "container-list reports valid states for every btcpay component" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local present=0
+  for c in "${btcpay_components[@]}"; do
+    local state
+    state=$(echo "$output" | jq -r --arg n "$c" '.[] | select(.name == $n) | .state')
+    [[ -n "$state" ]] || continue
+    present=$((present + 1))
+    [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] \
+      || fail "invalid state for $c: $state"
+  done
+  (( present > 0 )) || skip "btcpay stack not installed"
+}
+
+@test "no orphan btcpay-related containers beyond the known set" {
+  local total known
+  total=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(btcpay|archy-btcpay|archy-nbxplorer)' || true)
+  known=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(btcpay-server|archy-btcpay-db|archy-nbxplorer)$' || true)
+  [ "$total" -eq "$known" ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions btcpay-server to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
+    || skip "btcpay-server not installed"
+
+  run rpc_result package.stop '{"id":"btcpay-server"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status btcpay-server stopped 60
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings btcpay-server back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
+    || skip "btcpay-server not installed"
+
+  run rpc_result package.start '{"id":"btcpay-server"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status btcpay-server running 180
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves btcpay-server in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
+    || skip "btcpay-server not installed"
+
+  run rpc_result package.restart '{"id":"btcpay-server"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status btcpay-server running 180
+  [ "$status" -eq 0 ]
+}
+
+@test "db + nbxplorer remain running across btcpay-server restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  for c in archy-btcpay-db archy-nbxplorer; do
+    podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1 \
+      || skip "btcpay supporting container $c not installed"
+  done
+
+  for c in archy-btcpay-db archy-nbxplorer; do
+    local state
+    state=$(podman inspect --format '{{.State.Status}}' "$c" 2>/dev/null)
+    [[ "$state" == "running" ]] \
+      || fail "supporting btcpay container $c is not running (state=$state) — package.restart cascaded into it"
+  done
+}
+
+@test "package.uninstall removes the whole btcpay stack" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
+    || skip "btcpay-server not installed"
+
+  run rpc_result package.uninstall '{"id":"btcpay-server","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  for c in "${btcpay_components[@]}"; do
+    run wait_for_container_status "$c" absent 120
+    [ "$status" -eq 0 ] || fail "btcpay component $c not removed by uninstall"
+  done
+}
+
+@test "package.install restores the whole btcpay stack" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install '{"manifest_path":"btcpay-server/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  for c in "${btcpay_components[@]}"; do
+    run wait_for_container_status "$c" running 240
+    [ "$status" -eq 0 ] || fail "btcpay component $c never reached running after reinstall"
+  done
+}
@@ -0,0 +1,220 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/cascade-uninstall.bats
+#
+# CASCADE-tier regression guard for the uninstall → reinstall lifecycle — the
+# exact bug class the gate's DESTRUCTIVE tier never exercised:
+#   #13 "uninstall ghost"  — app stayed in My Apps after uninstall because the
+#                            package state entry wasn't cleared when teardown hit
+#                            cleanup residue (returned Err before removing it).
+#   #14 "reinstall stops"  — a reinstall stalled partway on the stale state/data
+#                            left behind by the broken uninstall.
+#
+# Uses a THROWAWAY app (default grafana — not installed on prod/test nodes, no
+# user data) so it can drive the FULL teardown path (no preserve_data), which is
+# where #13 actually bit. Precondition-skips if the app is already installed, so
+# it can NEVER destroy real data on a populated node.
+#
+# "No ghost" is asserted against server.get-state's package-data (literally the
+# My Apps map) — the entry must disappear, not linger with a stale state /
+# stuck uninstall stage.
+#
+# Gated on ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1. RPC-based, so it works on-node or
+# against a remote ARCHY_HOST (the data-dir residue check is on-node only).
+
+load '../lib/rpc.bash'
+
+CASCADE_APP="${ARCHY_CASCADE_APP:-grafana}"
+CASCADE_IMAGE="${ARCHY_CASCADE_IMAGE:-docker.io/grafana/grafana:10.2.0}"
+CASCADE_CONFIG="${ARCHY_CASCADE_CONFIG:-{\"ports\":[\"3000:3000\"],\"volumes\":[\"/var/lib/archipelago/grafana:/var/lib/grafana\"],\"env\":[\"GF_PATHS_DATA=/var/lib/grafana\",\"GF_USERS_ALLOW_SIGN_UP=false\"]}}"
+CASCADE_DATA_DIR="${ARCHY_CASCADE_DATA_DIR:-/var/lib/archipelago/${CASCADE_APP}}"
+
+setup_file() {
+  cascade_enabled || return 0
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+cascade_enabled() {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]]
+}
+
+# True when CASCADE_APP has an entry in My Apps (server.get-state package-data).
+app_in_my_apps() {
+  rpc_result server.get-state '{}' 2>/dev/null \
+    | jq -e --arg id "$CASCADE_APP" '.data["package-data"] | has($id)' >/dev/null 2>&1
+}
+
+# Top-level state of CASCADE_APP in My Apps, or "absent" when the entry is gone.
+app_state() {
+  rpc_result server.get-state '{}' 2>/dev/null \
+    | jq -r --arg id "$CASCADE_APP" '.data["package-data"][$id].state // "absent"'
+}
+
+# Live uninstall stage shown by My Apps, or an empty string when the entry is gone
+# or the backend has not emitted a stage yet.
+app_uninstall_stage() {
+  rpc_result server.get-state '{}' 2>/dev/null \
+    | jq -r --arg id "$CASCADE_APP" '.data["package-data"][$id]["uninstall-stage"] // ""'
+}
+
+# Mirror the frontend's AppCard.vue mapping so the gate proves the UI has
+# backend data that can render as a monotonic, non-fake progress bar.
+uninstall_stage_percent() {
+  local stage="$1"
+  if [[ "$stage" =~ \(([0-9]+)[[:space:]]*/[[:space:]]*([0-9]+)\) ]]; then
+    local done="${BASH_REMATCH[1]}" total="${BASH_REMATCH[2]}"
+    if (( total > 0 )); then
+      (( done > total )) && done="$total"
+      echo $(( 10 + (done * 40 / total) ))
+      return 0
+    fi
+  fi
+  if [[ "$stage" =~ [Vv]olume ]]; then echo 70; return 0; fi
+  if [[ "$stage" =~ [Dd]ata ]]; then echo 90; return 0; fi
+  return 1
+}
+
+# Poll until CASCADE_APP disappears while enforcing the progress contract:
+# stages must be parseable, monotonic, below 100 before terminal absence, and
+# the operation must emit at least one visible stage instead of silently hanging.
+wait_absent_with_truthful_uninstall_progress() {
+  local timeout="${1:-180}"
+  local deadline=$(( $(date +%s) + timeout ))
+  local saw_stage=0 last_percent=0
+  while (( $(date +%s) < deadline )); do
+    local state stage percent
+    state="$(app_state)"
+    [[ "$state" == "absent" ]] && {
+      (( saw_stage == 1 )) || {
+        echo "uninstall progress: no uninstall-stage observed before terminal absence" >&2
+        return 1
+      }
+      return 0
+    }
+
+    stage="$(app_uninstall_stage)"
+    if [[ -n "$stage" ]]; then
+      if ! percent="$(uninstall_stage_percent "$stage")"; then
+        echo "uninstall progress: unparseable stage '$stage'" >&2
+        return 1
+      fi
+      (( percent >= last_percent )) || {
+        echo "uninstall progress regressed: ${percent}% after ${last_percent}% (stage '$stage')" >&2
+        return 1
+      }
+      (( percent < 100 )) || {
+        echo "uninstall progress reached ${percent}% before terminal absence (stage '$stage')" >&2
+        return 1
+      }
+      saw_stage=1
+      last_percent="$percent"
+    fi
+    sleep 2
+  done
+  echo "wait_absent_with_truthful_uninstall_progress: $CASCADE_APP did not disappear within ${timeout}s (last='$(app_state)', stage='$(app_uninstall_stage)')" >&2
+  return 1
+}
+
+# Poll My Apps until CASCADE_APP reaches $1 (a state, or "absent").
+wait_app_state() {
+  local target="$1" timeout="${2:-180}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    [[ "$(app_state)" == "$target" ]] && return 0
+    sleep 3
+  done
+  echo "wait_app_state: $CASCADE_APP never reached '$target' (last='$(app_state)') within ${timeout}s" >&2
+  return 1
+}
+
+# ────────────────────────────────────────────────────────────────────
+@test "cascade gate enabled" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+}
+
+@test "precondition: ${CASCADE_APP} is not already installed (protects real data)" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  if app_in_my_apps; then
+    skip "${CASCADE_APP} already installed here — refusing to uninstall (would destroy data); set ARCHY_CASCADE_APP to an uninstalled throwaway"
+  fi
+}
+
+@test "install ${CASCADE_APP} (fresh) reaches running with a truthful, non-silent progression" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  app_in_my_apps && skip "already installed (precondition skip)"
+
+  run rpc_result package.install "{\"id\":\"${CASCADE_APP}\",\"dockerImage\":\"${CASCADE_IMAGE}\",\"containerConfig\":${CASCADE_CONFIG}}"
+  [ "$status" -eq 0 ]
+
+  # Progress truthfulness: must pass through a transitional install state (not a
+  # silent no-op) and land on running. A warm image cache can blow through the
+  # transitional states between polls, so a missed transitional is a warn, not a
+  # failure; reaching running is the hard assertion.
+  local saw_transitional=0 deadline=$(( $(date +%s) + 300 ))
+  while (( $(date +%s) < deadline )); do
+    case "$(app_state)" in
+      installing|pulling-image|pulling|downloading|starting|created) saw_transitional=1 ;;
+      running) break ;;
+    esac
+    sleep 2
+  done
+  [ "$(app_state)" == "running" ]
+  [ "$saw_transitional" -eq 1 ] || echo "# note: no transitional install state observed (image likely cached)" >&3
+}
+
+@test "uninstall ${CASCADE_APP} reports truthful progress and clears My Apps — NO ghost (#13)" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  app_in_my_apps || skip "${CASCADE_APP} not installed (install step must have failed)"
+
+  run rpc_result package.uninstall "{\"id\":\"${CASCADE_APP}\"}"
+  [ "$status" -eq 0 ]
+
+  # The container must go away…
+  run wait_for_container_status "$CASCADE_APP" absent 180
+  [ "$status" -eq 0 ]
+
+  # …AND the My Apps entry must be GONE — the #13 ghost was the entry lingering
+  # with a stale state / stuck uninstall stage. While polling, prove the backend
+  # emits stage data the UI can render as monotonic, non-full progress.
+  run wait_absent_with_truthful_uninstall_progress 120
+  [ "$status" -eq 0 ]
+
+  # Belt-and-suspenders: the key is truly absent from package-data.
+  run app_in_my_apps
+  [ "$status" -ne 0 ]
+}
+
+@test "uninstall removed the data dir (full teardown, no residue)" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  # Needs the local filesystem — on-node runs only.
+  case "${ARCHY_HOST:-127.0.0.1}" in
+    127.0.0.1|localhost) : ;;
+    *) skip "data-dir residue check is on-node only (ARCHY_HOST=${ARCHY_HOST})" ;;
+  esac
+  [[ ! -e "$CASCADE_DATA_DIR" ]]
+}
+
+@test "reinstall ${CASCADE_APP} returns to running (#14)" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install "{\"id\":\"${CASCADE_APP}\",\"dockerImage\":\"${CASCADE_IMAGE}\",\"containerConfig\":${CASCADE_CONFIG}}"
+  [ "$status" -eq 0 ]
+  run wait_app_state running 300
+  [ "$status" -eq 0 ]
+}
+
+@test "cleanup: uninstall ${CASCADE_APP} to leave the node as found" {
+  cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  run rpc_result package.uninstall "{\"id\":\"${CASCADE_APP}\"}"
+  [ "$status" -eq 0 ]
+  run wait_for_container_status "$CASCADE_APP" absent 180
+  [ "$status" -eq 0 ]
+  run wait_absent_with_truthful_uninstall_progress 120
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,146 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/companion-survives-archipelago-restart.bats
+#
+# Quadlet promise: companion UIs (archy-bitcoin-ui, archy-lnd-ui,
+# archy-electrs-ui) are managed by systemd, not archipelago. Restarting
+# the archipelago user service must NOT take them down.
+#
+# This is the regression gate for the .228 incident in
+# feedback_container_lifecycle_failure_modes.md (FM1: companions vanished
+# from `podman ps -a` after archipelago crash-loop).
+#
+# Gated by ARCHY_ALLOW_DESTRUCTIVE=1 because it bounces archipelago.
+
+companion_units=(
+  "archy-bitcoin-ui"
+  "archy-lnd-ui"
+  "archy-electrs-ui"
+)
+
+unit_dir="$HOME/.config/containers/systemd"
+
+unit_file_present() {
+  local name="$1"
+  [[ -f "$unit_dir/$name.container" ]]
+}
+
+service_active() {
+  local name="$1"
+  systemctl --user is-active --quiet "$name.service"
+}
+
+container_running() {
+  local name="$1"
+  [[ "$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" == "true" ]]
+}
+
+wait_service_active() {
+  local name="$1"
+  local timeout="${2:-60}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    if service_active "$name"; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+wait_archipelago_back() {
+  local timeout="${1:-60}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    if curl -fsS -o /dev/null "http://127.0.0.1:5678/health" 2>/dev/null; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+@test "destructive gate enabled" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+}
+
+@test "every installed companion has a quadlet unit on disk" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  local present=0
+  for c in "${companion_units[@]}"; do
+    if container_running "$c"; then
+      run unit_file_present "$c"
+      [ "$status" -eq 0 ]
+      present=$(( present + 1 ))
+    fi
+  done
+  (( present > 0 )) || skip "No companions installed on this node"
+}
+
+@test "every installed companion service is active before restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  for c in "${companion_units[@]}"; do
+    if container_running "$c"; then
+      run service_active "$c"
+      [ "$status" -eq 0 ]
+    fi
+  done
+}
+
+@test "companions survive archipelago restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # Snapshot: which companions were up before we touched anything.
+  local before=()
+  for c in "${companion_units[@]}"; do
+    if container_running "$c"; then
+      before+=("$c")
+    fi
+  done
+  (( ${#before[@]} > 0 )) || skip "No companions installed on this node"
+
+  # Bounce archipelago. The user service is the production canonical name;
+  # fall back to the system service for older nodes.
+  if systemctl --user list-units --no-legend archipelago.service | grep -q archipelago; then
+    systemctl --user restart archipelago.service
+  else
+    sudo systemctl restart archipelago.service
+  fi
+
+  run wait_archipelago_back 60
+  [ "$status" -eq 0 ]
+
+  # Every companion that was up before must still be up + healthy after.
+  for c in "${before[@]}"; do
+    run service_active "$c"
+    [ "$status" -eq 0 ]
+    run container_running "$c"
+    [ "$status" -eq 0 ]
+  done
+}
+
+@test "deleted unit file is recreated within one reconcile tick" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # Pick a companion that's currently running.
+  local target=""
+  for c in "${companion_units[@]}"; do
+    if container_running "$c"; then
+      target="$c"
+      break
+    fi
+  done
+  [[ -n "$target" ]] || skip "No companions installed on this node"
+
+  # Delete the unit file behind systemd's back. The reconciler should
+  # notice and rewrite it within one 30s tick, then start the service.
+  rm -f "$unit_dir/$target.container"
+  systemctl --user daemon-reload >/dev/null 2>&1 || true
+  systemctl --user stop "$target.service" >/dev/null 2>&1 || true
+
+  # Allow up to two reconcile ticks (60s + grace).
+  run wait_service_active "$target" 90
+  [ "$status" -eq 0 ]
+
+  run unit_file_present "$target"
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,146 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/electrumx.bats
+#
+# Lifecycle tests for the electrumx package (containers are named
+# `electrumx` + `archy-electrs-ui`). Mirrors bitcoin-knots.bats /
+# lnd.bats so the 5× release-gate run exercises electrumx through
+# the same state matrix.
+#
+# Tiers:
+#   - Read-only (always runs):        presence, valid state, TCP reachable
+#   - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
+#   - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
+#
+# Pre-req: electrumx is installed and bitcoin-knots is running (electrumx
+# depends on bitcoind RPC for headers).
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Read-only tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "container-list includes electrumx" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "electrumx")' >/dev/null
+}
+
+@test "container-list reports a valid state for electrumx" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local state
+  state=$(echo "$output" | jq -r '.[] | select(.name == "electrumx") | .state')
+  [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
+}
+
+@test "electrumx TCP port accepts connections when running" {
+  local state
+  state=$(rpc_result container-list | jq -r '.[] | select(.name == "electrumx") | .state')
+  if [[ "$state" != "running" ]]; then
+    skip "electrumx not running (state=$state)"
+  fi
+
+  # Same probe required-stack.bats uses — divergence flags a real regression.
+  run python3 - <<'PY'
+import socket
+s = socket.create_connection(("127.0.0.1", 50001), 3)
+s.close()
+print("ok")
+PY
+  [ "$status" -eq 0 ]
+}
+
+@test "no orphan electrumx-related containers beyond the known set" {
+  # FM4 guard: known-good electrumx-package set is {electrumx, archy-electrs-ui}.
+  local total known
+  total=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(electrumx|electrs|archy-electrs(-[a-z]+)?)$' || true)
+  known=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(electrumx|archy-electrs-ui)$' || true)
+  [ "$total" -eq "$known" ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier (stop → start → restart on the same container)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions electrumx to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.stop '{"id":"electrumx"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status electrumx stopped 60
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings electrumx back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.start '{"id":"electrumx"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status electrumx running 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves electrumx in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.restart '{"id":"electrumx"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status electrumx running 120
+  [ "$status" -eq 0 ]
+}
+
+@test "electrumx TCP port recovers after restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # electrumx replays its index against bitcoind on cold start; allow 120s.
+  local deadline=$(( $(date +%s) + 120 ))
+  while (( $(date +%s) < deadline )); do
+    if python3 -c 'import socket; socket.create_connection(("127.0.0.1", 50001), 3).close()' \
+        >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 3
+  done
+  fail "electrumx TCP port never reopened after restart"
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade-destructive tier (uninstall + reinstall)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes electrumx" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.uninstall '{"id":"electrumx","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status electrumx absent 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.install electrumx returns to running" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install '{"manifest_path":"electrumx/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status electrumx running 240
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,117 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/fedimint.bats
+#
+# Lifecycle tests for the fedimint package. The fedimint federation
+# daemon runs as a single container; the gateway is its own package
+# (fedimint-gateway). Mirrors the single-container pattern of
+# lnd.bats / electrumx.bats for L1 (RPC API) + L3 (lifecycle survival).
+# UI URL coverage is in ui-coverage.bats.
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+fedimint_skip_if_absent() {
+  podman inspect fedimint --format '{{.State.Status}}' >/dev/null 2>&1 \
+    || skip "fedimint not installed"
+}
+
+@test "container-list includes fedimint" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "fedimint")' >/dev/null \
+    || skip "fedimint not installed"
+}
+
+@test "container-list reports a valid state for fedimint" {
+  fedimint_skip_if_absent
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local state
+  state=$(echo "$output" | jq -r '.[] | select(.name == "fedimint") | .state')
+  [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
+}
+
+@test "no orphan fedimint-related containers beyond the known set" {
+  local total known
+  total=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(fedimint|fedimintd|fedimint-gateway)' || true)
+  # `fedimint-clientd` (the dual-ecash HTTP bridge) is a legitimate, known
+  # container — and the unanchored `total` regex above counts it (it starts
+  # with "fedimint"). It must therefore be in the known set too, or every node
+  # running fedimint-clientd false-fails this orphan check.
+  known=$(podman ps -a --format '{{.Names}}' \
+    | grep -Ec '^(fedimint|fedimint-clientd|fedimint-gateway)$' || true)
+  [ "$total" -eq "$known" ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions fedimint to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  fedimint_skip_if_absent
+
+  run rpc_result package.stop '{"id":"fedimint"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status fedimint stopped 60
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings fedimint back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  fedimint_skip_if_absent
+
+  run rpc_result package.start '{"id":"fedimint"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status fedimint running 180
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves fedimint in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  fedimint_skip_if_absent
+
+  run rpc_result package.restart '{"id":"fedimint"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status fedimint running 180
+  [ "$status" -eq 0 ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade-destructive tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes fedimint" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  fedimint_skip_if_absent
+
+  run rpc_result package.uninstall '{"id":"fedimint","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status fedimint absent 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.install fedimint returns to running" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install '{"manifest_path":"fedimint/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status fedimint running 240
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,126 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/immich.bats
+#
+# Lifecycle tests for the manifest-driven immich stack. The user-facing package is
+# "immich" (catalog title + icon); container-list reports it package-level as
+# "immich". Its containers are named immich_server / immich_postgres /
+# immich_redis (underscore) to match the runtime's per-app lifecycle references.
+#
+# Tiers:
+#   - Read-only (always):                      presence + valid state
+#   - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
+#   - Cascade (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall (preserve_data)
+#
+# RPC-based, so correct whether run on the host or against a remote ARCHY_HOST.
+
+load '../lib/rpc.bash'
+
+IMMICH_IMAGE="146.59.87.168:3000/lfg2025/immich-server:release"
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Read-only tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "container-list includes immich" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "immich")' >/dev/null
+}
+
+@test "container-list reports a valid state for immich" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local state
+  state=$(echo "$output" | jq -r '.[] | select(.name == "immich") | .state')
+  [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
+}
+
+@test "immich exposes its web UI lan-address (port 2283)" {
+  # Poll briefly: lan_address is derived from the published host port, which is
+  # momentarily absent (null) while immich_server is mid-recreate (e.g. a
+  # health-monitor bounce during the read-only tier). A genuinely unexposed
+  # immich never publishes 2283, so this still catches real port drift; it only
+  # absorbs the transient null seen under churn.
+  # 90s (not 30s): the immich stack (postgres→redis→server with DB migrations on
+  # boot) can take >30s to publish its host port after a churn-induced recreate,
+  # and the destructive-tier immich tests already allow 180–240s for the same
+  # stack. A genuinely unexposed immich still never publishes 2283, so this keeps
+  # catching real port drift while tolerating slow-but-healthy boots.
+  local deadline=$(( $(date +%s) + 90 ))
+  while (( $(date +%s) < deadline )); do
+    run rpc_result container-list
+    [ "$status" -eq 0 ]
+    if echo "$output" \
+      | jq -e '.[] | select(.name == "immich") | .lan_address // "" | test("2283")' >/dev/null; then
+      return 0
+    fi
+    sleep 3
+  done
+  echo "immich never reported a lan_address containing 2283 within 90s" >&2
+  return 1
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier (stop → start → restart)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions immich to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  # package.stop is async ({"status":"stopping"}) and a stack stop can race a
+  # still-settling prior op, so the end state — not the immediate RPC return — is
+  # the assertion.
+  rpc_call package.stop '{"id":"immich"}' >/dev/null 2>&1 || true
+  run wait_for_container_status immich stopped 90
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings immich back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  # Async start; the server comes up only after postgres is ready (~30s+), so wait.
+  rpc_call package.start '{"id":"immich"}' >/dev/null 2>&1 || true
+  run wait_for_container_status immich running 180
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves immich in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  run rpc_result package.restart '{"id":"immich"}'
+  [ "$status" -eq 0 ]
+  # Restart = ordered stop+start of the whole 3-container stack (postgres→redis→
+  # server, with the server doing DB-readiness + migrations on boot), so it needs
+  # at least as long as `start` (180s) — more, since it stops first. The old 120s
+  # was inconsistent with the start test and false-failed on heavily-loaded nodes.
+  run wait_for_container_status immich running 240
+  [ "$status" -eq 0 ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade tier (uninstall + reinstall the stack)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes immich (data preserved)" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  run rpc_result package.uninstall '{"id":"immich","preserve_data":true}'
+  [ "$status" -eq 0 ]
+  run wait_for_container_status immich absent 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.install immich returns to running" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  run rpc_result package.install "{\"id\":\"immich\",\"dockerImage\":\"${IMMICH_IMAGE}\"}"
+  [ "$status" -eq 0 ]
+  run wait_for_container_status immich running 180
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,154 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/lnd.bats
+#
+# Lifecycle tests for the lnd package. Mirrors bitcoin-knots.bats so the
+# 5× release-gate run exercises lnd through the same state matrix.
+#
+# Tiers:
+#   - Read-only (always runs):        presence, state-reporting consistency, RPC reachable
+#   - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
+#   - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
+#
+# Pre-req: lnd is installed. Reinstall path is gated separately because it
+# wipes the wallet macaroons and forces re-onboarding.
+
+load '../lib/rpc.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Read-only tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "container-list includes lnd" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.[] | select(.name == "lnd")' >/dev/null
+}
+
+@test "container-list reports a valid state for lnd" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local state
+  state=$(echo "$output" | jq -r '.[] | select(.name == "lnd") | .state')
+  [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
+}
+
+@test "lnd cli getinfo succeeds when lnd is running" {
+  local state
+  state=$(rpc_result container-list | jq -r '.[] | select(.name == "lnd") | .state')
+  if [[ "$state" != "running" ]]; then
+    skip "lnd not running (state=$state)"
+  fi
+
+  # lnd's RPC readiness LAGS the container "running" state: after a (re)start the
+  # wallet must auto-unlock before lncli answers, so a single-shot getinfo races
+  # that window and false-fails. Retry until ready (~90s), like a health probe.
+  # `timeout 10` per attempt: a wedged lnd RPC (e.g. chain-blind after its
+  # bitcoin backend was recreated under it, .228 2026-07-08) otherwise hangs
+  # a single exec — and with it the whole suite — indefinitely.
+  run sh -lc 'for i in $(seq 1 80); do
+    timeout 10 podman exec lnd lncli \
+      --tlscertpath /root/.lnd/tls.cert \
+      --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon \
+      --rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
+    sleep 3
+  done; exit 1'
+  [ "$status" -eq 0 ]
+}
+
+@test "no orphan lnd-related containers beyond the known set" {
+  # FM4 guard: rolling updates have left ghost containers behind in the past.
+  # Known-good lnd-package container set is {lnd, archy-lnd-ui}.
+  local total known
+  total=$(podman ps -a --format '{{.Names}}' | grep -Ec '^(archy-)?lnd(-[a-z]+)?$' || true)
+  known=$(podman ps -a --format '{{.Names}}' | grep -Ec '^(lnd|archy-lnd-ui)$' || true)
+  [ "$total" -eq "$known" ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier (stop → start → restart on the same container)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions lnd to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.stop '{"id":"lnd"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status lnd stopped 60
+  [ "$status" -eq 0 ]
+}
+
+@test "package.start brings lnd back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.start '{"id":"lnd"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status lnd running 240
+  [ "$status" -eq 0 ]
+}
+
+@test "package.restart leaves lnd in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  run rpc_result package.restart '{"id":"lnd"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status lnd running 240
+  [ "$status" -eq 0 ]
+}
+
+@test "lncli getinfo recovers after restart" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  # lnd takes longer than bitcoind to accept RPC after cold restart because
+  # the wallet has to be unlocked first, then it reconnects to bitcoind and
+  # re-syncs the graph. On a loaded node this exceeds 90s (observed ~2min on
+  # .228, then synced_to_chain:true). Give it 240s.
+  local deadline=$(( $(date +%s) + 240 ))
+  while (( $(date +%s) < deadline )); do
+    if sh -lc 'podman exec lnd lncli \
+        --tlscertpath /root/.lnd/tls.cert \
+        --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon \
+        --rpcserver localhost:10009 getinfo >/dev/null' 2>/dev/null; then
+      return 0
+    fi
+    sleep 3
+  done
+  fail "lncli getinfo never recovered after restart"
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade-destructive tier (uninstall + reinstall)
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes lnd" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.uninstall '{"id":"lnd","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status lnd absent 120
+  [ "$status" -eq 0 ]
+}
+
+@test "package.install lnd returns to running" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install '{"manifest_path":"lnd/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  run wait_for_container_status lnd running 180
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,197 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/mempool.bats
+#
+# Lifecycle tests for the mempool stack:
+#   - mempool          (legacy install path; the frontend container)
+#   - mempool-api      (orchestrator-managed; the backend api)
+#   - archy-mempool-db (orchestrator-managed; the mariadb)
+#   - archy-mempool-web (orchestrator-managed; the proxy/static layer)
+#
+# The mempool stack is split between the legacy install path (mempool itself)
+# and orchestrator-managed sub-containers — see uses_orchestrator_install_flow
+# in install.rs. Tests here treat them as one stack at the package.install/stop
+# level, addressed by id "mempool". UI URL coverage is in ui-coverage.bats.
+
+load '../lib/rpc.bash'
+
+# bats-assert is not loaded in this suite (only rpc.bash), so provide a minimal
+# `fail` so the `|| fail "..."` guards below report a real assertion failure
+# instead of an undefined-command status 127 that masks the actual reason.
+fail() { echo "$@" >&2; return 1; }
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+mempool_components=(
+  "mempool-api"
+  "archy-mempool-db"
+)
+
+mempool_optional_components=(
+  "mempool"
+  "archy-mempool-web"
+)
+
+mempool_skip_if_absent() {
+  for c in "${mempool_components[@]}"; do
+    podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1 && return 0
+  done
+  skip "mempool stack not installed"
+}
+
+@test "container-list includes the core mempool components" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local found=0
+  for c in "${mempool_components[@]}"; do
+    if echo "$output" | jq -e --arg n "$c" '.[] | select(.name == $n)' >/dev/null; then
+      found=$((found + 1))
+    fi
+  done
+  (( found > 0 )) || skip "mempool stack not installed"
+}
+
+@test "every present mempool component reports a valid state" {
+  run rpc_result container-list
+  [ "$status" -eq 0 ]
+  local present=0
+  for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
+    local state
+    state=$(echo "$output" | jq -r --arg n "$c" '.[] | select(.name == $n) | .state')
+    [[ -n "$state" ]] || continue
+    present=$((present + 1))
+    [[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] \
+      || fail "invalid state for $c: $state"
+  done
+  (( present > 0 )) || skip "mempool stack not installed"
+}
+
+@test "no orphan mempool-related containers beyond the known set" {
+  # Poll for steady state (don't single-shot): a stack restart in a prior tier
+  # briefly leaves a recreated member visible alongside its replacement, so a
+  # one-shot count can momentarily see total>known even though the reconciler
+  # converges within seconds. A genuine orphan never clears, so this still
+  # catches it — it just tolerates the transient recreate window.
+  local total known deadline=$(( $(date +%s) + 30 ))
+  while (( $(date +%s) < deadline )); do
+    total=$(podman ps -a --format '{{.Names}}' \
+      | grep -Ec '^(mempool|archy-mempool)' || true)
+    known=$(podman ps -a --format '{{.Names}}' \
+      | grep -Ec '^(mempool|mempool-api|archy-mempool-db|archy-mempool-web)$' || true)
+    [ "$total" -eq "$known" ] && return 0
+    sleep 3
+  done
+  echo "orphan mempool container persisted >30s (total=$total known=$known):" >&2
+  podman ps -a --format '{{.Names}}' | grep -E '^(mempool|archy-mempool)' \
+    | grep -vE '^(mempool|mempool-api|archy-mempool-db|archy-mempool-web)$' >&2 || true
+  return 1
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Destructive tier — operate on the package id "mempool" which the
+# legacy install path treats as the whole stack
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.stop transitions mempool stack to stopped" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  mempool_skip_if_absent
+
+  run rpc_result package.stop '{"id":"mempool"}'
+  [ "$status" -eq 0 ]
+
+  # The frontend container is the user-visible target; supporting
+  # services may stay running depending on orchestrator policy.
+  if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
+    run wait_for_container_status mempool stopped 60
+    [ "$status" -eq 0 ]
+  fi
+}
+
+@test "package.start brings mempool stack back to running" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  mempool_skip_if_absent
+
+  run rpc_result package.start '{"id":"mempool"}'
+  [ "$status" -eq 0 ]
+
+  if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
+    run wait_for_container_status mempool running 180
+    [ "$status" -eq 0 ]
+  fi
+}
+
+@test "package.restart leaves mempool stack in running state" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  mempool_skip_if_absent
+
+  run rpc_result package.restart '{"id":"mempool"}'
+  [ "$status" -eq 0 ]
+
+  if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
+    run wait_for_container_status mempool running 180
+    [ "$status" -eq 0 ]
+  fi
+}
+
+@test "mempool api backend remains queryable when stack is up" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+  mempool_skip_if_absent
+
+  # mempool-api on :8999 — same probe required-stack.bats uses for parity.
+  # This case runs immediately after package.restart, so mempool-api has just
+  # dropped + must re-establish its electrs/bitcoin connection (it reports
+  # "offline" in the frontend during this window). Give it the same recovery
+  # budget the passing parity probes use (required-stack-destructive: 240s,
+  # package-update-smoke: 300s) — 180s was too tight for the post-restart path.
+  local deadline=$(( $(date +%s) + 300 ))
+  while (( $(date +%s) < deadline )); do
+    if curl -fsS -m 5 "http://127.0.0.1:8999/api/v1/backend-info" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 3
+  done
+  # NB: bats-assert's `fail` is not loaded in this file (only ../lib/rpc.bash),
+  # so emit + return non-zero directly rather than calling an undefined helper.
+  echo "mempool-api never responded on :8999 within 300s" >&2
+  return 1
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Cascade-destructive tier
+# ────────────────────────────────────────────────────────────────────
+
+@test "package.uninstall removes the mempool stack" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+  mempool_skip_if_absent
+
+  run rpc_result package.uninstall '{"id":"mempool","preserve_data":true}'
+  [ "$status" -eq 0 ]
+
+  for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
+    if podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1; then
+      run wait_for_container_status "$c" absent 120
+      [ "$status" -eq 0 ] || fail "mempool component $c not removed by uninstall"
+    fi
+  done
+}
+
+@test "package.install restores the mempool stack" {
+  [[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
+
+  run rpc_result package.install '{"manifest_path":"mempool/manifest.yaml"}'
+  [ "$status" -eq 0 ]
+
+  # At minimum the core orchestrator-managed components must come back.
+  for c in "${mempool_components[@]}"; do
+    run wait_for_container_status "$c" running 240
+    [ "$status" -eq 0 ] || fail "mempool component $c never reached running after reinstall"
+  done
+}
@@ -0,0 +1,135 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/package-update-smoke.bats
+#
+# Destructive update smoke checks.
+# Requires RPC auth (ARCHY_PASSWORD) and ARCHY_ALLOW_DESTRUCTIVE=1.
+
+load '../lib/rpc.bash'
+
+require_destructive() {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+}
+
+require_auth() {
+  [[ -n "${ARCHY_PASSWORD:-}" ]] || skip "ARCHY_PASSWORD not set"
+}
+
+wait_http_ok() {
+  local url="$1"
+  local timeout="${2:-240}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    if curl -fsS "$url" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+wait_started_at_change() {
+  local name="$1"
+  local old_started_at="$2"
+  local timeout="${3:-300}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    local started_at running
+    started_at=$(podman inspect --format '{{.State.StartedAt}}' "$name" 2>/dev/null || true)
+    running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
+    if [[ -n "$started_at" && "$started_at" != "$old_started_at" && "$running" == "true" ]]; then
+      return 0
+    fi
+    sleep 3
+  done
+  return 1
+}
+
+wait_running() {
+  local name="$1"
+  local timeout="${2:-240}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    local running
+    running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
+    if [[ "$running" == "true" ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+setup_file() {
+  require_auth
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+@test "package.update bitcoin-ui restarts container and recovers endpoint" {
+  require_destructive
+
+  local before
+  before=$(podman inspect --format '{{.State.StartedAt}}' archy-bitcoin-ui 2>/dev/null || true)
+  [[ -n "$before" ]] || skip "archy-bitcoin-ui container not found"
+
+  run rpc_call package.update '{"id":"bitcoin-ui"}'
+  [ "$status" -eq 0 ]
+
+  local err
+  err=$(echo "$output" | jq -r '.error.message // empty')
+  if [[ -z "$err" ]]; then
+    echo "$output" | jq -e '.result.status == "updating"' >/dev/null
+    run wait_started_at_change archy-bitcoin-ui "$before" 360
+    if [[ "$status" -ne 0 ]]; then
+      run wait_running archy-bitcoin-ui 120
+      [ "$status" -eq 0 ]
+    fi
+  elif [[ "$err" == *"already updating"* ]]; then
+    :
+  else
+    echo "unexpected package.update error: $err" >&2
+    return 1
+  fi
+
+  run wait_http_ok "http://127.0.0.1:8334/" 180
+  [ "$status" -eq 0 ]
+}
+
+@test "package.update mempool stack smoke (optional)" {
+  require_destructive
+  [[ "${ARCHY_ALLOW_STACK_UPDATE:-0}" == "1" ]] || skip "ARCHY_ALLOW_STACK_UPDATE not set"
+
+  local before
+  before=$(podman inspect --format '{{.State.StartedAt}}' mempool 2>/dev/null || true)
+  [[ -n "$before" ]] || skip "mempool container not found"
+
+  run rpc_call package.update '{"id":"mempool"}'
+  [ "$status" -eq 0 ]
+
+  local err
+  err=$(echo "$output" | jq -r '.error.message // empty')
+  if [[ -z "$err" ]]; then
+    echo "$output" | jq -e '.result.status == "updating"' >/dev/null
+    run wait_started_at_change mempool "$before" 420
+    if [[ "$status" -ne 0 ]]; then
+      run wait_running mempool 120
+      [ "$status" -eq 0 ]
+    fi
+  elif [[ "$err" == *"already updating"* ]]; then
+    :
+  else
+    echo "unexpected package.update error: $err" >&2
+    return 1
+  fi
+
+  run wait_http_ok "http://127.0.0.1:4080/" 240
+  [ "$status" -eq 0 ]
+
+  run wait_http_ok "http://127.0.0.1:8999/api/v1/backend-info" 300
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,78 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/port-drift.bats
+#
+# Regression guard for the .116 failure class: a backend container that is
+# "Up" but publishes its ports to the WRONG host ports because the manifest
+# changed after the container was created (e.g. lnd REST stuck on host 8080
+# while the manifest — and every in-process client — expects 18080).
+#
+# This mirrors the orchestrator's `host_port_bindings_drifted` check, but from
+# the outside: it compares the live `podman inspect` PortBindings against the
+# manifest `ports:` for each installed backend. Runs on the archy host.
+#
+# Tiers: read-only.
+
+_apps_dir() {
+  local d
+  for d in "${ARCHIPELAGO_APPS_DIR:-}" /opt/archipelago/apps \
+           "$BATS_TEST_DIRNAME/../../../apps"; do
+    [[ -n "$d" && -d "$d" ]] && { echo "$d"; return 0; }
+  done
+  return 1
+}
+
+_manifest_for() {
+  local app="$1" dir
+  dir=$(_apps_dir) || return 1
+  local mf
+  for mf in "$dir/$app/manifest.yml" "$dir/$app/manifest.yaml"; do
+    [[ -r "$mf" ]] && { echo "$mf"; return 0; }
+  done
+  return 1
+}
+
+# Emit "host container" pairs from a manifest's ports: block.
+_manifest_ports() {
+  awk '
+    /^[[:space:]]*ports:/ { inports=1; next }
+    inports && /^[[:space:]]*[a-z_]+:[[:space:]]*$/ && !/protocol:|host:|container:/ { inports=0 }
+    inports && /- host:/    { host=$3 }
+    inports && /container:/  { print host, $2 }
+  ' "$1"
+}
+
+# For a given container + (host,container) port, emit a "DRIFT: …" line on
+# mismatch (and nothing otherwise). Stays silent for unpublished / host-net
+# ports — those are handled elsewhere and must never be treated as drift.
+_drift_line() {
+  local cname="$1" want_host="$2" cport="$3"
+  local bindings actual
+  bindings=$(podman inspect "$cname" --format '{{json .HostConfig.PortBindings}}' 2>/dev/null) || return 0
+  actual=$(echo "$bindings" | jq -r --arg k "${cport}/tcp" '.[$k][]?.HostPort // empty' 2>/dev/null)
+  [[ -n "$actual" ]] || return 0
+  echo "$actual" | grep -qx "$want_host" && return 0
+  echo "DRIFT: $cname container-port $cport published on host [$actual] but manifest wants $want_host"
+}
+
+@test "backend containers publish ports that match their manifest" {
+  command -v podman >/dev/null 2>&1 || skip "podman not available"
+  local checked=0 violations="" app cname mf line
+  # container-name : manifest-app-id
+  for pair in "lnd:lnd" "bitcoin-knots:bitcoin-knots" "electrumx:electrumx"; do
+    cname="${pair%%:*}"; app="${pair##*:}"
+    podman container exists "$cname" 2>/dev/null || continue
+    mf=$(_manifest_for "$app") || continue
+    while read -r host cport; do
+      [[ -n "$host" && -n "$cport" ]] || continue
+      checked=$((checked + 1))
+      line=$(_drift_line "$cname" "$host" "$cport")
+      [[ -n "$line" ]] && violations+="${line}"$'\n'
+    done < <(_manifest_ports "$mf")
+  done
+  [[ "$checked" -gt 0 ]] || skip "no installed backend containers with published ports to check"
+  if [[ -n "$violations" ]]; then
+    echo "published-port drift detected:" >&2
+    echo "$violations" >&2
+    return 1
+  fi
+}
@@ -0,0 +1,128 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/required-stack-destructive.bats
+#
+# Controlled destructive lifecycle checks for required stack containers.
+# Runs only when ARCHY_ALLOW_DESTRUCTIVE=1.
+
+required_containers=(
+  "archy-bitcoin-ui"
+  "archy-lnd-ui"
+  "archy-electrs-ui"
+  "mempool"
+  "mempool-api"
+)
+
+container_installed() {
+  podman ps -a --format '{{.Names}}' | grep -Fx "$1" >/dev/null
+}
+
+# Only the subset of required_containers actually installed on this node —
+# a node without the mempool stack (or another optional app) shouldn't
+# hard-fail restarting/probing something it was never meant to have.
+installed_required_containers() {
+  local c
+  for c in "${required_containers[@]}"; do
+    container_installed "$c" && echo "$c"
+  done
+  # Always succeed — under `set -e`, the function's own exit code is that of
+  # its last statement, so if the last array entry happens to be a container
+  # NOT installed on this node, the whole function (and any bare
+  # `x="$(installed_required_containers)"` caller) would spuriously fail even
+  # though earlier entries matched fine.
+  return 0
+}
+
+wait_running() {
+  local name="$1"
+  local timeout="${2:-120}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    local running
+    running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
+    if [[ "$running" == "true" ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+wait_http_ok() {
+  local url="$1"
+  local timeout="${2:-180}"
+  local deadline=$(( $(date +%s) + timeout ))
+  while (( $(date +%s) < deadline )); do
+    if curl -fsS "$url" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+restart_with_retry() {
+  local name="$1"
+  local attempts="${2:-3}"
+  local i
+  for ((i=1; i<=attempts; i++)); do
+    if podman restart "$name" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 3
+  done
+  return 1
+}
+
+@test "required-stack destructive gate enabled" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+}
+
+@test "restart each required service container and verify it recovers" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  local targets; targets="$(installed_required_containers)"
+  [[ -n "$targets" ]] || skip "none of required_containers installed on this node"
+  while IFS= read -r c; do
+    run restart_with_retry "$c" 4
+    [ "$status" -eq 0 ]
+    run wait_running "$c" 180
+    [ "$status" -eq 0 ]
+  done <<< "$targets"
+}
+
+@test "required endpoints still respond after restarts" {
+  [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
+
+  if container_installed archy-bitcoin-ui; then
+    run wait_http_ok "http://127.0.0.1:8334/" 180
+    [ "$status" -eq 0 ]
+  fi
+
+  # :8081 is nginx-proxy-manager — an OPTIONAL app (not in required_containers).
+  # Only assert it when NPM is actually installed on this node; otherwise the
+  # required-endpoints check false-fails on nodes that don't run NPM.
+  if podman ps --format '{{.Names}}' | grep -q '^nginx-proxy-manager$'; then
+    run wait_http_ok "http://127.0.0.1:8081/" 180
+    [ "$status" -eq 0 ]
+  fi
+
+  if container_installed mempool; then
+    run wait_http_ok "http://127.0.0.1:4080/" 180
+    [ "$status" -eq 0 ]
+  fi
+
+  if container_installed mempool-api; then
+    run wait_http_ok "http://127.0.0.1:8999/api/v1/backend-info" 240
+    [ "$status" -eq 0 ]
+  fi
+
+  if container_installed lnd; then
+    # lnd RPC readiness lags container 'running' (wallet unlock + graph sync) —
+    # retry rather than single-shot. See lnd.bats.
+    run sh -lc 'for i in $(seq 1 60); do
+      podman exec lnd lncli --tlscertpath /root/.lnd/tls.cert --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon --rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
+      sleep 3
+    done; exit 1'
+    [ "$status" -eq 0 ]
+  fi
+}
@@ -0,0 +1,201 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/required-stack.bats
+#
+# Read-only release-gate checks for the Bitcoin/electrum/lnd/mempool stack.
+# Originally written against .116's fixed app roster; the "present"/"running"
+# checks below now only require containers actually installed on THIS node
+# (podman_all_names — present in `podman ps -a` even if stopped), so a node
+# with a different app subset (e.g. no mempool stack) doesn't hard-fail on
+# apps it was never meant to have. Per-app checks further down (mempool,
+# filebrowser, ...) skip individually if that app isn't installed, matching
+# the mempool_skip_if_absent idiom in mempool.bats.
+#
+# This suite is intentionally non-destructive and does not use RPC auth;
+# it can run anytime as a health gate during long sync/reindex windows.
+
+required_containers=(
+  "bitcoin-knots"
+  "electrumx"
+  "lnd"
+  "archy-mempool-db"
+  "mempool-api"
+  "mempool"
+  "filebrowser"
+  "archy-bitcoin-ui"
+  "archy-lnd-ui"
+  "archy-electrs-ui"
+)
+
+fail() { echo "$@" >&2; return 1; }
+
+podman_names() {
+  podman ps --format '{{.Names}}'
+}
+
+podman_all_names() {
+  podman ps -a --format '{{.Names}}'
+}
+
+container_running() {
+  local name="$1"
+  podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null
+}
+
+container_installed() {
+  local name="$1"
+  podman_all_names | grep -Fx "$name" >/dev/null
+}
+
+skip_if_not_installed() {
+  container_installed "$1" || skip "$1 not installed on this node"
+}
+
+# The subset of required_containers actually installed on this node.
+installed_required_containers() {
+  local c
+  for c in "${required_containers[@]}"; do
+    container_installed "$c" && echo "$c"
+  done
+  # Always succeed — see the identical comment in required-stack-destructive.bats.
+  return 0
+}
+
+bitcoin_rpc() {
+  curl -fsS --max-time 60 \
+    --user "archipelago:$(cat /var/lib/archipelago/secrets/bitcoin-rpc-password)" \
+    --data-binary '{"jsonrpc":"1.0","id":"required-stack","method":"getblockchaininfo","params":[]}' \
+    -H 'content-type: text/plain;' \
+    http://127.0.0.1:8332/
+}
+
+bitcoin_json() {
+  python3 -c 'import json,sys; r=json.load(sys.stdin)["result"]; print(r[sys.argv[1]])' "$1"
+}
+
+@test "required containers are present" {
+  # Under sustained 5× churn an app may still be mid-restart when this runs;
+  # wait for the whole required set rather than single-shot. Only checks
+  # containers actually installed on this node (see installed_required_containers).
+  local targets; targets="$(installed_required_containers)"
+  [[ -n "$targets" ]] || skip "none of required_containers installed on this node"
+  local deadline=$(( $(date +%s) + 180 )) names missing
+  while (( $(date +%s) < deadline )); do
+    names="$(podman_names)"; missing=""
+    while IFS= read -r c; do
+      echo "$names" | grep -Fx "$c" >/dev/null || missing="$missing $c"
+    done <<< "$targets"
+    [[ -z "$missing" ]] && return 0
+    sleep 3
+  done
+  fail "required containers never all present; missing:$missing"
+}
+
+@test "required containers are running" {
+  local targets; targets="$(installed_required_containers)"
+  [[ -n "$targets" ]] || skip "none of required_containers installed on this node"
+  local deadline=$(( $(date +%s) + 180 )) notrunning
+  while (( $(date +%s) < deadline )); do
+    notrunning=""
+    while IFS= read -r c; do
+      [[ "$(container_running "$c" 2>/dev/null)" == "true" ]] || notrunning="$notrunning $c"
+    done <<< "$targets"
+    [[ -z "$notrunning" ]] && return 0
+    sleep 3
+  done
+  fail "required containers never all running; not-running:$notrunning"
+}
+
+@test "bitcoin-knots RPC responds" {
+  skip_if_not_installed bitcoin-knots
+  run bitcoin_rpc
+  [ "$status" -eq 0 ]
+  echo "$output" | python3 -c 'import json,sys; r=json.load(sys.stdin)["result"]; assert r["chain"] == "main" and r["blocks"] >= 0'
+}
+
+@test "bitcoin backend is synced archival for electrumx/lnd gate" {
+  skip_if_not_installed bitcoin-knots
+  run bitcoin_rpc
+  [ "$status" -eq 0 ]
+
+  local pruned ibd blocks headers
+  pruned="$(echo "$output" | bitcoin_json pruned)"
+  ibd="$(echo "$output" | bitcoin_json initialblockdownload)"
+  blocks="$(echo "$output" | bitcoin_json blocks)"
+  headers="$(echo "$output" | bitcoin_json headers)"
+
+  if [ "$pruned" = "True" ] || [ "$pruned" = "true" ]; then
+    echo "bitcoin is pruned (blocks=$blocks headers=$headers); electrumx cannot index pruned historical blocks"
+    return 1
+  fi
+  if [ "$ibd" = "True" ] || [ "$ibd" = "true" ]; then
+    echo "bitcoin is still in initial block download (blocks=$blocks headers=$headers)"
+    return 1
+  fi
+}
+
+@test "electrumx TCP port accepts connections" {
+  skip_if_not_installed electrumx
+  run python3 - <<'PY'
+import socket
+s = socket.create_connection(("127.0.0.1", 50001), 3)
+s.close()
+print("ok")
+PY
+  [ "$status" -eq 0 ]
+}
+
+@test "lnd CLI getinfo succeeds" {
+  skip_if_not_installed lnd
+  # lnd RPC readiness lags the container "running" state (wallet auto-unlock on
+  # start), so retry until ready rather than single-shot. See lnd.bats note.
+  run sh -lc 'for i in $(seq 1 30); do
+    timeout 20 podman exec lnd lncli --tlscertpath /root/.lnd/tls.cert --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon --rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
+    sleep 3
+  done; exit 1'
+  [ "$status" -eq 0 ]
+}
+
+@test "lnd REST port accepts connections" {
+  skip_if_not_installed lnd
+  run python3 - <<'PY'
+import socket
+s = socket.create_connection(("127.0.0.1", 18080), 3)
+s.close()
+print("ok")
+PY
+  [ "$status" -eq 0 ]
+}
+
+@test "mempool api endpoint responds" {
+  skip_if_not_installed mempool-api
+  # mempool-api reconnects to electrumx after a stack restart — retry ~180s.
+  run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:8999/api/v1/backend-info" && exit 0; sleep 3; done; exit 1'
+  [ "$status" -eq 0 ]
+}
+
+@test "mempool frontend responds" {
+  skip_if_not_installed mempool
+  run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:4080/" && exit 0; sleep 3; done; exit 1'
+  [ "$status" -eq 0 ]
+}
+
+@test "bitcoin ui responds" {
+  skip_if_not_installed archy-bitcoin-ui
+  # The companion (archy-bitcoin-ui) may have just been recreated by an earlier
+  # companion-survives test; its nginx takes a moment to serve. Retry ~120s
+  # rather than single-shot.
+  run sh -lc 'for i in $(seq 1 40); do curl -fsS -o /dev/null "http://127.0.0.1:8334/" && exit 0; sleep 3; done; exit 1'
+  [ "$status" -eq 0 ]
+}
+
+@test "lnd ui responds" {
+  skip_if_not_installed archy-lnd-ui
+  run curl -fsS "http://127.0.0.1:18083/"
+  [ "$status" -eq 0 ]
+}
+
+@test "filebrowser responds" {
+  skip_if_not_installed filebrowser
+  run curl -fsS "http://127.0.0.1:8083/"
+  [ "$status" -eq 0 ]
+}
@@ -0,0 +1,73 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/secret-completeness.bats
+#
+# Regression guard for the .198 failure class: a manifest references a
+# `secret_env.secret_file` that was never generated on the node, so secret
+# resolution hard-fails and the container won't start — cascading the whole
+# Bitcoin stack. (bitcoin-knots gained `bitcoin-rpc-txrelay-rpcauth`, which old
+# nodes lacked, so bitcoind never came up and reinstall "just stopped".)
+#
+# For every installed backend, assert every secret_file it references exists in
+# the secrets dir. Runs on the archy host.
+#
+# Tiers: read-only.
+
+SECRETS_DIR="${ARCHY_SECRETS_DIR:-/var/lib/archipelago/secrets}"
+
+_apps_dir() {
+  local d
+  for d in "${ARCHIPELAGO_APPS_DIR:-}" /opt/archipelago/apps \
+           "$BATS_TEST_DIRNAME/../../../apps"; do
+    [[ -n "$d" && -d "$d" ]] && { echo "$d"; return 0; }
+  done
+  return 1
+}
+
+_manifest_for() {
+  local app="$1" dir mf
+  dir=$(_apps_dir) || return 1
+  for mf in "$dir/$app/manifest.yml" "$dir/$app/manifest.yaml"; do
+    [[ -r "$mf" ]] && { echo "$mf"; return 0; }
+  done
+  return 1
+}
+
+_secret_files_in() {
+  # Emit each `secret_file:` value referenced by the manifest.
+  grep -E '^[[:space:]]*secret_file:' "$1" 2>/dev/null | awk '{print $2}'
+}
+
+_secret_exists() {
+  local f="$SECRETS_DIR/$1"
+  [[ -e "$f" ]] && return 0
+  sudo -n test -f "$f" 2>/dev/null
+}
+
+@test "every installed backend's referenced secrets exist on disk" {
+  command -v podman >/dev/null 2>&1 || skip "podman not available"
+  [[ -d "$SECRETS_DIR" ]] || sudo -n test -d "$SECRETS_DIR" 2>/dev/null || skip "secrets dir not present"
+
+  local checked=0 missing="" app cname mf sf
+  # container-name : manifest-app-id  (the bitcoin stack that cascades)
+  for pair in \
+    "bitcoin-knots:bitcoin-knots" "lnd:lnd" "electrumx:electrumx" \
+    "mempool-api:mempool-api" "btcpay-server:btcpay-server" \
+    "archy-nbxplorer:archy-nbxplorer" "fedimint:fedimint" \
+    "fedimint-gateway:fedimint-gateway"; do
+    cname="${pair%%:*}"; app="${pair##*:}"
+    podman container exists "$cname" 2>/dev/null || continue
+    mf=$(_manifest_for "$app") || continue
+    while read -r sf; do
+      [[ -n "$sf" ]] || continue
+      checked=$((checked + 1))
+      _secret_exists "$sf" || missing+="${app} -> ${sf}\n"
+    done < <(_secret_files_in "$mf")
+  done
+
+  [[ "$checked" -gt 0 ]] || skip "no installed backends with secret references to check"
+  if [[ -n "$missing" ]]; then
+    echo "installed apps reference missing secrets:" >&2
+    echo -e "$missing" >&2
+    return 1
+  fi
+}
@@ -0,0 +1,93 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/ui-coverage.bats
+#
+# UI surface tests — exercises the URLs a real user actually clicks
+# through, not just the JSON-RPC API. Fills the coverage gap where the
+# previous bats suites would report "container is up" while the iframe
+# behind /app/<id>/ was returning 502 because nginx had a stale upstream
+# or the proxy port was wrong.
+#
+# URL map sourced from neode-ui/src/views/appSession/appSessionConfig.ts
+# (the frontend's own resolveAppUrl). Tests here MUST stay in sync with
+# that file — divergence is the whole bug class we're guarding against.
+#
+# Each app probe is gated on its container being running:
+#   - container down  → skip (clean dependency report, no false-fail)
+#   - container up    → URL MUST return 200 with non-empty body
+#
+# Looped 5× via tests/lifecycle/run-gate.sh.
+
+load '../lib/rpc.bash'
+load '../lib/ui-probes.bash'
+
+setup_file() {
+  : "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
+  export ARCHY_FORCE_LOGIN=1
+  rpc_login
+  unset ARCHY_FORCE_LOGIN
+  HOST="${ARCHY_HOST:-127.0.0.1}"
+  export HOST
+}
+
+teardown_file() {
+  rpc_logout_local
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Dashboard shell + catalog (always required)
+# ────────────────────────────────────────────────────────────────────
+
+@test "dashboard https://host/ returns the Vue SPA shell" {
+  run probe_dashboard_shell
+  [ "$status" -eq 0 ]
+}
+
+@test "dashboard catalog endpoint responds with apps" {
+  run probe_dashboard_catalog
+  [ "$status" -eq 0 ]
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Bitcoin UI — direct host port (8334), companion container
+# ────────────────────────────────────────────────────────────────────
+
+@test "bitcoin-ui is reachable on :8334 when archy-bitcoin-ui is running" {
+  probe_app_url archy-bitcoin-ui "http://$HOST:8334/" "bitcoin-ui (direct port 8334)"
+}
+
+# ────────────────────────────────────────────────────────────────────
+# HTTPS proxy paths — match HTTPS_PROXY_PATHS in appSessionConfig.ts
+# ────────────────────────────────────────────────────────────────────
+
+@test "lnd proxy https://host/app/lnd/ responds when lnd is running" {
+  probe_app_url lnd "https://$HOST/app/lnd/" "lnd (proxy /app/lnd/)"
+}
+
+@test "electrumx proxy https://host/app/electrumx/ responds when electrumx is running" {
+  # electrumx companion (archy-electrs-ui) is what serves the iframe HTML;
+  # the electrumx daemon is just the TCP backend.
+  probe_app_url archy-electrs-ui "https://$HOST/app/electrumx/" "electrumx (proxy /app/electrumx/)"
+}
+
+@test "mempool proxy https://host/app/mempool/ responds when mempool is running" {
+  probe_app_url mempool "https://$HOST/app/mempool/" "mempool (proxy /app/mempool/)"
+}
+
+@test "fedimint proxy https://host/app/fedimint/ responds when fedimint is running" {
+  probe_app_url fedimint "https://$HOST/app/fedimint/" "fedimint (proxy /app/fedimint/)"
+}
+
+@test "btcpay proxy https://host/app/btcpay/ responds when btcpay-server is running" {
+  probe_app_url btcpay-server "https://$HOST/app/btcpay/" "btcpay (proxy /app/btcpay/)"
+}
+
+@test "filebrowser proxy https://host/app/filebrowser/ responds when filebrowser is running" {
+  probe_app_url filebrowser "https://$HOST/app/filebrowser/" "filebrowser (proxy /app/filebrowser/)"
+}
+
+# ────────────────────────────────────────────────────────────────────
+# Companion-served URLs that aren't in HTTPS_PROXY_PATHS but show up
+# in the dashboard. archy-lnd-ui shares lnd's iframe path; archy-electrs-ui
+# shares electrumx's. The earlier test already covers those — leaving
+# this section for future companion-direct probes (none today).
+# ────────────────────────────────────────────────────────────────────
@@ -0,0 +1,198 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/use-quadlet-backends-install.bats
+#
+# Validates the post-condition of Phase 3.2's `use_quadlet_backends`
+# install path. When the orchestrator routed at least one backend
+# install through `install_via_quadlet`, this suite asserts that the
+# resulting state has the four properties the Phase 3 design promises:
+#
+#   1. A `.container` unit file exists in ~/.config/containers/systemd/
+#      and is well-formed (required sections + directives).
+#   2. The corresponding `.service` is active under `systemctl --user`.
+#   3. The container is in `podman ps` (running).
+#   4. The container's cgroup is under `user.slice/...`, NOT under
+#      `archipelago.service` — proving FM3 (cgroup cascade SIGKILL on
+#      archipelago restart) is structurally fixed for that container.
+#
+# Auto-skips if no Quadlet-managed backend exists yet — so it runs as a
+# no-op on nodes where `use_quadlet_backends` is still false (today's
+# default), and turns into a hard regression gate as soon as anyone
+# flips the flag and reinstalls.
+#
+# Run on a node with rootless podman + systemd-user (every alpha-fleet
+# box). No env vars required for the read-only checks. The cleanup
+# section at the bottom is gated by ARCHY_ALLOW_DESTRUCTIVE=1.
+
+# bats-core ships no `fail`; bats-assert isn't installed on the alpha fleet.
+# Define the same minimal helper the other suites use (see mempool.bats) so a
+# tripped assertion reports as a real test failure, not a status-127 crash.
+fail() { echo "$@" >&2; return 1; }
+
+quadlet_dir() {
+  echo "${XDG_CONFIG_HOME:-$HOME/.config}/containers/systemd"
+}
+
+# List Quadlet `.container` units that correspond to backend containers
+# (i.e., NOT companions like archy-*-ui, which already shipped via Quadlet
+# in v1.7.41 and have their own coverage in companion-survives-archipelago-
+# restart.bats). Echoes one container name per line; empty if none found.
+backend_quadlet_units() {
+  local d
+  d="$(quadlet_dir)"
+  [[ -d "$d" ]] || return 0
+  # Strip the .container extension; filter out archy-*-ui companions.
+  # wyoming-* (piper/whisper voice services) are mid-integration and not yet
+  # part of the platform contract — exclude until their packaging lands.
+  for f in "$d"/*.container; do
+    [[ -e "$f" ]] || continue
+    local name
+    name="$(basename "$f" .container)"
+    [[ "$name" =~ ^archy-.*-ui$ ]] && continue
+    [[ "$name" =~ ^wyoming- ]] && continue
+    echo "$name"
+  done
+}
+
+# A unit file on disk does NOT imply the app should be running: an
+# explicitly user-stopped app keeps its .container file (e.g. the inactive
+# half of the bitcoin-core/bitcoin-knots multi-version pair), and its
+# .service being inactive / container absent is the CORRECT state. The
+# orchestrator persists that intent in user-stopped.json; honour it here so
+# the active-state assertions below don't false-fail on stopped-on-purpose
+# apps (gate tests 123/124, .228 2026-07-09).
+USER_STOPPED_FILE="${ARCHY_DATA_DIR:-/var/lib/archipelago}/user-stopped.json"
+
+is_user_stopped() {
+  local name="$1"
+  [[ -r "$USER_STOPPED_FILE" ]] || return 1
+  jq -e --arg n "$name" --arg s "${name#archy-}" \
+    'index($n) != null or index($s) != null' "$USER_STOPPED_FILE" >/dev/null 2>&1
+}
+
+# Read the cgroup path of a running container's main process. For
+# rootless podman the conmon-run target lands the container's pid1 in
+# the cgroup that owns its supervising .service.
+container_cgroup_path() {
+  local name="$1"
+  local pid
+  pid="$(podman inspect --format '{{.State.Pid}}' "$name" 2>/dev/null)"
+  [[ -n "$pid" && "$pid" != "0" ]] || return 1
+  # cgroup v2 line: "0::/path/to/cgroup"
+  awk -F: '$1=="0"{print $3}' "/proc/$pid/cgroup" 2>/dev/null
+}
+
+# Per-test gate. Each @test calls this so the suite is a clean no-op on
+# nodes where use_quadlet_backends is still false (today's default) —
+# bats doesn't propagate setup-level skip semantics across @test blocks.
+require_quadlet_backends() {
+  local count
+  count="$(backend_quadlet_units | wc -l)"
+  (( count > 0 )) || skip "no backend .container units in $(quadlet_dir) — use_quadlet_backends not enabled or no backends installed"
+}
+
+@test "Quadlet unit dir exists or is plausibly creatable" {
+  local d
+  d="$(quadlet_dir)"
+  # Either it already exists, or its parent does (so quadlet can mkdir it).
+  [[ -d "$d" ]] || [[ -d "$(dirname "$d")" ]] \
+    || skip "no XDG_CONFIG_HOME and no \$HOME/.config — not a desktop-style host"
+}
+
+@test "each backend Quadlet unit has the required sections + directives" {
+  require_quadlet_backends
+  local d
+  d="$(quadlet_dir)"
+  while read -r name; do
+    [[ -z "$name" ]] && continue
+    local body
+    body="$(<"$d/$name.container")"
+    # [Container] section + Image=
+    [[ "$body" == *"[Container]"* ]]   || fail "$name: missing [Container] section"
+    [[ "$body" == *"Image="*       ]]  || fail "$name: missing Image= directive"
+    # [Service] section with the Phase 3.2 backend invariant: Restart=on-failure.
+    # Companions use Restart=always; backends use on-failure so an operator-issued
+    # `systemctl stop` actually stays stopped.
+    [[ "$body" == *"[Service]"*       ]] || fail "$name: missing [Service] section"
+    [[ "$body" == *"Restart=on-failure"* ]] \
+      || fail "$name: backend unit must use Restart=on-failure (got companion-style Restart=always)"
+    # [Install] section so `systemctl --user enable` is well-defined.
+    [[ "$body" == *"[Install]"* ]]  || fail "$name: missing [Install] section"
+    [[ "$body" == *"WantedBy="*  ]] || fail "$name: missing WantedBy= in [Install]"
+  done < <(backend_quadlet_units)
+}
+
+@test "health is app-level state, NOT a systemd start gate (no Notify=healthy)" {
+  require_quadlet_backends
+  # Phase 3.4 originally emitted Notify=healthy so `systemctl start` blocked
+  # until the healthcheck passed. That was deliberately reverted: gating start
+  # on health hung boot reconciliation for dependency-waiting apps (fedimint
+  # idles its entrypoint until Bitcoin IBD finishes; lnd until the macaroon
+  # unlocks), leaving units stuck in "deactivating". The renderer now emits
+  # HealthCmd= for Podman's health state but TimeoutStartSec=0 and NO
+  # Notify=healthy (see quadlet.rs render() + contains_stale_health_gate()).
+  # This asserts the current invariant: no backend unit gates start on health.
+  local d
+  d="$(quadlet_dir)"
+  while read -r name; do
+    [[ -z "$name" ]] && continue
+    local body
+    body="$(<"$d/$name.container")"
+    [[ "$body" != *"Notify=healthy"* ]] \
+      || fail "$name: emits Notify=healthy — stale health gate; start would block on health and can hang boot reconcile"
+  done < <(backend_quadlet_units)
+}
+
+@test "every backend Quadlet unit's .service is active in systemctl --user" {
+  require_quadlet_backends
+  while read -r name; do
+    [[ -z "$name" ]] && continue
+    is_user_stopped "$name" && continue
+    # Converges-to-active, not instantly-active: a dependency-degraded app
+    # (mempool-api while electrumx catches up to the daemon) exits at startup
+    # and flaps through 'activating' for a couple of minutes after a lifecycle
+    # cycle; systemd's Restart=on-failure heals it. A genuine crash-loop still
+    # fails after the settle window (gate 2026-07-09, .228 iterations 1+2).
+    local state="" deadline=$((SECONDS + 180))
+    while (( SECONDS < deadline )); do
+      state="$(systemctl --user is-active "$name.service" 2>&1)" && break
+      sleep 5
+    done
+    [[ "$state" == "active" ]] \
+      || fail "$name.service is '$state' — did not reach 'active' within 180s"
+  done < <(backend_quadlet_units)
+}
+
+@test "every backend Quadlet unit has a running podman container" {
+  require_quadlet_backends
+  while read -r name; do
+    [[ -z "$name" ]] && continue
+    is_user_stopped "$name" && continue
+    # Same settle window as the active-state assert above: a quadlet --rm
+    # container is absent for a few seconds around each systemd retry.
+    local state="" deadline=$((SECONDS + 180))
+    while (( SECONDS < deadline )); do
+      state="$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" \
+        && [[ "$state" == "true" ]] && break
+      sleep 5
+    done
+    [[ "$state" == "true" ]] \
+      || fail "$name has no running container within 180s (state=${state:-absent})"
+  done < <(backend_quadlet_units)
+}
+
+@test "FM3 fix: backend cgroup is under user.slice, not archipelago.service" {
+  require_quadlet_backends
+  # The whole point of Phase 3 — verify the kernel-level invariant.
+  while read -r name; do
+    [[ -z "$name" ]] && continue
+    local cg
+    cg="$(container_cgroup_path "$name")" || skip "$name has no readable PID; container may have crashed mid-test"
+    [[ -n "$cg" ]] || fail "$name: empty cgroup path"
+    # Acceptable: anything under user.slice (rootless podman lands here when
+    # quadlet-managed). Forbidden: anything under archipelago.service's tree.
+    [[ "$cg" == *"user.slice"* ]] \
+      || fail "$name: cgroup '$cg' is not under user.slice — FM3 cascade still possible"
+    [[ "$cg" != *"archipelago.service"* ]] \
+      || fail "$name: cgroup '$cg' is under archipelago.service — Phase 3 promise broken"
+  done < <(backend_quadlet_units)
+}