test(lifecycle): match the approved Restart policy, and stop the suite racing itself

Two fixes from the first full 125-test run (121 ok / 4 not ok).

1. The Quadlet unit test still asserted Restart=on-failure for backends,
   which 8908fb4f deliberately changed. All 32 units on the node are
   Restart=always; only archy-btcpay-db showed up because `fail` aborts at
   the first offender. The old rationale — "backends need on-failure so an
   operator `systemctl stop` stays stopped" — was wrong twice over:
   systemd never applies Restart= to a unit stopped via `systemctl stop`,
   and because quadlet renders --rm, a cleanly-exited container is deleted
   and on-failure never brings it back (bitcoind exits 0 on SIGTERM, so
   backends vanished). Verified on-device before that change: `podman stop
   bitcoin-knots` came back in 12s, a dashboard stop stayed stopped 90s.
   Now asserts Restart=always, with the corrected reasoning in-place.

2. all-apps-lifecycle cycled ten apps back-to-back and manufactured the
   contention that then failed later tests: load went 10.8 at preflight to
   17.6 mid-loop, and btcpay's recovery stretched from 52s on a quiet box
   to 216s and then 512s, defeating waits of 180s and 300s. Widening waits
   cannot win when the run sets the load they must survive, so the loop now
   drains between apps — waits for load1 under 2x nproc, capped at 90s.
   ARCHY_APP_SETTLE_SECS=0 disables; ARCHY_APP_SETTLE_LOAD overrides.

Verified: run.sh use-quadlet-backends-install → 6/6 (was 1 failure).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/lifecycle/bats/all-apps-lifecycle.bats

@@ -100,6 +100,32 @@ wait_state() {
   return 1
 }
 
+# Let the node drain before cycling the NEXT app.
+#
+# Without this the suite competes with itself. Cycling ten apps back-to-back
+# drove this 4-core node from load 10.8 at preflight to 17.6 mid-loop, and the
+# contention then landed on whatever ran next: btcpay's recovery stretched from
+# 52s measured on a quiet box to 216s and then 512s, blowing through waits of
+# 180s and 300s. Widening those waits is a losing game, because the run itself
+# sets the load they have to survive — so pace the source instead.
+#
+# Waits for load1 to fall below the ceiling, capped so a genuinely busy node
+# can't stall the suite forever. ARCHY_APP_SETTLE_SECS=0 disables it.
+settle_between_apps() {
+  local cap="${ARCHY_APP_SETTLE_SECS:-90}"
+  (( cap > 0 )) || return 0
+  local cores ceiling
+  cores=$(nproc 2>/dev/null || echo 4)
+  ceiling="${ARCHY_APP_SETTLE_LOAD:-$(( cores * 2 ))}"
+  local deadline=$(( $(date +%s) + cap )) load1
+  while (( $(date +%s) < deadline )); do
+    load1=$(awk '{print $1}' /proc/loadavg)
+    awk -v l="$load1" -v m="$ceiling" 'BEGIN { exit !(l < m) }' && return 0
+    sleep 5
+  done
+  echo "# settle: load1 $load1 still >= $ceiling after ${cap}s — continuing anyway" >&3
+}
+
 # Build a package.install payload for $1 from the catalog, or fail (no spec).
 catalog_install_payload() {
   local id="$1" img cfg
@@ -126,9 +152,12 @@ catalog_install_payload() {
 
 @test "lifecycle: stop → start → restart every non-protected app" {
   [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
-  local fails="" id
+  local fails="" id first=1
   for id in $(target_apps); do
     [[ "$(app_state "$id")" == "running" ]] || continue   # only cycle running apps
+    # Drain before each app after the first. Placed at the TOP of the body so it
+    # still runs when a previous iteration bailed out via `continue`.
+    (( first )) && first=0 || settle_between_apps
     # Each rpc_result must be allowed to fail without aborting the loop.
     # rpc_result returns non-zero whenever the response carries .error, and
     # under bats' errexit a bare call ends the test right there — so a single

tests/lifecycle/bats/use-quadlet-backends-install.bats

@@ -109,12 +109,23 @@ require_quadlet_backends() {
     # [Container] section + Image=
     [[ "$body" == *"[Container]"* ]]   || fail "$name: missing [Container] section"
     [[ "$body" == *"Image="*       ]]  || fail "$name: missing Image= directive"
-    # [Service] section with the Phase 3.2 backend invariant: Restart=on-failure.
-    # Companions use Restart=always; backends use on-failure so an operator-issued
-    # `systemctl stop` actually stays stopped.
+    # [Service] section with Restart=always (backends AND companions alike).
+    #
+    # This asserted Restart=on-failure until 8908fb4f, on the rationale that
+    # backends needed it "so an operator-issued `systemctl stop` actually stays
+    # stopped". That rationale was wrong on two counts. systemd never applies
+    # Restart= to a unit stopped via `systemctl stop`, which is how archipelago
+    # stops apps — so on-failure bought nothing there. And because quadlet
+    # renders --rm, a container that exits CLEANLY is deleted and on-failure
+    # will not bring it back: bitcoind exits 0 on SIGTERM, so backends were
+    # vanishing after a clean exit.
+    #
+    # Verified on-device before the change landed: `podman stop bitcoin-knots`
+    # -> back in 12s, while a dashboard-issued stop stayed stopped for 90s.
+    # Operator approved it explicitly, gated on exactly that verification.
     [[ "$body" == *"[Service]"*       ]] || fail "$name: missing [Service] section"
-    [[ "$body" == *"Restart=on-failure"* ]] \
-      || fail "$name: backend unit must use Restart=on-failure (got companion-style Restart=always)"
+    [[ "$body" == *"Restart=always"* ]] \
+      || fail "$name: backend unit must use Restart=always (--rm deletes a cleanly-exited container, and on-failure never restarts it)"
     # [Install] section so `systemctl --user enable` is well-defined.
     [[ "$body" == *"[Install]"* ]]  || fail "$name: missing [Install] section"
     [[ "$body" == *"WantedBy="*  ]] || fail "$name: missing WantedBy= in [Install]"