diff --git a/tests/lifecycle/bats/all-apps-lifecycle.bats b/tests/lifecycle/bats/all-apps-lifecycle.bats index d35c4c6b..3fc99872 100644 --- a/tests/lifecycle/bats/all-apps-lifecycle.bats +++ b/tests/lifecycle/bats/all-apps-lifecycle.bats @@ -100,6 +100,32 @@ wait_state() { return 1 } +# Let the node drain before cycling the NEXT app. +# +# Without this the suite competes with itself. Cycling ten apps back-to-back +# drove this 4-core node from load 10.8 at preflight to 17.6 mid-loop, and the +# contention then landed on whatever ran next: btcpay's recovery stretched from +# 52s measured on a quiet box to 216s and then 512s, blowing through waits of +# 180s and 300s. Widening those waits is a losing game, because the run itself +# sets the load they have to survive — so pace the source instead. +# +# Waits for load1 to fall below the ceiling, capped so a genuinely busy node +# can't stall the suite forever. ARCHY_APP_SETTLE_SECS=0 disables it. +settle_between_apps() { + local cap="${ARCHY_APP_SETTLE_SECS:-90}" + (( cap > 0 )) || return 0 + local cores ceiling + cores=$(nproc 2>/dev/null || echo 4) + ceiling="${ARCHY_APP_SETTLE_LOAD:-$(( cores * 2 ))}" + local deadline=$(( $(date +%s) + cap )) load1 + while (( $(date +%s) < deadline )); do + load1=$(awk '{print $1}' /proc/loadavg) + awk -v l="$load1" -v m="$ceiling" 'BEGIN { exit !(l < m) }' && return 0 + sleep 5 + done + echo "# settle: load1 $load1 still >= $ceiling after ${cap}s — continuing anyway" >&3 +} + # Build a package.install payload for $1 from the catalog, or fail (no spec). catalog_install_payload() { local id="$1" img cfg @@ -126,9 +152,12 @@ catalog_install_payload() { @test "lifecycle: stop → start → restart every non-protected app" { [[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set" - local fails="" id + local fails="" id first=1 for id in $(target_apps); do [[ "$(app_state "$id")" == "running" ]] || continue # only cycle running apps + # Drain before each app after the first. Placed at the TOP of the body so it + # still runs when a previous iteration bailed out via `continue`. + (( first )) && first=0 || settle_between_apps # Each rpc_result must be allowed to fail without aborting the loop. # rpc_result returns non-zero whenever the response carries .error, and # under bats' errexit a bare call ends the test right there — so a single diff --git a/tests/lifecycle/bats/use-quadlet-backends-install.bats b/tests/lifecycle/bats/use-quadlet-backends-install.bats index ddd660e1..acde81c9 100644 --- a/tests/lifecycle/bats/use-quadlet-backends-install.bats +++ b/tests/lifecycle/bats/use-quadlet-backends-install.bats @@ -109,12 +109,23 @@ require_quadlet_backends() { # [Container] section + Image= [[ "$body" == *"[Container]"* ]] || fail "$name: missing [Container] section" [[ "$body" == *"Image="* ]] || fail "$name: missing Image= directive" - # [Service] section with the Phase 3.2 backend invariant: Restart=on-failure. - # Companions use Restart=always; backends use on-failure so an operator-issued - # `systemctl stop` actually stays stopped. + # [Service] section with Restart=always (backends AND companions alike). + # + # This asserted Restart=on-failure until 8908fb4f, on the rationale that + # backends needed it "so an operator-issued `systemctl stop` actually stays + # stopped". That rationale was wrong on two counts. systemd never applies + # Restart= to a unit stopped via `systemctl stop`, which is how archipelago + # stops apps — so on-failure bought nothing there. And because quadlet + # renders --rm, a container that exits CLEANLY is deleted and on-failure + # will not bring it back: bitcoind exits 0 on SIGTERM, so backends were + # vanishing after a clean exit. + # + # Verified on-device before the change landed: `podman stop bitcoin-knots` + # -> back in 12s, while a dashboard-issued stop stayed stopped for 90s. + # Operator approved it explicitly, gated on exactly that verification. [[ "$body" == *"[Service]"* ]] || fail "$name: missing [Service] section" - [[ "$body" == *"Restart=on-failure"* ]] \ - || fail "$name: backend unit must use Restart=on-failure (got companion-style Restart=always)" + [[ "$body" == *"Restart=always"* ]] \ + || fail "$name: backend unit must use Restart=always (--rm deletes a cleanly-exited container, and on-failure never restarts it)" # [Install] section so `systemctl --user enable` is well-defined. [[ "$body" == *"[Install]"* ]] || fail "$name: missing [Install] section" [[ "$body" == *"WantedBy="* ]] || fail "$name: missing WantedBy= in [Install]"