test(lifecycle): don't let one RPC hiccup hide which app failed

The stop/start/restart loop carefully accumulates per-app failures into
$fails and prints them, but the three rpc_result calls were bare. Under
bats' errexit a bare call ends the test immediately, so the summary that
names the app never ran.

On 2026-08-08 that turned a single transient error into an unattributable
failure: the test died at package.stop with no indication which of the
ten targets was involved. It was mempool, and the identical call returned
{"status":"stopping"} by hand a few minutes later.

Each call now records <id>:<phase>-rpc and moves to the next app, so the
run reports what actually broke.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/lifecycle/bats/all-apps-lifecycle.bats

@@ -129,11 +129,21 @@ catalog_install_payload() {
   local fails="" id
   for id in $(target_apps); do
     [[ "$(app_state "$id")" == "running" ]] || continue   # only cycle running apps
-    rpc_result package.stop "{\"id\":\"$id\"}" >/dev/null 2>&1
+    # Each rpc_result must be allowed to fail without aborting the loop.
+    # rpc_result returns non-zero whenever the response carries .error, and
+    # under bats' errexit a bare call ends the test right there — so a single
+    # transient RPC hiccup killed the run BEFORE the $fails summary below could
+    # name the app. That is exactly what happened on 2026-08-08: the whole test
+    # died at package.stop with no indication of which of the ten targets it was
+    # (it was mempool, and the same call succeeded by hand moments later).
+    rpc_result package.stop "{\"id\":\"$id\"}" >/dev/null 2>&1 \
+      || { fails+="$id:stop-rpc "; continue; }
     wait_state "$id" stopped 120 || { fails+="$id:stop "; }
-    rpc_result package.start "{\"id\":\"$id\"}" >/dev/null 2>&1
+    rpc_result package.start "{\"id\":\"$id\"}" >/dev/null 2>&1 \
+      || { fails+="$id:start-rpc "; continue; }
     wait_state "$id" running 240 || { fails+="$id:start "; continue; }
-    rpc_result package.restart "{\"id\":\"$id\"}" >/dev/null 2>&1
+    rpc_result package.restart "{\"id\":\"$id\"}" >/dev/null 2>&1 \
+      || { fails+="$id:restart-rpc "; continue; }
     wait_state "$id" running 240 || { fails+="$id:restart "; }
   done
   [[ -z "$fails" ]] || { echo "# lifecycle failures: $fails" >&3; false; }

tests/lifecycle/bats/ui-coverage.bats

@@ -27,6 +27,15 @@ setup_file() {
   unset ARCHY_FORCE_LOGIN
   HOST="${ARCHY_HOST:-127.0.0.1}"
   export HOST
+  # Honour ARCHY_SCHEME like lib/rpc.bash does, defaulting to https so nodes
+  # that already pass keep testing the TLS path. These probes used to hardcode
+  # https, which made the whole suite unrunnable on a node that serves the
+  # dashboard over http: on archi-dev-box :443 is bound to the Tailscale /
+  # WireGuard / LAN interface addresses but NOT to loopback, while :80 is bound
+  # on 0.0.0.0 — so every proxy probe failed "curl failed (network/timeout)"
+  # against http-reachable endpoints that were in fact serving 200.
+  UI_BASE="${ARCHY_SCHEME:-https}://$HOST"
+  export UI_BASE
 }
 
 teardown_file() {
@@ -37,7 +46,7 @@ teardown_file() {
 # Dashboard shell + catalog (always required)
 # ────────────────────────────────────────────────────────────────────
 
-@test "dashboard https://host/ returns the Vue SPA shell" {
+@test "dashboard / returns the Vue SPA shell" {
   run probe_dashboard_shell
   [ "$status" -eq 0 ]
 }
@@ -59,30 +68,30 @@ teardown_file() {
 # HTTPS proxy paths — match HTTPS_PROXY_PATHS in appSessionConfig.ts
 # ────────────────────────────────────────────────────────────────────
 
-@test "lnd proxy https://host/app/lnd/ responds when lnd is running" {
-  probe_app_url lnd "https://$HOST/app/lnd/" "lnd (proxy /app/lnd/)"
+@test "lnd proxy /app/lnd/ responds when lnd is running" {
+  probe_app_url lnd "$UI_BASE/app/lnd/" "lnd (proxy /app/lnd/)"
 }
 
-@test "electrumx proxy https://host/app/electrumx/ responds when electrumx is running" {
+@test "electrumx proxy /app/electrumx/ responds when electrumx is running" {
   # electrumx companion (archy-electrs-ui) is what serves the iframe HTML;
   # the electrumx daemon is just the TCP backend.
-  probe_app_url archy-electrs-ui "https://$HOST/app/electrumx/" "electrumx (proxy /app/electrumx/)"
+  probe_app_url archy-electrs-ui "$UI_BASE/app/electrumx/" "electrumx (proxy /app/electrumx/)"
 }
 
-@test "mempool proxy https://host/app/mempool/ responds when mempool is running" {
-  probe_app_url mempool "https://$HOST/app/mempool/" "mempool (proxy /app/mempool/)"
+@test "mempool proxy /app/mempool/ responds when mempool is running" {
+  probe_app_url mempool "$UI_BASE/app/mempool/" "mempool (proxy /app/mempool/)"
 }
 
-@test "fedimint proxy https://host/app/fedimint/ responds when fedimint is running" {
-  probe_app_url fedimint "https://$HOST/app/fedimint/" "fedimint (proxy /app/fedimint/)"
+@test "fedimint proxy /app/fedimint/ responds when fedimint is running" {
+  probe_app_url fedimint "$UI_BASE/app/fedimint/" "fedimint (proxy /app/fedimint/)"
 }
 
-@test "btcpay proxy https://host/app/btcpay/ responds when btcpay-server is running" {
-  probe_app_url btcpay-server "https://$HOST/app/btcpay/" "btcpay (proxy /app/btcpay/)"
+@test "btcpay proxy /app/btcpay/ responds when btcpay-server is running" {
+  probe_app_url btcpay-server "$UI_BASE/app/btcpay/" "btcpay (proxy /app/btcpay/)"
 }
 
-@test "filebrowser proxy https://host/app/filebrowser/ responds when filebrowser is running" {
-  probe_app_url filebrowser "https://$HOST/app/filebrowser/" "filebrowser (proxy /app/filebrowser/)"
+@test "filebrowser proxy /app/filebrowser/ responds when filebrowser is running" {
+  probe_app_url filebrowser "$UI_BASE/app/filebrowser/" "filebrowser (proxy /app/filebrowser/)"
 }
 
 # ────────────────────────────────────────────────────────────────────

tests/lifecycle/lib/ui-probes.bash

@@ -85,7 +85,7 @@ probe_app_url() {
 # layout" — see feedback_release_tarball_layout.md.
 probe_dashboard_shell() {
   local host="${ARCHY_HOST:-127.0.0.1}"
-  local url="https://$host/"
+  local url="${ARCHY_SCHEME:-https}://$host/"
   local body
   body=$(curl "${PROBE_CURL_OPTS[@]}" "$url" 2>/dev/null) || {
     echo "probe_dashboard_shell: $url — curl failed" >&2
@@ -106,7 +106,7 @@ probe_dashboard_shell() {
 probe_dashboard_catalog() {
   local host="${ARCHY_HOST:-127.0.0.1}"
   local body
-  body=$(curl "${PROBE_CURL_OPTS[@]}" "https://$host/catalog.json" 2>/dev/null) || {
+  body=$(curl "${PROBE_CURL_OPTS[@]}" "${ARCHY_SCHEME:-https}://$host/catalog.json" 2>/dev/null) || {
     echo "probe_dashboard_catalog: /catalog.json fetch failed" >&2
     return 1
   }

tests/lifecycle/run-gate.sh

@@ -11,6 +11,10 @@
 #   ARCHY_GATE_CASCADE=1                after the 5× loop, run ONE cascade pass
 #                                       (uninstall→no-ghost→reinstall a throwaway
 #                                       app); requires ARCHY_ALLOW_DESTRUCTIVE=1
+#   ARCHY_PREFLIGHT=0                   skip the host-readiness preflight
+#   ARCHY_MAX_LOAD                      load1 ceiling (default: nproc + 1)
+#   ARCHY_PREFLIGHT_SECS                how long to wait for load to fall
+#                                       (default: 900)
 #   plus everything run.sh / lib/rpc.bash respects
 #     (ARCHY_PASSWORD, ARCHY_HOST, ARCHY_SCHEME, ARCHY_ALLOW_DESTRUCTIVE,
 #      ARCHY_ALLOW_CASCADE_DESTRUCTIVE, ARCHY_ALLOW_NOAUTH)
@@ -77,6 +81,87 @@ settle_stack() {
   echo "  (stack settle deadline reached — proceeding anyway)"
 }
 
+# Host readiness, checked ONCE before iteration 1.
+#
+# Why this exists: on 2026-08-08 a gate run on a box at load ~14 failed five
+# times over, every failure reading "could not create a container" (searxng:start,
+# package.start btcpay-server, 3× electrumx) and never a lifecycle fault. That
+# sent two separate sessions hunting a phantom host-wide cgroup failure. It was
+# load. Measured on that 4-core box: at load ~14 podman runs 9-16 processes deep
+# and healthchecks time out 3-8/min; at load ~3.7, podman ~1 and zero timeouts.
+#
+# So: refuse to start on a loaded host rather than emit misleading failures.
+# Note what is deliberately NOT checked — the count of "Failed to create
+# container" in the journal. Those lines are emitted by healthcheck exec churn
+# and by settling after a boot; they never reach 0 on a busy node, and gating on
+# them blocks the gate forever. Prove container creation POSITIVELY instead.
+preflight_host() {
+  [[ "${ARCHY_PREFLIGHT:-1}" == "1" ]] || return 0
+  command -v podman >/dev/null 2>&1 || return 0   # remote/off-node run
+
+  local cores max_load
+  cores=$(nproc 2>/dev/null || echo 4)
+  max_load="${ARCHY_MAX_LOAD:-$((cores + 1))}"
+
+  echo "── preflight: host readiness ──"
+
+  # 1. aardvark-dns must be singular. Two of them serve divergent state and make
+  #    container-name resolution flaky, which then looks like a lifecycle bug.
+  local dns
+  dns=$(pgrep -c aardvark-dns 2>/dev/null || echo 0)
+  if (( dns > 1 )); then
+    echo "  FAIL: $dns aardvark-dns processes running (expected 1)." >&2
+    echo "        Duplicate DNS servers desync container-name resolution." >&2
+    return 1
+  fi
+  echo "  aardvark-dns: $dns"
+
+  # 2. Wait for load to fall. It oscillates on nodes doing IBD or media
+  #    indexing, so a brief spike is not fatal — a sustained one is.
+  local deadline=$(( $(date +%s) + ${ARCHY_PREFLIGHT_SECS:-900} ))
+  local load1
+  while :; do
+    load1=$(awk '{print $1}' /proc/loadavg)
+    awk -v l="$load1" -v m="$max_load" 'BEGIN { exit !(l < m) }' && break
+    if (( $(date +%s) >= deadline )); then
+      echo "  FAIL: load1 $load1 still above $max_load after ${ARCHY_PREFLIGHT_SECS:-900}s." >&2
+      echo "        Quiesce the node (bitcoind IBD, electrumx indexing, CI runners)" >&2
+      echo "        or override with ARCHY_MAX_LOAD=. Running now yields failures" >&2
+      echo "        that look like lifecycle bugs but are contention." >&2
+      return 1
+    fi
+    echo "  load1 $load1 > $max_load — waiting…"
+    sleep 20
+  done
+  echo "  load1: $load1 (ceiling $max_load, $cores cores)"
+
+  # 3. Prove the host can actually create a container, 3× — the positive test
+  #    that the journal grep only ever approximated.
+  local img
+  img=$(podman images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null \
+        | grep -v '<none>' | head -1)
+  if [[ -z "$img" ]]; then
+    echo "  (no local image — skipping container-create probe)"
+  else
+    local n
+    for n in 1 2 3; do
+      if ! timeout 60 podman run --rm "$img" /bin/true >/dev/null 2>&1; then
+        echo "  FAIL: container-create probe $n/3 failed using $img." >&2
+        echo "        The host genuinely cannot create containers; fix that first." >&2
+        return 1
+      fi
+    done
+    echo "  container-create probe: 3/3 via $img"
+  fi
+
+  echo "── preflight: OK ──"
+}
+
+if ! preflight_host; then
+  echo "Preflight failed — refusing to start the gate. (ARCHY_PREFLIGHT=0 to skip.)" >&2
+  exit 3
+fi
+
 # One initial teardown so a previous run's cookies don't poison iteration 1.
 ./setup-teardown.sh