test(gate): harden readiness for sustained 5x churn + inter-iteration settle

The 1x gate is green; the 5x failed iters 1-2 on readiness-under-churn (apps DO
recover — lnd synced, mempool just mid-restart when probed — but slower than the
windows when restarted back-to-back). Hardening:
- run-20x.sh: best-effort settle_stack() before each iteration (wait for
  mempool-api/frontend + lnd RPC healthy, 180s, on-node, never fails the run).
- required containers present/running (80/81): wait-loops (180s) not single-shot.
- mempool api/frontend (87/88): retry ~180s not single-shot.
- mempool queryable (74): 60s->180s. lnd restart-running (64): 120s->240s.
  lnd getinfo (60): 90s->240s retry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
2026-06-22 17:11:15 -04:00
co-authored by Claude Opus 4.8
parent 22b05de6d9
commit 98f4fa44a8
4 changed files with 50 additions and 14 deletions
+23 -10
View File
@@ -41,19 +41,31 @@ bitcoin_json() {
}
@test "required containers are present" {
local names
names="$(podman_names)"
for c in "${required_containers[@]}"; do
echo "$names" | grep -Fx "$c" >/dev/null
# Under sustained 5× churn an app may still be mid-restart when this runs;
# wait for the whole required set rather than single-shot.
local deadline=$(( $(date +%s) + 180 )) names missing
while (( $(date +%s) < deadline )); do
names="$(podman_names)"; missing=""
for c in "${required_containers[@]}"; do
echo "$names" | grep -Fx "$c" >/dev/null || missing="$missing $c"
done
[[ -z "$missing" ]] && return 0
sleep 3
done
fail "required containers never all present; missing:$missing"
}
@test "required containers are running" {
for c in "${required_containers[@]}"; do
run container_running "$c"
[ "$status" -eq 0 ]
[ "$output" = "true" ]
local deadline=$(( $(date +%s) + 180 )) notrunning
while (( $(date +%s) < deadline )); do
notrunning=""
for c in "${required_containers[@]}"; do
[[ "$(container_running "$c" 2>/dev/null)" == "true" ]] || notrunning="$notrunning $c"
done
[[ -z "$notrunning" ]] && return 0
sleep 3
done
fail "required containers never all running; not-running:$notrunning"
}
@test "bitcoin-knots RPC responds" {
@@ -113,12 +125,13 @@ PY
}
@test "mempool api endpoint responds" {
run curl -fsS "http://127.0.0.1:8999/api/v1/backend-info"
# mempool-api reconnects to electrumx after a stack restart — retry ~180s.
run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:8999/api/v1/backend-info" && exit 0; sleep 3; done; exit 1'
[ "$status" -eq 0 ]
}
@test "mempool frontend responds" {
run curl -fsS "http://127.0.0.1:4080/"
run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:4080/" && exit 0; sleep 3; done; exit 1'
[ "$status" -eq 0 ]
}