diff --git a/apps/archy-mempool-web/manifest.yml b/apps/archy-mempool-web/manifest.yml index 182c004d..4aab841d 100644 --- a/apps/archy-mempool-web/manifest.yml +++ b/apps/archy-mempool-web/manifest.yml @@ -1,7 +1,7 @@ app: id: archy-mempool-web name: Mempool Web - version: 3.0.1 + version: 3.3.1-archy1 # Where this app comes from, so scripts/check-upstream-releases.py can # tell us when the pin below has fallen behind. Without it nothing can: # container.image names our mirror, not the project it was mirrored from. @@ -12,7 +12,7 @@ app: container_name: mempool container: - image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1 + image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1 pull_policy: if-not-present network: archy-net @@ -45,7 +45,9 @@ app: # first, but nginx binds 0.0.0.0:8080 (IPv4) only -> localhost probe gets # "connection refused" -> perpetual unhealthy -> health_monitor restart loop. endpoint: http://127.0.0.1:8080 - path: / + # Probe the backend through nginx: a static page can be healthy while + # every API/WebSocket request is stuck on a dead backend address. + path: /api/v1/backend-info interval: 30s timeout: 5s retries: 3 diff --git a/apps/mempool/manifest.yml b/apps/mempool/manifest.yml index 885f8fd2..c6333a6e 100644 --- a/apps/mempool/manifest.yml +++ b/apps/mempool/manifest.yml @@ -1,7 +1,7 @@ app: id: mempool name: Mempool Explorer - version: 3.0.0 + version: 3.3.1-archy1 # Where this app comes from, so scripts/check-upstream-releases.py can # tell us when the pin below has fallen behind. Without it nothing can: # container.image names our mirror, not the project it was mirrored from. @@ -11,7 +11,7 @@ app: description: Bitcoin mempool and blockchain explorer. Real-time transaction and block visualization. container: - image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1 + image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1 image_signature: cosign://... pull_policy: if-not-present diff --git a/docker/mempool-frontend/Dockerfile b/docker/mempool-frontend/Dockerfile index b773591e..9dc7d675 100644 --- a/docker/mempool-frontend/Dockerfile +++ b/docker/mempool-frontend/Dockerfile @@ -1,14 +1,13 @@ # Archipelago mempool frontend — adds a resilient nginx backend proxy. # -# The only delta vs the upstream image is /patch/entrypoint.sh, which rewrites -# the generated nginx-mempool.conf to use `resolver` + a variable proxy_pass so -# the frontend re-resolves the backend (mempool-api) via DNS on every request. -# Without this, nginx pins the backend IP at startup and serves 502 / "offline" -# after any backend restart (podman reassigns the IP). See the script header. -ARG BASE=source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.0.0 +# Keep the upstream startup logic; repair its rendered proxy configuration. +# Publish this derived image under an Archipelago-specific tag, never the +# upstream version tag that the registry mirror can overwrite. +ARG BASE=source.archipelago-foundation.org/lfg2025/mempool-frontend@sha256:d63498a109622475c913db4e3199d893f2440a451450e542923d2e55a38407a0 FROM ${BASE} # --chmod keeps the exec bit (build runs as USER 1000, plain COPY lands root:0644 # → "not executable"). Base USER/ENTRYPOINT/CMD (1000 / /patch/entrypoint.sh / # nginx -g "daemon off;") are inherited unchanged. -COPY --chmod=0755 entrypoint.sh /patch/entrypoint.sh +RUN cp /patch/entrypoint.sh /patch/upstream-entrypoint.sh +COPY --chmod=0755 entrypoint.sh start-nginx.sh repair-nginx.sh /patch/ diff --git a/docker/mempool-frontend/README.md b/docker/mempool-frontend/README.md new file mode 100644 index 00000000..b0359358 --- /dev/null +++ b/docker/mempool-frontend/README.md @@ -0,0 +1,29 @@ +# Mempool frontend DNS recovery + +The stock v3.3.1 nginx configuration resolves `mempool-api` only when workers +start. Recreating the backend can change its Podman address while the frontend +continues to serve its static page, leaving all API/WebSocket requests offline. + +Build and test the derived image before publishing: + +```sh +podman build --pull=never -t source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1 docker/mempool-frontend +python3 scripts/test-mempool-dns-recovery.py +``` + +The base is pinned by digest. The wrapper preserves upstream runtime options, +then repairs all four local API/WebSocket routes after placeholder rendering. +DNS is cached for five seconds using the container network resolver. Explicit +rewrites preserve API prefixes and query arguments; backend absence does not +prevent nginx startup. An unexpected upstream configuration fails startup +instead of silently omitting the fix. + +Use an Archipelago-specific image tag. Do not replace it with a stock upstream +mirror when updating mempool. Every upstream update must rebuild this wrapper +and pass the recovery test (backend absent, changed IP, HTTP and WebSocket +mapping, repeated repair, and frontend restart). + +Publish the tested image before publishing the signed app catalog. Both the +mempool umbrella image mapping and the archy-mempool-web embedded manifest must +point at the patched image. Keep scripts/image-versions.sh in sync. The frontend +health check must reach `/api/v1/backend-info` through nginx, not only `/`. diff --git a/docker/mempool-frontend/entrypoint.sh b/docker/mempool-frontend/entrypoint.sh old mode 100644 new mode 100755 index 1c37e31b..7443ac9e --- a/docker/mempool-frontend/entrypoint.sh +++ b/docker/mempool-frontend/entrypoint.sh @@ -1,137 +1,5 @@ #!/bin/sh -__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1} -__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999} -__MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080} - -CONF=/etc/nginx/conf.d/nginx-mempool.conf - -# ─── archipelago patch ──────────────────────────────────────────────────── -# The stock frontend writes `proxy_pass http://:8999` with a literal -# hostname and NO resolver, so nginx resolves the backend IP ONCE at worker -# start and caches it for the process lifetime. Podman reassigns the backend -# container's IP whenever it is restarted/recreated (gate, OTA, crash, reboot -# re-IPAM), after which nginx keeps proxying to the dead IP → /api hangs, the -# websocket 502s, and the mempool UI shows "offline" until nginx is reloaded. -# -# Fix: force per-request DNS re-resolution via `resolver` + a variable in -# proxy_pass. Because a variable in proxy_pass disables nginx's automatic -# location→URI rewriting, each block is rewritten to preserve its original -# path mapping exactly: -# /api/v1/ws, /ws → "/" (var + "/" replaces the whole URI) -# /api/v1 → identity (no-URI proxy_pass passes $uri unchanged) -# /api/ → /api/v1/$1 (explicit rewrite, then no-URI proxy_pass) -# Operates on the __PLACEHOLDER__ tokens so the host/port sed below fills in -# the concrete values (incl. the `set $mp_backend` line). Idempotent. -# Resolver address: podman's aardvark-dns answers on the network gateway -# (e.g. 10.89.0.1), NOT Docker's 127.0.0.11. Read it from resolv.conf so this -# works on any podman network/subnet (and still falls back for Docker). -ARCHY_RESOLVER=$(awk '/^nameserver/ { print $2; exit }' /etc/resolv.conf 2>/dev/null) -ARCHY_RESOLVER=${ARCHY_RESOLVER:-127.0.0.11} - -if ! grep -q 'set \$mp_backend' "$CONF"; then - awk -v res_addr="$ARCHY_RESOLVER" ' - BEGIN { res = 0 } - /^[[:space:]]*location / && res == 0 { - print "\tresolver " res_addr " valid=10s ipv6=off;" - res = 1 - } - /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/;/ { - print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;" - print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;" - next - } - /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/api\/v1\/;/ { - print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;" - print "\t\trewrite ^/api/(.*)$ /api/v1/$1 break;" - print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__;" - next - } - /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/api\/v1;/ { - print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;" - print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__;" - next - } - { print } - ' "$CONF" > "$CONF.archy" && mv "$CONF.archy" "$CONF" -fi -# ─── end archipelago patch ──────────────────────────────────────────────── - -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf - -cp /etc/nginx/nginx.conf /patch/nginx.conf -sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf -cat /patch/nginx.conf > /etc/nginx/nginx.conf - -if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then - export LIGHTNING=true -fi - -# Runtime overrides - read env vars defined in docker compose - -__MAINNET_ENABLED__=${MAINNET_ENABLED:=true} -__TESTNET_ENABLED__=${TESTNET_ENABLED:=false} -__TESTNET4_ENABLED__=${TESTNET_ENABLED:=false} -__SIGNET_ENABLED__=${SIGNET_ENABLED:=false} -__LIQUID_ENABLED__=${LIQUID_ENABLED:=false} -__LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false} -__ITEMS_PER_PAGE__=${ITEMS_PER_PAGE:=10} -__KEEP_BLOCKS_AMOUNT__=${KEEP_BLOCKS_AMOUNT:=8} -__NGINX_PROTOCOL__=${NGINX_PROTOCOL:=http} -__NGINX_HOSTNAME__=${NGINX_HOSTNAME:=localhost} -__NGINX_PORT__=${NGINX_PORT:=8999} -__BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000} -__MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8} -__BASE_MODULE__=${BASE_MODULE:=mempool} -__ROOT_NETWORK__=${ROOT_NETWORK:=} -__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space} -__LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network} -__MINING_DASHBOARD__=${MINING_DASHBOARD:=true} -__LIGHTNING__=${LIGHTNING:=false} -__AUDIT__=${AUDIT:=false} -__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0} -__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0} -__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0} -__ACCELERATOR__=${ACCELERATOR:=false} -__ACCELERATOR_BUTTON__=${ACCELERATOR_BUTTON:=true} -__SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services} -__PUBLIC_ACCELERATIONS__=${PUBLIC_ACCELERATIONS:=false} -__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true} -__ADDITIONAL_CURRENCIES__=${ADDITIONAL_CURRENCIES:=false} - -# Export as environment variables to be used by envsubst -export __MAINNET_ENABLED__ -export __TESTNET_ENABLED__ -export __TESTNET4_ENABLED__ -export __SIGNET_ENABLED__ -export __LIQUID_ENABLED__ -export __LIQUID_TESTNET_ENABLED__ -export __ITEMS_PER_PAGE__ -export __KEEP_BLOCKS_AMOUNT__ -export __NGINX_PROTOCOL__ -export __NGINX_HOSTNAME__ -export __NGINX_PORT__ -export __BLOCK_WEIGHT_UNITS__ -export __MEMPOOL_BLOCKS_AMOUNT__ -export __BASE_MODULE__ -export __ROOT_NETWORK__ -export __MEMPOOL_WEBSITE_URL__ -export __LIQUID_WEBSITE_URL__ -export __MINING_DASHBOARD__ -export __LIGHTNING__ -export __AUDIT__ -export __MAINNET_BLOCK_AUDIT_START_HEIGHT__ -export __TESTNET_BLOCK_AUDIT_START_HEIGHT__ -export __SIGNET_BLOCK_AUDIT_START_HEIGHT__ -export __ACCELERATOR__ -export __ACCELERATOR_BUTTON__ -export __SERVICES_API__ -export __PUBLIC_ACCELERATIONS__ -export __HISTORICAL_PRICE__ -export __ADDITIONAL_CURRENCIES__ - -folder=$(find /var/www/mempool -name "config.js" | xargs dirname) -echo ${folder} -envsubst < ${folder}/config.template.js > ${folder}/config.js - -exec "$@" +set -eu +# Preserve the pinned upstream entrypoint (including new runtime options). +# Apply our DNS repair only after it has rendered the nginx configuration. +exec /patch/upstream-entrypoint.sh /patch/start-nginx.sh "$@" diff --git a/docker/mempool-frontend/repair-nginx.sh b/docker/mempool-frontend/repair-nginx.sh new file mode 100755 index 00000000..df03565c --- /dev/null +++ b/docker/mempool-frontend/repair-nginx.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Resolve the backend again after container IP changes. Run after upstream +# placeholder substitution, so the repair also works on an existing container. +set -eu +conf=${1:-/etc/nginx/conf.d/nginx-mempool.conf} +resolv=${2:-/etc/resolv.conf} +backend=${BACKEND_MAINNET_HTTP_HOST:-127.0.0.1} +port=${BACKEND_MAINNET_HTTP_PORT:-8999} +resolver=$(awk '/^nameserver/ { print $2; exit }' "$resolv") +[ -n "$resolver" ] || { echo 'No DNS resolver configured' >&2; exit 1; } +case "$resolver" in *:*) resolver="[$resolver]" ;; esac +case "$backend" in *[!a-zA-Z0-9._-]*|'') echo 'Invalid backend hostname' >&2; exit 1 ;; esac +case "$port" in *[!0-9]*|'') echo 'Invalid backend port' >&2; exit 1 ;; esac + +tmp=$(mktemp "${conf}.archy.XXXXXX") +trap 'rm -f "$tmp"' EXIT HUP INT TERM +awk -v backend="$backend" -v port="$port" -v resolver="$resolver" ' + BEGIN { + base = "http://" backend ":" port + print "# Archipelago: refresh backend DNS after container replacement." + print "resolver " resolver " valid=5s ipv6=off; # archy-dns" + print "resolver_timeout 3s; # archy-dns" + } + /# Archipelago: refresh backend DNS/ || /# archy-dns/ { next } + /^[[:space:]]*location[[:space:]]/ { location = $2 } + /^[[:space:]]*proxy_pass[[:space:]]/ && index($2, base) == 1 { + target = $2 + sub(/;$/, "", target) + path = substr(target, length(base) + 1) + if (location != "/api/v1/ws" && location != "/ws" && location != "/api/v1" && location != "/api/") { + print "Unexpected backend location: " location > "/dev/stderr" + failed = 1; exit 1 + } + if (path != "/" && path != "/api/v1" && path != "/api/v1/") { + print "Unexpected backend URI mapping" > "/dev/stderr" + failed = 1; exit 1 + } + # Explicitly preserve prefix substitution and query arguments. A variable + # proxy_pass without a URI forwards the rewritten URI and original args. + print "\t\tset $mp_backend " backend ";" + if (path != location) + print "\t\trewrite ^" location "(.*)$ " path "$1 break;" + print "\t\tproxy_pass http://$mp_backend:" port ";" + count++ + next + } + /proxy_pass http:\/\/\$mp_backend:/ { count++ } + { print } + END { + if (failed || count != 4) { + print "Expected four backend proxies; refusing an incomplete DNS repair" > "/dev/stderr" + exit 1 + } + } +' "$conf" > "$tmp" +cat "$tmp" > "$conf" diff --git a/docker/mempool-frontend/start-nginx.sh b/docker/mempool-frontend/start-nginx.sh new file mode 100755 index 00000000..115c9f47 --- /dev/null +++ b/docker/mempool-frontend/start-nginx.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +/patch/repair-nginx.sh +nginx -t +exec "$@" diff --git a/scripts/image-versions.sh b/scripts/image-versions.sh index 81ed49f6..b74b2fdb 100644 --- a/scripts/image-versions.sh +++ b/scripts/image-versions.sh @@ -33,7 +33,7 @@ ELECTRUMX_IMAGE="$ARCHY_REGISTRY/electrumx:v1.18.0" # Mempool stack MEMPOOL_BACKEND_IMAGE="$ARCHY_REGISTRY/mempool-backend:v3.3.1" -MEMPOOL_WEB_IMAGE="$ARCHY_REGISTRY/mempool-frontend:v3.3.1" +MEMPOOL_WEB_IMAGE="$ARCHY_REGISTRY/mempool-frontend:v3.3.1-archy1" MARIADB_IMAGE="$ARCHY_REGISTRY/mariadb:11.4.10" # BTCPay diff --git a/scripts/test-mempool-dns-recovery.py b/scripts/test-mempool-dns-recovery.py new file mode 100644 index 00000000..dedf8a49 --- /dev/null +++ b/scripts/test-mempool-dns-recovery.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Exercise the built frontend against a backend that disappears and changes IP. + +Uses an isolated Podman network and disposable containers, never the node stack. +Usage: python3 scripts/test-mempool-dns-recovery.py [frontend-image] +""" +import ipaddress +import json +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid + +IMAGE = sys.argv[1] if len(sys.argv) > 1 else ( + "source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1" +) +BACKEND = "source.archipelago-foundation.org/lfg2025/mempool-backend:v3.3.1" +prefix = "mempool-dns-test-" + uuid.uuid4().hex[:8] +network, frontend, backend = prefix, prefix + "-web", prefix + "-api" + + +def podman(*args, check=True): + return subprocess.run(["podman", *args], capture_output=True, text=True, + check=check, timeout=60).stdout.strip() + + +def eventually(check, timeout=25): + deadline = time.monotonic() + timeout + while True: + try: + return check() + except (AssertionError, OSError, urllib.error.URLError): + if time.monotonic() >= deadline: + raise + time.sleep(1) + + +server = r""" +const http = require('http'), crypto = require('crypto'); +const server = http.createServer((req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({url: req.url, instance: process.env.INSTANCE})); +}); +server.on('upgrade', (req, socket) => { + const key = crypto.createHash('sha1') + .update(req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11') + .digest('base64'); + socket.end('HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n' + + 'Connection: Upgrade\r\nSec-WebSocket-Accept: ' + key + '\r\n' + + 'X-Upstream-Url: ' + req.url + '\r\nX-Instance: ' + process.env.INSTANCE + '\r\n\r\n'); +}); +server.listen(8999, '0.0.0.0'); +""" + +try: + podman("network", "create", network) + subnet = ipaddress.ip_network(json.loads(podman("network", "inspect", network))[0]["subnets"][0]["subnet"]) + podman("run", "-d", "--name", frontend, "--network", network, + "-p", "127.0.0.1::8080", "-e", "BACKEND_MAINNET_HTTP_HOST=mempool-api", + "-e", "FRONTEND_HTTP_PORT=8080", IMAGE) + port = int(podman("port", frontend, "8080/tcp").rsplit(":", 1)[1]) + url = f"http://127.0.0.1:{port}" + + def static_ready(): + assert urllib.request.urlopen(url, timeout=4).status == 200 + + eventually(static_ready) + started = podman("inspect", frontend, "--format", "{{.State.StartedAt}}") + try: + urllib.request.urlopen(url + "/api/v1/backend-info", timeout=6) + raise AssertionError("An absent backend must not appear healthy") + except urllib.error.HTTPError as error: + assert error.code == 502 + print("PASS: frontend starts while backend DNS is absent", flush=True) + + for instance, offset in [("first", 10), ("replacement", 11)]: + if instance == "replacement": + podman("rm", "-f", backend) + # Ensure the cached address has expired while the backend is absent. + time.sleep(6) + podman("run", "-d", "--name", backend, "--network", network, + "--network-alias", "mempool-api", "--ip", str(subnet[offset]), + "-e", "INSTANCE=" + instance, "--entrypoint", "node", BACKEND, + "-e", server) + + for path, expected in [ + ("/api/blocks/tip/height?probe=one", "/api/v1/blocks/tip/height?probe=one"), + ("/api/v1/fees/recommended?probe=two", "/api/v1/fees/recommended?probe=two"), + ]: + def check_http(): + with urllib.request.urlopen(url + path, timeout=4) as response: + result = json.load(response) + assert result == {"url": expected, "instance": instance}, result + eventually(check_http) + + for path in ["/api/v1/ws?probe=ws", "/ws?probe=ws"]: + def check_ws(): + with socket.create_connection(("127.0.0.1", port), timeout=4) as sock: + sock.sendall((f"GET {path} HTTP/1.1\r\nHost: localhost\r\n" + "Upgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Version: 13\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n").encode()) + response = b"" + while b"\r\n\r\n" not in response: + part = sock.recv(4096) + assert part, response + response += part + assert b"101 Switching Protocols" in response, response + assert b"X-Upstream-Url: /?probe=ws" in response, response + assert ("X-Instance: " + instance).encode() in response, response + eventually(check_ws) + assert podman("inspect", frontend, "--format", "{{.State.StartedAt}}") == started + print(f"PASS: {instance} backend at {subnet[offset]}: HTTP paths, query strings, both WebSocket routes; frontend never restarted", flush=True) + + before = podman("exec", frontend, "cat", "/etc/nginx/conf.d/nginx-mempool.conf") + podman("exec", frontend, "/patch/repair-nginx.sh") + assert podman("exec", frontend, "cat", "/etc/nginx/conf.d/nginx-mempool.conf") == before + podman("exec", frontend, "nginx", "-t") + print("PASS: repeated repair is idempotent and nginx configuration is valid", flush=True) + podman("restart", frontend) + eventually(static_ready) + + def after_restart(): + with urllib.request.urlopen(url + "/api/blocks/tip/height?restart=1", timeout=4) as response: + assert json.load(response) == { + "url": "/api/v1/blocks/tip/height?restart=1", "instance": "replacement" + } + eventually(after_restart) + eventually(check_ws) + print("PASS: frontend restart preserves DNS recovery and HTTP/WebSocket routing", flush=True) +finally: + podman("rm", "-f", frontend, backend, check=False) + podman("network", "rm", network, check=False)