fix: recover mempool frontend after backend address changes

2026-09-15 02:49:06 -04:00
parent 700d39c425
commit b35409ca74
9 changed files with 244 additions and 149 deletions
@@ -1,7 +1,7 @@
 app:
  id: archy-mempool-web
  name: Mempool Web
-  version: 3.0.1
+  version: 3.3.1-archy1
  # Where this app comes from, so scripts/check-upstream-releases.py can
  # tell us when the pin below has fallen behind. Without it nothing can:
  # container.image names our mirror, not the project it was mirrored from.
@@ -12,7 +12,7 @@ app:
  container_name: mempool

  container:
-    image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1
+    image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1
    pull_policy: if-not-present
    network: archy-net

@@ -45,7 +45,9 @@ app:
    # first, but nginx binds 0.0.0.0:8080 (IPv4) only -> localhost probe gets
    # "connection refused" -> perpetual unhealthy -> health_monitor restart loop.
    endpoint: http://127.0.0.1:8080
-    path: /
+    # Probe the backend through nginx: a static page can be healthy while
+    # every API/WebSocket request is stuck on a dead backend address.
+    path: /api/v1/backend-info
    interval: 30s
    timeout: 5s
    retries: 3
@@ -1,7 +1,7 @@
 app:
  id: mempool
  name: Mempool Explorer
-  version: 3.0.0
+  version: 3.3.1-archy1
  # Where this app comes from, so scripts/check-upstream-releases.py can
  # tell us when the pin below has fallen behind. Without it nothing can:
  # container.image names our mirror, not the project it was mirrored from.
@@ -11,7 +11,7 @@ app:
  description: Bitcoin mempool and blockchain explorer. Real-time transaction and block visualization.
  
  container:
-    image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1
+    image: source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1
    image_signature: cosign://...
    pull_policy: if-not-present
    
@@ -1,14 +1,13 @@
 # Archipelago mempool frontend — adds a resilient nginx backend proxy.
 #
-# The only delta vs the upstream image is /patch/entrypoint.sh, which rewrites
-# the generated nginx-mempool.conf to use `resolver` + a variable proxy_pass so
-# the frontend re-resolves the backend (mempool-api) via DNS on every request.
-# Without this, nginx pins the backend IP at startup and serves 502 / "offline"
-# after any backend restart (podman reassigns the IP). See the script header.
-ARG BASE=source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.0.0
+# Keep the upstream startup logic; repair its rendered proxy configuration.
+# Publish this derived image under an Archipelago-specific tag, never the
+# upstream version tag that the registry mirror can overwrite.
+ARG BASE=source.archipelago-foundation.org/lfg2025/mempool-frontend@sha256:d63498a109622475c913db4e3199d893f2440a451450e542923d2e55a38407a0
 FROM ${BASE}

 # --chmod keeps the exec bit (build runs as USER 1000, plain COPY lands root:0644
 # → "not executable"). Base USER/ENTRYPOINT/CMD (1000 / /patch/entrypoint.sh /
 # nginx -g "daemon off;") are inherited unchanged.
-COPY --chmod=0755 entrypoint.sh /patch/entrypoint.sh
+RUN cp /patch/entrypoint.sh /patch/upstream-entrypoint.sh
+COPY --chmod=0755 entrypoint.sh start-nginx.sh repair-nginx.sh /patch/
@@ -0,0 +1,29 @@
+# Mempool frontend DNS recovery
+
+The stock v3.3.1 nginx configuration resolves `mempool-api` only when workers
+start. Recreating the backend can change its Podman address while the frontend
+continues to serve its static page, leaving all API/WebSocket requests offline.
+
+Build and test the derived image before publishing:
+
+```sh
+podman build --pull=never -t source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1 docker/mempool-frontend
+python3 scripts/test-mempool-dns-recovery.py
+```
+
+The base is pinned by digest. The wrapper preserves upstream runtime options,
+then repairs all four local API/WebSocket routes after placeholder rendering.
+DNS is cached for five seconds using the container network resolver. Explicit
+rewrites preserve API prefixes and query arguments; backend absence does not
+prevent nginx startup. An unexpected upstream configuration fails startup
+instead of silently omitting the fix.
+
+Use an Archipelago-specific image tag. Do not replace it with a stock upstream
+mirror when updating mempool. Every upstream update must rebuild this wrapper
+and pass the recovery test (backend absent, changed IP, HTTP and WebSocket
+mapping, repeated repair, and frontend restart).
+
+Publish the tested image before publishing the signed app catalog. Both the
+mempool umbrella image mapping and the archy-mempool-web embedded manifest must
+point at the patched image. Keep scripts/image-versions.sh in sync. The frontend
+health check must reach `/api/v1/backend-info` through nginx, not only `/`.
@@ -1,137 +1,5 @@
 #!/bin/sh
-__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
-__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
-__MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080}
-
-CONF=/etc/nginx/conf.d/nginx-mempool.conf
-
-# ─── archipelago patch ────────────────────────────────────────────────────
-# The stock frontend writes `proxy_pass http://<backend>:8999` with a literal
-# hostname and NO resolver, so nginx resolves the backend IP ONCE at worker
-# start and caches it for the process lifetime. Podman reassigns the backend
-# container's IP whenever it is restarted/recreated (gate, OTA, crash, reboot
-# re-IPAM), after which nginx keeps proxying to the dead IP → /api hangs, the
-# websocket 502s, and the mempool UI shows "offline" until nginx is reloaded.
-#
-# Fix: force per-request DNS re-resolution via `resolver` + a variable in
-# proxy_pass. Because a variable in proxy_pass disables nginx's automatic
-# location→URI rewriting, each block is rewritten to preserve its original
-# path mapping exactly:
-#   /api/v1/ws, /ws → "/"            (var + "/" replaces the whole URI)
-#   /api/v1         → identity       (no-URI proxy_pass passes $uri unchanged)
-#   /api/           → /api/v1/$1     (explicit rewrite, then no-URI proxy_pass)
-# Operates on the __PLACEHOLDER__ tokens so the host/port sed below fills in
-# the concrete values (incl. the `set $mp_backend` line). Idempotent.
-# Resolver address: podman's aardvark-dns answers on the network gateway
-# (e.g. 10.89.0.1), NOT Docker's 127.0.0.11. Read it from resolv.conf so this
-# works on any podman network/subnet (and still falls back for Docker).
-ARCHY_RESOLVER=$(awk '/^nameserver/ { print $2; exit }' /etc/resolv.conf 2>/dev/null)
-ARCHY_RESOLVER=${ARCHY_RESOLVER:-127.0.0.11}
-
-if ! grep -q 'set \$mp_backend' "$CONF"; then
-  awk -v res_addr="$ARCHY_RESOLVER" '
-    BEGIN { res = 0 }
-    /^[[:space:]]*location / && res == 0 {
-      print "\tresolver " res_addr " valid=10s ipv6=off;"
-      res = 1
-    }
-    /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/;/ {
-      print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;"
-      print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/;"
-      next
-    }
-    /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/api\/v1\/;/ {
-      print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;"
-      print "\t\trewrite ^/api/(.*)$ /api/v1/$1 break;"
-      print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__;"
-      next
-    }
-    /proxy_pass http:\/\/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__\/api\/v1;/ {
-      print "\t\tset $mp_backend __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__;"
-      print "\t\tproxy_pass http://$mp_backend:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__;"
-      next
-    }
-    { print }
-  ' "$CONF" > "$CONF.archy" && mv "$CONF.archy" "$CONF"
-fi
-# ─── end archipelago patch ────────────────────────────────────────────────
-
-sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf
-sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf
-
-cp /etc/nginx/nginx.conf /patch/nginx.conf
-sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
-cat /patch/nginx.conf > /etc/nginx/nginx.conf
-
-if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then
-  export LIGHTNING=true
-fi
-
-# Runtime overrides - read env vars defined in docker compose
-
-__MAINNET_ENABLED__=${MAINNET_ENABLED:=true}
-__TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
-__TESTNET4_ENABLED__=${TESTNET_ENABLED:=false}
-__SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
-__LIQUID_ENABLED__=${LIQUID_ENABLED:=false}
-__LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false}
-__ITEMS_PER_PAGE__=${ITEMS_PER_PAGE:=10}
-__KEEP_BLOCKS_AMOUNT__=${KEEP_BLOCKS_AMOUNT:=8}
-__NGINX_PROTOCOL__=${NGINX_PROTOCOL:=http}
-__NGINX_HOSTNAME__=${NGINX_HOSTNAME:=localhost}
-__NGINX_PORT__=${NGINX_PORT:=8999}
-__BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000}
-__MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8}
-__BASE_MODULE__=${BASE_MODULE:=mempool}
-__ROOT_NETWORK__=${ROOT_NETWORK:=}
-__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space}
-__LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
-__MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
-__LIGHTNING__=${LIGHTNING:=false}
-__AUDIT__=${AUDIT:=false}
-__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__ACCELERATOR__=${ACCELERATOR:=false}
-__ACCELERATOR_BUTTON__=${ACCELERATOR_BUTTON:=true}
-__SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services}
-__PUBLIC_ACCELERATIONS__=${PUBLIC_ACCELERATIONS:=false}
-__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
-__ADDITIONAL_CURRENCIES__=${ADDITIONAL_CURRENCIES:=false}
-
-# Export as environment variables to be used by envsubst
-export __MAINNET_ENABLED__
-export __TESTNET_ENABLED__
-export __TESTNET4_ENABLED__
-export __SIGNET_ENABLED__
-export __LIQUID_ENABLED__
-export __LIQUID_TESTNET_ENABLED__
-export __ITEMS_PER_PAGE__
-export __KEEP_BLOCKS_AMOUNT__
-export __NGINX_PROTOCOL__
-export __NGINX_HOSTNAME__
-export __NGINX_PORT__
-export __BLOCK_WEIGHT_UNITS__
-export __MEMPOOL_BLOCKS_AMOUNT__
-export __BASE_MODULE__
-export __ROOT_NETWORK__
-export __MEMPOOL_WEBSITE_URL__
-export __LIQUID_WEBSITE_URL__
-export __MINING_DASHBOARD__
-export __LIGHTNING__
-export __AUDIT__
-export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
-export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
-export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
-export __ACCELERATOR__
-export __ACCELERATOR_BUTTON__
-export __SERVICES_API__
-export __PUBLIC_ACCELERATIONS__
-export __HISTORICAL_PRICE__
-export __ADDITIONAL_CURRENCIES__
-
-folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
-echo ${folder}
-envsubst < ${folder}/config.template.js > ${folder}/config.js
-
-exec "$@"
+set -eu
+# Preserve the pinned upstream entrypoint (including new runtime options).
+# Apply our DNS repair only after it has rendered the nginx configuration.
+exec /patch/upstream-entrypoint.sh /patch/start-nginx.sh "$@"
@@ -0,0 +1,56 @@
+#!/bin/sh
+# Resolve the backend again after container IP changes. Run after upstream
+# placeholder substitution, so the repair also works on an existing container.
+set -eu
+conf=${1:-/etc/nginx/conf.d/nginx-mempool.conf}
+resolv=${2:-/etc/resolv.conf}
+backend=${BACKEND_MAINNET_HTTP_HOST:-127.0.0.1}
+port=${BACKEND_MAINNET_HTTP_PORT:-8999}
+resolver=$(awk '/^nameserver/ { print $2; exit }' "$resolv")
+[ -n "$resolver" ] || { echo 'No DNS resolver configured' >&2; exit 1; }
+case "$resolver" in *:*) resolver="[$resolver]" ;; esac
+case "$backend" in *[!a-zA-Z0-9._-]*|'') echo 'Invalid backend hostname' >&2; exit 1 ;; esac
+case "$port" in *[!0-9]*|'') echo 'Invalid backend port' >&2; exit 1 ;; esac
+
+tmp=$(mktemp "${conf}.archy.XXXXXX")
+trap 'rm -f "$tmp"' EXIT HUP INT TERM
+awk -v backend="$backend" -v port="$port" -v resolver="$resolver" '
+  BEGIN {
+    base = "http://" backend ":" port
+    print "# Archipelago: refresh backend DNS after container replacement."
+    print "resolver " resolver " valid=5s ipv6=off; # archy-dns"
+    print "resolver_timeout 3s; # archy-dns"
+  }
+  /# Archipelago: refresh backend DNS/ || /# archy-dns/ { next }
+  /^[[:space:]]*location[[:space:]]/ { location = $2 }
+  /^[[:space:]]*proxy_pass[[:space:]]/ && index($2, base) == 1 {
+    target = $2
+    sub(/;$/, "", target)
+    path = substr(target, length(base) + 1)
+    if (location != "/api/v1/ws" && location != "/ws" && location != "/api/v1" && location != "/api/") {
+      print "Unexpected backend location: " location > "/dev/stderr"
+      failed = 1; exit 1
+    }
+    if (path != "/" && path != "/api/v1" && path != "/api/v1/") {
+      print "Unexpected backend URI mapping" > "/dev/stderr"
+      failed = 1; exit 1
+    }
+    # Explicitly preserve prefix substitution and query arguments. A variable
+    # proxy_pass without a URI forwards the rewritten URI and original args.
+    print "\t\tset $mp_backend " backend ";"
+    if (path != location)
+      print "\t\trewrite ^" location "(.*)$ " path "$1 break;"
+    print "\t\tproxy_pass http://$mp_backend:" port ";"
+    count++
+    next
+  }
+  /proxy_pass http:\/\/\$mp_backend:/ { count++ }
+  { print }
+  END {
+    if (failed || count != 4) {
+      print "Expected four backend proxies; refusing an incomplete DNS repair" > "/dev/stderr"
+      exit 1
+    }
+  }
+' "$conf" > "$tmp"
+cat "$tmp" > "$conf"
@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+/patch/repair-nginx.sh
+nginx -t
+exec "$@"
@@ -33,7 +33,7 @@ ELECTRUMX_IMAGE="$ARCHY_REGISTRY/electrumx:v1.18.0"

 # Mempool stack
 MEMPOOL_BACKEND_IMAGE="$ARCHY_REGISTRY/mempool-backend:v3.3.1"
-MEMPOOL_WEB_IMAGE="$ARCHY_REGISTRY/mempool-frontend:v3.3.1"
+MEMPOOL_WEB_IMAGE="$ARCHY_REGISTRY/mempool-frontend:v3.3.1-archy1"
 MARIADB_IMAGE="$ARCHY_REGISTRY/mariadb:11.4.10"

 # BTCPay
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Exercise the built frontend against a backend that disappears and changes IP.
+
+Uses an isolated Podman network and disposable containers, never the node stack.
+Usage: python3 scripts/test-mempool-dns-recovery.py [frontend-image]
+"""
+import ipaddress
+import json
+import socket
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+import uuid
+
+IMAGE = sys.argv[1] if len(sys.argv) > 1 else (
+    "source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.3.1-archy1"
+)
+BACKEND = "source.archipelago-foundation.org/lfg2025/mempool-backend:v3.3.1"
+prefix = "mempool-dns-test-" + uuid.uuid4().hex[:8]
+network, frontend, backend = prefix, prefix + "-web", prefix + "-api"
+
+
+def podman(*args, check=True):
+    return subprocess.run(["podman", *args], capture_output=True, text=True,
+                          check=check, timeout=60).stdout.strip()
+
+
+def eventually(check, timeout=25):
+    deadline = time.monotonic() + timeout
+    while True:
+        try:
+            return check()
+        except (AssertionError, OSError, urllib.error.URLError):
+            if time.monotonic() >= deadline:
+                raise
+            time.sleep(1)
+
+
+server = r"""
+const http = require('http'), crypto = require('crypto');
+const server = http.createServer((req, res) => {
+  res.setHeader('Content-Type', 'application/json');
+  res.end(JSON.stringify({url: req.url, instance: process.env.INSTANCE}));
+});
+server.on('upgrade', (req, socket) => {
+  const key = crypto.createHash('sha1')
+    .update(req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11')
+    .digest('base64');
+  socket.end('HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n' +
+    'Connection: Upgrade\r\nSec-WebSocket-Accept: ' + key + '\r\n' +
+    'X-Upstream-Url: ' + req.url + '\r\nX-Instance: ' + process.env.INSTANCE + '\r\n\r\n');
+});
+server.listen(8999, '0.0.0.0');
+"""
+
+try:
+    podman("network", "create", network)
+    subnet = ipaddress.ip_network(json.loads(podman("network", "inspect", network))[0]["subnets"][0]["subnet"])
+    podman("run", "-d", "--name", frontend, "--network", network,
+           "-p", "127.0.0.1::8080", "-e", "BACKEND_MAINNET_HTTP_HOST=mempool-api",
+           "-e", "FRONTEND_HTTP_PORT=8080", IMAGE)
+    port = int(podman("port", frontend, "8080/tcp").rsplit(":", 1)[1])
+    url = f"http://127.0.0.1:{port}"
+
+    def static_ready():
+        assert urllib.request.urlopen(url, timeout=4).status == 200
+
+    eventually(static_ready)
+    started = podman("inspect", frontend, "--format", "{{.State.StartedAt}}")
+    try:
+        urllib.request.urlopen(url + "/api/v1/backend-info", timeout=6)
+        raise AssertionError("An absent backend must not appear healthy")
+    except urllib.error.HTTPError as error:
+        assert error.code == 502
+    print("PASS: frontend starts while backend DNS is absent", flush=True)
+
+    for instance, offset in [("first", 10), ("replacement", 11)]:
+        if instance == "replacement":
+            podman("rm", "-f", backend)
+            # Ensure the cached address has expired while the backend is absent.
+            time.sleep(6)
+        podman("run", "-d", "--name", backend, "--network", network,
+               "--network-alias", "mempool-api", "--ip", str(subnet[offset]),
+               "-e", "INSTANCE=" + instance, "--entrypoint", "node", BACKEND,
+               "-e", server)
+
+        for path, expected in [
+            ("/api/blocks/tip/height?probe=one", "/api/v1/blocks/tip/height?probe=one"),
+            ("/api/v1/fees/recommended?probe=two", "/api/v1/fees/recommended?probe=two"),
+        ]:
+            def check_http():
+                with urllib.request.urlopen(url + path, timeout=4) as response:
+                    result = json.load(response)
+                assert result == {"url": expected, "instance": instance}, result
+            eventually(check_http)
+
+        for path in ["/api/v1/ws?probe=ws", "/ws?probe=ws"]:
+            def check_ws():
+                with socket.create_connection(("127.0.0.1", port), timeout=4) as sock:
+                    sock.sendall((f"GET {path} HTTP/1.1\r\nHost: localhost\r\n"
+                                  "Upgrade: websocket\r\nConnection: Upgrade\r\n"
+                                  "Sec-WebSocket-Version: 13\r\n"
+                                  "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n").encode())
+                    response = b""
+                    while b"\r\n\r\n" not in response:
+                        part = sock.recv(4096)
+                        assert part, response
+                        response += part
+                assert b"101 Switching Protocols" in response, response
+                assert b"X-Upstream-Url: /?probe=ws" in response, response
+                assert ("X-Instance: " + instance).encode() in response, response
+            eventually(check_ws)
+        assert podman("inspect", frontend, "--format", "{{.State.StartedAt}}") == started
+        print(f"PASS: {instance} backend at {subnet[offset]}: HTTP paths, query strings, both WebSocket routes; frontend never restarted", flush=True)
+
+    before = podman("exec", frontend, "cat", "/etc/nginx/conf.d/nginx-mempool.conf")
+    podman("exec", frontend, "/patch/repair-nginx.sh")
+    assert podman("exec", frontend, "cat", "/etc/nginx/conf.d/nginx-mempool.conf") == before
+    podman("exec", frontend, "nginx", "-t")
+    print("PASS: repeated repair is idempotent and nginx configuration is valid", flush=True)
+    podman("restart", frontend)
+    eventually(static_ready)
+
+    def after_restart():
+        with urllib.request.urlopen(url + "/api/blocks/tip/height?restart=1", timeout=4) as response:
+            assert json.load(response) == {
+                "url": "/api/v1/blocks/tip/height?restart=1", "instance": "replacement"
+            }
+    eventually(after_restart)
+    eventually(check_ws)
+    print("PASS: frontend restart preserves DNS recovery and HTTP/WebSocket routing", flush=True)
+finally:
+    podman("rm", "-f", frontend, backend, check=False)
+    podman("network", "rm", network, check=False)