fix(lnd): pay through Router.SendPaymentV2 — LND 0.21 removed the old route

LND 0.21.2 removed the deprecated Lightning.SendPaymentSync REST route
(/v1/channels/transactions). The backend still called it, so every
Lightning send answered literal HTTP 404 and the wallet UI reported
'Payment failed: Not Found' fleet-wide right after the pin bump —
receive worked, which made it look intermittent.

Pay through the supported Router.SendPaymentV2 route (/v2/router/send)
instead, keeping the existing contract with the UI:
- single-record responses (no_inflight_updates) unwrapped from the
  grpc-gateway result envelope, transport errors from the nested error
- a slow multi-hop payment still resolves as pending + payment hash
  (only LND may declare failure), never a false 'Payment failed'
- LND's failure_reason codes translated to the same plain-language
  advice, invoice-expiry still says 'ask for a fresh invoice'

Guard it at the gate: tests/lifecycle/bats/lnd-api-compat.bats POSTs a
deliberately-invalid invoice to /v2/router/send on the RUNNING LND and
fails if the route answers 404 — the image/backend skew that shipped
silently last time because no test ever spoke the payment endpoint.
Also bumps the stale lnd image expectation in remote-lifecycle.sh.

core/archipelago/src/api/rpc/lnd/payments.rs

@@ -4,6 +4,44 @@ use tracing::info;
 
 use super::LND_REST_BASE_URL;
 
+fn router_error_message(body: &serde_json::Value) -> Option<&str> {
+    body.get("error")
+        .and_then(|e| e.get("message"))
+        .and_then(|v| v.as_str())
+        .or_else(|| body.get("message").and_then(|v| v.as_str()))
+}
+
+fn payment_error(message: &str) -> anyhow::Error {
+    if message.to_ascii_lowercase().contains("invoice expired") {
+        anyhow::anyhow!(
+            "Payment failed: this invoice has expired ({}). Ask the recipient for a fresh invoice and try again.",
+            message.trim_start_matches("invoice expired. ")
+        )
+    } else {
+        anyhow::anyhow!("Payment failed: {message}")
+    }
+}
+
+fn payment_failure_reason(reason: &str) -> &'static str {
+    match reason {
+        "FAILURE_REASON_NO_ROUTE" => "No route to the recipient",
+        "FAILURE_REASON_INSUFFICIENT_BALANCE" => "Insufficient channel balance",
+        "FAILURE_REASON_TIMEOUT" => "Payment timed out in the network",
+        "FAILURE_REASON_INCORRECT_PAYMENT_DETAILS" => {
+            "Recipient rejected the payment (wrong details or expired invoice)"
+        }
+        _ => "Payment failed",
+    }
+}
+
+fn json_i64(value: &serde_json::Value, key: &str) -> Option<i64> {
+    value.get(key).and_then(|v| {
+        v.as_str()
+            .and_then(|s| s.parse().ok())
+            .or_else(|| v.as_i64())
+    })
+}
+
 impl RpcHandler {
     /// Pay a Lightning invoice.
     pub(in crate::api::rpc) async fn handle_lnd_payinvoice(
@@ -65,23 +103,22 @@ impl RpcHandler {
 
         let mut pay_body = serde_json::json!({
             "payment_request": payment_request,
+            // Suppress intermediate stream records: one terminal Payment is
+            // enough, and it makes grpc-gateway's response a single JSON value.
+            "no_inflight_updates": true,
+            "timeout_seconds": 120,
         });
         if let Some(amt) = amount_sats {
             pay_body["amt"] = serde_json::json!(amt.to_string());
         }
 
-        // `/v1/channels/transactions` is SYNCHRONOUS: it blocks until the
+        // LND 0.21 removed the deprecated Lightning.SendPaymentSync REST route
-        // payment settles or definitively fails, and multi-hop routing with
+        // (`/v1/channels/transactions`). Router.SendPaymentV2 is its supported
-        // retries routinely takes longer than the shared client's 15s budget.
+        // replacement. The old route now returns literal 404 "Not Found" on
-        // That 15s abort used to surface as "Payment failed" while LND kept
+        // every payment — the fleet failure seen immediately after the 0.21.2
-        // paying in the background — only LND may declare a payment failed,
+        // update. Keep the short browser-facing wait: after LND accepts a slow
-        // so a post-connect timeout is IN FLIGHT (status: pending), never
+        // payment we return pending and the UI follows it through
-        // failure. The window is deliberately SHORT: most payments settle in
+        // lnd.paymentstatus instead of declaring a transport timeout a failure.
-        // a couple of seconds and still get their answer in one round trip,
-        // while a slow multi-hop route flips the UI into its "settling…"
-        // polling state (lnd.paymentstatus every 3s) after ~8s instead of
-        // freezing the modal for two minutes with no feedback (a test node
-        // user report, 2026-07-29).
         let pay_client = reqwest::Client::builder()
             .no_proxy()
             .connect_timeout(std::time::Duration::from_secs(10))
@@ -91,7 +128,7 @@ impl RpcHandler {
             .context("Failed to create HTTP client")?;
 
         let resp = match pay_client
-            .post(format!("{LND_REST_BASE_URL}/v1/channels/transactions"))
+            .post(format!("{LND_REST_BASE_URL}/v2/router/send"))
             .header("Grpc-Metadata-macaroon", &macaroon_hex)
             .json(&pay_body)
             .send()
@@ -119,49 +156,42 @@ impl RpcHandler {
         let body: serde_json::Value = resp
             .json()
             .await
-            .context("Failed to parse payment response")?;
+            .context("Failed to parse Router.SendPaymentV2 response")?;
 
+        // grpc-gateway wraps server-streaming records as {"result": ...} and
+        // transport/RPC failures as {"error": {"message": ...}}. Do not look
+        // only for the old endpoint's top-level `message`: that turns useful
+        // LND errors into "Unknown error".
         if !status.is_success() {
-            let msg = body
+            let msg = router_error_message(&body).unwrap_or("Unknown error");
-                .get("message")
+            return Err(payment_error(msg));
-                .and_then(|v| v.as_str())
+        }
-                .unwrap_or("Unknown error");
+        let payment = body.get("result").unwrap_or(&body);
-            // Invoices are short-lived; retrying the same one can never
+        match payment.get("status").and_then(|v| v.as_str()).unwrap_or("") {
-            // succeed, so tell the user the way out instead of just the fact.
+            "SUCCEEDED" => {}
-            if msg.contains("invoice expired") {
+            "FAILED" => {
-                return Err(anyhow::anyhow!(
+                let reason = payment
-                    "Payment failed: this invoice has expired ({}). Ask the recipient for a fresh invoice and try again.",
+                    .get("failure_reason")
-                    msg.trim_start_matches("invoice expired. ")
+                    .and_then(|v| v.as_str())
-                ));
+                    .map(payment_failure_reason)
+                    .unwrap_or("Payment failed");
+                return Err(anyhow::anyhow!("Payment failed: {reason}"));
+            }
+            _ => {
+                return Ok(serde_json::json!({
+                    "status": "pending",
+                    "payment_hash": decoded_hash,
+                    "amount_sats": decoded_amt,
+                }));
             }
-            return Err(anyhow::anyhow!("Payment failed: {}", msg));
         }
 
-        let payment_error = body
+        let amount_sat = json_i64(payment, "value_sat").unwrap_or(decoded_amt);
-            .get("payment_error")
-            .and_then(|v| v.as_str())
-            .unwrap_or("");
-        if !payment_error.is_empty() {
-            return Err(anyhow::anyhow!("Payment failed: {}", payment_error));
-        }
-
-        let amount_sat = body
-            .get("payment_route")
-            .and_then(|r| r.get("total_amt"))
-            .and_then(|v| v.as_str())
-            .and_then(|s| s.parse::<i64>().ok())
-            .unwrap_or(decoded_amt);
-
-        let payment_hash = body
-            .get("payment_hash")
-            .and_then(|v| v.as_str())
-            .filter(|s| !s.is_empty())
-            .map(|s| s.to_string())
-            .unwrap_or(decoded_hash);
-
         Ok(serde_json::json!({
             "status": "succeeded",
-            "payment_hash": payment_hash,
+            // The decode endpoint returns the canonical hex hash used by our
+            // polling/list APIs. Router's bytes field is base64 in REST JSON.
+            "payment_hash": decoded_hash,
             "amount_sats": amount_sat,
         }))
     }
@@ -482,3 +512,42 @@ impl RpcHandler {
         Ok(serde_json::json!({ "transactions": transactions }))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unwraps_grpc_gateway_router_success() {
+        let body = serde_json::json!({
+            "result": { "status": "SUCCEEDED", "value_sat": "1000" }
+        });
+        let payment = body.get("result").unwrap_or(&body);
+        assert_eq!(
+            payment.get("status").and_then(|v| v.as_str()),
+            Some("SUCCEEDED")
+        );
+        assert_eq!(json_i64(payment, "value_sat"), Some(1000));
+    }
+
+    #[test]
+    fn reads_nested_router_error() {
+        let body = serde_json::json!({
+            "error": { "code": 2, "message": "invoice expired. valid until yesterday" }
+        });
+        let msg = router_error_message(&body).unwrap();
+        assert!(payment_error(msg).to_string().contains("fresh invoice"));
+    }
+
+    #[test]
+    fn router_failure_reasons_are_actionable() {
+        assert_eq!(
+            payment_failure_reason("FAILURE_REASON_NO_ROUTE"),
+            "No route to the recipient"
+        );
+        assert_eq!(
+            payment_failure_reason("FAILURE_REASON_INSUFFICIENT_BALANCE"),
+            "Insufficient channel balance"
+        );
+    }
+}

tests/lifecycle/bats/lnd-api-compat.bats

@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+# tests/lifecycle/bats/lnd-api-compat.bats
+#
+# Regression guard for the 2026-09-01 fleet breakage: LND 0.21 REMOVED the
+# deprecated Lightning.SendPaymentSync REST route (`/v1/channels/transactions`)
+# that the backend paid through — every Lightning send answered the literal
+# HTTP 404 "Not Found" and the wallet UI showed "Payment failed: Not Found".
+# The backend now pays via Router.SendPaymentV2 (`/v2/router/send`).
+#
+# This test does not send sats. It POSTs a deliberately-invalid invoice to the
+# v2 route on the RUNNING LND and asserts the route itself answers: a
+# 400/500 "cannot parse" proves the endpoint exists; a 404 means the pinned
+# image no longer serves the route the backend calls — the exact image/backend
+# skew that shipped silently last time because no gate test ever spoke the
+# payment endpoint.
+#
+# Tiers: read-only (invalid payment request; nothing is sent).
+#
+# Runs on the archy host (sudo for the macaroon, curl to localhost).
+
+LND_MAINNET_DIR="/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet"
+
+_lnd_rest_host_port() {
+  local mf
+  for mf in \
+    "${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yml" \
+    "${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yaml" \
+    "$BATS_TEST_DIRNAME/../../../apps/lnd/manifest.yml"; do
+    [[ -r "$mf" ]] || continue
+    awk '
+      /- host:/        { host=$3 }
+      /container:/     { if ($2 == 8080 && host != "") { print host; exit } }
+    ' "$mf"
+    return 0
+  done
+}
+
+@test "running LND serves /v2/router/send (the route the backend pays through)" {
+  if ! podman ps --format '{{.Names}}' 2>/dev/null | grep -qx lnd; then
+    skip "lnd not running"
+  fi
+  local port
+  port=$(_lnd_rest_host_port)
+  [[ -n "$port" ]] || skip "could not resolve LND REST host port from manifest"
+
+  local mac
+  mac=$(sudo cat "$LND_MAINNET_DIR/admin.macaroon" 2>/dev/null | od -An -tx1 -v | tr -d " \n")
+  [[ -n "$mac" ]] || skip "LND admin macaroon not readable (LND installed but wallet not initialized?)"
+
+  local code body
+  body=$(mktemp)
+  code=$(curl -sk -o "$body" -w '%{http_code}' --max-time 10 -X POST \
+    -H "Grpc-Metadata-macaroon: $mac" \
+    -H 'Content-Type: application/json' \
+    --data '{"payment_request":"lnbc1notarealinvoice","timeout_seconds":5,"no_inflight_updates":true}' \
+    "https://127.0.0.1:${port}/v2/router/send" || echo 000)
+  rm -f "$body"
+
+  # 000 = LND REST unreachable at all — that is port-drift's failure class
+  # (port-drift.bats), but it also breaks payments, so fail loudly here too.
+  if [[ "$code" == "000" ]]; then
+    fail "LND REST not reachable on ${port} — payments cannot be sent at all"
+  fi
+  if [[ "$code" == "404" ]]; then
+    fail "running LND does not serve /v2/router/send (HTTP 404) — the backend's payment route is gone; every Lightning send fails 'Not Found'"
+  fi
+}
+
+@test "backend no longer references the removed /v1/channels/transactions route" {
+  # Source-level guard: the removed route must not creep back into the
+  # payment path (the runtime fix is in api/rpc/lnd/payments.rs).
+  local src="$BATS_TEST_DIRNAME/../../../core/archipelago/src/api/rpc/lnd/payments.rs"
+  [[ -r "$src" ]] || skip "source tree not present"
+  if grep -q 'v1/channels/transactions' "$src"; then
+    fail "payments.rs references /v1/channels/transactions — removed in LND 0.21, answers 404"
+  fi
+}

tests/lifecycle/remote-lifecycle.sh

@@ -136,7 +136,7 @@ image_for() {
     bitcoin-knots) echo "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest" ;;
     bitcoin-core) echo "docker.io/bitcoin/bitcoin:28.4" ;;
     btcpay-server) echo "docker.io/btcpayserver/btcpayserver:2.4.2" ;;
-    lnd) echo "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta" ;;
+    lnd) echo "source.archipelago-foundation.org/lfg2025/lnd:v0.21.2-beta" ;;
     mempool) echo "source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.0.0" ;;
     homeassistant) echo "source.archipelago-foundation.org/lfg2025/home-assistant:2024.1" ;;
     grafana) echo "source.archipelago-foundation.org/lfg2025/grafana:10.2.0" ;;