fix(wallet): prioritize LND boot and reject unavailable balances

This commit is contained in:
archipelago
2026-09-15 15:09:08 -04:00
parent 4302138b4f
commit 4237fb5e79
7 changed files with 284 additions and 175 deletions
+129 -51
View File
@@ -73,6 +73,42 @@ struct LndChannelBalanceResponse {
pending_open_local_balance: Option<LndAmount>,
}
/// Reject unavailable LND data before it can be decoded as an empty, zero wallet.
async fn get_lnd_json<T: serde::de::DeserializeOwned>(
client: &reqwest::Client,
url: &str,
macaroon_hex: &str,
) -> Result<T> {
client
.get(url)
.header("Grpc-Metadata-macaroon", macaroon_hex)
.send()
.await
.context("LND is unavailable; balance could not be checked")?
.error_for_status()
.context("LND is not ready; balance could not be checked")?
.json()
.await
.context("LND returned invalid wallet data")
}
fn checked_balances(
wallet: LndBalanceResponse,
channels: LndChannelBalanceResponse,
) -> Result<(i64, i64, i64)> {
fn sats(value: Option<String>) -> Result<i64> {
let value = value.context("LND omitted a balance; balance is unavailable")?;
let amount: i64 = value.parse().context("LND returned an invalid balance")?;
anyhow::ensure!(amount >= 0, "LND returned a negative balance");
Ok(amount)
}
Ok((
sats(wallet.total_balance)?,
sats(channels.local_balance.and_then(|a| a.sat))?,
sats(channels.pending_open_local_balance.and_then(|a| a.sat))?,
))
}
impl RpcHandler {
pub(in crate::api::rpc) async fn handle_lnd_getinfo(&self) -> Result<serde_json::Value> {
let macaroon_bytes = read_lnd_admin_macaroon().await?;
@@ -85,45 +121,26 @@ impl RpcHandler {
.build()
.context("Failed to create HTTP client")?;
let get_info: LndGetInfoResponse = client
.get(format!("{LND_REST_BASE_URL}/v1/getinfo"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
.context("LND REST connection failed")?
.json()
.await
.context("Failed to parse LND getinfo response")?;
let channel_balance: LndChannelBalanceResponse = match client
.get(format!("{LND_REST_BASE_URL}/v1/balance/channels"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
{
Ok(resp) => resp.json().await.unwrap_or(LndChannelBalanceResponse {
local_balance: None,
pending_open_local_balance: None,
}),
Err(_) => LndChannelBalanceResponse {
local_balance: None,
pending_open_local_balance: None,
},
};
let wallet_balance: LndBalanceResponse = match client
.get(format!("{LND_REST_BASE_URL}/v1/balance/blockchain"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
{
Ok(resp) => resp.json().await.unwrap_or(LndBalanceResponse {
total_balance: None,
}),
Err(_) => LndBalanceResponse {
total_balance: None,
},
};
let get_info: LndGetInfoResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/getinfo"),
&macaroon_hex,
)
.await?;
let channel_balance: LndChannelBalanceResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/balance/channels"),
&macaroon_hex,
)
.await?;
let wallet_balance: LndBalanceResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/balance/blockchain"),
&macaroon_hex,
)
.await?;
let (balance_sats, channel_balance_sats, pending_open_balance) =
checked_balances(wallet_balance, channel_balance)?;
let (identity_pubkey, uris) = map_identity(&get_info);
@@ -135,18 +152,9 @@ impl RpcHandler {
num_peers: get_info.num_peers.unwrap_or(0),
synced_to_chain: get_info.synced_to_chain.unwrap_or(false),
block_height: get_info.block_height.unwrap_or(0),
balance_sats: wallet_balance
.total_balance
.and_then(|s| s.parse().ok())
.unwrap_or(0),
channel_balance_sats: channel_balance
.local_balance
.and_then(|a| a.sat.and_then(|s| s.parse().ok()))
.unwrap_or(0),
pending_open_balance: channel_balance
.pending_open_local_balance
.and_then(|a| a.sat.and_then(|s| s.parse().ok()))
.unwrap_or(0),
balance_sats,
channel_balance_sats,
pending_open_balance,
};
Ok(serde_json::to_value(info)?)
@@ -268,6 +276,76 @@ impl RpcHandler {
mod tests {
use super::*;
#[test]
fn unavailable_balances_are_not_zero() {
for body in [r#"{}"#, r#"{"code":14,"message":"wallet locked"}"#] {
assert!(checked_balances(
serde_json::from_str(body).unwrap(),
serde_json::from_str(body).unwrap(),
)
.is_err());
}
for value in ["bad", "-1", "9223372036854775808"] {
let wallet = LndBalanceResponse {
total_balance: Some(value.into()),
};
let channels = serde_json::from_str(
r#"{"local_balance":{"sat":"5"},"pending_open_local_balance":{"sat":"0"}}"#,
)
.unwrap();
assert!(checked_balances(wallet, channels).is_err());
}
}
#[test]
fn verified_zero_and_nonzero_balances_survive() {
for expected in [0, 42] {
let wallet = LndBalanceResponse {
total_balance: Some(expected.to_string()),
};
let channels = serde_json::from_value(serde_json::json!({
"local_balance":{"sat":expected.to_string()},
"pending_open_local_balance":{"sat":"0"}
}))
.unwrap();
assert_eq!(
checked_balances(wallet, channels).unwrap(),
(expected, expected, 0)
);
}
}
#[tokio::test]
async fn locked_wallet_http_response_is_not_successful_getinfo() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let mut buf = [0; 2048];
stream.read(&mut buf).await.unwrap();
let body =
r#"{"code":9,"message":"wallet locked, unlock it to enable full RPC access"}"#;
stream.write_all(format!(
"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(), body
).as_bytes()).await.unwrap();
});
let client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(2))
.build()
.unwrap();
assert!(get_lnd_json::<LndGetInfoResponse>(
&client,
&format!("http://{addr}/v1/getinfo"),
"test"
)
.await
.is_err());
server.await.unwrap();
}
/// A real compressed secp256k1 pubkey shape: 66 hex characters.
const GOOD_PUBKEY: &str = "03a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
+7 -121
View File
@@ -96,129 +96,21 @@ pub async fn ensure_wallet_initialized() -> Result<()> {
if file_exists_as_root(admin_macaroon).await && lnd_getinfo_ready(admin_macaroon).await {
return Ok(());
}
match unlock_existing_wallet().await? {
true => {
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
false => {
// Every candidate password was actively rejected: this wallet was
// created with a password this node no longer has, so it can never
// auto-unlock unattended. Alpha nodes hold no real funds and a wallet
// locked with an unknown password is already inaccessible, so wipe +
// recreate it on the per-node secret to self-heal at boot.
recreate_wallet_destructively().await?;
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
}
unlock_existing_wallet_no_wipe().await?;
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
init_wallet_via_rest().await?;
wait_for_admin_macaroon(admin_macaroon).await
}
/// LND data subdirectories holding wallet + channel + graph state. Removing them
/// returns LND to a NON_EXISTING wallet state. Funds-bearing data lives here too,
/// so deletion is destructive — only done once the wallet is already unrecoverable.
const LND_STATE_DIRS: &[&str] = &[
"/var/lib/archipelago/lnd/data/chain",
"/var/lib/archipelago/lnd/data/graph",
];
/// Podman container name for the core LND app (see `compute_container_name`:
/// non-UI core apps keep their bare id). LND runs as a plain bridge-network
/// container, not a Quadlet unit, so it is restarted via `podman`, not systemctl.
const LND_CONTAINER: &str = "lnd";
/// Canonical on-host admin macaroon — same path the RPC layer reads.
const LND_ADMIN_MACAROON: &str =
"/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
/// Archipelago data dir (default; not overridden in prod). Holds the
/// `user-stopped.json` that gates health-monitor auto-restart.
const ARCHY_DATA_DIR: &str = "/var/lib/archipelago";
/// Destroy an unrecoverable LND wallet and recreate a fresh one keyed to the
/// per-node secret. Suppresses health-monitor auto-restart for the wipe window,
/// stops LND, deletes its wallet/chain/graph state as root, restarts it, waits
/// for NON_EXISTING, then inits a fresh wallet. Destructive — only called when no
/// candidate password can open the existing wallet.
async fn recreate_wallet_destructively() -> Result<()> {
tracing::warn!(
"[lnd] wallet is locked with an unknown password and cannot auto-unlock; \
wiping and recreating it on the per-node secret (DESTRUCTIVE)"
);
// The health monitor restarts any container it sees stopped; mark LND
// user-stopped so it doesn't re-launch (and re-open the wallet) mid-wipe.
// Always cleared below so LND auto-recovers normally afterwards.
let data_dir = std::path::Path::new(ARCHY_DATA_DIR);
crate::crash_recovery::mark_user_stopped(data_dir, LND_CONTAINER).await;
let result = wipe_and_reinit_wallet().await;
crate::crash_recovery::clear_user_stopped(data_dir, LND_CONTAINER).await;
result
}
async fn wipe_and_reinit_wallet() -> Result<()> {
podman_user_scoped(&["stop", LND_CONTAINER])
.await
.context("stopping lnd before wallet wipe")?;
for dir in LND_STATE_DIRS {
let status = host_sudo(&["rm", "-rf", dir])
.await
.with_context(|| format!("removing {dir}"))?;
if !status.success() {
anyhow::bail!("removing {dir} exited with {status}");
}
}
podman_user_scoped(&["start", LND_CONTAINER])
.await
.context("restarting lnd after wallet wipe")?;
wait_for_wallet_state("NON_EXISTING").await?;
init_wallet_via_rest().await
}
/// Run `podman <args>` inside a transient `systemd-run --user --scope`, matching
/// how the orchestrator/health-monitor manage rootless containers (keeps the
/// container out of the archipelago service's cgroup).
async fn podman_user_scoped(args: &[&str]) -> Result<()> {
let out = tokio::process::Command::new("systemd-run")
.args(["--user", "--scope", "--quiet", "--collect", "podman"])
.args(args)
.output()
.await
.with_context(|| format!("systemd-run --user --scope podman {}", args.join(" ")))?;
if !out.status.success() {
anyhow::bail!(
"podman {} failed: {}",
args.join(" "),
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(())
}
/// Poll `/v1/state` until LND reports `target`, or time out after ~120s.
async fn wait_for_wallet_state(target: &str) -> Result<()> {
let client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(5))
.danger_accept_invalid_certs(true)
.build()
.context("building LND REST client")?;
for _ in 0..120 {
if wallet_state(&client).await.as_deref() == Some(target) {
return Ok(());
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!("LND did not reach state {target} after wallet wipe")
}
async fn file_exists_as_root(path: &str) -> bool {
if std::path::Path::new(path).exists() {
return true;
@@ -390,14 +282,8 @@ async fn unlock_existing_wallet_via_rest() -> Result<bool> {
)
}
/// Unlock an existing wallet WITHOUT the destructive fallback.
///
/// `ensure_wallet_initialized` wipes and recreates a wallet no candidate
/// password can open — correct for a boot path that must self-heal, and exactly
/// wrong for macaroon rotation, which restarts LND against a wallet the operator
/// still wants. Rotation calls this instead, so there is no code path from
/// "rotate my credentials" to "delete my wallet": a rejected password surfaces
/// as an error the caller reports, never as a wipe.
/// Unlock the existing wallet, preserving its identity and channel data when
/// passwords are unavailable or rejected. Used by boot and credential rotation.
pub(crate) async fn unlock_existing_wallet_no_wipe() -> Result<()> {
match unlock_existing_wallet().await? {
true => Ok(()),
@@ -538,7 +424,7 @@ async fn init_wallet_via_rest() -> Result<()> {
{
UnlockerResponse::Value(seed) => seed,
UnlockerResponse::WalletAlreadyExists => {
unlock_existing_wallet().await?;
unlock_existing_wallet_no_wipe().await?;
return Ok(());
}
};
@@ -569,7 +455,7 @@ async fn init_wallet_via_rest() -> Result<()> {
.await;
}
UnlockerResponse::WalletAlreadyExists => {
unlock_existing_wallet().await?;
unlock_existing_wallet_no_wipe().await?;
}
}
@@ -1864,7 +1864,7 @@ impl ProdContainerOrchestrator {
// Durable installation record, consulted alongside the perishable
// `was_running` snapshot for desired-state recovery below.
let installed_apps = crate::crash_recovery::load_installed_apps(&self.data_dir).await;
let (manifests, container_name_by_app_id): (
let (mut manifests, container_name_by_app_id): (
Vec<LoadedManifest>,
std::collections::HashMap<String, String>,
) = {
@@ -1895,6 +1895,15 @@ impl ProdContainerOrchestrator {
.collect();
(filtered, names)
};
// Wallet readiness must not wait behind unrelated image pulls/builds.
// A running LND container can still be locked after boot; its post-start
// hook must run promptly. Reconcile Bitcoin first, then LND, before the
// rest of the catalog. Each app still honors stopped/uninstalled markers.
manifests.sort_by_key(|lm| match lm.manifest.app.id.as_str() {
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => 0,
"lnd" => 1,
_ => 2,
});
// Live container names (any state), for the same recovery check.
let present_containers: std::collections::HashSet<String> = self
.runtime
@@ -6398,6 +6407,42 @@ app:
assert!(cascade_pairs_for_report(&r, &none).is_empty());
}
#[tokio::test]
async fn reconcile_wallet_start_precedes_unrelated_failed_image_pull() {
let rt = Arc::new(MockRuntime::default());
rt.set_state("bitcoin-knots", ContainerState::Exited);
rt.set_state("lnd", ContainerState::Exited);
*rt.fail_pull.lock().unwrap() = Some("registry unreachable".into());
let mut orch = orch_with(rt.clone()).await;
orch.set_disk_gb_for_test(2000);
for id in ["unrelated", "lnd", "bitcoin-knots"] {
orch.insert_manifest_for_test(
pull_manifest(id, &format!("docker.io/example/{id}:1")),
PathBuf::from(format!("/tmp/{id}")),
)
.await;
}
let report = orch.reconcile_all().await;
assert!(report.failures.iter().any(|(id, _)| id == "unrelated"));
let calls = rt.calls();
let bitcoin = calls
.iter()
.position(|c| c == "start_container:bitcoin-knots")
.unwrap();
let lnd = calls
.iter()
.position(|c| c == "start_container:lnd")
.unwrap();
let pull = calls
.iter()
.position(|c| c.starts_with("pull_image:"))
.unwrap();
assert!(
bitcoin < lnd && lnd < pull,
"wallet startup was delayed by unrelated recovery: {calls:?}"
);
}
#[tokio::test]
async fn reconcile_starts_exited_container() {
let rt = Arc::new(MockRuntime::default());
+40 -1
View File
@@ -1,6 +1,6 @@
# Framework: LND startup, missing Receive address, false zero balance
**Status: OPEN — live Framework access required; not fixed.**
**Status: OPEN — Framework inspected live; candidate fix under test; not fixed yet.**
Reported: 2026-09-15. Source inspected: main at `3b9b74da` (v1.8.17-alpha publication).
The Framework's installed version and exact incident time have not been verified.
@@ -131,3 +131,42 @@ tab and response once the user clarifies and the node can be inspected.
2026-09-15: source investigation and persistent session-start instructions only.
No code fix, release, node deployment, or live reproduction for this incident yet.
## Live evidence captured 2026-09-15
Access was provided during the same session. Read-only inspection confirmed:
- Framework runs `1.8.17-alpha-dev`; the current full boot began at 18:40:09 UTC.
- LND opened its databases in 6.7 seconds and requested its wallet password at
18:40:20. It then rejected GetInfo/ChannelBalance/WalletBalance as wallet locked.
- The management service's first sequential reconcile pass was occupied by
unrelated image recovery, including a missing voice image from 18:40:24 and
later a missing Core Lightning image. Manifests are iterated from a HashMap;
wallet readiness has no initial priority. Boot recovery itself completed at
18:40:18; the first full app-reconcile report appeared at 18:44:34.
- The user's manual LND restart was recorded at 18:42:33. The replacement LND
process started at 18:42:40, requested its password at 18:43:05, and unlocked
at 18:43:07 through the explicit restart hook. This supports delayed unlock
behind unrelated recovery, rather than a missing wallet or bad password.
- At inspection, `/v1/state` reports SERVER_ACTIVE; getinfo reports chain and
graph sync and two active channels. Both authenticated balance endpoints
report nonzero balances. No wallet-recreation event was found in captured logs.
- The Minibits RPC separately fails with “The ecash wallet has no seed yet”.
`wallet/cashu_seed.json` and `wallet/minibits.json` are absent. The existing
ecash wallet is present with proofs and an August modification timestamp.
Do not overwrite it or generate an unrelated recovery identity. Still identify
which Receive item the user meant before declaring this part repaired.
Private raw evidence: `/home/archipelago/.local/state/archy-incidents/framework-lnd-20260915/`.
Files have mode 0600 and the directory 0700. Do not commit or publish raw logs.
Candidate changes on `investigate/framework-lnd-startup`:
- Run Bitcoin and LND reconciliation before unrelated image pulls/builds.
- Reject failed/incomplete LND balance responses instead of manufacturing zeros.
- Preserve known Home balances on invalid responses, visibly label unavailable
balances, and clear the warning after a successful refresh.
- Remove automatic destructive wallet recreation; failed unlock preserves data.
- Add backend outage/zero/ordering regressions and UI failure/recovery coverage.
These changes are not yet deployed or verified through a Framework reboot.
+14 -1
View File
@@ -133,6 +133,7 @@
class="order-2 lg:order-none"
:animate="animateCards"
:wallet-connected="walletConnected"
:wallet-balance-unavailable="walletBalanceUnavailable"
:wallet-onchain="walletOnchain"
:wallet-lightning="walletLightning"
:wallet-ecash="walletEcash"
@@ -685,6 +686,7 @@ async function devFaucet() { try { await rpcClient.call({ method: 'dev.faucet',
// readout instead; a rail only becomes a number when a call actually
// succeeds, so a real 0 is still a real 0.
const walletConnected = ref(false)
const walletBalanceUnavailable = ref(false)
const walletOnchain = ref<number | null>(null)
const walletLightning = ref<number | null>(null)
const walletEcash = ref<number | null>(null)
@@ -775,13 +777,24 @@ async function loadWeb5Status() {
// call, which is what makes the card feel like an app launch.
const balances = Promise.allSettled([
rpcClient.call<{ balance_sats: number; channel_balance_sats: number }>({ method: 'lnd.getinfo', timeout: 5000, dedup: true })
.then(res => { walletOnchain.value = res.balance_sats || 0; walletLightning.value = res.channel_balance_sats || 0; walletConnected.value = true; walletInfoFailures = 0 })
.then(res => {
if (!Number.isSafeInteger(res.balance_sats) || res.balance_sats < 0 ||
!Number.isSafeInteger(res.channel_balance_sats) || res.channel_balance_sats < 0) {
throw new Error('LND balance is unavailable')
}
walletOnchain.value = res.balance_sats
walletLightning.value = res.channel_balance_sats
walletConnected.value = true
walletBalanceUnavailable.value = false
walletInfoFailures = 0
})
.catch(() => {
// A single slow poll must NOT flip the card to "disconnected" and
// hide balances the user already knows — busy nodes routinely blow
// the 5s budget mid-payment or during IO storms (a test node user
// report: balances vanished while a payment settled). Only call it
// disconnected after three consecutive failures (~30s of silence).
walletBalanceUnavailable.value = true
walletInfoFailures += 1
if (walletInfoFailures >= 3) walletConnected.value = false
}),
@@ -238,6 +238,47 @@ describe('Home tab cache (Task 2): system/update/storage groups + wallet freshne
wrapper.unmount()
})
it.each(['failure', 'missing', 'partial'])('preserves known balances during %s and clears the warning on recovery', async (failure) => {
const wrapper = mountHomeHost()
await settle()
const home = wrapper.findComponent(Home)
const refresh = () => (home.vm as unknown as { loadWeb5Status: () => Promise<void> }).loadWeb5Status()
rpcCallMock.mockImplementationOnce(async () => {
if (failure === 'failure') throw new Error('wallet locked')
return failure === 'partial' ? { balance_sats: 0 } : {}
})
await refresh()
await settle()
const card = wrapper.findComponent(HomeWalletCard)
expect(card.props('walletOnchain')).toBe(5000)
expect(card.props('walletLightning')).toBe(2500)
expect(card.find('[data-testid="wallet-balance-unavailable"]').text()).toContain('last known')
const snapshot = JSON.parse(localStorage.getItem('archy-wallet-snapshot-v1')!)
expect(snapshot.onchain).toBe(5000)
expect(snapshot.lightning).toBe(2500)
rpcCallMock.mockImplementationOnce(async () => ({ balance_sats: 0, channel_balance_sats: 0, synced_to_chain: true }))
await refresh()
await settle()
expect(card.props('walletOnchain')).toBe(0)
expect(card.props('walletLightning')).toBe(0)
expect(card.find('[data-testid="wallet-balance-unavailable"]').exists()).toBe(false)
wrapper.unmount()
})
it('shows unknown rather than zero when the first LND request fails', async () => {
rpcCallMock.mockImplementation(async (request) => {
if (request.method === 'lnd.getinfo') throw new Error('wallet locked')
return defaultRpcCallImpl(request)
})
const wrapper = mountHomeHost()
await settle()
const card = wrapper.findComponent(HomeWalletCard)
expect(card.props('walletOnchain')).toBeNull()
expect(card.props('walletLightning')).toBeNull()
expect(card.find('[data-testid="wallet-balance-unavailable"]').text()).toContain('unavailable')
wrapper.unmount()
})
it('no sessionStorage key exists for the wallet resource after a mount and reactivation cycle', async () => {
const wrapper = mountHomeHost()
await settle()
@@ -54,6 +54,12 @@
</div>
</div>
<p v-if="walletBalanceUnavailable" data-testid="wallet-balance-unavailable" class="text-sm text-amber-200 mb-3" role="status">
{{ walletOnchain != null || walletLightning != null
? 'Bitcoin and Lightning balances could not be refreshed. Showing last known amounts.'
: 'Bitcoin and Lightning balances are unavailable while the wallet starts or reconnects.' }}
</p>
<!-- Incoming Transactions Panel -->
<transition name="incoming-tx-slide">
<div v-if="showIncomingTxPanel && incomingTransactions.length > 0" class="mb-4 rounded-xl overflow-hidden border border-green-500/20">
@@ -221,6 +227,7 @@ export interface WalletTransaction {
const props = defineProps<{
animate: boolean
walletConnected: boolean
walletBalanceUnavailable?: boolean
// `null` = not loaded yet, `0` = genuinely empty. Keeping those apart is
// what lets the card show a pixel readout instead of claiming a figure.
walletOnchain: number | null