lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

fix(wallet): prioritize LND boot and reject unavailable balances

Browse Source
This commit is contained in:
archipelago
2026-09-15 15:09:08 -04:00
parent 4302138b4f
commit 4237fb5e79
7 changed files with 284 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/lnd/info.rs
+129 -51
View File
@@ -73,6 +73,42 @@ struct LndChannelBalanceResponse {
pending_open_local_balance: Option<LndAmount>,
}
/// Reject unavailable LND data before it can be decoded as an empty, zero wallet.
async fn get_lnd_json<T: serde::de::DeserializeOwned>(
client: &reqwest::Client,
url: &str,
macaroon_hex: &str,
) -> Result<T> {
client
.get(url)
.header("Grpc-Metadata-macaroon", macaroon_hex)
.send()
.await
.context("LND is unavailable; balance could not be checked")?
.error_for_status()
.context("LND is not ready; balance could not be checked")?
.json()
.await
.context("LND returned invalid wallet data")
}
fn checked_balances(
wallet: LndBalanceResponse,
channels: LndChannelBalanceResponse,
) -> Result<(i64, i64, i64)> {
fn sats(value: Option<String>) -> Result<i64> {
let value = value.context("LND omitted a balance; balance is unavailable")?;
let amount: i64 = value.parse().context("LND returned an invalid balance")?;
anyhow::ensure!(amount >= 0, "LND returned a negative balance");
Ok(amount)
}
Ok((
sats(wallet.total_balance)?,
sats(channels.local_balance.and_then(|a| a.sat))?,
sats(channels.pending_open_local_balance.and_then(|a| a.sat))?,
))
}
impl RpcHandler {
pub(in crate::api::rpc) async fn handle_lnd_getinfo(&self) -> Result<serde_json::Value> {
let macaroon_bytes = read_lnd_admin_macaroon().await?;
@@ -85,45 +121,26 @@ impl RpcHandler {
.build()
.context("Failed to create HTTP client")?;
let get_info: LndGetInfoResponse = client
.get(format!("{LND_REST_BASE_URL}/v1/getinfo"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
.context("LND REST connection failed")?
.json()
.await
.context("Failed to parse LND getinfo response")?;
let channel_balance: LndChannelBalanceResponse = match client
.get(format!("{LND_REST_BASE_URL}/v1/balance/channels"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
{
Ok(resp) => resp.json().await.unwrap_or(LndChannelBalanceResponse {
local_balance: None,
pending_open_local_balance: None,
}),
Err(_) => LndChannelBalanceResponse {
local_balance: None,
pending_open_local_balance: None,
},
};
let wallet_balance: LndBalanceResponse = match client
.get(format!("{LND_REST_BASE_URL}/v1/balance/blockchain"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
{
Ok(resp) => resp.json().await.unwrap_or(LndBalanceResponse {
total_balance: None,
}),
Err(_) => LndBalanceResponse {
total_balance: None,
},
};
let get_info: LndGetInfoResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/getinfo"),
&macaroon_hex,
)
.await?;
let channel_balance: LndChannelBalanceResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/balance/channels"),
&macaroon_hex,
)
.await?;
let wallet_balance: LndBalanceResponse = get_lnd_json(
&client,
&format!("{LND_REST_BASE_URL}/v1/balance/blockchain"),
&macaroon_hex,
)
.await?;
let (balance_sats, channel_balance_sats, pending_open_balance) =
checked_balances(wallet_balance, channel_balance)?;
let (identity_pubkey, uris) = map_identity(&get_info);
@@ -135,18 +152,9 @@ impl RpcHandler {
num_peers: get_info.num_peers.unwrap_or(0),
synced_to_chain: get_info.synced_to_chain.unwrap_or(false),
block_height: get_info.block_height.unwrap_or(0),
balance_sats: wallet_balance
.total_balance
.and_then(|s| s.parse().ok())
.unwrap_or(0),
channel_balance_sats: channel_balance
.local_balance
.and_then(|a| a.sat.and_then(|s| s.parse().ok()))
.unwrap_or(0),
pending_open_balance: channel_balance
.pending_open_local_balance
.and_then(|a| a.sat.and_then(|s| s.parse().ok()))
.unwrap_or(0),
balance_sats,
channel_balance_sats,
pending_open_balance,
};
Ok(serde_json::to_value(info)?)
@@ -268,6 +276,76 @@ impl RpcHandler {
mod tests {
use super::*;
#[test]
fn unavailable_balances_are_not_zero() {
for body in [r#"{}"#, r#"{"code":14,"message":"wallet locked"}"#] {
assert!(checked_balances(
serde_json::from_str(body).unwrap(),
serde_json::from_str(body).unwrap(),
)
.is_err());
}
for value in ["bad", "-1", "9223372036854775808"] {
let wallet = LndBalanceResponse {
total_balance: Some(value.into()),
};
let channels = serde_json::from_str(
r#"{"local_balance":{"sat":"5"},"pending_open_local_balance":{"sat":"0"}}"#,
)
.unwrap();
assert!(checked_balances(wallet, channels).is_err());
}
}
#[test]
fn verified_zero_and_nonzero_balances_survive() {
for expected in [0, 42] {
let wallet = LndBalanceResponse {
total_balance: Some(expected.to_string()),
};
let channels = serde_json::from_value(serde_json::json!({
"local_balance":{"sat":expected.to_string()},
"pending_open_local_balance":{"sat":"0"}
}))
.unwrap();
assert_eq!(
checked_balances(wallet, channels).unwrap(),
(expected, expected, 0)
);
}
}
#[tokio::test]
async fn locked_wallet_http_response_is_not_successful_getinfo() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let mut buf = [0; 2048];
stream.read(&mut buf).await.unwrap();
let body =
r#"{"code":9,"message":"wallet locked, unlock it to enable full RPC access"}"#;
stream.write_all(format!(
"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(), body
).as_bytes()).await.unwrap();
});
let client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(2))
.build()
.unwrap();
assert!(get_lnd_json::<LndGetInfoResponse>(
&client,
&format!("http://{addr}/v1/getinfo"),
"test"
)
.await
.is_err());
server.await.unwrap();
}
/// A real compressed secp256k1 pubkey shape: 66 hex characters.
const GOOD_PUBKEY: &str = "03a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
core/archipelago/src/container/lnd.rs
+7 -121
View File
@@ -96,129 +96,21 @@ pub async fn ensure_wallet_initialized() -> Result<()> {
if file_exists_as_root(admin_macaroon).await && lnd_getinfo_ready(admin_macaroon).await {
return Ok(());
}
match unlock_existing_wallet().await? {
true => {
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
false => {
// Every candidate password was actively rejected: this wallet was
// created with a password this node no longer has, so it can never
// auto-unlock unattended. Alpha nodes hold no real funds and a wallet
// locked with an unknown password is already inaccessible, so wipe +
// recreate it on the per-node secret to self-heal at boot.
recreate_wallet_destructively().await?;
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
}
unlock_existing_wallet_no_wipe().await?;
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
init_wallet_via_rest().await?;
wait_for_admin_macaroon(admin_macaroon).await
}
/// LND data subdirectories holding wallet + channel + graph state. Removing them
/// returns LND to a NON_EXISTING wallet state. Funds-bearing data lives here too,
/// so deletion is destructive — only done once the wallet is already unrecoverable.
const LND_STATE_DIRS: &[&str] = &[
"/var/lib/archipelago/lnd/data/chain",
"/var/lib/archipelago/lnd/data/graph",
];
/// Podman container name for the core LND app (see `compute_container_name`:
/// non-UI core apps keep their bare id). LND runs as a plain bridge-network
/// container, not a Quadlet unit, so it is restarted via `podman`, not systemctl.
const LND_CONTAINER: &str = "lnd";
/// Canonical on-host admin macaroon — same path the RPC layer reads.
const LND_ADMIN_MACAROON: &str =
"/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
/// Archipelago data dir (default; not overridden in prod). Holds the
/// `user-stopped.json` that gates health-monitor auto-restart.
const ARCHY_DATA_DIR: &str = "/var/lib/archipelago";
/// Destroy an unrecoverable LND wallet and recreate a fresh one keyed to the
/// per-node secret. Suppresses health-monitor auto-restart for the wipe window,
/// stops LND, deletes its wallet/chain/graph state as root, restarts it, waits
/// for NON_EXISTING, then inits a fresh wallet. Destructive — only called when no
/// candidate password can open the existing wallet.
async fn recreate_wallet_destructively() -> Result<()> {
tracing::warn!(
"[lnd] wallet is locked with an unknown password and cannot auto-unlock; \
wiping and recreating it on the per-node secret (DESTRUCTIVE)"
);
// The health monitor restarts any container it sees stopped; mark LND
// user-stopped so it doesn't re-launch (and re-open the wallet) mid-wipe.
// Always cleared below so LND auto-recovers normally afterwards.
let data_dir = std::path::Path::new(ARCHY_DATA_DIR);
crate::crash_recovery::mark_user_stopped(data_dir, LND_CONTAINER).await;
let result = wipe_and_reinit_wallet().await;
crate::crash_recovery::clear_user_stopped(data_dir, LND_CONTAINER).await;
result
}
async fn wipe_and_reinit_wallet() -> Result<()> {
podman_user_scoped(&["stop", LND_CONTAINER])
.await
.context("stopping lnd before wallet wipe")?;
for dir in LND_STATE_DIRS {
let status = host_sudo(&["rm", "-rf", dir])
.await
.with_context(|| format!("removing {dir}"))?;
if !status.success() {
anyhow::bail!("removing {dir} exited with {status}");
}
}
podman_user_scoped(&["start", LND_CONTAINER])
.await
.context("restarting lnd after wallet wipe")?;
wait_for_wallet_state("NON_EXISTING").await?;
init_wallet_via_rest().await
}
/// Run `podman <args>` inside a transient `systemd-run --user --scope`, matching
/// how the orchestrator/health-monitor manage rootless containers (keeps the
/// container out of the archipelago service's cgroup).
async fn podman_user_scoped(args: &[&str]) -> Result<()> {
let out = tokio::process::Command::new("systemd-run")
.args(["--user", "--scope", "--quiet", "--collect", "podman"])
.args(args)
.output()
.await
.with_context(|| format!("systemd-run --user --scope podman {}", args.join(" ")))?;
if !out.status.success() {
anyhow::bail!(
"podman {} failed: {}",
args.join(" "),
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(())
}
/// Poll `/v1/state` until LND reports `target`, or time out after ~120s.
async fn wait_for_wallet_state(target: &str) -> Result<()> {
let client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(5))
.danger_accept_invalid_certs(true)
.build()
.context("building LND REST client")?;
for _ in 0..120 {
if wallet_state(&client).await.as_deref() == Some(target) {
return Ok(());
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!("LND did not reach state {target} after wallet wipe")
}
async fn file_exists_as_root(path: &str) -> bool {
if std::path::Path::new(path).exists() {
return true;
@@ -390,14 +282,8 @@ async fn unlock_existing_wallet_via_rest() -> Result<bool> {
)
}
/// Unlock an existing wallet WITHOUT the destructive fallback.
///
/// `ensure_wallet_initialized` wipes and recreates a wallet no candidate
/// password can open — correct for a boot path that must self-heal, and exactly
/// wrong for macaroon rotation, which restarts LND against a wallet the operator
/// still wants. Rotation calls this instead, so there is no code path from
/// "rotate my credentials" to "delete my wallet": a rejected password surfaces
/// as an error the caller reports, never as a wipe.
/// Unlock the existing wallet, preserving its identity and channel data when
/// passwords are unavailable or rejected. Used by boot and credential rotation.
pub(crate) async fn unlock_existing_wallet_no_wipe() -> Result<()> {
match unlock_existing_wallet().await? {
true => Ok(()),
@@ -538,7 +424,7 @@ async fn init_wallet_via_rest() -> Result<()> {
{
UnlockerResponse::Value(seed) => seed,
UnlockerResponse::WalletAlreadyExists => {
unlock_existing_wallet().await?;
unlock_existing_wallet_no_wipe().await?;
return Ok(());
}
};
@@ -569,7 +455,7 @@ async fn init_wallet_via_rest() -> Result<()> {
.await;
}
UnlockerResponse::WalletAlreadyExists => {
unlock_existing_wallet().await?;
unlock_existing_wallet_no_wipe().await?;
}
}
core/archipelago/src/container/prod_orchestrator.rs
+46 -1
View File
@@ -1864,7 +1864,7 @@ impl ProdContainerOrchestrator {
// Durable installation record, consulted alongside the perishable
// `was_running` snapshot for desired-state recovery below.
let installed_apps = crate::crash_recovery::load_installed_apps(&self.data_dir).await;
let (manifests, container_name_by_app_id): (
let (mut manifests, container_name_by_app_id): (
Vec<LoadedManifest>,
std::collections::HashMap<String, String>,
) = {
@@ -1895,6 +1895,15 @@ impl ProdContainerOrchestrator {
.collect();
(filtered, names)
};
// Wallet readiness must not wait behind unrelated image pulls/builds.
// A running LND container can still be locked after boot; its post-start
// hook must run promptly. Reconcile Bitcoin first, then LND, before the
// rest of the catalog. Each app still honors stopped/uninstalled markers.
manifests.sort_by_key(|lm| match lm.manifest.app.id.as_str() {
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => 0,
"lnd" => 1,
_ => 2,
});
// Live container names (any state), for the same recovery check.
let present_containers: std::collections::HashSet<String> = self
.runtime
@@ -6398,6 +6407,42 @@ app:
assert!(cascade_pairs_for_report(&r, &none).is_empty());
}
#[tokio::test]
async fn reconcile_wallet_start_precedes_unrelated_failed_image_pull() {
let rt = Arc::new(MockRuntime::default());
rt.set_state("bitcoin-knots", ContainerState::Exited);
rt.set_state("lnd", ContainerState::Exited);
*rt.fail_pull.lock().unwrap() = Some("registry unreachable".into());
let mut orch = orch_with(rt.clone()).await;
orch.set_disk_gb_for_test(2000);
for id in ["unrelated", "lnd", "bitcoin-knots"] {
orch.insert_manifest_for_test(
pull_manifest(id, &format!("docker.io/example/{id}:1")),
PathBuf::from(format!("/tmp/{id}")),
)
.await;
}
let report = orch.reconcile_all().await;
assert!(report.failures.iter().any(|(id, _)| id == "unrelated"));
let calls = rt.calls();
let bitcoin = calls
.iter()
.position(|c| c == "start_container:bitcoin-knots")
.unwrap();
let lnd = calls
.iter()
.position(|c| c == "start_container:lnd")
.unwrap();
let pull = calls
.iter()
.position(|c| c.starts_with("pull_image:"))
.unwrap();
assert!(
bitcoin < lnd && lnd < pull,
"wallet startup was delayed by unrelated recovery: {calls:?}"
);
}
#[tokio::test]
async fn reconcile_starts_exited_container() {
let rt = Arc::new(MockRuntime::default());