From 0b2c36f095e159f377ea096ed5e853da0bfc092f Mon Sep 17 00:00:00 2001 From: ssmithx Date: Sat, 1 Aug 2026 19:28:25 +0000 Subject: [PATCH] fix(bitcoin): stop writing a datadir bitcoin.conf that conflicts with -conf=/tmp/rpc.conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since a597c1d9 (bitcoind RPC creds off argv), bitcoin-core and bitcoin-knots launch bitcoind with -conf=/tmp/rpc.conf and pass all other settings as CLI args; bitcoind never reads /var/lib/archipelago/bitcoin/bitcoin.conf again. write_bitcoin_conf, ensure_bitcoin_rpc_config, and bootstrap's run_bitcoin_rpc_repair were never updated to match — they kept writing/ "repairing" server=/rpcbind=/rpcallowip=/listen= into that datadir file on every install, reinstall, and service restart. Bitcoin Core's own datadir-conflict safety check then refuses to start whenever that file exists alongside an explicit -conf= arg, so the write and every repair of it directly caused the crash it was trying to prevent. Also drop the "restart already-running container after bitcoin.conf repair" adoption-path branch: it assumed bind settings live in that file and needs a restart to pick them up, which hasn't been true since a597c1d9 — the running container's CLI args are already correct. Replaces both writers with remove_stale_bitcoin_conf(), which renames (not deletes) any leftover file so already-affected nodes self-heal on next install/restart instead of staying permanently broken. bitcoin_data_volume_gb is removed as dead code (it only fed the deleted prune= line in write_bitcoin_conf, itself unused since a597c1d9 hardcoded -prune=550 in the manifest's small-disk branch). Investigated after a crash loop on archy-x250-beta; full incident timeline and patch rationale in bitcoin-conf-crash-patch.md. --- .../src/api/rpc/package/install.rs | 244 ++++-------------- core/archipelago/src/bootstrap.rs | 52 ++-- 2 files changed, 69 insertions(+), 227 deletions(-) diff --git a/core/archipelago/src/api/rpc/package/install.rs b/core/archipelago/src/api/rpc/package/install.rs index 589b0b09..0261a091 100644 --- a/core/archipelago/src/api/rpc/package/install.rs +++ b/core/archipelago/src/api/rpc/package/install.rs @@ -307,19 +307,24 @@ impl RpcHandler { let deps = self.gate_install_deps(package_id).await?; check_bitcoin_pruning_compatibility(package_id).await?; log_optional_dep_info(package_id, &deps); - let repaired_bitcoin_conf = - if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") { - // Materialise the RPC password file before any install path - // runs. The orchestrator path resolves secret_env from - // /var/lib/archipelago/secrets/bitcoin-rpc-password at start - // time; if the file is missing, bitcoind exits within ms. - // bitcoin_rpc_credentials() generates + persists on first - // call (OnceCell-cached), so this is idempotent. - let _ = crate::bitcoin_rpc::bitcoin_rpc_credentials().await; - ensure_bitcoin_rpc_config().await? - } else { - false - }; + if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") { + // Materialise the RPC password file before any install path + // runs. The orchestrator path resolves secret_env from + // /var/lib/archipelago/secrets/bitcoin-rpc-password at start + // time; if the file is missing, bitcoind exits within ms. + // bitcoin_rpc_credentials() generates + persists on first + // call (OnceCell-cached), so this is idempotent. + let _ = crate::bitcoin_rpc::bitcoin_rpc_credentials().await; + // A stale datadir bitcoin.conf from an older install conflicts + // with the container's -conf=/tmp/rpc.conf launch (see + // apps/bitcoin-core & bitcoin-knots manifest.yml) and makes + // Bitcoin Core refuse to start at all. Clear it before + // (re)install. Unlike the old bind-setting "repair" this was + // replacing, it never requires restarting an already-running + // container — bitcoind doesn't read this file, so removing it + // changes nothing at runtime. + remove_stale_bitcoin_conf().await?; + } // For orchestrator-managed apps, skip the legacy "container exists → // adopt + return" probe entirely. The orchestrator's own install path @@ -389,37 +394,7 @@ impl RpcHandler { .trim() .to_string(); - if state == "running" && repaired_bitcoin_conf { - info!( - "Restarting existing container {} after bitcoin.conf RPC repair", - package_id - ); - let restart_output = tokio::process::Command::new("podman") - .args(["restart", package_id]) - .output() - .await - .context( - "Failed to restart existing container after bitcoin.conf repair", - )?; - if !restart_output.status.success() { - let stderr = String::from_utf8_lossy(&restart_output.stderr); - install_log(&format!( - "INSTALL ADOPT FAIL: {} - restart after RPC repair failed: {}", - package_id, stderr - )) - .await; - return Err(anyhow::anyhow!( - "Container {} exists but failed to restart after RPC repair: {}", - package_id, - stderr - )); - } - let _ = tokio::process::Command::new("podman") - .args(["restart", "archy-bitcoin-ui"]) - .output() - .await; - wait_for_adopted_container(package_id, package_id).await?; - } else if state != "running" { + if state != "running" { // Start the stopped/exited container info!("Starting existing container {} (was {})", package_id, state); let start_output = tokio::process::Command::new("podman") @@ -707,9 +682,13 @@ impl RpcHandler { } } - // Pre-install: write config files BEFORE chown (dir is still owned by archipelago user) + // Pre-install: clear a stale datadir bitcoin.conf BEFORE chown (dir is + // still owned by archipelago user). bitcoind is launched with + // -conf=/tmp/rpc.conf (see apps/bitcoin-core & bitcoin-knots + // manifest.yml) and never reads a datadir bitcoin.conf — if one + // exists, Bitcoin Core's own safety check refuses to start at all. if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") { - self.write_bitcoin_conf(&rpc_user, &rpc_pass).await?; + remove_stale_bitcoin_conf().await?; } if package_id == "lnd" { @@ -1418,96 +1397,13 @@ impl RpcHandler { } } - /// Write bitcoin.conf with rpcauth (salted HMAC hash, no plaintext password). - async fn write_bitcoin_conf(&self, rpc_user: &str, rpc_pass: &str) -> Result<()> { - let bitcoin_dir = "/var/lib/archipelago/bitcoin"; - let conf_path = format!("{}/bitcoin.conf", bitcoin_dir); - - // Idempotent: once bitcoin-knots (or a prior install) has started, - // the data dir is chowned into the container's user namespace - // (e.g. UID 100100 on the host) with 700 perms — the archipelago - // daemon can no longer stat or write there. Treat any non-NotFound - // error on the conf as "conf already provisioned by the container - // user" and skip. Matches the lnd.conf behavior below. - match tokio::fs::metadata(&conf_path).await { - Ok(_) => { - ensure_bitcoin_rpc_config().await?; - info!("bitcoin.conf already exists, ensured Bitcoin RPC config"); - return Ok(()); - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - ensure_bitcoin_rpc_config().await?; - info!("bitcoin.conf path inaccessible, ensured Bitcoin RPC config via host helper"); - return Ok(()); - } - } - - use hmac::{Hmac, Mac}; - use sha2::Sha256; - let salt_bytes: [u8; 16] = rand::random(); - let salt_hex = hex::encode(salt_bytes); - let mut mac = Hmac::::new_from_slice(salt_hex.as_bytes()) - .expect("HMAC accepts any key length"); - mac.update(rpc_pass.as_bytes()); - let hash_hex = hex::encode(mac.finalize().into_bytes()); - let rpcauth_line = format!("rpcauth={}:{}${}", rpc_user, salt_hex, hash_hex); - - // Default to full archive — operators with 2TB+ drives shouldn't be - // silently pruned down to 550 MB. Users who want a pruned node can - // set `prune=N` in bitcoin.conf themselves after install. - // - // printtoconsole=0: bitcoind already writes debug.log in the datadir - // (self-shrunk on restart); duplicating it to stdout pushed every IBD - // "UpdateTip" line through conmon into journald (>1 GB/day). Deep - // debugging uses /var/lib/archipelago/bitcoin/debug.log. - // rpcbind=0.0.0.0 is REQUIRED inside a container: with rpcallowip set - // but no rpcbind, bitcoind binds RPC to 127.0.0.1 in the container - // netns only — LND / the Bitcoin UI dialing bitcoin-knots:8332 over - // the bridge get connection refused (fresh-install LND crash-loop + - // bitcoin-rpc 502, seen on the 1.7.99 ISO). The port publish stays - // 127.0.0.1-only on the host, so exposure is unchanged. - // Prune sized to the data volume. A full archive needs ~810 GB and - // grows; silently writing an unpruned config onto a small disk fills - // it mid-IBD (framework node 2026-07-14: unpruned mainnet on a 205 GB - // volume). Volumes with real archival headroom (≥1.2 TB) stay full - // archive; smaller ones get prune = 25% of the volume, clamped to - // [550 MB, 100 GB], leaving room for LND/apps sharing the disk. - let prune_line = match bitcoin_data_volume_gb().await { - Some(total_gb) if total_gb > 0 && total_gb < 1200 => { - let prune_mb = ((total_gb as f64 * 0.25 * 1024.0) as u64).clamp(550, 100_000); - info!( - volume_gb = total_gb, - prune_mb, "Data volume below archival size — enabling sized bitcoin prune" - ); - format!("prune={}\n", prune_mb) - } - _ => String::new(), - }; - - let bitcoin_conf = format!( - "\ -# rpcauth: salted hash only - no plaintext password in config or CLI\n\ -{}\n\ -server=1\n\ -rpcbind=0.0.0.0\n\ -rpcallowip=0.0.0.0/0\n\ -listen=1\n\ -rpcthreads=16\n\ -rpcworkqueue=256\n\ -printtoconsole=0\n\ -{}", - rpcauth_line, prune_line - ); - tokio::fs::create_dir_all(bitcoin_dir) - .await - .context("Failed to create bitcoin data directory")?; - tokio::fs::write(&conf_path, bitcoin_conf) - .await - .context("Failed to write bitcoin.conf")?; - info!("Created bitcoin.conf with rpcauth (no plaintext credentials)"); - Ok(()) - } + // write_bitcoin_conf removed: bitcoind is launched with -conf=/tmp/rpc.conf + // (see apps/bitcoin-core & bitcoin-knots manifest.yml, commit a597c1d9) + // and never reads a datadir bitcoin.conf. Writing one here created a + // fatal "-conf vs default bitcoin.conf" conflict on every subsequent + // start (Bitcoin Core's own datadir-conflict safety check). See + // `remove_stale_bitcoin_conf` below, which replaces both this and + // `ensure_bitcoin_rpc_config`. /// Write LND config file with Bitcoin RPC credentials. async fn write_lnd_conf(&self, rpc_user: &str, rpc_pass: &str) -> Result<()> { @@ -2602,28 +2498,12 @@ async fn wait_for_adopted_container(package_id: &str, container_name: &str) -> R )) } -/// Total size (GB) of the filesystem holding the bitcoin data dir, via -/// `df -k`. None when df fails (containers, exotic mounts) — callers treat -/// unknown as "don't prune" to preserve archival defaults on big iron. -async fn bitcoin_data_volume_gb() -> Option { - let target = if std::path::Path::new("/var/lib/archipelago").exists() { - "/var/lib/archipelago" - } else { - "/" - }; - let output = tokio::process::Command::new("df") - .args(["-k", target]) - .output() - .await - .ok()?; - if !output.status.success() { - return None; - } - let stdout = String::from_utf8_lossy(&output.stdout); - let line = stdout.lines().nth(1)?; - let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; - Some(kb / 1024 / 1024) -} +// bitcoin_data_volume_gb removed with write_bitcoin_conf: it only fed that +// function's volume-aware `prune=` line, which bitcoind never read either +// (see remove_stale_bitcoin_conf). The manifest's shell entrypoint already +// computes DISK_GB_VALUE and hardcodes -prune=550 on small volumes — a +// real volume-aware prune fix belongs there, not in a conf file nothing +// reads. Tracked as follow-up in bitcoin-conf-crash-patch.md. /// One-shot probe: does bitcoind answer an authenticated getblockchaininfo? /// Works during IBD (the call answers with progress while syncing). Goes via @@ -2701,52 +2581,36 @@ async fn wait_for_bitcoin_rpc_gate(package_id: &str) -> Result<()> { Ok(()) } -async fn ensure_bitcoin_rpc_config() -> Result { +/// bitcoind reads only `/tmp/rpc.conf` + CLI args at container start (see +/// apps/bitcoin-core & bitcoin-knots manifest.yml, commit a597c1d9) — it +/// never reads a datadir bitcoin.conf. A leftover file from an older install +/// (or a manual edit) makes Bitcoin Core's own datadir-conflict safety check +/// refuse to start ("-conf=... vs default bitcoin.conf"). Remove it — via +/// the same host-privileged path the old writer/repairer used, since the +/// dir may already be chowned into the container's UID namespace by a +/// previous start — instead of "repairing" it into existence. +async fn remove_stale_bitcoin_conf() -> Result { let script = r#" set -eu conf=/var/lib/archipelago/bitcoin/bitcoin.conf [ -f "$conf" ] || exit 0 -changed=0 -tmp=$(mktemp) -awk -F= ' - /^(server|txindex|rpcbind|rpcallowip|rpcport|listen|bind|dbcache|rpcthreads|rpcworkqueue)=/ { - if (seen[$1]++) next - } - { print } -' "$conf" > "$tmp" -if ! cmp -s "$conf" "$tmp"; then - cat "$tmp" > "$conf" - changed=1 -fi -rm -f "$tmp" -ensure_line() { - line="$1" - key="${line%%=*}" - if ! grep -q "^${key}=" "$conf"; then - printf '%s\n' "$line" >> "$conf" - changed=1 - fi -} -ensure_line server=1 -ensure_line rpcbind=0.0.0.0 -ensure_line rpcallowip=0.0.0.0/0 -ensure_line listen=1 -ensure_line rpcthreads=16 -ensure_line rpcworkqueue=256 -[ "$changed" -eq 0 ] && exit 0 +mv "$conf" "$conf.disabled-$(date +%s)" exit 2 "#; let status = host_sudo(&["sh", "-lc", script]) .await - .context("ensure bitcoin.conf RPC bind settings")?; + .context("remove stale bitcoin.conf")?; match status.code() { Some(0) => Ok(false), Some(2) => { - install_log("INSTALL REPAIR: bitcoin.conf RPC bind settings added").await; + install_log( + "INSTALL REPAIR: removed stale bitcoin.conf (conflicts with -conf=/tmp/rpc.conf launch)", + ) + .await; Ok(true) } _ => Err(anyhow::anyhow!( - "bitcoin.conf RPC repair helper exited with {}", + "bitcoin.conf removal helper exited with {}", status )), } diff --git a/core/archipelago/src/bootstrap.rs b/core/archipelago/src/bootstrap.rs index 2739ff1c..644f806f 100644 --- a/core/archipelago/src/bootstrap.rs +++ b/core/archipelago/src/bootstrap.rs @@ -154,9 +154,9 @@ pub async fn ensure_doctor_installed() { } match run_bitcoin_rpc_repair().await { Ok(true) => { - info!("Repaired Bitcoin RPC bind settings; running Bitcoin containers left untouched") + info!("Removed stale bitcoin.conf; running Bitcoin containers left untouched") } - Ok(false) => debug!("Bitcoin RPC bind settings already usable"), + Ok(false) => debug!("No stale bitcoin.conf found"), Err(e) => warn!("Bitcoin RPC repair failed (non-fatal): {:#}", e), } match run_apps_dir_repair().await { @@ -577,52 +577,30 @@ exit 2 } async fn run_bitcoin_rpc_repair() -> Result { - // Older installs can have a container-owned bitcoin.conf with only rpcauth - // and printtoconsole. Repair it at startup so OTA fixes existing nodes - // without a manual uninstall/reinstall. Bind/port stay in the container - // command line to avoid duplicate RPC endpoint definitions. + // bitcoind is launched with -conf=/tmp/rpc.conf and never reads a + // datadir bitcoin.conf (apps/bitcoin-core & bitcoin-knots manifest.yml, + // commit a597c1d9 — bind/port live only on the container command line). + // A leftover file from an older install makes Bitcoin Core's own + // datadir-conflict safety check refuse to start on every subsequent + // start. Remove it instead of "repairing" it into existence — this + // previously wrote server=/rpcbind=/rpcallowip=/listen= into the file, + // which is exactly what caused the conflict. let script = r#" set -eu conf=/var/lib/archipelago/bitcoin/bitcoin.conf [ -f "$conf" ] || exit 0 -changed=0 -ensure_line() { - line="$1" - key="${line%%=*}" - if ! grep -q "^${key}=" "$conf"; then - printf '%s\n' "$line" >> "$conf" - changed=1 - fi -} -ensure_line server=1 -# rpcbind=0.0.0.0 is required inside the container: with rpcallowip set but -# no rpcbind, bitcoind binds RPC to the container's loopback only and every -# dial over the container network (LND, bitcoin-ui) is refused — the fresh- -# install "LND took 5 attempts" / bitcoin-rpc 502 failure (host publish stays -# 127.0.0.1-only, so exposure is unchanged). -ensure_line rpcbind=0.0.0.0 -ensure_line rpcallowip=0.0.0.0/0 -ensure_line listen=1 -# Log-volume fix: printtoconsole=1 duplicated every log line (incl. per-block -# IBD "UpdateTip" spam) into journald via conmon on top of the datadir -# debug.log bitcoind already writes. Console off; debug.log stays (bitcoind -# self-shrinks it on restart). -if grep -q '^printtoconsole=1' "$conf"; then - sed -i 's/^printtoconsole=1$/printtoconsole=0/' "$conf" - changed=1 -fi -[ "$changed" -eq 0 ] && exit 0 +mv "$conf" "$conf.disabled-$(date +%s)" exit 2 "#; let status = host_sudo(&["sh", "-lc", script]) .await - .context("repair bitcoin.conf RPC bind settings")?; + .context("remove stale bitcoin.conf RPC bind settings")?; match status.code() { Some(0) => Ok(false), // Do not restart Bitcoin from bootstrap. During IBD, an automatic - // restart can cost hours of progress. The repaired file is only a - // fallback for future starts; current containers keep their command-line - // RPC args until an operator or update intentionally restarts them. + // restart can cost hours of progress. Removing the stale file is + // only a fallback for future starts; current containers keep their + // command-line RPC args regardless. Some(2) => Ok(true), _ => { warn!("Bitcoin RPC repair helper exited with {}", status);