diff --git a/core/archipelago/src/api/rpc/mod.rs b/core/archipelago/src/api/rpc/mod.rs index 8ecf144c..fc5e1623 100644 --- a/core/archipelago/src/api/rpc/mod.rs +++ b/core/archipelago/src/api/rpc/mod.rs @@ -37,7 +37,7 @@ mod security; mod seed_rpc; mod streaming; mod system; -mod tor; +pub(crate) mod tor; mod totp; mod transitional; mod transport; diff --git a/core/archipelago/src/api/rpc/tor/mod.rs b/core/archipelago/src/api/rpc/tor/mod.rs index 1dab358f..0ec6a611 100644 --- a/core/archipelago/src/api/rpc/tor/mod.rs +++ b/core/archipelago/src/api/rpc/tor/mod.rs @@ -129,7 +129,7 @@ pub(in crate::api::rpc) async fn restart_tor() -> Result<()> { .await } -pub(super) async fn check_tor_running() -> bool { +pub(crate) async fn check_tor_running() -> bool { tokio::net::TcpStream::connect("127.0.0.1:9050") .await .is_ok() @@ -209,6 +209,58 @@ async fn archy_net_gateway_and_subnet() -> Option<(String, String)> { Some((gateway, subnet)) } +/// Boot-time Tor self-heal: rebuild torrc from current config and apply it if +/// the live file has drifted or Tor is not actually answering. +/// +/// Without this, shipping a torrc-generation fix does NOT repair the nodes it +/// was written for. `regenerate_torrc` only runs from the Tor RPC handlers and +/// package install, so a node carrying a bad torrc keeps it until somebody +/// happens to install an app or toggle a Tor setting — its Tor stays down +/// indefinitely and an OTA changes nothing. That is exactly what happened with +/// the unbindable `SocksPort ` line: two of three reachable +/// nodes had Tor dead, one of them unnoticed. +/// +/// Deliberately conservative — it only applies (which restarts Tor) when the +/// staged torrc differs from the live one or Tor is not answering on 9050. +/// A healthy node with a matching torrc is left completely alone. +pub(crate) async fn heal_on_boot(data_dir: &Path) -> Result { + let config_dir = data_dir.join("tor-config"); + let config = load_services_config(&config_dir).await; + + // Stage what the torrc SHOULD be with the current (fixed) generator. + regenerate_torrc(&config).await?; + + let staged = tokio::fs::read_to_string("/var/lib/archipelago/tor-config/torrc.staged") + .await + .unwrap_or_default(); + if staged.is_empty() { + return Ok(false); + } + let live = tokio::fs::read_to_string("/etc/tor/torrc") + .await + .unwrap_or_default(); + + let drifted = live.trim() != staged.trim(); + let answering = check_tor_running().await; + if !drifted && answering { + debug!("Tor healthy and torrc in sync — nothing to heal"); + return Ok(false); + } + + info!( + drifted, + answering, "Healing Tor: rewriting torrc from current config and restarting" + ); + restart_tor().await?; + + // Report the truth rather than assuming the restart worked. + let ok = check_tor_running().await; + if !ok { + warn!("Tor still not answering on 127.0.0.1:9050 after heal"); + } + Ok(ok) +} + /// Can this host actually bind `addr`? Binds an ephemeral port, the same /// operation Tor performs, so the answer matches Tor's own behaviour rather /// than inferring it from interface listings. diff --git a/core/archipelago/src/bootstrap.rs b/core/archipelago/src/bootstrap.rs index 5a4c7b59..a60ef86d 100644 --- a/core/archipelago/src/bootstrap.rs +++ b/core/archipelago/src/bootstrap.rs @@ -166,6 +166,11 @@ pub async fn ensure_doctor_installed() { Ok(false) => debug!("/opt/archipelago/apps already populated (or no installer copy)"), Err(e) => warn!("Apps dir repair failed (non-fatal): {:#}", e), } + match run_tor_torrc_repair().await { + Ok(true) => info!("Tor healed at boot (torrc rebuilt and/or daemon restarted)"), + Ok(false) => debug!("Tor healthy and torrc in sync — no heal needed"), + Err(e) => warn!("Tor boot heal failed (non-fatal): {:#}", e), + } match run_polkit_networkmanager_repair().await { Ok(true) => info!( "Installed NetworkManager polkit rule for the archipelago user — Wi-Fi setup enabled" @@ -620,6 +625,33 @@ exit 2 } } +/// Repair Tor at boot so a torrc-generation fix actually reaches the nodes it +/// was written for. +/// +/// `regenerate_torrc` only ran from the Tor RPC handlers and package install, +/// so a node carrying a bad torrc kept it indefinitely: shipping a fixed binary +/// changed nothing until somebody happened to install an app or toggle a Tor +/// setting. On 2026-08-09 two of the three reachable nodes had Tor completely +/// down from an unbindable `SocksPort ` line, one of them +/// unnoticed, while the dashboard reported "connected". +/// +/// Non-fatal and conservative: it only restarts Tor when the regenerated torrc +/// differs from the live one, or Tor is not answering on 9050. +async fn run_tor_torrc_repair() -> Result { + // Same location the RPC handlers use (Config::data_dir); bootstrap runs + // before the server owns a Config, and this path is fixed on real installs. + let data_dir = Path::new("/var/lib/archipelago"); + if !data_dir.exists() { + debug!("No {} — skipping Tor boot heal", data_dir.display()); + return Ok(false); + } + if !Path::new("/etc/tor/torrc").exists() { + debug!("No /etc/tor/torrc — Tor not installed here, skipping boot heal"); + return Ok(false); + } + crate::api::rpc::tor::heal_on_boot(data_dir).await +} + async fn run_bitcoin_rpc_repair() -> Result { // bitcoind is launched with -conf=/tmp/rpc.conf and never reads a // datadir bitcoin.conf (apps/bitcoin-core & bitcoin-knots manifest.yml, diff --git a/core/archipelago/src/data_model.rs b/core/archipelago/src/data_model.rs index 55d458ec..45f2191f 100644 --- a/core/archipelago/src/data_model.rs +++ b/core/archipelago/src/data_model.rs @@ -51,6 +51,15 @@ pub struct ServerInfo { pub lan_address: Option, #[serde(rename = "tor-address")] pub tor_address: Option, + /// Is the Tor daemon actually answering — NOT "was an onion provisioned". + /// + /// `tor_address` is read from the hidden-service hostname file on disk and + /// survives Tor being dead, so the dashboard's Network card reported + /// "Connected" on three fleet nodes whose Tor had been down for days + /// (2026-08-09). Liveness has to come from a probe; the address is a + /// separate fact and must not be used as a proxy for it. + #[serde(rename = "tor-running")] + pub tor_running: bool, #[serde(rename = "node-address", skip_serializing_if = "Option::is_none")] pub node_address: Option, pub unread: u32, @@ -354,6 +363,7 @@ impl DataModel { }, lan_address: Some("http://localhost:8100".to_string()), tor_address: None, + tor_running: false, node_address: None, unread: 0, wifi_ssids: vec![], diff --git a/core/archipelago/src/server.rs b/core/archipelago/src/server.rs index 5ab39ca7..87db6d6f 100644 --- a/core/archipelago/src/server.rs +++ b/core/archipelago/src/server.rs @@ -96,6 +96,11 @@ impl Server { } } data.server_info.tor_address = docker_packages::read_tor_address("archipelago").await; + // Liveness comes from a probe, never from the presence of an onion + // address: read_tor_address reads a hostname file that outlives the + // daemon, which is why the dashboard showed "Connected" on nodes whose + // Tor had been dead for days. + data.server_info.tor_running = crate::api::rpc::tor::check_tor_running().await; if let Some(ref tor) = data.server_info.tor_address { data.server_info.node_address = Some(identity.node_address(tor)); }