diff --git a/core/archipelago/src/api/rpc/content.rs b/core/archipelago/src/api/rpc/content.rs index a5f43f0c..1eb89233 100644 --- a/core/archipelago/src/api/rpc/content.rs +++ b/core/archipelago/src/api/rpc/content.rs @@ -279,6 +279,7 @@ impl RpcHandler { .service(crate::settings::transport::PeerService::PeerFiles) .header("X-Federation-DID", local_did) .timeout(std::time::Duration::from_secs(120)) + .fips_timeout(std::time::Duration::from_secs(8)) .send_get() .await .context("Failed to connect to peer")?; @@ -364,6 +365,11 @@ impl RpcHandler { crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), onion, "/content") .service(crate::settings::transport::PeerService::PeerFiles) .timeout(std::time::Duration::from_secs(30)) + // The Cloud page's hottest call: without a fast-fail cap a + // cold FIPS path burned ~16.6s before Tor even started, + // against the UI's 30s deadline — users saw errors, not + // fallback. + .fips_timeout(std::time::Duration::from_secs(6)) .send_get() .await .context("Failed to connect to peer")?; @@ -1137,6 +1143,7 @@ impl RpcHandler { crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), onion, &path) .service(crate::settings::transport::PeerService::PeerFiles) .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)) .send_get() .await .context("Failed to connect to peer for preview")?; diff --git a/core/archipelago/src/api/rpc/federation/handlers.rs b/core/archipelago/src/api/rpc/federation/handlers.rs index 39fac73f..6b8f1bec 100644 --- a/core/archipelago/src/api/rpc/federation/handlers.rs +++ b/core/archipelago/src/api/rpc/federation/handlers.rs @@ -865,7 +865,8 @@ impl RpcHandler { "/rpc/v1", ) .service(crate::settings::transport::PeerService::Peers) - .timeout(std::time::Duration::from_secs(30)); + .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)); match req.send_json(&body).await { Ok((resp, transport)) if resp.status().is_success() => { diff --git a/core/archipelago/src/api/rpc/mesh/typed_messages.rs b/core/archipelago/src/api/rpc/mesh/typed_messages.rs index 28cc66f6..12e9ac78 100644 --- a/core/archipelago/src/api/rpc/mesh/typed_messages.rs +++ b/core/archipelago/src/api/rpc/mesh/typed_messages.rs @@ -820,6 +820,7 @@ impl RpcHandler { crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), &onion_bare, &path) .service(crate::settings::transport::PeerService::MeshFileSharing) .timeout(std::time::Duration::from_secs(120)) + .fips_timeout(std::time::Duration::from_secs(8)) .send_get() .await .map_err(|e| anyhow::anyhow!("Fetch failed: {}", e))?; diff --git a/core/archipelago/src/api/rpc/tor/mod.rs b/core/archipelago/src/api/rpc/tor/mod.rs index 0e3d60d4..ca0bcb2e 100644 --- a/core/archipelago/src/api/rpc/tor/mod.rs +++ b/core/archipelago/src/api/rpc/tor/mod.rs @@ -498,7 +498,8 @@ pub(super) async fn notify_federation_peers_address_change( "/rpc/v1", ) .service(crate::settings::transport::PeerService::Peers) - .timeout(std::time::Duration::from_secs(30)); + .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)); match req.send_json(&payload).await { Ok((_, transport)) => { info!(peer_did = %peer.did, transport = %transport, "Notified peer of address change") diff --git a/core/archipelago/src/fips/anchors.rs b/core/archipelago/src/fips/anchors.rs index 1e33ffe8..e58746d7 100644 --- a/core/archipelago/src/fips/anchors.rs +++ b/core/archipelago/src/fips/anchors.rs @@ -290,12 +290,12 @@ pub struct ApplyResult { /// FIPS UDP transport port (matches `transports.udp.bind_addr` in the generated /// `fips.yaml`). Direct peer links dial this, NOT the HTTP/LAN messaging port. -const FIPS_UDP_PORT: u16 = 8668; +const FIPS_UDP_PORT: u16 = crate::fips::PUBLISHED_UDP_PORT; /// Build transient seed-anchor entries that dial LAN-discovered federation peers /// directly over their FIPS UDP transport. For each peer the registry knows both /// a LAN socket address AND a FIPS npub for, point a `udp` anchor at -/// `:8668`. This lets co-located federation nodes form a DIRECT FIPS link +/// `:`. This lets co-located federation nodes form a DIRECT FIPS link /// instead of depending on the global anchor's spanning tree to route between /// them (the cause of every dial falling back to Tor when the anchor link flaps). /// @@ -448,4 +448,39 @@ mod tests { assert_eq!(a.transport, "udp"); assert_eq!(a.label, ""); } + + #[test] + fn lan_fips_anchor_port_matches_daemon_bind() { + // Drift guard: direct LAN anchors must dial the UDP port the + // generated fips.yaml actually binds. These were out of sync for + // months (anchors dialed 8668, the daemon bound 2121), making the + // whole direct-peering feature dial a dead port. + let yaml = crate::fips::config::render_config_yaml(); + assert!( + yaml.contains(&format!("0.0.0.0:{FIPS_UDP_PORT}")), + "lan_fips_anchors dials :{FIPS_UDP_PORT} but the daemon config binds elsewhere" + ); + } + + #[test] + fn lan_fips_anchors_builds_direct_entry() { + let peer = crate::transport::PeerRecord { + did: "did:key:zpeer".to_string(), + lan_address: Some("192.168.63.198:5678".to_string()), + fips_npub: Some("npub1peer".to_string()), + ..Default::default() + }; + let out = lan_fips_anchors(&[peer]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].address, format!("192.168.63.198:{FIPS_UDP_PORT}")); + assert_eq!(out[0].transport, "udp"); + + // Peers missing either the LAN address or the npub produce nothing. + let no_npub = crate::transport::PeerRecord { + did: "did:key:zother".to_string(), + lan_address: Some("192.168.63.199:5678".to_string()), + ..Default::default() + }; + assert!(lan_fips_anchors(&[no_npub]).is_empty()); + } } diff --git a/core/archipelago/src/fips/config.rs b/core/archipelago/src/fips/config.rs index 3dff85ce..9ebc19cf 100644 --- a/core/archipelago/src/fips/config.rs +++ b/core/archipelago/src/fips/config.rs @@ -240,11 +240,21 @@ pub async fn install(identity_dir: &Path) -> Result<()> { // framework-pt). Ship the allowance as a fips.d drop-in on every // install/upgrade so no node ever regresses to a UI-less mesh. sudo_install_dir("/etc/fips/fips.d").await?; - let dropin = "# Written by archipelago on every daemon config install.\n\ - # Allows the web UI + peer API through the fips0\n\ - # default-deny inbound baseline (fips.nft).\n\ - tcp dport 80 accept\n\ - tcp dport 8443 accept\n"; + // PEER_PORT (5679) carries ALL federation sync, cloud browse/download, + // mesh envelopes, DWN and invoices. It was missing from this allowlist + // while the comment claimed "web UI + peer API" — so every hardened + // node silently dropped peers' FIPS dials at the firewall and the whole + // fleet fell back to Tor (root-caused live 2026-07-27: 28k drops on + // .198's counter; :5679 answered in 0.35s once the rule was inserted). + let dropin = format!( + "# Written by archipelago on every daemon config install.\n\ + # Allows the web UI + peer API through the fips0\n\ + # default-deny inbound baseline (fips.nft).\n\ + tcp dport 80 accept\n\ + tcp dport 8443 accept\n\ + tcp dport {peer_port} accept\n", + peer_port = crate::fips::dial::PEER_PORT + ); let nft_stage = std::env::temp_dir().join(format!("fips-webui-{}.nft", std::process::id())); tokio::fs::write(&nft_stage, dropin) .await diff --git a/core/archipelago/src/fips/dial.rs b/core/archipelago/src/fips/dial.rs index 5364dda7..fd337a9c 100644 --- a/core/archipelago/src/fips/dial.rs +++ b/core/archipelago/src/fips/dial.rs @@ -447,14 +447,26 @@ impl<'a> PeerRequest<'a> { } }; let url = format!("{}{}", base, self.path); - let c = client_with_timeout(self.fips_attempt_timeout()); + let budget = self.fips_attempt_timeout(); + // With an explicit fast-fail cap, halve the per-attempt client + // timeout so the one-retry path in send_with_retry fits inside the + // budget instead of silently doubling it ("fips_timeout(6s)" used + // to really mean ~12.6s). Without one (long streaming downloads), + // keep the full budget per attempt — the client timeout also + // governs body streaming and must not truncate a real transfer. + let per_attempt = if self.fips_timeout.is_some() { + budget / 2 + } else { + budget + }; + let c = client_with_timeout(per_attempt); let mut rb = c.post(&url).json(body); for (k, v) in &self.headers { rb = rb.header(*k, v); } - match send_with_retry(rb).await { - Ok(r) => Ok(Some(r)), - Err(e) => { + match tokio::time::timeout(budget, send_with_retry(rb)).await { + Ok(Ok(r)) => Ok(Some(r)), + Ok(Err(e)) => { tracing::debug!( "FIPS POST {} failed after retry: {}, falling back to Tor", url, @@ -462,6 +474,14 @@ impl<'a> PeerRequest<'a> { ); Ok(None) } + Err(_) => { + tracing::debug!( + "FIPS POST {} exceeded attempt budget {:?}, falling back to Tor", + url, + budget + ); + Ok(None) + } } } @@ -480,14 +500,22 @@ impl<'a> PeerRequest<'a> { } }; let url = format!("{}{}", base, self.path); - let c = client_with_timeout(self.fips_attempt_timeout()); + let budget = self.fips_attempt_timeout(); + // Same budget discipline as the POST path: halve per attempt only + // under an explicit fast-fail cap; hard-cap the retry sequence. + let per_attempt = if self.fips_timeout.is_some() { + budget / 2 + } else { + budget + }; + let c = client_with_timeout(per_attempt); let mut rb = c.get(&url); for (k, v) in &self.headers { rb = rb.header(*k, v); } - match send_with_retry(rb).await { - Ok(r) => Ok(Some(r)), - Err(e) => { + match tokio::time::timeout(budget, send_with_retry(rb)).await { + Ok(Ok(r)) => Ok(Some(r)), + Ok(Err(e)) => { tracing::debug!( "FIPS GET {} failed after retry: {}, falling back to Tor", url, @@ -495,6 +523,14 @@ impl<'a> PeerRequest<'a> { ); Ok(None) } + Err(_) => { + tracing::debug!( + "FIPS GET {} exceeded attempt budget {:?}, falling back to Tor", + url, + budget + ); + Ok(None) + } } } diff --git a/core/archipelago/src/network/dwn_sync.rs b/core/archipelago/src/network/dwn_sync.rs index cec6feed..5a140351 100644 --- a/core/archipelago/src/network/dwn_sync.rs +++ b/core/archipelago/src/network/dwn_sync.rs @@ -186,6 +186,7 @@ async fn sync_single_peer( let (health_resp, _) = PeerRequest::new(fips_npub, onion, "/dwn/health") .service(crate::settings::transport::PeerService::Federation) .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)) .send_get() .await .context("Peer DWN unreachable")?; @@ -211,6 +212,7 @@ async fn sync_single_peer( let (pull_res, _) = PeerRequest::new(fips_npub, onion, "/dwn") .service(crate::settings::transport::PeerService::Federation) .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)) .send_json(&pull_body) .await .context("Failed to query peer DWN")?; @@ -269,6 +271,7 @@ async fn sync_single_peer( match PeerRequest::new(fips_npub, onion, "/dwn") .service(crate::settings::transport::PeerService::Federation) .timeout(std::time::Duration::from_secs(30)) + .fips_timeout(std::time::Duration::from_secs(6)) .send_json(&push_body) .await { diff --git a/core/archipelago/src/node_message.rs b/core/archipelago/src/node_message.rs index c08df91e..2b2ed27f 100644 --- a/core/archipelago/src/node_message.rs +++ b/core/archipelago/src/node_message.rs @@ -374,6 +374,7 @@ pub async fn send_to_peer( crate::fips::dial::PeerRequest::new(fips_npub, onion, "/archipelago/node-message") .service(crate::settings::transport::PeerService::Messaging) .timeout(std::time::Duration::from_secs(60)) + .fips_timeout(std::time::Duration::from_secs(8)) .send_json(&body) .await .map_err(|e| { @@ -410,6 +411,7 @@ pub async fn check_peer_reachable(onion: &str, fips_npub: Option<&str>) -> Resul // circuit that hasn't answered /health in 12s is "offline" for UI // purposes; the old 30s made the Connected Nodes probes crawl. .timeout(std::time::Duration::from_secs(12)) + .fips_timeout(std::time::Duration::from_secs(4)) .send_get() .await { diff --git a/core/archipelago/src/server.rs b/core/archipelago/src/server.rs index 44aba302..b9dbfef4 100644 --- a/core/archipelago/src/server.rs +++ b/core/archipelago/src/server.rs @@ -433,8 +433,18 @@ impl Server { ), )); - // LAN transport (mDNS discovery) - let mut lan = crate::transport::lan::LanTransport::new(&did, &pubkey_hex, 5678); + // LAN transport (mDNS discovery). Advertise our FIPS npub in + // the TXT record so co-located peers can form a direct FIPS + // link (see `lan_fips_anchors`). + let local_fips_npub = crate::identity::fips_npub(&data_dir.join("identity")) + .await + .unwrap_or(None); + let mut lan = crate::transport::lan::LanTransport::new( + &did, + &pubkey_hex, + 5678, + local_fips_npub, + ); match lan.start(registry.clone()) { Ok(()) => info!("📡 LAN transport (mDNS) started"), Err(e) => debug!("LAN transport init (non-fatal): {}", e), @@ -753,12 +763,25 @@ impl Server { // (often flaky) global anchor's spanning tree to route to each // other. For every peer the registry knows both a LAN address // AND a FIPS npub for, dial it on its FIPS UDP transport port - // (8668) at its LAN IP. This is FIPS's own transport over the + // at its LAN IP. This is FIPS's own transport over the // LAN — NOT Tailscale, NOT the HTTP/LAN messaging port. Pure // FIPS. `fipsctl connect` is idempotent, so re-applying every // tick just keeps the direct link warm; unknown/remote peers // (no LAN address) are left to the anchor as before. if let Some(reg) = fips_peer_registry.as_ref() { + // Hydrate FIPS npubs into the registry from federation + // storage (did-keyed). Peers discovered before the mDNS + // TXT `fips` key existed — or running builds that don't + // advertise it yet — would otherwise never satisfy the + // `fips_npub` requirement in lan_fips_anchors(), leaving + // direct LAN peering a no-op. + if let Ok(nodes) = crate::federation::load_nodes(&data_dir).await { + for n in &nodes { + if let Some(npub) = n.fips_npub.as_deref() { + reg.set_fips_npub(&n.did, npub).await; + } + } + } let direct = crate::fips::anchors::lan_fips_anchors(®.all_peers().await); if !direct.is_empty() { let _ = crate::fips::anchors::apply(&direct).await; @@ -1241,6 +1264,12 @@ pub fn is_peer_allowed_path(path: &str) -> bool { ) // Prefix-matched content endpoints (peer file browse + fetch) || path.starts_with("/content/") + // Mesh file sharing — blob fetch by CID, signature-gated in the + // handler. Absent from this list it 404'd over FIPS and the feature + // was 100% Tor by construction. + || path.starts_with("/blob/") + // DWN sync — /dwn/health is step 1 of every sync; same story. + || path.starts_with("/dwn/") } async fn accept_loop( @@ -1944,10 +1973,17 @@ mod merge_tests { ); assert!(is_peer_allowed_path("/rpc/v1")); assert!(is_peer_allowed_path("/health")); + // Mesh blob fetch + DWN sync — both were missing from the allowlist, + // which made them deterministically 404 over FIPS and therefore + // 100% Tor by construction. + assert!(is_peer_allowed_path("/blob/abc123"), "blob fetch by CID"); + assert!(is_peer_allowed_path("/dwn/health"), "DWN sync step 1"); // Not on the allow-list → rejected (no broad surface over the mesh). assert!(!is_peer_allowed_path("/contention"), "must not prefix-leak"); assert!(!is_peer_allowed_path("/")); assert!(!is_peer_allowed_path("/rpc/v2")); + assert!(!is_peer_allowed_path("/blobber"), "must not prefix-leak"); + assert!(!is_peer_allowed_path("/dwnx"), "must not prefix-leak"); } #[test] diff --git a/core/archipelago/src/transport/lan.rs b/core/archipelago/src/transport/lan.rs index 610948e7..1b3ce419 100644 --- a/core/archipelago/src/transport/lan.rs +++ b/core/archipelago/src/transport/lan.rs @@ -24,17 +24,27 @@ pub struct LanTransport { our_did: String, our_pubkey_hex: String, our_port: u16, + /// This node's FIPS npub, advertised in the mDNS TXT record so + /// co-located peers can form a direct FIPS link (`lan_fips_anchors`) + /// without waiting for federation storage to sync. + our_fips_npub: Option, daemon: Option, available: AtomicBool, } impl LanTransport { /// Create a new LAN transport. Does not start discovery yet. - pub fn new(our_did: &str, our_pubkey_hex: &str, port: u16) -> Self { + pub fn new( + our_did: &str, + our_pubkey_hex: &str, + port: u16, + our_fips_npub: Option, + ) -> Self { Self { our_did: our_did.to_string(), our_pubkey_hex: our_pubkey_hex.to_string(), our_port: port, + our_fips_npub, daemon: None, available: AtomicBool::new(false), } @@ -47,11 +57,14 @@ impl LanTransport { // Advertise our service let hostname = format!("archy-{}.local.", &self.our_pubkey_hex[..8]); - let properties = vec![ + let mut properties = vec![ ("did".to_string(), self.our_did.clone()), ("pubkey".to_string(), self.our_pubkey_hex.clone()), ("version".to_string(), "0.1.0".to_string()), ]; + if let Some(npub) = &self.our_fips_npub { + properties.push(("fips".to_string(), npub.clone())); + } let service_info = ServiceInfo::new( SERVICE_TYPE, @@ -93,6 +106,11 @@ impl LanTransport { .map(|v| v.val_str().to_string()); let addresses = info.get_addresses(); + let fips_npub = info + .get_properties() + .get("fips") + .map(|v| v.val_str().to_string()); + if let (Some(did), Some(pubkey)) = (did, pubkey) { if let Some(scoped_ip) = addresses.iter().next() { let ip: std::net::IpAddr = match scoped_ip.to_string().parse() { @@ -106,6 +124,9 @@ impl LanTransport { .await; registry_clone.set_lan_address(&did, socket_addr).await; registry_clone.set_name(&did, info.get_fullname()).await; + if let Some(npub) = fips_npub.as_deref() { + registry_clone.set_fips_npub(&did, npub).await; + } } } } diff --git a/core/archipelago/src/transport/mod.rs b/core/archipelago/src/transport/mod.rs index 99158562..2b700fe0 100644 --- a/core/archipelago/src/transport/mod.rs +++ b/core/archipelago/src/transport/mod.rs @@ -106,7 +106,7 @@ pub enum PeerSource { } /// Unified peer record with per-transport capabilities. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct PeerRecord { pub did: String, pub pubkey_hex: String,