lfg2025/archy
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects Releases 61 Wiki Activity

fix(fips): P0 uptime fixes — open peer port 5679, allow /blob+/dwn, fix LAN anchor port, un-deaden direct peering, fast-fail budgets

Browse Source 
Phase A1 of docs/FIPS-UPTIME-AND-UI-STATE-PLAN.md — the five changes that
made FIPS fall back to Tor even when a FIPS path existed:

- RC0: the fips.d drop-in now opens PEER_PORT 5679 (was 80+8443 only, so
  every hardened node firewalled peers' FIPS dials; 28k drops on .198)
- RC4: /blob/ and /dwn/ added to the peer-path allowlist — mesh file
  sharing and DWN sync were 404 → 100% Tor by construction
- RC2-G2: lan_fips_anchors dials PUBLISHED_UDP_PORT (2121) instead of the
  dead 8668, with a drift-guard test against the rendered daemon config
- RC2-G1: direct LAN peering actually runs now — mDNS TXT advertises the
  FIPS npub, discovery calls set_fips_npub, and the anchor tick hydrates
  npubs from federation storage for peers on older builds
- RC3: FIPS attempt budget is a hard cap (retry no longer doubles it) and
  the 12 hot call sites get explicit fips_timeout fast-fail so Tor keeps
  its full budget (browse-peer, preview, /blob, DWN, node-message,
  rotation notifies)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-27 16:42:44 -04:00
parent 94b5374f66
commit eb2fc0f37b
12 changed files with 176 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/archipelago/src/api/rpc/content.rs
View File

@ -279,6 +279,7 @@ impl RpcHandler {
.service(crate::settings::transport::PeerService::PeerFiles)
.header("X-Federation-DID", local_did)
.timeout(std::time::Duration::from_secs(120))
.fips_timeout(std::time::Duration::from_secs(8))
.send_get()
.await
.context("Failed to connect to peer")?;
@ -364,6 +365,11 @@ impl RpcHandler {
crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), onion, "/content")
.service(crate::settings::transport::PeerService::PeerFiles)
.timeout(std::time::Duration::from_secs(30))
// The Cloud page's hottest call: without a fast-fail cap a
// cold FIPS path burned ~16.6s before Tor even started,
// against the UI's 30s deadline — users saw errors, not
// fallback.
.fips_timeout(std::time::Duration::from_secs(6))
.send_get()
.await
.context("Failed to connect to peer")?;
@ -1137,6 +1143,7 @@ impl RpcHandler {
crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), onion, &path)
.service(crate::settings::transport::PeerService::PeerFiles)
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6))
.send_get()
.await
.context("Failed to connect to peer for preview")?;

3
core/archipelago/src/api/rpc/federation/handlers.rs
View File

@ -865,7 +865,8 @@ impl RpcHandler {
"/rpc/v1",
)
.service(crate::settings::transport::PeerService::Peers)
.timeout(std::time::Duration::from_secs(30));
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6));
match req.send_json(&body).await {
Ok((resp, transport)) if resp.status().is_success() => {

1
core/archipelago/src/api/rpc/mesh/typed_messages.rs
View File

@ -820,6 +820,7 @@ impl RpcHandler {
crate::fips::dial::PeerRequest::new(fips_npub.as_deref(), &onion_bare, &path)
.service(crate::settings::transport::PeerService::MeshFileSharing)
.timeout(std::time::Duration::from_secs(120))
.fips_timeout(std::time::Duration::from_secs(8))
.send_get()
.await
.map_err(|e| anyhow::anyhow!("Fetch failed: {}", e))?;

3
core/archipelago/src/api/rpc/tor/mod.rs
View File

@ -498,7 +498,8 @@ pub(super) async fn notify_federation_peers_address_change(
"/rpc/v1",
)
.service(crate::settings::transport::PeerService::Peers)
.timeout(std::time::Duration::from_secs(30));
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6));
match req.send_json(&payload).await {
Ok((_, transport)) => {
info!(peer_did = %peer.did, transport = %transport, "Notified peer of address change")

39
core/archipelago/src/fips/anchors.rs
View File

@ -290,12 +290,12 @@ pub struct ApplyResult {
/// FIPS UDP transport port (matches `transports.udp.bind_addr` in the generated
/// `fips.yaml`). Direct peer links dial this, NOT the HTTP/LAN messaging port.
const FIPS_UDP_PORT: u16 = 8668;
const FIPS_UDP_PORT: u16 = crate::fips::PUBLISHED_UDP_PORT;
/// Build transient seed-anchor entries that dial LAN-discovered federation peers
/// directly over their FIPS UDP transport. For each peer the registry knows both
/// a LAN socket address AND a FIPS npub for, point a `udp` anchor at
/// `<lan-ip>:8668`. This lets co-located federation nodes form a DIRECT FIPS link
/// `<lan-ip>:<FIPS_UDP_PORT>`. This lets co-located federation nodes form a DIRECT FIPS link
/// instead of depending on the global anchor's spanning tree to route between
/// them (the cause of every dial falling back to Tor when the anchor link flaps).
///
@ -448,4 +448,39 @@ mod tests {
assert_eq!(a.transport, "udp");
assert_eq!(a.label, "");
}
#[test]
fn lan_fips_anchor_port_matches_daemon_bind() {
// Drift guard: direct LAN anchors must dial the UDP port the
// generated fips.yaml actually binds. These were out of sync for
// months (anchors dialed 8668, the daemon bound 2121), making the
// whole direct-peering feature dial a dead port.
let yaml = crate::fips::config::render_config_yaml();
assert!(
yaml.contains(&format!("0.0.0.0:{FIPS_UDP_PORT}")),
"lan_fips_anchors dials :{FIPS_UDP_PORT} but the daemon config binds elsewhere"
);
}
#[test]
fn lan_fips_anchors_builds_direct_entry() {
let peer = crate::transport::PeerRecord {
did: "did:key:zpeer".to_string(),
lan_address: Some("192.168.63.198:5678".to_string()),
fips_npub: Some("npub1peer".to_string()),
..Default::default()
};
let out = lan_fips_anchors(&[peer]);
assert_eq!(out.len(), 1);
assert_eq!(out[0].address, format!("192.168.63.198:{FIPS_UDP_PORT}"));
assert_eq!(out[0].transport, "udp");
// Peers missing either the LAN address or the npub produce nothing.
let no_npub = crate::transport::PeerRecord {
did: "did:key:zother".to_string(),
lan_address: Some("192.168.63.199:5678".to_string()),
..Default::default()
};
assert!(lan_fips_anchors(&[no_npub]).is_empty());
}
}

20
core/archipelago/src/fips/config.rs
View File

@ -240,11 +240,21 @@ pub async fn install(identity_dir: &Path) -> Result<()> {
// framework-pt). Ship the allowance as a fips.d drop-in on every
// install/upgrade so no node ever regresses to a UI-less mesh.
sudo_install_dir("/etc/fips/fips.d").await?;
let dropin = "# Written by archipelago on every daemon config install.\n\
# Allows the web UI + peer API through the fips0\n\
# default-deny inbound baseline (fips.nft).\n\
tcp dport 80 accept\n\
tcp dport 8443 accept\n";
// PEER_PORT (5679) carries ALL federation sync, cloud browse/download,
// mesh envelopes, DWN and invoices. It was missing from this allowlist
// while the comment claimed "web UI + peer API" — so every hardened
// node silently dropped peers' FIPS dials at the firewall and the whole
// fleet fell back to Tor (root-caused live 2026-07-27: 28k drops on
// .198's counter; :5679 answered in 0.35s once the rule was inserted).
let dropin = format!(
"# Written by archipelago on every daemon config install.\n\
# Allows the web UI + peer API through the fips0\n\
# default-deny inbound baseline (fips.nft).\n\
tcp dport 80 accept\n\
tcp dport 8443 accept\n\
tcp dport {peer_port} accept\n",
peer_port = crate::fips::dial::PEER_PORT
);
let nft_stage = std::env::temp_dir().join(format!("fips-webui-{}.nft", std::process::id()));
tokio::fs::write(&nft_stage, dropin)
.await

52
core/archipelago/src/fips/dial.rs
View File

@ -447,14 +447,26 @@ impl<'a> PeerRequest<'a> {
}
};
let url = format!("{}{}", base, self.path);
let c = client_with_timeout(self.fips_attempt_timeout());
let budget = self.fips_attempt_timeout();
// With an explicit fast-fail cap, halve the per-attempt client
// timeout so the one-retry path in send_with_retry fits inside the
// budget instead of silently doubling it ("fips_timeout(6s)" used
// to really mean ~12.6s). Without one (long streaming downloads),
// keep the full budget per attempt — the client timeout also
// governs body streaming and must not truncate a real transfer.
let per_attempt = if self.fips_timeout.is_some() {
budget / 2
} else {
budget
};
let c = client_with_timeout(per_attempt);
let mut rb = c.post(&url).json(body);
for (k, v) in &self.headers {
rb = rb.header(*k, v);
}
match send_with_retry(rb).await {
Ok(r) => Ok(Some(r)),
Err(e) => {
match tokio::time::timeout(budget, send_with_retry(rb)).await {
Ok(Ok(r)) => Ok(Some(r)),
Ok(Err(e)) => {
tracing::debug!(
"FIPS POST {} failed after retry: {}, falling back to Tor",
url,
@ -462,6 +474,14 @@ impl<'a> PeerRequest<'a> {
);
Ok(None)
}
Err(_) => {
tracing::debug!(
"FIPS POST {} exceeded attempt budget {:?}, falling back to Tor",
url,
budget
);
Ok(None)
}
}
}
@ -480,14 +500,22 @@ impl<'a> PeerRequest<'a> {
}
};
let url = format!("{}{}", base, self.path);
let c = client_with_timeout(self.fips_attempt_timeout());
let budget = self.fips_attempt_timeout();
// Same budget discipline as the POST path: halve per attempt only
// under an explicit fast-fail cap; hard-cap the retry sequence.
let per_attempt = if self.fips_timeout.is_some() {
budget / 2
} else {
budget
};
let c = client_with_timeout(per_attempt);
let mut rb = c.get(&url);
for (k, v) in &self.headers {
rb = rb.header(*k, v);
}
match send_with_retry(rb).await {
Ok(r) => Ok(Some(r)),
Err(e) => {
match tokio::time::timeout(budget, send_with_retry(rb)).await {
Ok(Ok(r)) => Ok(Some(r)),
Ok(Err(e)) => {
tracing::debug!(
"FIPS GET {} failed after retry: {}, falling back to Tor",
url,
@ -495,6 +523,14 @@ impl<'a> PeerRequest<'a> {
);
Ok(None)
}
Err(_) => {
tracing::debug!(
"FIPS GET {} exceeded attempt budget {:?}, falling back to Tor",
url,
budget
);
Ok(None)
}
}
}

3
core/archipelago/src/network/dwn_sync.rs
View File

@ -186,6 +186,7 @@ async fn sync_single_peer(
let (health_resp, _) = PeerRequest::new(fips_npub, onion, "/dwn/health")
.service(crate::settings::transport::PeerService::Federation)
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6))
.send_get()
.await
.context("Peer DWN unreachable")?;
@ -211,6 +212,7 @@ async fn sync_single_peer(
let (pull_res, _) = PeerRequest::new(fips_npub, onion, "/dwn")
.service(crate::settings::transport::PeerService::Federation)
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6))
.send_json(&pull_body)
.await
.context("Failed to query peer DWN")?;
@ -269,6 +271,7 @@ async fn sync_single_peer(
match PeerRequest::new(fips_npub, onion, "/dwn")
.service(crate::settings::transport::PeerService::Federation)
.timeout(std::time::Duration::from_secs(30))
.fips_timeout(std::time::Duration::from_secs(6))
.send_json(&push_body)
.await
{

2
core/archipelago/src/node_message.rs
View File

@ -374,6 +374,7 @@ pub async fn send_to_peer(
crate::fips::dial::PeerRequest::new(fips_npub, onion, "/archipelago/node-message")
.service(crate::settings::transport::PeerService::Messaging)
.timeout(std::time::Duration::from_secs(60))
.fips_timeout(std::time::Duration::from_secs(8))
.send_json(&body)
.await
.map_err(|e| {
@ -410,6 +411,7 @@ pub async fn check_peer_reachable(onion: &str, fips_npub: Option<&str>) -> Resul
// circuit that hasn't answered /health in 12s is "offline" for UI
// purposes; the old 30s made the Connected Nodes probes crawl.
.timeout(std::time::Duration::from_secs(12))
.fips_timeout(std::time::Duration::from_secs(4))
.send_get()
.await
{

42
core/archipelago/src/server.rs
View File

@ -433,8 +433,18 @@ impl Server {
),
));
// LAN transport (mDNS discovery)
let mut lan = crate::transport::lan::LanTransport::new(&did, &pubkey_hex, 5678);
// LAN transport (mDNS discovery). Advertise our FIPS npub in
// the TXT record so co-located peers can form a direct FIPS
// link (see `lan_fips_anchors`).
let local_fips_npub = crate::identity::fips_npub(&data_dir.join("identity"))
.await
.unwrap_or(None);
let mut lan = crate::transport::lan::LanTransport::new(
&did,
&pubkey_hex,
5678,
local_fips_npub,
);
match lan.start(registry.clone()) {
Ok(()) => info!("📡 LAN transport (mDNS) started"),
Err(e) => debug!("LAN transport init (non-fatal): {}", e),
@ -753,12 +763,25 @@ impl Server {
// (often flaky) global anchor's spanning tree to route to each
// other. For every peer the registry knows both a LAN address
// AND a FIPS npub for, dial it on its FIPS UDP transport port
// (8668) at its LAN IP. This is FIPS's own transport over the
// at its LAN IP. This is FIPS's own transport over the
// LAN — NOT Tailscale, NOT the HTTP/LAN messaging port. Pure
// FIPS. `fipsctl connect` is idempotent, so re-applying every
// tick just keeps the direct link warm; unknown/remote peers
// (no LAN address) are left to the anchor as before.
if let Some(reg) = fips_peer_registry.as_ref() {
// Hydrate FIPS npubs into the registry from federation
// storage (did-keyed). Peers discovered before the mDNS
// TXT `fips` key existed — or running builds that don't
// advertise it yet — would otherwise never satisfy the
// `fips_npub` requirement in lan_fips_anchors(), leaving
// direct LAN peering a no-op.
if let Ok(nodes) = crate::federation::load_nodes(&data_dir).await {
for n in &nodes {
if let Some(npub) = n.fips_npub.as_deref() {
reg.set_fips_npub(&n.did, npub).await;
}
}
}
let direct = crate::fips::anchors::lan_fips_anchors(&reg.all_peers().await);
if !direct.is_empty() {
let _ = crate::fips::anchors::apply(&direct).await;
@ -1241,6 +1264,12 @@ pub fn is_peer_allowed_path(path: &str) -> bool {
)
// Prefix-matched content endpoints (peer file browse + fetch)
|| path.starts_with("/content/")
// Mesh file sharing — blob fetch by CID, signature-gated in the
// handler. Absent from this list it 404'd over FIPS and the feature
// was 100% Tor by construction.
|| path.starts_with("/blob/")
// DWN sync — /dwn/health is step 1 of every sync; same story.
|| path.starts_with("/dwn/")
}
async fn accept_loop(
@ -1944,10 +1973,17 @@ mod merge_tests {
);
assert!(is_peer_allowed_path("/rpc/v1"));
assert!(is_peer_allowed_path("/health"));
// Mesh blob fetch + DWN sync — both were missing from the allowlist,
// which made them deterministically 404 over FIPS and therefore
// 100% Tor by construction.
assert!(is_peer_allowed_path("/blob/abc123"), "blob fetch by CID");
assert!(is_peer_allowed_path("/dwn/health"), "DWN sync step 1");
// Not on the allow-list → rejected (no broad surface over the mesh).
assert!(!is_peer_allowed_path("/contention"), "must not prefix-leak");
assert!(!is_peer_allowed_path("/"));
assert!(!is_peer_allowed_path("/rpc/v2"));
assert!(!is_peer_allowed_path("/blobber"), "must not prefix-leak");
assert!(!is_peer_allowed_path("/dwnx"), "must not prefix-leak");
}
#[test]

25
core/archipelago/src/transport/lan.rs
View File

@ -24,17 +24,27 @@ pub struct LanTransport {
our_did: String,
our_pubkey_hex: String,
our_port: u16,
/// This node's FIPS npub, advertised in the mDNS TXT record so
/// co-located peers can form a direct FIPS link (`lan_fips_anchors`)
/// without waiting for federation storage to sync.
our_fips_npub: Option<String>,
daemon: Option<ServiceDaemon>,
available: AtomicBool,
}
impl LanTransport {
/// Create a new LAN transport. Does not start discovery yet.
pub fn new(our_did: &str, our_pubkey_hex: &str, port: u16) -> Self {
pub fn new(
our_did: &str,
our_pubkey_hex: &str,
port: u16,
our_fips_npub: Option<String>,
) -> Self {
Self {
our_did: our_did.to_string(),
our_pubkey_hex: our_pubkey_hex.to_string(),
our_port: port,
our_fips_npub,
daemon: None,
available: AtomicBool::new(false),
}
@ -47,11 +57,14 @@ impl LanTransport {
// Advertise our service
let hostname = format!("archy-{}.local.", &self.our_pubkey_hex[..8]);
let properties = vec![
let mut properties = vec![
("did".to_string(), self.our_did.clone()),
("pubkey".to_string(), self.our_pubkey_hex.clone()),
("version".to_string(), "0.1.0".to_string()),
];
if let Some(npub) = &self.our_fips_npub {
properties.push(("fips".to_string(), npub.clone()));
}
let service_info = ServiceInfo::new(
SERVICE_TYPE,
@ -93,6 +106,11 @@ impl LanTransport {
.map(|v| v.val_str().to_string());
let addresses = info.get_addresses();
let fips_npub = info
.get_properties()
.get("fips")
.map(|v| v.val_str().to_string());
if let (Some(did), Some(pubkey)) = (did, pubkey) {
if let Some(scoped_ip) = addresses.iter().next() {
let ip: std::net::IpAddr = match scoped_ip.to_string().parse() {
@ -106,6 +124,9 @@ impl LanTransport {
.await;
registry_clone.set_lan_address(&did, socket_addr).await;
registry_clone.set_name(&did, info.get_fullname()).await;
if let Some(npub) = fips_npub.as_deref() {
registry_clone.set_fips_npub(&did, npub).await;
}
}
}
}

2
core/archipelago/src/transport/mod.rs
View File

@ -106,7 +106,7 @@ pub enum PeerSource {
}
/// Unified peer record with per-transport capabilities.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PeerRecord {
pub did: String,
pub pubkey_hex: String,