fix(federation): reliable symmetric auto-federation across LAN/Tor/FIPS

Federated nodes failed to converge to full-mesh across the LAN<->Tailscale
boundary: nodes were invisible to peers, sync 'took ages'/timed out, and
names only updated on a manual sync. Onions were healthy in both directions
(~3-5s); the failures were app-layer.

- B: federation dials fast-fail a dead FIPS path via .fips_timeout(6s) in
  sync_with_peer + notify_join, so the Tor fallback isn't stuck behind the
  full 30s FIPS budget when LAN and remote peers share no FIPS path.
- A: notify_join (peer-joined) now spawns with retries+backoff instead of a
  single awaited best-effort POST, so the join RPC returns instantly (no
  'Request timeout') and the inviter reliably learns the joiner (was
  asymmetric).
- C: new 90s periodic federation auto-sync (none existed) so renamed nodes
  and roster changes propagate without a manual Sync click.
- self-heal: each auto-sync re-asserts membership to any peer that doesn't
  list us back, converging the fleet to full-mesh and healing pre-existing
  asymmetry with no manual re-joins.

Validated live across 7 nodes: a previously fleet-invisible node became
fully meshed automatically (logs: 'auto-sync ... reasserted=1',
'peer-joined ... delivered').

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
2026-06-19 09:52:26 -04:00
co-authored by Claude Opus 4.8
parent 1bce694ebb
commit 837cc02812
4 changed files with 153 additions and 5 deletions
+93
View File
@@ -428,6 +428,99 @@ impl Server {
});
}
// Periodic federation auto-sync. Pulls every federated peer's state on a
// timer so renamed nodes and roster changes propagate WITHOUT a manual
// "Sync" click. Each sync now fast-fails a dead FIPS path and falls back
// to Tor (~3-5s), so a full pass over a handful of peers is quick.
{
let data_dir = config.data_dir.clone();
let state = state_manager.clone();
tokio::spawn(async move {
// Delay the first pass so Tor/onion publishing settles after boot.
tokio::time::sleep(Duration::from_secs(20)).await;
let mut interval = tokio::time::interval(Duration::from_secs(90));
loop {
interval.tick().await;
let nodes = match crate::federation::load_nodes(&data_dir).await {
Ok(n) if !n.is_empty() => n,
_ => continue,
};
let (snap, _) = state.get_snapshot().await;
let local_did =
match crate::identity::did_key_from_pubkey_hex(&snap.server_info.pubkey) {
Ok(d) => d,
Err(_) => continue,
};
let identity_dir = data_dir.join("identity");
let node_identity =
match crate::identity::NodeIdentity::load_or_create(&identity_dir).await {
Ok(id) => id,
Err(_) => continue,
};
// Our own identity, for re-asserting membership to any peer
// that doesn't list us back (asymmetry self-heal, below).
let local_onion = snap.server_info.tor_address.clone().unwrap_or_default();
let local_pubkey = snap.server_info.pubkey.clone();
let local_name = snap.server_info.name.clone();
let local_fips_npub =
crate::identity::fips_npub(&identity_dir).await.unwrap_or(None);
let mut ok = 0usize;
let mut healed = 0usize;
for node in &nodes {
if node.trust_level == crate::federation::TrustLevel::Untrusted {
continue;
}
match crate::federation::sync_with_peer(&data_dir, node, &local_did, |b| {
node_identity.sign(b)
})
.await
{
Ok(state) => {
ok += 1;
// Asymmetry self-heal: if this peer's exported
// trusted list doesn't include us, our original
// peer-joined never landed (e.g. it was sent
// before the reliable-notify fix, or the peer was
// down). Re-assert membership over the now
// FIPS-fast-failing/Tor path so they add us back.
// Without this, a node that joined everyone stays
// invisible to the whole fleet until a manual
// re-add (the "X250-EXP missing everywhere" case).
let they_list_us = state
.federated_peers
.iter()
.any(|h| h.did == local_did);
if !they_list_us && !local_onion.is_empty() {
crate::federation::notify_join(
&node.onion,
node.fips_npub.as_deref(),
&local_did,
&local_onion,
&local_pubkey,
local_fips_npub.as_deref(),
local_name.as_deref(),
|b| node_identity.sign(b),
)
.await
.ok();
healed += 1;
}
}
Err(e) => {
debug!(peer = %node.did, error = %e, "federation auto-sync (non-fatal)")
}
}
}
debug!(
synced = ok,
reasserted = healed,
total = nodes.len(),
"federation auto-sync pass complete"
);
}
});
}
// Initialize container scanner — discovers installed apps from Podman/Docker
{
let scanner = create_docker_scanner(&config).await?;