fix(federation): end the perpetual peer-joined "Invalid signature" storm
Root cause observed live 2026-08-16: onboarding/seed-restore rewrite identity/node_key on disk but server_info.pubkey is only seeded at boot, so until the next restart every peer-joined advertised the stale boot key while signing with the new seed-derived key — deterministically rejected by every receiver, once per 90s heal tick, forever. - seed.generate / seed.restore now refresh server_info.pubkey in the live snapshot immediately (mirrors the DID-rotation handler). - The 90s heal loop advertises the SAME key it signs with (disk identity, like federation sync already did) instead of the boot snapshot. - notify_join no longer logs "delivered" for an HTTP-200 JSON-RPC rejection; in-band errors are terminal (identical signed bytes can never succeed on retry). - The heal loop backs off per peer (doubling toward a daily re-assert) instead of re-notifying every 90s forever — Observer-held peers never appear in Trusted-only exported hints, so they_list_us could never become true for them. - Receiver now binds the DID to the advertised pubkey (the old check was self-referential) and logs malformed signatures distinctly from genuine mismatches. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
809f7649a4
commit
519fa68c72
@@ -696,6 +696,22 @@ impl RpcHandler {
|
|||||||
anyhow::bail!("Refusing to peer with self");
|
anyhow::bail!("Refusing to peer with self");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Bind the DID to the advertised pubkey. Without this the signature
|
||||||
|
// check below is self-referential (the caller signs over a pubkey it
|
||||||
|
// also supplies), so a consistent-but-unrelated keypair would pass.
|
||||||
|
// The DID-rotation handler already enforces the same invariant.
|
||||||
|
match identity::did_key_from_pubkey_hex(pubkey) {
|
||||||
|
Ok(derived) if derived == did => {}
|
||||||
|
Ok(derived) => {
|
||||||
|
tracing::warn!(peer_did = %did, derived_did = %derived, "Rejected peer-joined: DID does not match pubkey");
|
||||||
|
anyhow::bail!("DID does not match pubkey");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(peer_did = %did, error = %e, "Rejected peer-joined: invalid pubkey");
|
||||||
|
anyhow::bail!("Invalid pubkey");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Verify ed25519 signature to prevent federation spoofing (H2 security fix)
|
// Verify ed25519 signature to prevent federation spoofing (H2 security fix)
|
||||||
let signature = params.get("signature").and_then(|v| v.as_str());
|
let signature = params.get("signature").and_then(|v| v.as_str());
|
||||||
match signature {
|
match signature {
|
||||||
@@ -703,10 +719,16 @@ impl RpcHandler {
|
|||||||
let sign_data = format!("peer-joined:{}:{}:{}", did, onion, pubkey);
|
let sign_data = format!("peer-joined:{}:{}:{}", did, onion, pubkey);
|
||||||
match identity::NodeIdentity::verify(pubkey, sign_data.as_bytes(), sig) {
|
match identity::NodeIdentity::verify(pubkey, sign_data.as_bytes(), sig) {
|
||||||
Ok(true) => {}
|
Ok(true) => {}
|
||||||
_ => {
|
Ok(false) => {
|
||||||
tracing::warn!(peer_did = %did, "Rejected peer-joined: invalid signature");
|
tracing::warn!(peer_did = %did, "Rejected peer-joined: invalid signature");
|
||||||
anyhow::bail!("Invalid signature");
|
anyhow::bail!("Invalid signature");
|
||||||
}
|
}
|
||||||
|
Err(e) => {
|
||||||
|
// Malformed hex / wrong length — distinguish from a
|
||||||
|
// genuine mismatch so the log tells us which it was.
|
||||||
|
tracing::warn!(peer_did = %did, error = %e, "Rejected peer-joined: malformed signature");
|
||||||
|
anyhow::bail!("Invalid signature");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
|
|||||||
@@ -176,6 +176,29 @@ pub(in crate::api::rpc) async fn restore_node_identity_from_words(
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RpcHandler {
|
impl RpcHandler {
|
||||||
|
/// Push the on-disk node identity's pubkey into the live `server_info`
|
||||||
|
/// snapshot. `seed.generate` / `seed.restore` rewrite `identity/node_key`,
|
||||||
|
/// but `server_info.pubkey` was only seeded at boot — until the next
|
||||||
|
/// restart every federation peer-joined advertised the stale boot key
|
||||||
|
/// while signing with the new seed-derived key, so receivers rejected it
|
||||||
|
/// ("Invalid signature") on every 90s heal tick (2026-08-16). Mirrors the
|
||||||
|
/// DID-rotation handler, which already does this refresh.
|
||||||
|
async fn refresh_server_pubkey_from_disk(&self) {
|
||||||
|
let identity_dir = self.config.data_dir.join("identity");
|
||||||
|
let identity = match crate::identity::NodeIdentity::load_or_create(&identity_dir).await {
|
||||||
|
Ok(id) => id,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(error = %e, "Could not reload node identity after seed write — server_info.pubkey stays stale until restart");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (mut data, _) = self.state_manager.get_snapshot().await;
|
||||||
|
if data.server_info.pubkey != identity.pubkey_hex() {
|
||||||
|
data.server_info.pubkey = identity.pubkey_hex();
|
||||||
|
self.state_manager.update_data(data).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Generate a new 24-word BIP-39 mnemonic, derive and persist node keys.
|
/// Generate a new 24-word BIP-39 mnemonic, derive and persist node keys.
|
||||||
/// Returns the words for the user to write down.
|
/// Returns the words for the user to write down.
|
||||||
pub(in crate::api::rpc) async fn handle_seed_generate(&self) -> Result<serde_json::Value> {
|
pub(in crate::api::rpc) async fn handle_seed_generate(&self) -> Result<serde_json::Value> {
|
||||||
@@ -242,6 +265,10 @@ impl RpcHandler {
|
|||||||
// Initialize identity index at 0.
|
// Initialize identity index at 0.
|
||||||
crate::seed::save_identity_index(&self.config.data_dir, 0).await?;
|
crate::seed::save_identity_index(&self.config.data_dir, 0).await?;
|
||||||
|
|
||||||
|
// The node key on disk just changed — keep the live snapshot's pubkey
|
||||||
|
// in lockstep (see refresh_server_pubkey_from_disk for why).
|
||||||
|
self.refresh_server_pubkey_from_disk().await;
|
||||||
|
|
||||||
// fips_key is now on disk — auto-activate FIPS so the user doesn't
|
// fips_key is now on disk — auto-activate FIPS so the user doesn't
|
||||||
// have to hit a manual Start button. Detached task;
|
// have to hit a manual Start button. Detached task;
|
||||||
// the onboarding RPC returns immediately.
|
// the onboarding RPC returns immediately.
|
||||||
@@ -350,7 +377,11 @@ impl RpcHandler {
|
|||||||
)
|
)
|
||||||
.context("Invalid words array")?;
|
.context("Invalid words array")?;
|
||||||
|
|
||||||
restore_node_identity_from_words(&self.config.data_dir, &self.auth_manager, &words).await
|
let result =
|
||||||
|
restore_node_identity_from_words(&self.config.data_dir, &self.auth_manager, &words)
|
||||||
|
.await?;
|
||||||
|
self.refresh_server_pubkey_from_disk().await;
|
||||||
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Encrypt and save the mnemonic to disk for convenience backup.
|
/// Encrypt and save the mnemonic to disk for convenience backup.
|
||||||
|
|||||||
@@ -326,6 +326,26 @@ pub(crate) async fn notify_join(
|
|||||||
.await;
|
.await;
|
||||||
match res {
|
match res {
|
||||||
Ok((resp, transport)) if resp.status().is_success() => {
|
Ok((resp, transport)) if resp.status().is_success() => {
|
||||||
|
// A JSON-RPC-level rejection still arrives as HTTP 200
|
||||||
|
// (the RPC layer returns errors in-band), so checking the
|
||||||
|
// status alone logged "delivered" for calls the peer had
|
||||||
|
// just rejected. Read the body: an in-band error is
|
||||||
|
// terminal — the signed payload is deterministic, so
|
||||||
|
// retrying identical bytes can never succeed.
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
let rpc_err = serde_json::from_str::<serde_json::Value>(&body)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.get("error").cloned())
|
||||||
|
.filter(|e| !e.is_null());
|
||||||
|
if let Some(err) = rpc_err {
|
||||||
|
tracing::warn!(
|
||||||
|
attempt,
|
||||||
|
transport = %transport,
|
||||||
|
error = %err,
|
||||||
|
"peer-joined notification rejected by peer — giving up (retrying identical payload cannot succeed)"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
attempt,
|
attempt,
|
||||||
transport = %transport,
|
transport = %transport,
|
||||||
|
|||||||
@@ -553,6 +553,15 @@ impl Server {
|
|||||||
// heavy load) don't fire a burst of catch-up ticks back-to-back,
|
// heavy load) don't fire a burst of catch-up ticks back-to-back,
|
||||||
// just resume the cadence from now.
|
// just resume the cadence from now.
|
||||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||||
|
// Peers that never list us back would otherwise be re-notified
|
||||||
|
// every 90s forever (e.g. they hold us at Observer and their
|
||||||
|
// exported hints are Trusted-only, so `they_list_us` can never
|
||||||
|
// become true). Back off per peer, doubling toward a daily
|
||||||
|
// re-assert; reset the moment they do list us.
|
||||||
|
let mut notify_backoff: std::collections::HashMap<
|
||||||
|
String,
|
||||||
|
(u32, tokio::time::Instant),
|
||||||
|
> = std::collections::HashMap::new();
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
// Zero federated nodes is a clean no-op: nothing is read
|
// Zero federated nodes is a clean no-op: nothing is read
|
||||||
@@ -568,21 +577,27 @@ impl Server {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let (snap, _) = state.get_snapshot().await;
|
let (snap, _) = state.get_snapshot().await;
|
||||||
let local_did =
|
|
||||||
match crate::identity::did_key_from_pubkey_hex(&snap.server_info.pubkey) {
|
|
||||||
Ok(d) => d,
|
|
||||||
Err(_) => continue,
|
|
||||||
};
|
|
||||||
let identity_dir = data_dir.join("identity");
|
let identity_dir = data_dir.join("identity");
|
||||||
let node_identity =
|
let node_identity =
|
||||||
match crate::identity::NodeIdentity::load_or_create(&identity_dir).await {
|
match crate::identity::NodeIdentity::load_or_create(&identity_dir).await {
|
||||||
Ok(id) => id,
|
Ok(id) => id,
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
};
|
};
|
||||||
|
// Advertise the SAME key we sign with. server_info.pubkey
|
||||||
|
// is only seeded at boot; onboarding/seed-restore rewrite
|
||||||
|
// identity/node_key on disk without touching the snapshot,
|
||||||
|
// and a peer-joined that advertises the stale boot key
|
||||||
|
// while signing with the new seed-derived key is
|
||||||
|
// deterministically rejected ("Invalid signature") by
|
||||||
|
// every receiver, once per tick, forever (2026-08-16).
|
||||||
|
let local_pubkey = node_identity.pubkey_hex();
|
||||||
|
let local_did = match crate::identity::did_key_from_pubkey_hex(&local_pubkey) {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
// Our own identity, for re-asserting membership to any peer
|
// Our own identity, for re-asserting membership to any peer
|
||||||
// that doesn't list us back (asymmetry self-heal, below).
|
// that doesn't list us back (asymmetry self-heal, below).
|
||||||
let local_onion = snap.server_info.tor_address.clone().unwrap_or_default();
|
let local_onion = snap.server_info.tor_address.clone().unwrap_or_default();
|
||||||
let local_pubkey = snap.server_info.pubkey.clone();
|
|
||||||
let local_name = snap.server_info.name.clone();
|
let local_name = snap.server_info.name.clone();
|
||||||
let local_fips_npub = crate::identity::fips_npub(&identity_dir)
|
let local_fips_npub = crate::identity::fips_npub(&identity_dir)
|
||||||
.await
|
.await
|
||||||
@@ -618,24 +633,44 @@ impl Server {
|
|||||||
// re-add (the "peer missing everywhere" case).
|
// re-add (the "peer missing everywhere" case).
|
||||||
let they_list_us =
|
let they_list_us =
|
||||||
state.federated_peers.iter().any(|h| h.did == local_did);
|
state.federated_peers.iter().any(|h| h.did == local_did);
|
||||||
if !they_list_us && !local_onion.is_empty() {
|
if they_list_us {
|
||||||
crate::federation::notify_join(
|
notify_backoff.remove(&node.did);
|
||||||
&node.onion,
|
} else if !local_onion.is_empty() {
|
||||||
node.fips_npub.as_deref(),
|
let now = tokio::time::Instant::now();
|
||||||
&local_did,
|
let due = notify_backoff
|
||||||
&local_onion,
|
.get(&node.did)
|
||||||
&local_pubkey,
|
.map(|(_, next)| *next <= now)
|
||||||
local_fips_npub.as_deref(),
|
.unwrap_or(true);
|
||||||
local_name.as_deref(),
|
if due {
|
||||||
// Re-assert at the level WE hold for
|
crate::federation::notify_join(
|
||||||
// this peer; no invite token on heal.
|
&node.onion,
|
||||||
None,
|
node.fips_npub.as_deref(),
|
||||||
node.trust_level,
|
&local_did,
|
||||||
|b| node_identity.sign(b),
|
&local_onion,
|
||||||
)
|
&local_pubkey,
|
||||||
.await
|
local_fips_npub.as_deref(),
|
||||||
.ok();
|
local_name.as_deref(),
|
||||||
healed += 1;
|
// Re-assert at the level WE hold for
|
||||||
|
// this peer; no invite token on heal.
|
||||||
|
None,
|
||||||
|
node.trust_level,
|
||||||
|
|b| node_identity.sign(b),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
healed += 1;
|
||||||
|
let attempts = notify_backoff
|
||||||
|
.get(&node.did)
|
||||||
|
.map(|(a, _)| *a)
|
||||||
|
.unwrap_or(0)
|
||||||
|
+ 1;
|
||||||
|
let delay =
|
||||||
|
(90u64 << attempts.min(10)).min(86_400);
|
||||||
|
notify_backoff.insert(
|
||||||
|
node.did.clone(),
|
||||||
|
(attempts, now + Duration::from_secs(delay)),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user