archipelago eb2fc0f37b fix(fips): P0 uptime fixes — open peer port 5679, allow /blob+/dwn, fix LAN anchor port, un-deaden direct peering, fast-fail budgets
Phase A1 of docs/FIPS-UPTIME-AND-UI-STATE-PLAN.md — the five changes that
made FIPS fall back to Tor even when a FIPS path existed:

- RC0: the fips.d drop-in now opens PEER_PORT 5679 (was 80+8443 only, so
  every hardened node firewalled peers' FIPS dials; 28k drops on .198)
- RC4: /blob/ and /dwn/ added to the peer-path allowlist — mesh file
  sharing and DWN sync were 404 → 100% Tor by construction
- RC2-G2: lan_fips_anchors dials PUBLISHED_UDP_PORT (2121) instead of the
  dead 8668, with a drift-guard test against the rendered daemon config
- RC2-G1: direct LAN peering actually runs now — mDNS TXT advertises the
  FIPS npub, discovery calls set_fips_npub, and the anchor tick hydrates
  npubs from federation storage for peers on older builds
- RC3: FIPS attempt budget is a hard cap (retry no longer doubles it) and
  the 12 hot call sites get explicit fips_timeout fast-fail so Tor keeps
  its full budget (browse-peer, preview, /blob, DWN, node-message,
  rotation notifies)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-27 16:42:44 -04:00

497 lines
19 KiB
Rust

//! FIPS daemon config + key materialisation.
//!
//! Writes `/etc/fips/fips.yaml`, `/etc/fips/fips.key`, and
//! `/etc/fips/fips.pub` from the archipelago node's seed-derived FIPS
//! keypair, then chmod 0600 the private key.
//!
//! Privileged filesystem writes go through a `sudo install` invocation
//! rather than opening `/etc/fips/*` directly — the archipelago service
//! user cannot write `/etc` itself. The sudoers policy in the ISO
//! whitelists `install` into `/etc/fips/`.
use anyhow::{Context, Result};
use serde::Serialize;
use std::path::Path;
use tokio::process::Command;
use super::{
DAEMON_CONFIG_PATH, DAEMON_KEY_PATH, DAEMON_PUB_PATH, DEFAULT_TCP_PORT, PUBLISHED_UDP_PORT,
};
/// Header prepended to the generated YAML. serde doesn't emit comments, so
/// this is concatenated onto the serialised body.
const CONFIG_HEADER: &str = "# Generated by archipelago — do not edit by hand.\n\
# Regenerated on every key change and daemon upgrade.\n";
/// Typed mirror of the subset of upstream `fips.yaml` that archipelago owns.
///
/// This was previously built by `format!`-ing a string literal. Upstream's
/// config structs are `#[serde(deny_unknown_fields)]`, so a key we get wrong
/// doesn't degrade gracefully — the daemon refuses to start and the node drops
/// off the mesh. Serialising from typed structs lets the compiler and the
/// tests below catch drift, instead of a node discovering it at boot after an
/// upgrade.
///
/// Schema verified field-by-field against jmcorgan/fips **v0.4.1** (2026-07-20).
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct FipsConfig {
pub node: NodeSection,
pub tun: TunSection,
pub dns: DnsSection,
pub transports: TransportsSection,
/// Static peers. Always empty: archipelago feeds peers dynamically via the
/// seed-anchors apply loop and federation-invite hooks.
pub peers: Vec<PeerEntry>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct NodeSection {
pub identity: IdentitySection,
pub discovery: DiscoverySection,
pub retry: RetrySection,
pub rate_limit: RateLimitSection,
}
/// Fast-reconnect profile (`node.retry.*`). Upstream defaults (5s base
/// doubling to 300s) are tuned for stable always-on links; a node redialing
/// a recycled anchor sat off-mesh for ~90s. Field names verified live on
/// 2026-07-24: a daemon restarted with these keys in fips.yaml and peered.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RetrySection {
pub base_interval_secs: u64,
pub max_backoff_secs: u64,
pub max_retries: u32,
}
/// Session-handshake resend pacing (`node.rate_limit.*`). Stock 1s x2.0
/// gaps out to 8-16s between resends exactly when a route has just
/// appeared; 400ms x1.5 keeps continuous coverage through the connect
/// window (phone measured session-after-route: 225ms).
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RateLimitSection {
pub handshake_resend_interval_ms: u64,
pub handshake_resend_backoff: f64,
pub handshake_max_resends: u32,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct IdentitySection {
/// With `persistent: true` the daemon reuses the key file at
/// config-dir/fips.key (= `DAEMON_KEY_PATH`) instead of generating an
/// ephemeral identity on every start.
pub persistent: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DiscoverySection {
/// Lookup completion timeout. Lookups fired while the tree position is
/// still settling are doomed; failing them fast (5s, not 10s) lets the
/// 1s-backoff retry find the route the moment it exists.
pub timeout_secs: u64,
pub backoff_base_secs: u64,
pub backoff_max_secs: u64,
pub retry_interval_secs: u64,
pub max_attempts: u8,
pub lan: LanDiscoverySection,
}
/// mDNS / DNS-SD discovery on the local link (`node.discovery.lan.*`), added
/// upstream in v0.4.0 and opt-in there (upstream default is `false`).
///
/// We enable it so co-located nodes peer directly instead of depending on the
/// public anchor being reachable — an anchor blackhole on one network segment
/// otherwise islands a node completely.
///
/// Emitted unconditionally rather than version-gated: v0.3.0's `DiscoveryConfig`
/// has no `lan` field *and* no `deny_unknown_fields`, so a v0.3.0 daemon ignores
/// this key harmlessly (verified against the v0.3.0 source). It therefore starts
/// working on its own when a node upgrades, with no second config migration.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct LanDiscoverySection {
pub enabled: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct TunSection {
pub enabled: bool,
pub name: String,
pub mtu: u16,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DnsSection {
pub enabled: bool,
pub bind_addr: String,
}
/// Both UDP and TCP are enabled: the public anchor answers on TCP/8443 only,
/// and networks that block outbound UDP can still bootstrap over TCP.
/// Upstream dropped the `tor:` transport variant — archipelago's own Tor
/// fallback handles that layer.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct TransportsSection {
pub udp: TransportBind,
pub tcp: TransportBind,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct TransportBind {
/// Upstream takes `bind_addr` ("host:port"), not `enabled` + `port`.
pub bind_addr: String,
}
/// A static peer entry. Never constructed today (see `FipsConfig::peers`), but
/// typed so the shape is checked if static peering is ever needed.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct PeerEntry {
pub npub: String,
pub address: String,
pub transport: String,
}
impl Default for FipsConfig {
fn default() -> Self {
Self {
node: NodeSection {
identity: IdentitySection { persistent: true },
discovery: DiscoverySection {
timeout_secs: 5,
backoff_base_secs: 1,
backoff_max_secs: 30,
retry_interval_secs: 2,
max_attempts: 3,
lan: LanDiscoverySection { enabled: true },
},
retry: RetrySection {
base_interval_secs: 1,
max_backoff_secs: 30,
max_retries: 30,
},
rate_limit: RateLimitSection {
handshake_resend_interval_ms: 400,
handshake_resend_backoff: 1.5,
handshake_max_resends: 10,
},
},
tun: TunSection {
enabled: true,
name: "fips0".to_string(),
mtu: 1280,
},
dns: DnsSection {
enabled: true,
bind_addr: "127.0.0.1".to_string(),
},
transports: TransportsSection {
udp: TransportBind {
bind_addr: format!("0.0.0.0:{PUBLISHED_UDP_PORT}"),
},
tcp: TransportBind {
bind_addr: format!("0.0.0.0:{DEFAULT_TCP_PORT}"),
},
},
peers: Vec::new(),
}
}
}
/// Render the FIPS daemon config. Overwrites any existing file — callers
/// re-run this whenever the key or daemon version changes.
///
/// Node identity comes from the key file on disk; the static peer list stays
/// empty because peers are fed dynamically at runtime.
pub fn render_config_yaml() -> String {
let body = serde_yaml::to_string(&FipsConfig::default())
.expect("FipsConfig is a plain struct tree and cannot fail to serialise");
format!("{CONFIG_HEADER}{body}")
}
/// Install the local FIPS key + rendered config into `/etc/fips/`.
/// Requires the seed-derived key to already exist at `identity_dir/fips_key`.
pub async fn install(identity_dir: &Path) -> Result<()> {
let src_key = identity_dir.join("fips_key");
let src_pub = identity_dir.join("fips_key.pub");
if !src_key.exists() {
anyhow::bail!(
"FIPS key not materialised at {} — run seed onboarding first",
src_key.display()
);
}
// Ensure /etc/fips exists with mode 0755.
sudo_install_dir("/etc/fips").await?;
// Render + write the yaml via a staging file the archipelago user owns,
// then `sudo install` it into place so we never need to write to
// /etc directly.
let yaml = render_config_yaml();
let stage = std::env::temp_dir().join(format!("fips-{}.yaml", std::process::id()));
tokio::fs::write(&stage, yaml)
.await
.context("Failed to stage fips.yaml")?;
let install_result = sudo_install_file(&stage, DAEMON_CONFIG_PATH, "0644").await;
let _ = tokio::fs::remove_file(&stage).await;
install_result?;
// The release-hardening firewall (/etc/fips/fips.nft, provisioned
// out-of-band) default-denies inbound on fips0 — without an explicit
// allowance the node's web UI is unreachable over the mesh (phones got
// RST on :80 with a healthy session; root-caused 2026-07-26 on
// framework-pt). Ship the allowance as a fips.d drop-in on every
// install/upgrade so no node ever regresses to a UI-less mesh.
sudo_install_dir("/etc/fips/fips.d").await?;
// PEER_PORT (5679) carries ALL federation sync, cloud browse/download,
// mesh envelopes, DWN and invoices. It was missing from this allowlist
// while the comment claimed "web UI + peer API" — so every hardened
// node silently dropped peers' FIPS dials at the firewall and the whole
// fleet fell back to Tor (root-caused live 2026-07-27: 28k drops on
// .198's counter; :5679 answered in 0.35s once the rule was inserted).
let dropin = format!(
"# Written by archipelago on every daemon config install.\n\
# Allows the web UI + peer API through the fips0\n\
# default-deny inbound baseline (fips.nft).\n\
tcp dport 80 accept\n\
tcp dport 8443 accept\n\
tcp dport {peer_port} accept\n",
peer_port = crate::fips::dial::PEER_PORT
);
let nft_stage = std::env::temp_dir().join(format!("fips-webui-{}.nft", std::process::id()));
tokio::fs::write(&nft_stage, dropin)
.await
.context("Failed to stage web-ui nft drop-in")?;
let nft_install = sudo_install_file(&nft_stage, "/etc/fips/fips.d/80-web-ui.nft", "0644").await;
let _ = tokio::fs::remove_file(&nft_stage).await;
nft_install?;
// App launch ports: the companion opens catalog apps by direct port
// over the mesh. Ports of apps that aren't installed have no listener,
// so the allowance is inert until an app exists to answer.
let port_list = super::app_ports::APP_LAUNCH_PORTS
.iter()
.map(|p| p.to_string())
.collect::<Vec<_>>()
.join(", ");
let app_dropin = format!(
"# Written by archipelago on every daemon config install.\n\
# Catalog app launch ports (web UIs) allowed through the fips0\n\
# default-deny inbound baseline. Service/RPC ports stay closed.\n\
tcp dport {{ {port_list} }} accept\n"
);
let app_stage = std::env::temp_dir().join(format!("fips-appports-{}.nft", std::process::id()));
tokio::fs::write(&app_stage, app_dropin)
.await
.context("Failed to stage app-ports nft drop-in")?;
let app_install =
sudo_install_file(&app_stage, "/etc/fips/fips.d/85-app-ports.nft", "0644").await;
let _ = tokio::fs::remove_file(&app_stage).await;
app_install?;
// Make the allowance live immediately; a no-op error when the
// hardening baseline isn't installed on this node yet.
if tokio::fs::try_exists("/etc/fips/fips.nft").await.unwrap_or(false) {
match Command::new("sudo")
.args(["nft", "-f", "/etc/fips/fips.nft"])
.output()
.await
{
Ok(out) if !out.status.success() => tracing::warn!(
"nft reload after web-ui drop-in failed: {}",
String::from_utf8_lossy(&out.stderr).trim()
),
Err(e) => tracing::warn!("nft reload after web-ui drop-in failed: {e}"),
_ => {}
}
}
sudo_install_file(&src_key, DAEMON_KEY_PATH, "0600").await?;
// Heal a legacy fips_key.pub that was written as bech32 npub text
// (pre-fix identity::write_fips_key_from_seed did this). Upstream
// fips expects 32 raw bytes; a text file silently passes through
// and then the daemon can't identify itself to peers. This
// rewrites the source file in place with the correct binary form
// derived from fips_key before staging it to /etc/fips/fips.pub.
normalize_pub_file(&src_key, &src_pub).await?;
sudo_install_file(&src_pub, DAEMON_PUB_PATH, "0644").await?;
Ok(())
}
/// Ensure `fips_key.pub` is 32 raw bytes. If it's a bech32 npub text
/// file (from the pre-fix writer), decode it and rewrite in place. If
/// the file is missing or its content doesn't match either format,
/// re-derive the public key from `fips_key` and write that.
pub async fn normalize_pub_file(key_path: &Path, pub_path: &Path) -> Result<()> {
// Happy path: already 32 raw bytes.
if let Ok(bytes) = tokio::fs::read(pub_path).await {
if bytes.len() == 32 {
return Ok(());
}
// bech32 npub text from the pre-fix writer: decode in place.
if let Ok(s) = std::str::from_utf8(&bytes) {
let trimmed = s.trim();
if trimmed.starts_with("npub1") {
if let Ok(pk) = nostr_sdk::PublicKey::parse(trimmed) {
let raw: [u8; 32] = pk.to_bytes();
tokio::fs::write(pub_path, raw)
.await
.context("rewriting fips_key.pub as 32 raw bytes")?;
tracing::info!(
"Migrated legacy bech32 fips_key.pub to raw-byte form at {}",
pub_path.display()
);
return Ok(());
}
}
}
}
// Fallback: no pub file, or unreadable format. Re-derive from the
// private key file (already validated by load_fips_keys).
let secret_bytes = tokio::fs::read(key_path)
.await
.with_context(|| format!("read {} to derive public", key_path.display()))?;
let text = std::str::from_utf8(&secret_bytes)
.context("fips_key is not UTF-8 — can't derive public")?;
let secret = nostr_sdk::SecretKey::parse(text.trim())
.context("fips_key not parseable as bech32 nsec")?;
let keys = nostr_sdk::Keys::new(secret);
let raw: [u8; 32] = keys.public_key().to_bytes();
tokio::fs::write(pub_path, raw)
.await
.context("writing re-derived fips_key.pub")?;
tracing::info!("Re-derived fips_key.pub from fips_key");
Ok(())
}
async fn sudo_install_dir(path: &str) -> Result<()> {
let out = Command::new("sudo")
.args(["install", "-d", "-m", "0755", path])
.output()
.await
.with_context(|| format!("sudo install -d {}", path))?;
if !out.status.success() {
anyhow::bail!(
"sudo install -d {}: {}",
path,
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(())
}
async fn sudo_install_file(src: &Path, dest: &str, mode: &str) -> Result<()> {
let out = Command::new("sudo")
.args([
"install",
"-m",
mode,
src.to_str().context("Non-UTF8 source path")?,
dest,
])
.output()
.await
.with_context(|| format!("sudo install {} -> {}", src.display(), dest))?;
if !out.status.success() {
anyhow::bail!(
"sudo install {} -> {}: {}",
src.display(),
dest,
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rendered_yaml_matches_upstream_schema() {
let yaml = render_config_yaml();
assert!(yaml.contains("persistent: true"));
assert!(yaml.contains(&format!("0.0.0.0:{}", PUBLISHED_UDP_PORT)));
assert!(yaml.contains(&format!("0.0.0.0:{}", DEFAULT_TCP_PORT)));
assert!(yaml.contains("udp:"));
assert!(yaml.contains("tcp:"));
assert!(yaml.contains("tun:"));
assert!(yaml.contains("name: fips0"));
// Upstream fips dropped the `tor:` transport variant; archipelago
// handles Tor fallback itself. Make sure we didn't regress.
assert!(!yaml.contains("tor:"));
}
/// Exact-output snapshot. Upstream's config structs are
/// `deny_unknown_fields`, so an accidental key rename/addition means the
/// daemon won't start. Pinning the full rendering makes any such change
/// fail here — where it's cheap — instead of on a node after an upgrade.
/// If this fails, re-verify against the upstream schema before updating it.
#[test]
fn test_rendered_yaml_exact_snapshot() {
let expected = "\
# Generated by archipelago — do not edit by hand.
# Regenerated on every key change and daemon upgrade.
node:
identity:
persistent: true
discovery:
timeout_secs: 5
backoff_base_secs: 1
backoff_max_secs: 30
retry_interval_secs: 2
max_attempts: 3
lan:
enabled: true
retry:
base_interval_secs: 1
max_backoff_secs: 30
max_retries: 30
rate_limit:
handshake_resend_interval_ms: 400
handshake_resend_backoff: 1.5
handshake_max_resends: 10
tun:
enabled: true
name: fips0
mtu: 1280
dns:
enabled: true
bind_addr: 127.0.0.1
transports:
udp:
bind_addr: 0.0.0.0:2121
tcp:
bind_addr: 0.0.0.0:8443
peers: []
";
assert_eq!(render_config_yaml(), expected);
}
/// The rendered config must parse as YAML and carry the mDNS opt-in at the
/// exact path upstream reads (`node.discovery.lan.enabled`) — a typo there
/// would silently leave LAN discovery off rather than erroring.
#[test]
fn test_lan_discovery_enabled_at_upstream_path() {
let yaml = render_config_yaml();
let parsed: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("renders valid YAML");
assert_eq!(
parsed["node"]["discovery"]["lan"]["enabled"],
serde_yaml::Value::Bool(true),
);
}
/// Rendering is deterministic: the startup drift check in server.rs compares
/// the freshly rendered config against what's on disk, so any instability
/// here would cause an endless reinstall+restart loop of the daemon.
#[test]
fn test_render_is_deterministic() {
assert_eq!(render_config_yaml(), render_config_yaml());
}
#[tokio::test]
async fn test_install_refuses_when_key_missing() {
let dir = tempfile::tempdir().unwrap();
let err = install(dir.path()).await.unwrap_err();
assert!(err.to_string().contains("FIPS key not materialised"));
}
}