archy/core/archipelago/src/fips/telemetry.rs
archipelago a3f07d5ac6 feat(fips): resilience — connectivity watcher with immediate anchor re-apply, rebindable peer listener, cached service probe, warm-path union
Phase A3 of docs/FIPS-UPTIME-AND-UI-STATE-PLAN.md (RC5), measured against
the A2 dial_stats baseline:

- 25s connectivity watcher in the fips supervisor: re-applies seed anchors
  immediately on an anchor-link drop, on startup-disconnected, AND on
  silent data-path death (connect_fails growing with zero fips_ok — the
  live .198 failure where the daemon reported "connected" while every
  dial blackholed and the 300s tick never healed it). Bounded to one
  re-apply per 60s.
- anchors::apply is now concurrent with a 15s per-connect cap — the old
  serial loop waited unbounded on each `sudo fipsctl connect`, so one
  hung subprocess stalled the whole periodic tick.
- rebindable peer listener: the accept loop returns after persistent
  accept errors (was: continue forever = inbound-dead until restart) and
  peer_late_bind_loop rebinds — also on fips0 ULA change.
- is_service_active gets a 10s TTL cache (was up to 2 systemctl spawns
  per dial attempt and per warm-tick peer).
- the warm tick now warms the union of federation peers + configured
  seed anchors (direct anchor links used to go cold between 300s ticks),
  skipping the redundant per-peer service check.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-28 03:57:12 -04:00

156 lines
5.8 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! In-process counters for FIPS dial outcomes.
//!
//! Every peer dial that could have used FIPS either succeeds over FIPS or
//! falls back to Tor for one of six reasons (F1F6). Before these counters
//! existed, fallbacks were `debug!`-only and invisible in production, which
//! made "FIPS uptime" unfalsifiable — several paths were 100% Tor for months
//! (dead ports, firewalled listeners, allowlist 404s) and nothing surfaced
//! it. The counters are process-lifetime (reset on restart) and exposed via
//! `fips.status` as `dial_stats`, so a fleet-wide fallback regression shows
//! up on the dashboard instead of as vague slowness.
use std::sync::atomic::{AtomicU64, Ordering};
/// Why a FIPS-capable dial fell back to Tor.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FallbackReason {
/// F1 — no FIPS npub known for the peer (never meshed, or pre-npub
/// federation record). Expected for non-FIPS peers; high counts here
/// mean npub propagation is broken, not the transport.
NoNpub,
/// F2 — the local FIPS daemon service isn't active.
ServiceInactive,
/// F3 — the local FIPS DNS resolver couldn't resolve the peer's npub
/// (daemon up but peer not in the identity cache / mesh unreachable).
DnsFail,
/// F4 — TCP/HTTP dial to the peer's ULA failed or exceeded the FIPS
/// attempt budget (firewalled :5679, cold hole-punch, peer down).
ConnectFail,
/// F5 — peer answered over FIPS with 404: its listener doesn't serve
/// this path (older build / stricter allowlist).
Http404,
/// F6 — peer answered over FIPS with a 5xx server error.
Http5xx,
}
impl FallbackReason {
pub fn key(self) -> &'static str {
match self {
Self::NoNpub => "no_npub",
Self::ServiceInactive => "service_inactive",
Self::DnsFail => "dns_fail",
Self::ConnectFail => "connect_fail",
Self::Http404 => "http_404",
Self::Http5xx => "http_5xx",
}
}
}
static FIPS_OK: AtomicU64 = AtomicU64::new(0);
static NO_NPUB: AtomicU64 = AtomicU64::new(0);
static SERVICE_INACTIVE: AtomicU64 = AtomicU64::new(0);
static DNS_FAIL: AtomicU64 = AtomicU64::new(0);
static CONNECT_FAIL: AtomicU64 = AtomicU64::new(0);
static HTTP_404: AtomicU64 = AtomicU64::new(0);
static HTTP_5XX: AtomicU64 = AtomicU64::new(0);
fn counter(reason: FallbackReason) -> &'static AtomicU64 {
match reason {
FallbackReason::NoNpub => &NO_NPUB,
FallbackReason::ServiceInactive => &SERVICE_INACTIVE,
FallbackReason::DnsFail => &DNS_FAIL,
FallbackReason::ConnectFail => &CONNECT_FAIL,
FallbackReason::Http404 => &HTTP_404,
FallbackReason::Http5xx => &HTTP_5XX,
}
}
/// A dial completed over FIPS (any HTTP status that wasn't a fallback
/// trigger — the peer was reached on the mesh).
pub fn record_fips_ok() {
FIPS_OK.fetch_add(1, Ordering::Relaxed);
}
/// A FIPS-capable dial fell back to Tor.
pub fn record_fallback(reason: FallbackReason) {
counter(reason).fetch_add(1, Ordering::Relaxed);
}
/// `(fips_ok, connect_fail)` totals for the connectivity watcher: a window
/// where connect_fail grows while fips_ok doesn't is a degraded data path —
/// including the "daemon says connected but packets blackhole" failure the
/// link-state check alone can't see (observed live 2026-07-27 on .198).
pub fn totals() -> (u64, u64) {
(
FIPS_OK.load(Ordering::Relaxed),
CONNECT_FAIL.load(Ordering::Relaxed),
)
}
/// Snapshot for `fips.status` (`dial_stats`). Process-lifetime counts.
pub fn snapshot() -> serde_json::Value {
let f1 = NO_NPUB.load(Ordering::Relaxed);
let f2 = SERVICE_INACTIVE.load(Ordering::Relaxed);
let f3 = DNS_FAIL.load(Ordering::Relaxed);
let f4 = CONNECT_FAIL.load(Ordering::Relaxed);
let f5 = HTTP_404.load(Ordering::Relaxed);
let f6 = HTTP_5XX.load(Ordering::Relaxed);
serde_json::json!({
"fips_ok": FIPS_OK.load(Ordering::Relaxed),
"fallbacks": {
"no_npub": f1,
"service_inactive": f2,
"dns_fail": f3,
"connect_fail": f4,
"http_404": f5,
"http_5xx": f6,
"total": f1 + f2 + f3 + f4 + f5 + f6,
},
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn snapshot_counts_recorded_events() {
// Counters are global; assert deltas rather than absolutes so this
// test stays correct alongside any other test that dials.
let before = snapshot();
record_fips_ok();
record_fallback(FallbackReason::ConnectFail);
record_fallback(FallbackReason::Http404);
let after = snapshot();
let d = |v: &serde_json::Value, path: &[&str]| -> u64 {
let mut cur = v;
for p in path {
cur = &cur[p];
}
cur.as_u64().unwrap()
};
assert_eq!(d(&after, &["fips_ok"]) - d(&before, &["fips_ok"]), 1);
assert_eq!(
d(&after, &["fallbacks", "connect_fail"]) - d(&before, &["fallbacks", "connect_fail"]),
1
);
assert_eq!(
d(&after, &["fallbacks", "http_404"]) - d(&before, &["fallbacks", "http_404"]),
1
);
assert!(d(&after, &["fallbacks", "total"]) >= 2);
}
#[test]
fn reason_keys_are_stable() {
// These strings are the fips.status API surface — renaming one is a
// breaking change for the UI.
assert_eq!(FallbackReason::NoNpub.key(), "no_npub");
assert_eq!(FallbackReason::ServiceInactive.key(), "service_inactive");
assert_eq!(FallbackReason::DnsFail.key(), "dns_fail");
assert_eq!(FallbackReason::ConnectFail.key(), "connect_fail");
assert_eq!(FallbackReason::Http404.key(), "http_404");
assert_eq!(FallbackReason::Http5xx.key(), "http_5xx");
}
}