feat(fips): fallback telemetry — per-reason counters in fips.status + last-transport recording on all dial sites

Phase A2 of docs/FIPS-UPTIME-AND-UI-STATE-PLAN.md (RC6). Fallbacks to Tor
were debug!-only and uncounted, so "FIPS uptime" was unfalsifiable and
paths that were 100% Tor by construction went unnoticed for months.

- fips::telemetry: process-lifetime counters for FIPS successes and the
  six fallback reasons (no_npub, service_inactive, dns_fail, connect_fail,
  http_404, http_5xx), exposed as `dial_stats` in fips.status
- dial.rs: every fallback branch now counts + logs at info! with a
  `reason` field (resolve/connect/status branches)
- PeerRequest::record_transport(data_dir): opt-in hook that writes the
  transport actually used to federation storage off the hot path — wired
  into the dial sites that never recorded (DWN sync ×3, mesh blob fetch,
  federation deploy notify, onion-rotation notify, node messages via a
  new send_to_peer data-dir param)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/archipelago/src/fips/dial.rs

@@ -24,6 +24,7 @@
 //! ```
 #![allow(dead_code)]
 
+use super::telemetry::{self, FallbackReason};
 use anyhow::{Context, Result};
 use std::net::{IpAddr, Ipv6Addr};
 use std::time::Duration;
@@ -317,6 +318,11 @@ pub struct PeerRequest<'a> {
     /// large content download needs so its long FIPS transfer isn't truncated.
     pub fips_timeout: Option<std::time::Duration>,
     pub service: Option<crate::settings::transport::PeerService>,
+    /// When set, the transport that actually served this request is written
+    /// to federation storage (`record_peer_transport`, matched by onion) so
+    /// the per-peer FIPS/Tor badge reflects reality. Opt-in because not
+    /// every caller has a data dir in scope.
+    pub record_data_dir: Option<std::path::PathBuf>,
 }
 
 impl<'a> PeerRequest<'a> {
@@ -329,6 +335,31 @@ impl<'a> PeerRequest<'a> {
             timeout: std::time::Duration::from_secs(30),
             fips_timeout: None,
             service: None,
+            record_data_dir: None,
+        }
+    }
+
+    /// Record the transport that serves this request into federation storage
+    /// (matched by this request's onion host). Best-effort, off the hot path.
+    pub fn record_transport(mut self, data_dir: impl Into<std::path::PathBuf>) -> Self {
+        self.record_data_dir = Some(data_dir.into());
+        self
+    }
+
+    fn spawn_record(&self, kind: crate::transport::TransportKind) {
+        if let Some(dir) = &self.record_data_dir {
+            let dir = dir.clone();
+            let onion = self.onion_host.to_string();
+            let transport = kind.to_string();
+            tokio::spawn(async move {
+                let _ = crate::federation::record_peer_transport(
+                    &dir,
+                    None,
+                    Some(&onion),
+                    &transport,
+                )
+                .await;
+            });
         }
     }
 
@@ -389,8 +420,22 @@ impl<'a> PeerRequest<'a> {
                     // fix (404 path-not-served / 5xx) and we're allowed to
                     // fall back. FIPS-only never falls back.
                     if pref == TransportPref::Fips || !fips_should_fall_back(resp.status()) {
+                        telemetry::record_fips_ok();
+                        self.spawn_record(crate::transport::TransportKind::Fips);
                         return Ok((resp, crate::transport::TransportKind::Fips));
                     }
+                    let reason = if resp.status() == reqwest::StatusCode::NOT_FOUND {
+                        FallbackReason::Http404
+                    } else {
+                        FallbackReason::Http5xx
+                    };
+                    telemetry::record_fallback(reason);
+                    tracing::info!(
+                        reason = reason.key(),
+                        status = %resp.status(),
+                        "FIPS POST {} answered but status triggers Tor fallback",
+                        self.path
+                    );
                 }
                 None => {
                     if pref == TransportPref::Fips {
@@ -402,6 +447,7 @@ impl<'a> PeerRequest<'a> {
             }
         }
         let resp = self.send_tor_post_json(body).await?;
+        self.spawn_record(crate::transport::TransportKind::Tor);
         Ok((resp, crate::transport::TransportKind::Tor))
     }
 
@@ -413,8 +459,22 @@ impl<'a> PeerRequest<'a> {
             match self.try_fips_get().await? {
                 Some(resp) => {
                     if pref == TransportPref::Fips || !fips_should_fall_back(resp.status()) {
+                        telemetry::record_fips_ok();
+                        self.spawn_record(crate::transport::TransportKind::Fips);
                         return Ok((resp, crate::transport::TransportKind::Fips));
                     }
+                    let reason = if resp.status() == reqwest::StatusCode::NOT_FOUND {
+                        FallbackReason::Http404
+                    } else {
+                        FallbackReason::Http5xx
+                    };
+                    telemetry::record_fallback(reason);
+                    tracing::info!(
+                        reason = reason.key(),
+                        status = %resp.status(),
+                        "FIPS GET {} answered but status triggers Tor fallback",
+                        self.path
+                    );
                 }
                 None => {
                     if pref == TransportPref::Fips {
@@ -426,6 +486,7 @@ impl<'a> PeerRequest<'a> {
             }
         }
         let resp = self.send_tor_get().await?;
+        self.spawn_record(crate::transport::TransportKind::Tor);
         Ok((resp, crate::transport::TransportKind::Tor))
     }
 
@@ -434,15 +495,23 @@ impl<'a> PeerRequest<'a> {
         body: &B,
     ) -> Result<Option<reqwest::Response>> {
         let Some(npub) = self.fips_npub else {
+            telemetry::record_fallback(FallbackReason::NoNpub);
             return Ok(None);
         };
         if !is_service_active().await {
+            telemetry::record_fallback(FallbackReason::ServiceInactive);
             return Ok(None);
         }
         let base = match peer_base_url(npub).await {
             Ok(b) => b,
             Err(e) => {
-                tracing::debug!("FIPS resolve for {} failed: {}", npub, e);
+                telemetry::record_fallback(FallbackReason::DnsFail);
+                tracing::info!(
+                    reason = FallbackReason::DnsFail.key(),
+                    "FIPS resolve for {} failed: {}, falling back to Tor",
+                    npub,
+                    e
+                );
                 return Ok(None);
             }
         };
@@ -467,7 +536,9 @@ impl<'a> PeerRequest<'a> {
         match tokio::time::timeout(budget, send_with_retry(rb)).await {
             Ok(Ok(r)) => Ok(Some(r)),
             Ok(Err(e)) => {
-                tracing::debug!(
+                telemetry::record_fallback(FallbackReason::ConnectFail);
+                tracing::info!(
+                    reason = FallbackReason::ConnectFail.key(),
                     "FIPS POST {} failed after retry: {}, falling back to Tor",
                     url,
                     e
@@ -475,7 +546,9 @@ impl<'a> PeerRequest<'a> {
                 Ok(None)
             }
             Err(_) => {
-                tracing::debug!(
+                telemetry::record_fallback(FallbackReason::ConnectFail);
+                tracing::info!(
+                    reason = FallbackReason::ConnectFail.key(),
                     "FIPS POST {} exceeded attempt budget {:?}, falling back to Tor",
                     url,
                     budget
@@ -487,15 +560,23 @@ impl<'a> PeerRequest<'a> {
 
     async fn try_fips_get(&self) -> Result<Option<reqwest::Response>> {
         let Some(npub) = self.fips_npub else {
+            telemetry::record_fallback(FallbackReason::NoNpub);
             return Ok(None);
         };
         if !is_service_active().await {
+            telemetry::record_fallback(FallbackReason::ServiceInactive);
             return Ok(None);
         }
         let base = match peer_base_url(npub).await {
             Ok(b) => b,
             Err(e) => {
-                tracing::debug!("FIPS resolve for {} failed: {}", npub, e);
+                telemetry::record_fallback(FallbackReason::DnsFail);
+                tracing::info!(
+                    reason = FallbackReason::DnsFail.key(),
+                    "FIPS resolve for {} failed: {}, falling back to Tor",
+                    npub,
+                    e
+                );
                 return Ok(None);
             }
         };
@@ -516,7 +597,9 @@ impl<'a> PeerRequest<'a> {
         match tokio::time::timeout(budget, send_with_retry(rb)).await {
             Ok(Ok(r)) => Ok(Some(r)),
             Ok(Err(e)) => {
-                tracing::debug!(
+                telemetry::record_fallback(FallbackReason::ConnectFail);
+                tracing::info!(
+                    reason = FallbackReason::ConnectFail.key(),
                     "FIPS GET {} failed after retry: {}, falling back to Tor",
                     url,
                     e
@@ -524,7 +607,9 @@ impl<'a> PeerRequest<'a> {
                 Ok(None)
             }
             Err(_) => {
-                tracing::debug!(
+                telemetry::record_fallback(FallbackReason::ConnectFail);
+                tracing::info!(
+                    reason = FallbackReason::ConnectFail.key(),
                     "FIPS GET {} exceeded attempt budget {:?}, falling back to Tor",
                     url,
                     budget

core/archipelago/src/fips/mod.rs

@@ -31,6 +31,7 @@ pub mod config;
 pub mod dial;
 pub mod iface;
 pub mod service;
+pub mod telemetry;
 pub mod update;
 
 use serde::{Deserialize, Serialize};

core/archipelago/src/fips/telemetry.rs

@@ -0,0 +1,144 @@
+//! In-process counters for FIPS dial outcomes.
+//!
+//! Every peer dial that could have used FIPS either succeeds over FIPS or
+//! falls back to Tor for one of six reasons (F1–F6). Before these counters
+//! existed, fallbacks were `debug!`-only and invisible in production, which
+//! made "FIPS uptime" unfalsifiable — several paths were 100% Tor for months
+//! (dead ports, firewalled listeners, allowlist 404s) and nothing surfaced
+//! it. The counters are process-lifetime (reset on restart) and exposed via
+//! `fips.status` as `dial_stats`, so a fleet-wide fallback regression shows
+//! up on the dashboard instead of as vague slowness.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Why a FIPS-capable dial fell back to Tor.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum FallbackReason {
+    /// F1 — no FIPS npub known for the peer (never meshed, or pre-npub
+    /// federation record). Expected for non-FIPS peers; high counts here
+    /// mean npub propagation is broken, not the transport.
+    NoNpub,
+    /// F2 — the local FIPS daemon service isn't active.
+    ServiceInactive,
+    /// F3 — the local FIPS DNS resolver couldn't resolve the peer's npub
+    /// (daemon up but peer not in the identity cache / mesh unreachable).
+    DnsFail,
+    /// F4 — TCP/HTTP dial to the peer's ULA failed or exceeded the FIPS
+    /// attempt budget (firewalled :5679, cold hole-punch, peer down).
+    ConnectFail,
+    /// F5 — peer answered over FIPS with 404: its listener doesn't serve
+    /// this path (older build / stricter allowlist).
+    Http404,
+    /// F6 — peer answered over FIPS with a 5xx server error.
+    Http5xx,
+}
+
+impl FallbackReason {
+    pub fn key(self) -> &'static str {
+        match self {
+            Self::NoNpub => "no_npub",
+            Self::ServiceInactive => "service_inactive",
+            Self::DnsFail => "dns_fail",
+            Self::ConnectFail => "connect_fail",
+            Self::Http404 => "http_404",
+            Self::Http5xx => "http_5xx",
+        }
+    }
+}
+
+static FIPS_OK: AtomicU64 = AtomicU64::new(0);
+static NO_NPUB: AtomicU64 = AtomicU64::new(0);
+static SERVICE_INACTIVE: AtomicU64 = AtomicU64::new(0);
+static DNS_FAIL: AtomicU64 = AtomicU64::new(0);
+static CONNECT_FAIL: AtomicU64 = AtomicU64::new(0);
+static HTTP_404: AtomicU64 = AtomicU64::new(0);
+static HTTP_5XX: AtomicU64 = AtomicU64::new(0);
+
+fn counter(reason: FallbackReason) -> &'static AtomicU64 {
+    match reason {
+        FallbackReason::NoNpub => &NO_NPUB,
+        FallbackReason::ServiceInactive => &SERVICE_INACTIVE,
+        FallbackReason::DnsFail => &DNS_FAIL,
+        FallbackReason::ConnectFail => &CONNECT_FAIL,
+        FallbackReason::Http404 => &HTTP_404,
+        FallbackReason::Http5xx => &HTTP_5XX,
+    }
+}
+
+/// A dial completed over FIPS (any HTTP status that wasn't a fallback
+/// trigger — the peer was reached on the mesh).
+pub fn record_fips_ok() {
+    FIPS_OK.fetch_add(1, Ordering::Relaxed);
+}
+
+/// A FIPS-capable dial fell back to Tor.
+pub fn record_fallback(reason: FallbackReason) {
+    counter(reason).fetch_add(1, Ordering::Relaxed);
+}
+
+/// Snapshot for `fips.status` (`dial_stats`). Process-lifetime counts.
+pub fn snapshot() -> serde_json::Value {
+    let f1 = NO_NPUB.load(Ordering::Relaxed);
+    let f2 = SERVICE_INACTIVE.load(Ordering::Relaxed);
+    let f3 = DNS_FAIL.load(Ordering::Relaxed);
+    let f4 = CONNECT_FAIL.load(Ordering::Relaxed);
+    let f5 = HTTP_404.load(Ordering::Relaxed);
+    let f6 = HTTP_5XX.load(Ordering::Relaxed);
+    serde_json::json!({
+        "fips_ok": FIPS_OK.load(Ordering::Relaxed),
+        "fallbacks": {
+            "no_npub": f1,
+            "service_inactive": f2,
+            "dns_fail": f3,
+            "connect_fail": f4,
+            "http_404": f5,
+            "http_5xx": f6,
+            "total": f1 + f2 + f3 + f4 + f5 + f6,
+        },
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn snapshot_counts_recorded_events() {
+        // Counters are global; assert deltas rather than absolutes so this
+        // test stays correct alongside any other test that dials.
+        let before = snapshot();
+        record_fips_ok();
+        record_fallback(FallbackReason::ConnectFail);
+        record_fallback(FallbackReason::Http404);
+        let after = snapshot();
+        let d = |v: &serde_json::Value, path: &[&str]| -> u64 {
+            let mut cur = v;
+            for p in path {
+                cur = &cur[p];
+            }
+            cur.as_u64().unwrap()
+        };
+        assert_eq!(d(&after, &["fips_ok"]) - d(&before, &["fips_ok"]), 1);
+        assert_eq!(
+            d(&after, &["fallbacks", "connect_fail"]) - d(&before, &["fallbacks", "connect_fail"]),
+            1
+        );
+        assert_eq!(
+            d(&after, &["fallbacks", "http_404"]) - d(&before, &["fallbacks", "http_404"]),
+            1
+        );
+        assert!(d(&after, &["fallbacks", "total"]) >= 2);
+    }
+
+    #[test]
+    fn reason_keys_are_stable() {
+        // These strings are the fips.status API surface — renaming one is a
+        // breaking change for the UI.
+        assert_eq!(FallbackReason::NoNpub.key(), "no_npub");
+        assert_eq!(FallbackReason::ServiceInactive.key(), "service_inactive");
+        assert_eq!(FallbackReason::DnsFail.key(), "dns_fail");
+        assert_eq!(FallbackReason::ConnectFail.key(), "connect_fail");
+        assert_eq!(FallbackReason::Http404.key(), "http_404");
+        assert_eq!(FallbackReason::Http5xx.key(), "http_5xx");
+    }
+}