feat(fips): fallback telemetry — per-reason counters in fips.status + last-transport recording on all dial sites
Phase A2 of docs/FIPS-UPTIME-AND-UI-STATE-PLAN.md (RC6). Fallbacks to Tor were debug!-only and uncounted, so "FIPS uptime" was unfalsifiable and paths that were 100% Tor by construction went unnoticed for months. - fips::telemetry: process-lifetime counters for FIPS successes and the six fallback reasons (no_npub, service_inactive, dns_fail, connect_fail, http_404, http_5xx), exposed as `dial_stats` in fips.status - dial.rs: every fallback branch now counts + logs at info! with a `reason` field (resolve/connect/status branches) - PeerRequest::record_transport(data_dir): opt-in hook that writes the transport actually used to federation storage off the hot path — wired into the dial sites that never recorded (DWN sync ×3, mesh blob fetch, federation deploy notify, onion-rotation notify, node messages via a new send_to_peer data-dir param) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago
co-authored by
Claude Fable 5
Claude Fable 5
parent
eb2fc0f37b
commit
e24e0a6473
@@ -24,6 +24,7 @@
|
||||
//! ```
|
||||
#![allow(dead_code)]
|
||||
|
||||
use super::telemetry::{self, FallbackReason};
|
||||
use anyhow::{Context, Result};
|
||||
use std::net::{IpAddr, Ipv6Addr};
|
||||
use std::time::Duration;
|
||||
@@ -317,6 +318,11 @@ pub struct PeerRequest<'a> {
|
||||
/// large content download needs so its long FIPS transfer isn't truncated.
|
||||
pub fips_timeout: Option<std::time::Duration>,
|
||||
pub service: Option<crate::settings::transport::PeerService>,
|
||||
/// When set, the transport that actually served this request is written
|
||||
/// to federation storage (`record_peer_transport`, matched by onion) so
|
||||
/// the per-peer FIPS/Tor badge reflects reality. Opt-in because not
|
||||
/// every caller has a data dir in scope.
|
||||
pub record_data_dir: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
impl<'a> PeerRequest<'a> {
|
||||
@@ -329,6 +335,31 @@ impl<'a> PeerRequest<'a> {
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
fips_timeout: None,
|
||||
service: None,
|
||||
record_data_dir: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the transport that serves this request into federation storage
|
||||
/// (matched by this request's onion host). Best-effort, off the hot path.
|
||||
pub fn record_transport(mut self, data_dir: impl Into<std::path::PathBuf>) -> Self {
|
||||
self.record_data_dir = Some(data_dir.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn spawn_record(&self, kind: crate::transport::TransportKind) {
|
||||
if let Some(dir) = &self.record_data_dir {
|
||||
let dir = dir.clone();
|
||||
let onion = self.onion_host.to_string();
|
||||
let transport = kind.to_string();
|
||||
tokio::spawn(async move {
|
||||
let _ = crate::federation::record_peer_transport(
|
||||
&dir,
|
||||
None,
|
||||
Some(&onion),
|
||||
&transport,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -389,8 +420,22 @@ impl<'a> PeerRequest<'a> {
|
||||
// fix (404 path-not-served / 5xx) and we're allowed to
|
||||
// fall back. FIPS-only never falls back.
|
||||
if pref == TransportPref::Fips || !fips_should_fall_back(resp.status()) {
|
||||
telemetry::record_fips_ok();
|
||||
self.spawn_record(crate::transport::TransportKind::Fips);
|
||||
return Ok((resp, crate::transport::TransportKind::Fips));
|
||||
}
|
||||
let reason = if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
FallbackReason::Http404
|
||||
} else {
|
||||
FallbackReason::Http5xx
|
||||
};
|
||||
telemetry::record_fallback(reason);
|
||||
tracing::info!(
|
||||
reason = reason.key(),
|
||||
status = %resp.status(),
|
||||
"FIPS POST {} answered but status triggers Tor fallback",
|
||||
self.path
|
||||
);
|
||||
}
|
||||
None => {
|
||||
if pref == TransportPref::Fips {
|
||||
@@ -402,6 +447,7 @@ impl<'a> PeerRequest<'a> {
|
||||
}
|
||||
}
|
||||
let resp = self.send_tor_post_json(body).await?;
|
||||
self.spawn_record(crate::transport::TransportKind::Tor);
|
||||
Ok((resp, crate::transport::TransportKind::Tor))
|
||||
}
|
||||
|
||||
@@ -413,8 +459,22 @@ impl<'a> PeerRequest<'a> {
|
||||
match self.try_fips_get().await? {
|
||||
Some(resp) => {
|
||||
if pref == TransportPref::Fips || !fips_should_fall_back(resp.status()) {
|
||||
telemetry::record_fips_ok();
|
||||
self.spawn_record(crate::transport::TransportKind::Fips);
|
||||
return Ok((resp, crate::transport::TransportKind::Fips));
|
||||
}
|
||||
let reason = if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
FallbackReason::Http404
|
||||
} else {
|
||||
FallbackReason::Http5xx
|
||||
};
|
||||
telemetry::record_fallback(reason);
|
||||
tracing::info!(
|
||||
reason = reason.key(),
|
||||
status = %resp.status(),
|
||||
"FIPS GET {} answered but status triggers Tor fallback",
|
||||
self.path
|
||||
);
|
||||
}
|
||||
None => {
|
||||
if pref == TransportPref::Fips {
|
||||
@@ -426,6 +486,7 @@ impl<'a> PeerRequest<'a> {
|
||||
}
|
||||
}
|
||||
let resp = self.send_tor_get().await?;
|
||||
self.spawn_record(crate::transport::TransportKind::Tor);
|
||||
Ok((resp, crate::transport::TransportKind::Tor))
|
||||
}
|
||||
|
||||
@@ -434,15 +495,23 @@ impl<'a> PeerRequest<'a> {
|
||||
body: &B,
|
||||
) -> Result<Option<reqwest::Response>> {
|
||||
let Some(npub) = self.fips_npub else {
|
||||
telemetry::record_fallback(FallbackReason::NoNpub);
|
||||
return Ok(None);
|
||||
};
|
||||
if !is_service_active().await {
|
||||
telemetry::record_fallback(FallbackReason::ServiceInactive);
|
||||
return Ok(None);
|
||||
}
|
||||
let base = match peer_base_url(npub).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::debug!("FIPS resolve for {} failed: {}", npub, e);
|
||||
telemetry::record_fallback(FallbackReason::DnsFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::DnsFail.key(),
|
||||
"FIPS resolve for {} failed: {}, falling back to Tor",
|
||||
npub,
|
||||
e
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
@@ -467,7 +536,9 @@ impl<'a> PeerRequest<'a> {
|
||||
match tokio::time::timeout(budget, send_with_retry(rb)).await {
|
||||
Ok(Ok(r)) => Ok(Some(r)),
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!(
|
||||
telemetry::record_fallback(FallbackReason::ConnectFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::ConnectFail.key(),
|
||||
"FIPS POST {} failed after retry: {}, falling back to Tor",
|
||||
url,
|
||||
e
|
||||
@@ -475,7 +546,9 @@ impl<'a> PeerRequest<'a> {
|
||||
Ok(None)
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!(
|
||||
telemetry::record_fallback(FallbackReason::ConnectFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::ConnectFail.key(),
|
||||
"FIPS POST {} exceeded attempt budget {:?}, falling back to Tor",
|
||||
url,
|
||||
budget
|
||||
@@ -487,15 +560,23 @@ impl<'a> PeerRequest<'a> {
|
||||
|
||||
async fn try_fips_get(&self) -> Result<Option<reqwest::Response>> {
|
||||
let Some(npub) = self.fips_npub else {
|
||||
telemetry::record_fallback(FallbackReason::NoNpub);
|
||||
return Ok(None);
|
||||
};
|
||||
if !is_service_active().await {
|
||||
telemetry::record_fallback(FallbackReason::ServiceInactive);
|
||||
return Ok(None);
|
||||
}
|
||||
let base = match peer_base_url(npub).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::debug!("FIPS resolve for {} failed: {}", npub, e);
|
||||
telemetry::record_fallback(FallbackReason::DnsFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::DnsFail.key(),
|
||||
"FIPS resolve for {} failed: {}, falling back to Tor",
|
||||
npub,
|
||||
e
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
@@ -516,7 +597,9 @@ impl<'a> PeerRequest<'a> {
|
||||
match tokio::time::timeout(budget, send_with_retry(rb)).await {
|
||||
Ok(Ok(r)) => Ok(Some(r)),
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!(
|
||||
telemetry::record_fallback(FallbackReason::ConnectFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::ConnectFail.key(),
|
||||
"FIPS GET {} failed after retry: {}, falling back to Tor",
|
||||
url,
|
||||
e
|
||||
@@ -524,7 +607,9 @@ impl<'a> PeerRequest<'a> {
|
||||
Ok(None)
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!(
|
||||
telemetry::record_fallback(FallbackReason::ConnectFail);
|
||||
tracing::info!(
|
||||
reason = FallbackReason::ConnectFail.key(),
|
||||
"FIPS GET {} exceeded attempt budget {:?}, falling back to Tor",
|
||||
url,
|
||||
budget
|
||||
|
||||
Reference in New Issue
Block a user