feat: Phase 4 backend hardening — container reliability + security audit

Container Management (CONT-01 through CONT-06):
- Fix needs_archy_net: add lnd, nbxplorer to archy-net list
- Add StartupTier dependency ordering to health monitor (DB→Core→Dependent→App→UI)
- Add exponential backoff (10s/30s/90s) with 1hr stability reset
- Add get_health_check_args() with health checks for 20+ apps
- Add get_memory_limit() with per-app limits (128m-4g vs blanket 2g)
- Create docs/network-topology.md
- Fix fedimint containers on both nodes (moved to archy-net)

Security Audit (SEC-01 through SEC-06):
- Add sanitize_error_message() — strips internal paths from RPC errors
- Add validate_identity_id() — blocks path traversal on identity operations
- Add validate_did() — blocks path traversal on federation operations
- Add message size limits: node-send-message (1MB), dwn.write-message (10MB)
- Add rate limits for federation endpoints (join: 5/60s, invite: 10/300s)
- Configure journald (500MB max, 7 day retention) on both nodes
- Add /etc/logrotate.d/archipelago for backend + crowdsec logs
- Verify all 4 nginx security headers on both nodes

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

core/archipelago/src/health_monitor.rs

@@ -1,5 +1,6 @@
 // Container Health Monitor
-// Checks container health every 60s, auto-restarts unhealthy containers (max 3 times),
+// Checks container health every 60s, auto-restarts unhealthy containers (max 3 times)
+// with exponential backoff (10s, 30s, 90s), dependency-aware startup ordering,
 // and sends WebSocket notifications to the UI on failure.
 
 use crate::data_model::{Notification, NotificationLevel};
@@ -8,20 +9,67 @@ use crate::webhooks::{self, WebhookEvent};
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::time::Instant;
 use tracing::{debug, info, warn};
 
 const MAX_RESTART_ATTEMPTS: u32 = 3;
 const CHECK_INTERVAL_SECS: u64 = 60;
+/// Backoff delays per attempt: 10s, 30s, 90s
+const BACKOFF_DELAYS_SECS: [u64; 3] = [10, 30, 90];
+/// Reset restart counter after 1 hour of stability
+const STABILITY_RESET_SECS: u64 = 3600;
 
-/// Track restart attempts per container to avoid infinite restart loops.
+/// Container startup tier for dependency ordering.
+/// Lower tiers start first. Containers in the same tier start together.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum StartupTier {
+    /// Databases: postgres, redis, mariadb, mysql
+    Database = 0,
+    /// Core infrastructure: bitcoin-knots, bitcoin-core
+    CoreInfra = 1,
+    /// Services depending on core: lnd, electrs, nbxplorer
+    DependentService = 2,
+    /// Application layer: mempool-api, btcpay-server, fedimint, nextcloud, etc.
+    Application = 3,
+    /// UI/frontend containers: mempool-web, bitcoin-ui, lnd-ui
+    Frontend = 4,
+}
+
+fn container_tier(name: &str) -> StartupTier {
+    let id = name.strip_prefix("archy-").unwrap_or(name);
+    match id {
+        // Tier 0: Databases
+        "btcpay-db" | "mempool-db" | "penpot-postgres" | "immich_postgres"
+        | "immich_redis" | "penpot-valkey" | "endurain-db" | "nextcloud-db" => StartupTier::Database,
+
+        // Tier 1: Core infrastructure
+        "bitcoin-knots" | "bitcoin-core" | "bitcoin" => StartupTier::CoreInfra,
+
+        // Tier 2: Dependent services
+        "lnd" | "mempool-electrs" | "electrs" | "nbxplorer" => StartupTier::DependentService,
+
+        // Tier 4: Frontend/UI
+        "mempool-web" | "bitcoin-ui" | "lnd-ui" | "electrs-ui"
+        | "penpot-frontend" | "penpot-exporter" => StartupTier::Frontend,
+
+        // Tier 3: Everything else
+        _ => StartupTier::Application,
+    }
+}
+
+/// Track restart attempts per container with exponential backoff and stability reset.
 struct RestartTracker {
     attempts: HashMap<String, u32>,
+    last_failure: HashMap<String, Instant>,
+    last_healthy: HashMap<String, Instant>,
 }
 
 impl RestartTracker {
     fn new() -> Self {
         Self {
             attempts: HashMap::new(),
+            last_failure: HashMap::new(),
+            last_healthy: HashMap::new(),
         }
     }
 
@@ -29,17 +77,50 @@ impl RestartTracker {
     fn record_attempt(&mut self, name: &str) -> bool {
         let count = self.attempts.entry(name.to_string()).or_insert(0);
         *count += 1;
+        self.last_failure.insert(name.to_string(), Instant::now());
         *count <= MAX_RESTART_ATTEMPTS
     }
 
     /// Clear restart count when a container is healthy again.
     fn clear(&mut self, name: &str) {
         self.attempts.remove(name);
+        self.last_failure.remove(name);
+        self.last_healthy.insert(name.to_string(), Instant::now());
     }
 
     fn attempt_count(&self, name: &str) -> u32 {
         *self.attempts.get(name).unwrap_or(&0)
     }
+
+    /// Get the backoff delay in seconds for the current attempt number.
+    fn backoff_delay_secs(&self, name: &str) -> u64 {
+        let attempts = self.attempt_count(name);
+        if attempts == 0 {
+            return BACKOFF_DELAYS_SECS[0];
+        }
+        let idx = (attempts as usize).saturating_sub(1).min(BACKOFF_DELAYS_SECS.len() - 1);
+        BACKOFF_DELAYS_SECS[idx]
+    }
+
+    /// Check if enough time has passed since last failure for the backoff delay.
+    fn backoff_elapsed(&self, name: &str) -> bool {
+        let delay = self.backoff_delay_secs(name);
+        match self.last_failure.get(name) {
+            Some(last) => last.elapsed().as_secs() >= delay,
+            None => true,
+        }
+    }
+
+    /// Check if a failed container should have its counter reset (1h stability window).
+    fn should_reset_failed(&self, name: &str) -> bool {
+        if self.attempt_count(name) < MAX_RESTART_ATTEMPTS {
+            return false;
+        }
+        match self.last_failure.get(name) {
+            Some(last) => last.elapsed().as_secs() >= STABILITY_RESET_SECS,
+            None => false,
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -152,77 +233,107 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
                 continue;
             }
 
+            // Sort containers by startup tier so databases restart before dependent services
+            let mut unhealthy: Vec<&ContainerHealth> = Vec::new();
             let mut state_changed = false;
             let (mut data, _) = state.get_snapshot().await;
 
             for container in &containers {
                 if container.healthy {
-                    // Clear restart tracker if container recovered
                     if tracker.attempt_count(&container.name) > 0 {
                         info!("Container {} is healthy again after restart", container.name);
                         tracker.clear(&container.name);
                     }
                     continue;
                 }
-
-                // Container is unhealthy (exited/stopped)
-                // Only try auto-restart if we haven't exceeded max attempts
                 if container.state == "exited" || container.state == "stopped" {
-                    let attempts = tracker.attempt_count(&container.name);
+                    unhealthy.push(container);
+                }
+            }
 
-                    if attempts >= MAX_RESTART_ATTEMPTS {
-                        // Already notified, skip
-                        debug!("Container {} exceeded max restart attempts ({})", container.name, MAX_RESTART_ATTEMPTS);
-                        continue;
+            // Sort by startup tier: databases first, then core, then dependent, then apps, then UIs
+            unhealthy.sort_by_key(|c| container_tier(&c.name));
+
+            let mut prev_tier: Option<StartupTier> = None;
+            for container in &unhealthy {
+                let tier = container_tier(&container.name);
+                let attempts = tracker.attempt_count(&container.name);
+
+                // Reset counter after 1 hour for permanently failed containers
+                if tracker.should_reset_failed(&container.name) {
+                    info!("Resetting restart counter for {} after {}s stability window", container.name, STABILITY_RESET_SECS);
+                    tracker.clear(&container.name);
+                }
+
+                if tracker.attempt_count(&container.name) >= MAX_RESTART_ATTEMPTS {
+                    debug!("Container {} exceeded max restart attempts ({})", container.name, MAX_RESTART_ATTEMPTS);
+                    continue;
+                }
+
+                // Wait for backoff delay before retrying
+                if !tracker.backoff_elapsed(&container.name) {
+                    let delay = tracker.backoff_delay_secs(&container.name);
+                    debug!("Container {} waiting for backoff ({}s)", container.name, delay);
+                    continue;
+                }
+
+                // When transitioning to a higher tier, wait briefly for previous tier to stabilize
+                if let Some(prev) = prev_tier {
+                    if tier > prev {
+                        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
                     }
+                }
+                prev_tier = Some(tier);
 
-                    if tracker.record_attempt(&container.name) {
-                        let restarted = restart_container(&container.name).await;
-                        let attempt = tracker.attempt_count(&container.name);
+                if tracker.record_attempt(&container.name) {
+                    let attempt = tracker.attempt_count(&container.name);
+                    info!("Restarting {} (tier {:?}, attempt {}/{}, backoff {}s)",
+                        container.name, tier, attempt, MAX_RESTART_ATTEMPTS,
+                        BACKOFF_DELAYS_SECS.get(attempt.saturating_sub(1) as usize).unwrap_or(&90));
 
-                        if !restarted || attempt >= MAX_RESTART_ATTEMPTS {
-                            // Push notification to UI
-                            let notification = Notification {
-                                id: format!("health-{}-{}", container.app_id, chrono::Utc::now().timestamp()),
-                                level: NotificationLevel::Error,
-                                title: format!("{} is unhealthy", container.app_id),
-                                message: if restarted {
-                                    format!(
-                                        "Container restarted ({}/{} attempts). May need manual attention.",
-                                        attempt, MAX_RESTART_ATTEMPTS
-                                    )
-                                } else {
-                                    format!(
-                                        "Auto-restart failed (attempt {}/{}). Container state: {}",
-                                        attempt, MAX_RESTART_ATTEMPTS, container.state
-                                    )
-                                },
-                                timestamp: chrono::Utc::now().to_rfc3339(),
-                                app_id: Some(container.app_id.clone()),
-                            };
+                    let restarted = restart_container(&container.name).await;
 
-                            // Keep only the latest 20 notifications
-                            data.notifications.push(notification.clone());
-                            if data.notifications.len() > 20 {
-                                data.notifications = data.notifications.split_off(data.notifications.len() - 20);
-                            }
-                            state_changed = true;
+                    if !restarted || attempt >= MAX_RESTART_ATTEMPTS {
+                        let notification = Notification {
+                            id: format!("health-{}-{}", container.app_id, chrono::Utc::now().timestamp()),
+                            level: NotificationLevel::Error,
+                            title: format!("{} is unhealthy", container.app_id),
+                            message: if restarted {
+                                format!(
+                                    "Container restarted ({}/{} attempts). May need manual attention.",
+                                    attempt, MAX_RESTART_ATTEMPTS
+                                )
+                            } else {
+                                format!(
+                                    "Auto-restart failed (attempt {}/{}). Container state: {}",
+                                    attempt, MAX_RESTART_ATTEMPTS, container.state
+                                )
+                            },
+                            timestamp: chrono::Utc::now().to_rfc3339(),
+                            app_id: Some(container.app_id.clone()),
+                        };
 
-                            // Fire-and-forget webhook delivery (checks config internally)
-                            let webhook_payload = webhooks::WebhookPayload {
-                                event: WebhookEvent::ContainerCrash,
-                                title: notification.title,
-                                message: notification.message,
-                                timestamp: notification.timestamp,
-                                node_id: String::new(),
-                                details: Some(serde_json::json!({
-                                    "container": container.name,
-                                    "app_id": container.app_id,
-                                    "state": container.state,
-                                })),
-                            };
-                            webhooks::send_webhook(&data_dir, webhook_payload).await;
+                        data.notifications.push(notification.clone());
+                        if data.notifications.len() > 20 {
+                            data.notifications = data.notifications.split_off(data.notifications.len() - 20);
                         }
+                        state_changed = true;
+
+                        let webhook_payload = webhooks::WebhookPayload {
+                            event: WebhookEvent::ContainerCrash,
+                            title: notification.title,
+                            message: notification.message,
+                            timestamp: notification.timestamp,
+                            node_id: String::new(),
+                            details: Some(serde_json::json!({
+                                "container": container.name,
+                                "app_id": container.app_id,
+                                "state": container.state,
+                                "attempt": attempt,
+                                "tier": format!("{:?}", tier),
+                            })),
+                        };
+                        webhooks::send_webhook(&data_dir, webhook_payload).await;
                     }
                 }
             }
@@ -261,7 +372,6 @@ mod tests {
     #[test]
     fn test_restart_tracker_max_attempts_exceeded() {
         let mut tracker = RestartTracker::new();
-        // First MAX_RESTART_ATTEMPTS attempts should return true
         for i in 1..=MAX_RESTART_ATTEMPTS {
             assert!(
                 tracker.record_attempt("container-a"),
@@ -269,7 +379,6 @@ mod tests {
                 i
             );
         }
-        // Next attempt exceeds max, returns false
         assert!(!tracker.record_attempt("container-a"));
         assert_eq!(tracker.attempt_count("container-a"), MAX_RESTART_ATTEMPTS + 1);
     }
@@ -300,13 +409,11 @@ mod tests {
     #[test]
     fn test_restart_tracker_clear_allows_new_attempts() {
         let mut tracker = RestartTracker::new();
-        // Exhaust attempts
         for _ in 0..=MAX_RESTART_ATTEMPTS {
             tracker.record_attempt("container-y");
         }
         assert!(!tracker.record_attempt("container-y"));
 
-        // Clear and try again
         tracker.clear("container-y");
         assert!(tracker.record_attempt("container-y"));
         assert_eq!(tracker.attempt_count("container-y"), 1);
@@ -315,7 +422,6 @@ mod tests {
     #[test]
     fn test_restart_tracker_clear_nonexistent_is_safe() {
         let mut tracker = RestartTracker::new();
-        // Should not panic
         tracker.clear("nonexistent");
         assert_eq!(tracker.attempt_count("nonexistent"), 0);
     }
@@ -348,7 +454,6 @@ mod tests {
 
     #[test]
     fn test_max_restart_attempts_constant() {
-        // Ensure the constant is a reasonable value (not 0, not too high)
         assert!(MAX_RESTART_ATTEMPTS >= 1);
         assert!(MAX_RESTART_ATTEMPTS <= 10);
         assert_eq!(MAX_RESTART_ATTEMPTS, 3);
@@ -358,4 +463,75 @@ mod tests {
     fn test_check_interval_constant() {
         assert_eq!(CHECK_INTERVAL_SECS, 60);
     }
+
+    #[test]
+    fn test_backoff_delays() {
+        let tracker = RestartTracker::new();
+        // Before any attempts, delay is first backoff
+        assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[0]);
+    }
+
+    #[test]
+    fn test_backoff_delays_escalate() {
+        let mut tracker = RestartTracker::new();
+        tracker.record_attempt("test");
+        assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[0]); // 10s
+        tracker.record_attempt("test");
+        assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[1]); // 30s
+        tracker.record_attempt("test");
+        assert_eq!(tracker.backoff_delay_secs("test"), BACKOFF_DELAYS_SECS[2]); // 90s
+    }
+
+    #[test]
+    fn test_backoff_elapsed_true_for_new() {
+        let tracker = RestartTracker::new();
+        assert!(tracker.backoff_elapsed("test"));
+    }
+
+    #[test]
+    fn test_stability_reset_not_triggered_early() {
+        let mut tracker = RestartTracker::new();
+        tracker.record_attempt("test");
+        assert!(!tracker.should_reset_failed("test"));
+    }
+
+    #[test]
+    fn test_container_tier_database() {
+        assert_eq!(container_tier("archy-btcpay-db"), StartupTier::Database);
+        assert_eq!(container_tier("immich_postgres"), StartupTier::Database);
+        assert_eq!(container_tier("penpot-valkey"), StartupTier::Database);
+    }
+
+    #[test]
+    fn test_container_tier_core() {
+        assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
+    }
+
+    #[test]
+    fn test_container_tier_dependent() {
+        assert_eq!(container_tier("lnd"), StartupTier::DependentService);
+        assert_eq!(container_tier("mempool-electrs"), StartupTier::DependentService);
+        assert_eq!(container_tier("archy-nbxplorer"), StartupTier::DependentService);
+    }
+
+    #[test]
+    fn test_container_tier_frontend() {
+        assert_eq!(container_tier("archy-mempool-web"), StartupTier::Frontend);
+        assert_eq!(container_tier("archy-bitcoin-ui"), StartupTier::Frontend);
+    }
+
+    #[test]
+    fn test_container_tier_application_default() {
+        assert_eq!(container_tier("nextcloud"), StartupTier::Application);
+        assert_eq!(container_tier("grafana"), StartupTier::Application);
+        assert_eq!(container_tier("btcpay-server"), StartupTier::Application);
+    }
+
+    #[test]
+    fn test_tier_ordering() {
+        assert!(StartupTier::Database < StartupTier::CoreInfra);
+        assert!(StartupTier::CoreInfra < StartupTier::DependentService);
+        assert!(StartupTier::DependentService < StartupTier::Application);
+        assert!(StartupTier::Application < StartupTier::Frontend);
+    }
 }