fix(orchestrator,content): bound repair-recreate loops; self-heal stale content catalog entries

- prod_orchestrator.rs: the boot reconciler's zombie-guard and start-failed
  recreate paths (Created/Stopped/Exited states) had no attempt cap, unlike
  health_monitor's independent restart tracker. A container whose entrypoint
  fatally crashes right after `podman start` succeeds got stop+remove+
  install_fresh'd every ~30s reconcile tick forever (portainer on .198,
  2026-07-01: a DB schema newer than the pinned binary could read -- no
  amount of recreating fixes that). Added a 5-attempts/30-minute circuit
  breaker; once exhausted the container is left alone with an error! log
  instead of looping, and an explicit install/start clears the counter.
- content_server.rs: serve_content now prunes a catalog entry whose backing
  file is missing on disk, instead of leaving it advertised to every peer
  forever with no way to distinguish "gone" from "transient failure."

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

core/archipelago/src/content_server.rs

@@ -7,7 +7,7 @@ use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 use tokio::fs;
-use tracing::debug;
+use tracing::{debug, warn};
 
 const CATALOG_FILE: &str = "content/catalog.json";
 const CONTENT_DIR: &str = "content/files";
@@ -86,6 +86,22 @@ pub async fn save_catalog(data_dir: &Path, catalog: &ContentCatalog) -> Result<(
     Ok(())
 }
 
+/// Removes `id` from the on-disk catalog. Best-effort: a failure here just
+/// means the entry gets pruned again next time it's requested, so errors are
+/// logged rather than propagated.
+async fn prune_missing_content_entry(data_dir: &Path, id: &str) {
+    let Ok(mut catalog) = load_catalog(data_dir).await else {
+        return;
+    };
+    let before = catalog.items.len();
+    catalog.items.retain(|i| i.id != id);
+    if catalog.items.len() != before {
+        if let Err(e) = save_catalog(data_dir, &catalog).await {
+            warn!(error = %e, content_id = %id, "failed to save catalog after pruning missing content entry");
+        }
+    }
+}
+
 /// Get the full filesystem path for a content item.
 /// Checks the dedicated content/files/ directory first, then falls back to the
 /// FileBrowser data directory (where users manage files via the web UI).
@@ -268,6 +284,19 @@ pub async fn serve_content(
 
     let file_path = content_file_path(data_dir, item);
     if !file_path.exists() {
+        // The catalog entry survived (it's a separate JSON file) but its
+        // backing file is gone — most likely lost in an unrelated data-dir
+        // reset (a shared filebrowser file, 2026-07-01: two catalog entries
+        // outlived a filebrowser reinstall that wiped the files themselves).
+        // Leaving the entry in place would keep advertising it as available
+        // to every peer forever, each hitting the exact same dead end this
+        // one just did. Prune it so it stops being offered.
+        warn!(
+            content_id = %id,
+            filename = %item.filename,
+            "content catalog entry's file is missing on disk — pruning the stale entry"
+        );
+        prune_missing_content_entry(data_dir, id).await;
         return Ok(ServeResult::NotFound);
     }
 
@@ -555,3 +584,95 @@ mod faststart_tests {
         assert_eq!(mp4_is_faststart(&p).await, Some(false));
     }
 }
+
+#[cfg(test)]
+mod prune_missing_content_tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn serve_content_prunes_catalog_entry_whose_file_is_missing() {
+        // Simulates a catalog entry that outlived its backing file (a shared
+        // filebrowser file lost in an unrelated data-dir reset, 2026-07-01) —
+        // every peer request for it would otherwise 404 forever with no way
+        // to tell it apart from a transient failure.
+        let dir = tempfile::tempdir().unwrap();
+        let data_dir = dir.path();
+        let item = ContentItem {
+            id: "missing-item".to_string(),
+            filename: "gone.mp4".to_string(),
+            mime_type: "video/mp4".to_string(),
+            size_bytes: 123,
+            description: String::new(),
+            access: AccessControl::Free,
+            availability: Availability::AllPeers,
+            added_at: "2026-01-01T00:00:00Z".to_string(),
+        };
+        save_catalog(
+            data_dir,
+            &ContentCatalog {
+                items: vec![item],
+            },
+        )
+        .await
+        .unwrap();
+
+        // File was never written to disk under content/files/ or filebrowser/.
+        let result = serve_content(data_dir, "missing-item", None, None, None, None)
+            .await
+            .unwrap();
+        assert!(matches!(result, ServeResult::NotFound));
+
+        let reloaded = load_catalog(data_dir).await.unwrap();
+        assert!(
+            reloaded.items.is_empty(),
+            "stale entry should have been pruned after the 404"
+        );
+    }
+
+    #[tokio::test]
+    async fn serve_content_leaves_other_entries_untouched_when_pruning() {
+        let dir = tempfile::tempdir().unwrap();
+        let data_dir = dir.path();
+        let missing = ContentItem {
+            id: "missing-item".to_string(),
+            filename: "gone.mp4".to_string(),
+            mime_type: "video/mp4".to_string(),
+            size_bytes: 123,
+            description: String::new(),
+            access: AccessControl::Free,
+            availability: Availability::AllPeers,
+            added_at: "2026-01-01T00:00:00Z".to_string(),
+        };
+        let present = ContentItem {
+            id: "present-item".to_string(),
+            filename: "here.mp4".to_string(),
+            mime_type: "video/mp4".to_string(),
+            size_bytes: 4,
+            description: String::new(),
+            access: AccessControl::Free,
+            availability: Availability::AllPeers,
+            added_at: "2026-01-01T00:00:00Z".to_string(),
+        };
+        save_catalog(
+            data_dir,
+            &ContentCatalog {
+                items: vec![missing, present],
+            },
+        )
+        .await
+        .unwrap();
+        let content_dir = data_dir.join("content").join("files");
+        tokio::fs::create_dir_all(&content_dir).await.unwrap();
+        tokio::fs::write(content_dir.join("here.mp4"), b"data")
+            .await
+            .unwrap();
+
+        let _ = serve_content(data_dir, "missing-item", None, None, None, None)
+            .await
+            .unwrap();
+
+        let reloaded = load_catalog(data_dir).await.unwrap();
+        assert_eq!(reloaded.items.len(), 1);
+        assert_eq!(reloaded.items[0].id, "present-item");
+    }
+}