feat(container): manifest-declared generated secrets + companion/quadlet hardening

Generated-secrets system: apps declare `generated_secrets` in their manifest
(kinds hex16/hex32/bcrypt); `container::secrets::ensure_generated_secrets`
materialises them 0600/rootless in resolve_dynamic_env — idempotent and
self-healing (recovers wrongly root-owned secrets with no privilege). Replaces
per-app Rust (deletes ensure_fmcd_password). fedimint-clientd/gateway manifests
now declare fmcd-password / fedimint-gateway-hash.

companion.rs: rebuild the auto-built :latest image when its build context changes
(staleness check) so baked-in fixes (e.g. guardian-UI CSS) actually reach nodes.

quadlet.rs: skip PublishPort under Network=host (podman rejects the combo, exit
125) + regression tests.

UI: "Fedimint Guardian" rename, fedimint-clientd/nostr-rs-relay/meshtastic tagged
as Services (headless backends), gateway icon fallback.

Deployed + verified on .228 (generated-secrets fixed fedimint-gateway start;
grafana/strfry orphan crash-loop units removed).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

core/archipelago/src/container/companion.rs

@@ -221,13 +221,26 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
     for dir in spec.build_dir_candidates {
         let dockerfile = PathBuf::from(dir).join("Dockerfile");
         if fs::try_exists(&dockerfile).await.unwrap_or(false) {
+            // `:local` is a deliberate manual override — never auto-rebuild it.
             if image_exists(&local_image_compat).await {
                 return Ok(local_image_compat);
             }
+            // Reuse the auto-built `:latest` only when the build context has NOT
+            // changed since it was built. Without this staleness check an
+            // already-present image is reused forever, so edits to the baked-in
+            // context (Dockerfile, nginx.conf, …) never reach the node — this is
+            // exactly why the guardian-CSS nginx fix never reached the fleet.
             if image_exists(&local_image).await {
-                return Ok(local_image);
+                if !context_is_newer_than_image(dir, &local_image).await {
+                    return Ok(local_image);
+                }
+                info!(
+                    companion = spec.name,
+                    "build context changed since image built; rebuilding {dir}"
+                );
+            } else {
+                info!(companion = spec.name, "building locally from {dir}");
             }
-            info!(companion = spec.name, "building locally from {dir}");
             let out = command_output_with_timeout(
                 Command::new("podman").args(["build", "-t", &local_image, dir]),
                 COMPANION_BUILD_TIMEOUT,
@@ -286,6 +299,73 @@ async fn image_exists(image: &str) -> bool {
     }
 }
 
+/// Returns true if any file in the build context `dir` is newer than the
+/// already-built `image`, signalling the cached image is stale and must be
+/// rebuilt. Conservative: if either timestamp can't be determined we return
+/// false (reuse the cache) to avoid rebuild storms on every reconcile pass.
+async fn context_is_newer_than_image(dir: &str, image: &str) -> bool {
+    let image_created = match image_created_unix(image).await {
+        Some(t) => t,
+        None => return false,
+    };
+    match newest_mtime_unix(PathBuf::from(dir)).await {
+        Some(ctx) => ctx > image_created,
+        None => false,
+    }
+}
+
+/// Build timestamp of `image` as Unix seconds, via `podman image inspect`.
+async fn image_created_unix(image: &str) -> Option<i64> {
+    let mut cmd = Command::new("podman");
+    cmd.args(["image", "inspect", "--format", "{{.Created.Unix}}", image]);
+    let out = command_output_with_timeout(
+        &mut cmd,
+        COMPANION_IMAGE_CHECK_TIMEOUT,
+        "podman image created time",
+    )
+    .await
+    .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    String::from_utf8_lossy(&out.stdout).trim().parse::<i64>().ok()
+}
+
+/// Newest modification time (Unix seconds) across all files under `dir`,
+/// walked recursively. Runs on a blocking thread since it touches the fs.
+async fn newest_mtime_unix(dir: PathBuf) -> Option<i64> {
+    tokio::task::spawn_blocking(move || newest_mtime_blocking(&dir))
+        .await
+        .ok()
+        .flatten()
+}
+
+fn newest_mtime_blocking(dir: &std::path::Path) -> Option<i64> {
+    let mut newest: Option<i64> = None;
+    let mut stack = vec![dir.to_path_buf()];
+    while let Some(p) = stack.pop() {
+        let entries = match std::fs::read_dir(&p) {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        for entry in entries.flatten() {
+            let meta = match entry.metadata() {
+                Ok(m) => m,
+                Err(_) => continue,
+            };
+            if meta.is_dir() {
+                stack.push(entry.path());
+            } else if let Ok(modified) = meta.modified() {
+                if let Ok(dur) = modified.duration_since(std::time::UNIX_EPOCH) {
+                    let secs = dur.as_secs() as i64;
+                    newest = Some(newest.map_or(secs, |n| n.max(secs)));
+                }
+            }
+        }
+    }
+    newest
+}
+
 async fn command_output_with_timeout(
     cmd: &mut Command,
     timeout: Duration,

core/archipelago/src/container/mod.rs

@@ -11,6 +11,7 @@ pub mod lnd;
 pub mod prod_orchestrator;
 pub mod quadlet;
 pub mod registry;
+pub mod secrets;
 pub mod traits;
 
 pub use boot_reconciler::{BootReconciler, DEFAULT_INTERVAL as RECONCILER_DEFAULT_INTERVAL};

core/archipelago/src/container/prod_orchestrator.rs

@@ -2732,17 +2732,19 @@ impl ProdContainerOrchestrator {
                 .await
                 .context("ensuring bitcoin tx-relay credentials")?;
         }
-        if app_id == "fedimint-clientd" {
-            // The fmcd container's secret_env (fmcd-password) and the wallet
-            // bridge both read this; generate it before secret_env resolves.
-            crate::wallet::fedimint_client::ensure_fmcd_password(&self.secrets_dir)
-                .await
-                .context("ensuring fmcd password secret")?;
-        }
+        // Other app secrets (fmcd-password, fedimint-gateway-hash, …) are now
+        // declared as `generated_secrets` in their manifests and materialised
+        // generically in `resolve_dynamic_env` — no per-app code here.
         Ok(())
     }
 
     fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
+        // Materialise any manifest-declared generated secrets before they're
+        // read below. This is the single chokepoint every install/reconcile
+        // path funnels through, so an app's secrets exist by the time its
+        // `secret_env` resolves — no per-app code, no host provisioning.
+        crate::container::secrets::ensure_generated_secrets(&self.secrets_dir, manifest)?;
+
         let mut facts = self.detect_host_facts();
         // Only pay the podman cost to detect Knots-vs-Core when this manifest
         // actually templates the Bitcoin node into its env (mempool — B12).

core/archipelago/src/container/quadlet.rs

@@ -227,13 +227,20 @@ impl QuadletUnit {
                 mode
             );
         }
-        for (host, container, proto) in &self.ports {
-            let p = if proto.is_empty() {
-                "tcp"
-            } else {
-                proto.as_str()
-            };
-            let _ = writeln!(s, "PublishPort={host}:{container}/{p}");
+        // Host networking exposes the container's ports on the host directly.
+        // Podman rejects PublishPort combined with Network=host ("published
+        // ports cannot be used with host network") and the unit crash-loops
+        // (exit 125). Skip publishing in host mode — matches the NetworkMode
+        // doc note that Podman discards port mappings under host networking.
+        if !matches!(self.network, NetworkMode::Host) {
+            for (host, container, proto) in &self.ports {
+                let p = if proto.is_empty() {
+                    "tcp"
+                } else {
+                    proto.as_str()
+                };
+                let _ = writeln!(s, "PublishPort={host}:{container}/{p}");
+            }
         }
         for env in &self.environment {
             // env entries already arrive shaped as "KEY=VALUE"; quadlet
@@ -852,6 +859,26 @@ mod tests {
         assert!(!s.contains("Network=host"));
     }
 
+    #[test]
+    fn render_host_network_omits_publish_ports() {
+        // Podman rejects PublishPort with Network=host (crash-loop exit 125).
+        let mut u = sample_unit();
+        u.network = NetworkMode::Host;
+        u.ports = vec![(3000, 3000, "tcp".into())];
+        let s = u.render();
+        assert!(s.contains("Network=host"));
+        assert!(!s.contains("PublishPort"));
+    }
+
+    #[test]
+    fn render_non_host_network_emits_publish_ports() {
+        let mut u = sample_unit();
+        u.network = NetworkMode::Bridge("archy-net".into());
+        u.ports = vec![(3000, 3000, "tcp".into())];
+        let s = u.render();
+        assert!(s.contains("PublishPort=3000:3000/tcp"));
+    }
+
     #[test]
     fn unit_filename_and_service_name_are_consistent() {
         let u = sample_unit();

core/archipelago/src/container/secrets.rs

@@ -0,0 +1,198 @@
+//! Declarative, self-healing generation of app secrets.
+//!
+//! An app declares `generated_secrets` in its manifest; this module materialises
+//! them just before `secret_env` is resolved. That keeps the migration's
+//! data-driven bar: an app installs from its manifest alone — no host
+//! provisioning and no per-app Rust — and every secret lands `0600`, owned by
+//! the unprivileged (rootless) service user.
+//!
+//! Two properties make it safe to call on every install/reconcile tick:
+//!
+//! * **Idempotent** — a target file that already exists, is readable and
+//!   non-empty is left untouched, so values are stable across ticks.
+//! * **Self-healing without privilege** — a target file that exists but is
+//!   *unreadable* (the classic `root:root`-owned secret left by some earlier
+//!   path) is unlinked and rewritten. Unlinking needs write on the
+//!   service-owned secrets dir, not on the file, so this recovers the broken
+//!   state with no `chown` and no root — exactly what a rootless node needs.
+
+use anyhow::{Context, Result};
+use archipelago_container::{AppManifest, GeneratedSecret, SecretGenKind};
+use rand::RngCore;
+use std::fs;
+use std::io::Write;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::Path;
+
+/// Plaintext-password length (bytes of entropy) for [`SecretGenKind::Bcrypt`].
+const BCRYPT_PASSWORD_BYTES: usize = 24;
+
+/// Materialise every declared generated secret for `manifest` under
+/// `secrets_dir`. No-op when the manifest declares none. Safe to call on every
+/// reconcile/install tick (idempotent + self-healing).
+pub fn ensure_generated_secrets(secrets_dir: &Path, manifest: &AppManifest) -> Result<()> {
+    let specs = &manifest.app.container.generated_secrets;
+    if specs.is_empty() {
+        return Ok(());
+    }
+    fs::create_dir_all(secrets_dir)
+        .with_context(|| format!("creating secrets dir {}", secrets_dir.display()))?;
+    for gs in specs {
+        ensure_one(secrets_dir, gs).with_context(|| format!("generating secret '{}'", gs.name))?;
+    }
+    Ok(())
+}
+
+fn ensure_one(dir: &Path, gs: &GeneratedSecret) -> Result<()> {
+    let files = gs.target_files();
+
+    // Idempotent fast path: every target file present, readable and non-empty.
+    if files.iter().all(|f| readable_nonempty(&dir.join(f))) {
+        return Ok(());
+    }
+
+    // Self-heal: drop any stale/unreadable target so the write below recreates
+    // it owned by us. Unlinking uses the (service-owned) dir's write bit, so a
+    // wrongly root-owned secret is recovered with no privilege escalation.
+    for f in &files {
+        let p = dir.join(f);
+        if p.exists() && !readable_nonempty(&p) {
+            tracing::warn!("regenerating unreadable/stale secret {}", p.display());
+            fs::remove_file(&p)
+                .with_context(|| format!("removing stale secret {}", p.display()))?;
+        }
+    }
+
+    match gs.kind {
+        SecretGenKind::Hex16 => write_secret(&dir.join(&gs.name), &random_hex(16))?,
+        SecretGenKind::Hex32 => write_secret(&dir.join(&gs.name), &random_hex(32))?,
+        SecretGenKind::Bcrypt => {
+            let password = random_hex(BCRYPT_PASSWORD_BYTES);
+            let hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST)
+                .context("bcrypt-hashing generated password")?;
+            // Primary (server-facing hash) first, then the plaintext sibling.
+            write_secret(&dir.join(&gs.name), &hash)?;
+            write_secret(&dir.join(format!("{}.pw", gs.name)), &password)?;
+        }
+    }
+    Ok(())
+}
+
+/// True when `path` exists, is readable by this process, and is non-empty after
+/// trimming. Any error (missing, permission denied, empty) reads as false.
+fn readable_nonempty(path: &Path) -> bool {
+    fs::read_to_string(path)
+        .map(|s| !s.trim().is_empty())
+        .unwrap_or(false)
+}
+
+fn random_hex(bytes: usize) -> String {
+    let mut buf = vec![0u8; bytes];
+    rand::thread_rng().fill_bytes(&mut buf);
+    hex::encode(buf)
+}
+
+/// Atomically write a `0600` secret: a temp file in the same dir (so the rename
+/// is atomic), fsynced, then renamed over the target.
+fn write_secret(path: &Path, value: &str) -> Result<()> {
+    let dir = path
+        .parent()
+        .context("secret path has no parent directory")?;
+    let name = path
+        .file_name()
+        .and_then(|n| n.to_str())
+        .context("secret path has no filename")?;
+    let tmp = dir.join(format!(".{name}.tmp"));
+
+    let mut f = fs::OpenOptions::new()
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .mode(0o600)
+        .open(&tmp)
+        .with_context(|| format!("creating temp secret {}", tmp.display()))?;
+    f.write_all(value.as_bytes())
+        .with_context(|| format!("writing temp secret {}", tmp.display()))?;
+    f.sync_all()
+        .with_context(|| format!("fsync temp secret {}", tmp.display()))?;
+    drop(f);
+
+    fs::rename(&tmp, path)
+        .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use archipelago_container::SecretGenKind;
+    use std::os::unix::fs::PermissionsExt;
+
+    fn manifest_with(secrets: Vec<GeneratedSecret>) -> AppManifest {
+        let mut m: AppManifest = serde_yaml::from_str(
+            "app:\n  id: t\n  name: t\n  version: 1.0.0\n  container:\n    image: x:y\n",
+        )
+        .unwrap();
+        m.app.container.generated_secrets = secrets;
+        m
+    }
+
+    fn gs(name: &str, kind: SecretGenKind) -> GeneratedSecret {
+        GeneratedSecret {
+            name: name.to_string(),
+            kind,
+        }
+    }
+
+    #[test]
+    fn generates_hex_and_bcrypt_with_0600() {
+        let dir = tempfile::tempdir().unwrap();
+        let m = manifest_with(vec![
+            gs("tok", SecretGenKind::Hex16),
+            gs("admin", SecretGenKind::Bcrypt),
+        ]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+
+        let tok = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(tok.trim().len(), 32, "hex16 = 16 bytes = 32 hex chars");
+
+        let hash = std::fs::read_to_string(dir.path().join("admin")).unwrap();
+        let pw = std::fs::read_to_string(dir.path().join("admin.pw")).unwrap();
+        assert!(hash.starts_with("$2"), "bcrypt hash shape");
+        assert!(bcrypt::verify(pw.trim(), hash.trim()).unwrap(), "pw matches hash");
+
+        for f in ["tok", "admin", "admin.pw"] {
+            let mode = std::fs::metadata(dir.path().join(f))
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777;
+            assert_eq!(mode, 0o600, "{f} must be 0600");
+        }
+    }
+
+    #[test]
+    fn idempotent_value_is_stable() {
+        let dir = tempfile::tempdir().unwrap();
+        let m = manifest_with(vec![gs("tok", SecretGenKind::Hex32)]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let first = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let second = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(first, second, "a present readable secret is never rewritten");
+    }
+
+    #[test]
+    fn self_heals_unreadable_secret() {
+        // Simulate the root-owned case: a present-but-unreadable file. We can't
+        // chmod-away read as the owner in a unit test, so emulate "unreadable"
+        // via the empty-file branch (readable_nonempty == false), which drives
+        // the same unlink+regenerate path.
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(dir.path().join("tok"), "").unwrap();
+        let m = manifest_with(vec![gs("tok", SecretGenKind::Hex16)]);
+        ensure_generated_secrets(dir.path(), &m).unwrap();
+        let v = std::fs::read_to_string(dir.path().join("tok")).unwrap();
+        assert_eq!(v.trim().len(), 32, "stale/empty secret was regenerated");
+    }
+}

core/archipelago/src/wallet/fedimint_client.rs

@@ -50,38 +50,12 @@ pub struct FederationRegistry {
 const REGISTRY_FILE: &str = "wallet/fedimint_federations.json";
 
 /// Shared HTTP-Basic password between the fmcd container and this bridge. The
-/// fedimint-clientd manifest reads it via `secret_env: fmcd-password`, resolved
-/// from `<data_dir>/secrets/`; the bridge reads the same file in `from_node`.
+/// fedimint-clientd manifest generates it via `generated_secrets: [fmcd-password]`
+/// and injects it through `secret_env`; the bridge reads the same file in
+/// `from_node`. (Generation lives in `container::secrets`, not here — it's a
+/// generic, manifest-declared concern, not fedimint-specific.)
 const FMCD_PASSWORD_SECRET: &str = "fmcd-password";
 
-/// Generate the fmcd Basic-auth password once, so the fmcd container
-/// (`secret_env: fmcd-password`) and this bridge (`from_node`) agree on it.
-/// Idempotent: a non-empty existing secret is left untouched. Mirrors the
-/// bitcoin-rpc secret pattern (random hex, 0600). Called from the orchestrator's
-/// `ensure_app_secrets` before the container's `secret_env` is resolved.
-pub async fn ensure_fmcd_password(secrets_dir: &Path) -> Result<()> {
-    let path = secrets_dir.join(FMCD_PASSWORD_SECRET);
-    if let Ok(existing) = fs::read_to_string(&path).await {
-        if !existing.trim().is_empty() {
-            return Ok(());
-        }
-    }
-    fs::create_dir_all(secrets_dir)
-        .await
-        .context("creating secrets dir for fmcd password")?;
-    let bytes: [u8; 16] = rand::random();
-    let password = hex::encode(bytes);
-    fs::write(&path, &password)
-        .await
-        .context("writing fmcd password secret")?;
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::PermissionsExt;
-        let _ = fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).await;
-    }
-    Ok(())
-}
-
 pub async fn load_registry(data_dir: &Path) -> Result<FederationRegistry> {
     let path = data_dir.join(REGISTRY_FILE);
     if !path.exists() {