feat(security): move secret env out of podman inspect and Quadlet unit files

Secret env used to merge into manifest.app.environment, landing in
'podman inspect' Config.Env on the API backend and — worse — as
plaintext Environment= lines in Quadlet unit files on disk. Now:

- expand_and_partition_env (container crate, pure + tested) expands
  ${KEY} placeholders and splits env into plain entries and
  secret-bearing pairs. Plain entries that interpolate a secret
  (btcpay's Password=${BTCPAY_DB_PASS} connection strings) are
  tainted and travel as secrets too. Secret values themselves are
  never expanded (a generated value containing '${' passes verbatim).
- values register as podman secrets: stdin (never argv/tempfile),
  --replace, content-hash label to skip no-op rewrites; a per-app hash
  cache in the orchestrator makes steady-state reconciles free of
  podman secret calls. Registration goes through the runtime trait
  (default no-op keeps mocks/docker inert).
- containers reference secrets by name: secret_env map in the libpod
  create spec, Secret=<name>,type=env,target=<KEY> in Quadlet units.
  Verified empirically on fleet podman 5.4.2: value absent from
  inspect Config.Env, runtime injection works rootless.
- rotation detection: io.archipelago.secret-env-hash container label
  (API) / the changed unit bytes (Quadlet). Pre-upgrade containers
  lack the label, so every secret-bearing app recreates ONCE on the
  first reconcile after deploy — deliberate, it scrubs the plaintext
  secrets out of existing container configs. Data dirs untouched.
- docker dev fallback keeps plain -e injection (no secret store);
  podman secrets persist across uninstall, matching the
  preserve-credentials invariant (reinstall re-registers by hash).

In-container /proc/<pid>/environ is unchanged — env remains the
app-compat contract; the closed leaks are inspect output and unit
files on disk.

Tests: archipelago-container 61/61 (3 new: taint partition, verbatim
secrets, hash order-independence), archipelago container:: 160/160
(fedimint install test now asserts the secret arrives as a ref, not
env; quadlet render test asserts Secret=/Label= lines). NEEDS the
on-node gate re-run before the item counts as verified.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/container/src/manifest.rs

@@ -243,6 +243,22 @@ pub struct ContainerConfig {
     /// Example: `"100070:100070"` for Postgres' mapped subuid.
     #[serde(default)]
     pub data_uid: Option<String>,
+
+    /// Runtime-resolved secret env entries (never serialized, never in a
+    /// manifest file). Populated by the orchestrator's env-resolution
+    /// chokepoint; the backends turn each ref into a podman secret
+    /// reference (`secret_env` in the API spec / `Secret=…,type=env` in
+    /// Quadlet) so the VALUE never lands in `podman inspect` output or a
+    /// unit file on disk. The dev-only docker fallback injects `value` as
+    /// plain env — docker has no rootless secret store.
+    #[serde(skip)]
+    pub secret_env_refs: Vec<SecretEnvRef>,
+
+    /// sha256 over all resolved secret env pairs, stamped onto the
+    /// container as a label so rotation is detectable as drift without
+    /// exposing values. None when the app has no secret env.
+    #[serde(skip)]
+    pub secret_env_hash: Option<String>,
 }
 
 /// Derived-env entry. The template is rendered against `HostFacts` at
@@ -265,6 +281,75 @@ pub struct SecretEnv {
     pub secret_file: String,
 }
 
+/// A fully resolved secret env entry, produced at apply time. `value` lives
+/// only in memory on its way to the podman secret store (or, on the dev
+/// docker fallback, straight into the container env).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SecretEnvRef {
+    pub env_key: String,
+    /// Podman secret object name: `archy-env-<app>-<KEY>`.
+    pub secret_name: String,
+    pub value: String,
+}
+
+/// Container label carrying the combined secret-env content hash, used by
+/// the reconciler to detect secret rotation as env drift.
+pub const SECRET_ENV_HASH_LABEL: &str = "io.archipelago.secret-env-hash";
+
+/// Podman secret label carrying the individual secret's content hash, used
+/// to skip re-registration when nothing changed.
+pub const SECRET_HASH_LABEL: &str = "io.archipelago.hash";
+
+/// Expand `${KEY}` placeholders across plain + secret values, then split
+/// the result into (plain env, secret-bearing pairs). Any plain entry whose
+/// value interpolates a secret key is *tainted* — it embeds the secret and
+/// must travel as a secret itself (btcpay's
+/// `BTCPAY_POSTGRES=…Password=${BTCPAY_DB_PASS};…` pattern). Secret values
+/// themselves are taken verbatim: a generated secret that happens to
+/// contain `${` must not be mangled by expansion.
+pub fn expand_and_partition_env(
+    plain: Vec<String>,
+    secrets: Vec<(String, String)>,
+) -> (Vec<String>, Vec<(String, String)>) {
+    let plain_values: std::collections::HashMap<String, String> = plain
+        .iter()
+        .filter_map(|entry| {
+            let (key, value) = entry.split_once('=')?;
+            Some((key.to_string(), value.to_string()))
+        })
+        .collect();
+
+    let mut out_plain = Vec::with_capacity(plain.len());
+    let mut out_secret: Vec<(String, String)> = Vec::with_capacity(secrets.len());
+
+    for entry in plain {
+        let Some((key, value)) = entry.split_once('=') else {
+            out_plain.push(entry);
+            continue;
+        };
+        let mut expanded = value.to_string();
+        let mut tainted = false;
+        for (k, v) in &plain_values {
+            expanded = expanded.replace(&format!("${{{k}}}"), v);
+        }
+        for (k, v) in &secrets {
+            let placeholder = format!("${{{k}}}");
+            if expanded.contains(&placeholder) {
+                expanded = expanded.replace(&placeholder, v);
+                tainted = true;
+            }
+        }
+        if tainted {
+            out_secret.push((key.to_string(), expanded));
+        } else {
+            out_plain.push(format!("{key}={expanded}"));
+        }
+    }
+
+    out_secret.extend(secrets);
+    (out_plain, out_secret)
+}
+
 /// How a [`GeneratedSecret`] is produced. Each kind is deterministic in shape
 /// (so the orchestrator knows which files to expect) but random in value.
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
@@ -1209,6 +1294,19 @@ impl ContainerConfig {
         &self,
         provider: &dyn SecretsProvider,
     ) -> Result<Vec<String>, ManifestError> {
+        Ok(self
+            .resolve_secret_env_pairs(provider)?
+            .into_iter()
+            .map(|(k, v)| format!("{k}={v}"))
+            .collect())
+    }
+
+    /// Like `resolve_secret_env` but returns (key, value) pairs — the shape
+    /// the podman-secret pipeline needs.
+    pub fn resolve_secret_env_pairs(
+        &self,
+        provider: &dyn SecretsProvider,
+    ) -> Result<Vec<(String, String)>, ManifestError> {
         let mut out = Vec::with_capacity(self.secret_env.len());
         for e in &self.secret_env {
             let v = provider.read(&e.secret_file)?;
@@ -1220,12 +1318,28 @@ impl ContainerConfig {
                     e.key, e.secret_file
                 )));
             }
-            out.push(format!("{}={}", e.key, v));
+            out.push((e.key.clone(), v));
         }
         Ok(out)
     }
 }
 
+/// Deterministic content hash over resolved secret env pairs (sorted by
+/// key), used for the container drift label and per-secret labels.
+pub fn secret_env_content_hash(pairs: &[(String, String)]) -> String {
+    use sha2::{Digest, Sha256};
+    let mut sorted: Vec<&(String, String)> = pairs.iter().collect();
+    sorted.sort_by(|a, b| a.0.cmp(&b.0));
+    let mut hasher = Sha256::new();
+    for (k, v) in sorted {
+        hasher.update(k.as_bytes());
+        hasher.update([0u8]);
+        hasher.update(v.as_bytes());
+        hasher.update([0u8]);
+    }
+    hex::encode(hasher.finalize())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1233,6 +1347,53 @@ mod tests {
     use std::fs;
     use std::path::{Path, PathBuf};
 
+    #[test]
+    fn partition_taints_plain_entries_that_interpolate_secrets() {
+        let plain = vec![
+            "PLAIN=1".to_string(),
+            "COMPOSED=${BASE}/x".to_string(),
+            "DB_URL=postgres://u:${DB_PASS}@db/x".to_string(),
+            "BASE=/srv".to_string(),
+            "UNKNOWN=${NOT_DEFINED}".to_string(),
+        ];
+        let secrets = vec![("DB_PASS".to_string(), "s3cret".to_string())];
+        let (p, s) = expand_and_partition_env(plain, secrets);
+
+        // plain-from-plain expansion stays plain; unknown placeholders stay
+        // literal (legacy expander parity)
+        assert!(p.contains(&"PLAIN=1".to_string()));
+        assert!(p.contains(&"COMPOSED=/srv/x".to_string()));
+        assert!(p.contains(&"UNKNOWN=${NOT_DEFINED}".to_string()));
+        // the tainted entry moved out of plain, fully expanded
+        assert!(!p.iter().any(|e| e.starts_with("DB_URL=")));
+        assert!(s.contains(&("DB_URL".to_string(), "postgres://u:s3cret@db/x".to_string())));
+        // the original secret rides along verbatim
+        assert!(s.contains(&("DB_PASS".to_string(), "s3cret".to_string())));
+    }
+
+    #[test]
+    fn secret_values_are_never_expanded() {
+        // A generated secret containing `${` must pass through untouched.
+        let secrets = vec![("WEIRD".to_string(), "pa${PLAIN}ss".to_string())];
+        let (_, s) = expand_and_partition_env(vec!["PLAIN=1".to_string()], secrets);
+        assert!(s.contains(&("WEIRD".to_string(), "pa${PLAIN}ss".to_string())));
+    }
+
+    #[test]
+    fn secret_env_hash_is_order_independent() {
+        let a = vec![
+            ("K1".to_string(), "v1".to_string()),
+            ("K2".to_string(), "v2".to_string()),
+        ];
+        let b = vec![
+            ("K2".to_string(), "v2".to_string()),
+            ("K1".to_string(), "v1".to_string()),
+        ];
+        assert_eq!(secret_env_content_hash(&a), secret_env_content_hash(&b));
+        let c = vec![("K1".to_string(), "CHANGED".to_string())];
+        assert_ne!(secret_env_content_hash(&a), secret_env_content_hash(&c));
+    }
+
     #[test]
     fn hooks_parse_and_validate() {
         let yaml = r#"
@@ -1765,6 +1926,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let facts = HostFacts {
             host_ip: "192.168.1.116".to_string(),
@@ -1817,6 +1980,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let p = MapSecretsProvider {
             data: HashMap::from([
@@ -1855,6 +2020,8 @@ app:
             generated_secrets: vec![],
             generated_certs: vec![],
             data_uid: None,
+            secret_env_refs: vec![],
+            secret_env_hash: None,
         };
         let p = MapSecretsProvider {
             data: HashMap::from([("bitcoin-rpc-password".to_string(), "   \n".to_string())]),

core/container/src/podman_client.rs

@@ -382,12 +382,32 @@ impl PodmanClient {
             manifest.app.security.network_policy.as_str(),
         );
 
+        // Secret env travels by reference (podman injects the value at
+        // start), so it never shows up in `podman inspect` output. The
+        // combined content hash rides as a label for rotation-drift checks.
+        let mut secret_env_map = serde_json::Map::new();
+        for r in &manifest.app.container.secret_env_refs {
+            secret_env_map.insert(
+                r.env_key.clone(),
+                serde_json::Value::String(r.secret_name.clone()),
+            );
+        }
+        let mut labels_map = serde_json::Map::new();
+        if let Some(hash) = &manifest.app.container.secret_env_hash {
+            labels_map.insert(
+                crate::manifest::SECRET_ENV_HASH_LABEL.to_string(),
+                serde_json::Value::String(hash.clone()),
+            );
+        }
+
         let mut body = serde_json::json!({
             "name": name,
             "image": image_ref,
             "portmappings": port_mappings,
             "mounts": mounts,
             "env": env_map,
+            "secret_env": secret_env_map,
+            "labels": labels_map,
             "entrypoint": manifest.app.container.entrypoint.clone(),
             "command": manifest.app.container.custom_args.clone(),
             "hostadd": [

core/container/src/runtime.rs

@@ -82,6 +82,18 @@ pub trait ContainerRuntime: Send + Sync {
     /// `create_container` / `image_exists` calls. Stdout/stderr are collected
     /// and included in the error on failure; on success they are discarded.
     async fn build_image(&self, config: &BuildConfig) -> Result<()>;
+
+    /// Register the app's resolved secret-env entries in the runtime's
+    /// secret store (idempotent — skips entries whose content hash already
+    /// matches). Called before `create_container` for manifests with
+    /// `secret_env_refs`, so the create can reference secrets by name and
+    /// the values never appear in inspect output or unit files. Default is
+    /// a no-op for runtimes without a secret store (mocks, docker — the
+    /// docker fallback injects plain env in `create_container` instead).
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        let _ = refs;
+        Ok(())
+    }
 }
 
 pub struct PodmanRuntime {
@@ -131,6 +143,68 @@ impl ContainerRuntime for PodmanRuntime {
         self.client.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        use crate::manifest::{secret_env_content_hash, SECRET_HASH_LABEL};
+        use tokio::io::AsyncWriteExt;
+
+        for r in refs {
+            let hash = secret_env_content_hash(&[(r.env_key.clone(), r.value.clone())]);
+
+            // Skip the write when the stored secret already has this content
+            // (label round-trip beats rewriting the secret store every
+            // reconcile). Any inspect failure just falls through to create.
+            let fmt = format!("{{{{ index .Spec.Labels \"{SECRET_HASH_LABEL}\" }}}}");
+            if let Ok(out) = self
+                .podman_cli(&["secret", "inspect", &r.secret_name, "--format", &fmt])
+                .await
+            {
+                if out.status.success()
+                    && String::from_utf8_lossy(&out.stdout).trim() == hash
+                {
+                    continue;
+                }
+            }
+
+            // Value goes via stdin — never argv, never a temp file.
+            let mut cmd = TokioCommand::new("podman");
+            cmd.args([
+                "secret",
+                "create",
+                "--replace",
+                "--label",
+                &format!("{SECRET_HASH_LABEL}={hash}"),
+                &r.secret_name,
+                "-",
+            ]);
+            cmd.stdin(std::process::Stdio::piped());
+            cmd.stdout(std::process::Stdio::piped());
+            cmd.stderr(std::process::Stdio::piped());
+            cmd.kill_on_drop(true);
+            let mut child = cmd
+                .spawn()
+                .with_context(|| format!("spawning podman secret create {}", r.secret_name))?;
+            {
+                let mut stdin = child
+                    .stdin
+                    .take()
+                    .context("podman secret create stdin unavailable")?;
+                stdin.write_all(r.value.as_bytes()).await?;
+                // drop closes the pipe so podman sees EOF
+            }
+            let out = tokio::time::timeout(PODMAN_CLI_DEFAULT_TIMEOUT, child.wait_with_output())
+                .await
+                .with_context(|| format!("podman secret create {} timed out", r.secret_name))??;
+            if !out.status.success() {
+                anyhow::bail!(
+                    "podman secret create {} failed: {}",
+                    r.secret_name,
+                    String::from_utf8_lossy(&out.stderr)
+                );
+            }
+        }
+        Ok(())
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,
@@ -621,6 +695,12 @@ impl ContainerRuntime for DockerRuntime {
         for env in &manifest.app.environment {
             cmd.arg("-e").arg(env);
         }
+        // Dev-only fallback: docker has no rootless secret store, so secret
+        // env rides as plain env here. The podman path (production) passes
+        // these by secret reference instead — see ensure_env_secrets.
+        for r in &manifest.app.container.secret_env_refs {
+            cmd.arg("-e").arg(format!("{}={}", r.env_key, r.value));
+        }
 
         // Resource limits
         if let Some(cpu) = manifest.app.resources.cpu_limit {
@@ -893,6 +973,10 @@ impl ContainerRuntime for AutoRuntime {
         self.runtime.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        self.runtime.ensure_env_secrets(refs).await
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,