feat(security): move secret env out of podman inspect and Quadlet unit files

Secret env used to merge into manifest.app.environment, landing in
'podman inspect' Config.Env on the API backend and — worse — as
plaintext Environment= lines in Quadlet unit files on disk. Now:

- expand_and_partition_env (container crate, pure + tested) expands
  ${KEY} placeholders and splits env into plain entries and
  secret-bearing pairs. Plain entries that interpolate a secret
  (btcpay's Password=${BTCPAY_DB_PASS} connection strings) are
  tainted and travel as secrets too. Secret values themselves are
  never expanded (a generated value containing '${' passes verbatim).
- values register as podman secrets: stdin (never argv/tempfile),
  --replace, content-hash label to skip no-op rewrites; a per-app hash
  cache in the orchestrator makes steady-state reconciles free of
  podman secret calls. Registration goes through the runtime trait
  (default no-op keeps mocks/docker inert).
- containers reference secrets by name: secret_env map in the libpod
  create spec, Secret=<name>,type=env,target=<KEY> in Quadlet units.
  Verified empirically on fleet podman 5.4.2: value absent from
  inspect Config.Env, runtime injection works rootless.
- rotation detection: io.archipelago.secret-env-hash container label
  (API) / the changed unit bytes (Quadlet). Pre-upgrade containers
  lack the label, so every secret-bearing app recreates ONCE on the
  first reconcile after deploy — deliberate, it scrubs the plaintext
  secrets out of existing container configs. Data dirs untouched.
- docker dev fallback keeps plain -e injection (no secret store);
  podman secrets persist across uninstall, matching the
  preserve-credentials invariant (reinstall re-registers by hash).

In-container /proc/<pid>/environ is unchanged — env remains the
app-compat contract; the closed leaks are inspect output and unit
files on disk.

Tests: archipelago-container 61/61 (3 new: taint partition, verbatim
secrets, hash order-independence), archipelago container:: 160/160
(fedimint install test now asserts the secret arrives as a ref, not
env; quadlet render test asserts Secret=/Label= lines). NEEDS the
on-node gate re-run before the item counts as verified.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

core/container/src/runtime.rs

@@ -82,6 +82,18 @@ pub trait ContainerRuntime: Send + Sync {
     /// `create_container` / `image_exists` calls. Stdout/stderr are collected
     /// and included in the error on failure; on success they are discarded.
     async fn build_image(&self, config: &BuildConfig) -> Result<()>;
+
+    /// Register the app's resolved secret-env entries in the runtime's
+    /// secret store (idempotent — skips entries whose content hash already
+    /// matches). Called before `create_container` for manifests with
+    /// `secret_env_refs`, so the create can reference secrets by name and
+    /// the values never appear in inspect output or unit files. Default is
+    /// a no-op for runtimes without a secret store (mocks, docker — the
+    /// docker fallback injects plain env in `create_container` instead).
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        let _ = refs;
+        Ok(())
+    }
 }
 
 pub struct PodmanRuntime {
@@ -131,6 +143,68 @@ impl ContainerRuntime for PodmanRuntime {
         self.client.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        use crate::manifest::{secret_env_content_hash, SECRET_HASH_LABEL};
+        use tokio::io::AsyncWriteExt;
+
+        for r in refs {
+            let hash = secret_env_content_hash(&[(r.env_key.clone(), r.value.clone())]);
+
+            // Skip the write when the stored secret already has this content
+            // (label round-trip beats rewriting the secret store every
+            // reconcile). Any inspect failure just falls through to create.
+            let fmt = format!("{{{{ index .Spec.Labels \"{SECRET_HASH_LABEL}\" }}}}");
+            if let Ok(out) = self
+                .podman_cli(&["secret", "inspect", &r.secret_name, "--format", &fmt])
+                .await
+            {
+                if out.status.success()
+                    && String::from_utf8_lossy(&out.stdout).trim() == hash
+                {
+                    continue;
+                }
+            }
+
+            // Value goes via stdin — never argv, never a temp file.
+            let mut cmd = TokioCommand::new("podman");
+            cmd.args([
+                "secret",
+                "create",
+                "--replace",
+                "--label",
+                &format!("{SECRET_HASH_LABEL}={hash}"),
+                &r.secret_name,
+                "-",
+            ]);
+            cmd.stdin(std::process::Stdio::piped());
+            cmd.stdout(std::process::Stdio::piped());
+            cmd.stderr(std::process::Stdio::piped());
+            cmd.kill_on_drop(true);
+            let mut child = cmd
+                .spawn()
+                .with_context(|| format!("spawning podman secret create {}", r.secret_name))?;
+            {
+                let mut stdin = child
+                    .stdin
+                    .take()
+                    .context("podman secret create stdin unavailable")?;
+                stdin.write_all(r.value.as_bytes()).await?;
+                // drop closes the pipe so podman sees EOF
+            }
+            let out = tokio::time::timeout(PODMAN_CLI_DEFAULT_TIMEOUT, child.wait_with_output())
+                .await
+                .with_context(|| format!("podman secret create {} timed out", r.secret_name))??;
+            if !out.status.success() {
+                anyhow::bail!(
+                    "podman secret create {} failed: {}",
+                    r.secret_name,
+                    String::from_utf8_lossy(&out.stderr)
+                );
+            }
+        }
+        Ok(())
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,
@@ -621,6 +695,12 @@ impl ContainerRuntime for DockerRuntime {
         for env in &manifest.app.environment {
             cmd.arg("-e").arg(env);
         }
+        // Dev-only fallback: docker has no rootless secret store, so secret
+        // env rides as plain env here. The podman path (production) passes
+        // these by secret reference instead — see ensure_env_secrets.
+        for r in &manifest.app.container.secret_env_refs {
+            cmd.arg("-e").arg(format!("{}={}", r.env_key, r.value));
+        }
 
         // Resource limits
         if let Some(cpu) = manifest.app.resources.cpu_limit {
@@ -893,6 +973,10 @@ impl ContainerRuntime for AutoRuntime {
         self.runtime.pull_image(image, signature).await
     }
 
+    async fn ensure_env_secrets(&self, refs: &[crate::manifest::SecretEnvRef]) -> Result<()> {
+        self.runtime.ensure_env_secrets(refs).await
+    }
+
     async fn create_container(
         &self,
         manifest: &AppManifest,