fix(stop): honour per-app graceful-stop grace in orchestrator stop path

package.stop left slow-to-SIGTERM apps (fedimint/electrumx/bitcoin/btcpay/immich)
running: the orchestrator path hardcoded podman API ?t=10 / CLI -t 30 and the CLI
wrapper deadline (30s) equalled the -t grace, so the await fired exactly as podman
SIGKILLed -> stop reported failed -> state reverted to running. Reproduced live on
clean .198 (fedimint).

- container/runtime.rs: add ContainerRuntime::stop_container_with_grace (defaulted
  so mock/dev impls are unchanged); PodmanRuntime honours grace for API + CLI with
  deadline = grace + 15s buffer; AutoRuntime delegates. New canonical per-app table
  stop_grace_secs_for() + DEFAULT_STOP_GRACE_SECS / STOP_GRACE_DEADLINE_BUFFER_SECS.
- podman_client.rs: stop_container_with_grace uses ?t=<grace> + longer HTTP deadline.
- prod_orchestrator::stop: resolve grace = manifest stop_grace_secs (north-star) else
  the table; pass to quadlet::stop_service_with_timeout AND stop_container_with_grace.
- quadlet.rs: stop_service_with_timeout so slow apps aren't SIGKILLed at 45s.
- rpc/package/runtime.rs: doc-note its &str stop_timeout_secs mirrors the canonical table.
- tests: resolve_stop_grace_secs (manifest field wins / table fallback / default 30).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

core/container/src/podman_client.rs

@@ -422,11 +422,22 @@ impl PodmanClient {
     }
 
     pub async fn stop_container(&self, name: &str) -> Result<()> {
+        self.stop_container_with_grace(name, 10).await
+    }
+
+    /// Stop via libpod honouring a per-app grace (seconds). The HTTP deadline is
+    /// kept above the grace so the post-grace SIGKILL lands before we give up —
+    /// otherwise slow-to-SIGTERM apps (fedimint, bitcoin-core, electrumx…) time
+    /// out at exactly the grace boundary and the stop is reported as failed.
+    pub async fn stop_container_with_grace(&self, name: &str, grace_secs: u64) -> Result<()> {
+        let deadline = std::time::Duration::from_secs(
+            grace_secs + crate::runtime::STOP_GRACE_DEADLINE_BUFFER_SECS,
+        );
         self.api_request(
             "POST",
-            &format!("libpod/containers/{}/stop?t=10", name),
+            &format!("libpod/containers/{}/stop?t={}", name, grace_secs),
             None,
-            DEFAULT_TIMEOUT,
+            deadline,
         )
         .await
         .map(|_| ())

core/container/src/runtime.rs

@@ -10,6 +10,35 @@ const PODMAN_CLI_DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
 const PODMAN_CLI_IMAGE_CHECK_TIMEOUT: Duration = Duration::from_secs(10);
 const PODMAN_CLI_BUILD_TIMEOUT: Duration = Duration::from_secs(900);
 
+/// Default graceful-stop grace (seconds) when a caller doesn't supply a per-app
+/// value. Mirrors the historical `podman stop -t 30`.
+pub const DEFAULT_STOP_GRACE_SECS: u64 = 30;
+/// Headroom added to a stop grace to form the await/HTTP deadline, so podman's
+/// post-grace SIGKILL completes before the wrapper times out.
+pub const STOP_GRACE_DEADLINE_BUFFER_SECS: u64 = 15;
+
+/// Canonical per-app graceful-stop grace (seconds), keyed by container name.
+/// Slow-to-SIGTERM apps need far longer than the 30s default: bitcoin-core
+/// flushes its chainstate, lnd closes channels, electrumx finishes indexing,
+/// stack DBs checkpoint. Used as the fallback when a manifest doesn't declare
+/// `stop_grace_secs`. NOTE: the RPC layer's `stop_timeout_secs` mirrors this
+/// (returns the same values as `&str` for legacy `podman stop -t` call sites) —
+/// keep the two in sync until that path is retired.
+pub fn stop_grace_secs_for(container_name: &str) -> u64 {
+    let id = container_name
+        .strip_prefix("archy-")
+        .unwrap_or(container_name);
+    match id {
+        "bitcoin-knots" | "bitcoin-core" | "bitcoin" => 600,
+        "lnd" => 330,
+        "electrumx" | "electrs" | "mempool-electrs" => 300,
+        "btcpay-db" | "mempool-db" | "penpot-postgres" | "immich_postgres" | "nextcloud-db"
+        | "endurain-db" => 120,
+        "btcpay-server" | "nbxplorer" | "fedimint" | "fedimint-gateway" => 60,
+        _ => DEFAULT_STOP_GRACE_SECS,
+    }
+}
+
 #[async_trait]
 pub trait ContainerRuntime: Send + Sync {
     async fn pull_image(&self, image: &str, signature: Option<&str>) -> Result<()>;
@@ -21,6 +50,19 @@ pub trait ContainerRuntime: Send + Sync {
     ) -> Result<String>;
     async fn start_container(&self, name: &str) -> Result<()>;
     async fn stop_container(&self, name: &str) -> Result<()>;
+    /// Stop a container honouring a per-app graceful-shutdown grace (seconds).
+    ///
+    /// Slow-to-SIGTERM apps (bitcoin-core, lnd, electrumx, fedimint, immich…)
+    /// need a longer `podman stop -t` than the default 30s, or `podman stop`
+    /// returns before the container exits and the orchestrator treats the stop
+    /// as failed (the container keeps running). The wrapping deadline is always
+    /// kept strictly greater than `grace_secs` so podman's post-grace SIGKILL
+    /// lands inside the await. The default impl ignores the grace and calls
+    /// `stop_container` — only the real podman runtime honours it.
+    async fn stop_container_with_grace(&self, name: &str, grace_secs: u64) -> Result<()> {
+        let _ = grace_secs;
+        self.stop_container(name).await
+    }
     async fn remove_container(&self, name: &str) -> Result<()>;
     async fn get_container_status(&self, name: &str) -> Result<ContainerStatus>;
     async fn get_container_logs(&self, name: &str, lines: u32) -> Result<Vec<String>>;
@@ -122,10 +164,23 @@ impl ContainerRuntime for PodmanRuntime {
     }
 
     async fn stop_container(&self, name: &str) -> Result<()> {
-        match self.client.stop_container(name).await {
+        self.stop_container_with_grace(name, DEFAULT_STOP_GRACE_SECS)
+            .await
+    }
+
+    async fn stop_container_with_grace(&self, name: &str, grace_secs: u64) -> Result<()> {
+        match self.client.stop_container_with_grace(name, grace_secs).await {
             Ok(()) => Ok(()),
             Err(api_err) => {
-                let output = self.podman_cli(&["stop", "-t", "30", name]).await?;
+                // CLI fallback. Keep the wrapper deadline strictly above the
+                // `-t` grace so podman's post-grace SIGKILL completes before the
+                // await gives up (otherwise a deadline == grace races the kill
+                // and reports a spurious timeout).
+                let grace = grace_secs.to_string();
+                let deadline = Duration::from_secs(grace_secs + STOP_GRACE_DEADLINE_BUFFER_SECS);
+                let output = self
+                    .podman_cli_timeout(&["stop", "-t", &grace, name], deadline)
+                    .await?;
                 if output.status.success() {
                     Ok(())
                 } else {
@@ -841,6 +896,10 @@ impl ContainerRuntime for AutoRuntime {
         self.runtime.stop_container(name).await
     }
 
+    async fn stop_container_with_grace(&self, name: &str, grace_secs: u64) -> Result<()> {
+        self.runtime.stop_container_with_grace(name, grace_secs).await
+    }
+
     async fn remove_container(&self, name: &str) -> Result<()> {
         self.runtime.remove_container(name).await
     }