Prepare stack update images before downtime and reuse private digest imports

core/archipelago/src/api/rpc/package/update.rs

@@ -180,10 +180,17 @@ impl RpcHandler {
             return Err(anyhow::anyhow!("No containers found for {}", package_id));
         }
 
-        // Execute update — on failure, attempt rollback by restarting old containers
+        // Resolve every image while the old stack is still available. A
-        match self
+        // registry outage or missing private import must not stop the app or
-            .execute_update(package_id, &containers, &images_to_pull)
+        // enter rollback (which could start a deliberately stopped member).
-            .await
+        self.set_install_phase(package_id, InstallPhase::PullingImage)
+            .await;
+        match preflighted_stack_update(
+            &images_to_pull,
+            |image| async move { self.pull_update_image(package_id, &image).await },
+            || self.execute_update(package_id, &containers, &images_to_pull),
+        )
+        .await
         {
             Ok(()) => {
                 install_log(&format!("UPDATE OK: {}", package_id)).await;
@@ -193,7 +200,12 @@ impl RpcHandler {
                     "package_id": package_id,
                 }))
             }
-            Err(e) => {
+            Err(UpdateFailure::Preparation(error)) => {
+                self.clear_install_progress(package_id).await;
+                self.clear_update_state(package_id).await;
+                Err(error)
+            }
+            Err(UpdateFailure::Execution(e)) => {
                 error!("Update {} failed: {}. Attempting rollback.", package_id, e);
                 install_log(&format!(
                     "UPDATE FAIL: {} — {}. Rolling back.",
@@ -248,7 +260,7 @@ impl RpcHandler {
         }
     }
 
-    /// Core update execution: stop → pull → remove → recreate → verify.
+    /// Images are prepared first; then stop → remove → recreate → verify.
     async fn execute_update(
         &self,
         package_id: &str,
@@ -289,29 +301,6 @@ impl RpcHandler {
             }
         }
 
-        // Phase: PullingImage — about to fetch each pinned image in turn.
-        self.set_install_phase(package_id, InstallPhase::PullingImage)
-            .await;
-
-        // 2. Pull new images with progress
-        info!(
-            "Update {}: pulling {} images",
-            package_id,
-            images_to_pull.len()
-        );
-        for (i, (name, image)) in images_to_pull.iter().enumerate() {
-            info!(
-                "Update {}: pulling image {}/{} ({})",
-                package_id,
-                i + 1,
-                images_to_pull.len(),
-                image
-            );
-            self.pull_update_image(package_id, image)
-                .await
-                .context(format!("Failed to pull {} for {}", image, name))?;
-        }
-
         // 3. Remove old containers
         info!("Update {}: removing old containers", package_id);
         for name in containers {
@@ -430,6 +419,32 @@ impl RpcHandler {
     async fn pull_update_image(&self, package_id: &str, image: &str) -> Result<()> {
         self.set_install_progress(package_id, 0, 0).await;
 
+        if immutable_update_image(image) {
+            // A digest-addressed lookup asks Podman for these exact bytes,
+            // unlike a mutable tag lookup. Private imports need no registry.
+            let local = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                tokio::process::Command::new("podman")
+                    .args(["image", "exists", image])
+                    .kill_on_drop(true)
+                    .output(),
+            )
+            .await
+            .context("Local image lookup timed out; existing app remains unchanged")??;
+            match local.status.code() {
+                Some(0) => {
+                    self.set_install_progress(package_id, 100, 100).await;
+                    return Ok(());
+                }
+                Some(1) => {}
+                _ => anyhow::bail!("Cannot inspect local image storage; update cancelled"),
+            }
+        }
+        anyhow::ensure!(
+            !image.starts_with("localhost/"),
+            "The exact private image must be imported before updating this app"
+        );
+
         let mut cmd = tokio::process::Command::new("podman");
         cmd.arg("pull");
         if archipelago_container::image_uses_insecure_registry(image) {
@@ -655,6 +670,44 @@ fn should_try_orchestrator_update(package_id: &str, orchestrator_available: bool
     orchestrator_available && !uses_legacy_update_flow(package_id)
 }
 
+fn immutable_update_image(image: &str) -> bool {
+    image.rsplit_once("@sha256:").is_some_and(|(name, digest)| {
+        !name.is_empty()
+            && !name.contains('@')
+            && digest.len() == 64
+            && digest.bytes().all(|byte| byte.is_ascii_hexdigit())
+    })
+}
+
+#[derive(Debug)]
+enum UpdateFailure {
+    Preparation(anyhow::Error),
+    Execution(anyhow::Error),
+}
+
+/// A preparation failure never enters the lifecycle/rollback path. Keep this
+/// sequencing injectable so failed second-image pulls are tested without apps.
+async fn preflighted_stack_update<P, PF, E, EF>(
+    images: &[(String, String)],
+    mut prepare: P,
+    execute: E,
+) -> std::result::Result<(), UpdateFailure>
+where
+    P: FnMut(String) -> PF,
+    PF: std::future::Future<Output = Result<()>>,
+    E: FnOnce() -> EF,
+    EF: std::future::Future<Output = Result<()>>,
+{
+    for (name, image) in images {
+        prepare(image.clone()).await.map_err(|error| {
+            UpdateFailure::Preparation(error.context(format!(
+                "Cannot prepare image for {name}; existing containers were left unchanged"
+            )))
+        })?;
+    }
+    execute().await.map_err(UpdateFailure::Execution)
+}
+
 fn orchestrator_update_app_id(package_id: &str) -> &str {
     match package_id {
         "electrs" | "mempool-electrs" => "electrumx",
@@ -716,11 +769,85 @@ fn candidate_app_ids_for_container(container_name: &str) -> Vec<String> {
 #[cfg(test)]
 mod tests {
     use super::{
-        candidate_app_ids_for_container, orchestrator_update_app_id,
+        candidate_app_ids_for_container, immutable_update_image, orchestrator_update_app_id,
         should_try_orchestrator_update, update_targets_need_change, uses_legacy_update_flow,
         verify_update_targets,
     };
 
+    #[tokio::test]
+    async fn stack_image_failure_precedes_every_lifecycle_action() {
+        use std::sync::{Arc, Mutex};
+        for failed_second in [true, false] {
+            let calls = Arc::new(Mutex::new(Vec::new()));
+            let preparing = calls.clone();
+            let executing = calls.clone();
+            let images = vec![
+                ("web".into(), "web-image".into()),
+                ("api".into(), "api-image".into()),
+            ];
+            let result = super::preflighted_stack_update(
+                &images,
+                move |image| {
+                    let preparing = preparing.clone();
+                    async move {
+                        preparing.lock().unwrap().push(format!("prepare:{image}"));
+                        anyhow::ensure!(!(failed_second && image == "api-image"), "import missing");
+                        Ok(())
+                    }
+                },
+                move || async move {
+                    executing.lock().unwrap().extend([
+                        "stop".into(),
+                        "remove".into(),
+                        "start".into(),
+                    ]);
+                    Ok(())
+                },
+            )
+            .await;
+            if failed_second {
+                assert!(matches!(result, Err(super::UpdateFailure::Preparation(_))));
+                assert_eq!(
+                    *calls.lock().unwrap(),
+                    ["prepare:web-image", "prepare:api-image"]
+                );
+            } else {
+                assert!(result.is_ok());
+                assert_eq!(
+                    *calls.lock().unwrap(),
+                    [
+                        "prepare:web-image",
+                        "prepare:api-image",
+                        "stop",
+                        "remove",
+                        "start"
+                    ]
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn only_exact_digest_refs_may_skip_update_registry_pull() {
+        let digest = "ab".repeat(32);
+        assert!(immutable_update_image(&format!(
+            "localhost/lfg2025/indeedhub:1.0.1@sha256:{digest}"
+        )));
+        assert!(immutable_update_image(&format!(
+            "registry.example/app@sha256:{digest}"
+        )));
+        for mutable_or_invalid in [
+            "localhost/lfg2025/indeedhub:1.0.1".to_string(),
+            "registry.example/app:latest".to_string(),
+            format!("registry.example/app@sha256:{}", "g".repeat(64)),
+            "registry.example/app@sha256:abcd".to_string(),
+            format!("@sha256:{digest}"),
+            format!("registry.example/app@other@sha256:{digest}"),
+        ] {
+            assert!(!immutable_update_image(&mutable_or_invalid));
+        }
+    }
+
     #[test]
     fn mempool_update_preflight_rejects_stale_catalog_without_reinstalling() {
         let installed = vec![(

docs/node-demo-catalog-and-media.md

@@ -322,3 +322,34 @@ This deployment includes file availability and minimum on-chain amount checks
 and the tested recovery primitives. The complete new purchase caller and
 IndeeHub publication/playback integration remain under development; these are
 not claimed accepted. No new real payment or public publication was performed.
+
+## Private IndeeHub packaging follow-up — 7 October
+
+The separately prepared IndeeHub frontend/API images at local source `3b09b81`
+were built and exported privately with all 671 recorded source inputs unchanged.
+An isolated restore of Yaya's database preserved all 32 original public application
+table hashes, advanced exactly three migrations (107 to 110), and passed an
+idempotent second migration run. This did not change live application data.
+Evidence is in `~/.local/state/archipelago/session-recovery/indeehub-private-assets-build`
+and `indeehub-yaya-restored-qualification`.
+
+OCI export changes the manifest digest while preserving the image config ID.
+The catalog must pin the archive/import digest verified against the config ID,
+not the pre-export build manifest digest. An isolated API import and higher-version
+local alias check confirmed that the exact alias@archive-digest resolves to the
+original image ID. The API archive/import digest is
+`sha256:58f8461f59205ab562a11a888337a8ff79bd47cac017733888825eeb50c42834`.
+Original build receipts remain unchanged; alias provenance is a separate receipt.
+
+The built frontend still uses playback protocol 1. A separately qualified protocol
+2 frontend is required with the next rental-readiness host; no incompatible pair
+will be activated. The API image and migration evidence can be retained when its
+source inputs remain unchanged. New private catalog signing and deployment remain
+pending the matched frontend, imported digest checks and backend qualification.
+
+The current legacy stack updater unconditionally pulls images after stopping
+containers. The draft now prepares every image first and reuses exact digest-pinned
+local imports; missing private images or a failed second-image preflight cannot
+enter lifecycle/rollback. Added tests exercise that production sequencing.
+Formatting checks pass; backend tests/compilation are pending. This narrow fix
+is not acceptance of the separate stopped-app staging/rollback work.