fix(apps): preserve state across runtime repairs and restore Gitea SSH

apps/gitea/manifest.yml

@@ -15,6 +15,9 @@ app:
     image: source.archipelago-foundation.org/lfg2025/gitea:1.27.3
     pull_policy: if-not-present
 
+  # Preserve repositories, database, keys and configuration during runtime repairs.
+  backup_before_runtime_change: true
+
   dependencies:
     # Source history, LFS objects, release artifacts and OCI layers all share
     # this persistent store. 500Mi was only suitable for an empty demo node.
@@ -25,7 +28,7 @@ app:
     disk_limit: 50Gi
 
   security:
-    capabilities: [CHOWN, FOWNER, SETUID, SETGID, DAC_OVERRIDE, NET_BIND_SERVICE]
+    capabilities: [CHOWN, FOWNER, SETUID, SETGID, DAC_OVERRIDE, NET_BIND_SERVICE, SYS_CHROOT]
     readonly_root: false
     no_new_privileges: false
     network_policy: bridge
@@ -62,6 +65,17 @@ app:
       target: /etc/gitea
       options: [rw]
 
+  # Seed a fresh installation with the same origin advertised by the app gate.
+  # Existing app.ini (including custom HTTPS/domain settings) is never replaced.
+  files:
+    - path: /var/lib/archipelago/gitea/data/gitea/conf/app.ini
+      overwrite: false
+      content: |
+        [server]
+        DOMAIN = {{HOST_IP}}
+        SSH_DOMAIN = {{HOST_IP}}
+        ROOT_URL = http://{{HOST_IP}}:3001/
+
   environment:
     - GITEA__database__DB_TYPE=sqlite3
     - GITEA__server__SSH_PORT=2222

apps/portainer/manifest.yml

@@ -22,7 +22,7 @@ app:
     data_uid: "1000:1000"
 
   # Snapshot state before an upgrade recreates this app with new networking.
-  backup_on_network_change: true
+  backup_before_runtime_change: true
 
   dependencies:
     - storage: 1Gi

core/archipelago/src/container/app_catalog.rs

@@ -118,7 +118,7 @@ fn selected_manifest(entry: AppCatalogEntry) -> Option<serde_json::Value> {
     // Never let an unknown future requirement become an unsafe partial match.
     for variant in entry.manifest_variants.into_iter().rev() {
         if !variant.requires.is_empty() && variant.requires.iter().all(|capability| {
-            capability == "network-migration-backup-v1"
+            capability == "runtime-migration-backup-v1"
         }) {
             return Some(variant.manifest);
         }
@@ -583,8 +583,8 @@ mod tests {
     fn catalog_migration_variant_is_compatible_with_old_and_future_daemons() {
         let raw = serde_json::json!({
             "version": "2.45.0", "manifest": {"app": {"id": "portainer", "container": {}}},
-            "manifest_variants": [{"requires": ["network-migration-backup-v1"],
+            "manifest_variants": [{"requires": ["runtime-migration-backup-v1"],
-                "manifest": {"app": {"id": "portainer", "container": {"network": "slirp4netns"}, "backup_on_network_change": true}}}]
+                "manifest": {"app": {"id": "portainer", "container": {"network": "slirp4netns"}, "backup_before_runtime_change": true}}}]
         });
         #[derive(Deserialize)]
         struct OldEntry { manifest: serde_json::Value }
@@ -593,7 +593,7 @@ mod tests {
         let current: AppCatalogEntry = serde_json::from_value(raw.clone()).unwrap();
         let chosen = selected_manifest(current).unwrap();
         assert_eq!(chosen["app"]["container"]["network"], "slirp4netns");
-        assert_eq!(chosen["app"]["backup_on_network_change"], true);
+        assert_eq!(chosen["app"]["backup_before_runtime_change"], true);
         let mut future = raw;
         future["manifest_variants"][0]["requires"].as_array_mut().unwrap().push(serde_json::json!("unknown-next-capability"));
         let chosen = selected_manifest(serde_json::from_value(future).unwrap()).unwrap();

core/archipelago/src/container/migration_backup.rs

@@ -1,15 +1,15 @@
-//! Consistent, private snapshots for declaratively opted-in network migrations.
+//! Consistent, private snapshots for declaratively opted-in runtime migrations.
 use anyhow::{bail, Context, Result};
 use archipelago_container::AppManifest;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
 
 pub fn enabled(manifest: &AppManifest) -> Result<bool> {
-    match manifest.app.extensions.get("backup_on_network_change") {
+    match manifest.app.extensions.get("backup_before_runtime_change") {
         None => Ok(false),
         Some(value) => value
             .as_bool()
-            .context("backup_on_network_change must be boolean"),
+            .context("backup_before_runtime_change must be boolean"),
     }
 }
 
@@ -24,12 +24,12 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
             continue;
         }
         if volume.volume_type != "bind" {
-            bail!("network migration backup requires bind-mounted persistent state");
+            bail!("runtime migration backup requires bind-mounted persistent state");
         }
         let path = Path::new(&volume.source);
         let relative = path
             .strip_prefix(data_dir)
-            .context("network migration state must be inside the node data directory")?;
+            .context("runtime migration state must be inside the node data directory")?;
         if relative.starts_with("migration-backups") {
             bail!("migration backup cannot include its own archive directory");
         }
@@ -38,7 +38,7 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
                 .components()
                 .any(|c| !matches!(c, std::path::Component::Normal(_)))
         {
-            bail!("invalid network migration state path");
+            bail!("invalid runtime migration state path");
         }
         sources.push(relative.to_path_buf());
     }
@@ -51,7 +51,7 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
         }
     }
     if roots.is_empty() {
-        bail!("network migration backup has no persistent state mounts");
+        bail!("runtime migration backup has no persistent state mounts");
     }
     Ok(roots)
 }
@@ -83,11 +83,11 @@ async fn snapshot_with_command(
             .file_type()
             .is_symlink()
         {
-            bail!("network migration state mount is a symlink; explicit backup required");
+            bail!("runtime migration state mount is a symlink; explicit backup required");
         }
         let canonical = tokio::fs::canonicalize(&path).await?;
         if !canonical.starts_with(&canonical_root) {
-            bail!("network migration state path resolves outside node data directory");
+            bail!("runtime migration state path resolves outside node data directory");
         }
     }
     let root = data_dir.join("migration-backups");
@@ -129,7 +129,7 @@ async fn snapshot_with_command(
     tokio::fs::File::open(&partial).await?.sync_all().await?;
     tokio::fs::rename(&partial, &archive).await?;
     let metadata = serde_json::json!({"app": manifest.app.id, "version": manifest.app.version,
-        "network": manifest.app.container.network, "sources": sources});
+        "network": manifest.app.container.network, "capabilities": manifest.app.security.capabilities, "sources": sources});
     tokio::fs::write(
         dir.join("metadata.json"),
         serde_json::to_vec_pretty(&metadata)?,

core/archipelago/src/container/prod_orchestrator.rs

@@ -99,6 +99,13 @@ fn rootless_network_mode_drifted(expected: Option<&str>, actual: &str) -> bool {
         && actual.trim().split(':').next() != expected
 }
 
+fn missing_declared_capability(expected: &[String], actual: &[String]) -> bool {
+    expected.iter().any(|required| {
+        let required = required.strip_prefix("CAP_").unwrap_or(required);
+        !actual.iter().any(|cap| cap.strip_prefix("CAP_").unwrap_or(cap) == required)
+    })
+}
+
 fn uses_pasta_network(manifest: &AppManifest) -> bool {
     manifest.app.container.network.as_deref() == Some("pasta")
 }
@@ -2472,6 +2479,7 @@ impl ProdContainerOrchestrator {
                             .await
                         {
                             tracing::info!(app_id = %app_id, container = %name, "container published-port drift detected — recreating");
+                            self.backup_runtime_change(&name, &resolved_manifest).await?;
                             let _ = self.runtime.stop_container(&name).await;
                             let _ = self.runtime.remove_container(&name).await;
                             self.install_fresh(lm).await?;
@@ -2507,7 +2515,7 @@ impl ProdContainerOrchestrator {
                                 return Ok(ReconcileAction::NoOp);
                             }
                             tracing::info!(app_id = %app_id, container = %name, "container env drift detected — recreating");
-                            self.backup_network_change(&name, &resolved_manifest).await?;
+                            self.backup_runtime_change(&name, &resolved_manifest).await?;
                             let _ = self.runtime.stop_container(&name).await;
                             let _ = self.runtime.remove_container(&name).await;
                             self.install_fresh(lm).await?;
@@ -2564,7 +2572,7 @@ impl ProdContainerOrchestrator {
                                 .await
                         {
                             tracing::info!(app_id = %app_id, container = %name, "stopped container env/port drift detected — recreating");
-                            self.backup_network_change(&name, &resolved_manifest).await?;
+                            self.backup_runtime_change(&name, &resolved_manifest).await?;
                             let _ = self.runtime.remove_container(&name).await;
                             self.install_fresh(lm).await?;
                             return Ok(ReconcileAction::Installed);
@@ -2621,6 +2629,7 @@ impl ProdContainerOrchestrator {
                         self.prepare_for_start(&resolved_manifest).await?;
                         if self.container_env_drifted(&name, &resolved_manifest).await {
                             tracing::info!(app_id = %app_id, container = %name, "created container env drift detected — recreating");
+                            self.backup_runtime_change(&name, &resolved_manifest).await?;
                             let _ = self.runtime.remove_container(&name).await;
                             self.install_fresh(lm).await?;
                             return Ok(ReconcileAction::Installed);
@@ -3128,11 +3137,13 @@ impl ProdContainerOrchestrator {
             quadlet::network_aliases_changed(&old_body, &new_body);
         let restart_for_exec_change = quadlet::exec_changed(&old_body, &new_body);
         let restart_for_health_change = quadlet::health_cmd_changed(&old_body, &new_body);
+        let restart_for_security_change = quadlet::security_changed(&old_body, &new_body);
         let needs_restart = restart_required
             || restart_for_port_change
             || restart_for_network_alias_change
             || restart_for_exec_change
-            || restart_for_health_change;
+            || restart_for_health_change
+            || restart_for_security_change;
         // Record the obligation BEFORE replacing the unit. A failed reload or
         // restart must not become a no-op on the next tick just because the
         // generated file already matches the manifest.
@@ -3140,8 +3151,8 @@ impl ProdContainerOrchestrator {
         if pending.is_pending() {
             self.ensure_resolved_source_available(lm).await?;
         }
-        if restart_for_network_alias_change {
+        if needs_restart {
-            self.backup_network_change(name, &resolved).await?;
+            self.backup_runtime_change(name, &resolved).await?;
         }
         let changed = quadlet::write_if_changed(&unit, &unit_dir)
             .await
@@ -3870,18 +3881,21 @@ impl ProdContainerOrchestrator {
         Ok(())
     }
 
-    async fn backup_network_change(&self, name: &str, manifest: &AppManifest) -> Result<()> {
+    async fn backup_runtime_change(&self, name: &str, manifest: &AppManifest) -> Result<()> {
         if !crate::container::migration_backup::enabled(manifest)? {
             return Ok(());
         }
-        // Only back up an actual network migration, not ordinary env drift.
+        // A persistent disk/permission failure must not repeatedly stop a
+        // working old service. Reuse the reconciler's bounded repair budget.
+        if !self.should_attempt_repair(name).await {
+            anyhow::bail!("runtime migration retry budget exhausted; original service retained, inspect backup failure before retrying");
+        }
+        // Called only before a known runtime change. No app-specific commands;
+        // opted-in manifests identify their persistent state through bind mounts.
         let output = tokio::process::Command::new("podman")
             .args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
             .output().await.context("inspect network before migration backup")?;
         let present = if output.status.success() {
-            if !rootless_network_mode_drifted(manifest.app.container.network.as_deref(), &String::from_utf8_lossy(&output.stdout)) {
-                return Ok(());
-            }
             true
         } else {
             // A crash after gracefully stopping a --rm Quadlet container can
@@ -3934,9 +3948,29 @@ impl ProdContainerOrchestrator {
             return true;
         }
 
+        // Generated-unit drift handles managed services; preserve deliberate
+        // systemd drop-in overrides instead of recreating them every tick.
+        let unmanaged = !quadlet::unit_exists(name).await;
+        // Podman's effective bounding set, not Docker-compatible CapAdd (which
+        // can be empty even when Quadlet supplied capabilities).
+        if unmanaged && !manifest.app.security.capabilities.is_empty() {
+            if let Ok(output) = tokio::process::Command::new("podman")
+                .args(["inspect", name, "--format", "{{json .BoundingCaps}}"])
+                .output().await
+            {
+                if output.status.success() {
+                    if let Ok(actual) = serde_json::from_slice::<Vec<String>>(&output.stdout) {
+                        if missing_declared_capability(&manifest.app.security.capabilities, &actual) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+
         // Quadlet handles declarative Network= drift above. Legacy rootless
         // Podman containers need the same convergence when no unit owns them.
-        if matches!(manifest.app.container.network.as_deref(), Some("slirp4netns" | "pasta")) {
+        if unmanaged && matches!(manifest.app.container.network.as_deref(), Some("slirp4netns" | "pasta")) {
             if let Ok(output) = tokio::process::Command::new("podman")
                 .args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
                 .output()
@@ -4993,6 +5027,33 @@ mod tests {
     /// recovered when its siblings have live containers (the stack is
     /// installed), and left alone when the whole stack is gone or the app
     /// is not a stack member at all.
+    #[tokio::test]
+    async fn gitea_fresh_url_seed_preserves_operator_config_and_reports_write_failure() {
+        let manifest = AppManifest::parse(include_str!("../../../../apps/gitea/manifest.yml")).unwrap();
+        let seed = &manifest.app.files[0];
+        assert!(!seed.overwrite);
+        let content = seed.content.replace("{{HOST_IP}}", "192.0.2.1");
+        assert!(content.contains("ROOT_URL = http://192.0.2.1:3001/"));
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("fresh/app.ini");
+        assert_eq!(ensure_rendered_file(path.to_str().unwrap(), &content, seed.overwrite).await.unwrap(), HookOutcome::Rewritten);
+        assert!(tokio::fs::read_to_string(&path).await.unwrap().contains("ROOT_URL"));
+        let custom = "[server]\nROOT_URL = https://git.example.test/\n[database]\nDB_TYPE = postgres\n";
+        tokio::fs::write(&path, custom).await.unwrap();
+        assert_eq!(ensure_rendered_file(path.to_str().unwrap(), &content, seed.overwrite).await.unwrap(), HookOutcome::Unchanged);
+        assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), custom);
+        let impossible = path.join("app.ini");
+        assert!(ensure_rendered_file(impossible.to_str().unwrap(), &content, seed.overwrite).await.is_err());
+    }
+
+    #[test]
+    fn ssh_sandbox_capability_repair_uses_bounding_set_and_preserves_extra_overrides() {
+        let required = vec!["CHOWN".into(), "SYS_CHROOT".into()];
+        assert!(missing_declared_capability(&required, &["CAP_CHOWN".into()]));
+        assert!(!missing_declared_capability(&required, &["CAP_CHOWN".into(), "CAP_SYS_CHROOT".into()]));
+        assert!(!missing_declared_capability(&required, &["CHOWN".into(), "SYS_CHROOT".into(), "CAP_KILL".into()]));
+    }
+
     #[test]
     fn explicit_rootless_network_change_converges_without_guessing_defaults() {
         assert!(rootless_network_mode_drifted(Some("slirp4netns"), "pasta"));

core/archipelago/src/container/quadlet.rs

@@ -991,6 +991,16 @@ pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
     old_ports != new_ports
 }
 
+pub fn security_changed(old_body: &str, new_body: &str) -> bool {
+    ["AddCapability=", "DropCapability=", "NoNewPrivileges=", "ReadOnly=", "User="]
+        .iter().any(|directive| {
+            let mut old = directive_values(old_body, directive);
+            let mut new = directive_values(new_body, directive);
+            old.sort(); new.sort();
+            old != new
+        })
+}
+
 pub fn network_aliases_changed(old_body: &str, new_body: &str) -> bool {
     let old_network = directive_values(old_body, "Network=");
     let new_network = directive_values(new_body, "Network=");
@@ -1989,6 +1999,18 @@ app:
         assert!(pending.complete().await.is_err());
     }
 
+    #[test]
+    fn gitea_ssh_sandbox_capability_is_applied_as_a_runtime_change() {
+        let manifest = AppManifest::parse(include_str!("../../../../apps/gitea/manifest.yml")).unwrap();
+        manifest.validate().unwrap();
+        let new = QuadletUnit::from_manifest(&manifest, "gitea").render();
+        assert!(new.contains("AddCapability=SYS_CHROOT\n"));
+        let old = new.replace("AddCapability=SYS_CHROOT\n", "");
+        assert!(security_changed(&old, &new));
+        assert!(!security_changed(&new, &new));
+        assert!(!security_changed("AddCapability=CHOWN\nAddCapability=SETUID\n", "AddCapability=SETUID\nAddCapability=CHOWN\n"));
+    }
+
     #[test]
     fn network_aliases_changed_detects_network_mode_drift() {
         let old = "[Container]\nNetwork=slirp4netns\n";

core/container/src/manifest.rs

@@ -1074,6 +1074,12 @@ impl AppManifest {
         // `..` copy sources). See docs/manifest-hooks-design.md.
         self.app.hooks.validate()?;
 
+        if let Some(value) = self.app.extensions.get("backup_before_runtime_change") {
+            if value.as_bool().is_none() {
+                return Err(ManifestError::Invalid("backup_before_runtime_change must be boolean".into()));
+            }
+        }
+
         Ok(())
     }
 }
@@ -1111,6 +1117,7 @@ fn validate_security(policy: &SecurityPolicy) -> Result<(), ManifestError> {
         "SETGID",
         "SETUID",
         "SYS_ADMIN",
+        "SYS_CHROOT",
     ];
     let mut seen = HashSet::new();
     for cap in &policy.capabilities {

docs/TODO.md

@@ -14,6 +14,8 @@ doc. See [`ROADMAP.md`](ROADMAP.md) for the curated, public-facing direction.
 
 ## Next release after 1.8.21 — reported 2026-09-30
 
+Release status and acceptance gates: [execution checklist](next-release-20260930.md).
+
 - [ ] **Release blocker: Gitea → Portainer repository integration.** Diagnose
   smart-HTTP reachability from Portainer's actual request namespace, then provide
   one declarative topology and idempotent migration for fresh installs and
@@ -36,8 +38,9 @@ doc. See [`ROADMAP.md`](ROADMAP.md) for the curated, public-facing direction.
   documentation establishes a UI. Verify Bitcoin/Mempool requirements, decide
   whether an existing first-class relay meets Angor's requirements or a relay
   must be packaged with the indexer, and use the Angor logo from angor.io for its
-  service icon. The mentioned setup-documentation link was not included; asked
+  service icon. Official current deployment documentation located and reviewed: stock Mempool
-  the operator for it. Include this service in the next-release scope.
+  plus an optional strfry relay. Reuse of existing indexing services is the
+  proposed approach; implementation and acceptance remain pending. Include this service in the next-release scope.
 
 - [ ] **App lifecycle: keep installed apps visible through restart and hard
   refresh; gate embedded/browser launches on actual web and listener readiness.**

docs/app-manifest-spec.md

@@ -291,21 +291,22 @@ Validate with `scripts/validate-app-manifest.sh` and regenerate the catalog
 with `scripts/generate-app-catalog.py` (drift-checked in CI by
 `scripts/check-app-catalog-drift.py`).
 
-### Persistent-state backup for network migrations
+### Persistent-state backup for runtime repairs
 
-`app.backup_on_network_change: true` opts an app into a stopped-state snapshot
+`app.backup_before_runtime_change: true` opts an app into a stopped-state snapshot
-before an explicitly selected rootless network mode is migrated. The orchestrator
+before reconciliation changes a service’s network, ports, security settings,
+command or health configuration. Image-upgrade backup policy remains separate. The orchestrator
 archives writable persistent bind mounts under the node data directory, collapses
 nested mounts, excludes the runtime Podman socket, and preserves the previous
 Quadlet definition for rollback. Named volumes, outside-data-root state and
 symlinked mount roots fail closed rather than silently producing an incomplete
 backup. A failed snapshot resumes the original service and leaves migration
 pending. Private archives are retained under `migration-backups/`; fresh installs
-and unchanged network configurations do not create migration snapshots.
+and unchanged runtime configurations do not create migration snapshots.
 
 Catalog generation preserves the previously published base manifest for older
 daemons and puts opted-in network changes in a signed `manifest_variants` entry
-requiring `network-migration-backup-v1`. New runtimes select only variants whose
+requiring `runtime-migration-backup-v1`. New runtimes select only variants whose
 complete requirement list they support. Supply `BASE_CATALOG` when generating
 against a different reviewed pre-migration catalog. This keeps catalog refresh
 from applying a migration before the matching OTA code is installed.

docs/gitea-portainer-repair-20260930.md

@@ -1,7 +1,7 @@
 # Same-node Gitea sources in Portainer
 
-Status: root cause reproduced and network repair verified in disposable Portainer
+Status: root cause reproduced and network repair verified in disposable and actual
-instances; final migration integration and release acceptance remain in progress.
+production Portainer instances; final migration integration and release acceptance remain in progress.
 This change belongs to the next signed catalog, OTA and ISO. It does not modify
 published 1.8.21 artifacts.
 
@@ -37,12 +37,16 @@ this public record.
   authentication. Remove obsolete port-3000 nginx metadata/template and the old
   best-effort installer commands which silently rewrote app.ini and falsely
   claimed success. Gitea owns first-run setup and operator configuration.
+- Gitea SSH also failed before authentication: OpenSSH logged a denied
+  `chroot("/var/empty")` because the manifest dropped `SYS_CHROOT`. Add that
+  specific sandbox capability and reconcile security-directive changes. A
+  disposable fixture then passed SSH clone/push with host-key checking enabled.
 - Existing Quadlet reconciliation applies Network= drift. Record a durable
   pending restart before updating the unit and clear it only after a successful
   restart, so failed reloads/restarts and management interruptions retry.
 - Detect explicit rootless network-mode drift in the older Podman runtime too.
   Unspecified networks do not trigger inferred changes to unrelated apps.
-- Portainer opts into `backup_on_network_change`. Before recreation, gracefully
+- Portainer and Gitea opt into `backup_before_runtime_change`. Before recreation, gracefully
   stop the app and archive its writable persistent bind mounts, including nested
   Compose state, once each. Runtime sockets are excluded. Save the previous
   Quadlet definition, where present. Archives live under the node data directory's
@@ -75,14 +79,16 @@ verification stays enabled and API redirects are refused.
 
 The signed catalog embeds manifests and overrides installed disk copies.
 Capability-gated manifest variants keep the previous Portainer manifest as the
-base for older daemons; only daemons supporting `network-migration-backup-v1`
+base for older daemons; only daemons supporting `runtime-migration-backup-v1`
 select the network repair. This prevents catalog refresh from triggering an
 unbacked recreation before the OTA is installed. A disk
 edit alone cannot deliver this fix. Publish the matching catalog with the tested
 runtime, then verify the generated unit, actual network mode and Source API.
 Expect a Portainer interruption while the snapshot and recreation run; duration
 depends on its saved state size.
-Gitea does not need recreation or an app.ini rewrite for this repair.
+The Portainer routing repair does not require a Gitea configuration change.
+The separate SSH capability repair does recreate Gitea, preserving and snapshotting
+both data/config mounts first. Supported systemd drop-in overrides remain intact.
 
 Keep the previous trusted catalog/runtime for rollback. Restore that catalog
 before restoring the saved `previous.container`, reloading user systemd and
@@ -99,14 +105,42 @@ repositories or the production Portainer database with disposable test data.
 - Invalid Git credentials produce a repository-authentication error, distinct
   from TCP refusal. Requested branch and Compose file read from Portainer context.
 - Combined backend suite including the reviewed paid-download PRs and catalog
-  rollout guard: 1,602 passed, zero failed, four existing ignored
+  rollout guard: 1,605 passed, zero failed, four existing ignored
   tests, including stopped-state archive round trips and failure preservation. Container runtime suite: 78 passed.
   Five diagnostic regression tests passed; catalog regeneration is idempotent
   and the generated catalog has zero manifest metadata drift.
 - Fresh managed Gitea and Portainer fixtures: authenticated private Source
   creation, invalid-token rejection, workstation clone/push and exact branch
-  lookup from Portainer namespace passed.
+  lookup from Portainer namespace passed. LFS batch/upload/download and OCI
+  registry authentication/blob/manifest round trips passed. Desktop and mobile
+  login/private-repository/assets/hard-refresh checks passed.
 - Still required before release: live automatic migration with the new runtime,
   snapshot/rollback verification and reversed install-order acceptance,
   lifecycle/reboot convergence, and signed-catalog delivery to the existing app.
   Record LFS/registry/SSH/browser checks and actual hardware/runtime coverage.
+
+### Affected X250: production routing repair verified
+
+Applied the tested rootless network setting to the actual installed Portainer
+through a persistent Quadlet drop-in, after gracefully stopping it and creating a
+private archive of its database and Compose directory. Compared the archive
+against the stopped original before changing configuration; retained the original
+unit and a rollback path. A verification helper initially compared mount list
+order rather than mount identity and safely rolled back; the corrected check
+compares sorted source/destination/write-mode tuples and passed.
+
+The actual production Portainer namespace reproduced connection refusal before
+repair. After repair it received a Git smart-HTTP advertisement, fetched the
+requested branch at its current tip and read its Compose file. Repeating these
+checks after restarting the managed Portainer service passed. All original data
+and socket mounts and the loopback-only HTTP binding are retained. Gitea,
+Bitcoin and the wallet container IDs and start times were unchanged. No stack
+was deployed and no repository credential was changed.
+
+This establishes the routing repair on the affected hardware. A logged-in
+production Portainer Source UI/API acceptance has not yet been recorded; the
+corresponding API checks passed on disposable instances as documented above.
+The installed-node drop-in persists through service restart/reboot but is not the
+fleet delivery mechanism. Automatic migration and signed catalog/OTA/ISO release
+validation remain pending; the source manifest declares the same network mode.
+Private deployment addresses, branch details and state archives are not committed.

docs/next-release-20260930.md

@@ -0,0 +1,63 @@
+# Next OTA and raw ISO after 1.8.21
+
+**Status: implementation and acceptance in progress; NOT ready to release.**
+
+This is the consolidated execution checklist for the operator's chat requests.
+A targeted node repair is not completion of the release. Finish the remaining
+acceptance gates, preserve live wallets and app data, and publish both artifacts
+through git and ngit. No universal absence of future failures is claimed.
+
+## Changes already shipped in 1.8.21 or earlier
+
+Keep these fixes in the next build and include relevant regressions:
+
+- Mempool image/catalog version agreement and update-button behavior.
+- Minibits integration; Framework automatic LND startup and safe unavailable
+  balances. Framework incident closed with operator acceptance.
+- Shorter, single-column ecash backup messaging.
+- AIUI transparent background on desktop/mobile.
+- Cashu paid-file keyset/mint/error/refund corrections, with live purchases.
+- mempool.space explorer fallback, preserving local/custom explorer settings.
+- Bitcoin install pruning choice and matching automatic-pruning behavior.
+- Friendly Bitcoin warmup and LND install/start/sync waiting states.
+- Raw ISO publishing and upload support.
+
+The Primal automatic LNURL comment problem was traced to sender behavior and
+Minibits metadata. The user accepted clearing the sender's automatic comment;
+no unsupported local metadata rewrite or wallet-identity replacement is planned.
+See the Framework incident and 1.8.21 execution records for evidence/limits.
+
+## New release scope and gates
+
+| Task | Implemented/verified | Remaining before release |
+| --- | --- | --- |
+| X250 Bitcoin picker | Inline choices; actual Chromium kiosk selection, readability and pruning layout passed | Include in final UI/build checks |
+| App disappearance/readiness | Durable inventory and safe lifecycle repair; delayed HTTP and desktop/mobile hard-refresh checks passed | Final lifecycle/reboot gate on candidate |
+| X250 GitWorkshop/Nginx | Missing build contexts restored, dependency/build checks and live UI passed; Nginx slow pull diagnosed; truthful progress label | Verify both artifact payloads contain all build contexts |
+| PRs 161/162 | Reviewed, repaired, merged/closed normally; combined regression suite passed | Candidate funded Tor-only purchase, change and Files acceptance |
+| Gitea/Portainer | Root cause confirmed; source network/backup/retry/catalog changes; real X250 routing repair and restart verified; private Git, SSH, LFS, registry and browser fixture checks passed | Automatic migration/rollback, failure retry, reverse install order, reboot convergence, production Source API/UI, signed delivery |
+| Angor headless store service | Current official guide reviewed: standard Mempool with optional strfry relay | Implement using app-development docs; safe dependency/relay integration; official logo; API and lifecycle acceptance |
+
+Durable payment receipts after a lost seller response remain a separately
+recorded design follow-up. Preserve the truthful unconfirmed-refund warning and
+prevent duplicate automatic payment; do not describe an unconfirmed refund as
+completed. See PR review for the accepted scope and coverage limits.
+
+## Final release checklist
+
+- [ ] Finish all new-scope implementation and specific acceptance above.
+- [ ] Remove disposable fixtures and temporary test overrides; verify native
+      Bitcoin/LND identity and start-state baselines remain protected.
+- [ ] Commit and push completed source changes to git and ngit.
+- [ ] Run final backend/UI/regression/release gates on the final source; inspect
+      skipped tests and report actual hardware/runtime coverage.
+- [ ] Prepare compatible signed app catalog; old runtimes must not apply a
+      migration before they have backup/recovery support.
+- [ ] Version/changelog and OTA payload prepared, validated and signed by user.
+- [ ] Raw ISO built; payload hashes/content verified; installer boot tested.
+- [ ] User signs ISO checksums; publish OTA and ISO plus verification files on
+      git and ngit; independently read back hashes and update discovery.
+- [ ] Provide LAN scp command for the new raw ISO.
+
+Latest backend source verification: 1,605 passed, zero failed, four existing
+ignored tests. This is one layer of evidence, not a substitute for live gates.

scripts/check-app-catalog-drift.py

@@ -83,7 +83,7 @@ def load_catalog(path: Path) -> dict[str, dict[str, Any]]:
             manifest = entry.get("manifest")
             for variant in reversed(entry.get("manifest_variants", [])):
                 requires = variant.get("requires", [])
-                if requires and all(cap == "network-migration-backup-v1" for cap in requires):
+                if requires and all(cap == "runtime-migration-backup-v1" for cap in requires):
                     manifest = variant.get("manifest")
                     break
             if isinstance(manifest, dict) and isinstance(manifest.get("app"), dict):

scripts/generate-app-catalog.sh

@@ -187,13 +187,13 @@ if os.environ.get("EMBED_MANIFESTS") and apps_dir:
         entry = apps.setdefault(str(app_id), {})
         entry.setdefault("version", str(app.get("version", "")) or "0")
         rendered = _retarget_registry(data)
-        if data["app"].get("backup_on_network_change"):
+        if data["app"].get("backup_before_runtime_change"):
             baseline = baseline_entries.get(app_id, {}).get("manifest")
-            if not baseline or baseline.get("app", {}).get("backup_on_network_change"):
+            if not baseline or baseline.get("app", {}).get("backup_before_runtime_change"):
                 raise SystemExit(f"{app_id}: a pre-migration BASE_CATALOG manifest is required for old-node compatibility")
             entry["manifest"] = baseline
             entry["manifest_variants"] = [{
-                "requires": ["network-migration-backup-v1"], "manifest": rendered,
+                "requires": ["runtime-migration-backup-v1"], "manifest": rendered,
             }]
         else:
             entry["manifest"] = rendered