Isolate backend tests from live node wallets and services

AGENTS.md

@@ -21,3 +21,11 @@ While its status is OPEN:
 This priority comes from the user's explicit instruction on 2026-09-15. It remains
 in effect across sessions until the documented acceptance criteria are met or the
 user explicitly changes it.
+
+## Unit tests on a live node
+
+Run backend unit tests through `scripts/test-backend-isolated.sh`. Do not run
+unrestricted `cargo test` on a node with installed apps: older mocked-runtime
+tests still reached real service commands. The runner isolates wallet data,
+service buses, container storage, networking, and process IDs. Compilation with
+`cargo test --no-run` is safe. Keep separately authorized live checks explicit.

core/archipelago/src/container/prod_orchestrator.rs

@@ -798,6 +798,10 @@ fn host_port_bindings_drifted(
 }
 
 async fn ensure_user_podman_socket() -> Result<()> {
+    // Unit tests inject a runtime; they must not restart the host Podman API.
+    if cfg!(test) {
+        return Ok(());
+    }
     let socket_path = "/run/user/1000/podman/podman.sock";
     if podman_socket_accepts_connections(socket_path).await {
         return Ok(());
@@ -3231,6 +3235,9 @@ impl ProdContainerOrchestrator {
     }
 
     async fn ensure_container_network(&self, manifest: &AppManifest) -> Result<()> {
+        if cfg!(test) {
+            return Ok(());
+        }
         let Some(network) = manifest.app.container.network.as_deref() else {
             return Ok(());
         };

core/archipelago/src/container/quadlet.rs

@@ -676,6 +676,13 @@ pub async fn unit_exists(name: &str) -> bool {
 
 /// Resolve the per-user quadlet dir under $HOME. Created if missing.
 pub async fn unit_dir() -> Result<PathBuf> {
+    #[cfg(test)]
+    {
+        static TEST_UNITS: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
+        return Ok(TEST_UNITS
+            .get_or_init(|| tempfile::tempdir().unwrap().keep())
+            .clone());
+    }
     let home = std::env::var_os("HOME")
         .map(PathBuf::from)
         .ok_or_else(|| anyhow!("HOME not set; cannot locate quadlet unit dir"))?;
@@ -810,6 +817,11 @@ async fn systemctl_user_status(
     args: &[&str],
     timeout: Duration,
 ) -> Result<std::process::ExitStatus> {
+    #[cfg(test)]
+    {
+        use std::os::unix::process::ExitStatusExt;
+        return Ok(std::process::ExitStatus::from_raw(0));
+    }
     let mut cmd = Command::new("systemctl");
     cmd.arg("--user").args(args);
     cmd.kill_on_drop(true);
@@ -856,6 +868,10 @@ async fn wait_not_deactivating(service: &str, timeout: Duration) -> bool {
 }
 
 async fn systemctl_user_output(args: &[&str], timeout: Duration) -> Result<std::process::Output> {
+    #[cfg(test)]
+    {
+        anyhow::bail!("Unit tests have no real user service manager");
+    }
     let mut cmd = Command::new("systemctl");
     cmd.arg("--user").args(args);
     cmd.kill_on_drop(true);
@@ -960,6 +976,9 @@ pub async fn disable_remove(unit_name: &str, dir: &Path) -> Result<()> {
 
 /// Is the quadlet-generated service currently active?
 pub async fn is_active(service: &str) -> bool {
+    if cfg!(test) {
+        return false;
+    }
     Command::new("systemctl")
         .args(["--user", "is-active", "--quiet", service])
         .status()

core/archipelago/src/update.rs

@@ -1481,6 +1481,21 @@ pub async fn cancel_download(data_dir: &Path) -> Result<()> {
 /// service unit that inherits systemd's default protections (i.e. none
 /// of ours), escaping the namespace.
 pub(crate) async fn host_sudo(args: &[&str]) -> Result<std::process::ExitStatus> {
+    #[cfg(test)]
+    {
+        anyhow::ensure!(
+            std::env::var("ARCHY_TEST_ISOLATED").as_deref() == Ok("1"),
+            "Host-operation tests require scripts/test-backend-isolated.sh"
+        );
+        let (program, args) = args.split_first().context("Missing test command")?;
+        // Run inside the test namespace, never escape through sudo/systemd-run.
+        return tokio::process::Command::new(program)
+            .args(args)
+            .status()
+            .await
+            .context("isolated test command failed");
+    }
+
     let mut full: Vec<&str> = vec![
         "systemd-run",
         "--wait",
@@ -1505,6 +1520,21 @@ pub(crate) async fn host_sudo(args: &[&str]) -> Result<std::process::ExitStatus>
 /// Same mechanism as `host_sudo` but captures stdout — for read-only probes
 /// (e.g. `stat`) where the answer is in the output, not the exit status.
 pub(crate) async fn host_sudo_output(args: &[&str]) -> Result<std::process::Output> {
+    #[cfg(test)]
+    {
+        anyhow::ensure!(
+            std::env::var("ARCHY_TEST_ISOLATED").as_deref() == Ok("1"),
+            "Host-operation tests require scripts/test-backend-isolated.sh"
+        );
+        let (program, args) = args.split_first().context("Missing test command")?;
+        // Run inside the test namespace, never escape through sudo/systemd-run.
+        return tokio::process::Command::new(program)
+            .args(args)
+            .output()
+            .await
+            .context("isolated test command failed");
+    }
+
     let mut full: Vec<&str> = vec![
         "systemd-run",
         "--wait",

docs/repair-release-20260929.md

@@ -156,3 +156,36 @@ Follow-up applies the existing source-mtime/stamp checks to both :local and
 image-ID comparison then restarts the UI companion onto the new image. This does
 not restart LND itself. Regression covers every companion's two local tags; final
 backend suite is running. Verify the resulting live rebuilt image before release.
+
+### Test isolation finding — release remains blocked
+
+The next full run passed 1,552 tests but one existing boot-loop timing test failed.
+Its output and node logs exposed an independent test defect: MockRuntime tests
+still invoked real Quadlet service operations and Podman socket recovery. These
+caused further LND/companion restarts during unrestricted unit runs. They were not
+a recurrence of the repaired doctor port check. Stopped unrestricted testing;
+LND has remained running since 19:02:46 UTC during isolated test execution.
+
+New isolated runner hides live wallets, service buses, container storage and host
+process IDs, supplies a private network and temporary writable fixture paths,
+and keeps host filesystems read-only. An independent boundary probe passed.
+Test-only service helpers use a temporary Quadlet directory and simulated service
+results; mocked runtimes skip real Podman socket/network provisioning. Host file
+helpers require the isolated-runner marker and execute inside the namespace
+instead of escaping through sudo/systemd-run. Release harness and AGENTS now
+require this runner. Initial isolation trials correctly blocked host operations
+and exposed fixture permission assumptions; final runner compiles and executes
+the full suite with those fixture paths isolated. No final pass claimed yet.
+
+Main dashboard candidate and AIUI build at b634f41a are now deployed on dev; served
+index SHA matches the build. Live package.versions returns bitcoinPrune=false
+for Core and Knots, preserving current automatic mode. Existing full chain stays
+unpruned. Final backend (Files/cached response/legacy UI delivery follow-ups) is
+not yet deployed; earlier 0f85f588 backend remains live on both nodes.
+
+Final isolated backend run: **1,553 passed, zero failed, four existing ignored**
+in 13 seconds after compilation. Boundary probe confirms no host service buses,
+live wallet data, host process IDs, or external network. Bitcoin/LND start times
+remained unchanged during isolated execution. Production helpers are unchanged;
+the namespace-specific command behavior is compiled only into unit tests.
+Release and ISO gates now use the isolated runner.

scripts/build-iso-release.sh

@@ -112,10 +112,9 @@ VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*
 if [ "$SKIP_GATES" = "0" ]; then
     stage "release-gate-harness" bash tests/release/run.sh
     stage "catalog-drift-strict" python3 scripts/check-app-catalog-drift.py --release --strict
-    # Full Rust suite — the release harness only runs a 6-module slice;
+    # The release harness runs the full backend suite inside namespaces.
-    # ~1000 tests otherwise go unverified at ISO time (hardening plan §H).
+    # Never execute unrestricted tests on a node with live wallets/services.
-    stage "cargo-test-full" timeout 5400 env CARGO_INCREMENTAL=0 \
+
-        nice -n 10 cargo test --manifest-path core/Cargo.toml -p archipelago --bin archipelago
 else
     echo; echo "═══ [gates] SKIPPED (--skip-gates)"
 fi

scripts/test-backend-isolated.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Compile normally; execute unit tests away from real wallets, service buses,
+# container storage, processes and networking. Never silently fall back to host.
+set -euo pipefail
+REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+command -v systemd-run >/dev/null
+command -v unshare >/dev/null
+command -v setpriv >/dev/null
+sudo -n true || { echo 'Isolated backend tests require noninteractive sudo for systemd namespaces.' >&2; exit 1; }
+metadata=$(mktemp)
+trap 'rm -f "$metadata"' EXIT
+if ! cargo test --manifest-path "$REPO/core/Cargo.toml" -p archipelago --bin archipelago \
+  --locked --no-run --message-format=json --config 'profile.test.package.archipelago.opt-level=0' > "$metadata"; then
+    python3 - "$metadata" <<'PYDIAG'
+import json,sys
+for line in open(sys.argv[1]):
+ try: item=json.loads(line)
+ except json.JSONDecodeError: continue
+ rendered=item.get('message',{}).get('rendered') if item.get('reason')=='compiler-message' else None
+ if rendered: print(rendered,file=sys.stderr,end='')
+PYDIAG
+    exit 1
+fi
+executable=$(python3 - "$metadata" <<'PY'
+import json,sys
+found=[]
+for line in open(sys.argv[1]):
+ try: item=json.loads(line)
+ except json.JSONDecodeError: continue
+ if item.get('reason')=='compiler-artifact' and item.get('profile',{}).get('test') and item.get('executable'):
+  found.append(item['executable'])
+assert len(found)==1, f'Expected one unit test executable, got {len(found)}'
+print(found[0])
+PY
+)
+[[ -x "$executable" ]]
+unit="archy-isolated-tests-$(date +%s)-$$"
+sudo -n systemd-run --unit="$unit" --wait --pipe --collect \
+  --property="WorkingDirectory=$REPO/core" \
+  --property=PrivateNetwork=yes --property=PrivateTmp=yes --property=PrivateDevices=yes \
+  --property=ProtectSystem=strict --property=ProtectHome=read-only \
+  --property=NoNewPrivileges=yes \
+  --property='TemporaryFileSystem=/run:rw /var/lib/archipelago:rw /var/lib/containers:rw /root:rw' \
+  --setenv=ARCHY_TEST_ISOLATED=1 \
+  /usr/bin/unshare --pid --fork --mount-proc --kill-child \
+  /usr/bin/setpriv --bounding-set=-all,+chown,+dac_override,+fowner,+setuid,+setgid,+kill \
+  "$executable" --test-threads=4 "$@"

tests/release/run.sh

@@ -169,9 +169,7 @@ stage "cargo-check"      timeout 580 cargo check --manifest-path core/Cargo.toml
 # 2026-08-20 1500s died at unit 427/429 (the archipelago bin test, the biggest
 # link) on a loaded, swapping dev box, again without running a single test.
 # 3600s leaves headroom; a warm target/ finishes in a fraction of it.
-stage "cargo-test-weekly" timeout 3600 env CARGO_INCREMENTAL=0 \
+stage "cargo-test-isolated" timeout 3600 bash scripts/test-backend-isolated.sh
-  cargo test --manifest-path core/Cargo.toml -p archipelago -- \
-  update:: lnd container::image_versions upgrade_preserves_container scanner drift missing_secret collision payment_tests bitcoin_storage bitcoin_status
 
 # ── Stage 4: live node smoke ─────────────────────────────────────────
 if [[ $LIVE -eq 1 ]]; then