From 1f9abefc358eef243985e026119d60ab2dc7bd29 Mon Sep 17 00:00:00 2001 From: archipelago Date: Tue, 29 Sep 2026 15:15:51 -0400 Subject: [PATCH] Isolate backend tests from live node wallets and services --- AGENTS.md | 8 ++++ .../src/container/prod_orchestrator.rs | 7 +++ core/archipelago/src/container/quadlet.rs | 19 ++++++++ core/archipelago/src/update.rs | 30 ++++++++++++ docs/repair-release-20260929.md | 33 +++++++++++++ scripts/build-iso-release.sh | 7 ++- scripts/test-backend-isolated.sh | 47 +++++++++++++++++++ tests/release/run.sh | 4 +- 8 files changed, 148 insertions(+), 7 deletions(-) create mode 100755 scripts/test-backend-isolated.sh diff --git a/AGENTS.md b/AGENTS.md index 6b75350c..46fbf04c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,3 +21,11 @@ While its status is OPEN: This priority comes from the user's explicit instruction on 2026-09-15. It remains in effect across sessions until the documented acceptance criteria are met or the user explicitly changes it. + +## Unit tests on a live node + +Run backend unit tests through `scripts/test-backend-isolated.sh`. Do not run +unrestricted `cargo test` on a node with installed apps: older mocked-runtime +tests still reached real service commands. The runner isolates wallet data, +service buses, container storage, networking, and process IDs. Compilation with +`cargo test --no-run` is safe. Keep separately authorized live checks explicit. diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 712e660e..f25d8dbd 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -798,6 +798,10 @@ fn host_port_bindings_drifted( } async fn ensure_user_podman_socket() -> Result<()> { + // Unit tests inject a runtime; they must not restart the host Podman API. + if cfg!(test) { + return Ok(()); + } let socket_path = "/run/user/1000/podman/podman.sock"; if podman_socket_accepts_connections(socket_path).await { return Ok(()); @@ -3231,6 +3235,9 @@ impl ProdContainerOrchestrator { } async fn ensure_container_network(&self, manifest: &AppManifest) -> Result<()> { + if cfg!(test) { + return Ok(()); + } let Some(network) = manifest.app.container.network.as_deref() else { return Ok(()); }; diff --git a/core/archipelago/src/container/quadlet.rs b/core/archipelago/src/container/quadlet.rs index 00a5b7fc..34216acc 100644 --- a/core/archipelago/src/container/quadlet.rs +++ b/core/archipelago/src/container/quadlet.rs @@ -676,6 +676,13 @@ pub async fn unit_exists(name: &str) -> bool { /// Resolve the per-user quadlet dir under $HOME. Created if missing. pub async fn unit_dir() -> Result { + #[cfg(test)] + { + static TEST_UNITS: std::sync::OnceLock = std::sync::OnceLock::new(); + return Ok(TEST_UNITS + .get_or_init(|| tempfile::tempdir().unwrap().keep()) + .clone()); + } let home = std::env::var_os("HOME") .map(PathBuf::from) .ok_or_else(|| anyhow!("HOME not set; cannot locate quadlet unit dir"))?; @@ -810,6 +817,11 @@ async fn systemctl_user_status( args: &[&str], timeout: Duration, ) -> Result { + #[cfg(test)] + { + use std::os::unix::process::ExitStatusExt; + return Ok(std::process::ExitStatus::from_raw(0)); + } let mut cmd = Command::new("systemctl"); cmd.arg("--user").args(args); cmd.kill_on_drop(true); @@ -856,6 +868,10 @@ async fn wait_not_deactivating(service: &str, timeout: Duration) -> bool { } async fn systemctl_user_output(args: &[&str], timeout: Duration) -> Result { + #[cfg(test)] + { + anyhow::bail!("Unit tests have no real user service manager"); + } let mut cmd = Command::new("systemctl"); cmd.arg("--user").args(args); cmd.kill_on_drop(true); @@ -960,6 +976,9 @@ pub async fn disable_remove(unit_name: &str, dir: &Path) -> Result<()> { /// Is the quadlet-generated service currently active? pub async fn is_active(service: &str) -> bool { + if cfg!(test) { + return false; + } Command::new("systemctl") .args(["--user", "is-active", "--quiet", service]) .status() diff --git a/core/archipelago/src/update.rs b/core/archipelago/src/update.rs index 51455303..8c27b1f9 100644 --- a/core/archipelago/src/update.rs +++ b/core/archipelago/src/update.rs @@ -1481,6 +1481,21 @@ pub async fn cancel_download(data_dir: &Path) -> Result<()> { /// service unit that inherits systemd's default protections (i.e. none /// of ours), escaping the namespace. pub(crate) async fn host_sudo(args: &[&str]) -> Result { + #[cfg(test)] + { + anyhow::ensure!( + std::env::var("ARCHY_TEST_ISOLATED").as_deref() == Ok("1"), + "Host-operation tests require scripts/test-backend-isolated.sh" + ); + let (program, args) = args.split_first().context("Missing test command")?; + // Run inside the test namespace, never escape through sudo/systemd-run. + return tokio::process::Command::new(program) + .args(args) + .status() + .await + .context("isolated test command failed"); + } + let mut full: Vec<&str> = vec![ "systemd-run", "--wait", @@ -1505,6 +1520,21 @@ pub(crate) async fn host_sudo(args: &[&str]) -> Result /// Same mechanism as `host_sudo` but captures stdout — for read-only probes /// (e.g. `stat`) where the answer is in the output, not the exit status. pub(crate) async fn host_sudo_output(args: &[&str]) -> Result { + #[cfg(test)] + { + anyhow::ensure!( + std::env::var("ARCHY_TEST_ISOLATED").as_deref() == Ok("1"), + "Host-operation tests require scripts/test-backend-isolated.sh" + ); + let (program, args) = args.split_first().context("Missing test command")?; + // Run inside the test namespace, never escape through sudo/systemd-run. + return tokio::process::Command::new(program) + .args(args) + .output() + .await + .context("isolated test command failed"); + } + let mut full: Vec<&str> = vec![ "systemd-run", "--wait", diff --git a/docs/repair-release-20260929.md b/docs/repair-release-20260929.md index cd605f5b..02c1a42d 100644 --- a/docs/repair-release-20260929.md +++ b/docs/repair-release-20260929.md @@ -156,3 +156,36 @@ Follow-up applies the existing source-mtime/stamp checks to both :local and image-ID comparison then restarts the UI companion onto the new image. This does not restart LND itself. Regression covers every companion's two local tags; final backend suite is running. Verify the resulting live rebuilt image before release. + +### Test isolation finding — release remains blocked + +The next full run passed 1,552 tests but one existing boot-loop timing test failed. +Its output and node logs exposed an independent test defect: MockRuntime tests +still invoked real Quadlet service operations and Podman socket recovery. These +caused further LND/companion restarts during unrestricted unit runs. They were not +a recurrence of the repaired doctor port check. Stopped unrestricted testing; +LND has remained running since 19:02:46 UTC during isolated test execution. + +New isolated runner hides live wallets, service buses, container storage and host +process IDs, supplies a private network and temporary writable fixture paths, +and keeps host filesystems read-only. An independent boundary probe passed. +Test-only service helpers use a temporary Quadlet directory and simulated service +results; mocked runtimes skip real Podman socket/network provisioning. Host file +helpers require the isolated-runner marker and execute inside the namespace +instead of escaping through sudo/systemd-run. Release harness and AGENTS now +require this runner. Initial isolation trials correctly blocked host operations +and exposed fixture permission assumptions; final runner compiles and executes +the full suite with those fixture paths isolated. No final pass claimed yet. + +Main dashboard candidate and AIUI build at b634f41a are now deployed on dev; served +index SHA matches the build. Live package.versions returns bitcoinPrune=false +for Core and Knots, preserving current automatic mode. Existing full chain stays +unpruned. Final backend (Files/cached response/legacy UI delivery follow-ups) is +not yet deployed; earlier 0f85f588 backend remains live on both nodes. + +Final isolated backend run: **1,553 passed, zero failed, four existing ignored** +in 13 seconds after compilation. Boundary probe confirms no host service buses, +live wallet data, host process IDs, or external network. Bitcoin/LND start times +remained unchanged during isolated execution. Production helpers are unchanged; +the namespace-specific command behavior is compiled only into unit tests. +Release and ISO gates now use the isolated runner. diff --git a/scripts/build-iso-release.sh b/scripts/build-iso-release.sh index 1b5462dc..879bf19a 100755 --- a/scripts/build-iso-release.sh +++ b/scripts/build-iso-release.sh @@ -112,10 +112,9 @@ VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".* if [ "$SKIP_GATES" = "0" ]; then stage "release-gate-harness" bash tests/release/run.sh stage "catalog-drift-strict" python3 scripts/check-app-catalog-drift.py --release --strict - # Full Rust suite — the release harness only runs a 6-module slice; - # ~1000 tests otherwise go unverified at ISO time (hardening plan §H). - stage "cargo-test-full" timeout 5400 env CARGO_INCREMENTAL=0 \ - nice -n 10 cargo test --manifest-path core/Cargo.toml -p archipelago --bin archipelago + # The release harness runs the full backend suite inside namespaces. + # Never execute unrestricted tests on a node with live wallets/services. + else echo; echo "═══ [gates] SKIPPED (--skip-gates)" fi diff --git a/scripts/test-backend-isolated.sh b/scripts/test-backend-isolated.sh new file mode 100755 index 00000000..2138b8c7 --- /dev/null +++ b/scripts/test-backend-isolated.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Compile normally; execute unit tests away from real wallets, service buses, +# container storage, processes and networking. Never silently fall back to host. +set -euo pipefail +REPO=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +command -v systemd-run >/dev/null +command -v unshare >/dev/null +command -v setpriv >/dev/null +sudo -n true || { echo 'Isolated backend tests require noninteractive sudo for systemd namespaces.' >&2; exit 1; } +metadata=$(mktemp) +trap 'rm -f "$metadata"' EXIT +if ! cargo test --manifest-path "$REPO/core/Cargo.toml" -p archipelago --bin archipelago \ + --locked --no-run --message-format=json --config 'profile.test.package.archipelago.opt-level=0' > "$metadata"; then + python3 - "$metadata" <<'PYDIAG' +import json,sys +for line in open(sys.argv[1]): + try: item=json.loads(line) + except json.JSONDecodeError: continue + rendered=item.get('message',{}).get('rendered') if item.get('reason')=='compiler-message' else None + if rendered: print(rendered,file=sys.stderr,end='') +PYDIAG + exit 1 +fi +executable=$(python3 - "$metadata" <<'PY' +import json,sys +found=[] +for line in open(sys.argv[1]): + try: item=json.loads(line) + except json.JSONDecodeError: continue + if item.get('reason')=='compiler-artifact' and item.get('profile',{}).get('test') and item.get('executable'): + found.append(item['executable']) +assert len(found)==1, f'Expected one unit test executable, got {len(found)}' +print(found[0]) +PY +) +[[ -x "$executable" ]] +unit="archy-isolated-tests-$(date +%s)-$$" +sudo -n systemd-run --unit="$unit" --wait --pipe --collect \ + --property="WorkingDirectory=$REPO/core" \ + --property=PrivateNetwork=yes --property=PrivateTmp=yes --property=PrivateDevices=yes \ + --property=ProtectSystem=strict --property=ProtectHome=read-only \ + --property=NoNewPrivileges=yes \ + --property='TemporaryFileSystem=/run:rw /var/lib/archipelago:rw /var/lib/containers:rw /root:rw' \ + --setenv=ARCHY_TEST_ISOLATED=1 \ + /usr/bin/unshare --pid --fork --mount-proc --kill-child \ + /usr/bin/setpriv --bounding-set=-all,+chown,+dac_override,+fowner,+setuid,+setgid,+kill \ + "$executable" --test-threads=4 "$@" diff --git a/tests/release/run.sh b/tests/release/run.sh index dd9520d2..b9607461 100755 --- a/tests/release/run.sh +++ b/tests/release/run.sh @@ -169,9 +169,7 @@ stage "cargo-check" timeout 580 cargo check --manifest-path core/Cargo.toml # 2026-08-20 1500s died at unit 427/429 (the archipelago bin test, the biggest # link) on a loaded, swapping dev box, again without running a single test. # 3600s leaves headroom; a warm target/ finishes in a fraction of it. -stage "cargo-test-weekly" timeout 3600 env CARGO_INCREMENTAL=0 \ - cargo test --manifest-path core/Cargo.toml -p archipelago -- \ - update:: lnd container::image_versions upgrade_preserves_container scanner drift missing_secret collision payment_tests bitcoin_storage bitcoin_status +stage "cargo-test-isolated" timeout 3600 bash scripts/test-backend-isolated.sh # ── Stage 4: live node smoke ───────────────────────────────────────── if [[ $LIVE -eq 1 ]]; then