Compare commits

...
Author SHA1 Message Date
archipelago 6d5f3ffb85 fix: select NPM admin port regardless of binding order
Demo images / Build & push demo images (push) Failing after 34s
2026-09-30 18:24:35 -04:00
archipelago d1bc1273d4 fix: replace cached container doctor before ISO first boot 2026-09-30 17:52:30 -04:00
archipelago 96fb5a4f19 fix: prevent stale snapshots resurrecting orphaned dashboards
Demo images / Build & push demo images (push) Failing after 36s
2026-09-30 17:45:18 -04:00
archipelago f91c1f33db fix: keep apps running when network diagnostics fail 2026-09-30 17:34:11 -04:00
20 changed files with 460 additions and 107 deletions
+6
View File
@@ -2,6 +2,12 @@
## v1.8.22-alpha (2026-09-30)
- Fixed Nginx Proxy Manager launch readiness choosing a proxy listener instead of its admin port after container recreation.
- Network diagnostic failures no longer stop all apps or rebuild shared container networking.
- Prevented orphaned companion dashboards from repeatedly reinstalling themselves after their backend app was removed.
- Fixed companion dashboard builds still referencing a retired image registry.
- Fixed Angor Indexer health checks choosing IPv6 localhost for an IPv4 listener and unnecessarily restarting the working service.
- Prevented false app restarts by probing each published port at its actual bind address; Nginx Proxy Manager now checks its internal admin API.
@@ -103,6 +103,15 @@ pub fn companions_for(package_id: &str) -> &'static [CompanionSpec] {
}
}
/// Missing companion UIs are provisioned here, never by snapshot recovery.
/// A stale running-container snapshot must not resurrect an orphaned UI.
pub fn is_companion_app(app_id: &str) -> bool {
ALL_COMPANIONS
.iter()
.flat_map(|specs| specs.iter())
.any(|spec| spec.image_base == app_id)
}
/// Every companion this build knows how to provision. Kept beside
/// `companions_for` — a new companion must be added to both, or the reaper
/// will not recognise it as one of ours and will leave it running forever.
@@ -978,14 +978,20 @@ fn extract_lan_address(ports: &[String]) -> Option<String> {
let mut first_candidate = None;
for port_str in ports {
// Parse port strings like "0.0.0.0:18443->18443/tcp" or "0.0.0.0:18443-18444->18443-18444/tcp"
let Some(public_part) = port_str.split("->").next() else {
let Some((public_part, _)) = port_str.split_once("->") else {
continue;
};
let Some(port_part) = public_part.split(':').nth(1) else {
let Some((_, port_part)) = public_part.rsplit_once(':') else {
continue;
};
// Extract just the first port if it's a range (e.g., "18443-18444" -> "18443")
let host_port = port_part.split('-').next().unwrap_or(port_part);
let Ok(host_port) = host_port.parse::<u16>() else {
continue;
};
if host_port == 0 {
continue;
}
let candidate = format!("http://localhost:{}", host_port);
if first_candidate.is_none() {
first_candidate = Some(candidate.clone());
@@ -1113,6 +1119,29 @@ fn package_launch_candidate(
if let Some(companion) = companion_lan_address(app_id) {
return Some(companion);
}
if app_id == "nginx-proxy-manager" {
// 80/443 serve users' proxy hosts; only container port 81 serves the
// admin UI. Podman's binding order is unstable across recreation.
// Resolve its actual host allocation rather than guessing the first
// HTTP port or hardcoding the default host port 8081.
let admin_ports: Vec<String> = ports
.iter()
.filter(|port| {
port.split_once("->")
.is_some_and(|(_, target)| target == "81/tcp")
})
.cloned()
.collect();
// With published bindings, a missing admin mapping is not evidence
// that some unrelated service on the default host port is this UI.
return extract_lan_address(&admin_ports).or_else(|| {
if ports.is_empty() {
known
} else {
None
}
});
}
if uses_allocated_launch_port(app_id) {
extract_lan_address(ports).or(known)
} else {
@@ -1241,6 +1270,98 @@ mod extract_lan_address_tests {
);
}
#[test]
fn npm_admin_launch_is_independent_of_proxy_binding_order() {
let mappings = [
"10.77.0.2:18081->80/tcp",
"10.77.0.2:18443->443/tcp",
"127.0.0.1:8081->81/tcp",
];
for order in [
[0, 1, 2],
[0, 2, 1],
[1, 0, 2],
[1, 2, 0],
[2, 0, 1],
[2, 1, 0],
] {
let ports: Vec<String> = order.iter().map(|&i| mappings[i].into()).collect();
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&ports,
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:8081")
);
}
}
#[test]
fn npm_admin_launch_respects_host_allocation_and_ipv6_bindings() {
for binding in ["127.0.0.1", "0.0.0.0", "[::1]", "[::]"] {
let ports = vec![
"10.77.0.2:18081->80/tcp".into(),
format!("{binding}:28081->81/tcp"),
];
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&ports,
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:28081")
);
}
let proxies = vec![
"10.77.0.2:18081->80/tcp".into(),
"10.77.0.2:18443->443/tcp".into(),
];
assert_eq!(
package_launch_candidate("nginx-proxy-manager", &proxies, None),
None
);
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&proxies,
Some("http://localhost:8081/".into())
),
None
);
}
#[test]
fn npm_without_port_information_uses_declared_admin_url() {
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&[],
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:8081/")
);
}
#[test]
fn malformed_published_ports_do_not_become_launch_urls() {
for port in [
"81/tcp",
"127.0.0.1:bad->81/tcp",
"[::1]:0->81/tcp",
"[::]:65536->81/tcp",
"127.0.0.1:8081->81/udp",
] {
assert_eq!(
package_launch_candidate("nginx-proxy-manager", &[port.into()], None),
None
);
}
}
#[test]
fn skips_ssh_port_when_web_port_is_published() {
// gitea: SSH published before the web port, in podman's list order.
@@ -2071,6 +2071,10 @@ impl ProdContainerOrchestrator {
Ok(ReconcileAction::Left(reason))
if mode == ReconcileMode::ExistingOnly
&& reason == "absent"
// companion.rs owns missing UI provisioning/removal.
// Never resurrect an orphan from a stale snapshot.
// Existing UIs still pass through security config repair.
&& !super::companion::is_companion_app(&app_id)
&& (was_running.contains(&compute_container_name(&lm.manifest))
// The durable answer, and the one that does not
// erode. `was_running` only records what was
@@ -7225,6 +7229,52 @@ app:
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
}
#[tokio::test]
async fn reconcile_existing_does_not_resurrect_orphaned_companions() {
let rt = Arc::new(MockRuntime::default());
let mut orch = orch_with(rt.clone()).await;
orch.set_disk_gb_for_test(500);
let companions = [
"bitcoin-ui",
"electrs-ui",
"lnd-ui",
"fedimint-ui",
"cuprate-ui",
];
let mut names = Vec::new();
for id in companions {
let manifest = pull_manifest(id, "localhost/companion:local");
names.push(compute_container_name(&manifest));
orch.insert_manifest_for_test(manifest, PathBuf::from("/tmp/companion"))
.await;
}
let refs: Vec<&str> = names.iter().map(String::as_str).collect();
crate::crash_recovery::save_container_snapshot_for_test(&orch.data_dir, &refs).await;
// Repeated passes must leave lifecycle ownership with companion.rs.
for _ in 0..3 {
let report = orch.reconcile_existing().await;
assert_eq!(report.actions.len(), companions.len());
assert!(report
.actions
.iter()
.all(|(_, action)| *action == ReconcileAction::Left("absent".into())));
assert!(report.failures.is_empty());
}
let calls = rt.calls();
for operation in [
"pull_image:",
"create_container:",
"start_container:",
"stop_container:",
"remove_container:",
] {
assert!(
!calls.iter().any(|call| call.starts_with(operation)),
"{calls:?}"
);
}
}
#[tokio::test]
async fn reconcile_existing_self_heals_missing_optional_installed_app() {
// A non-baseline app (gitea) self-heals ONLY with installation
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY tailwind.css /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
COPY qrcode.js /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/index.html
COPY nginx.conf /etc/nginx/conf.d/default.conf
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
#
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Copy the HTML file
COPY index.html /usr/share/nginx/html/
+96
View File
@@ -334,3 +334,99 @@ repository branch and Compose content from its own network namespace.
Version preparation is 1.8.22-alpha. No new release tag or fleet-visible update
manifest is published by the version commit. Optimized candidate deployment,
artifact inspection, ISO smoke/boot checks and offline signatures follow.
### Release blocker discovered during candidate observation: scheduled doctor
The initial `02b840f2` 1.8.22 candidate is rejected for release. On the X250,
2026-09-30 21:10–21:11 UTC, the scheduled `archipelago-doctor.service` explicitly
ran `podman stop --all --time 30`, killed rootless network helpers and ran
`podman system migrate` after a two-attempt external network probe failed.
The journal attributes the stop to that unit, not the app health monitor or a
host reboot. All apps restarted, including Bitcoin, LND and the production site.
The earlier unchanged-container acceptance applies only to immediate deployment;
the later observation failed and must not be represented as a stability pass.
No persistent-data loss has been established. Keep this distinct from the closed
Framework incident; do not wipe or recreate any wallet as a recovery action.
Containment: stopped doctor timers on both test boxes, installed a safe diagnostic
script into both the executable and runtime payload, and rejected/stopped the
old ISO build. Network failure now produces a warning without stopping apps,
killing network processes, migrating Podman or deleting network state. Repeated
failures remain warnings, never a successful repair/check. Regression cases cover
healthy, absent network, non-root invocation, host failure, transient recovery,
repeated endpoint failure and namespace access failure, with mutation tripwires.
Live scheduled-cycle observation and final rebuilt-artifact acceptance are pending.
Recovery also exposed retired `git.tx1138.com` nginx base references in six
companion UI Dockerfiles. They now use the existing primary registry at the same
pinned version. All six images built successfully against that registry; payload
validation rejects the retired host before OTA/ISO packaging.
The post-recovery X250 check passes: Bitcoin authenticated RPC responds and IBD
advances; NPM/Gitea/Portainer APIs respond; Portainer's real namespace fetches
`demo-portainer` at `3ae171d6b0c728665a860520fe393c0abb772798` and its Compose
file; Portainer's original persistent mounts match the earlier backup evidence;
LND wallet/channel databases remain present on their persistent mount. No new
pre-incident cryptographic wallet-identity baseline was available, so these checks
must not be described as an exact identity/balance comparison.
The dev all-container observation also caught a separate Cuprate UI orphan loop:
`companion.rs` removed it because Cuprate was not installed, while generic desired-
state recovery resurrected it from an old running snapshot, using a unit without
nginx's required capabilities. Generic desired-state recovery now excludes missing companions
owned by `companion.rs`; existing companion provisioning/reaping remains the
single owner. Running UIs still receive the existing security configuration repairs. Regression runs repeated reconciliation against stale companion
snapshots and checks that no image/container lifecycle operations occur.
Safe-doctor live acceptance: the X250 completed a 12-minute observation with all
running container IDs, start times and data mounts unchanged. Its journal records
successful doctor runs at 21:22:06, 21:27:51 and 21:33:10 UTC. Both doctor timers
are restored with the safe script. Dev's native Bitcoin/LND stayed running;
all-container dev acceptance remains pending the companion-loop backend fix.
Final-source UI suite: 1,133 passed. Heavy backend compilation is serialized with
remaining build steps to reduce memory/IO pressure on the syncing dev node.
Final source release gates at `96fb5a4f`: 1,613 backend tests passed, zero failed,
four explicitly ignored; 1,133 UI tests passed; type-check, production UI build,
catalog/trust, shell, pruning, LND readiness, NPM migration and doctor regressions
passed. The isolated companion-loop regression passed independently as well.
ISO cache hardening: the installer now carries the current doctor script and
service/timer separately from rootfs.tar and overwrites both historical and active
script locations before first boot. This prevents a cached base image restoring
the old recovery code. A regression executes the actual installer block against
stale disposable files twice and confirms a missing safety payload fails closed.
The mounted-ISO smoke test also compares all three overlay files to source.
The final ISO build captures the exact newly deployed OTA UI/runtime payload.
### Final kiosk acceptance found nondeterministic NPM launch selection
Do not publish the staged `d1bc1273` candidate. NPM itself remains healthy and its
API, Portainer integration, site, and native services passed stability checks.
However, final kiosk acceptance found its card stuck at "Web UI not ready".
The runtime reported bindings in proxy-HTTP, proxy-HTTPS, admin order. The scanner
chose the first non-database/SSH binding, then rejected its tunnel-only host port
as unreachable on loopback, leaving the launch address empty. Earlier tests had
passed with admin first. This is a confirmed order-dependent scanner defect.
The candidate fix explicitly resolves NPM container port 81 to its actual host
allocation. Proxy ports never become the admin URL. Missing/malformed admin
bindings do not fall back to another service when published bindings are present.
Port parsing handles IPv6 authorities and rejects invalid ports. Regressions
cover all six three-port permutations, allocated admin ports, IPv4/IPv6 binding
strings, missing admin mappings, missing runtime port information, and malformed
or UDP bindings. Full backend regression execution is pending for this change.
The ISO build is frozen at installer-environment creation; no release was signed
or published. Rebuild/revalidate the OTA and ISO with this correction.
The companion orphan fix worked live: Cuprate UI was automatically removed and
all installed app container IDs remained unchanged. One observation helper raced
that expected removal between `podman ps` and `inspect`; it now excludes that
known orphan before inspection and repeats the stability check. This was a test
snapshot race, not another installed-app restart.
NPM selector final backend gate passed: 1,617 tests, zero failures, four explicitly
ignored, through the isolated runner. This includes all new port-selection cases
and the existing companion security/configuration and lifecycle regressions.
Rebuild the release binary and UI metadata, deploy those exact OTA bytes to both
boxes, and require actual kiosk hard-refresh/Launch acceptance before ISO assembly.
@@ -2607,6 +2607,11 @@ if [ -f "$SCRIPT_DIR/../../scripts/image-versions.sh" ]; then
echo " ✅ Bundled image-versions.sh"
fi
# Always overlay the current doctor, including when rootfs.tar is cached.
cp "$SCRIPT_DIR/../../scripts/container-doctor.sh" "$ARCH_DIR/scripts/"
cp "$SCRIPT_DIR/../configs/archipelago-doctor.service" "$ARCH_DIR/scripts/"
cp "$SCRIPT_DIR/../configs/archipelago-doctor.timer" "$ARCH_DIR/scripts/"
# Build-source apps need their complete contexts even on unbundled ISOs.
# Keep this identical to the OTA runtime payload; a per-app allowlist silently
# omitted GitWorkshop, FIPS and Cuprate and made fresh installs fail at 70%.
@@ -3230,6 +3235,18 @@ for test_script in run-e2e-tests.sh run-post-install-tests.sh; do
fi
done
# BEGIN DOCTOR OVERLAY
# Replace both the active and historical script locations before first boot.
# A cached rootfs can contain the unsafe network recovery implementation.
mkdir -p /mnt/target/opt/archipelago/scripts /mnt/target/home/archipelago/archy/scripts
for doctor_dir in /mnt/target/opt/archipelago/scripts /mnt/target/home/archipelago/archy/scripts; do
install -m 755 "$BOOT_MEDIA/archipelago/scripts/container-doctor.sh" "$doctor_dir/container-doctor.sh" || exit 1
done
for doctor_unit in archipelago-doctor.service archipelago-doctor.timer; do
install -m 644 "$BOOT_MEDIA/archipelago/scripts/$doctor_unit" "/mnt/target/etc/systemd/system/$doctor_unit" || exit 1
done
# END DOCTOR OVERLAY
# Copy self-update script
if [ -f "$BOOT_MEDIA/archipelago/scripts/self-update.sh" ]; then
cp "$BOOT_MEDIA/archipelago/scripts/self-update.sh" /mnt/target/opt/archipelago/scripts/
@@ -369,6 +369,10 @@ init()
<span class="text-xs text-white/40">September 30, 2026</span>
</div>
<div class="space-y-3 text-sm text-white/80 pl-3 border-l border-white/10">
<p>Fixed Nginx Proxy Manager launch readiness choosing a proxy listener instead of its admin port after container recreation.</p>
<p>Network diagnostic failures no longer stop all apps or rebuild shared container networking.</p>
<p>Prevented orphaned companion dashboards from repeatedly reinstalling themselves after their backend app was removed.</p>
<p>Fixed companion dashboard builds still referencing a retired image registry.</p>
<p>Fixed Angor Indexer health checks choosing IPv6 localhost for an IPv4 listener and unnecessarily restarting the working service.</p>
<p>Prevented false app restarts by probing each published port at its actual bind address; Nginx Proxy Manager now checks its internal admin API.</p>
<p>Added a backed-up migration for the recognized legacy Nginx Proxy Manager tunnel/LND port conflict in both OTA and ISO startup paths.</p>
+2
View File
@@ -27,6 +27,8 @@ def check(root: Path) -> int:
dockerfile = (context / build.get('dockerfile', 'Dockerfile')).resolve()
if not dockerfile.is_relative_to(context) or not dockerfile.is_file():
raise ValueError(f'{app["id"]}: missing or out-of-context Dockerfile: {dockerfile}')
if 'git.tx1138.com/' in dockerfile.read_text():
raise ValueError(f'{app["id"]}: Dockerfile references retired registry git.tx1138.com')
count += 1
return count
+28 -99
View File
@@ -32,6 +32,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
FIXES_APPLIED=0
CHECKS_PASSED=0
CHECKS_WARNED=0
WARNING_NAMES=()
FIX_NAMES=()
log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
@@ -83,7 +85,13 @@ run_fix() {
FIXES_APPLIED=$((FIXES_APPLIED + 1))
FIX_NAMES+=("$name")
else
CHECKS_PASSED=$((CHECKS_PASSED + 1))
local status=$?
if [ "$status" = 1 ]; then
CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
CHECKS_WARNED=$((CHECKS_WARNED + 1))
WARNING_NAMES+=("$name")
fi
fi
}
@@ -446,114 +454,33 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
[ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1
}
# ── Fix 8: Rootless netns egress lost ────────────────────────
# Rootless podman uses pasta to give containers internet egress. If pasta's
# tap vanishes (host link flap, mount churn, pasta dying during a boot-time
# restart storm), the rootless-netns keeps inter-container traffic working
# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls
# fail. The repair must rebuild the netns from scratch: merely cycling the
# containers reuses the existing (broken) netns because its holders
# (aardvark-dns, podman's pause process) survive — observed on a test node
# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers
# every timer run for ~an hour without ever restoring egress. So: stop the
# containers, kill the netns holders, `podman system migrate`, clear the
# stale netns state, then start everything back up.
#
# Destructive-action latch: cycling the whole fleet is a last resort. After
# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly)
# until a run observes egress healthy again, which resets the counter.
NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures"
NETNS_CYCLE_MAX=3
fix_rootless_netns_egress() {
# Needs root for nsenter. When doctor runs as the rootless container owner,
# a failed nsenter probe is a permissions artifact, not evidence of broken
# egress; do not cycle the fleet from that context.
# ── Check 8: Rootless network egress (diagnostic only) ──────
# A single external endpoint or nsenter failure cannot establish that the
# containers have lost connectivity. In particular, entering only the network
# namespace can fail for rootless user namespaces. Never stop apps, kill network
# helpers, migrate Podman, or remove network state in response to this probe.
# Return 1 for healthy/not applicable and 2 for an inconclusive warning.
check_rootless_netns_egress() {
[ "$(id -u)" = "0" ] || return 1
local archi_uid
local archi_uid aardvark_pid
archi_uid=$(id -u archipelago 2>/dev/null) || return 1
# Locate the rootless-netns via aardvark-dns (it lives inside it).
local aardvark_pid
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1 # no rootless network active
[ -n "$aardvark_pid" ] || return 1
# Host precheck: if the host itself can't reach the internet, no point
# cycling containers — this is an upstream problem.
if ! timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
return 1
log "WARNING: host connectivity probe failed; external endpoint may be unavailable. Apps left running."
return 2
fi
# Probe egress from inside the rootless-netns. One probe is noisy;
# require two consecutive failures 10s apart to rule out transients.
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch
return 1 # first probe succeeded
return 1
fi
sleep 10
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE"
return 1 # recovered on its own
fi
# Latch: don't keep bouncing the fleet when the rebuild demonstrably
# isn't fixing it.
local failures
failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0)
case "$failures" in *[!0-9]*|"") failures=0;; esac
if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then
log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)"
return 1
fi
log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns"
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman"
local running
running=$($PODMANCMD ps --format '{{.Names}}' 2>/dev/null)
if [ -z "$running" ]; then
log " No running containers to cycle — skipping"
return 1
fi
local count
count=$(echo "$running" | wc -l)
log " Stopping $count running containers (graceful, 30s)..."
$PODMANCMD stop --all --time 30 >/dev/null 2>&1
sleep 5
# Tear the broken netns down for real: kill its holders and drop the
# stale state so the first container start rebuilds pasta + aardvark-dns
# from scratch. Without this, podman re-enters the old netns and the
# missing pasta tap never comes back.
log " Rebuilding rootless netns (killing holders, clearing state)..."
pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null
pkill -U "$archi_uid" -x pasta 2>/dev/null
pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null
pkill -U "$archi_uid" -x slirp4netns 2>/dev/null
sleep 2
$PODMANCMD system migrate >/dev/null 2>&1
rm -rf "/run/user/$archi_uid/containers/networks"
log " Starting containers back up..."
for c in $running; do
$PODMANCMD start "$c" >/dev/null 2>&1 &
done
wait
sleep 5
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
log " Rootless-netns egress restored ($count containers cycled)"
rm -f "$NETNS_CYCLE_STATE"
else
failures=$((failures + 1))
echo "$failures" > "$NETNS_CYCLE_STATE"
log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention"
return 1
fi
return 0
log "WARNING: rootless network probe inconclusive (endpoint, connectivity, or namespace access). Inspect affected apps before repair. Apps left running."
return 2
}
# ── Fix 9: Restart stopped core containers ──────────────────
@@ -731,7 +658,7 @@ run_fix "tor-permissions" fix_tor_permissions
run_fix "searxng" fix_searxng
run_fix "bitcoin-txindex" fix_bitcoin_txindex
run_fix "exit-127" fix_exit_127
run_fix "netns-egress" fix_rootless_netns_egress
run_fix "netns-egress" check_rootless_netns_egress
run_fix "stopped-core" fix_stopped_core_containers
run_fix "rootless-ports" fix_missing_rootless_ports
run_fix "npm-public-hosts" fix_npm_public_hosts
@@ -740,7 +667,9 @@ run_fix "catatonit" fix_missing_catatonit
run_fix "dialout" fix_archipelago_dialout
echo ""
if [ $FIXES_APPLIED -gt 0 ]; then
if [ "$CHECKS_WARNED" -gt 0 ]; then
log "Done: $CHECKS_WARNED unresolved warnings (${WARNING_NAMES[*]}), $FIXES_APPLIED fixes applied, $CHECKS_PASSED checks passed"
elif [ $FIXES_APPLIED -gt 0 ]; then
log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed"
else
log "Done: all $CHECKS_PASSED checks passed — no fixes needed"
+19
View File
@@ -71,6 +71,25 @@ else
bad "incomplete app build payload"
fi
# The cached rootfs must never restore the unsafe historical doctor on boot.
for doctor_file in container-doctor.sh archipelago-doctor.service archipelago-doctor.timer; do
if [[ "$doctor_file" == container-doctor.sh ]]; then
doctor_source="$REPO/scripts/$doctor_file"
else
doctor_source="$REPO/image-recipe/configs/$doctor_file"
fi
if cmp -s "$doctor_source" "$MNT/archipelago/scripts/$doctor_file"; then
ok "current doctor payload: $doctor_file"
else
bad "missing/stale doctor overlay: $doctor_file"
fi
done
if grep -Fq '# BEGIN DOCTOR OVERLAY' "$MNT/archipelago/auto-install.sh"; then
ok "installer replaces cached doctor before first boot"
else
bad "installer lacks cached doctor replacement"
fi
# ── GRUB must boot the live system ───────────────────────────────────
if grep -q "boot=live" "$MNT/boot/grub/grub.cfg" 2>/dev/null; then
ok "grub.cfg has boot=live"
+6
View File
@@ -40,6 +40,12 @@ class BuildPayloadTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'out-of-payload'):
contexts.check(self.root)
def test_retired_registry_rejected(self):
target = self.root / 'docker/lnd-ui/Dockerfile'
target.write_text('FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine\n')
with self.assertRaisesRegex(ValueError, 'lnd-ui.*retired registry'):
contexts.check(self.root)
def test_empty_payload_rejected(self):
shutil.rmtree(self.root / 'apps')
with self.assertRaisesRegex(ValueError, 'No app manifests'):
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Simulate failures without namespaces, network access, or real repair commands.
set -euo pipefail
source "$(dirname "$0")/../../scripts/container-doctor.sh"
id() { if [[ "$*" == '-u archipelago' ]]; then echo 1000; else echo "${TEST_UID:-0}"; fi; }
pgrep() { if [[ "$HAS_NETWORK" == 1 ]]; then echo 123; else return 1; fi; }
sleep() { :; }
# Any mutation fails the test immediately, including within command substitution.
tripwire() { echo 'FAIL: diagnostic attempted a mutation' >&2; exit 99; }
podman() { tripwire; }
podman_rootless() { tripwire; }
sudo() { tripwire; }
systemctl() { tripwire; }
pkill() { tripwire; }
kill() { tripwire; }
rm() { tripwire; }
mkdir() { tripwire; }
timeout() {
if [[ "$2" == bash ]]; then return "$HOST_STATUS"; fi
[[ "$2" == nsenter ]] || exit 98
PROBES=$((PROBES + 1))
if [[ "$PROBES" == 1 ]]; then return "$FIRST_STATUS"; fi
return "$SECOND_STATUS"
}
check_case() {
local label=$1 expected=$2 expected_probes=$3
PROBES=0
local status=0
check_rootless_netns_egress > /dev/null || status=$?
[[ "$status" == "$expected" && "$PROBES" == "$expected_probes" ]] || {
echo "FAIL: $label status=$status probes=$PROBES"; exit 1;
}
echo "PASS: $label"
}
HAS_NETWORK=1 HOST_STATUS=0 FIRST_STATUS=0 SECOND_STATUS=0
check_case healthy 1 1
TEST_UID=1000 check_case rootless-caller 1 0
HAS_NETWORK=0 check_case no-network 1 0
HOST_STATUS=1 check_case host-offline 2 0
FIRST_STATUS=1 check_case transient-recovery 1 2
FIRST_STATUS=1 SECOND_STATUS=1 check_case repeated-egress-failure 2 2
FIRST_STATUS=126 SECOND_STATUS=126 check_case namespace-access-failure 2 2
# Failure must remain an unresolved warning on every scheduled invocation.
FIRST_STATUS=1 SECOND_STATUS=1
for attempt in 1 2 3 4 5; do
PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
done
[[ "$CHECKS_WARNED" == 5 && "$FIXES_APPLIED" == 0 && "$CHECKS_PASSED" == 0 ]]
FIRST_STATUS=0 PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
[[ "$CHECKS_PASSED" == 1 && "$FIXES_APPLIED" == 0 ]]
echo 'PASS: repeated failure warnings never trigger repair or report a successful check'
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""Execute the installer's actual overlay against a stale disposable rootfs."""
import pathlib, subprocess, tempfile, shutil, unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
class DoctorOverlayTests(unittest.TestCase):
def test_cached_rootfs_and_missing_payload(self):
source = (ROOT / 'image-recipe/_archived/build-auto-installer-iso.sh').read_text()
block = source.split('# BEGIN DOCTOR OVERLAY\n', 1)[1].split('# END DOCTOR OVERLAY', 1)[0]
with tempfile.TemporaryDirectory() as temp:
base = pathlib.Path(temp)
target = base / 'target'
media = base / 'media'
payload = media / 'archipelago/scripts'
payload.mkdir(parents=True)
units = target / 'etc/systemd/system'
units.mkdir(parents=True)
for directory in ['opt/archipelago/scripts', 'home/archipelago/archy/scripts']:
dest = target / directory
dest.mkdir(parents=True)
(dest / 'container-doctor.sh').write_text('UNSAFE OLD SCRIPT')
files = [ROOT / 'scripts/container-doctor.sh',
ROOT / 'image-recipe/configs/archipelago-doctor.service',
ROOT / 'image-recipe/configs/archipelago-doctor.timer']
for path in files:
shutil.copyfile(path, payload / path.name)
script = block.replace('/mnt/target', str(target))
for _ in range(2):
subprocess.run(['bash', '-c', script], env={'BOOT_MEDIA': str(media), 'PATH': '/usr/bin:/bin'}, check=True)
for directory in ['opt/archipelago/scripts', 'home/archipelago/archy/scripts']:
dest = target / directory / 'container-doctor.sh'
self.assertEqual(dest.read_bytes(), files[0].read_bytes())
self.assertEqual(dest.stat().st_mode & 0o777, 0o755)
for path in files[1:]:
self.assertEqual((units / path.name).read_bytes(), path.read_bytes())
(payload / 'container-doctor.sh').unlink()
failed = subprocess.run(['bash', '-c', script], env={'BOOT_MEDIA': str(media), 'PATH': '/usr/bin:/bin'}, capture_output=True)
self.assertNotEqual(failed.returncode, 0, 'Missing safety overlay must fail installation')
if __name__ == '__main__':
unittest.main()
+2
View File
@@ -74,6 +74,8 @@ stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml -
stage "app-build-contexts" python3 tests/regression/app-build-contexts.py
stage "manifest-shell" python3 scripts/check-manifest-shell.py
stage "npm-tunnel-migration" python3 -m unittest discover -s scripts/tests -p test_repair_npm_tunnel.py
stage "iso-doctor-overlay" python3 tests/regression/iso-doctor-overlay.py
stage "doctor-egress" bash tests/regression/container-doctor-egress.sh
stage "doctor-ports" bash tests/regression/container-doctor-ports.sh
stage "bitcoin-pruning" python3 tests/regression/bitcoin-prune-entrypoint.py
stage "lnd-ui-readiness" node --test tests/regression/lnd-ui-readiness.cjs