Compare commits

...
Author SHA1 Message Date
archipelago 0be7aee49d docs: record final 1.8.22 OTA and ISO acceptance 2026-09-30 19:50:01 -04:00
archipelago 6d5f3ffb85 fix: select NPM admin port regardless of binding order
Demo images / Build & push demo images (push) Failing after 34s
2026-09-30 18:24:35 -04:00
archipelago d1bc1273d4 fix: replace cached container doctor before ISO first boot 2026-09-30 17:52:30 -04:00
archipelago 96fb5a4f19 fix: prevent stale snapshots resurrecting orphaned dashboards
Demo images / Build & push demo images (push) Failing after 36s
2026-09-30 17:45:18 -04:00
archipelago f91c1f33db fix: keep apps running when network diagnostics fail 2026-09-30 17:34:11 -04:00
21 changed files with 574 additions and 111 deletions
+6
View File
@@ -2,6 +2,12 @@
## v1.8.22-alpha (2026-09-30)
- Fixed Nginx Proxy Manager launch readiness choosing a proxy listener instead of its admin port after container recreation.
- Network diagnostic failures no longer stop all apps or rebuild shared container networking.
- Prevented orphaned companion dashboards from repeatedly reinstalling themselves after their backend app was removed.
- Fixed companion dashboard builds still referencing a retired image registry.
- Fixed Angor Indexer health checks choosing IPv6 localhost for an IPv4 listener and unnecessarily restarting the working service.
- Prevented false app restarts by probing each published port at its actual bind address; Nginx Proxy Manager now checks its internal admin API.
@@ -103,6 +103,15 @@ pub fn companions_for(package_id: &str) -> &'static [CompanionSpec] {
}
}
/// Missing companion UIs are provisioned here, never by snapshot recovery.
/// A stale running-container snapshot must not resurrect an orphaned UI.
pub fn is_companion_app(app_id: &str) -> bool {
ALL_COMPANIONS
.iter()
.flat_map(|specs| specs.iter())
.any(|spec| spec.image_base == app_id)
}
/// Every companion this build knows how to provision. Kept beside
/// `companions_for` — a new companion must be added to both, or the reaper
/// will not recognise it as one of ours and will leave it running forever.
@@ -978,14 +978,20 @@ fn extract_lan_address(ports: &[String]) -> Option<String> {
let mut first_candidate = None;
for port_str in ports {
// Parse port strings like "0.0.0.0:18443->18443/tcp" or "0.0.0.0:18443-18444->18443-18444/tcp"
let Some(public_part) = port_str.split("->").next() else {
let Some((public_part, _)) = port_str.split_once("->") else {
continue;
};
let Some(port_part) = public_part.split(':').nth(1) else {
let Some((_, port_part)) = public_part.rsplit_once(':') else {
continue;
};
// Extract just the first port if it's a range (e.g., "18443-18444" -> "18443")
let host_port = port_part.split('-').next().unwrap_or(port_part);
let Ok(host_port) = host_port.parse::<u16>() else {
continue;
};
if host_port == 0 {
continue;
}
let candidate = format!("http://localhost:{}", host_port);
if first_candidate.is_none() {
first_candidate = Some(candidate.clone());
@@ -1113,6 +1119,29 @@ fn package_launch_candidate(
if let Some(companion) = companion_lan_address(app_id) {
return Some(companion);
}
if app_id == "nginx-proxy-manager" {
// 80/443 serve users' proxy hosts; only container port 81 serves the
// admin UI. Podman's binding order is unstable across recreation.
// Resolve its actual host allocation rather than guessing the first
// HTTP port or hardcoding the default host port 8081.
let admin_ports: Vec<String> = ports
.iter()
.filter(|port| {
port.split_once("->")
.is_some_and(|(_, target)| target == "81/tcp")
})
.cloned()
.collect();
// With published bindings, a missing admin mapping is not evidence
// that some unrelated service on the default host port is this UI.
return extract_lan_address(&admin_ports).or_else(|| {
if ports.is_empty() {
known
} else {
None
}
});
}
if uses_allocated_launch_port(app_id) {
extract_lan_address(ports).or(known)
} else {
@@ -1241,6 +1270,98 @@ mod extract_lan_address_tests {
);
}
#[test]
fn npm_admin_launch_is_independent_of_proxy_binding_order() {
let mappings = [
"10.77.0.2:18081->80/tcp",
"10.77.0.2:18443->443/tcp",
"127.0.0.1:8081->81/tcp",
];
for order in [
[0, 1, 2],
[0, 2, 1],
[1, 0, 2],
[1, 2, 0],
[2, 0, 1],
[2, 1, 0],
] {
let ports: Vec<String> = order.iter().map(|&i| mappings[i].into()).collect();
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&ports,
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:8081")
);
}
}
#[test]
fn npm_admin_launch_respects_host_allocation_and_ipv6_bindings() {
for binding in ["127.0.0.1", "0.0.0.0", "[::1]", "[::]"] {
let ports = vec![
"10.77.0.2:18081->80/tcp".into(),
format!("{binding}:28081->81/tcp"),
];
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&ports,
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:28081")
);
}
let proxies = vec![
"10.77.0.2:18081->80/tcp".into(),
"10.77.0.2:18443->443/tcp".into(),
];
assert_eq!(
package_launch_candidate("nginx-proxy-manager", &proxies, None),
None
);
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&proxies,
Some("http://localhost:8081/".into())
),
None
);
}
#[test]
fn npm_without_port_information_uses_declared_admin_url() {
assert_eq!(
package_launch_candidate(
"nginx-proxy-manager",
&[],
Some("http://localhost:8081/".into())
)
.as_deref(),
Some("http://localhost:8081/")
);
}
#[test]
fn malformed_published_ports_do_not_become_launch_urls() {
for port in [
"81/tcp",
"127.0.0.1:bad->81/tcp",
"[::1]:0->81/tcp",
"[::]:65536->81/tcp",
"127.0.0.1:8081->81/udp",
] {
assert_eq!(
package_launch_candidate("nginx-proxy-manager", &[port.into()], None),
None
);
}
}
#[test]
fn skips_ssh_port_when_web_port_is_published() {
// gitea: SSH published before the web port, in podman's list order.
@@ -2071,6 +2071,10 @@ impl ProdContainerOrchestrator {
Ok(ReconcileAction::Left(reason))
if mode == ReconcileMode::ExistingOnly
&& reason == "absent"
// companion.rs owns missing UI provisioning/removal.
// Never resurrect an orphan from a stale snapshot.
// Existing UIs still pass through security config repair.
&& !super::companion::is_companion_app(&app_id)
&& (was_running.contains(&compute_container_name(&lm.manifest))
// The durable answer, and the one that does not
// erode. `was_running` only records what was
@@ -7225,6 +7229,52 @@ app:
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
}
#[tokio::test]
async fn reconcile_existing_does_not_resurrect_orphaned_companions() {
let rt = Arc::new(MockRuntime::default());
let mut orch = orch_with(rt.clone()).await;
orch.set_disk_gb_for_test(500);
let companions = [
"bitcoin-ui",
"electrs-ui",
"lnd-ui",
"fedimint-ui",
"cuprate-ui",
];
let mut names = Vec::new();
for id in companions {
let manifest = pull_manifest(id, "localhost/companion:local");
names.push(compute_container_name(&manifest));
orch.insert_manifest_for_test(manifest, PathBuf::from("/tmp/companion"))
.await;
}
let refs: Vec<&str> = names.iter().map(String::as_str).collect();
crate::crash_recovery::save_container_snapshot_for_test(&orch.data_dir, &refs).await;
// Repeated passes must leave lifecycle ownership with companion.rs.
for _ in 0..3 {
let report = orch.reconcile_existing().await;
assert_eq!(report.actions.len(), companions.len());
assert!(report
.actions
.iter()
.all(|(_, action)| *action == ReconcileAction::Left("absent".into())));
assert!(report.failures.is_empty());
}
let calls = rt.calls();
for operation in [
"pull_image:",
"create_container:",
"start_container:",
"stop_container:",
"remove_container:",
] {
assert!(
!calls.iter().any(|call| call.starts_with(operation)),
"{calls:?}"
);
}
}
#[tokio::test]
async fn reconcile_existing_self_heals_missing_optional_installed_app() {
// A non-baseline app (gitea) self-heals ONLY with installation
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY tailwind.css /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/
COPY 50x.html /usr/share/nginx/html/
COPY qrcode.js /usr/share/nginx/html/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
COPY index.html /usr/share/nginx/html/index.html
COPY nginx.conf /etc/nginx/conf.d/default.conf
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Static site content.
COPY index.html /usr/share/nginx/html/
#
+1 -1
View File
@@ -1,4 +1,4 @@
FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine
FROM source.archipelago-foundation.org/lfg2025/nginx:1.27.4-alpine
# Copy the HTML file
COPY index.html /usr/share/nginx/html/
+127 -4
View File
@@ -1,6 +1,10 @@
# Next OTA and raw ISO after 1.8.21
**Status: implementation and acceptance in progress; NOT ready to release.**
**Status: implementation and final OTA/raw ISO acceptance passed; draft upload verification and offline signing/publication remain.**
Current acceptance evidence: [1.8.22 release acceptance](release-1.8.22-acceptance.md).
The chronological notes below retain earlier failures and superseded candidates;
the final tested source is `6d5f3ffb`.
This is the consolidated execution checklist for the operator's chat requests.
A targeted node repair is not completion of the release. Finish the remaining
@@ -45,16 +49,18 @@ completed. See PR review for the accepted scope and coverage limits.
## Final release checklist
- [ ] Finish all new-scope implementation and specific acceptance above.
- [x] Finish new-scope implementation and release acceptance; full-chain Angor
indexing still depends on the dev node finishing initial sync.
- [x] Remove disposable fixtures and temporary test overrides; verify native
Bitcoin/LND identity and start-state baselines remain protected.
- [x] Commit and push completed source changes to git and ngit.
- [ ] Run final backend/UI/regression/release gates on the final source; inspect
- [x] Run final backend/UI/regression/release gates on the final source; inspect
skipped tests and report actual hardware/runtime coverage.
- [ ] Prepare compatible signed app catalog; old runtimes must not apply a
migration before they have backup/recovery support.
- [ ] Version/changelog and OTA payload prepared, validated and signed by user.
- [ ] Raw ISO built; payload hashes/content verified; installer boot tested.
- [x] Raw ISO built; payload hashes/content verified; full installation and
installed-system boot tested in QEMU/KVM without network.
- [ ] User signs ISO checksums; publish OTA and ISO plus verification files on
git and ngit; independently read back hashes and update discovery.
- [ ] Provide LAN scp command for the new raw ISO.
@@ -334,3 +340,120 @@ repository branch and Compose content from its own network namespace.
Version preparation is 1.8.22-alpha. No new release tag or fleet-visible update
manifest is published by the version commit. Optimized candidate deployment,
artifact inspection, ISO smoke/boot checks and offline signatures follow.
### Release blocker discovered during candidate observation: scheduled doctor
The initial `02b840f2` 1.8.22 candidate is rejected for release. On the X250,
2026-09-30 21:10–21:11 UTC, the scheduled `archipelago-doctor.service` explicitly
ran `podman stop --all --time 30`, killed rootless network helpers and ran
`podman system migrate` after a two-attempt external network probe failed.
The journal attributes the stop to that unit, not the app health monitor or a
host reboot. All apps restarted, including Bitcoin, LND and the production site.
The earlier unchanged-container acceptance applies only to immediate deployment;
the later observation failed and must not be represented as a stability pass.
No persistent-data loss has been established. Keep this distinct from the closed
Framework incident; do not wipe or recreate any wallet as a recovery action.
Containment: stopped doctor timers on both test boxes, installed a safe diagnostic
script into both the executable and runtime payload, and rejected/stopped the
old ISO build. Network failure now produces a warning without stopping apps,
killing network processes, migrating Podman or deleting network state. Repeated
failures remain warnings, never a successful repair/check. Regression cases cover
healthy, absent network, non-root invocation, host failure, transient recovery,
repeated endpoint failure and namespace access failure, with mutation tripwires.
Live scheduled-cycle observation and final rebuilt-artifact acceptance are pending.
Recovery also exposed retired `git.tx1138.com` nginx base references in six
companion UI Dockerfiles. They now use the existing primary registry at the same
pinned version. All six images built successfully against that registry; payload
validation rejects the retired host before OTA/ISO packaging.
The post-recovery X250 check passes: Bitcoin authenticated RPC responds and IBD
advances; NPM/Gitea/Portainer APIs respond; Portainer's real namespace fetches
`demo-portainer` at `3ae171d6b0c728665a860520fe393c0abb772798` and its Compose
file; Portainer's original persistent mounts match the earlier backup evidence;
LND wallet/channel databases remain present on their persistent mount. No new
pre-incident cryptographic wallet-identity baseline was available, so these checks
must not be described as an exact identity/balance comparison.
The dev all-container observation also caught a separate Cuprate UI orphan loop:
`companion.rs` removed it because Cuprate was not installed, while generic desired-
state recovery resurrected it from an old running snapshot, using a unit without
nginx's required capabilities. Generic desired-state recovery now excludes missing companions
owned by `companion.rs`; existing companion provisioning/reaping remains the
single owner. Running UIs still receive the existing security configuration repairs. Regression runs repeated reconciliation against stale companion
snapshots and checks that no image/container lifecycle operations occur.
Safe-doctor live acceptance: the X250 completed a 12-minute observation with all
running container IDs, start times and data mounts unchanged. Its journal records
successful doctor runs at 21:22:06, 21:27:51 and 21:33:10 UTC. Both doctor timers
are restored with the safe script. Dev's native Bitcoin/LND stayed running;
all-container dev acceptance remains pending the companion-loop backend fix.
Final-source UI suite: 1,133 passed. Heavy backend compilation is serialized with
remaining build steps to reduce memory/IO pressure on the syncing dev node.
Final source release gates at `96fb5a4f`: 1,613 backend tests passed, zero failed,
four explicitly ignored; 1,133 UI tests passed; type-check, production UI build,
catalog/trust, shell, pruning, LND readiness, NPM migration and doctor regressions
passed. The isolated companion-loop regression passed independently as well.
ISO cache hardening: the installer now carries the current doctor script and
service/timer separately from rootfs.tar and overwrites both historical and active
script locations before first boot. This prevents a cached base image restoring
the old recovery code. A regression executes the actual installer block against
stale disposable files twice and confirms a missing safety payload fails closed.
The mounted-ISO smoke test also compares all three overlay files to source.
The final ISO build captures the exact newly deployed OTA UI/runtime payload.
### Final kiosk acceptance found nondeterministic NPM launch selection
Do not publish the staged `d1bc1273` candidate. NPM itself remains healthy and its
API, Portainer integration, site, and native services passed stability checks.
However, final kiosk acceptance found its card stuck at "Web UI not ready".
The runtime reported bindings in proxy-HTTP, proxy-HTTPS, admin order. The scanner
chose the first non-database/SSH binding, then rejected its tunnel-only host port
as unreachable on loopback, leaving the launch address empty. Earlier tests had
passed with admin first. This is a confirmed order-dependent scanner defect.
The candidate fix explicitly resolves NPM container port 81 to its actual host
allocation. Proxy ports never become the admin URL. Missing/malformed admin
bindings do not fall back to another service when published bindings are present.
Port parsing handles IPv6 authorities and rejects invalid ports. Regressions
cover all six three-port permutations, allocated admin ports, IPv4/IPv6 binding
strings, missing admin mappings, missing runtime port information, and malformed
or UDP bindings. Full backend regression execution is pending for this change.
The ISO build is frozen at installer-environment creation; no release was signed
or published. Rebuild/revalidate the OTA and ISO with this correction.
The companion orphan fix worked live: Cuprate UI was automatically removed and
all installed app container IDs remained unchanged. One observation helper raced
that expected removal between `podman ps` and `inspect`; it now excludes that
known orphan before inspection and repeats the stability check. This was a test
snapshot race, not another installed-app restart.
NPM selector final backend gate passed: 1,617 tests, zero failures, four explicitly
ignored, through the isolated runner. This includes all new port-selection cases
and the existing companion security/configuration and lifecycle regressions.
Rebuild the release binary and UI metadata, deploy those exact OTA bytes to both
boxes, and require actual kiosk hard-refresh/Launch acceptance before ISO assembly.
## Final accepted artifacts — 1.8.22-alpha
Source `6d5f3ffb` passed 1,617 backend tests (four explicit opt-in exclusions),
1,133 frontend tests and final release gates. Exact OTA bytes were deployed to
both boxes. Actual X250 kiosk NPM Launch, version/pruning, desktop/mobile
readiness/AIUI, production Portainer Git/Compose and 12-minute stability checks
on both boxes passed. No installed app was restarted by the safe diagnostics,
and the Cuprate orphan stayed absent. Native Bitcoin/LND and the production site
were preserved during final management deployment.
The raw ISO passed mounted payload checks and matches all 653 OTA frontend/runtime
files plus the backend. Full offline installation and installed UEFI boot to the
visible setup screen passed in a disposable QEMU/KVM VM. Both installed doctor
paths and the installed backend have the expected hashes. No VM wallet was set up.
See `release-1.8.22-acceptance.md` for exact artifact hashes, hardware/runtime
coverage and limits. Draft upload verification, offline signatures, publication
and public readback remain; the fleet still advertises 1.8.21 until those gates
finish. Do not confuse a draft asset or source push with completed publication.
+83
View File
@@ -0,0 +1,83 @@
# Archipelago 1.8.22-alpha acceptance
Source: `6d5f3ffb850bfd3dcd396bac986ba770935d1daa`.
## Verified application and runtime changes
- Full isolated backend suite: 1,617 passed, zero failed, four explicit opt-in exclusions.
- Frontend suite: 1,133 passed. Final frontend and AIUI production builds succeeded.
- Container suite: 79 passed. Catalog compatibility/trust, release manifest, build contexts, pruning, Lightning readiness, NPM migration, safe doctor, companion recovery and ISO doctor-overlay regressions passed.
- Six companion dashboard images built using the current registry.
- Final OTA backend SHA-256: `e108b78bbbd21cb7d5d47c8d0b7b9b19b63fb0c44678773603202440ec7d6f5b`.
- Final OTA frontend SHA-256: `2da485a2da75ff2fbe4aba52d6f217150e303be43a031723480c9c4ff9d43f41`.
## Live acceptance
The exact OTA bytes were deployed to the development box and ThinkPad X250.
Native Bitcoin/LND and the X250 production site retained their container identity
and start time during these final management deployments. Both boxes completed
12-minute observations including scheduled diagnostics with running containers
and persistent mounts unchanged. The orphaned Cuprate dashboard stayed absent.
Actual X250 Chromium kiosk: hard refresh, NPM Launch to the correct admin URL,
visible login/admin page, readable inline Bitcoin version choices and pruning
checkbox passed. No Bitcoin installation was triggered by this test.
Final desktop/mobile checks passed for Bitcoin's IBD dashboard, one Mempool card,
headless Phoenixd, LND waiting/unknown-balance behavior and all five transparent
AIUI embedding layers. Standalone AIUI retains its wallpaper.
Portainer's actual production network namespace fetched Git refs and the Compose
file after final deployment. Original mounts were preserved. Earlier disposable
fresh/reverse-install and migration/rollback tests, and the production host's
operator-initiated reboot check, passed.
Live Tor-only Cashu paid-file acceptance verified a one-satoshi net purchase,
change, rejected-payment refund, exact file bytes, Files access and free repeat
delivery. No native Bitcoin/LND funds were moved. PRs 161/162 are merged and
closed; the open pull-request list is empty.
## Boundaries
- Angor's real dev API, fees, block tip, CORS and rootless/headless configuration
passed. Full-chain indexing remains dependent on initial Bitcoin sync finishing.
- Optional live AI providers, physical RNode hardware, the opt-in Reticulum TCP
subprocess test and creation of a production Minibits profile were not run.
- The previously recorded unsafe-doctor incident changed X250 container start
times before the final fix. Persistent databases were present after recovery,
but no pre-incident cryptographic wallet-identity baseline was available.
Do not describe recovery evidence as an exact pre-incident balance comparison.
- No claim of perfect behavior on every device, network or future failure is made.
## Raw ISO acceptance
The raw ISO is 2,755,072,000 bytes. SHA-256:
`cf7be6378dcd52f6f62774523341fa75dd453fa73a9cadff5390846f483e0140`.
Mounted-artifact smoke checks passed, including BIOS/UEFI boot files, live-boot
hooks, build contexts, current doctor overlay, crash-capture configuration,
version and frontend payload. The ISO backend and all 653 OTA frontend/runtime
files match exactly. AIUI metadata names the tested source commit.
A disposable QEMU/KVM x86_64 VM with UEFI firmware, 3 GiB RAM, two vCPUs, a fresh
64 GiB NVMe virtual disk and no network completed the full installation. This
covered partitioning, LUKS2 data encryption, swap, system configuration, UEFI
bootloader and initramfs generation. Cold boot with the ISO detached reached the
visible Welcome to Archipelago setup screen. The installed backend and both
historical/current doctor paths matched source hashes. Backend/nginx were active;
health reported RPC/sessions ready, crash recovery complete and version 1.8.22.
No wallet was initialized in this disposable VM.
The first automatic VM reboot selected the still-attached installer ISO. That
was corrected in the test configuration by detaching the ISO and explicitly
booting NVMe. It was not accepted as an installed-system boot. The subsequent
cold boot above is the successful acceptance run.
The dev native Bitcoin/LND identity/start-time baseline also remained unchanged
after the ISO build and VM acceptance.
## Publication pending
Artifacts are staged in a draft release. Upload/readback verification and the
operator's offline signatures must complete before promoting the fleet manifest
and signed app catalog or publishing the Git/ngit releases.
@@ -2607,6 +2607,11 @@ if [ -f "$SCRIPT_DIR/../../scripts/image-versions.sh" ]; then
echo " ✅ Bundled image-versions.sh"
fi
# Always overlay the current doctor, including when rootfs.tar is cached.
cp "$SCRIPT_DIR/../../scripts/container-doctor.sh" "$ARCH_DIR/scripts/"
cp "$SCRIPT_DIR/../configs/archipelago-doctor.service" "$ARCH_DIR/scripts/"
cp "$SCRIPT_DIR/../configs/archipelago-doctor.timer" "$ARCH_DIR/scripts/"
# Build-source apps need their complete contexts even on unbundled ISOs.
# Keep this identical to the OTA runtime payload; a per-app allowlist silently
# omitted GitWorkshop, FIPS and Cuprate and made fresh installs fail at 70%.
@@ -3230,6 +3235,18 @@ for test_script in run-e2e-tests.sh run-post-install-tests.sh; do
fi
done
# BEGIN DOCTOR OVERLAY
# Replace both the active and historical script locations before first boot.
# A cached rootfs can contain the unsafe network recovery implementation.
mkdir -p /mnt/target/opt/archipelago/scripts /mnt/target/home/archipelago/archy/scripts
for doctor_dir in /mnt/target/opt/archipelago/scripts /mnt/target/home/archipelago/archy/scripts; do
install -m 755 "$BOOT_MEDIA/archipelago/scripts/container-doctor.sh" "$doctor_dir/container-doctor.sh" || exit 1
done
for doctor_unit in archipelago-doctor.service archipelago-doctor.timer; do
install -m 644 "$BOOT_MEDIA/archipelago/scripts/$doctor_unit" "/mnt/target/etc/systemd/system/$doctor_unit" || exit 1
done
# END DOCTOR OVERLAY
# Copy self-update script
if [ -f "$BOOT_MEDIA/archipelago/scripts/self-update.sh" ]; then
cp "$BOOT_MEDIA/archipelago/scripts/self-update.sh" /mnt/target/opt/archipelago/scripts/
@@ -369,6 +369,10 @@ init()
<span class="text-xs text-white/40">September 30, 2026</span>
</div>
<div class="space-y-3 text-sm text-white/80 pl-3 border-l border-white/10">
<p>Fixed Nginx Proxy Manager launch readiness choosing a proxy listener instead of its admin port after container recreation.</p>
<p>Network diagnostic failures no longer stop all apps or rebuild shared container networking.</p>
<p>Prevented orphaned companion dashboards from repeatedly reinstalling themselves after their backend app was removed.</p>
<p>Fixed companion dashboard builds still referencing a retired image registry.</p>
<p>Fixed Angor Indexer health checks choosing IPv6 localhost for an IPv4 listener and unnecessarily restarting the working service.</p>
<p>Prevented false app restarts by probing each published port at its actual bind address; Nginx Proxy Manager now checks its internal admin API.</p>
<p>Added a backed-up migration for the recognized legacy Nginx Proxy Manager tunnel/LND port conflict in both OTA and ISO startup paths.</p>
+2
View File
@@ -27,6 +27,8 @@ def check(root: Path) -> int:
dockerfile = (context / build.get('dockerfile', 'Dockerfile')).resolve()
if not dockerfile.is_relative_to(context) or not dockerfile.is_file():
raise ValueError(f'{app["id"]}: missing or out-of-context Dockerfile: {dockerfile}')
if 'git.tx1138.com/' in dockerfile.read_text():
raise ValueError(f'{app["id"]}: Dockerfile references retired registry git.tx1138.com')
count += 1
return count
+28 -99
View File
@@ -32,6 +32,8 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
FIXES_APPLIED=0
CHECKS_PASSED=0
CHECKS_WARNED=0
WARNING_NAMES=()
FIX_NAMES=()
log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
@@ -83,7 +85,13 @@ run_fix() {
FIXES_APPLIED=$((FIXES_APPLIED + 1))
FIX_NAMES+=("$name")
else
CHECKS_PASSED=$((CHECKS_PASSED + 1))
local status=$?
if [ "$status" = 1 ]; then
CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
CHECKS_WARNED=$((CHECKS_WARNED + 1))
WARNING_NAMES+=("$name")
fi
fi
}
@@ -446,114 +454,33 @@ print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
[ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1
}
# ── Fix 8: Rootless netns egress lost ────────────────────────
# Rootless podman uses pasta to give containers internet egress. If pasta's
# tap vanishes (host link flap, mount churn, pasta dying during a boot-time
# restart storm), the rootless-netns keeps inter-container traffic working
# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls
# fail. The repair must rebuild the netns from scratch: merely cycling the
# containers reuses the existing (broken) netns because its holders
# (aardvark-dns, podman's pause process) survive — observed on a test node
# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers
# every timer run for ~an hour without ever restoring egress. So: stop the
# containers, kill the netns holders, `podman system migrate`, clear the
# stale netns state, then start everything back up.
#
# Destructive-action latch: cycling the whole fleet is a last resort. After
# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly)
# until a run observes egress healthy again, which resets the counter.
NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures"
NETNS_CYCLE_MAX=3
fix_rootless_netns_egress() {
# Needs root for nsenter. When doctor runs as the rootless container owner,
# a failed nsenter probe is a permissions artifact, not evidence of broken
# egress; do not cycle the fleet from that context.
# ── Check 8: Rootless network egress (diagnostic only) ──────
# A single external endpoint or nsenter failure cannot establish that the
# containers have lost connectivity. In particular, entering only the network
# namespace can fail for rootless user namespaces. Never stop apps, kill network
# helpers, migrate Podman, or remove network state in response to this probe.
# Return 1 for healthy/not applicable and 2 for an inconclusive warning.
check_rootless_netns_egress() {
[ "$(id -u)" = "0" ] || return 1
local archi_uid
local archi_uid aardvark_pid
archi_uid=$(id -u archipelago 2>/dev/null) || return 1
# Locate the rootless-netns via aardvark-dns (it lives inside it).
local aardvark_pid
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1 # no rootless network active
[ -n "$aardvark_pid" ] || return 1
# Host precheck: if the host itself can't reach the internet, no point
# cycling containers — this is an upstream problem.
if ! timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
return 1
log "WARNING: host connectivity probe failed; external endpoint may be unavailable. Apps left running."
return 2
fi
# Probe egress from inside the rootless-netns. One probe is noisy;
# require two consecutive failures 10s apart to rule out transients.
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch
return 1 # first probe succeeded
return 1
fi
sleep 10
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
[ -z "$aardvark_pid" ] && return 1
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
rm -f "$NETNS_CYCLE_STATE"
return 1 # recovered on its own
fi
# Latch: don't keep bouncing the fleet when the rebuild demonstrably
# isn't fixing it.
local failures
failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0)
case "$failures" in *[!0-9]*|"") failures=0;; esac
if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then
log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)"
return 1
fi
log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns"
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman"
local running
running=$($PODMANCMD ps --format '{{.Names}}' 2>/dev/null)
if [ -z "$running" ]; then
log " No running containers to cycle — skipping"
return 1
fi
local count
count=$(echo "$running" | wc -l)
log " Stopping $count running containers (graceful, 30s)..."
$PODMANCMD stop --all --time 30 >/dev/null 2>&1
sleep 5
# Tear the broken netns down for real: kill its holders and drop the
# stale state so the first container start rebuilds pasta + aardvark-dns
# from scratch. Without this, podman re-enters the old netns and the
# missing pasta tap never comes back.
log " Rebuilding rootless netns (killing holders, clearing state)..."
pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null
pkill -U "$archi_uid" -x pasta 2>/dev/null
pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null
pkill -U "$archi_uid" -x slirp4netns 2>/dev/null
sleep 2
$PODMANCMD system migrate >/dev/null 2>&1
rm -rf "/run/user/$archi_uid/containers/networks"
log " Starting containers back up..."
for c in $running; do
$PODMANCMD start "$c" >/dev/null 2>&1 &
done
wait
sleep 5
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
log " Rootless-netns egress restored ($count containers cycled)"
rm -f "$NETNS_CYCLE_STATE"
else
failures=$((failures + 1))
echo "$failures" > "$NETNS_CYCLE_STATE"
log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention"
return 1
fi
return 0
log "WARNING: rootless network probe inconclusive (endpoint, connectivity, or namespace access). Inspect affected apps before repair. Apps left running."
return 2
}
# ── Fix 9: Restart stopped core containers ──────────────────
@@ -731,7 +658,7 @@ run_fix "tor-permissions" fix_tor_permissions
run_fix "searxng" fix_searxng
run_fix "bitcoin-txindex" fix_bitcoin_txindex
run_fix "exit-127" fix_exit_127
run_fix "netns-egress" fix_rootless_netns_egress
run_fix "netns-egress" check_rootless_netns_egress
run_fix "stopped-core" fix_stopped_core_containers
run_fix "rootless-ports" fix_missing_rootless_ports
run_fix "npm-public-hosts" fix_npm_public_hosts
@@ -740,7 +667,9 @@ run_fix "catatonit" fix_missing_catatonit
run_fix "dialout" fix_archipelago_dialout
echo ""
if [ $FIXES_APPLIED -gt 0 ]; then
if [ "$CHECKS_WARNED" -gt 0 ]; then
log "Done: $CHECKS_WARNED unresolved warnings (${WARNING_NAMES[*]}), $FIXES_APPLIED fixes applied, $CHECKS_PASSED checks passed"
elif [ $FIXES_APPLIED -gt 0 ]; then
log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed"
else
log "Done: all $CHECKS_PASSED checks passed — no fixes needed"
+19
View File
@@ -71,6 +71,25 @@ else
bad "incomplete app build payload"
fi
# The cached rootfs must never restore the unsafe historical doctor on boot.
for doctor_file in container-doctor.sh archipelago-doctor.service archipelago-doctor.timer; do
if [[ "$doctor_file" == container-doctor.sh ]]; then
doctor_source="$REPO/scripts/$doctor_file"
else
doctor_source="$REPO/image-recipe/configs/$doctor_file"
fi
if cmp -s "$doctor_source" "$MNT/archipelago/scripts/$doctor_file"; then
ok "current doctor payload: $doctor_file"
else
bad "missing/stale doctor overlay: $doctor_file"
fi
done
if grep -Fq '# BEGIN DOCTOR OVERLAY' "$MNT/archipelago/auto-install.sh"; then
ok "installer replaces cached doctor before first boot"
else
bad "installer lacks cached doctor replacement"
fi
# ── GRUB must boot the live system ───────────────────────────────────
if grep -q "boot=live" "$MNT/boot/grub/grub.cfg" 2>/dev/null; then
ok "grub.cfg has boot=live"
+6
View File
@@ -40,6 +40,12 @@ class BuildPayloadTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'out-of-payload'):
contexts.check(self.root)
def test_retired_registry_rejected(self):
target = self.root / 'docker/lnd-ui/Dockerfile'
target.write_text('FROM git.tx1138.com/lfg2025/nginx:1.27.4-alpine\n')
with self.assertRaisesRegex(ValueError, 'lnd-ui.*retired registry'):
contexts.check(self.root)
def test_empty_payload_rejected(self):
shutil.rmtree(self.root / 'apps')
with self.assertRaisesRegex(ValueError, 'No app manifests'):
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Simulate failures without namespaces, network access, or real repair commands.
set -euo pipefail
source "$(dirname "$0")/../../scripts/container-doctor.sh"
id() { if [[ "$*" == '-u archipelago' ]]; then echo 1000; else echo "${TEST_UID:-0}"; fi; }
pgrep() { if [[ "$HAS_NETWORK" == 1 ]]; then echo 123; else return 1; fi; }
sleep() { :; }
# Any mutation fails the test immediately, including within command substitution.
tripwire() { echo 'FAIL: diagnostic attempted a mutation' >&2; exit 99; }
podman() { tripwire; }
podman_rootless() { tripwire; }
sudo() { tripwire; }
systemctl() { tripwire; }
pkill() { tripwire; }
kill() { tripwire; }
rm() { tripwire; }
mkdir() { tripwire; }
timeout() {
if [[ "$2" == bash ]]; then return "$HOST_STATUS"; fi
[[ "$2" == nsenter ]] || exit 98
PROBES=$((PROBES + 1))
if [[ "$PROBES" == 1 ]]; then return "$FIRST_STATUS"; fi
return "$SECOND_STATUS"
}
check_case() {
local label=$1 expected=$2 expected_probes=$3
PROBES=0
local status=0
check_rootless_netns_egress > /dev/null || status=$?
[[ "$status" == "$expected" && "$PROBES" == "$expected_probes" ]] || {
echo "FAIL: $label status=$status probes=$PROBES"; exit 1;
}
echo "PASS: $label"
}
HAS_NETWORK=1 HOST_STATUS=0 FIRST_STATUS=0 SECOND_STATUS=0
check_case healthy 1 1
TEST_UID=1000 check_case rootless-caller 1 0
HAS_NETWORK=0 check_case no-network 1 0
HOST_STATUS=1 check_case host-offline 2 0
FIRST_STATUS=1 check_case transient-recovery 1 2
FIRST_STATUS=1 SECOND_STATUS=1 check_case repeated-egress-failure 2 2
FIRST_STATUS=126 SECOND_STATUS=126 check_case namespace-access-failure 2 2
# Failure must remain an unresolved warning on every scheduled invocation.
FIRST_STATUS=1 SECOND_STATUS=1
for attempt in 1 2 3 4 5; do
PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
done
[[ "$CHECKS_WARNED" == 5 && "$FIXES_APPLIED" == 0 && "$CHECKS_PASSED" == 0 ]]
FIRST_STATUS=0 PROBES=0
run_fix netns-egress check_rootless_netns_egress > /dev/null
[[ "$CHECKS_PASSED" == 1 && "$FIXES_APPLIED" == 0 ]]
echo 'PASS: repeated failure warnings never trigger repair or report a successful check'
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""Execute the installer's actual overlay against a stale disposable rootfs."""
import pathlib, subprocess, tempfile, shutil, unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
class DoctorOverlayTests(unittest.TestCase):
def test_cached_rootfs_and_missing_payload(self):
source = (ROOT / 'image-recipe/_archived/build-auto-installer-iso.sh').read_text()
block = source.split('# BEGIN DOCTOR OVERLAY\n', 1)[1].split('# END DOCTOR OVERLAY', 1)[0]
with tempfile.TemporaryDirectory() as temp:
base = pathlib.Path(temp)
target = base / 'target'
media = base / 'media'
payload = media / 'archipelago/scripts'
payload.mkdir(parents=True)
units = target / 'etc/systemd/system'
units.mkdir(parents=True)
for directory in ['opt/archipelago/scripts', 'home/archipelago/archy/scripts']:
dest = target / directory
dest.mkdir(parents=True)
(dest / 'container-doctor.sh').write_text('UNSAFE OLD SCRIPT')
files = [ROOT / 'scripts/container-doctor.sh',
ROOT / 'image-recipe/configs/archipelago-doctor.service',
ROOT / 'image-recipe/configs/archipelago-doctor.timer']
for path in files:
shutil.copyfile(path, payload / path.name)
script = block.replace('/mnt/target', str(target))
for _ in range(2):
subprocess.run(['bash', '-c', script], env={'BOOT_MEDIA': str(media), 'PATH': '/usr/bin:/bin'}, check=True)
for directory in ['opt/archipelago/scripts', 'home/archipelago/archy/scripts']:
dest = target / directory / 'container-doctor.sh'
self.assertEqual(dest.read_bytes(), files[0].read_bytes())
self.assertEqual(dest.stat().st_mode & 0o777, 0o755)
for path in files[1:]:
self.assertEqual((units / path.name).read_bytes(), path.read_bytes())
(payload / 'container-doctor.sh').unlink()
failed = subprocess.run(['bash', '-c', script], env={'BOOT_MEDIA': str(media), 'PATH': '/usr/bin:/bin'}, capture_output=True)
self.assertNotEqual(failed.returncode, 0, 'Missing safety overlay must fail installation')
if __name__ == '__main__':
unittest.main()
+2
View File
@@ -74,6 +74,8 @@ stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml -
stage "app-build-contexts" python3 tests/regression/app-build-contexts.py
stage "manifest-shell" python3 scripts/check-manifest-shell.py
stage "npm-tunnel-migration" python3 -m unittest discover -s scripts/tests -p test_repair_npm_tunnel.py
stage "iso-doctor-overlay" python3 tests/regression/iso-doctor-overlay.py
stage "doctor-egress" bash tests/regression/container-doctor-egress.sh
stage "doctor-ports" bash tests/regression/container-doctor-ports.sh
stage "bitcoin-pruning" python3 tests/regression/bitcoin-prune-entrypoint.py
stage "lnd-ui-readiness" node --test tests/regression/lnd-ui-readiness.cjs