archipelagoandClaude Fable 5
c5cd751bcf
setup-node-ca.sh writes one 'listen <addr>:443 ssl;' per LAN address at
the moment it runs (per-address on purpose — Tailscale holds :443 on the
tailnet address) and its idempotency guard never revisits them. nginx
REFUSES TO START while any listen address is missing, so this takes the
entire dashboard down, not just HTTPS:
1. the node moves networks and the old address is gone; or
2. nginx starts before DHCP assigns the address — and nginx.service
ships no Restart=, making that single race permanent.
Both hit archi-dev-box today: nginx dead since boot on 'bind() to
192.168.63.240:443 failed (99: Cannot assign requested address)', the
dashboard simply unreachable, which is exactly the symptom a user with
no screen cannot diagnose.
run_nginx_listener_repair drops listeners for absent addresses, adds one
per present address (CGNAT excluded), installs behind with
rollback, then starts nginx if it is down and gives it a
Restart=on-failure drop-in so the boot race stops being fatal.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1920 lines
85 KiB
Rust
1920 lines
85 KiB
Rust
//! Bootstrap host-side artifacts on every archipelago startup.
|
|
//!
|
|
//! The update pipeline swaps the archipelago binary but does not touch
|
|
//! scripts, systemd units, or nginx configuration — those are installed
|
|
//! once by the ISO builder. Without this module, changes to
|
|
//! `container-doctor.sh`, the doctor service/timer, or the nginx config
|
|
//! never reach boxes installed before the change.
|
|
//!
|
|
//! Two things are synced on startup:
|
|
//! 1. Doctor artifacts (container-doctor.sh + service + timer).
|
|
//! 2. Missing nginx backend proxy blocks required for frontend fetches to
|
|
//! reach the backend instead of the SPA fallback.
|
|
//!
|
|
//! Idempotent: no-ops on boxes that are already in sync. All work is
|
|
//! best-effort — failures are logged but never abort the backend.
|
|
|
|
use anyhow::{Context, Result};
|
|
use std::path::{Path, PathBuf};
|
|
use tokio::fs;
|
|
use tracing::{debug, info, warn};
|
|
|
|
use crate::update::host_sudo;
|
|
|
|
const DOCTOR_SH: &str = include_str!("../../../scripts/container-doctor.sh");
|
|
const DOCTOR_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-doctor.service");
|
|
const DOCTOR_TIMER: &str = include_str!("../../../image-recipe/configs/archipelago-doctor.timer");
|
|
|
|
const DOCTOR_SH_PATH: &str = "/home/archipelago/archy/scripts/container-doctor.sh";
|
|
const DOCTOR_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-doctor.service";
|
|
const DOCTOR_TIMER_PATH: &str = "/etc/systemd/system/archipelago-doctor.timer";
|
|
|
|
// Kiosk hardening (#36): keep the deployed unit + launcher in sync with the
|
|
// repo so the CPU/memory cap and the GPU-vs-headless flag selection reach
|
|
// already-installed nodes via OTA, not just fresh ISOs.
|
|
const KIOSK_SERVICE: &str = include_str!("../../../image-recipe/configs/archipelago-kiosk.service");
|
|
const KIOSK_LAUNCHER: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-kiosk-launcher.sh");
|
|
const KIOSK_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-kiosk.service";
|
|
const KIOSK_LAUNCHER_PATH: &str = "/usr/local/bin/archipelago-kiosk-launcher";
|
|
|
|
// HDMI audio (kiosk nodes): ISOs built before 2026-07-23 shipped no audio
|
|
// stack at all even though the kiosk launcher expects PipeWire-Pulse, and the
|
|
// kiosk's boot-time modeset can race the i915→HDA audio-component bind so the
|
|
// HDMI ELD is lost and every HDMI profile stays unavailable (silent failure).
|
|
// The router daemon handles routing + the ELD re-modeset nudge; this heal
|
|
// installs the packages, group membership, script and unit on deployed nodes.
|
|
const AUDIO_ROUTER: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-audio-router.sh");
|
|
const AUDIO_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-audio-router.service");
|
|
const AUDIO_ROUTER_PATH: &str = "/usr/local/bin/archipelago-audio-router";
|
|
const AUDIO_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-audio-router.service";
|
|
|
|
// Gamepad→keyboard bridge (TV input inside every app iframe) — same
|
|
// splice-from-configs + self-heal pattern as the audio router.
|
|
const GAMEPAD_KEYS: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-gamepad-keys.py");
|
|
const GAMEPAD_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-gamepad-keys.service");
|
|
const GAMEPAD_KEYS_PATH: &str = "/usr/local/bin/archipelago-gamepad-keys";
|
|
const GAMEPAD_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-gamepad-keys.service";
|
|
|
|
// Journald log-volume policy (size cap + per-service rate limit). Fresh ISOs
|
|
// write the identical file at build time (image-recipe/_archived/
|
|
// build-auto-installer-iso.sh); this heals already-deployed nodes via OTA.
|
|
// A fresh node produced >1 GB/day of journal (bitcoind IBD console spam plus
|
|
// debug-level backend logging) — the cap bounds disk use and the rate limit
|
|
// keeps one chatty service from drowning everything else.
|
|
const JOURNALD_DROPIN: &str =
|
|
include_str!("../../../image-recipe/configs/journald-archipelago.conf");
|
|
const JOURNALD_DROPIN_PATH: &str = "/etc/systemd/journald.conf.d/10-archipelago-persistent.conf";
|
|
|
|
const NGINX_CONF_PATH: &str = "/etc/nginx/sites-available/archipelago";
|
|
const NGINX_ENABLED_CONF_PATH: &str = "/etc/nginx/sites-enabled/archipelago";
|
|
/// Per-app proxy snippet included by the HTTPS (:443) server block. Carries its
|
|
/// own `/app/fedimint/` location, so it needs the same B13 asset-rewrite heal as
|
|
/// the main conf — browsers reach fedimint over HTTPS via this snippet. Absent on
|
|
/// HTTP-only nodes, in which case the bootstrap loop skips it.
|
|
const NGINX_HTTPS_SNIPPET_PATH: &str = "/etc/nginx/snippets/archipelago-https-app-proxies.conf";
|
|
const RUNTIME_ASSETS_DIR: &str = "/opt/archipelago/web-ui/archipelago-runtime";
|
|
|
|
/// Inserted into every server block of the nginx config that lacks the
|
|
/// `/api/app-catalog` proxy. Kept in sync with the canonical block in
|
|
/// image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_APP_CATALOG_BLOCK: &str = "\n # App Store catalog proxy — backend fetches from configured registries\n # so the browser doesn't hit CORS/CSP. Without this block nginx falls\n # through to the SPA index.html and the frontend gets HTML back instead\n # of JSON.\n location /api/app-catalog {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header Cookie $http_cookie;\n proxy_connect_timeout 15s;\n proxy_read_timeout 30s;\n proxy_send_timeout 15s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n\n";
|
|
|
|
const NGINX_BITCOIN_STATUS_BLOCK: &str = "\n location /bitcoin-status {\n proxy_pass http://127.0.0.1:5678/bitcoin-status;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_connect_timeout 10s;\n proxy_read_timeout 10s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block that lacks the `/proxy/lnd/` proxy. Nodes
|
|
/// flashed before 2026-04-10 shipped an nginx config without this block, so the
|
|
/// browser's wallet fetches to `/proxy/lnd/*` fell through to the SPA
|
|
/// index.html and got HTML back instead of JSON ("failing to fetch"). Kept in
|
|
/// sync with the canonical block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_LND_PROXY_BLOCK: &str = "\n # LND REST proxy — backend handles auth + CORS\n location /proxy/lnd/ {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Cookie $http_cookie;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_connect_timeout 10s;\n proxy_read_timeout 10s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block lacking the peer-content streaming proxy.
|
|
/// Without it, the browser's `<video>`/`<audio>` Range requests to
|
|
/// `/api/peer-content/*` fall through to the SPA index.html (HTML, no Range)
|
|
/// and peer media won't play (B3). Forwards Cookie (session auth) + Range and
|
|
/// disables buffering so streaming works. Kept in sync with the canonical
|
|
/// block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_PEER_CONTENT_BLOCK: &str = "\n # Peer content streaming proxy (B3) — Range-streams a peer's media file.\n # Long read timeout: this path also serves full-file downloads of large\n # media (#38), which can take minutes over Tor; 120s aborted them.\n location /api/peer-content/ {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Cookie $http_cookie;\n proxy_set_header Range $http_range;\n proxy_buffering off;\n proxy_connect_timeout 10s;\n proxy_read_timeout 900s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block lacking the Pine node-status proxy.
|
|
/// `/api/pine/status` serves the Pine launcher page's live status card and
|
|
/// the seeded Home Assistant REST sensors (the sensitive tier is gated by a
|
|
/// bearer token at the backend, so nginx just forwards). Kept in sync with
|
|
/// the canonical block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_PINE_STATUS_BLOCK: &str = "\n # Pine node status — live node facts for the Pine launcher page and the\n # seeded Home Assistant sensors. Sensitive fields are token-gated at the\n # backend; nginx only forwards.\n location /api/pine/status {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Authorization $http_authorization;\n proxy_set_header Cookie $http_cookie;\n proxy_connect_timeout 10s;\n proxy_read_timeout 15s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// B13 — Fedimint UI asset rewrite. Pre-fix nodes proxy /app/fedimint/ with only
|
|
/// the nostr-provider injection (`sub_filter_once on`), so the UI's root-rooted
|
|
/// CSS/JS asset URLs (href="/…", url("/…")) miss the proxy and load the SPA shell
|
|
/// → unstyled UI. We swap that single sub_filter for the full rewrite set that
|
|
/// reroots every asset URL under /app/fedimint/. NEW matches the canonical block
|
|
/// in image-recipe/configs/nginx-archipelago.conf byte-for-byte so self-healed
|
|
/// nodes converge to the same config fresh ISOs ship with.
|
|
const NGINX_FEDIMINT_OLD: &str = " sub_filter_once on;\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';\n }\n location /app/fedimint-gateway/ {";
|
|
const NGINX_FEDIMINT_NEW: &str = " sub_filter_types text/css application/javascript application/json;\n sub_filter_once off;\n sub_filter 'href=\"/' 'href=\"/app/fedimint/';\n sub_filter 'src=\"/' 'src=\"/app/fedimint/';\n sub_filter \"href='/\" \"href='/app/fedimint/\";\n sub_filter \"src='/\" \"src='/app/fedimint/\";\n sub_filter 'url(\"/' 'url(\"/app/fedimint/';\n sub_filter \"url('/\" \"url('/app/fedimint/\";\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';\n }\n location /app/fedimint-gateway/ {";
|
|
|
|
/// B13 Style B — the HTTPS app-proxy snippet's fedimint block has NO sub_filter
|
|
/// at all (older than the main conf's), and the directive that follows it varies
|
|
/// per node (fedimint-gateway vs tailscale), so a full-block match is unreliable.
|
|
/// Instead we anchor on the unique :8175 proxy_pass (fedimint is the only block
|
|
/// proxying there) and insert the reroot set right after it — directive order
|
|
/// inside a location block is irrelevant to nginx. Idempotent via the same
|
|
/// `href="/app/fedimint/` marker the main-conf heal leaves behind.
|
|
const NGINX_FEDIMINT_SNIPPET_ANCHOR: &str = "proxy_pass http://127.0.0.1:8175/;";
|
|
const NGINX_FEDIMINT_SNIPPET_INSERT: &str = "proxy_pass http://127.0.0.1:8175/;\n proxy_set_header Accept-Encoding \"\";\n sub_filter_types text/css application/javascript application/json;\n sub_filter_once off;\n sub_filter 'href=\"/' 'href=\"/app/fedimint/';\n sub_filter 'src=\"/' 'src=\"/app/fedimint/';\n sub_filter \"href='/\" \"href='/app/fedimint/\";\n sub_filter \"src='/\" \"src='/app/fedimint/\";\n sub_filter 'url(\"/' 'url(\"/app/fedimint/';\n sub_filter \"url('/\" \"url('/app/fedimint/\";\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';";
|
|
|
|
/// Entry point called from main startup. Never returns an error to the caller —
|
|
/// failing to bootstrap host artifacts must not prevent the backend from serving.
|
|
pub async fn ensure_doctor_installed() {
|
|
match run_service_override_repair().await {
|
|
Ok(true) => info!("Removed stale Archipelago dev-mode service override"),
|
|
Ok(false) => debug!("No stale Archipelago dev-mode service override found"),
|
|
Err(e) => warn!("Service override repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_runtime_assets().await {
|
|
Ok(changed) if changed => info!("Runtime assets synchronized from OTA payload"),
|
|
Ok(_) => debug!("No OTA runtime payload to synchronize"),
|
|
Err(e) => warn!("Runtime asset bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run().await {
|
|
Ok(changed) if changed => info!("Doctor artifacts synchronized with binary"),
|
|
Ok(_) => debug!("Doctor artifacts already in sync"),
|
|
Err(e) => warn!("Doctor bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_nginx().await {
|
|
Ok(true) => info!("Patched nginx config to proxy missing backend endpoints"),
|
|
Ok(false) => debug!("Nginx backend endpoint proxy blocks already present"),
|
|
Err(e) => warn!("Nginx bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_bitcoin_rpc_repair().await {
|
|
Ok(true) => {
|
|
info!("Removed stale bitcoin.conf; running Bitcoin containers left untouched")
|
|
}
|
|
Ok(false) => debug!("No stale bitcoin.conf found"),
|
|
Err(e) => warn!("Bitcoin RPC repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_apps_dir_repair().await {
|
|
Ok(true) => {
|
|
info!("Populated /opt/archipelago/apps from installer copy at /etc/archipelago/apps")
|
|
}
|
|
Ok(false) => debug!("/opt/archipelago/apps already populated (or no installer copy)"),
|
|
Err(e) => warn!("Apps dir repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_tor_helper_sync().await {
|
|
Ok(true) => info!("tor-helper.sh synchronized with binary"),
|
|
Ok(false) => debug!("tor-helper.sh already current"),
|
|
Err(e) => warn!("tor-helper sync failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_welcome_banner_sync().await {
|
|
Ok(true) => info!(
|
|
"Console welcome banner synchronized (LAN address + .local name, not the WG tunnel IP)"
|
|
),
|
|
Ok(false) => debug!("Console welcome banner already current (or not an ISO node)"),
|
|
Err(e) => warn!("Welcome banner sync failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_nginx_listener_repair().await {
|
|
Ok(true) => info!("nginx HTTPS listeners retargeted to this host's current addresses"),
|
|
Ok(false) => debug!("nginx listeners already match this host's addresses"),
|
|
Err(e) => warn!("nginx listener repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_ha_rpc_proxy_bind_repair().await {
|
|
Ok(true) => info!(
|
|
"HA bitcoind RPC forwarder rebound dynamically — survives network moves now"
|
|
),
|
|
Ok(false) => debug!("HA bitcoind RPC forwarder absent or already dynamic"),
|
|
Err(e) => warn!("HA RPC forwarder bind repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_pull_never_image_repair().await {
|
|
Ok(n) if n > 0 => info!(retagged = n, "Healed quadlet image refs orphaned by registry rename"),
|
|
Ok(_) => debug!("All quadlet image refs resolve locally"),
|
|
Err(e) => warn!("Quadlet image ref repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_tor_torrc_repair().await {
|
|
Ok(true) => info!("Tor healed at boot (torrc rebuilt and/or daemon restarted)"),
|
|
Ok(false) => debug!("Tor healthy and torrc in sync — no heal needed"),
|
|
Err(e) => warn!("Tor boot heal failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_nginx_mempool_ws_repair().await {
|
|
Ok(true) => info!("nginx mempool websocket headers repaired and reloaded"),
|
|
Ok(false) => debug!("nginx mempool websocket headers already present"),
|
|
Err(e) => warn!("nginx mempool ws repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_polkit_networkmanager_repair().await {
|
|
Ok(true) => info!(
|
|
"Installed NetworkManager polkit rule for the archipelago user — Wi-Fi setup enabled"
|
|
),
|
|
Ok(false) => debug!("NetworkManager polkit rule already present"),
|
|
Err(e) => warn!("polkit NetworkManager repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_journald_dropin().await {
|
|
Ok(true) => info!("Installed journald log-volume policy drop-in"),
|
|
Ok(false) => debug!("journald log-volume policy already in place"),
|
|
Err(e) => warn!("journald drop-in bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match tighten_secrets_dir().await {
|
|
Ok(n) if n > 0 => info!(tightened = n, "Tightened mode on secret files"),
|
|
Ok(_) => debug!("Secrets directory already at expected mode"),
|
|
Err(e) => warn!("Secrets dir tightening failed (non-fatal): {:#}", e),
|
|
}
|
|
// Podman probing MUST be the last bootstrap stage. We used to delete
|
|
// transient runroot state here when `podman info` failed, but live nodes
|
|
// can still have rootlessport/conmon processes holding that state. Removing
|
|
// it automatically makes failures worse: containers lose `.containerenv`,
|
|
// ports stay bound, and later starts fail. Report the fault instead; repair
|
|
// must be deliberate/operator-driven.
|
|
match heal_podman_state().await {
|
|
Ok(PodmanHealOutcome::Healthy) => debug!("podman runtime state healthy"),
|
|
Ok(PodmanHealOutcome::Unhealthy) => warn!(
|
|
"podman runtime state is unhealthy at startup — skipping automatic runroot cleanup"
|
|
),
|
|
Err(e) => warn!(
|
|
"podman self-heal failed (non-fatal, will retry next boot): {:#}",
|
|
e
|
|
),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
enum PodmanHealOutcome {
|
|
Healthy,
|
|
Unhealthy,
|
|
}
|
|
|
|
async fn heal_podman_state() -> Result<PodmanHealOutcome> {
|
|
if probe_podman_ok().await {
|
|
return Ok(PodmanHealOutcome::Healthy);
|
|
}
|
|
Ok(PodmanHealOutcome::Unhealthy)
|
|
}
|
|
|
|
/// True iff `podman info` returns 0 within 5s. Any timeout, spawn
|
|
/// failure, or non-zero exit reads as "wedged" and triggers cleanup.
|
|
async fn probe_podman_ok() -> bool {
|
|
use std::time::Duration;
|
|
let probe = tokio::time::timeout(
|
|
Duration::from_secs(5),
|
|
tokio::process::Command::new("podman")
|
|
.arg("info")
|
|
.arg("--format=json")
|
|
.output(),
|
|
)
|
|
.await;
|
|
match probe {
|
|
Ok(Ok(out)) => out.status.success(),
|
|
Ok(Err(_)) | Err(_) => false,
|
|
}
|
|
}
|
|
|
|
/// Make sure /var/lib/archipelago/secrets/ stays 0700 owned by archipelago,
|
|
/// and every file inside is 0600. The parent dir mode is the load-bearing
|
|
/// boundary against host-side reads from other UIDs (rootless container
|
|
/// escapes get mapped to UID >= 100000 and can't traverse a 0700/uid=1000
|
|
/// directory). The per-file 0600 sweep is defense-in-depth in case some
|
|
/// installer wrote a 0644 file.
|
|
async fn tighten_secrets_dir() -> Result<u32> {
|
|
let dir = Path::new("/var/lib/archipelago/secrets");
|
|
if !dir.exists() {
|
|
return Ok(0);
|
|
}
|
|
use std::os::unix::fs::PermissionsExt;
|
|
fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))
|
|
.await
|
|
.with_context(|| format!("chmod 0700 {}", dir.display()))?;
|
|
|
|
let mut entries = fs::read_dir(dir)
|
|
.await
|
|
.with_context(|| format!("read_dir {}", dir.display()))?;
|
|
let mut tightened = 0u32;
|
|
while let Some(entry) = entries.next_entry().await? {
|
|
let path = entry.path();
|
|
let meta = match entry.metadata().await {
|
|
Ok(m) => m,
|
|
Err(_) => continue,
|
|
};
|
|
if !meta.is_file() {
|
|
continue;
|
|
}
|
|
if meta.permissions().mode() & 0o777 != 0o600 {
|
|
fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))
|
|
.await
|
|
.with_context(|| format!("chmod 0600 {}", path.display()))?;
|
|
tightened += 1;
|
|
}
|
|
}
|
|
Ok(tightened)
|
|
}
|
|
|
|
async fn run_service_override_repair() -> Result<bool> {
|
|
let override_path = Path::new("/etc/systemd/system/archipelago.service.d/override.conf");
|
|
let Ok(content) = fs::read_to_string(override_path).await else {
|
|
return Ok(false);
|
|
};
|
|
if !content.contains("ARCHIPELAGO_DEV_MODE=true") {
|
|
return Ok(false);
|
|
}
|
|
|
|
let only_dev_mode_override = content
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|line| !line.is_empty() && !line.starts_with('#'))
|
|
.all(|line| line == "[Service]" || line == "Environment=ARCHIPELAGO_DEV_MODE=true");
|
|
if !only_dev_mode_override {
|
|
warn!(
|
|
path = %override_path.display(),
|
|
"Archipelago service override contains ARCHIPELAGO_DEV_MODE=true plus other settings; leaving it untouched"
|
|
);
|
|
return Ok(false);
|
|
}
|
|
|
|
let path_s = override_path.to_string_lossy().to_string();
|
|
let status = host_sudo(&["rm", "-f", &path_s])
|
|
.await
|
|
.with_context(|| format!("remove {}", override_path.display()))?;
|
|
if !status.success() {
|
|
anyhow::bail!("remove {} exited with {}", override_path.display(), status);
|
|
}
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
Ok(true)
|
|
}
|
|
|
|
async fn run_runtime_assets() -> Result<bool> {
|
|
// The v1.7.50 OTA bridge puts scripts/apps/docker assets inside the
|
|
// frontend tarball because older binaries only know how to apply the
|
|
// backend binary and frontend archive. Once the new backend starts, it
|
|
// promotes that payload into /opt so app installs use the matching specs.
|
|
let runtime_dir = Path::new(RUNTIME_ASSETS_DIR);
|
|
if !runtime_dir.exists() {
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
for (relative, dest) in [
|
|
("apps", "/opt/archipelago/apps"),
|
|
("scripts", "/opt/archipelago/scripts"),
|
|
("docker", "/opt/archipelago/docker"),
|
|
] {
|
|
let src = runtime_dir.join(relative);
|
|
if src.exists() {
|
|
replace_dir_from_runtime(&src, dest).await?;
|
|
if relative == "scripts" {
|
|
let _ = host_sudo(&[
|
|
"find", dest, "-type", "f", "-name", "*.sh", "-exec", "chmod", "755", "{}", "+",
|
|
])
|
|
.await;
|
|
let image_versions = format!("{}/image-versions.sh", dest);
|
|
if Path::new(&image_versions).exists() {
|
|
let _ =
|
|
host_sudo(&["cp", &image_versions, "/opt/archipelago/image-versions.sh"])
|
|
.await;
|
|
}
|
|
}
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
let configs = runtime_dir.join("image-recipe/configs");
|
|
let nginx_src = configs.join("nginx-archipelago.conf");
|
|
if nginx_src.exists() {
|
|
let src_s = nginx_src.to_string_lossy().to_string();
|
|
let status = host_sudo(&[
|
|
"install",
|
|
"-m",
|
|
"644",
|
|
&src_s,
|
|
"/etc/nginx/sites-available/archipelago",
|
|
])
|
|
.await
|
|
.context("install nginx-archipelago.conf")?;
|
|
if !status.success() {
|
|
anyhow::bail!("install nginx-archipelago.conf exited with {}", status);
|
|
}
|
|
changed = true;
|
|
}
|
|
|
|
// archipelago-host-secrets-audit.service rides this same path (phase 10
|
|
// KEY-02 / audit F-03). Nodes already in the field never received 10-03's
|
|
// ISO-build fix — the first-boot script is installed by the installer, not
|
|
// shipped by OTA — so a node that hit the old fail-open path is still on
|
|
// the SSH host key and TLS private key baked into its published ISO, and
|
|
// will never try again. This unit is how such a node reports itself. It is
|
|
// DETECT ONLY (D-06: detect-report-then-apply); rotation is operator-driven
|
|
// via `--apply --yes` and never fires from a unit.
|
|
let mut host_secrets_unit_installed = false;
|
|
for unit in [
|
|
"archipelago-doctor.service",
|
|
"archipelago-doctor.timer",
|
|
"archipelago-host-secrets-audit.service",
|
|
] {
|
|
let src = configs.join(unit);
|
|
if src.exists() {
|
|
let src_s = src.to_string_lossy().to_string();
|
|
let dest = format!("/etc/systemd/system/{}", unit);
|
|
let status = host_sudo(&["install", "-m", "644", &src_s, &dest])
|
|
.await
|
|
.with_context(|| format!("install {}", unit))?;
|
|
if !status.success() {
|
|
anyhow::bail!("install {} exited with {}", unit, status);
|
|
}
|
|
if unit == "archipelago-host-secrets-audit.service" {
|
|
host_secrets_unit_installed = true;
|
|
}
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
// Packaged radio tools (v1.7.118+): OTA updates only apply the backend
|
|
// binary and frontend tarball, so these PyInstaller binaries ride the
|
|
// runtime payload and get promoted here. Without this, fleet nodes keep
|
|
// a stale archy-reticulum-daemon (which exits on flags it doesn't know —
|
|
// the v1.7.117 --enable-transport rollout killed their mesh sessions)
|
|
// and never receive archy-rnodeconf at all (Flash LoRa: "No such file
|
|
// or directory"). Skipped when byte-identical; a running daemon is
|
|
// unaffected (install replaces the inode) and picks the new binary up
|
|
// on its next spawn.
|
|
for tool in ["archy-reticulum-daemon", "archy-rnodeconf"] {
|
|
let src = runtime_dir.join("radio-tools").join(tool);
|
|
if !src.exists() {
|
|
continue;
|
|
}
|
|
let dest = format!("/usr/local/bin/{}", tool);
|
|
let same = match (fs::read(&src).await, fs::read(&dest).await) {
|
|
(Ok(a), Ok(b)) => a == b,
|
|
_ => false,
|
|
};
|
|
if same {
|
|
continue;
|
|
}
|
|
let src_s = src.to_string_lossy().to_string();
|
|
let status = host_sudo(&["install", "-m", "755", &src_s, &dest])
|
|
.await
|
|
.with_context(|| format!("install {}", tool))?;
|
|
if !status.success() {
|
|
anyhow::bail!("install {} exited with {}", tool, status);
|
|
}
|
|
info!(
|
|
tool,
|
|
"Promoted packaged radio tool from OTA runtime payload"
|
|
);
|
|
changed = true;
|
|
}
|
|
|
|
if changed {
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
if host_secrets_unit_installed {
|
|
// `--now` on purpose: the verdict is the whole deliverable, and
|
|
// waiting for the next reboot to learn whether a node is running
|
|
// fleet-shared key material wastes the OTA that just delivered the
|
|
// means to find out. The unit is Type=oneshot, read-only and exits
|
|
// in milliseconds on a healthy node. Best-effort: a node that
|
|
// cannot enable it still boots, and the next OTA retries.
|
|
match host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-host-secrets-audit.service",
|
|
])
|
|
.await
|
|
{
|
|
Ok(status) if status.success() => {
|
|
info!("Enabled archipelago-host-secrets-audit.service from OTA runtime payload")
|
|
}
|
|
Ok(status) => tracing::warn!(
|
|
"enabling archipelago-host-secrets-audit.service exited with {}",
|
|
status
|
|
),
|
|
Err(e) => tracing::warn!(
|
|
"failed to enable archipelago-host-secrets-audit.service: {}",
|
|
e
|
|
),
|
|
}
|
|
}
|
|
if nginx_src.exists() {
|
|
match host_sudo(&["nginx", "-t"]).await {
|
|
Ok(status) if status.success() => {
|
|
let _ = host_sudo(&["systemctl", "reload", "nginx"]).await;
|
|
}
|
|
Ok(status) => {
|
|
tracing::warn!("nginx config test failed after runtime sync: {}", status);
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!("failed to test nginx config after runtime sync: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn replace_dir_from_runtime(src: &Path, dest: &str) -> Result<()> {
|
|
let tmp = format!("{}.new.{}", dest, chrono::Utc::now().timestamp_millis());
|
|
let src_dot = path_dot(src);
|
|
let mkdir = host_sudo(&["mkdir", "-p", &tmp])
|
|
.await
|
|
.with_context(|| format!("mkdir {}", tmp))?;
|
|
if !mkdir.success() {
|
|
anyhow::bail!("mkdir {} exited with {}", tmp, mkdir);
|
|
}
|
|
let copy = host_sudo(&["cp", "-a", &src_dot, &tmp])
|
|
.await
|
|
.with_context(|| format!("copy runtime {} -> {}", src.display(), tmp))?;
|
|
if !copy.success() {
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
anyhow::bail!("copy runtime {} exited with {}", src.display(), copy);
|
|
}
|
|
let _ = host_sudo(&["mkdir", "-p", dest]).await;
|
|
let cleanup = host_sudo(&[
|
|
"find",
|
|
dest,
|
|
"-mindepth",
|
|
"1",
|
|
"-maxdepth",
|
|
"1",
|
|
"-exec",
|
|
"rm",
|
|
"-rf",
|
|
"{}",
|
|
"+",
|
|
])
|
|
.await
|
|
.with_context(|| format!("clean {}", dest))?;
|
|
if !cleanup.success() {
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
anyhow::bail!("clean {} exited with {}", dest, cleanup);
|
|
}
|
|
let tmp_dot = format!("{}/.", tmp);
|
|
let promote = host_sudo(&["cp", "-a", &tmp_dot, dest])
|
|
.await
|
|
.with_context(|| format!("promote {} -> {}", tmp, dest))?;
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
if !promote.success() {
|
|
anyhow::bail!("promote {} exited with {}", dest, promote);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn path_dot(path: &Path) -> String {
|
|
let mut p = PathBuf::from(path);
|
|
p.push(".");
|
|
p.to_string_lossy().to_string()
|
|
}
|
|
|
|
/// ISO installs before the auto-install.sh path fix copied the app manifests
|
|
/// to /etc/archipelago/apps while the backend loads them from
|
|
/// /opt/archipelago/apps — so fresh nodes had ZERO disk manifests and only
|
|
/// catalog-covered apps could install (netbird "manifests not available",
|
|
/// framework node 2026-07-14). Self-heal: when /opt has no manifests and the
|
|
/// installer copy exists, populate /opt from /etc. Never overwrites existing
|
|
/// /opt manifests (OTA runtime-assets sync owns those afterwards).
|
|
async fn run_apps_dir_repair() -> Result<bool> {
|
|
let script = r#"
|
|
set -eu
|
|
src=/etc/archipelago/apps
|
|
dst=/opt/archipelago/apps
|
|
[ -d "$src" ] || exit 0
|
|
# Only heal when the destination has no manifests at all.
|
|
if [ -d "$dst" ] && [ -n "$(ls -A "$dst" 2>/dev/null)" ]; then exit 0; fi
|
|
ls "$src"/*/manifest.yml >/dev/null 2>&1 || exit 0
|
|
mkdir -p "$dst"
|
|
cp -r "$src"/. "$dst"/
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("populate /opt/archipelago/apps from installer copy")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("Apps dir repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Self-heal Wi-Fi setup on nodes that predate the polkit fix (issue #99).
|
|
///
|
|
/// Archipelago drives NetworkManager from a system-level systemd service
|
|
/// (`User=archipelago`, no logind seat), so the stock NM polkit rule — which
|
|
/// only authorizes `subject.local && subject.active` sessions — denies it, and
|
|
/// "connect to Wi-Fi" fails with "Insufficient privileges". Fresh ISO installs
|
|
/// since 2026-05 ship the rule below, but nodes that reached this build over
|
|
/// OTA never got it (OTA replaces the binary + web UI, not host system config).
|
|
///
|
|
/// Install the scoped rule if it is missing, and best-effort ensure `polkitd`
|
|
/// itself is present (without the daemon the rule is inert). Both are wrapped
|
|
/// so an offline/locked apt or a missing package can never fail startup — the
|
|
/// rule is still written so it takes effect once polkitd arrives (e.g. after an
|
|
/// ISO reflash). Idempotent: keyed on the rule's unique `subject.user` marker.
|
|
async fn run_polkit_networkmanager_repair() -> Result<bool> {
|
|
let script = r#"
|
|
set -u
|
|
RULE=/etc/polkit-1/rules.d/49-archipelago-networkmanager.rules
|
|
MARKER='subject.user == "archipelago"'
|
|
# Rule already installed — nothing to do.
|
|
if grep -qF "$MARKER" "$RULE" 2>/dev/null; then
|
|
exit 0
|
|
fi
|
|
# The rule is inert without the polkit daemon. Older nodes (the ones that hit
|
|
# issue #99) shipped without it. Try to install it, but never let apt failure
|
|
# (offline node, locked dpkg, package unavailable) abort the heal — the rule is
|
|
# written regardless so it activates whenever polkitd lands.
|
|
if [ ! -d /usr/share/polkit-1 ] && ! command -v pkaction >/dev/null 2>&1; then
|
|
timeout 240 apt-get install -y --no-install-recommends polkitd >/dev/null 2>&1 \
|
|
|| timeout 240 sh -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends polkitd >/dev/null 2>&1' \
|
|
|| true
|
|
fi
|
|
mkdir -p /etc/polkit-1/rules.d
|
|
cat > "$RULE" <<'RULEEOF'
|
|
polkit.addRule(function(action, subject) {
|
|
if (subject.user == "archipelago" && action.id.indexOf("org.freedesktop.NetworkManager.") == 0) {
|
|
return polkit.Result.YES;
|
|
}
|
|
});
|
|
RULEEOF
|
|
chmod 644 "$RULE"
|
|
# Pick up the new rule. polkitd re-reads rules.d on reload; restart as a
|
|
# fallback. Non-fatal if the unit name differs or the daemon is absent.
|
|
systemctl reload polkit 2>/dev/null \
|
|
|| systemctl restart polkit 2>/dev/null \
|
|
|| systemctl restart polkit.service 2>/dev/null \
|
|
|| true
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("install NetworkManager polkit rule")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("polkit NetworkManager repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Repair Tor at boot so a torrc-generation fix actually reaches the nodes it
|
|
/// was written for.
|
|
///
|
|
/// `regenerate_torrc` only ran from the Tor RPC handlers and package install,
|
|
/// so a node carrying a bad torrc kept it indefinitely: shipping a fixed binary
|
|
/// changed nothing until somebody happened to install an app or toggle a Tor
|
|
/// setting. On 2026-08-09 two of the three reachable nodes had Tor completely
|
|
/// down from an unbindable `SocksPort <archy-net gateway>` line, one of them
|
|
/// unnoticed, while the dashboard reported "connected".
|
|
///
|
|
/// Non-fatal and conservative: it only restarts Tor when the regenerated torrc
|
|
/// differs from the live one, or Tor is not answering on 9050.
|
|
/// The privileged Tor helper, embedded so the OTA actually delivers it.
|
|
/// scripts/tor-helper.sh previously reached nodes only through ISO builds and
|
|
/// manual deploys — the 2026-08-09 helper fix (reset-failed + truthful result)
|
|
/// would have shipped to nobody. Same include_str! pattern as the doctor.
|
|
const TOR_HELPER_SH: &str = include_str!("../../../scripts/tor-helper.sh");
|
|
const TOR_HELPER_PATH: &str = "/opt/archipelago/scripts/tor-helper.sh";
|
|
|
|
/// Heal socat forwarder units that were generated with the node's LAN IP
|
|
/// baked into `bind=`.
|
|
///
|
|
/// `archy-ha-btc-rpc-proxy.service` (written on-node during the Pine/HA
|
|
/// integration) bound socat to the box's DHCP address at generation time —
|
|
/// pasta containers reach the host via its LAN address, so that was the
|
|
/// address that worked. Move the box to a new network and the address no
|
|
/// longer exists: `bind()` fails and the unit restart-loops forever
|
|
/// (framework-pt after relocating, 2026-08-15: restart counter 2446, and
|
|
/// Home Assistant's bitcoind sensor dead with it).
|
|
///
|
|
/// The rewrite computes the bind address at every service start instead, so
|
|
/// `Restart=always` itself becomes the heal: plug the box into any network
|
|
/// and the next restart binds to the new address.
|
|
const HA_RPC_PROXY_UNIT_PATH: &str = "/etc/systemd/system/archy-ha-btc-rpc-proxy.service";
|
|
|
|
/// Parse `TCP-LISTEN:<port>,bind=<ipv4>` + trailing `TCP:<target>` out of a
|
|
/// socat ExecStart line. Returns (listen_port, target).
|
|
fn parse_socat_static_bind(exec_line: &str) -> Option<(String, String)> {
|
|
let after_listen = exec_line.split("TCP-LISTEN:").nth(1)?;
|
|
let port = after_listen.split(',').next()?.trim();
|
|
if port.is_empty() || !port.chars().all(|c| c.is_ascii_digit()) {
|
|
return None;
|
|
}
|
|
// Only rewrite units pinned to a concrete address; a unit already using
|
|
// a computed bind (or none) needs no heal.
|
|
let bind = after_listen.split("bind=").nth(1)?.split(',').next()?.trim();
|
|
if !bind.chars().all(|c| c.is_ascii_digit() || c == '.') || bind.starts_with("127.") {
|
|
return None;
|
|
}
|
|
let target = exec_line.rsplit(" TCP:").next()?.trim();
|
|
if target.is_empty() || target == exec_line {
|
|
return None;
|
|
}
|
|
Some((port.to_string(), target.to_string()))
|
|
}
|
|
|
|
fn dynamic_bind_execstart(listen_port: &str, target: &str) -> String {
|
|
// `$$` survives systemd's own expansion as a literal `$`, so the command
|
|
// substitution runs in the shell at ExecStart time. If the box has no
|
|
// default route yet, exit non-zero and let Restart=always retry.
|
|
format!(
|
|
"ExecStart=/bin/sh -c 'IP=$$(ip -4 route get 1.1.1.1 | sed -n \"s/.*src \\([0-9.]*\\).*/\\1/p\"); \
|
|
[ -n \"$$IP\" ] || exit 1; \
|
|
exec /usr/bin/socat TCP-LISTEN:{listen_port},bind=$$IP,fork,reuseaddr TCP:{target}'"
|
|
)
|
|
}
|
|
|
|
async fn run_ha_rpc_proxy_bind_repair() -> Result<bool> {
|
|
let unit = match tokio::fs::read_to_string(HA_RPC_PROXY_UNIT_PATH).await {
|
|
Ok(s) => s,
|
|
Err(_) => return Ok(false), // node never grew the forwarder
|
|
};
|
|
let Some(exec_line) = unit.lines().find(|l| l.trim_start().starts_with("ExecStart=")) else {
|
|
return Ok(false);
|
|
};
|
|
let Some((port, target)) = parse_socat_static_bind(exec_line) else {
|
|
return Ok(false); // already dynamic (or not the shape we heal)
|
|
};
|
|
let healed = unit.replace(exec_line, &dynamic_bind_execstart(&port, &target));
|
|
let staged = "/var/lib/archipelago/ha-rpc-proxy.staged";
|
|
if let Some(dir) = Path::new(staged).parent() {
|
|
tokio::fs::create_dir_all(dir).await.ok();
|
|
}
|
|
tokio::fs::write(staged, &healed)
|
|
.await
|
|
.context("stage ha-rpc-proxy unit")?;
|
|
let script = format!(
|
|
"set -eu\ninstall -m 0644 {staged} {dest}\nsystemctl daemon-reload\nsystemctl restart archy-ha-btc-rpc-proxy 2>/dev/null || true\nexit 0\n",
|
|
staged = staged,
|
|
dest = HA_RPC_PROXY_UNIT_PATH
|
|
);
|
|
host_sudo(&["sh", "-lc", &script])
|
|
.await
|
|
.context("install ha-rpc-proxy unit")?;
|
|
Ok(true)
|
|
}
|
|
|
|
/// Re-point `--pull never` quadlets whose image ref no longer matches local
|
|
/// storage.
|
|
///
|
|
/// The catalog signing pass rewrites image refs (bare-IP registry → domain),
|
|
/// so a quadlet regenerated with the new ref points at an image the local
|
|
/// store only holds under the old name. With `--pull never` the app can
|
|
/// never start again on its own — Home Assistant looped 761 restarts on
|
|
/// "image not known" (framework-pt, 2026-08-15) while an identical
|
|
/// `name:tag` sat in storage under the bare-IP ref. If any local image
|
|
/// shares the wanted `name:tag`, retag it; pulling is deliberately NOT
|
|
/// attempted here (offline nodes, metered links — the doctor handles pulls).
|
|
async fn run_pull_never_image_repair() -> Result<usize> {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string());
|
|
let quadlet_dir = format!("{home}/.config/containers/systemd");
|
|
let mut wanted: Vec<String> = Vec::new();
|
|
let mut entries = match tokio::fs::read_dir(&quadlet_dir).await {
|
|
Ok(e) => e,
|
|
Err(_) => return Ok(0),
|
|
};
|
|
while let Ok(Some(entry)) = entries.next_entry().await {
|
|
let path = entry.path();
|
|
if path.extension().and_then(|e| e.to_str()) != Some("container") {
|
|
continue;
|
|
}
|
|
let Ok(text) = tokio::fs::read_to_string(&path).await else {
|
|
continue;
|
|
};
|
|
for line in text.lines() {
|
|
if let Some(image) = line.trim().strip_prefix("Image=") {
|
|
let image = image.trim();
|
|
if !image.is_empty() {
|
|
wanted.push(image.to_string());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if wanted.is_empty() {
|
|
return Ok(0);
|
|
}
|
|
let local = podman_stdout(&["images", "--format", "{{.Repository}}:{{.Tag}}"]).await;
|
|
let local: Vec<&str> = local
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|l| !l.is_empty() && !l.contains("<none>"))
|
|
.collect();
|
|
let mut retagged = 0usize;
|
|
for want in wanted {
|
|
if local.iter().any(|l| *l == want) {
|
|
continue;
|
|
}
|
|
// Same `name:tag`, any registry prefix, is the rename we heal.
|
|
let Some(name_tag) = want.rsplit('/').next() else {
|
|
continue;
|
|
};
|
|
if !name_tag.contains(':') {
|
|
continue;
|
|
}
|
|
let suffix = format!("/{name_tag}");
|
|
let Some(src) = local.iter().find(|l| l.ends_with(&suffix)) else {
|
|
continue;
|
|
};
|
|
let status = tokio::process::Command::new("podman")
|
|
.args(["tag", src, &want])
|
|
.status()
|
|
.await;
|
|
match status {
|
|
Ok(s) if s.success() => {
|
|
info!(from = %src, to = %want, "Retagged image for a --pull never quadlet");
|
|
retagged += 1;
|
|
}
|
|
_ => warn!(from = %src, to = %want, "Image retag failed (non-fatal)"),
|
|
}
|
|
}
|
|
Ok(retagged)
|
|
}
|
|
|
|
async fn podman_stdout(args: &[&str]) -> String {
|
|
match tokio::process::Command::new("podman").args(args).output().await {
|
|
Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
_ => String::new(),
|
|
}
|
|
}
|
|
|
|
/// Keep nginx's per-address HTTPS listeners in step with the addresses the
|
|
/// host actually has, and get nginx running if a boot race killed it.
|
|
///
|
|
/// `scripts/setup-node-ca.sh` writes one `listen <addr>:443 ssl;` per LAN
|
|
/// address at the moment it runs (per-address rather than wildcard on
|
|
/// purpose: Tailscale holds :443 on the tailnet address). Its idempotency
|
|
/// guard then never revisits them. Two ways that takes the WHOLE web UI
|
|
/// down — nginx refuses to start if any listen address is missing, so this
|
|
/// is not merely an HTTPS outage:
|
|
/// 1. The node moves networks and the old address no longer exists.
|
|
/// 2. Even in place, nginx starts before DHCP has assigned the address —
|
|
/// and nginx.service ships no `Restart=`, so that single failure is
|
|
/// permanent until a human intervenes.
|
|
/// Both observed on archi-dev-box, 2026-08-15: nginx dead since boot with
|
|
/// `bind() to 192.168.63.240:443 failed (99: Cannot assign requested
|
|
/// address)`, and the dashboard simply unreachable.
|
|
const NGINX_SITES: [&str; 2] = [
|
|
"/etc/nginx/sites-available/archipelago-http",
|
|
"/etc/nginx/sites-available/archipelago",
|
|
];
|
|
const NGINX_RESTART_DROPIN: &str = "/etc/systemd/system/nginx.service.d/10-archipelago-restart.conf";
|
|
|
|
/// Global IPv4 addresses on this host, minus Tailscale CGNAT (100.64/10) —
|
|
/// the same exclusion `setup-node-ca.sh` applies, for the same reason.
|
|
async fn host_lan_addrs() -> Vec<String> {
|
|
let out = tokio::process::Command::new("ip")
|
|
.args(["-o", "-4", "addr", "show", "scope", "global"])
|
|
.output()
|
|
.await;
|
|
let Ok(out) = out else { return Vec::new() };
|
|
String::from_utf8_lossy(&out.stdout)
|
|
.lines()
|
|
.filter_map(|l| l.split_whitespace().nth(3))
|
|
.filter_map(|cidr| cidr.split('/').next())
|
|
.filter(|a| !is_cgnat(a))
|
|
.map(str::to_string)
|
|
.collect()
|
|
}
|
|
|
|
fn is_cgnat(addr: &str) -> bool {
|
|
let mut parts = addr.split('.');
|
|
let (Some(100), Some(second)) = (
|
|
parts.next().and_then(|p| p.parse::<u8>().ok()),
|
|
parts.next().and_then(|p| p.parse::<u8>().ok()),
|
|
) else {
|
|
return false;
|
|
};
|
|
(64..=127).contains(&second)
|
|
}
|
|
|
|
/// Rewrite the `listen <ip>:443 ssl;` set for one config's text. Returns the
|
|
/// new text when it differs. Lines for absent addresses are dropped and one
|
|
/// line per present address is kept, preserving the file's indentation.
|
|
fn retarget_https_listeners(text: &str, present: &[String]) -> Option<String> {
|
|
let listen_of = |l: &str| -> Option<String> {
|
|
let t = l.trim();
|
|
let rest = t.strip_prefix("listen ")?.strip_suffix(":443 ssl;")?;
|
|
// Only per-address listeners; `listen 443 ssl ...` has no address.
|
|
rest.split('.').count().eq(&4).then(|| rest.to_string())
|
|
};
|
|
if !text.lines().any(|l| listen_of(l).is_some()) {
|
|
return None; // wildcard-only config; nothing address-pinned to heal
|
|
}
|
|
let stale: Vec<String> = text
|
|
.lines()
|
|
.filter_map(listen_of)
|
|
.filter(|a| !present.contains(a))
|
|
.collect();
|
|
let existing: Vec<String> = text.lines().filter_map(listen_of).collect();
|
|
let missing: Vec<&String> = present.iter().filter(|a| !existing.contains(a)).collect();
|
|
if stale.is_empty() && missing.is_empty() {
|
|
return None;
|
|
}
|
|
let indent = text
|
|
.lines()
|
|
.find(|l| listen_of(l).is_some())
|
|
.map(|l| l[..l.len() - l.trim_start().len()].to_string())
|
|
.unwrap_or_else(|| " ".to_string());
|
|
let mut out: Vec<String> = Vec::new();
|
|
let mut wrote_block = false;
|
|
for line in text.lines() {
|
|
match listen_of(line) {
|
|
Some(_) if !wrote_block => {
|
|
wrote_block = true;
|
|
for a in present {
|
|
out.push(format!("{indent}listen {a}:443 ssl;"));
|
|
}
|
|
}
|
|
Some(_) => {} // subsequent old listen lines are replaced by the block
|
|
None => out.push(line.to_string()),
|
|
}
|
|
}
|
|
Some(out.join("\n"))
|
|
}
|
|
|
|
async fn run_nginx_listener_repair() -> Result<bool> {
|
|
let present = host_lan_addrs().await;
|
|
if present.is_empty() {
|
|
return Ok(false); // no network yet; a later boot pass will do it
|
|
}
|
|
let mut changed = false;
|
|
for site in NGINX_SITES {
|
|
let Ok(text) = tokio::fs::read_to_string(site).await else {
|
|
continue;
|
|
};
|
|
let Some(healed) = retarget_https_listeners(&text, &present) else {
|
|
continue;
|
|
};
|
|
let staged = "/var/lib/archipelago/nginx-listeners.staged";
|
|
if let Some(dir) = Path::new(staged).parent() {
|
|
tokio::fs::create_dir_all(dir).await.ok();
|
|
}
|
|
tokio::fs::write(staged, &healed)
|
|
.await
|
|
.context("stage nginx listeners")?;
|
|
// Install behind `nginx -t`, and roll back if the test fails — a bad
|
|
// config here would take the dashboard down, which is the very
|
|
// failure this repair exists to prevent.
|
|
let script = format!(
|
|
"set -eu\ncp {site} {site}.bak-listeners\ninstall -m 0644 {staged} {site}\n\
|
|
if ! nginx -t 2>/dev/null; then cp {site}.bak-listeners {site}; exit 3; fi\nexit 0\n"
|
|
);
|
|
let status = host_sudo(&["sh", "-lc", &script]).await?;
|
|
match status.code() {
|
|
Some(0) => changed = true,
|
|
Some(3) => warn!(site, "nginx listener repair failed its config test — rolled back"),
|
|
_ => warn!(site, "nginx listener repair helper failed"),
|
|
}
|
|
}
|
|
// Whether or not the config changed: if nginx is down (the boot race, or
|
|
// it died on an address that has since arrived), start it. And give it a
|
|
// restart policy so the race stops being fatal in the first place.
|
|
let script = format!(
|
|
"set -eu\nmkdir -p $(dirname {dropin})\n\
|
|
cat > {dropin} <<'EOF'\n[Service]\nRestart=on-failure\nRestartSec=5\n\
|
|
[Unit]\nStartLimitIntervalSec=300\nStartLimitBurst=10\nEOF\n\
|
|
systemctl daemon-reload\n\
|
|
if ! systemctl is-active --quiet nginx; then systemctl reset-failed nginx 2>/dev/null || true; systemctl start nginx 2>/dev/null || true; \
|
|
elif [ \"${{RELOAD:-1}}\" = 1 ]; then systemctl reload nginx 2>/dev/null || true; fi\nexit 0\n",
|
|
dropin = NGINX_RESTART_DROPIN
|
|
);
|
|
host_sudo(&["sh", "-lc", &script])
|
|
.await
|
|
.context("nginx restart policy + start")?;
|
|
Ok(changed)
|
|
}
|
|
|
|
/// The console welcome banner, embedded so the OTA can fix it on deployed
|
|
/// nodes. `/etc/profile.d/archipelago.sh` is baked by the ISO installer and
|
|
/// no OTA path touched it, so every node kept whatever its ISO generation
|
|
/// shipped — including banners that print the node's own WireGuard address
|
|
/// (10.44.0.1, present on EVERY node) as the "web ui", which is unreachable
|
|
/// off-tunnel and actively misleading after a move to a new network
|
|
/// (framework-pt, 2026-08-15). Canonical copy: scripts/welcome-banner.sh;
|
|
/// the ISO builder inlines the same content for fresh installs.
|
|
const WELCOME_BANNER_SH: &str = include_str!("../../../scripts/welcome-banner.sh");
|
|
const WELCOME_BANNER_PATH: &str = "/etc/profile.d/archipelago.sh";
|
|
|
|
async fn run_welcome_banner_sync() -> Result<bool> {
|
|
let current = tokio::fs::read_to_string(WELCOME_BANNER_PATH)
|
|
.await
|
|
.unwrap_or_default();
|
|
// Only refresh a banner the installer put there: a dev machine running
|
|
// the backend from a checkout has no business growing one in /etc.
|
|
if current.is_empty() || current == WELCOME_BANNER_SH {
|
|
return Ok(false);
|
|
}
|
|
let staged = "/var/lib/archipelago/welcome-banner.staged";
|
|
if let Some(dir) = Path::new(staged).parent() {
|
|
tokio::fs::create_dir_all(dir).await.ok();
|
|
}
|
|
tokio::fs::write(staged, WELCOME_BANNER_SH)
|
|
.await
|
|
.context("stage welcome banner")?;
|
|
let script = format!(
|
|
"set -eu\ninstall -m 0755 {staged} {dest}\nexit 0\n",
|
|
staged = staged,
|
|
dest = WELCOME_BANNER_PATH
|
|
);
|
|
host_sudo(&["sh", "-lc", &script])
|
|
.await
|
|
.context("install welcome banner")?;
|
|
Ok(true)
|
|
}
|
|
|
|
async fn run_tor_helper_sync() -> Result<bool> {
|
|
let current = tokio::fs::read_to_string(TOR_HELPER_PATH)
|
|
.await
|
|
.unwrap_or_default();
|
|
if current == TOR_HELPER_SH {
|
|
return Ok(false);
|
|
}
|
|
let staged = "/var/lib/archipelago/tor-config/tor-helper.staged";
|
|
if let Some(dir) = Path::new(staged).parent() {
|
|
tokio::fs::create_dir_all(dir).await.ok();
|
|
}
|
|
tokio::fs::write(staged, TOR_HELPER_SH)
|
|
.await
|
|
.context("stage tor-helper.sh")?;
|
|
let script = format!(
|
|
"set -eu\ninstall -m 0755 {staged} {dest}\nexit 0\n",
|
|
staged = staged,
|
|
dest = TOR_HELPER_PATH
|
|
);
|
|
host_sudo(&["sh", "-lc", &script])
|
|
.await
|
|
.context("install tor-helper.sh")?;
|
|
Ok(true)
|
|
}
|
|
|
|
/// Existing nodes' nginx configs never receive repo snippet fixes — the OTA
|
|
/// updates the binary and web assets, not /etc/nginx. The mempool UI is
|
|
/// websocket-driven, and every fleet node's /app/mempool/ proxy block strips
|
|
/// the Upgrade handshake, so the page loads and never connects (three-layer
|
|
/// outage, 2026-08-09). Idempotently add the two headers to any mempool block
|
|
/// missing them, in every nginx file that has one, then reload once.
|
|
async fn run_nginx_mempool_ws_repair() -> Result<bool> {
|
|
let script = r#"
|
|
set -eu
|
|
changed=0
|
|
for f in /etc/nginx/sites-available/archipelago-http \
|
|
/etc/nginx/sites-available/archipelago \
|
|
/etc/nginx/snippets/archipelago-https-app-proxies.conf; do
|
|
[ -f "$f" ] || continue
|
|
grep -q 'location /app/mempool/' "$f" || continue
|
|
python3 - "$f" <<'PYEOF'
|
|
import re, sys
|
|
p = sys.argv[1]
|
|
src = open(p).read()
|
|
def fix(m):
|
|
b = m.group(0)
|
|
if 'Upgrade $http_upgrade' in b:
|
|
return b
|
|
return b.replace('proxy_http_version 1.1;',
|
|
'proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection "upgrade";', 1)
|
|
new = re.sub(r'location /app/mempool/ \{[^}]*\}', fix, src, flags=re.S)
|
|
if new != src:
|
|
open(p, 'w').write(new)
|
|
sys.exit(3)
|
|
PYEOF
|
|
rc=$?
|
|
[ "$rc" = 3 ] && changed=1
|
|
[ "$rc" = 0 ] || [ "$rc" = 3 ] || exit "$rc"
|
|
done
|
|
if [ "$changed" = 1 ]; then
|
|
nginx -t >/dev/null 2>&1 && systemctl reload nginx || true
|
|
exit 3
|
|
fi
|
|
exit 0
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("nginx mempool ws repair")?;
|
|
Ok(status.code() == Some(3))
|
|
}
|
|
|
|
async fn run_tor_torrc_repair() -> Result<bool> {
|
|
// Same location the RPC handlers use (Config::data_dir); bootstrap runs
|
|
// before the server owns a Config, and this path is fixed on real installs.
|
|
let data_dir = Path::new("/var/lib/archipelago");
|
|
if !data_dir.exists() {
|
|
debug!("No {} — skipping Tor boot heal", data_dir.display());
|
|
return Ok(false);
|
|
}
|
|
if !Path::new("/etc/tor/torrc").exists() {
|
|
debug!("No /etc/tor/torrc — Tor not installed here, skipping boot heal");
|
|
return Ok(false);
|
|
}
|
|
crate::api::rpc::tor::heal_on_boot(data_dir).await
|
|
}
|
|
|
|
async fn run_bitcoin_rpc_repair() -> Result<bool> {
|
|
// bitcoind is launched with -conf=/tmp/rpc.conf and never reads a
|
|
// datadir bitcoin.conf (apps/bitcoin-core & bitcoin-knots manifest.yml,
|
|
// commit a597c1d9 — bind/port live only on the container command line).
|
|
// A leftover file from an older install makes Bitcoin Core's own
|
|
// datadir-conflict safety check refuse to start on every subsequent
|
|
// start. Remove it instead of "repairing" it into existence — this
|
|
// previously wrote server=/rpcbind=/rpcallowip=/listen= into the file,
|
|
// which is exactly what caused the conflict.
|
|
let script = r#"
|
|
set -eu
|
|
conf=/var/lib/archipelago/bitcoin/bitcoin.conf
|
|
[ -f "$conf" ] || exit 0
|
|
mv "$conf" "$conf.disabled-$(date +%s)"
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("remove stale bitcoin.conf RPC bind settings")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
// Do not restart Bitcoin from bootstrap. During IBD, an automatic
|
|
// restart can cost hours of progress. Removing the stale file is
|
|
// only a fallback for future starts; current containers keep their
|
|
// command-line RPC args regardless.
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("Bitcoin RPC repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Install the journald log-volume policy drop-in (JOURNALD_DROPIN) so nodes
|
|
/// deployed before the ISO shipped it get the size cap + rate limit via OTA.
|
|
/// Idempotent; restarts journald only when the file actually changed (safe:
|
|
/// the sockets are held by pid1, so at most a few messages queue briefly).
|
|
async fn run_journald_dropin() -> Result<bool> {
|
|
// Same dev-box guards as the doctor bootstrap: never touch /etc on
|
|
// contributors' laptops (symlinked or absent /home/archipelago/archy).
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
debug!("/home/archipelago/archy is a symlink — skipping journald bootstrap (dev box)");
|
|
return Ok(false);
|
|
}
|
|
if fs::metadata(home_archy).await.is_err() {
|
|
debug!("/home/archipelago/archy missing — skipping journald bootstrap");
|
|
return Ok(false);
|
|
}
|
|
|
|
let dropin_dir = "/etc/systemd/journald.conf.d";
|
|
let status = host_sudo(&["mkdir", "-p", dropin_dir])
|
|
.await
|
|
.with_context(|| format!("mkdir {}", dropin_dir))?;
|
|
if !status.success() {
|
|
anyhow::bail!("mkdir {} exited with {}", dropin_dir, status);
|
|
}
|
|
|
|
let changed = write_root_if_needed(JOURNALD_DROPIN_PATH, JOURNALD_DROPIN).await?;
|
|
if changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "restart", "systemd-journald"]).await {
|
|
warn!("journald restart after drop-in update failed: {:#}", e);
|
|
}
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn run() -> Result<bool> {
|
|
// Dev-box guard: on contributors' laptops `/home/archipelago/archy` is
|
|
// typically a symlink into the git checkout, and writing through it
|
|
// would clobber the working tree with whatever the binary happens to
|
|
// have been compiled from. Production ISO installs materialize a real
|
|
// directory.
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
debug!("/home/archipelago/archy is a symlink — skipping doctor bootstrap (dev box)");
|
|
return Ok(false);
|
|
}
|
|
|
|
// Skip entirely on machines without the canonical scripts directory —
|
|
// writing orphan files there just causes confusion.
|
|
let scripts_dir = Path::new(DOCTOR_SH_PATH)
|
|
.parent()
|
|
.context("doctor script path has no parent")?;
|
|
if !scripts_dir.exists() {
|
|
debug!(
|
|
"Scripts dir {} missing — skipping doctor bootstrap",
|
|
scripts_dir.display()
|
|
);
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
|
|
// 1. Script — lives in archipelago's home dir, user-writable.
|
|
if needs_write(DOCTOR_SH_PATH, DOCTOR_SH).await {
|
|
fs::write(DOCTOR_SH_PATH, DOCTOR_SH)
|
|
.await
|
|
.with_context(|| format!("write {}", DOCTOR_SH_PATH))?;
|
|
let _ = tokio::process::Command::new("chmod")
|
|
.args(["+x", DOCTOR_SH_PATH])
|
|
.status()
|
|
.await;
|
|
info!("Updated {}", DOCTOR_SH_PATH);
|
|
changed = true;
|
|
}
|
|
|
|
// 2. Systemd unit files — /etc is restricted; route through host_sudo.
|
|
let service_changed = write_root_if_needed(DOCTOR_SERVICE_PATH, DOCTOR_SERVICE).await?;
|
|
let timer_changed = write_root_if_needed(DOCTOR_TIMER_PATH, DOCTOR_TIMER).await?;
|
|
changed = changed || service_changed || timer_changed;
|
|
|
|
// 3. Reload if units changed. Do not enable/start the timer here: lifecycle
|
|
// qualification and explicit app operations need deterministic Podman
|
|
// ownership, and the doctor can race those flows. Operators can enable it
|
|
// separately when they want periodic host repair.
|
|
if service_changed || timer_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn needs_write(path: &str, expected: &str) -> bool {
|
|
match fs::read_to_string(path).await {
|
|
Ok(current) => current != expected,
|
|
Err(_) => true,
|
|
}
|
|
}
|
|
|
|
/// Write content to a root-owned path via `sudo mv` of a user-owned tmp file.
|
|
/// Returns true if a write happened.
|
|
async fn write_root_if_needed(path: &str, content: &str) -> Result<bool> {
|
|
if !needs_write(path, content).await {
|
|
return Ok(false);
|
|
}
|
|
let tmp = format!(
|
|
"/tmp/archipelago-bootstrap-{}-{}.tmp",
|
|
std::process::id(),
|
|
Path::new(path)
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.unwrap_or("unit")
|
|
);
|
|
fs::write(&tmp, content)
|
|
.await
|
|
.with_context(|| format!("write tmp {}", tmp))?;
|
|
let status = host_sudo(&["mv", &tmp, path])
|
|
.await
|
|
.with_context(|| format!("sudo mv {} -> {}", tmp, path))?;
|
|
if !status.success() {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
anyhow::bail!("sudo mv to {} exited with {}", path, status);
|
|
}
|
|
info!("Updated {}", path);
|
|
Ok(true)
|
|
}
|
|
|
|
const ARCHIPELAGO_SERVICE_PATH: &str = "/etc/systemd/system/archipelago.service";
|
|
const MOUNT_REQUIRE_LINE: &str = "RequiresMountsFor=/var/lib/archipelago";
|
|
|
|
/// B17 self-heal: ensure the installed archipelago.service waits for the data
|
|
/// volume to mount before it starts. On production nodes `/var/lib/archipelago`
|
|
/// (the app data dir AND podman's graphroot) is a separate device-mapper volume;
|
|
/// without a mount dependency the service can start before `var-lib-archipelago.mount`,
|
|
/// write to the bare mountpoint on rootfs, fail every podman call, exit, and be
|
|
/// restarted every 5s until the volume mounts (~5 min of "[FAILED] Failed to start"
|
|
/// on cold boots). Fresh ISOs already ship the directive; this heals already-deployed
|
|
/// nodes. The change is boot-ordering only — it takes effect on the NEXT reboot, so we
|
|
/// never restart the running service here. Idempotent; no-op if the unit is absent
|
|
/// (dev runs) or already patched. Harmless when the data dir is on rootfs (systemd maps
|
|
/// the requirement to the always-mounted root).
|
|
pub async fn ensure_archipelago_mount_ordering() {
|
|
let current = match fs::read_to_string(ARCHIPELAGO_SERVICE_PATH).await {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
tracing::debug!(
|
|
"mount-ordering self-heal: {} not readable ({}) — skipping",
|
|
ARCHIPELAGO_SERVICE_PATH,
|
|
e
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
if current.contains(MOUNT_REQUIRE_LINE) {
|
|
return; // already healed
|
|
}
|
|
// Insert the directive into the [Unit] section, immediately before [Service].
|
|
let Some(idx) = current.find("\n[Service]") else {
|
|
tracing::warn!(
|
|
"mount-ordering self-heal: no [Service] section in {} — skipping",
|
|
ARCHIPELAGO_SERVICE_PATH
|
|
);
|
|
return;
|
|
};
|
|
let mut patched = String::with_capacity(current.len() + MOUNT_REQUIRE_LINE.len() + 96);
|
|
patched.push_str(¤t[..idx]);
|
|
patched.push_str("\n# B17: start only after the data volume (+ podman graphroot) is mounted\n");
|
|
patched.push_str(MOUNT_REQUIRE_LINE);
|
|
patched.push_str(¤t[idx..]);
|
|
match write_root_if_needed(ARCHIPELAGO_SERVICE_PATH, &patched).await {
|
|
Ok(true) => {
|
|
info!(
|
|
"B17: added '{}' to archipelago.service (effective next reboot)",
|
|
MOUNT_REQUIRE_LINE
|
|
);
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
tracing::warn!("B17 self-heal: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
Ok(false) => {}
|
|
Err(e) => tracing::warn!("B17 mount-ordering self-heal failed: {:#}", e),
|
|
}
|
|
}
|
|
|
|
/// #36 self-heal: keep the kiosk unit + launcher current on already-deployed
|
|
/// nodes so the CPU/memory cap (a runaway chromium was saturating the node and
|
|
/// starving the backend) and the GPU-vs-headless flag selection arrive via OTA.
|
|
/// No-op on nodes without the kiosk installed; only restarts the kiosk if it's
|
|
/// actually running (so it never re-enables an operator-disabled kiosk).
|
|
pub async fn ensure_kiosk_hardened() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return; // kiosk not installed on this node
|
|
}
|
|
let svc_changed = write_root_if_needed(KIOSK_SERVICE_PATH, KIOSK_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
let launcher_changed = write_root_if_needed(KIOSK_LAUNCHER_PATH, KIOSK_LAUNCHER)
|
|
.await
|
|
.unwrap_or(false);
|
|
if launcher_changed {
|
|
let _ = host_sudo(&["chmod", "+x", KIOSK_LAUNCHER_PATH]).await;
|
|
}
|
|
if svc_changed || launcher_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("kiosk hardening: daemon-reload failed: {:#}", e);
|
|
}
|
|
// try-restart only restarts a currently-active unit — leaves a stopped/
|
|
// disabled kiosk alone.
|
|
let _ = host_sudo(&["systemctl", "try-restart", "archipelago-kiosk.service"]).await;
|
|
info!("kiosk: applied resource cap + GPU-flag hardening (#36)");
|
|
}
|
|
}
|
|
|
|
/// HDMI-audio self-heal for kiosk nodes: install the PipeWire stack (older
|
|
/// ISOs shipped none), put the archipelago user in `audio` (PipeWire runs
|
|
/// under the lingering user manager — no logind seat, so no udev ACLs on
|
|
/// /dev/snd), and keep the audio-router daemon (HDMI routing + ELD boot-race
|
|
/// nudge) installed and current. No-op on nodes without the kiosk — audio
|
|
/// only matters where media plays on an attached display.
|
|
pub async fn ensure_audio_stack() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return; // no kiosk → no display audio to route
|
|
}
|
|
|
|
// Package install runs via systemd-run (host_sudo), outside the service
|
|
// sandbox — /usr and the dpkg database are read-only in our namespace.
|
|
if fs::metadata("/usr/bin/pactl").await.is_err() {
|
|
info!("audio: PipeWire stack missing — installing packages");
|
|
let _ = host_sudo(&["apt-get", "update", "-qq"]).await;
|
|
match host_sudo(&[
|
|
"apt-get",
|
|
"install",
|
|
"-y",
|
|
"-qq",
|
|
"--no-install-recommends",
|
|
"pipewire",
|
|
"pipewire-pulse",
|
|
"pipewire-alsa",
|
|
"wireplumber",
|
|
"alsa-utils",
|
|
])
|
|
.await
|
|
{
|
|
Ok(s) if s.success() => info!("audio: PipeWire stack installed"),
|
|
Ok(s) => {
|
|
warn!(
|
|
"audio: package install exited with {} — will retry next start",
|
|
s
|
|
);
|
|
return;
|
|
}
|
|
Err(e) => {
|
|
warn!(
|
|
"audio: package install failed: {:#} — will retry next start",
|
|
e
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
let _ = host_sudo(&["usermod", "-aG", "audio", "archipelago"]).await;
|
|
|
|
let unit_was_missing = fs::metadata(AUDIO_SERVICE_PATH).await.is_err();
|
|
let script_changed = write_root_if_needed(AUDIO_ROUTER_PATH, AUDIO_ROUTER)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed {
|
|
let _ = host_sudo(&["chmod", "+x", AUDIO_ROUTER_PATH]).await;
|
|
}
|
|
let unit_changed = write_root_if_needed(AUDIO_SERVICE_PATH, AUDIO_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
|
|
if script_changed || unit_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("audio: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
if unit_was_missing {
|
|
// First install on this node — bring it up now and on every boot.
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-audio-router.service",
|
|
])
|
|
.await;
|
|
info!("audio: router installed and enabled (HDMI routing + ELD heal)");
|
|
} else if script_changed || unit_changed {
|
|
// Content update: restart only if it's running — never re-enable a
|
|
// unit an operator deliberately disabled.
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"try-restart",
|
|
"archipelago-audio-router.service",
|
|
])
|
|
.await;
|
|
info!("audio: router updated");
|
|
}
|
|
}
|
|
|
|
/// Gamepad→keyboard bridge self-heal for kiosk nodes: keeps the evdev→uinput
|
|
/// daemon (controller works inside every app iframe on the TV) installed and
|
|
/// current. Same gating as audio: no kiosk → no display input to bridge.
|
|
pub async fn ensure_gamepad_keys() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return;
|
|
}
|
|
let unit_was_missing = fs::metadata(GAMEPAD_SERVICE_PATH).await.is_err();
|
|
let script_changed = write_root_if_needed(GAMEPAD_KEYS_PATH, GAMEPAD_KEYS)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed {
|
|
let _ = host_sudo(&["chmod", "+x", GAMEPAD_KEYS_PATH]).await;
|
|
}
|
|
let unit_changed = write_root_if_needed(GAMEPAD_SERVICE_PATH, GAMEPAD_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed || unit_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("gamepad bridge: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
if unit_was_missing {
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-gamepad-keys.service",
|
|
])
|
|
.await;
|
|
info!("gamepad: bridge installed and enabled (TV controller input)");
|
|
} else if script_changed || unit_changed {
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"try-restart",
|
|
"archipelago-gamepad-keys.service",
|
|
])
|
|
.await;
|
|
info!("gamepad: bridge updated");
|
|
}
|
|
}
|
|
|
|
/// Patch the nginx site config to add missing backend proxy blocks. Older ISO
|
|
/// configs shipped individual per-endpoint `location` blocks, so missing
|
|
/// endpoints silently fell through to the SPA `index.html` and the frontend
|
|
/// got HTML instead of JSON.
|
|
///
|
|
/// Validates via `nginx -t` before reloading. On failure the patch is
|
|
/// rolled back from a backup written just before the write.
|
|
async fn run_nginx() -> Result<bool> {
|
|
// Skip on dev symlinks — we don't want to touch `/etc/nginx` on laptops.
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
let mut patched_paths = Vec::<PathBuf>::new();
|
|
for path in [
|
|
NGINX_CONF_PATH,
|
|
NGINX_ENABLED_CONF_PATH,
|
|
NGINX_HTTPS_SNIPPET_PATH,
|
|
] {
|
|
let candidate = Path::new(path);
|
|
if !candidate.exists() {
|
|
debug!("{} missing — skipping nginx bootstrap", path);
|
|
continue;
|
|
}
|
|
let canonical = fs::canonicalize(candidate)
|
|
.await
|
|
.unwrap_or_else(|_| candidate.to_path_buf());
|
|
if patched_paths.iter().any(|p| p == &canonical) {
|
|
continue;
|
|
}
|
|
patched_paths.push(canonical);
|
|
changed |= patch_nginx_conf(path).await?;
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
/// Reflective CORS add_headers that older configs placed inside the
|
|
/// `/lnd-connect-info` location. The backend now sets a validated
|
|
/// `Access-Control-Allow-Origin` for that endpoint (api/handler/proxy.rs), so
|
|
/// leaving these in nginx emits a DUPLICATE header ("contains multiple values
|
|
/// … but only one is allowed") and the LND wallet UI's cross-origin fetch is
|
|
/// rejected. Stripped during nginx bootstrap so the backend solely owns CORS.
|
|
const NGINX_LND_DUP_CORS: &str = " add_header Access-Control-Allow-Origin $http_origin always;\n add_header Access-Control-Allow-Credentials \"true\" always;\n";
|
|
|
|
/// S4 follow-up (2026-08-07): pre-fix nodes proxy /aiui/api/web-search
|
|
/// STRAIGHT to SearXNG (127.0.0.1:8888) with no session check — anyone on the
|
|
/// LAN can run searches attributed to the node's IP. The canonical conf routes
|
|
/// it through the session-gated daemon (5678) and forwards the Cookie. The
|
|
/// stale target string is unique to that block, so a plain replace is safe.
|
|
/// Pure and testable; `None` when the stale shape is absent.
|
|
fn heal_stale_web_search_block(content: &str) -> Option<String> {
|
|
if !content.contains("proxy_pass http://127.0.0.1:8888/search;") {
|
|
return None;
|
|
}
|
|
Some(content.replace(
|
|
"proxy_pass http://127.0.0.1:8888/search;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;",
|
|
"proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header Cookie $http_cookie;",
|
|
))
|
|
}
|
|
|
|
async fn patch_nginx_conf(path: &str) -> Result<bool> {
|
|
let content = fs::read_to_string(path)
|
|
.await
|
|
.with_context(|| format!("read {}", path))?;
|
|
// Each "missing" flag is gated on the splice anchor actually being present,
|
|
// so an included snippet that legitimately has none of these endpoints (the
|
|
// HTTPS app-proxy snippet) neither tries to patch them nor logs warn-skips on
|
|
// every boot — it falls through to the fedimint heal alone.
|
|
let has_lnd_anchor = content.contains(" location /lnd-connect-info {")
|
|
|| content.contains(" location /electrs-status {");
|
|
let missing_app_catalog = content
|
|
.contains(" # DWN endpoints — peer access over Tor (no auth)")
|
|
&& !content.contains("location /api/app-catalog");
|
|
let missing_bitcoin_status = content.contains(" location /electrs-status {")
|
|
&& !content.contains("location /bitcoin-status");
|
|
let missing_lnd_proxy = has_lnd_anchor && !content.contains("location /proxy/lnd/");
|
|
let missing_peer_content = has_lnd_anchor && !content.contains("location /api/peer-content");
|
|
let missing_pine_status = has_lnd_anchor && !content.contains("location /api/pine/status");
|
|
let has_lnd_dup_cors = content.contains(NGINX_LND_DUP_CORS);
|
|
// B13: fedimint block present but lacking the asset-rewrite sub_filters.
|
|
let needs_fedimint_css = content.contains("location /app/fedimint/")
|
|
&& !content.contains("'href=\"/' 'href=\"/app/fedimint/'");
|
|
// Companion mesh access: phones reach this node over FIPS at its fips0
|
|
// ULA (http://[fdxx:…]). Configs shipped before 2026-07-23 listened on
|
|
// IPv4 only, so the ULA could never connect — nothing answered [::]:80.
|
|
let missing_v6_http =
|
|
content.contains("listen 80 default_server;") && !content.contains("listen [::]:80");
|
|
let missing_v6_https =
|
|
content.contains("listen 443 ssl default_server;") && !content.contains("listen [::]:443");
|
|
let stale_web_search = heal_stale_web_search_block(&content).is_some();
|
|
if !missing_app_catalog
|
|
&& !missing_bitcoin_status
|
|
&& !missing_lnd_proxy
|
|
&& !missing_peer_content
|
|
&& !missing_pine_status
|
|
&& !has_lnd_dup_cors
|
|
&& !needs_fedimint_css
|
|
&& !missing_v6_http
|
|
&& !missing_v6_https
|
|
&& !stale_web_search
|
|
{
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut patched = content.clone();
|
|
|
|
if let Some(p) = heal_stale_web_search_block(&patched) {
|
|
patched = p;
|
|
}
|
|
|
|
if missing_v6_http {
|
|
patched = patched.replace(
|
|
"listen 80 default_server;",
|
|
"listen 80 default_server;\n listen [::]:80 default_server;",
|
|
);
|
|
}
|
|
if missing_v6_https {
|
|
patched = patched.replace(
|
|
"listen 443 ssl default_server;",
|
|
"listen 443 ssl default_server;\n listen [::]:443 ssl default_server;",
|
|
);
|
|
}
|
|
|
|
if has_lnd_dup_cors {
|
|
// Drop the redundant nginx-side CORS headers so the backend's single
|
|
// validated Access-Control-Allow-Origin is the only one returned.
|
|
patched = patched.replace(NGINX_LND_DUP_CORS, "");
|
|
}
|
|
|
|
if needs_fedimint_css {
|
|
// Style A (main conf): the block already injects nostr-provider, so swap
|
|
// its single-sub_filter tail for the full asset-rewrite set. No-op if the
|
|
// node's fedimint block doesn't match OLD.
|
|
patched = patched.replace(NGINX_FEDIMINT_OLD, NGINX_FEDIMINT_NEW);
|
|
// Style B (HTTPS app-proxy snippet): the block has no sub_filter to swap,
|
|
// so insert the reroot set after the unique :8175 proxy_pass. Guarded on
|
|
// the marker so it can never double-apply after Style A already healed.
|
|
if !patched.contains("'href=\"/' 'href=\"/app/fedimint/'") {
|
|
patched = patched.replace(NGINX_FEDIMINT_SNIPPET_ANCHOR, NGINX_FEDIMINT_SNIPPET_INSERT);
|
|
}
|
|
}
|
|
|
|
if missing_lnd_proxy {
|
|
// Prefer the `/lnd-connect-info` anchor (present since 2026-03-17); fall
|
|
// back to `/electrs-status` (since 2026-03-08) for even older configs.
|
|
// Both appear once per archipelago server block, so the block is added
|
|
// to every server block that proxies to the backend.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing lnd-connect-info/electrs-status anchor — skipping /proxy/lnd patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_LND_PROXY_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if missing_pine_status {
|
|
// Same anchoring as the LND proxy: prepend to every server block that
|
|
// proxies to the backend.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if patched.contains(anchor) {
|
|
let replacement = format!("{}{}", NGINX_PINE_STATUS_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
} else {
|
|
warn!("nginx conf missing anchor — skipping /api/pine/status patch");
|
|
}
|
|
}
|
|
|
|
if missing_peer_content {
|
|
// Same anchoring as the LND proxy: prepend the block to every server
|
|
// block so /api/peer-content/* reaches the backend instead of the SPA.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if patched.contains(anchor) {
|
|
let replacement = format!("{}{}", NGINX_PEER_CONTENT_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
} else {
|
|
warn!("nginx conf missing anchor — skipping /api/peer-content patch");
|
|
}
|
|
}
|
|
|
|
if missing_bitcoin_status {
|
|
let anchor = " location /electrs-status {";
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing electrs-status anchor — skipping /bitcoin-status patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_BITCOIN_STATUS_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if missing_app_catalog {
|
|
// The DWN comment sits at the same indent right after the `/api/blob`
|
|
// block in both server blocks — a stable anchor that existed on every
|
|
// ISO shipped to date. If it's absent (config got heavily customized),
|
|
// skip rather than guess where to splice.
|
|
let anchor = " # DWN endpoints — peer access over Tor (no auth)";
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing DWN anchor — skipping /api/app-catalog patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_APP_CATALOG_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if patched == content {
|
|
return Ok(false);
|
|
}
|
|
|
|
// Write patched config via a user-owned tmp + sudo mv, after stashing
|
|
// a backup outside nginx include dirs so validation cannot load it too.
|
|
let pid = std::process::id();
|
|
let tmp = format!("/tmp/archipelago-nginx-{}.conf", pid);
|
|
fs::write(&tmp, &patched)
|
|
.await
|
|
.with_context(|| format!("write {}", tmp))?;
|
|
|
|
let backup = format!(
|
|
"/tmp/archipelago-nginx-backup-{}-{}.conf",
|
|
pid,
|
|
patched.len()
|
|
);
|
|
if let Err(e) = host_sudo(&["cp", path, &backup]).await {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
return Err(e.context("backup nginx conf"));
|
|
}
|
|
|
|
let mv = host_sudo(&["mv", &tmp, path]).await;
|
|
match mv {
|
|
Ok(s) if s.success() => {}
|
|
Ok(s) => {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
anyhow::bail!("sudo mv nginx conf to {} exited with {}", path, s);
|
|
}
|
|
Err(e) => {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
return Err(e.context("mv tmp -> nginx conf"));
|
|
}
|
|
}
|
|
|
|
// Validate.
|
|
let test = host_sudo(&["nginx", "-t"]).await;
|
|
let valid = matches!(&test, Ok(s) if s.success());
|
|
if !valid {
|
|
warn!("nginx -t failed after patch — reverting");
|
|
let _ = host_sudo(&["mv", &backup, path]).await;
|
|
if let Err(e) = test {
|
|
return Err(e.context("nginx -t"));
|
|
}
|
|
anyhow::bail!("nginx config invalid after patch — reverted");
|
|
}
|
|
|
|
// Reload nginx so the new block takes effect immediately. Reload (not
|
|
// restart) keeps in-flight connections alive.
|
|
if let Err(e) = host_sudo(&["systemctl", "reload", "nginx"]).await {
|
|
warn!("nginx reload failed (non-fatal): {:#}", e);
|
|
}
|
|
let _ = host_sudo(&["rm", "-f", &backup]).await;
|
|
Ok(true)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn podman_heal_outcome_no_longer_has_cleanup_variant() {
|
|
let outcome = PodmanHealOutcome::Unhealthy;
|
|
assert_ne!(outcome, PodmanHealOutcome::Healthy);
|
|
}
|
|
|
|
#[test]
|
|
fn stale_web_search_block_is_gated_and_idempotent() {
|
|
let stale = " location /aiui/api/web-search {\n proxy_pass http://127.0.0.1:8888/search;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_connect_timeout 30s;\n }";
|
|
let healed = heal_stale_web_search_block(stale).expect("stale block must heal");
|
|
assert!(healed.contains("proxy_pass http://127.0.0.1:5678;"));
|
|
assert!(healed.contains("proxy_set_header Cookie $http_cookie;"));
|
|
assert!(!healed.contains("8888"));
|
|
// Second pass is a no-op (idempotent self-heal).
|
|
assert!(heal_stale_web_search_block(&healed).is_none());
|
|
// A config without the block is untouched.
|
|
assert!(
|
|
heal_stale_web_search_block("location / { try_files $uri /index.html; }").is_none()
|
|
);
|
|
}
|
|
|
|
/// The exact ExecStart framework-pt shipped with must parse, and the
|
|
/// rewrite must preserve its listen port and forward target.
|
|
#[test]
|
|
fn static_socat_bind_is_parsed_and_rewritten_dynamically() {
|
|
let line = "ExecStart=/usr/bin/socat TCP-LISTEN:18332,bind=192.168.1.249,fork,reuseaddr TCP:127.0.0.1:8332";
|
|
let (port, target) = parse_socat_static_bind(line).expect("must parse");
|
|
assert_eq!(port, "18332");
|
|
assert_eq!(target, "127.0.0.1:8332");
|
|
let dynamic = dynamic_bind_execstart(&port, &target);
|
|
assert!(dynamic.contains("TCP-LISTEN:18332,bind=$$IP"));
|
|
assert!(dynamic.contains("TCP:127.0.0.1:8332"));
|
|
assert!(dynamic.contains("route get 1.1.1.1"));
|
|
// The heal is idempotent: its own output no longer parses as a
|
|
// static bind (bind=$$IP is not a concrete address).
|
|
assert!(parse_socat_static_bind(&dynamic).is_none());
|
|
}
|
|
|
|
/// The archi-dev-box config: one stale address (old network) beside the
|
|
/// WireGuard one. The stale listener must go — nginx refuses to START
|
|
/// while it names an address the host lacks — and the current LAN
|
|
/// address must appear.
|
|
#[test]
|
|
fn stale_https_listeners_are_retargeted_to_present_addresses() {
|
|
let cfg = "server {\n listen 80 default_server;\n listen 10.44.0.1:443 ssl;\n listen 192.168.63.240:443 ssl;\n ssl_certificate /x;\n}\n";
|
|
let present = vec!["10.44.0.1".to_string(), "192.168.1.50".to_string()];
|
|
let healed = retarget_https_listeners(cfg, &present).expect("must heal");
|
|
assert!(healed.contains("listen 192.168.1.50:443 ssl;"));
|
|
assert!(healed.contains("listen 10.44.0.1:443 ssl;"));
|
|
assert!(!healed.contains("192.168.63.240"), "stale listener must be dropped");
|
|
// Untouched lines survive, and the repair is idempotent.
|
|
assert!(healed.contains("listen 80 default_server;"));
|
|
assert!(healed.contains("ssl_certificate /x;"));
|
|
assert!(retarget_https_listeners(&healed, &present).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn wildcard_only_configs_and_cgnat_are_left_alone() {
|
|
// No address-pinned listener → nothing to heal (the ISO's own config).
|
|
assert!(retarget_https_listeners(
|
|
"server {\n listen 443 ssl default_server;\n}\n",
|
|
&["192.168.1.50".to_string()]
|
|
)
|
|
.is_none());
|
|
// Tailscale CGNAT must never become an nginx listener: tailscaled
|
|
// already holds :443 there, and binding it would fail nginx outright.
|
|
assert!(is_cgnat("100.69.68.39"));
|
|
assert!(is_cgnat("100.127.255.1"));
|
|
assert!(!is_cgnat("100.128.0.1"));
|
|
assert!(!is_cgnat("192.168.1.50"));
|
|
assert!(!is_cgnat("10.44.0.1"));
|
|
}
|
|
|
|
#[test]
|
|
fn socat_units_that_need_no_heal_are_left_alone() {
|
|
// Loopback bind is intentional (Tor bootstrap forwarder) — not ours.
|
|
assert!(parse_socat_static_bind(
|
|
"ExecStart=/usr/bin/socat TCP-LISTEN:18332,bind=127.0.0.1,reuseaddr,fork SOCKS4A:127.0.0.1:x.onion:8332,socksport=9050"
|
|
)
|
|
.is_none());
|
|
// No bind at all.
|
|
assert!(parse_socat_static_bind(
|
|
"ExecStart=/usr/bin/socat TCP-LISTEN:18332,fork,reuseaddr TCP:127.0.0.1:8332"
|
|
)
|
|
.is_none());
|
|
// Not a socat line.
|
|
assert!(parse_socat_static_bind("ExecStart=/usr/bin/true").is_none());
|
|
}
|
|
}
|
|
|
|
/// Repair this node's own systemd restart policy.
|
|
///
|
|
/// The in-process updater replaces the binary and then asks systemd to
|
|
/// restart the service, treating `Restart=always` on the unit as its second
|
|
/// net if that request is ever lost. On austin-sapien (2026-08-05) the unit
|
|
/// was an old one carrying `Restart=on-failure`: the daemon exited cleanly
|
|
/// (status 0), systemd read that as success, and the node sat dead for over
|
|
/// two hours after a routine update — "server starting" in the UI, with
|
|
/// nothing to start it.
|
|
///
|
|
/// A node cannot be relied on to fix this via `self-update.sh` (which does
|
|
/// refresh units) because the in-process update path never runs it. So the
|
|
/// daemon checks its own unit at boot: any node that starts even once ends
|
|
/// up with a policy that survives the next update. Deliberately narrow —
|
|
/// only the `Restart=` line is touched, so local edits elsewhere in the unit
|
|
/// are preserved.
|
|
pub async fn ensure_restart_policy() {
|
|
const UNIT: &str = "/etc/systemd/system/archipelago.service";
|
|
let Ok(body) = fs::read_to_string(UNIT).await else {
|
|
return; // not a systemd install (container, dev box) — nothing to do
|
|
};
|
|
if !body.lines().any(|l| {
|
|
let l = l.trim();
|
|
l.starts_with("Restart=") && l != "Restart=always"
|
|
}) {
|
|
return; // already correct, or no Restart= line to repair
|
|
}
|
|
let patched: String = body
|
|
.lines()
|
|
.map(|l| {
|
|
if l.trim().starts_with("Restart=") && l.trim() != "Restart=always" {
|
|
"Restart=always"
|
|
} else {
|
|
l
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
match write_root_if_needed(UNIT, &patched).await {
|
|
Ok(true) => {
|
|
tracing::warn!(
|
|
"repaired archipelago.service Restart= policy to always — this node would \
|
|
have stayed dead after an in-process update"
|
|
);
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
}
|
|
Ok(false) => {}
|
|
Err(e) => tracing::warn!(error = %e, "could not repair archipelago.service restart policy"),
|
|
}
|
|
}
|