archipelagoandClaude Fable 5
81033ed6f5
austin-sapien (100.70.96.88) sat dead for over two hours after taking v1.7.122 — 'server starting' in the UI, service inactive, exit status 0/SUCCESS. It did not crash: the in-process updater replaces the binary and exits cleanly for systemd to restart it, and that node's unit still carried Restart=on-failure from an older install. systemd read the clean exit as success and left it stopped. Every node with the old unit has this waiting for it on the next update. self-update.sh does refresh units, but the in-process update path never runs it, so nothing was repairing them. The daemon now checks its own unit at boot and rewrites only the Restart= line, so a node that starts even once ends up with a policy that survives the next update. Also carries the session-policy wiring: validate() now honours the configured idle and absolute limits and the per-device class, instead of the single hard-coded 24h constant. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1325 lines
59 KiB
Rust
1325 lines
59 KiB
Rust
//! Bootstrap host-side artifacts on every archipelago startup.
|
|
//!
|
|
//! The update pipeline swaps the archipelago binary but does not touch
|
|
//! scripts, systemd units, or nginx configuration — those are installed
|
|
//! once by the ISO builder. Without this module, changes to
|
|
//! `container-doctor.sh`, the doctor service/timer, or the nginx config
|
|
//! never reach boxes installed before the change.
|
|
//!
|
|
//! Two things are synced on startup:
|
|
//! 1. Doctor artifacts (container-doctor.sh + service + timer).
|
|
//! 2. Missing nginx backend proxy blocks required for frontend fetches to
|
|
//! reach the backend instead of the SPA fallback.
|
|
//!
|
|
//! Idempotent: no-ops on boxes that are already in sync. All work is
|
|
//! best-effort — failures are logged but never abort the backend.
|
|
|
|
use anyhow::{Context, Result};
|
|
use std::path::{Path, PathBuf};
|
|
use tokio::fs;
|
|
use tracing::{debug, info, warn};
|
|
|
|
use crate::update::host_sudo;
|
|
|
|
const DOCTOR_SH: &str = include_str!("../../../scripts/container-doctor.sh");
|
|
const DOCTOR_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-doctor.service");
|
|
const DOCTOR_TIMER: &str = include_str!("../../../image-recipe/configs/archipelago-doctor.timer");
|
|
|
|
const DOCTOR_SH_PATH: &str = "/home/archipelago/archy/scripts/container-doctor.sh";
|
|
const DOCTOR_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-doctor.service";
|
|
const DOCTOR_TIMER_PATH: &str = "/etc/systemd/system/archipelago-doctor.timer";
|
|
|
|
// Kiosk hardening (#36): keep the deployed unit + launcher in sync with the
|
|
// repo so the CPU/memory cap and the GPU-vs-headless flag selection reach
|
|
// already-installed nodes via OTA, not just fresh ISOs.
|
|
const KIOSK_SERVICE: &str = include_str!("../../../image-recipe/configs/archipelago-kiosk.service");
|
|
const KIOSK_LAUNCHER: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-kiosk-launcher.sh");
|
|
const KIOSK_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-kiosk.service";
|
|
const KIOSK_LAUNCHER_PATH: &str = "/usr/local/bin/archipelago-kiosk-launcher";
|
|
|
|
// HDMI audio (kiosk nodes): ISOs built before 2026-07-23 shipped no audio
|
|
// stack at all even though the kiosk launcher expects PipeWire-Pulse, and the
|
|
// kiosk's boot-time modeset can race the i915→HDA audio-component bind so the
|
|
// HDMI ELD is lost and every HDMI profile stays unavailable (silent failure).
|
|
// The router daemon handles routing + the ELD re-modeset nudge; this heal
|
|
// installs the packages, group membership, script and unit on deployed nodes.
|
|
const AUDIO_ROUTER: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-audio-router.sh");
|
|
const AUDIO_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-audio-router.service");
|
|
const AUDIO_ROUTER_PATH: &str = "/usr/local/bin/archipelago-audio-router";
|
|
const AUDIO_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-audio-router.service";
|
|
|
|
// Gamepad→keyboard bridge (TV input inside every app iframe) — same
|
|
// splice-from-configs + self-heal pattern as the audio router.
|
|
const GAMEPAD_KEYS: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-gamepad-keys.py");
|
|
const GAMEPAD_SERVICE: &str =
|
|
include_str!("../../../image-recipe/configs/archipelago-gamepad-keys.service");
|
|
const GAMEPAD_KEYS_PATH: &str = "/usr/local/bin/archipelago-gamepad-keys";
|
|
const GAMEPAD_SERVICE_PATH: &str = "/etc/systemd/system/archipelago-gamepad-keys.service";
|
|
|
|
// Journald log-volume policy (size cap + per-service rate limit). Fresh ISOs
|
|
// write the identical file at build time (image-recipe/_archived/
|
|
// build-auto-installer-iso.sh); this heals already-deployed nodes via OTA.
|
|
// A fresh node produced >1 GB/day of journal (bitcoind IBD console spam plus
|
|
// debug-level backend logging) — the cap bounds disk use and the rate limit
|
|
// keeps one chatty service from drowning everything else.
|
|
const JOURNALD_DROPIN: &str =
|
|
include_str!("../../../image-recipe/configs/journald-archipelago.conf");
|
|
const JOURNALD_DROPIN_PATH: &str = "/etc/systemd/journald.conf.d/10-archipelago-persistent.conf";
|
|
|
|
const NGINX_CONF_PATH: &str = "/etc/nginx/sites-available/archipelago";
|
|
const NGINX_ENABLED_CONF_PATH: &str = "/etc/nginx/sites-enabled/archipelago";
|
|
/// Per-app proxy snippet included by the HTTPS (:443) server block. Carries its
|
|
/// own `/app/fedimint/` location, so it needs the same B13 asset-rewrite heal as
|
|
/// the main conf — browsers reach fedimint over HTTPS via this snippet. Absent on
|
|
/// HTTP-only nodes, in which case the bootstrap loop skips it.
|
|
const NGINX_HTTPS_SNIPPET_PATH: &str = "/etc/nginx/snippets/archipelago-https-app-proxies.conf";
|
|
const RUNTIME_ASSETS_DIR: &str = "/opt/archipelago/web-ui/archipelago-runtime";
|
|
|
|
/// Inserted into every server block of the nginx config that lacks the
|
|
/// `/api/app-catalog` proxy. Kept in sync with the canonical block in
|
|
/// image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_APP_CATALOG_BLOCK: &str = "\n # App Store catalog proxy — backend fetches from configured registries\n # so the browser doesn't hit CORS/CSP. Without this block nginx falls\n # through to the SPA index.html and the frontend gets HTML back instead\n # of JSON.\n location /api/app-catalog {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header Cookie $http_cookie;\n proxy_connect_timeout 15s;\n proxy_read_timeout 30s;\n proxy_send_timeout 15s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n\n";
|
|
|
|
const NGINX_BITCOIN_STATUS_BLOCK: &str = "\n location /bitcoin-status {\n proxy_pass http://127.0.0.1:5678/bitcoin-status;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_connect_timeout 10s;\n proxy_read_timeout 10s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block that lacks the `/proxy/lnd/` proxy. Nodes
|
|
/// flashed before 2026-04-10 shipped an nginx config without this block, so the
|
|
/// browser's wallet fetches to `/proxy/lnd/*` fell through to the SPA
|
|
/// index.html and got HTML back instead of JSON ("failing to fetch"). Kept in
|
|
/// sync with the canonical block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_LND_PROXY_BLOCK: &str = "\n # LND REST proxy — backend handles auth + CORS\n location /proxy/lnd/ {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Cookie $http_cookie;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_connect_timeout 10s;\n proxy_read_timeout 10s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block lacking the peer-content streaming proxy.
|
|
/// Without it, the browser's `<video>`/`<audio>` Range requests to
|
|
/// `/api/peer-content/*` fall through to the SPA index.html (HTML, no Range)
|
|
/// and peer media won't play (B3). Forwards Cookie (session auth) + Range and
|
|
/// disables buffering so streaming works. Kept in sync with the canonical
|
|
/// block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_PEER_CONTENT_BLOCK: &str = "\n # Peer content streaming proxy (B3) — Range-streams a peer's media file.\n # Long read timeout: this path also serves full-file downloads of large\n # media (#38), which can take minutes over Tor; 120s aborted them.\n location /api/peer-content/ {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Cookie $http_cookie;\n proxy_set_header Range $http_range;\n proxy_buffering off;\n proxy_connect_timeout 10s;\n proxy_read_timeout 900s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// Inserted into every server block lacking the Pine node-status proxy.
|
|
/// `/api/pine/status` serves the Pine launcher page's live status card and
|
|
/// the seeded Home Assistant REST sensors (the sensitive tier is gated by a
|
|
/// bearer token at the backend, so nginx just forwards). Kept in sync with
|
|
/// the canonical block in image-recipe/configs/nginx-archipelago.conf.
|
|
const NGINX_PINE_STATUS_BLOCK: &str = "\n # Pine node status — live node facts for the Pine launcher page and the\n # seeded Home Assistant sensors. Sensitive fields are token-gated at the\n # backend; nginx only forwards.\n location /api/pine/status {\n proxy_pass http://127.0.0.1:5678;\n proxy_http_version 1.1;\n proxy_set_header Host $host;\n proxy_set_header Authorization $http_authorization;\n proxy_set_header Cookie $http_cookie;\n proxy_connect_timeout 10s;\n proxy_read_timeout 15s;\n proxy_send_timeout 5s;\n error_page 502 503 = @backend_unavailable;\n error_page 504 = @backend_timeout;\n }\n";
|
|
|
|
/// B13 — Fedimint UI asset rewrite. Pre-fix nodes proxy /app/fedimint/ with only
|
|
/// the nostr-provider injection (`sub_filter_once on`), so the UI's root-rooted
|
|
/// CSS/JS asset URLs (href="/…", url("/…")) miss the proxy and load the SPA shell
|
|
/// → unstyled UI. We swap that single sub_filter for the full rewrite set that
|
|
/// reroots every asset URL under /app/fedimint/. NEW matches the canonical block
|
|
/// in image-recipe/configs/nginx-archipelago.conf byte-for-byte so self-healed
|
|
/// nodes converge to the same config fresh ISOs ship with.
|
|
const NGINX_FEDIMINT_OLD: &str = " sub_filter_once on;\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';\n }\n location /app/fedimint-gateway/ {";
|
|
const NGINX_FEDIMINT_NEW: &str = " sub_filter_types text/css application/javascript application/json;\n sub_filter_once off;\n sub_filter 'href=\"/' 'href=\"/app/fedimint/';\n sub_filter 'src=\"/' 'src=\"/app/fedimint/';\n sub_filter \"href='/\" \"href='/app/fedimint/\";\n sub_filter \"src='/\" \"src='/app/fedimint/\";\n sub_filter 'url(\"/' 'url(\"/app/fedimint/';\n sub_filter \"url('/\" \"url('/app/fedimint/\";\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';\n }\n location /app/fedimint-gateway/ {";
|
|
|
|
/// B13 Style B — the HTTPS app-proxy snippet's fedimint block has NO sub_filter
|
|
/// at all (older than the main conf's), and the directive that follows it varies
|
|
/// per node (fedimint-gateway vs tailscale), so a full-block match is unreliable.
|
|
/// Instead we anchor on the unique :8175 proxy_pass (fedimint is the only block
|
|
/// proxying there) and insert the reroot set right after it — directive order
|
|
/// inside a location block is irrelevant to nginx. Idempotent via the same
|
|
/// `href="/app/fedimint/` marker the main-conf heal leaves behind.
|
|
const NGINX_FEDIMINT_SNIPPET_ANCHOR: &str = "proxy_pass http://127.0.0.1:8175/;";
|
|
const NGINX_FEDIMINT_SNIPPET_INSERT: &str = "proxy_pass http://127.0.0.1:8175/;\n proxy_set_header Accept-Encoding \"\";\n sub_filter_types text/css application/javascript application/json;\n sub_filter_once off;\n sub_filter 'href=\"/' 'href=\"/app/fedimint/';\n sub_filter 'src=\"/' 'src=\"/app/fedimint/';\n sub_filter \"href='/\" \"href='/app/fedimint/\";\n sub_filter \"src='/\" \"src='/app/fedimint/\";\n sub_filter 'url(\"/' 'url(\"/app/fedimint/';\n sub_filter \"url('/\" \"url('/app/fedimint/\";\n sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';";
|
|
|
|
/// Entry point called from main startup. Never returns an error to the caller —
|
|
/// failing to bootstrap host artifacts must not prevent the backend from serving.
|
|
pub async fn ensure_doctor_installed() {
|
|
match run_service_override_repair().await {
|
|
Ok(true) => info!("Removed stale Archipelago dev-mode service override"),
|
|
Ok(false) => debug!("No stale Archipelago dev-mode service override found"),
|
|
Err(e) => warn!("Service override repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_runtime_assets().await {
|
|
Ok(changed) if changed => info!("Runtime assets synchronized from OTA payload"),
|
|
Ok(_) => debug!("No OTA runtime payload to synchronize"),
|
|
Err(e) => warn!("Runtime asset bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run().await {
|
|
Ok(changed) if changed => info!("Doctor artifacts synchronized with binary"),
|
|
Ok(_) => debug!("Doctor artifacts already in sync"),
|
|
Err(e) => warn!("Doctor bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_nginx().await {
|
|
Ok(true) => info!("Patched nginx config to proxy missing backend endpoints"),
|
|
Ok(false) => debug!("Nginx backend endpoint proxy blocks already present"),
|
|
Err(e) => warn!("Nginx bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_bitcoin_rpc_repair().await {
|
|
Ok(true) => {
|
|
info!("Removed stale bitcoin.conf; running Bitcoin containers left untouched")
|
|
}
|
|
Ok(false) => debug!("No stale bitcoin.conf found"),
|
|
Err(e) => warn!("Bitcoin RPC repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_apps_dir_repair().await {
|
|
Ok(true) => {
|
|
info!("Populated /opt/archipelago/apps from installer copy at /etc/archipelago/apps")
|
|
}
|
|
Ok(false) => debug!("/opt/archipelago/apps already populated (or no installer copy)"),
|
|
Err(e) => warn!("Apps dir repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_polkit_networkmanager_repair().await {
|
|
Ok(true) => info!(
|
|
"Installed NetworkManager polkit rule for the archipelago user — Wi-Fi setup enabled"
|
|
),
|
|
Ok(false) => debug!("NetworkManager polkit rule already present"),
|
|
Err(e) => warn!("polkit NetworkManager repair failed (non-fatal): {:#}", e),
|
|
}
|
|
match run_journald_dropin().await {
|
|
Ok(true) => info!("Installed journald log-volume policy drop-in"),
|
|
Ok(false) => debug!("journald log-volume policy already in place"),
|
|
Err(e) => warn!("journald drop-in bootstrap failed (non-fatal): {:#}", e),
|
|
}
|
|
match tighten_secrets_dir().await {
|
|
Ok(n) if n > 0 => info!(tightened = n, "Tightened mode on secret files"),
|
|
Ok(_) => debug!("Secrets directory already at expected mode"),
|
|
Err(e) => warn!("Secrets dir tightening failed (non-fatal): {:#}", e),
|
|
}
|
|
// Podman probing MUST be the last bootstrap stage. We used to delete
|
|
// transient runroot state here when `podman info` failed, but live nodes
|
|
// can still have rootlessport/conmon processes holding that state. Removing
|
|
// it automatically makes failures worse: containers lose `.containerenv`,
|
|
// ports stay bound, and later starts fail. Report the fault instead; repair
|
|
// must be deliberate/operator-driven.
|
|
match heal_podman_state().await {
|
|
Ok(PodmanHealOutcome::Healthy) => debug!("podman runtime state healthy"),
|
|
Ok(PodmanHealOutcome::Unhealthy) => warn!(
|
|
"podman runtime state is unhealthy at startup — skipping automatic runroot cleanup"
|
|
),
|
|
Err(e) => warn!(
|
|
"podman self-heal failed (non-fatal, will retry next boot): {:#}",
|
|
e
|
|
),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
enum PodmanHealOutcome {
|
|
Healthy,
|
|
Unhealthy,
|
|
}
|
|
|
|
async fn heal_podman_state() -> Result<PodmanHealOutcome> {
|
|
if probe_podman_ok().await {
|
|
return Ok(PodmanHealOutcome::Healthy);
|
|
}
|
|
Ok(PodmanHealOutcome::Unhealthy)
|
|
}
|
|
|
|
/// True iff `podman info` returns 0 within 5s. Any timeout, spawn
|
|
/// failure, or non-zero exit reads as "wedged" and triggers cleanup.
|
|
async fn probe_podman_ok() -> bool {
|
|
use std::time::Duration;
|
|
let probe = tokio::time::timeout(
|
|
Duration::from_secs(5),
|
|
tokio::process::Command::new("podman")
|
|
.arg("info")
|
|
.arg("--format=json")
|
|
.output(),
|
|
)
|
|
.await;
|
|
match probe {
|
|
Ok(Ok(out)) => out.status.success(),
|
|
Ok(Err(_)) | Err(_) => false,
|
|
}
|
|
}
|
|
|
|
/// Make sure /var/lib/archipelago/secrets/ stays 0700 owned by archipelago,
|
|
/// and every file inside is 0600. The parent dir mode is the load-bearing
|
|
/// boundary against host-side reads from other UIDs (rootless container
|
|
/// escapes get mapped to UID >= 100000 and can't traverse a 0700/uid=1000
|
|
/// directory). The per-file 0600 sweep is defense-in-depth in case some
|
|
/// installer wrote a 0644 file.
|
|
async fn tighten_secrets_dir() -> Result<u32> {
|
|
let dir = Path::new("/var/lib/archipelago/secrets");
|
|
if !dir.exists() {
|
|
return Ok(0);
|
|
}
|
|
use std::os::unix::fs::PermissionsExt;
|
|
fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))
|
|
.await
|
|
.with_context(|| format!("chmod 0700 {}", dir.display()))?;
|
|
|
|
let mut entries = fs::read_dir(dir)
|
|
.await
|
|
.with_context(|| format!("read_dir {}", dir.display()))?;
|
|
let mut tightened = 0u32;
|
|
while let Some(entry) = entries.next_entry().await? {
|
|
let path = entry.path();
|
|
let meta = match entry.metadata().await {
|
|
Ok(m) => m,
|
|
Err(_) => continue,
|
|
};
|
|
if !meta.is_file() {
|
|
continue;
|
|
}
|
|
if meta.permissions().mode() & 0o777 != 0o600 {
|
|
fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))
|
|
.await
|
|
.with_context(|| format!("chmod 0600 {}", path.display()))?;
|
|
tightened += 1;
|
|
}
|
|
}
|
|
Ok(tightened)
|
|
}
|
|
|
|
async fn run_service_override_repair() -> Result<bool> {
|
|
let override_path = Path::new("/etc/systemd/system/archipelago.service.d/override.conf");
|
|
let Ok(content) = fs::read_to_string(override_path).await else {
|
|
return Ok(false);
|
|
};
|
|
if !content.contains("ARCHIPELAGO_DEV_MODE=true") {
|
|
return Ok(false);
|
|
}
|
|
|
|
let only_dev_mode_override = content
|
|
.lines()
|
|
.map(str::trim)
|
|
.filter(|line| !line.is_empty() && !line.starts_with('#'))
|
|
.all(|line| line == "[Service]" || line == "Environment=ARCHIPELAGO_DEV_MODE=true");
|
|
if !only_dev_mode_override {
|
|
warn!(
|
|
path = %override_path.display(),
|
|
"Archipelago service override contains ARCHIPELAGO_DEV_MODE=true plus other settings; leaving it untouched"
|
|
);
|
|
return Ok(false);
|
|
}
|
|
|
|
let path_s = override_path.to_string_lossy().to_string();
|
|
let status = host_sudo(&["rm", "-f", &path_s])
|
|
.await
|
|
.with_context(|| format!("remove {}", override_path.display()))?;
|
|
if !status.success() {
|
|
anyhow::bail!("remove {} exited with {}", override_path.display(), status);
|
|
}
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
Ok(true)
|
|
}
|
|
|
|
async fn run_runtime_assets() -> Result<bool> {
|
|
// The v1.7.50 OTA bridge puts scripts/apps/docker assets inside the
|
|
// frontend tarball because older binaries only know how to apply the
|
|
// backend binary and frontend archive. Once the new backend starts, it
|
|
// promotes that payload into /opt so app installs use the matching specs.
|
|
let runtime_dir = Path::new(RUNTIME_ASSETS_DIR);
|
|
if !runtime_dir.exists() {
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
for (relative, dest) in [
|
|
("apps", "/opt/archipelago/apps"),
|
|
("scripts", "/opt/archipelago/scripts"),
|
|
("docker", "/opt/archipelago/docker"),
|
|
] {
|
|
let src = runtime_dir.join(relative);
|
|
if src.exists() {
|
|
replace_dir_from_runtime(&src, dest).await?;
|
|
if relative == "scripts" {
|
|
let _ = host_sudo(&[
|
|
"find", dest, "-type", "f", "-name", "*.sh", "-exec", "chmod", "755", "{}", "+",
|
|
])
|
|
.await;
|
|
let image_versions = format!("{}/image-versions.sh", dest);
|
|
if Path::new(&image_versions).exists() {
|
|
let _ =
|
|
host_sudo(&["cp", &image_versions, "/opt/archipelago/image-versions.sh"])
|
|
.await;
|
|
}
|
|
}
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
let configs = runtime_dir.join("image-recipe/configs");
|
|
let nginx_src = configs.join("nginx-archipelago.conf");
|
|
if nginx_src.exists() {
|
|
let src_s = nginx_src.to_string_lossy().to_string();
|
|
let status = host_sudo(&[
|
|
"install",
|
|
"-m",
|
|
"644",
|
|
&src_s,
|
|
"/etc/nginx/sites-available/archipelago",
|
|
])
|
|
.await
|
|
.context("install nginx-archipelago.conf")?;
|
|
if !status.success() {
|
|
anyhow::bail!("install nginx-archipelago.conf exited with {}", status);
|
|
}
|
|
changed = true;
|
|
}
|
|
|
|
// archipelago-host-secrets-audit.service rides this same path (phase 10
|
|
// KEY-02 / audit F-03). Nodes already in the field never received 10-03's
|
|
// ISO-build fix — the first-boot script is installed by the installer, not
|
|
// shipped by OTA — so a node that hit the old fail-open path is still on
|
|
// the SSH host key and TLS private key baked into its published ISO, and
|
|
// will never try again. This unit is how such a node reports itself. It is
|
|
// DETECT ONLY (D-06: detect-report-then-apply); rotation is operator-driven
|
|
// via `--apply --yes` and never fires from a unit.
|
|
let mut host_secrets_unit_installed = false;
|
|
for unit in [
|
|
"archipelago-doctor.service",
|
|
"archipelago-doctor.timer",
|
|
"archipelago-host-secrets-audit.service",
|
|
] {
|
|
let src = configs.join(unit);
|
|
if src.exists() {
|
|
let src_s = src.to_string_lossy().to_string();
|
|
let dest = format!("/etc/systemd/system/{}", unit);
|
|
let status = host_sudo(&["install", "-m", "644", &src_s, &dest])
|
|
.await
|
|
.with_context(|| format!("install {}", unit))?;
|
|
if !status.success() {
|
|
anyhow::bail!("install {} exited with {}", unit, status);
|
|
}
|
|
if unit == "archipelago-host-secrets-audit.service" {
|
|
host_secrets_unit_installed = true;
|
|
}
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
// Packaged radio tools (v1.7.118+): OTA updates only apply the backend
|
|
// binary and frontend tarball, so these PyInstaller binaries ride the
|
|
// runtime payload and get promoted here. Without this, fleet nodes keep
|
|
// a stale archy-reticulum-daemon (which exits on flags it doesn't know —
|
|
// the v1.7.117 --enable-transport rollout killed their mesh sessions)
|
|
// and never receive archy-rnodeconf at all (Flash LoRa: "No such file
|
|
// or directory"). Skipped when byte-identical; a running daemon is
|
|
// unaffected (install replaces the inode) and picks the new binary up
|
|
// on its next spawn.
|
|
for tool in ["archy-reticulum-daemon", "archy-rnodeconf"] {
|
|
let src = runtime_dir.join("radio-tools").join(tool);
|
|
if !src.exists() {
|
|
continue;
|
|
}
|
|
let dest = format!("/usr/local/bin/{}", tool);
|
|
let same = match (fs::read(&src).await, fs::read(&dest).await) {
|
|
(Ok(a), Ok(b)) => a == b,
|
|
_ => false,
|
|
};
|
|
if same {
|
|
continue;
|
|
}
|
|
let src_s = src.to_string_lossy().to_string();
|
|
let status = host_sudo(&["install", "-m", "755", &src_s, &dest])
|
|
.await
|
|
.with_context(|| format!("install {}", tool))?;
|
|
if !status.success() {
|
|
anyhow::bail!("install {} exited with {}", tool, status);
|
|
}
|
|
info!(
|
|
tool,
|
|
"Promoted packaged radio tool from OTA runtime payload"
|
|
);
|
|
changed = true;
|
|
}
|
|
|
|
if changed {
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
if host_secrets_unit_installed {
|
|
// `--now` on purpose: the verdict is the whole deliverable, and
|
|
// waiting for the next reboot to learn whether a node is running
|
|
// fleet-shared key material wastes the OTA that just delivered the
|
|
// means to find out. The unit is Type=oneshot, read-only and exits
|
|
// in milliseconds on a healthy node. Best-effort: a node that
|
|
// cannot enable it still boots, and the next OTA retries.
|
|
match host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-host-secrets-audit.service",
|
|
])
|
|
.await
|
|
{
|
|
Ok(status) if status.success() => {
|
|
info!("Enabled archipelago-host-secrets-audit.service from OTA runtime payload")
|
|
}
|
|
Ok(status) => tracing::warn!(
|
|
"enabling archipelago-host-secrets-audit.service exited with {}",
|
|
status
|
|
),
|
|
Err(e) => tracing::warn!(
|
|
"failed to enable archipelago-host-secrets-audit.service: {}",
|
|
e
|
|
),
|
|
}
|
|
}
|
|
if nginx_src.exists() {
|
|
match host_sudo(&["nginx", "-t"]).await {
|
|
Ok(status) if status.success() => {
|
|
let _ = host_sudo(&["systemctl", "reload", "nginx"]).await;
|
|
}
|
|
Ok(status) => {
|
|
tracing::warn!("nginx config test failed after runtime sync: {}", status);
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!("failed to test nginx config after runtime sync: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn replace_dir_from_runtime(src: &Path, dest: &str) -> Result<()> {
|
|
let tmp = format!("{}.new.{}", dest, chrono::Utc::now().timestamp_millis());
|
|
let src_dot = path_dot(src);
|
|
let mkdir = host_sudo(&["mkdir", "-p", &tmp])
|
|
.await
|
|
.with_context(|| format!("mkdir {}", tmp))?;
|
|
if !mkdir.success() {
|
|
anyhow::bail!("mkdir {} exited with {}", tmp, mkdir);
|
|
}
|
|
let copy = host_sudo(&["cp", "-a", &src_dot, &tmp])
|
|
.await
|
|
.with_context(|| format!("copy runtime {} -> {}", src.display(), tmp))?;
|
|
if !copy.success() {
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
anyhow::bail!("copy runtime {} exited with {}", src.display(), copy);
|
|
}
|
|
let _ = host_sudo(&["mkdir", "-p", dest]).await;
|
|
let cleanup = host_sudo(&[
|
|
"find",
|
|
dest,
|
|
"-mindepth",
|
|
"1",
|
|
"-maxdepth",
|
|
"1",
|
|
"-exec",
|
|
"rm",
|
|
"-rf",
|
|
"{}",
|
|
"+",
|
|
])
|
|
.await
|
|
.with_context(|| format!("clean {}", dest))?;
|
|
if !cleanup.success() {
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
anyhow::bail!("clean {} exited with {}", dest, cleanup);
|
|
}
|
|
let tmp_dot = format!("{}/.", tmp);
|
|
let promote = host_sudo(&["cp", "-a", &tmp_dot, dest])
|
|
.await
|
|
.with_context(|| format!("promote {} -> {}", tmp, dest))?;
|
|
let _ = host_sudo(&["rm", "-rf", &tmp]).await;
|
|
if !promote.success() {
|
|
anyhow::bail!("promote {} exited with {}", dest, promote);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn path_dot(path: &Path) -> String {
|
|
let mut p = PathBuf::from(path);
|
|
p.push(".");
|
|
p.to_string_lossy().to_string()
|
|
}
|
|
|
|
/// ISO installs before the auto-install.sh path fix copied the app manifests
|
|
/// to /etc/archipelago/apps while the backend loads them from
|
|
/// /opt/archipelago/apps — so fresh nodes had ZERO disk manifests and only
|
|
/// catalog-covered apps could install (netbird "manifests not available",
|
|
/// framework node 2026-07-14). Self-heal: when /opt has no manifests and the
|
|
/// installer copy exists, populate /opt from /etc. Never overwrites existing
|
|
/// /opt manifests (OTA runtime-assets sync owns those afterwards).
|
|
async fn run_apps_dir_repair() -> Result<bool> {
|
|
let script = r#"
|
|
set -eu
|
|
src=/etc/archipelago/apps
|
|
dst=/opt/archipelago/apps
|
|
[ -d "$src" ] || exit 0
|
|
# Only heal when the destination has no manifests at all.
|
|
if [ -d "$dst" ] && [ -n "$(ls -A "$dst" 2>/dev/null)" ]; then exit 0; fi
|
|
ls "$src"/*/manifest.yml >/dev/null 2>&1 || exit 0
|
|
mkdir -p "$dst"
|
|
cp -r "$src"/. "$dst"/
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("populate /opt/archipelago/apps from installer copy")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("Apps dir repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Self-heal Wi-Fi setup on nodes that predate the polkit fix (issue #99).
|
|
///
|
|
/// Archipelago drives NetworkManager from a system-level systemd service
|
|
/// (`User=archipelago`, no logind seat), so the stock NM polkit rule — which
|
|
/// only authorizes `subject.local && subject.active` sessions — denies it, and
|
|
/// "connect to Wi-Fi" fails with "Insufficient privileges". Fresh ISO installs
|
|
/// since 2026-05 ship the rule below, but nodes that reached this build over
|
|
/// OTA never got it (OTA replaces the binary + web UI, not host system config).
|
|
///
|
|
/// Install the scoped rule if it is missing, and best-effort ensure `polkitd`
|
|
/// itself is present (without the daemon the rule is inert). Both are wrapped
|
|
/// so an offline/locked apt or a missing package can never fail startup — the
|
|
/// rule is still written so it takes effect once polkitd arrives (e.g. after an
|
|
/// ISO reflash). Idempotent: keyed on the rule's unique `subject.user` marker.
|
|
async fn run_polkit_networkmanager_repair() -> Result<bool> {
|
|
let script = r#"
|
|
set -u
|
|
RULE=/etc/polkit-1/rules.d/49-archipelago-networkmanager.rules
|
|
MARKER='subject.user == "archipelago"'
|
|
# Rule already installed — nothing to do.
|
|
if grep -qF "$MARKER" "$RULE" 2>/dev/null; then
|
|
exit 0
|
|
fi
|
|
# The rule is inert without the polkit daemon. Older nodes (the ones that hit
|
|
# issue #99) shipped without it. Try to install it, but never let apt failure
|
|
# (offline node, locked dpkg, package unavailable) abort the heal — the rule is
|
|
# written regardless so it activates whenever polkitd lands.
|
|
if [ ! -d /usr/share/polkit-1 ] && ! command -v pkaction >/dev/null 2>&1; then
|
|
timeout 240 apt-get install -y --no-install-recommends polkitd >/dev/null 2>&1 \
|
|
|| timeout 240 sh -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends polkitd >/dev/null 2>&1' \
|
|
|| true
|
|
fi
|
|
mkdir -p /etc/polkit-1/rules.d
|
|
cat > "$RULE" <<'RULEEOF'
|
|
polkit.addRule(function(action, subject) {
|
|
if (subject.user == "archipelago" && action.id.indexOf("org.freedesktop.NetworkManager.") == 0) {
|
|
return polkit.Result.YES;
|
|
}
|
|
});
|
|
RULEEOF
|
|
chmod 644 "$RULE"
|
|
# Pick up the new rule. polkitd re-reads rules.d on reload; restart as a
|
|
# fallback. Non-fatal if the unit name differs or the daemon is absent.
|
|
systemctl reload polkit 2>/dev/null \
|
|
|| systemctl restart polkit 2>/dev/null \
|
|
|| systemctl restart polkit.service 2>/dev/null \
|
|
|| true
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("install NetworkManager polkit rule")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("polkit NetworkManager repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn run_bitcoin_rpc_repair() -> Result<bool> {
|
|
// bitcoind is launched with -conf=/tmp/rpc.conf and never reads a
|
|
// datadir bitcoin.conf (apps/bitcoin-core & bitcoin-knots manifest.yml,
|
|
// commit a597c1d9 — bind/port live only on the container command line).
|
|
// A leftover file from an older install makes Bitcoin Core's own
|
|
// datadir-conflict safety check refuse to start on every subsequent
|
|
// start. Remove it instead of "repairing" it into existence — this
|
|
// previously wrote server=/rpcbind=/rpcallowip=/listen= into the file,
|
|
// which is exactly what caused the conflict.
|
|
let script = r#"
|
|
set -eu
|
|
conf=/var/lib/archipelago/bitcoin/bitcoin.conf
|
|
[ -f "$conf" ] || exit 0
|
|
mv "$conf" "$conf.disabled-$(date +%s)"
|
|
exit 2
|
|
"#;
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|
.await
|
|
.context("remove stale bitcoin.conf RPC bind settings")?;
|
|
match status.code() {
|
|
Some(0) => Ok(false),
|
|
// Do not restart Bitcoin from bootstrap. During IBD, an automatic
|
|
// restart can cost hours of progress. Removing the stale file is
|
|
// only a fallback for future starts; current containers keep their
|
|
// command-line RPC args regardless.
|
|
Some(2) => Ok(true),
|
|
_ => {
|
|
warn!("Bitcoin RPC repair helper exited with {}", status);
|
|
Ok(false)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Install the journald log-volume policy drop-in (JOURNALD_DROPIN) so nodes
|
|
/// deployed before the ISO shipped it get the size cap + rate limit via OTA.
|
|
/// Idempotent; restarts journald only when the file actually changed (safe:
|
|
/// the sockets are held by pid1, so at most a few messages queue briefly).
|
|
async fn run_journald_dropin() -> Result<bool> {
|
|
// Same dev-box guards as the doctor bootstrap: never touch /etc on
|
|
// contributors' laptops (symlinked or absent /home/archipelago/archy).
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
debug!("/home/archipelago/archy is a symlink — skipping journald bootstrap (dev box)");
|
|
return Ok(false);
|
|
}
|
|
if fs::metadata(home_archy).await.is_err() {
|
|
debug!("/home/archipelago/archy missing — skipping journald bootstrap");
|
|
return Ok(false);
|
|
}
|
|
|
|
let dropin_dir = "/etc/systemd/journald.conf.d";
|
|
let status = host_sudo(&["mkdir", "-p", dropin_dir])
|
|
.await
|
|
.with_context(|| format!("mkdir {}", dropin_dir))?;
|
|
if !status.success() {
|
|
anyhow::bail!("mkdir {} exited with {}", dropin_dir, status);
|
|
}
|
|
|
|
let changed = write_root_if_needed(JOURNALD_DROPIN_PATH, JOURNALD_DROPIN).await?;
|
|
if changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "restart", "systemd-journald"]).await {
|
|
warn!("journald restart after drop-in update failed: {:#}", e);
|
|
}
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn run() -> Result<bool> {
|
|
// Dev-box guard: on contributors' laptops `/home/archipelago/archy` is
|
|
// typically a symlink into the git checkout, and writing through it
|
|
// would clobber the working tree with whatever the binary happens to
|
|
// have been compiled from. Production ISO installs materialize a real
|
|
// directory.
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
debug!("/home/archipelago/archy is a symlink — skipping doctor bootstrap (dev box)");
|
|
return Ok(false);
|
|
}
|
|
|
|
// Skip entirely on machines without the canonical scripts directory —
|
|
// writing orphan files there just causes confusion.
|
|
let scripts_dir = Path::new(DOCTOR_SH_PATH)
|
|
.parent()
|
|
.context("doctor script path has no parent")?;
|
|
if !scripts_dir.exists() {
|
|
debug!(
|
|
"Scripts dir {} missing — skipping doctor bootstrap",
|
|
scripts_dir.display()
|
|
);
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
|
|
// 1. Script — lives in archipelago's home dir, user-writable.
|
|
if needs_write(DOCTOR_SH_PATH, DOCTOR_SH).await {
|
|
fs::write(DOCTOR_SH_PATH, DOCTOR_SH)
|
|
.await
|
|
.with_context(|| format!("write {}", DOCTOR_SH_PATH))?;
|
|
let _ = tokio::process::Command::new("chmod")
|
|
.args(["+x", DOCTOR_SH_PATH])
|
|
.status()
|
|
.await;
|
|
info!("Updated {}", DOCTOR_SH_PATH);
|
|
changed = true;
|
|
}
|
|
|
|
// 2. Systemd unit files — /etc is restricted; route through host_sudo.
|
|
let service_changed = write_root_if_needed(DOCTOR_SERVICE_PATH, DOCTOR_SERVICE).await?;
|
|
let timer_changed = write_root_if_needed(DOCTOR_TIMER_PATH, DOCTOR_TIMER).await?;
|
|
changed = changed || service_changed || timer_changed;
|
|
|
|
// 3. Reload if units changed. Do not enable/start the timer here: lifecycle
|
|
// qualification and explicit app operations need deterministic Podman
|
|
// ownership, and the doctor can race those flows. Operators can enable it
|
|
// separately when they want periodic host repair.
|
|
if service_changed || timer_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
|
|
Ok(changed)
|
|
}
|
|
|
|
async fn needs_write(path: &str, expected: &str) -> bool {
|
|
match fs::read_to_string(path).await {
|
|
Ok(current) => current != expected,
|
|
Err(_) => true,
|
|
}
|
|
}
|
|
|
|
/// Write content to a root-owned path via `sudo mv` of a user-owned tmp file.
|
|
/// Returns true if a write happened.
|
|
async fn write_root_if_needed(path: &str, content: &str) -> Result<bool> {
|
|
if !needs_write(path, content).await {
|
|
return Ok(false);
|
|
}
|
|
let tmp = format!(
|
|
"/tmp/archipelago-bootstrap-{}-{}.tmp",
|
|
std::process::id(),
|
|
Path::new(path)
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.unwrap_or("unit")
|
|
);
|
|
fs::write(&tmp, content)
|
|
.await
|
|
.with_context(|| format!("write tmp {}", tmp))?;
|
|
let status = host_sudo(&["mv", &tmp, path])
|
|
.await
|
|
.with_context(|| format!("sudo mv {} -> {}", tmp, path))?;
|
|
if !status.success() {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
anyhow::bail!("sudo mv to {} exited with {}", path, status);
|
|
}
|
|
info!("Updated {}", path);
|
|
Ok(true)
|
|
}
|
|
|
|
const ARCHIPELAGO_SERVICE_PATH: &str = "/etc/systemd/system/archipelago.service";
|
|
const MOUNT_REQUIRE_LINE: &str = "RequiresMountsFor=/var/lib/archipelago";
|
|
|
|
/// B17 self-heal: ensure the installed archipelago.service waits for the data
|
|
/// volume to mount before it starts. On production nodes `/var/lib/archipelago`
|
|
/// (the app data dir AND podman's graphroot) is a separate device-mapper volume;
|
|
/// without a mount dependency the service can start before `var-lib-archipelago.mount`,
|
|
/// write to the bare mountpoint on rootfs, fail every podman call, exit, and be
|
|
/// restarted every 5s until the volume mounts (~5 min of "[FAILED] Failed to start"
|
|
/// on cold boots). Fresh ISOs already ship the directive; this heals already-deployed
|
|
/// nodes. The change is boot-ordering only — it takes effect on the NEXT reboot, so we
|
|
/// never restart the running service here. Idempotent; no-op if the unit is absent
|
|
/// (dev runs) or already patched. Harmless when the data dir is on rootfs (systemd maps
|
|
/// the requirement to the always-mounted root).
|
|
pub async fn ensure_archipelago_mount_ordering() {
|
|
let current = match fs::read_to_string(ARCHIPELAGO_SERVICE_PATH).await {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
tracing::debug!(
|
|
"mount-ordering self-heal: {} not readable ({}) — skipping",
|
|
ARCHIPELAGO_SERVICE_PATH,
|
|
e
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
if current.contains(MOUNT_REQUIRE_LINE) {
|
|
return; // already healed
|
|
}
|
|
// Insert the directive into the [Unit] section, immediately before [Service].
|
|
let Some(idx) = current.find("\n[Service]") else {
|
|
tracing::warn!(
|
|
"mount-ordering self-heal: no [Service] section in {} — skipping",
|
|
ARCHIPELAGO_SERVICE_PATH
|
|
);
|
|
return;
|
|
};
|
|
let mut patched = String::with_capacity(current.len() + MOUNT_REQUIRE_LINE.len() + 96);
|
|
patched.push_str(¤t[..idx]);
|
|
patched.push_str("\n# B17: start only after the data volume (+ podman graphroot) is mounted\n");
|
|
patched.push_str(MOUNT_REQUIRE_LINE);
|
|
patched.push_str(¤t[idx..]);
|
|
match write_root_if_needed(ARCHIPELAGO_SERVICE_PATH, &patched).await {
|
|
Ok(true) => {
|
|
info!(
|
|
"B17: added '{}' to archipelago.service (effective next reboot)",
|
|
MOUNT_REQUIRE_LINE
|
|
);
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
tracing::warn!("B17 self-heal: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
Ok(false) => {}
|
|
Err(e) => tracing::warn!("B17 mount-ordering self-heal failed: {:#}", e),
|
|
}
|
|
}
|
|
|
|
/// #36 self-heal: keep the kiosk unit + launcher current on already-deployed
|
|
/// nodes so the CPU/memory cap (a runaway chromium was saturating the node and
|
|
/// starving the backend) and the GPU-vs-headless flag selection arrive via OTA.
|
|
/// No-op on nodes without the kiosk installed; only restarts the kiosk if it's
|
|
/// actually running (so it never re-enables an operator-disabled kiosk).
|
|
pub async fn ensure_kiosk_hardened() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return; // kiosk not installed on this node
|
|
}
|
|
let svc_changed = write_root_if_needed(KIOSK_SERVICE_PATH, KIOSK_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
let launcher_changed = write_root_if_needed(KIOSK_LAUNCHER_PATH, KIOSK_LAUNCHER)
|
|
.await
|
|
.unwrap_or(false);
|
|
if launcher_changed {
|
|
let _ = host_sudo(&["chmod", "+x", KIOSK_LAUNCHER_PATH]).await;
|
|
}
|
|
if svc_changed || launcher_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("kiosk hardening: daemon-reload failed: {:#}", e);
|
|
}
|
|
// try-restart only restarts a currently-active unit — leaves a stopped/
|
|
// disabled kiosk alone.
|
|
let _ = host_sudo(&["systemctl", "try-restart", "archipelago-kiosk.service"]).await;
|
|
info!("kiosk: applied resource cap + GPU-flag hardening (#36)");
|
|
}
|
|
}
|
|
|
|
/// HDMI-audio self-heal for kiosk nodes: install the PipeWire stack (older
|
|
/// ISOs shipped none), put the archipelago user in `audio` (PipeWire runs
|
|
/// under the lingering user manager — no logind seat, so no udev ACLs on
|
|
/// /dev/snd), and keep the audio-router daemon (HDMI routing + ELD boot-race
|
|
/// nudge) installed and current. No-op on nodes without the kiosk — audio
|
|
/// only matters where media plays on an attached display.
|
|
pub async fn ensure_audio_stack() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return; // no kiosk → no display audio to route
|
|
}
|
|
|
|
// Package install runs via systemd-run (host_sudo), outside the service
|
|
// sandbox — /usr and the dpkg database are read-only in our namespace.
|
|
if fs::metadata("/usr/bin/pactl").await.is_err() {
|
|
info!("audio: PipeWire stack missing — installing packages");
|
|
let _ = host_sudo(&["apt-get", "update", "-qq"]).await;
|
|
match host_sudo(&[
|
|
"apt-get",
|
|
"install",
|
|
"-y",
|
|
"-qq",
|
|
"--no-install-recommends",
|
|
"pipewire",
|
|
"pipewire-pulse",
|
|
"pipewire-alsa",
|
|
"wireplumber",
|
|
"alsa-utils",
|
|
])
|
|
.await
|
|
{
|
|
Ok(s) if s.success() => info!("audio: PipeWire stack installed"),
|
|
Ok(s) => {
|
|
warn!(
|
|
"audio: package install exited with {} — will retry next start",
|
|
s
|
|
);
|
|
return;
|
|
}
|
|
Err(e) => {
|
|
warn!(
|
|
"audio: package install failed: {:#} — will retry next start",
|
|
e
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
let _ = host_sudo(&["usermod", "-aG", "audio", "archipelago"]).await;
|
|
|
|
let unit_was_missing = fs::metadata(AUDIO_SERVICE_PATH).await.is_err();
|
|
let script_changed = write_root_if_needed(AUDIO_ROUTER_PATH, AUDIO_ROUTER)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed {
|
|
let _ = host_sudo(&["chmod", "+x", AUDIO_ROUTER_PATH]).await;
|
|
}
|
|
let unit_changed = write_root_if_needed(AUDIO_SERVICE_PATH, AUDIO_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
|
|
if script_changed || unit_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("audio: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
if unit_was_missing {
|
|
// First install on this node — bring it up now and on every boot.
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-audio-router.service",
|
|
])
|
|
.await;
|
|
info!("audio: router installed and enabled (HDMI routing + ELD heal)");
|
|
} else if script_changed || unit_changed {
|
|
// Content update: restart only if it's running — never re-enable a
|
|
// unit an operator deliberately disabled.
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"try-restart",
|
|
"archipelago-audio-router.service",
|
|
])
|
|
.await;
|
|
info!("audio: router updated");
|
|
}
|
|
}
|
|
|
|
/// Gamepad→keyboard bridge self-heal for kiosk nodes: keeps the evdev→uinput
|
|
/// daemon (controller works inside every app iframe on the TV) installed and
|
|
/// current. Same gating as audio: no kiosk → no display input to bridge.
|
|
pub async fn ensure_gamepad_keys() {
|
|
if fs::metadata(KIOSK_SERVICE_PATH).await.is_err() {
|
|
return;
|
|
}
|
|
let unit_was_missing = fs::metadata(GAMEPAD_SERVICE_PATH).await.is_err();
|
|
let script_changed = write_root_if_needed(GAMEPAD_KEYS_PATH, GAMEPAD_KEYS)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed {
|
|
let _ = host_sudo(&["chmod", "+x", GAMEPAD_KEYS_PATH]).await;
|
|
}
|
|
let unit_changed = write_root_if_needed(GAMEPAD_SERVICE_PATH, GAMEPAD_SERVICE)
|
|
.await
|
|
.unwrap_or(false);
|
|
if script_changed || unit_changed {
|
|
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
|
|
warn!("gamepad bridge: daemon-reload failed: {:#}", e);
|
|
}
|
|
}
|
|
if unit_was_missing {
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"enable",
|
|
"--now",
|
|
"archipelago-gamepad-keys.service",
|
|
])
|
|
.await;
|
|
info!("gamepad: bridge installed and enabled (TV controller input)");
|
|
} else if script_changed || unit_changed {
|
|
let _ = host_sudo(&[
|
|
"systemctl",
|
|
"try-restart",
|
|
"archipelago-gamepad-keys.service",
|
|
])
|
|
.await;
|
|
info!("gamepad: bridge updated");
|
|
}
|
|
}
|
|
|
|
/// Patch the nginx site config to add missing backend proxy blocks. Older ISO
|
|
/// configs shipped individual per-endpoint `location` blocks, so missing
|
|
/// endpoints silently fell through to the SPA `index.html` and the frontend
|
|
/// got HTML instead of JSON.
|
|
///
|
|
/// Validates via `nginx -t` before reloading. On failure the patch is
|
|
/// rolled back from a backup written just before the write.
|
|
async fn run_nginx() -> Result<bool> {
|
|
// Skip on dev symlinks — we don't want to touch `/etc/nginx` on laptops.
|
|
let home_archy = Path::new("/home/archipelago/archy");
|
|
if fs::symlink_metadata(home_archy)
|
|
.await
|
|
.map(|m| m.file_type().is_symlink())
|
|
.unwrap_or(false)
|
|
{
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut changed = false;
|
|
let mut patched_paths = Vec::<PathBuf>::new();
|
|
for path in [
|
|
NGINX_CONF_PATH,
|
|
NGINX_ENABLED_CONF_PATH,
|
|
NGINX_HTTPS_SNIPPET_PATH,
|
|
] {
|
|
let candidate = Path::new(path);
|
|
if !candidate.exists() {
|
|
debug!("{} missing — skipping nginx bootstrap", path);
|
|
continue;
|
|
}
|
|
let canonical = fs::canonicalize(candidate)
|
|
.await
|
|
.unwrap_or_else(|_| candidate.to_path_buf());
|
|
if patched_paths.iter().any(|p| p == &canonical) {
|
|
continue;
|
|
}
|
|
patched_paths.push(canonical);
|
|
changed |= patch_nginx_conf(path).await?;
|
|
}
|
|
Ok(changed)
|
|
}
|
|
|
|
/// Reflective CORS add_headers that older configs placed inside the
|
|
/// `/lnd-connect-info` location. The backend now sets a validated
|
|
/// `Access-Control-Allow-Origin` for that endpoint (api/handler/proxy.rs), so
|
|
/// leaving these in nginx emits a DUPLICATE header ("contains multiple values
|
|
/// … but only one is allowed") and the LND wallet UI's cross-origin fetch is
|
|
/// rejected. Stripped during nginx bootstrap so the backend solely owns CORS.
|
|
const NGINX_LND_DUP_CORS: &str = " add_header Access-Control-Allow-Origin $http_origin always;\n add_header Access-Control-Allow-Credentials \"true\" always;\n";
|
|
|
|
async fn patch_nginx_conf(path: &str) -> Result<bool> {
|
|
let content = fs::read_to_string(path)
|
|
.await
|
|
.with_context(|| format!("read {}", path))?;
|
|
// Each "missing" flag is gated on the splice anchor actually being present,
|
|
// so an included snippet that legitimately has none of these endpoints (the
|
|
// HTTPS app-proxy snippet) neither tries to patch them nor logs warn-skips on
|
|
// every boot — it falls through to the fedimint heal alone.
|
|
let has_lnd_anchor = content.contains(" location /lnd-connect-info {")
|
|
|| content.contains(" location /electrs-status {");
|
|
let missing_app_catalog = content
|
|
.contains(" # DWN endpoints — peer access over Tor (no auth)")
|
|
&& !content.contains("location /api/app-catalog");
|
|
let missing_bitcoin_status = content.contains(" location /electrs-status {")
|
|
&& !content.contains("location /bitcoin-status");
|
|
let missing_lnd_proxy = has_lnd_anchor && !content.contains("location /proxy/lnd/");
|
|
let missing_peer_content = has_lnd_anchor && !content.contains("location /api/peer-content");
|
|
let missing_pine_status = has_lnd_anchor && !content.contains("location /api/pine/status");
|
|
let has_lnd_dup_cors = content.contains(NGINX_LND_DUP_CORS);
|
|
// B13: fedimint block present but lacking the asset-rewrite sub_filters.
|
|
let needs_fedimint_css = content.contains("location /app/fedimint/")
|
|
&& !content.contains("'href=\"/' 'href=\"/app/fedimint/'");
|
|
// Companion mesh access: phones reach this node over FIPS at its fips0
|
|
// ULA (http://[fdxx:…]). Configs shipped before 2026-07-23 listened on
|
|
// IPv4 only, so the ULA could never connect — nothing answered [::]:80.
|
|
let missing_v6_http =
|
|
content.contains("listen 80 default_server;") && !content.contains("listen [::]:80");
|
|
let missing_v6_https =
|
|
content.contains("listen 443 ssl default_server;") && !content.contains("listen [::]:443");
|
|
if !missing_app_catalog
|
|
&& !missing_bitcoin_status
|
|
&& !missing_lnd_proxy
|
|
&& !missing_peer_content
|
|
&& !missing_pine_status
|
|
&& !has_lnd_dup_cors
|
|
&& !needs_fedimint_css
|
|
&& !missing_v6_http
|
|
&& !missing_v6_https
|
|
{
|
|
return Ok(false);
|
|
}
|
|
|
|
let mut patched = content.clone();
|
|
|
|
if missing_v6_http {
|
|
patched = patched.replace(
|
|
"listen 80 default_server;",
|
|
"listen 80 default_server;\n listen [::]:80 default_server;",
|
|
);
|
|
}
|
|
if missing_v6_https {
|
|
patched = patched.replace(
|
|
"listen 443 ssl default_server;",
|
|
"listen 443 ssl default_server;\n listen [::]:443 ssl default_server;",
|
|
);
|
|
}
|
|
|
|
if has_lnd_dup_cors {
|
|
// Drop the redundant nginx-side CORS headers so the backend's single
|
|
// validated Access-Control-Allow-Origin is the only one returned.
|
|
patched = patched.replace(NGINX_LND_DUP_CORS, "");
|
|
}
|
|
|
|
if needs_fedimint_css {
|
|
// Style A (main conf): the block already injects nostr-provider, so swap
|
|
// its single-sub_filter tail for the full asset-rewrite set. No-op if the
|
|
// node's fedimint block doesn't match OLD.
|
|
patched = patched.replace(NGINX_FEDIMINT_OLD, NGINX_FEDIMINT_NEW);
|
|
// Style B (HTTPS app-proxy snippet): the block has no sub_filter to swap,
|
|
// so insert the reroot set after the unique :8175 proxy_pass. Guarded on
|
|
// the marker so it can never double-apply after Style A already healed.
|
|
if !patched.contains("'href=\"/' 'href=\"/app/fedimint/'") {
|
|
patched = patched.replace(NGINX_FEDIMINT_SNIPPET_ANCHOR, NGINX_FEDIMINT_SNIPPET_INSERT);
|
|
}
|
|
}
|
|
|
|
if missing_lnd_proxy {
|
|
// Prefer the `/lnd-connect-info` anchor (present since 2026-03-17); fall
|
|
// back to `/electrs-status` (since 2026-03-08) for even older configs.
|
|
// Both appear once per archipelago server block, so the block is added
|
|
// to every server block that proxies to the backend.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing lnd-connect-info/electrs-status anchor — skipping /proxy/lnd patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_LND_PROXY_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if missing_pine_status {
|
|
// Same anchoring as the LND proxy: prepend to every server block that
|
|
// proxies to the backend.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if patched.contains(anchor) {
|
|
let replacement = format!("{}{}", NGINX_PINE_STATUS_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
} else {
|
|
warn!("nginx conf missing anchor — skipping /api/pine/status patch");
|
|
}
|
|
}
|
|
|
|
if missing_peer_content {
|
|
// Same anchoring as the LND proxy: prepend the block to every server
|
|
// block so /api/peer-content/* reaches the backend instead of the SPA.
|
|
let anchor = if patched.contains(" location /lnd-connect-info {") {
|
|
" location /lnd-connect-info {"
|
|
} else {
|
|
" location /electrs-status {"
|
|
};
|
|
if patched.contains(anchor) {
|
|
let replacement = format!("{}{}", NGINX_PEER_CONTENT_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
} else {
|
|
warn!("nginx conf missing anchor — skipping /api/peer-content patch");
|
|
}
|
|
}
|
|
|
|
if missing_bitcoin_status {
|
|
let anchor = " location /electrs-status {";
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing electrs-status anchor — skipping /bitcoin-status patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_BITCOIN_STATUS_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if missing_app_catalog {
|
|
// The DWN comment sits at the same indent right after the `/api/blob`
|
|
// block in both server blocks — a stable anchor that existed on every
|
|
// ISO shipped to date. If it's absent (config got heavily customized),
|
|
// skip rather than guess where to splice.
|
|
let anchor = " # DWN endpoints — peer access over Tor (no auth)";
|
|
if !patched.contains(anchor) {
|
|
warn!("nginx conf missing DWN anchor — skipping /api/app-catalog patch");
|
|
} else {
|
|
let replacement = format!("{}{}", NGINX_APP_CATALOG_BLOCK, anchor);
|
|
patched = patched.replace(anchor, &replacement);
|
|
}
|
|
}
|
|
|
|
if patched == content {
|
|
return Ok(false);
|
|
}
|
|
|
|
// Write patched config via a user-owned tmp + sudo mv, after stashing
|
|
// a backup outside nginx include dirs so validation cannot load it too.
|
|
let pid = std::process::id();
|
|
let tmp = format!("/tmp/archipelago-nginx-{}.conf", pid);
|
|
fs::write(&tmp, &patched)
|
|
.await
|
|
.with_context(|| format!("write {}", tmp))?;
|
|
|
|
let backup = format!(
|
|
"/tmp/archipelago-nginx-backup-{}-{}.conf",
|
|
pid,
|
|
patched.len()
|
|
);
|
|
if let Err(e) = host_sudo(&["cp", path, &backup]).await {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
return Err(e.context("backup nginx conf"));
|
|
}
|
|
|
|
let mv = host_sudo(&["mv", &tmp, path]).await;
|
|
match mv {
|
|
Ok(s) if s.success() => {}
|
|
Ok(s) => {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
anyhow::bail!("sudo mv nginx conf to {} exited with {}", path, s);
|
|
}
|
|
Err(e) => {
|
|
let _ = fs::remove_file(&tmp).await;
|
|
return Err(e.context("mv tmp -> nginx conf"));
|
|
}
|
|
}
|
|
|
|
// Validate.
|
|
let test = host_sudo(&["nginx", "-t"]).await;
|
|
let valid = matches!(&test, Ok(s) if s.success());
|
|
if !valid {
|
|
warn!("nginx -t failed after patch — reverting");
|
|
let _ = host_sudo(&["mv", &backup, path]).await;
|
|
if let Err(e) = test {
|
|
return Err(e.context("nginx -t"));
|
|
}
|
|
anyhow::bail!("nginx config invalid after patch — reverted");
|
|
}
|
|
|
|
// Reload nginx so the new block takes effect immediately. Reload (not
|
|
// restart) keeps in-flight connections alive.
|
|
if let Err(e) = host_sudo(&["systemctl", "reload", "nginx"]).await {
|
|
warn!("nginx reload failed (non-fatal): {:#}", e);
|
|
}
|
|
let _ = host_sudo(&["rm", "-f", &backup]).await;
|
|
Ok(true)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn podman_heal_outcome_no_longer_has_cleanup_variant() {
|
|
let outcome = PodmanHealOutcome::Unhealthy;
|
|
assert_ne!(outcome, PodmanHealOutcome::Healthy);
|
|
}
|
|
}
|
|
|
|
/// Repair this node's own systemd restart policy.
|
|
///
|
|
/// The in-process updater replaces the binary and then asks systemd to
|
|
/// restart the service, treating `Restart=always` on the unit as its second
|
|
/// net if that request is ever lost. On austin-sapien (2026-08-05) the unit
|
|
/// was an old one carrying `Restart=on-failure`: the daemon exited cleanly
|
|
/// (status 0), systemd read that as success, and the node sat dead for over
|
|
/// two hours after a routine update — "server starting" in the UI, with
|
|
/// nothing to start it.
|
|
///
|
|
/// A node cannot be relied on to fix this via `self-update.sh` (which does
|
|
/// refresh units) because the in-process update path never runs it. So the
|
|
/// daemon checks its own unit at boot: any node that starts even once ends
|
|
/// up with a policy that survives the next update. Deliberately narrow —
|
|
/// only the `Restart=` line is touched, so local edits elsewhere in the unit
|
|
/// are preserved.
|
|
pub async fn ensure_restart_policy() {
|
|
const UNIT: &str = "/etc/systemd/system/archipelago.service";
|
|
let Ok(body) = fs::read_to_string(UNIT).await else {
|
|
return; // not a systemd install (container, dev box) — nothing to do
|
|
};
|
|
if !body.lines().any(|l| {
|
|
let l = l.trim();
|
|
l.starts_with("Restart=") && l != "Restart=always"
|
|
}) {
|
|
return; // already correct, or no Restart= line to repair
|
|
}
|
|
let patched: String = body
|
|
.lines()
|
|
.map(|l| {
|
|
if l.trim().starts_with("Restart=") && l.trim() != "Restart=always" {
|
|
"Restart=always"
|
|
} else {
|
|
l
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join("\n");
|
|
match write_root_if_needed(UNIT, &patched).await {
|
|
Ok(true) => {
|
|
tracing::warn!(
|
|
"repaired archipelago.service Restart= policy to always — this node would \
|
|
have stayed dead after an in-process update"
|
|
);
|
|
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
|
|
}
|
|
Ok(false) => {}
|
|
Err(e) => tracing::warn!(error = %e, "could not repair archipelago.service restart policy"),
|
|
}
|
|
}
|