From e898f138d115b05335df714200e60d553dae77dc Mon Sep 17 00:00:00 2001 From: archipelago Date: Sun, 9 Aug 2026 12:48:59 -0400 Subject: [PATCH] fix(bootstrap): deliver the tor-helper and nginx ws fix to EXISTING nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An OTA updates the binary and web assets — not /etc/nginx, and not /opt/archipelago/scripts. Auditing the 2026-08-09 fixes' delivery paths found two that would silently reach nobody already installed: - scripts/tor-helper.sh (reset-failed + truthful restart result) shipped only via ISO builds and manual deploys. Now embedded via include_str! like the doctor script, staged and installed at boot when the on-disk copy differs. - The /app/mempool/ nginx Upgrade/Connection headers existed only in repo snippet sources consumed at image build time. A boot repair now idempotently patches any mempool location block missing them — in the live vhost (archipelago-http, the one sites-enabled actually links to), the legacy file, and the installed snippet — and reloads nginx once. Without this, every fleet node's mempool UI keeps loading-but-never- connecting after the OTA that supposedly fixed it. Both are non-fatal boot repairs in the existing bootstrap chain, no-ops when everything is already current. Co-Authored-By: Claude Fable 5 --- core/archipelago/src/bootstrap.rs | 88 +++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/core/archipelago/src/bootstrap.rs b/core/archipelago/src/bootstrap.rs index f1bbfae6..40301666 100644 --- a/core/archipelago/src/bootstrap.rs +++ b/core/archipelago/src/bootstrap.rs @@ -166,11 +166,21 @@ pub async fn ensure_doctor_installed() { Ok(false) => debug!("/opt/archipelago/apps already populated (or no installer copy)"), Err(e) => warn!("Apps dir repair failed (non-fatal): {:#}", e), } + match run_tor_helper_sync().await { + Ok(true) => info!("tor-helper.sh synchronized with binary"), + Ok(false) => debug!("tor-helper.sh already current"), + Err(e) => warn!("tor-helper sync failed (non-fatal): {:#}", e), + } match run_tor_torrc_repair().await { Ok(true) => info!("Tor healed at boot (torrc rebuilt and/or daemon restarted)"), Ok(false) => debug!("Tor healthy and torrc in sync — no heal needed"), Err(e) => warn!("Tor boot heal failed (non-fatal): {:#}", e), } + match run_nginx_mempool_ws_repair().await { + Ok(true) => info!("nginx mempool websocket headers repaired and reloaded"), + Ok(false) => debug!("nginx mempool websocket headers already present"), + Err(e) => warn!("nginx mempool ws repair failed (non-fatal): {:#}", e), + } match run_polkit_networkmanager_repair().await { Ok(true) => info!( "Installed NetworkManager polkit rule for the archipelago user — Wi-Fi setup enabled" @@ -637,6 +647,84 @@ exit 2 /// /// Non-fatal and conservative: it only restarts Tor when the regenerated torrc /// differs from the live one, or Tor is not answering on 9050. +/// The privileged Tor helper, embedded so the OTA actually delivers it. +/// scripts/tor-helper.sh previously reached nodes only through ISO builds and +/// manual deploys — the 2026-08-09 helper fix (reset-failed + truthful result) +/// would have shipped to nobody. Same include_str! pattern as the doctor. +const TOR_HELPER_SH: &str = include_str!("../../../scripts/tor-helper.sh"); +const TOR_HELPER_PATH: &str = "/opt/archipelago/scripts/tor-helper.sh"; + +async fn run_tor_helper_sync() -> Result { + let current = tokio::fs::read_to_string(TOR_HELPER_PATH) + .await + .unwrap_or_default(); + if current == TOR_HELPER_SH { + return Ok(false); + } + let staged = "/var/lib/archipelago/tor-config/tor-helper.staged"; + if let Some(dir) = Path::new(staged).parent() { + tokio::fs::create_dir_all(dir).await.ok(); + } + tokio::fs::write(staged, TOR_HELPER_SH) + .await + .context("stage tor-helper.sh")?; + let script = format!( + "set -eu\ninstall -m 0755 {staged} {dest}\nexit 0\n", + staged = staged, + dest = TOR_HELPER_PATH + ); + host_sudo(&["sh", "-lc", &script]) + .await + .context("install tor-helper.sh")?; + Ok(true) +} + +/// Existing nodes' nginx configs never receive repo snippet fixes — the OTA +/// updates the binary and web assets, not /etc/nginx. The mempool UI is +/// websocket-driven, and every fleet node's /app/mempool/ proxy block strips +/// the Upgrade handshake, so the page loads and never connects (three-layer +/// outage, 2026-08-09). Idempotently add the two headers to any mempool block +/// missing them, in every nginx file that has one, then reload once. +async fn run_nginx_mempool_ws_repair() -> Result { + let script = r#" +set -eu +changed=0 +for f in /etc/nginx/sites-available/archipelago-http \ + /etc/nginx/sites-available/archipelago \ + /etc/nginx/snippets/archipelago-https-app-proxies.conf; do + [ -f "$f" ] || continue + grep -q 'location /app/mempool/' "$f" || continue + python3 - "$f" <<'PYEOF' +import re, sys +p = sys.argv[1] +src = open(p).read() +def fix(m): + b = m.group(0) + if 'Upgrade $http_upgrade' in b: + return b + return b.replace('proxy_http_version 1.1;', + 'proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection "upgrade";', 1) +new = re.sub(r'location /app/mempool/ \{[^}]*\}', fix, src, flags=re.S) +if new != src: + open(p, 'w').write(new) + sys.exit(3) +PYEOF + rc=$? + [ "$rc" = 3 ] && changed=1 + [ "$rc" = 0 ] || [ "$rc" = 3 ] || exit "$rc" +done +if [ "$changed" = 1 ]; then + nginx -t >/dev/null 2>&1 && systemctl reload nginx || true + exit 3 +fi +exit 0 +"#; + let status = host_sudo(&["sh", "-lc", script]) + .await + .context("nginx mempool ws repair")?; + Ok(status.code() == Some(3)) +} + async fn run_tor_torrc_repair() -> Result { // Same location the RPC handlers use (Config::data_dir); bootstrap runs // before the server owns a Config, and this path is fixed on real installs.