diff --git a/core/archipelago/src/api/rpc/mesh/status.rs b/core/archipelago/src/api/rpc/mesh/status.rs index 688ca399..ade30aca 100644 --- a/core/archipelago/src/api/rpc/mesh/status.rs +++ b/core/archipelago/src/api/rpc/mesh/status.rs @@ -101,6 +101,21 @@ impl RpcHandler { detected.iter().any(|d| d == &path), "{path} is not a detected mesh-radio candidate port" ); + // Refuse to probe while a firmware flash is in flight. Confirmed + // live 2026-07-23: esptool ("multiple access on port?") and + // rnodeconf (OSError Errno 71 Protocol error on an RTS ioctl) both + // failed with symptoms consistent with a second process holding the + // same serial fd — the flash subprocess runs for minutes outside + // our own async runtime, so nothing previously stopped a concurrent + // `mesh.probe-device` call (e.g. the hot-swap modal's own re-probe) + // from opening the identical port at the same time and corrupting + // both operations' handshakes. + if let Some(job) = self.flash_job.read().await.as_ref() { + anyhow::ensure!( + job.snapshot().await.done, + "A firmware flash is in progress — refusing to probe the serial port until it finishes" + ); + } // Only hold the mesh_service lock long enough for the quick // active-path guard check — NEVER across the actual probe, which // can take 15-60s across its internal collision retries. Confirmed diff --git a/core/archipelago/src/mesh/flash.rs b/core/archipelago/src/mesh/flash.rs index 35d850dc..33a0f089 100644 --- a/core/archipelago/src/mesh/flash.rs +++ b/core/archipelago/src/mesh/flash.rs @@ -117,6 +117,34 @@ const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15 /// listener genuinely won't let go. const STOP_LISTENER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); +/// How long to keep retrying the port-free check before giving up. +const PORT_FREE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +/// Confirm nothing else has `path` open by actually opening (and immediately +/// closing) it ourselves. Retries across the timeout since a just-stopped +/// listener's fd can take a moment to actually release even after `stop()` +/// returns (task abort is a request, not an instant guarantee the OS-level +/// resource is gone yet). +async fn wait_for_port_free(path: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + PORT_FREE_TIMEOUT; + let mut last_err = None; + loop { + match serial2_tokio::SerialPort::open(path, 115200) { + Ok(_) => return Ok(()), + Err(e) => last_err = Some(e), + } + if tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + Err(anyhow::anyhow!( + "{path} is still held open by something else after {}s (last error: {}) — refusing to start the flasher against a contended port", + PORT_FREE_TIMEOUT.as_secs(), + last_err.map(|e| e.to_string()).unwrap_or_default() + )) +} + /// Live state for the one flash job that can run at a time. A single global /// slot is sufficient because flashing needs exclusive serial access to the /// one port being flashed — there is no meaningful concept of two concurrent @@ -352,6 +380,23 @@ pub async fn start_flash_job( return; } + // Belt-and-suspenders port-free check. `stop()` above should have + // fully released the port, but esptool/rnodeconf run as external + // subprocesses for minutes outside our own async runtime — if + // ANYTHING else still has it open (a racing probe, a not-yet-dropped + // fd from an aborted task, anything we haven't anticipated), handing + // the port to the flasher anyway risks exactly the corruption + // confirmed live 2026-07-23: esptool's "device disconnected or + // multiple access on port?" and rnodeconf's raw `OSError: [Errno 71] + // Protocol error` on an RTS ioctl are both textbook two-openers-on- + // one-fd symptoms. Verify by actually opening it ourselves — cheap, + // and definitive — before ever starting the flasher. + if let Err(e) = wait_for_port_free(&path).await { + bg_job.push_log(format!("ERROR: {e:#}")).await; + bg_job.fail(&e).await; + return; + } + // Outer ceiling on top of run_flash's own internal timeouts — // belt-and-suspenders so that no future hang (network, subprocess, // anything) can ever wedge the single-flash-job guard permanently diff --git a/core/archipelago/src/mesh/listener/session.rs b/core/archipelago/src/mesh/listener/session.rs index 9db19b43..9792bbeb 100644 --- a/core/archipelago/src/mesh/listener/session.rs +++ b/core/archipelago/src/mesh/listener/session.rs @@ -274,6 +274,7 @@ async fn auto_detect_and_open( if paths.is_empty() { anyhow::bail!("No serial devices found in /dev"); } + info!(candidates = ?paths, "Auto-detect candidate ports for this attempt"); for path in &paths { debug!(path = %path, "Probing for mesh radio device"); // Tried FIRST: `ReticulumLink::open()` gates its expensive daemon @@ -487,40 +488,19 @@ async fn open_preferred_path( }; } - // Reticulum first — see the matching comment on auto_detect_and_open: - // its cheap probe_rnode gate fails in ~1s for non-RNode firmware, while - // trying Meshcore/Meshtastic first was observed leaving a real RNode - // board unresponsive by the time Reticulum's turn came. - match ReticulumLink::open( - path, - data_dir, - Some(our_ed_pubkey_hex), - Some(our_x25519_pubkey_hex), - ) - .await - { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => return Ok((MeshRadioDevice::Reticulum(dev), info)), - Err(e) => { - debug!(path = %path, error = %e, "Preferred path is not a working Reticulum RNode") - } - }, - Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Reticulum"), - } - match MeshcoreDevice::open(path).await { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => return Ok((MeshRadioDevice::Meshcore(dev), info)), - Err(e) => debug!(path = %path, error = %e, "Preferred path is not Meshcore"), - }, - Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Meshcore"), - } - match MeshtasticDevice::open(path).await { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => Ok((MeshRadioDevice::Meshtastic(dev), info)), - Err(e) => Err(e).context("Preferred path is not a working Meshtastic device"), - }, - Err(e) => Err(e).context("Could not open preferred path as Meshtastic"), - } + // Unpinned: don't probe this path ourselves at all. Confirmed live + // 2026-07-23 — this function used to run its own Reticulum→Meshcore→ + // Meshtastic sequence here, and the caller (run_mesh_session) falls + // back to `auto_detect_and_open` on any error, which scans every + // candidate path (this one included) with the exact same three-protocol + // sequence. With a single physical radio — the overwhelmingly common + // case — `path` here IS the one candidate `auto_detect_and_open` is + // about to try, so every unpinned reconnect was resetting the board via + // Reticulum/Meshcore/Meshtastic's DTR/RTS toggle TWICE: once here, once + // again moments later in auto-detect. Bailing immediately (no port + // access at all) means auto-detect's single pass is the only one that + // ever touches the port when nothing is pinned yet. + anyhow::bail!("No device_kind pin — deferring to auto-detect for {path}") } /// Bring up a Reticulum daemon over plain TCP — no physical RNode, no