probing issues with flashing

This commit is contained in:
ssmithx 2026-07-23 23:43:09 +00:00
parent 078f3b3619
commit 8403f2233e
3 changed files with 74 additions and 34 deletions

View File

@ -101,6 +101,21 @@ impl RpcHandler {
detected.iter().any(|d| d == &path), detected.iter().any(|d| d == &path),
"{path} is not a detected mesh-radio candidate port" "{path} is not a detected mesh-radio candidate port"
); );
// Refuse to probe while a firmware flash is in flight. Confirmed
// live 2026-07-23: esptool ("multiple access on port?") and
// rnodeconf (OSError Errno 71 Protocol error on an RTS ioctl) both
// failed with symptoms consistent with a second process holding the
// same serial fd — the flash subprocess runs for minutes outside
// our own async runtime, so nothing previously stopped a concurrent
// `mesh.probe-device` call (e.g. the hot-swap modal's own re-probe)
// from opening the identical port at the same time and corrupting
// both operations' handshakes.
if let Some(job) = self.flash_job.read().await.as_ref() {
anyhow::ensure!(
job.snapshot().await.done,
"A firmware flash is in progress — refusing to probe the serial port until it finishes"
);
}
// Only hold the mesh_service lock long enough for the quick // Only hold the mesh_service lock long enough for the quick
// active-path guard check — NEVER across the actual probe, which // active-path guard check — NEVER across the actual probe, which
// can take 15-60s across its internal collision retries. Confirmed // can take 15-60s across its internal collision retries. Confirmed

View File

@ -117,6 +117,34 @@ const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15
/// listener genuinely won't let go. /// listener genuinely won't let go.
const STOP_LISTENER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); const STOP_LISTENER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
/// How long to keep retrying the port-free check before giving up.
const PORT_FREE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
/// Confirm nothing else has `path` open by actually opening (and immediately
/// closing) it ourselves. Retries across the timeout since a just-stopped
/// listener's fd can take a moment to actually release even after `stop()`
/// returns (task abort is a request, not an instant guarantee the OS-level
/// resource is gone yet).
async fn wait_for_port_free(path: &str) -> Result<()> {
let deadline = tokio::time::Instant::now() + PORT_FREE_TIMEOUT;
let mut last_err = None;
loop {
match serial2_tokio::SerialPort::open(path, 115200) {
Ok(_) => return Ok(()),
Err(e) => last_err = Some(e),
}
if tokio::time::Instant::now() >= deadline {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
Err(anyhow::anyhow!(
"{path} is still held open by something else after {}s (last error: {}) — refusing to start the flasher against a contended port",
PORT_FREE_TIMEOUT.as_secs(),
last_err.map(|e| e.to_string()).unwrap_or_default()
))
}
/// Live state for the one flash job that can run at a time. A single global /// Live state for the one flash job that can run at a time. A single global
/// slot is sufficient because flashing needs exclusive serial access to the /// slot is sufficient because flashing needs exclusive serial access to the
/// one port being flashed — there is no meaningful concept of two concurrent /// one port being flashed — there is no meaningful concept of two concurrent
@ -352,6 +380,23 @@ pub async fn start_flash_job(
return; return;
} }
// Belt-and-suspenders port-free check. `stop()` above should have
// fully released the port, but esptool/rnodeconf run as external
// subprocesses for minutes outside our own async runtime — if
// ANYTHING else still has it open (a racing probe, a not-yet-dropped
// fd from an aborted task, anything we haven't anticipated), handing
// the port to the flasher anyway risks exactly the corruption
// confirmed live 2026-07-23: esptool's "device disconnected or
// multiple access on port?" and rnodeconf's raw `OSError: [Errno 71]
// Protocol error` on an RTS ioctl are both textbook two-openers-on-
// one-fd symptoms. Verify by actually opening it ourselves — cheap,
// and definitive — before ever starting the flasher.
if let Err(e) = wait_for_port_free(&path).await {
bg_job.push_log(format!("ERROR: {e:#}")).await;
bg_job.fail(&e).await;
return;
}
// Outer ceiling on top of run_flash's own internal timeouts — // Outer ceiling on top of run_flash's own internal timeouts —
// belt-and-suspenders so that no future hang (network, subprocess, // belt-and-suspenders so that no future hang (network, subprocess,
// anything) can ever wedge the single-flash-job guard permanently // anything) can ever wedge the single-flash-job guard permanently

View File

@ -274,6 +274,7 @@ async fn auto_detect_and_open(
if paths.is_empty() { if paths.is_empty() {
anyhow::bail!("No serial devices found in /dev"); anyhow::bail!("No serial devices found in /dev");
} }
info!(candidates = ?paths, "Auto-detect candidate ports for this attempt");
for path in &paths { for path in &paths {
debug!(path = %path, "Probing for mesh radio device"); debug!(path = %path, "Probing for mesh radio device");
// Tried FIRST: `ReticulumLink::open()` gates its expensive daemon // Tried FIRST: `ReticulumLink::open()` gates its expensive daemon
@ -487,40 +488,19 @@ async fn open_preferred_path(
}; };
} }
// Reticulum first — see the matching comment on auto_detect_and_open: // Unpinned: don't probe this path ourselves at all. Confirmed live
// its cheap probe_rnode gate fails in ~1s for non-RNode firmware, while // 2026-07-23 — this function used to run its own Reticulum→Meshcore→
// trying Meshcore/Meshtastic first was observed leaving a real RNode // Meshtastic sequence here, and the caller (run_mesh_session) falls
// board unresponsive by the time Reticulum's turn came. // back to `auto_detect_and_open` on any error, which scans every
match ReticulumLink::open( // candidate path (this one included) with the exact same three-protocol
path, // sequence. With a single physical radio — the overwhelmingly common
data_dir, // case — `path` here IS the one candidate `auto_detect_and_open` is
Some(our_ed_pubkey_hex), // about to try, so every unpinned reconnect was resetting the board via
Some(our_x25519_pubkey_hex), // Reticulum/Meshcore/Meshtastic's DTR/RTS toggle TWICE: once here, once
) // again moments later in auto-detect. Bailing immediately (no port
.await // access at all) means auto-detect's single pass is the only one that
{ // ever touches the port when nothing is pinned yet.
Ok(mut dev) => match dev.initialize().await { anyhow::bail!("No device_kind pin — deferring to auto-detect for {path}")
Ok(info) => return Ok((MeshRadioDevice::Reticulum(dev), info)),
Err(e) => {
debug!(path = %path, error = %e, "Preferred path is not a working Reticulum RNode")
}
},
Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Reticulum"),
}
match MeshcoreDevice::open(path).await {
Ok(mut dev) => match dev.initialize().await {
Ok(info) => return Ok((MeshRadioDevice::Meshcore(dev), info)),
Err(e) => debug!(path = %path, error = %e, "Preferred path is not Meshcore"),
},
Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Meshcore"),
}
match MeshtasticDevice::open(path).await {
Ok(mut dev) => match dev.initialize().await {
Ok(info) => Ok((MeshRadioDevice::Meshtastic(dev), info)),
Err(e) => Err(e).context("Preferred path is not a working Meshtastic device"),
},
Err(e) => Err(e).context("Could not open preferred path as Meshtastic"),
}
} }
/// Bring up a Reticulum daemon over plain TCP — no physical RNode, no /// Bring up a Reticulum daemon over plain TCP — no physical RNode, no