probing issues with flashing
This commit is contained in:
parent
078f3b3619
commit
8403f2233e
@ -101,6 +101,21 @@ impl RpcHandler {
|
||||
detected.iter().any(|d| d == &path),
|
||||
"{path} is not a detected mesh-radio candidate port"
|
||||
);
|
||||
// Refuse to probe while a firmware flash is in flight. Confirmed
|
||||
// live 2026-07-23: esptool ("multiple access on port?") and
|
||||
// rnodeconf (OSError Errno 71 Protocol error on an RTS ioctl) both
|
||||
// failed with symptoms consistent with a second process holding the
|
||||
// same serial fd — the flash subprocess runs for minutes outside
|
||||
// our own async runtime, so nothing previously stopped a concurrent
|
||||
// `mesh.probe-device` call (e.g. the hot-swap modal's own re-probe)
|
||||
// from opening the identical port at the same time and corrupting
|
||||
// both operations' handshakes.
|
||||
if let Some(job) = self.flash_job.read().await.as_ref() {
|
||||
anyhow::ensure!(
|
||||
job.snapshot().await.done,
|
||||
"A firmware flash is in progress — refusing to probe the serial port until it finishes"
|
||||
);
|
||||
}
|
||||
// Only hold the mesh_service lock long enough for the quick
|
||||
// active-path guard check — NEVER across the actual probe, which
|
||||
// can take 15-60s across its internal collision retries. Confirmed
|
||||
|
||||
@ -117,6 +117,34 @@ const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15
|
||||
/// listener genuinely won't let go.
|
||||
const STOP_LISTENER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
|
||||
/// How long to keep retrying the port-free check before giving up.
|
||||
const PORT_FREE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// Confirm nothing else has `path` open by actually opening (and immediately
|
||||
/// closing) it ourselves. Retries across the timeout since a just-stopped
|
||||
/// listener's fd can take a moment to actually release even after `stop()`
|
||||
/// returns (task abort is a request, not an instant guarantee the OS-level
|
||||
/// resource is gone yet).
|
||||
async fn wait_for_port_free(path: &str) -> Result<()> {
|
||||
let deadline = tokio::time::Instant::now() + PORT_FREE_TIMEOUT;
|
||||
let mut last_err = None;
|
||||
loop {
|
||||
match serial2_tokio::SerialPort::open(path, 115200) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(e) => last_err = Some(e),
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(anyhow::anyhow!(
|
||||
"{path} is still held open by something else after {}s (last error: {}) — refusing to start the flasher against a contended port",
|
||||
PORT_FREE_TIMEOUT.as_secs(),
|
||||
last_err.map(|e| e.to_string()).unwrap_or_default()
|
||||
))
|
||||
}
|
||||
|
||||
/// Live state for the one flash job that can run at a time. A single global
|
||||
/// slot is sufficient because flashing needs exclusive serial access to the
|
||||
/// one port being flashed — there is no meaningful concept of two concurrent
|
||||
@ -352,6 +380,23 @@ pub async fn start_flash_job(
|
||||
return;
|
||||
}
|
||||
|
||||
// Belt-and-suspenders port-free check. `stop()` above should have
|
||||
// fully released the port, but esptool/rnodeconf run as external
|
||||
// subprocesses for minutes outside our own async runtime — if
|
||||
// ANYTHING else still has it open (a racing probe, a not-yet-dropped
|
||||
// fd from an aborted task, anything we haven't anticipated), handing
|
||||
// the port to the flasher anyway risks exactly the corruption
|
||||
// confirmed live 2026-07-23: esptool's "device disconnected or
|
||||
// multiple access on port?" and rnodeconf's raw `OSError: [Errno 71]
|
||||
// Protocol error` on an RTS ioctl are both textbook two-openers-on-
|
||||
// one-fd symptoms. Verify by actually opening it ourselves — cheap,
|
||||
// and definitive — before ever starting the flasher.
|
||||
if let Err(e) = wait_for_port_free(&path).await {
|
||||
bg_job.push_log(format!("ERROR: {e:#}")).await;
|
||||
bg_job.fail(&e).await;
|
||||
return;
|
||||
}
|
||||
|
||||
// Outer ceiling on top of run_flash's own internal timeouts —
|
||||
// belt-and-suspenders so that no future hang (network, subprocess,
|
||||
// anything) can ever wedge the single-flash-job guard permanently
|
||||
|
||||
@ -274,6 +274,7 @@ async fn auto_detect_and_open(
|
||||
if paths.is_empty() {
|
||||
anyhow::bail!("No serial devices found in /dev");
|
||||
}
|
||||
info!(candidates = ?paths, "Auto-detect candidate ports for this attempt");
|
||||
for path in &paths {
|
||||
debug!(path = %path, "Probing for mesh radio device");
|
||||
// Tried FIRST: `ReticulumLink::open()` gates its expensive daemon
|
||||
@ -487,40 +488,19 @@ async fn open_preferred_path(
|
||||
};
|
||||
}
|
||||
|
||||
// Reticulum first — see the matching comment on auto_detect_and_open:
|
||||
// its cheap probe_rnode gate fails in ~1s for non-RNode firmware, while
|
||||
// trying Meshcore/Meshtastic first was observed leaving a real RNode
|
||||
// board unresponsive by the time Reticulum's turn came.
|
||||
match ReticulumLink::open(
|
||||
path,
|
||||
data_dir,
|
||||
Some(our_ed_pubkey_hex),
|
||||
Some(our_x25519_pubkey_hex),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(mut dev) => match dev.initialize().await {
|
||||
Ok(info) => return Ok((MeshRadioDevice::Reticulum(dev), info)),
|
||||
Err(e) => {
|
||||
debug!(path = %path, error = %e, "Preferred path is not a working Reticulum RNode")
|
||||
}
|
||||
},
|
||||
Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Reticulum"),
|
||||
}
|
||||
match MeshcoreDevice::open(path).await {
|
||||
Ok(mut dev) => match dev.initialize().await {
|
||||
Ok(info) => return Ok((MeshRadioDevice::Meshcore(dev), info)),
|
||||
Err(e) => debug!(path = %path, error = %e, "Preferred path is not Meshcore"),
|
||||
},
|
||||
Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Meshcore"),
|
||||
}
|
||||
match MeshtasticDevice::open(path).await {
|
||||
Ok(mut dev) => match dev.initialize().await {
|
||||
Ok(info) => Ok((MeshRadioDevice::Meshtastic(dev), info)),
|
||||
Err(e) => Err(e).context("Preferred path is not a working Meshtastic device"),
|
||||
},
|
||||
Err(e) => Err(e).context("Could not open preferred path as Meshtastic"),
|
||||
}
|
||||
// Unpinned: don't probe this path ourselves at all. Confirmed live
|
||||
// 2026-07-23 — this function used to run its own Reticulum→Meshcore→
|
||||
// Meshtastic sequence here, and the caller (run_mesh_session) falls
|
||||
// back to `auto_detect_and_open` on any error, which scans every
|
||||
// candidate path (this one included) with the exact same three-protocol
|
||||
// sequence. With a single physical radio — the overwhelmingly common
|
||||
// case — `path` here IS the one candidate `auto_detect_and_open` is
|
||||
// about to try, so every unpinned reconnect was resetting the board via
|
||||
// Reticulum/Meshcore/Meshtastic's DTR/RTS toggle TWICE: once here, once
|
||||
// again moments later in auto-detect. Bailing immediately (no port
|
||||
// access at all) means auto-detect's single pass is the only one that
|
||||
// ever touches the port when nothing is pinned yet.
|
||||
anyhow::bail!("No device_kind pin — deferring to auto-detect for {path}")
|
||||
}
|
||||
|
||||
/// Bring up a Reticulum daemon over plain TCP — no physical RNode, no
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user