fix(mesh): stop the flash-triggered device boot-loop, fix stuck/wedged jobs
Three real incidents from live-testing the flash feature on archy-x250-dev,
each traced through logs on the node and fixed at the root:
- A failed flash left the mesh listener auto-resuming into a reconnect loop
whose backoff reset to its 5s minimum on any momentary connection, even
mid-boot-loop — every retry's open() toggles DTR/RTS, which resets many
ESP32 boards, so the retries were themselves sustaining the loop. Backoff
now only resets after a session runs stably for 20s+, and the flash job
no longer auto-resumes the listener after a failure (only on success,
after a settle delay).
- The HTTP client's blanket 30s request timeout covered entire downloads
(Meshtastic's zip is ~170MB), killing large transfers mid-stream; fixed
with a per-chunk stall timeout instead of a fixed total-transfer cap. But
the download's initial request had no timeout at all, so a slow-to-start
server hung it forever — wedging the single-flash-job guard permanently
("already in progress" on every subsequent attempt). Fixed with a bounded
wait for the response to start, plus an absolute ceiling around the whole
job as a last-resort safety net.
- archy-rnodeconf's PyInstaller freeze broke its own internal esptool
invocation (sys.executable pointed at the frozen binary itself instead of
a real interpreter) — fixed via a new runtime hook. esptool's own
auto-reset-into-bootloader handshake is separately known to be flaky on
some CP2102/CH340 boards; it now retries once at a conservative baud
rate instead of failing outright, and failure logs show the full error
chain instead of just the outer context message.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
7d31ca5d65
commit
beff5dd577
@ -96,6 +96,18 @@ pub struct FlashJobStatus {
|
||||
|
||||
const LOG_TAIL_MAX: usize = 200;
|
||||
|
||||
/// How long to wait after a successful flash before resuming the mesh
|
||||
/// listener, so the board finishes its own post-flash boot/reset before we
|
||||
/// start opening the port (which itself toggles DTR/RTS) again.
|
||||
const POST_FLASH_SETTLE_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Absolute ceiling on a whole flash job (download + erase + write, or
|
||||
/// autoinstall), regardless of what it's doing internally. Last-resort
|
||||
/// safety net so a hang anywhere can't wedge the single-flash-job guard
|
||||
/// forever — generous enough to never trigger on a legitimately slow
|
||||
/// multi-hundred-MB transfer.
|
||||
const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15 * 60);
|
||||
|
||||
/// Live state for the one flash job that can run at a time. A single global
|
||||
/// slot is sufficient because flashing needs exclusive serial access to the
|
||||
/// one port being flashed — there is no meaningful concept of two concurrent
|
||||
@ -198,14 +210,30 @@ fn firmware_cache_dir(data_dir: &Path) -> PathBuf {
|
||||
data_dir.join("mesh").join("firmware-cache")
|
||||
}
|
||||
|
||||
/// No blanket `.timeout()` here on purpose: reqwest's request timeout covers
|
||||
/// the *entire* request including streaming the response body, which would
|
||||
/// kill a legitimate large download partway through (Meshtastic's esp32s3
|
||||
/// zip is ~170MB) — not just a hung connection. `download_to_file` instead
|
||||
/// applies a per-chunk stall timeout, and metadata calls (small JSON
|
||||
/// responses) get their own short timeout at the call site.
|
||||
fn github_client() -> Result<reqwest::Client> {
|
||||
reqwest::Client::builder()
|
||||
.user_agent("archipelago-mesh-flash")
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.context("Failed to build HTTP client")
|
||||
}
|
||||
|
||||
/// Applied per-chunk while streaming a firmware download — if the transfer
|
||||
/// stalls (no bytes for this long) it's treated as a failure, but a slow
|
||||
/// download that's still making progress is never killed just for taking a
|
||||
/// while.
|
||||
const DOWNLOAD_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// Applied to metadata calls (GitHub release JSON) — these are small
|
||||
/// responses with no reason to ever take this long.
|
||||
const METADATA_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
|
||||
/// Resolve what firmware is available for a board+family. v1 only ever
|
||||
/// offers "latest" — MeshCore/Meshtastic latest GitHub release, or, for
|
||||
/// Reticulum, "latest" meaning "whatever archy-rnodeconf --autoinstall
|
||||
@ -293,7 +321,26 @@ pub async fn start_flash_job(
|
||||
let bg_job = Arc::clone(&job);
|
||||
let bg_service = Arc::clone(mesh_service);
|
||||
let task = tokio::spawn(async move {
|
||||
let result = run_flash(board, family, &data_dir, &path, &bg_job).await;
|
||||
// Outer ceiling on top of run_flash's own internal timeouts —
|
||||
// belt-and-suspenders so that no future hang (network, subprocess,
|
||||
// anything) can ever wedge the single-flash-job guard permanently
|
||||
// again the way a stuck download did on 2026-07-23 (every
|
||||
// subsequent mesh.flash-device call failed with "already in
|
||||
// progress" until the service was restarted). Generous enough that
|
||||
// a legitimately slow multi-hundred-MB transfer still completes.
|
||||
let result = match tokio::time::timeout(
|
||||
MAX_JOB_DURATION,
|
||||
run_flash(board, family, &data_dir, &path, &bg_job),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(inner) => inner,
|
||||
Err(_) => Err(anyhow::anyhow!(
|
||||
"Flash job exceeded the {}-minute ceiling — aborted",
|
||||
MAX_JOB_DURATION.as_secs() / 60
|
||||
)),
|
||||
};
|
||||
let succeeded = result.is_ok();
|
||||
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
@ -302,26 +349,58 @@ pub async fn start_flash_job(
|
||||
info!(path = %path, board = ?board, family = %family, "LoRa firmware flash succeeded");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(path = %path, error = %e, "LoRa firmware flash failed");
|
||||
// {:#} (alternate Display) walks the full anyhow context
|
||||
// chain — plain {} / %e only prints the outermost .context()
|
||||
// message, which made a real 2026-07-23 esptool failure
|
||||
// undiagnosable from journalctl alone (just "esptool
|
||||
// erase_flash failed", no actual esptool stderr).
|
||||
warn!(path = %path, error = %format!("{e:#}"), "LoRa firmware flash failed");
|
||||
bg_job.push_log(format!("ERROR: {e:#}")).await;
|
||||
bg_job.fail(e).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Whatever happened, the board's firmware may now differ from
|
||||
// whatever was pinned before — clear the pin so the listener's
|
||||
// strict auto-detect order picks up reality instead of getting
|
||||
// wedged trying the old protocol first, then resume the listener
|
||||
// (probe_device is called separately by the frontend re-showing the
|
||||
// hot-swap modal, reusing the existing non-destructive probe flow).
|
||||
let mut svc = bg_service.write().await;
|
||||
if let Some(s) = svc.as_mut() {
|
||||
// The board's firmware may now differ from whatever was pinned
|
||||
// before — clear the pin either way so a later reconnect's strict
|
||||
// auto-detect order picks up reality instead of getting wedged
|
||||
// trying the old protocol first.
|
||||
if let Ok(mut config) = super::load_config(&data_dir).await {
|
||||
config.device_kind = None;
|
||||
if let Err(e) = s.configure(config).await {
|
||||
if let Err(e) = super::save_config(&data_dir, &config).await {
|
||||
warn!(error = %e, "Failed to clear device_kind pin after flash");
|
||||
}
|
||||
}
|
||||
|
||||
if !succeeded {
|
||||
// Deliberately do NOT auto-restart the listener here. A failed
|
||||
// flash means we can't vouch for the board's state — reopening
|
||||
// the port immediately (esptool/rnodeconf's own reset sequence
|
||||
// plus our open() toggling DTR/RTS again right after) risks
|
||||
// hammering a marginal device with reconnect attempts. Confirmed
|
||||
// live 2026-07-23: exactly this sequence left a real Heltec V3
|
||||
// boot-looping for 5+ minutes after a failed flash. Leave mesh
|
||||
// stopped; the user reconnects explicitly via the hot-swap
|
||||
// modal/Mesh page once they've confirmed the board is alive.
|
||||
warn!(
|
||||
path = %path,
|
||||
"Leaving mesh listener stopped after failed flash — reconnect manually once the board is confirmed responsive"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// On success, give the board a moment to finish booting after the
|
||||
// flash tool's own reset sequence before we start hammering it with
|
||||
// connection attempts — same reasoning as above, just the
|
||||
// lower-risk (successful-flash) side of it.
|
||||
tokio::time::sleep(POST_FLASH_SETTLE_DELAY).await;
|
||||
|
||||
let mut svc = bg_service.write().await;
|
||||
if let Some(s) = svc.as_mut() {
|
||||
if let Ok(config) = super::load_config(&data_dir).await {
|
||||
if let Err(e) = s.configure(config).await {
|
||||
warn!(error = %e, "Failed to resume mesh listener after flash");
|
||||
}
|
||||
}
|
||||
if let Err(e) = s.start() {
|
||||
warn!(error = %e, "Failed to restart mesh listener after flash");
|
||||
}
|
||||
@ -344,7 +423,13 @@ async fn run_flash(
|
||||
let image = fetch_esptool_image(board, family, data_dir, job).await?;
|
||||
esptool_erase_and_write(path, &image, job).await
|
||||
}
|
||||
DeviceType::Reticulum => rnodeconf_autoinstall(path, job).await,
|
||||
DeviceType::Reticulum => {
|
||||
let lora_region = super::load_config(data_dir)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|c| c.lora_region);
|
||||
rnodeconf_autoinstall(path, board, lora_region.as_deref(), job).await
|
||||
}
|
||||
DeviceType::Unknown => anyhow::bail!("Pick a firmware family before flashing"),
|
||||
}
|
||||
}
|
||||
@ -378,6 +463,7 @@ async fn fetch_meshtastic_image(
|
||||
) -> Result<PathBuf> {
|
||||
let release: GithubRelease = client
|
||||
.get("https://api.github.com/repos/meshtastic/firmware/releases/latest")
|
||||
.timeout(METADATA_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.context("Fetching Meshtastic release list")?
|
||||
@ -459,6 +545,7 @@ async fn fetch_meshcore_image(
|
||||
) -> Result<PathBuf> {
|
||||
let release: GithubRelease = client
|
||||
.get("https://api.github.com/repos/meshcore-dev/MeshCore/releases/latest")
|
||||
.timeout(METADATA_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.context("Fetching MeshCore release list")?
|
||||
@ -504,10 +591,20 @@ async fn download_to_file(
|
||||
job: &Arc<FlashJob>,
|
||||
) -> Result<()> {
|
||||
job.set_stage(FlashStage::Downloading).await;
|
||||
let resp = client
|
||||
.get(url)
|
||||
.send()
|
||||
// Bound only the wait for the response to *start* (headers) — NOT a
|
||||
// request-level `.timeout()`, which would cap the whole body transfer
|
||||
// again (the bug this replaced: a blanket 30s client timeout killed
|
||||
// large downloads mid-stream). If the server never responds at all,
|
||||
// this is what stops the job from hanging forever; the per-chunk stall
|
||||
// timeout below is what guards the body once streaming starts. Without
|
||||
// this, a server that accepts the TCP connection but never sends
|
||||
// headers back hangs this call indefinitely — confirmed live
|
||||
// 2026-07-23: a stuck `.send()` here wedged the single-flash-job guard
|
||||
// for good, permanently blocking every subsequent flash attempt with
|
||||
// "already in progress" until the service was restarted.
|
||||
let resp = tokio::time::timeout(METADATA_TIMEOUT, client.get(url).send())
|
||||
.await
|
||||
.context("Firmware download server did not respond")?
|
||||
.context("Starting firmware download")?
|
||||
.error_for_status()
|
||||
.context("Firmware download returned an error status")?;
|
||||
@ -519,7 +616,11 @@ async fn download_to_file(
|
||||
let mut stream = resp.bytes_stream();
|
||||
let mut downloaded: u64 = 0;
|
||||
use futures_util::StreamExt;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
loop {
|
||||
let next = tokio::time::timeout(DOWNLOAD_STALL_TIMEOUT, stream.next())
|
||||
.await
|
||||
.context("Firmware download stalled")?;
|
||||
let Some(chunk) = next else { break };
|
||||
let chunk = chunk.context("Reading firmware download stream")?;
|
||||
file.write_all(&chunk)
|
||||
.await
|
||||
@ -547,25 +648,62 @@ async fn download_to_file(
|
||||
/// Both Heltec V3 and V4 are ESP32-S3 boards.
|
||||
const ESPTOOL_CHIP: &str = "esp32s3";
|
||||
|
||||
/// esptool's auto-reset-into-bootloader handshake (toggling DTR/RTS in a
|
||||
/// specific timed pattern) is well-known to be flaky on some CP2102/CH340
|
||||
/// board+adapter combinations — esptool's own docs recommend retrying at a
|
||||
/// lower baud rate when this happens. Rather than fail the whole job on the
|
||||
/// first hiccup, retry once at a conservative baud before giving up.
|
||||
const ESPTOOL_FALLBACK_BAUD: &str = "115200";
|
||||
|
||||
async fn esptool_erase_and_write(path: &str, image: &Path, job: &Arc<FlashJob>) -> Result<()> {
|
||||
job.set_stage(FlashStage::Erasing).await;
|
||||
let mut erase = Command::new("esptool");
|
||||
erase.args(["--chip", ESPTOOL_CHIP, "--port", path, "erase_flash"]);
|
||||
run_streamed(erase, job)
|
||||
esptool_with_retry(
|
||||
&["--chip", ESPTOOL_CHIP, "--port", path, "erase_flash"],
|
||||
job,
|
||||
)
|
||||
.await
|
||||
.context("esptool erase_flash failed")?;
|
||||
|
||||
job.set_stage(FlashStage::Writing).await;
|
||||
let mut write = Command::new("esptool");
|
||||
write.args(["--chip", ESPTOOL_CHIP, "--port", path, "write_flash", "0x0"]);
|
||||
write.arg(image);
|
||||
run_streamed(write, job)
|
||||
let image_str = image.to_string_lossy().to_string();
|
||||
esptool_with_retry(
|
||||
&[
|
||||
"--chip",
|
||||
ESPTOOL_CHIP,
|
||||
"--port",
|
||||
path,
|
||||
"write_flash",
|
||||
"0x0",
|
||||
&image_str,
|
||||
],
|
||||
job,
|
||||
)
|
||||
.await
|
||||
.context("esptool write_flash failed")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn esptool_with_retry(args: &[&str], job: &Arc<FlashJob>) -> Result<()> {
|
||||
let mut cmd = Command::new("esptool");
|
||||
cmd.args(args);
|
||||
match run_streamed(cmd, None, job).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(first_err) => {
|
||||
job.push_log(format!(
|
||||
"First attempt failed ({first_err:#}); retrying once at {ESPTOOL_FALLBACK_BAUD} baud"
|
||||
))
|
||||
.await;
|
||||
let mut retry = Command::new("esptool");
|
||||
retry.args(args);
|
||||
retry.args(["--baud", ESPTOOL_FALLBACK_BAUD]);
|
||||
run_streamed(retry, None, job)
|
||||
.await
|
||||
.context(format!("retry also failed (first attempt: {first_err:#})"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Reticulum/RNode: archy-rnodeconf ───────────────────────────────────
|
||||
|
||||
fn rnodeconf_bin() -> String {
|
||||
@ -573,12 +711,56 @@ fn rnodeconf_bin() -> String {
|
||||
.unwrap_or_else(|_| "/usr/local/bin/archy-rnodeconf".to_string())
|
||||
}
|
||||
|
||||
/// `--autoinstall`'s "which board is this" step is interactive by design —
|
||||
/// confirmed live against a real Heltec V4 (2026-07-23): even with a board
|
||||
/// given on the command line, rnodeconf can't always tell V3 from V4 apart
|
||||
/// (their bootstrap-time USB identity is often generic, same root cause as
|
||||
/// `resolve_flash_board`'s doc comment), so it always asks. The full prompt
|
||||
/// sequence observed for a Heltec board that already has *some* RNode
|
||||
/// firmware installed (the common case — a truly blank chip likely skips
|
||||
/// straight to the same "Device Selection" menu):
|
||||
/// 1. numbered device-type menu → answer with the menu number
|
||||
/// 2. "Hit enter to continue" → answer with a blank line
|
||||
/// 3. numbered band menu → answer with the menu number
|
||||
/// 4. "Is the above correct? [y/N]" → answer "y"
|
||||
/// Feeding all four answers up front (rather than watching stdout for each
|
||||
/// prompt text) works because the menu is always asked in this fixed order
|
||||
/// for every board that needs (re)provisioning — verified by driving it
|
||||
/// through an unprovisioned real V4 end-to-end (erase → flash → EEPROM
|
||||
/// bootstrap → "Device signature validated" on the next probe).
|
||||
fn rnodeconf_device_menu_number(board: FlashBoard) -> &'static str {
|
||||
match board {
|
||||
FlashBoard::HeltecV3 => "8",
|
||||
FlashBoard::HeltecV4 => "9",
|
||||
}
|
||||
}
|
||||
|
||||
/// rnodeconf's band choice is a coarse RF-frontend bootstrap parameter
|
||||
/// (868/915/923 MHz), not the final operating frequency — that's still
|
||||
/// configured later via the daemon's interface config, same as today. This
|
||||
/// is a best-effort mapping from the node's persisted Meshtastic-style
|
||||
/// region code (see `mesh::meshtastic::region_name_to_code`) down to
|
||||
/// rnodeconf's 3-way menu; regions with no exact 868/923 match fall back to
|
||||
/// 915 MHz as the broadest-compatibility default.
|
||||
fn rnodeconf_band_menu_number(lora_region: Option<&str>) -> &'static str {
|
||||
match lora_region.map(|s| s.trim().to_uppercase()) {
|
||||
Some(r) if r.contains("868") => "1",
|
||||
Some(r) if r.contains("923") => "3",
|
||||
_ => "2",
|
||||
}
|
||||
}
|
||||
|
||||
/// `--autoinstall` fetches, erases, flashes, and bootstraps the EEPROM for
|
||||
/// a detected board as one atomic step (confirmed via `archy-rnodeconf
|
||||
/// --help` on real hardware) — this is the RNode-side equivalent of our
|
||||
/// "always erase before write" default, since autoinstall doesn't try to
|
||||
/// preserve any existing on-device state.
|
||||
async fn rnodeconf_autoinstall(path: &str, job: &Arc<FlashJob>) -> Result<()> {
|
||||
/// --help` AND a real end-to-end flash on real hardware) — this is the
|
||||
/// RNode-side equivalent of our "always erase before write" default, since
|
||||
/// autoinstall doesn't try to preserve any existing on-device state.
|
||||
async fn rnodeconf_autoinstall(
|
||||
path: &str,
|
||||
board: FlashBoard,
|
||||
lora_region: Option<&str>,
|
||||
job: &Arc<FlashJob>,
|
||||
) -> Result<()> {
|
||||
job.set_stage(FlashStage::Autoinstalling).await;
|
||||
let bin = rnodeconf_bin();
|
||||
let mut cmd = if Path::new(&bin).exists() {
|
||||
@ -588,7 +770,12 @@ async fn rnodeconf_autoinstall(path: &str, job: &Arc<FlashJob>) -> Result<()> {
|
||||
Command::new("rnodeconf")
|
||||
};
|
||||
cmd.args(["--autoinstall", path]);
|
||||
run_streamed(cmd, job)
|
||||
let stdin = format!(
|
||||
"{}\n\n{}\ny\n",
|
||||
rnodeconf_device_menu_number(board),
|
||||
rnodeconf_band_menu_number(lora_region)
|
||||
);
|
||||
run_streamed(cmd, Some(stdin.into_bytes()), job)
|
||||
.await
|
||||
.context("archy-rnodeconf --autoinstall failed")
|
||||
}
|
||||
@ -600,15 +787,27 @@ fn percent_regex() -> &'static Regex {
|
||||
RE.get_or_init(|| Regex::new(r"\((\d{1,3})\s*%\)").expect("valid regex"))
|
||||
}
|
||||
|
||||
async fn run_streamed(mut cmd: Command, job: &Arc<FlashJob>) -> Result<()> {
|
||||
async fn run_streamed(mut cmd: Command, stdin: Option<Vec<u8>>, job: &Arc<FlashJob>) -> Result<()> {
|
||||
cmd.stdout(Stdio::piped());
|
||||
cmd.stderr(Stdio::piped());
|
||||
if stdin.is_some() {
|
||||
cmd.stdin(Stdio::piped());
|
||||
}
|
||||
// Deliberately NOT kill_on_drop: an interrupted erase/write can leave
|
||||
// the chip in a worse state than either finished or unstarted (see the
|
||||
// cancellation-safety note in mesh flashing docs). The job is expected
|
||||
// to run to completion or fail on its own.
|
||||
let mut child = cmd.spawn().context("Failed to start subprocess")?;
|
||||
|
||||
if let Some(bytes) = stdin {
|
||||
if let Some(mut child_stdin) = child.stdin.take() {
|
||||
child_stdin
|
||||
.write_all(&bytes)
|
||||
.await
|
||||
.context("Writing to subprocess stdin")?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
let job = Arc::clone(job);
|
||||
|
||||
@ -87,6 +87,18 @@ const RECONNECT_DELAY_INIT: Duration = Duration::from_secs(5);
|
||||
/// Maximum reconnect delay (cap for exponential backoff).
|
||||
const RECONNECT_DELAY_MAX: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Minimum time a session must run before we trust it enough to reset
|
||||
/// backoff to the minimum. Without this gate, a device that connects then
|
||||
/// fails again within a couple of seconds (e.g. mid-boot-loop) never backs
|
||||
/// off — every retry immediately re-opens the port, which toggles DTR/RTS
|
||||
/// (resets many ESP32 boards' MCU on native-USB and CP2102/CH340
|
||||
/// auto-reset-circuit boards alike), turning a device that's merely
|
||||
/// unstable into a self-sustaining boot loop that outlasts whatever
|
||||
/// triggered the original instability. Confirmed live 2026-07-23: a Heltec
|
||||
/// V3 stuck retrying every ~5-15s for 5+ minutes after a failed firmware
|
||||
/// flash left it in a marginal state.
|
||||
const STABLE_SESSION_THRESHOLD: Duration = Duration::from_secs(20);
|
||||
|
||||
/// Number of consecutive write failures before we consider the device dead
|
||||
/// and trigger a reconnection cycle.
|
||||
const MAX_CONSECUTIVE_WRITE_FAILURES: u32 = 3;
|
||||
@ -550,6 +562,7 @@ pub fn spawn_mesh_listener(
|
||||
return;
|
||||
}
|
||||
|
||||
let session_start = std::time::Instant::now();
|
||||
match session::run_mesh_session(
|
||||
&state,
|
||||
&data_dir,
|
||||
@ -572,13 +585,14 @@ pub fn spawn_mesh_listener(
|
||||
{
|
||||
Ok(()) => {
|
||||
info!("Mesh session ended cleanly");
|
||||
// Session was established before ending — reset backoff
|
||||
// Only trust a session that actually ran for a while —
|
||||
// see STABLE_SESSION_THRESHOLD's doc comment.
|
||||
if session_start.elapsed() >= STABLE_SESSION_THRESHOLD {
|
||||
reconnect_delay = RECONNECT_DELAY_INIT;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// Check if session was ever connected (vs failed to open)
|
||||
let was_connected = state.status.read().await.device_connected;
|
||||
if was_connected {
|
||||
if session_start.elapsed() >= STABLE_SESSION_THRESHOLD {
|
||||
reconnect_delay = RECONNECT_DELAY_INIT;
|
||||
}
|
||||
error!("Mesh session error: {} (retry in {:?})", e, reconnect_delay);
|
||||
|
||||
@ -150,6 +150,20 @@ tested on real hardware vs. code-reviewed only as this is run.
|
||||
10. ❑ Cancel button only appears (and only works) while still in the
|
||||
"Downloading firmware…" stage — once erasing/writing starts, no cancel
|
||||
affordance is offered.
|
||||
11. ❑ **Boot-loop regression (2026-07-23 incident)**: after a *failed* flash
|
||||
(e.g. kill network access mid-download to force a failure), confirm the
|
||||
mesh listener does NOT auto-resume — `journalctl -u archipelago` should
|
||||
show a single `Leaving mesh listener stopped after failed flash` line
|
||||
and then go quiet for that device, not a repeating `mesh::serial:
|
||||
Opened serial port... Starting Meshcore handshake` cycle every few
|
||||
seconds. Reconnect manually via the hot-swap modal afterward and confirm
|
||||
it connects normally (the board itself should be untouched — the
|
||||
download fails before esptool/rnodeconf ever runs).
|
||||
12. ❑ Separately, force a device to flap connected/disconnected a few times
|
||||
in under 20s each (e.g. a marginal USB connection) and confirm
|
||||
`reconnect_delay` in the logs actually escalates (5s → 10s → 20s → ...)
|
||||
rather than resetting to 5s on every attempt — see
|
||||
`STABLE_SESSION_THRESHOLD` in `mesh/listener/mod.rs`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -489,6 +489,40 @@ which esptool; ls -la /usr/local/bin/archy-rnodeconf
|
||||
for the next detection poll) — the hot-swap modal re-probes automatically
|
||||
and shows whatever firmware is actually on the board now.
|
||||
|
||||
**Known incident (2026-07-23) — reconnect storm / device boot-loop after a
|
||||
failed flash**: a real Heltec V3 got stuck cycling "connect → partial
|
||||
handshake → drop" every 5-15s for 5+ minutes after a `mesh.flash-device`
|
||||
attempt failed with `Reading firmware download stream`. Root cause was two
|
||||
compounding issues, both now fixed:
|
||||
1. `spawn_mesh_listener`'s reconnect backoff (`core/archipelago/src/mesh/listener/mod.rs`)
|
||||
reset to its 5s minimum any time the prior session had been `device_connected`
|
||||
at all, even for under a second — so a device that connects-then-drops
|
||||
repeatedly never actually backed off. Every retry's `open()` toggles
|
||||
DTR/RTS, which resets many ESP32 boards' MCU (native-USB *and*
|
||||
CP2102/CH340 auto-reset-circuit boards), so the aggressive retries were
|
||||
themselves *causing* the boot loop, not just observing one. Fixed by only
|
||||
resetting backoff when a session ran for at least `STABLE_SESSION_THRESHOLD`
|
||||
(20s) — see that constant's doc comment.
|
||||
2. `mesh::flash::start_flash_job`'s post-completion handler auto-resumed the
|
||||
listener unconditionally, even after a *failed* flash, immediately
|
||||
re-entering the reconnect loop above with no cooldown. Fixed: on failure
|
||||
the listener is now deliberately left stopped (reconnect manually via the
|
||||
UI once the board is confirmed alive); on success there's a 5s settle
|
||||
delay before resuming, so the board finishes booting from the flash
|
||||
tool's own reset before Archipelago starts probing it again.
|
||||
3. Separately, the download itself was failing because `mesh::flash`'s HTTP
|
||||
client had a blanket 30s request timeout that covered the *entire*
|
||||
download (including streaming a 170MB Meshtastic zip), not just
|
||||
connection setup — fixed with a per-chunk stall timeout instead of a
|
||||
fixed total-transfer cap.
|
||||
|
||||
If this symptom recurs (rapid repeating `mesh::serial: Opened serial
|
||||
port... Starting Meshcore handshake` lines in `journalctl -u archipelago`
|
||||
without a `LoRa firmware flash` job in progress), it's a NEW instance of the
|
||||
same class of bug, not the one above — check whether backoff is actually
|
||||
escalating (`Mesh session error: ... (retry in Xs)` — X should grow past 5s
|
||||
within a few cycles) before assuming it's flashing-related.
|
||||
|
||||
---
|
||||
|
||||
## General Maintenance
|
||||
|
||||
@ -47,11 +47,16 @@ if [ -n "$RNODECONF_SRC" ] && [ -f "$RNODECONF_SRC" ]; then
|
||||
# exit()/quit() builtins, which only exist in interactive Python (site.py
|
||||
# injects them) — a frozen app hits NameError right as it tries to quit
|
||||
# cleanly, after all the real work already succeeded. See
|
||||
# pyi_rthook_exit_builtins.py.
|
||||
# pyi_rthook_exit_builtins.py. A second hook fixes rnodeconf's board-flash
|
||||
# step, which shells out to a bundled esptool.py via `sys.executable` —
|
||||
# under a frozen binary that's the binary itself, not a real interpreter,
|
||||
# so the flash subprocess call breaks. See
|
||||
# pyi_rthook_fix_flasher_executable.py.
|
||||
.venv/bin/pyinstaller --onefile --name archy-rnodeconf --clean --noconfirm \
|
||||
--collect-submodules RNS \
|
||||
--collect-data RNS \
|
||||
--runtime-hook pyi_rthook_exit_builtins.py \
|
||||
--runtime-hook pyi_rthook_fix_flasher_executable.py \
|
||||
-d noarchive \
|
||||
"$RNODECONF_SRC"
|
||||
echo "Built dist/archy-rnodeconf ($(du -h dist/archy-rnodeconf | cut -f1))"
|
||||
|
||||
34
reticulum-daemon/pyi_rthook_fix_flasher_executable.py
Normal file
34
reticulum-daemon/pyi_rthook_fix_flasher_executable.py
Normal file
@ -0,0 +1,34 @@
|
||||
# PyInstaller runtime hook — see build.sh.
|
||||
#
|
||||
# rnodeconf's own board-flashing code shells out to a bundled esptool.py as
|
||||
# `[sys.executable, flasher_path, "--chip", ..., "write_flash", ...]` (RNS's
|
||||
# rnodeconf.py, ~line 2794 as of RNS 1.3.5). That's correct for a normal
|
||||
# `python rnodeconf.py` invocation, but under a frozen PyInstaller binary
|
||||
# `sys.executable` is the frozen binary itself, not a real interpreter — so
|
||||
# the "subprocess" just re-invokes archy-rnodeconf's OWN argparse CLI with
|
||||
# esptool-shaped flags, which it doesn't recognize, and the flash step fails
|
||||
# immediately with "unrecognized arguments: --chip ...". Confirmed live
|
||||
# against a real Heltec V4 (2026-07-23): device selection, band selection,
|
||||
# and firmware download all worked; only the final `write_flash` subprocess
|
||||
# call broke this way.
|
||||
#
|
||||
# Fix: point sys.executable at a real Python interpreter that has rnodeconf's
|
||||
# own runtime deps available (esptool.py only needs pyserial, which RNS
|
||||
# already depends on) before any of rnodeconf's code runs. Prefer the build
|
||||
# venv this exact binary was frozen from — see build.sh — falling back to a
|
||||
# bare `python3` on PATH if that venv isn't present on this node.
|
||||
import os
|
||||
import sys
|
||||
|
||||
if getattr(sys, "frozen", False):
|
||||
_candidates = [
|
||||
os.environ.get("ARCHY_RNODECONF_PYTHON", ""),
|
||||
os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "..", "reticulum-daemon", ".venv", "bin", "python3"),
|
||||
os.path.expanduser("~/archy/reticulum-daemon/.venv/bin/python3"),
|
||||
]
|
||||
for _candidate in _candidates:
|
||||
if _candidate and os.path.isfile(_candidate):
|
||||
sys.executable = _candidate
|
||||
break
|
||||
else:
|
||||
sys.executable = "python3"
|
||||
Loading…
x
Reference in New Issue
Block a user