diff --git a/core/Cargo.lock b/core/Cargo.lock index 1582d933..a49d1ecf 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -84,6 +84,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.1" @@ -161,6 +170,7 @@ dependencies = [ "uuid", "zbase32", "zeroize", + "zip", ] [[package]] @@ -1175,6 +1185,17 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -6895,8 +6916,37 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/core/archipelago/Cargo.toml b/core/archipelago/Cargo.toml index 7a03e0a1..0e7e5d8e 100644 --- a/core/archipelago/Cargo.toml +++ b/core/archipelago/Cargo.toml @@ -108,6 +108,10 @@ bytes = "1" # Mesh networking (Meshcore serial protocol over USB LoRa radios) serial2-tokio = "0.1" +# LoRa radio firmware flashing: Meshtastic ships per-board images inside a +# per-platform release zip (see mesh/flash.rs). +zip = { version = "2", default-features = false, features = ["deflate"] } + # Double Ratchet key derivation (Phase 3: encrypted mesh messaging) hkdf = "0.12.4" diff --git a/core/archipelago/src/api/rpc/dispatcher.rs b/core/archipelago/src/api/rpc/dispatcher.rs index fd4686d9..2dd8d60e 100644 --- a/core/archipelago/src/api/rpc/dispatcher.rs +++ b/core/archipelago/src/api/rpc/dispatcher.rs @@ -391,6 +391,10 @@ impl RpcHandler { // Mesh networking (Meshcore LoRa) "mesh.status" => self.handle_mesh_status().await, "mesh.probe-device" => self.handle_mesh_probe_device(params).await, + "mesh.flash-list-firmware" => self.handle_mesh_flash_list_firmware(params).await, + "mesh.flash-device" => self.handle_mesh_flash_device(params).await, + "mesh.flash-status" => self.handle_mesh_flash_status().await, + "mesh.flash-cancel" => self.handle_mesh_flash_cancel().await, "mesh.peers" => self.handle_mesh_peers().await, "mesh.refresh" => self.handle_mesh_refresh().await, "mesh.messages" => self.handle_mesh_messages(params).await, diff --git a/core/archipelago/src/api/rpc/mesh/flash.rs b/core/archipelago/src/api/rpc/mesh/flash.rs new file mode 100644 index 00000000..c4fd4fc0 --- /dev/null +++ b/core/archipelago/src/api/rpc/mesh/flash.rs @@ -0,0 +1,132 @@ +use super::super::RpcHandler; +use crate::mesh; +use crate::mesh::flash::{self, FlashBoard, FlashJobStatus}; +use crate::mesh::types::DeviceType; +use anyhow::Result; + +fn parse_family(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "meshcore" => Ok(DeviceType::Meshcore), + "meshtastic" => Ok(DeviceType::Meshtastic), + "reticulum" | "rnode" => Ok(DeviceType::Reticulum), + other => anyhow::bail!("Unknown firmware family: {other} (expected meshcore|meshtastic|reticulum)"), + } +} + +fn parse_board(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "heltec-v3" | "heltec_v3" | "heltecv3" => Ok(FlashBoard::HeltecV3), + "heltec-v4" | "heltec_v4" | "heltecv4" => Ok(FlashBoard::HeltecV4), + other => anyhow::bail!("Unknown board: {other} (expected heltec-v3|heltec-v4)"), + } +} + +impl RpcHandler { + /// mesh.flash-list-firmware — resolve the available firmware version(s) + /// for a given family. v1 only ever surfaces "latest". + pub(in crate::api::rpc) async fn handle_mesh_flash_list_firmware( + &self, + params: Option, + ) -> Result { + let family = params + .as_ref() + .and_then(|p| p.get("family")) + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("Missing family"))?; + let family = parse_family(family)?; + let versions = flash::list_firmware(family).await?; + Ok(serde_json::json!({ "versions": versions })) + } + + /// mesh.flash-device — erase and reflash a detected LoRa radio with the + /// latest firmware for the given family, defaulting to a full chip + /// erase before write. `board` is optional: if the port's USB vid:pid + /// unambiguously resolves to a known board, that's used; otherwise the + /// caller must supply it explicitly (see `flash::resolve_flash_board`'s + /// doc comment on why we refuse to guess). + pub(in crate::api::rpc) async fn handle_mesh_flash_device( + &self, + params: Option, + ) -> Result { + let path = params + .as_ref() + .and_then(|p| p.get("path")) + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("Missing path"))? + .to_string(); + let family = params + .as_ref() + .and_then(|p| p.get("family")) + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("Missing family"))?; + let family = parse_family(family)?; + + let detected = mesh::detect_devices().await; + anyhow::ensure!( + detected.iter().any(|d| d == &path), + "{path} is not a detected mesh-radio candidate port" + ); + + let board = match params + .as_ref() + .and_then(|p| p.get("board")) + .and_then(|v| v.as_str()) + { + Some(explicit) => parse_board(explicit)?, + None => { + let info = mesh::detect_devices_info() + .await + .into_iter() + .find(|d| d.path == path); + info.as_ref() + .and_then(flash::resolve_flash_board) + .ok_or_else(|| { + anyhow::anyhow!( + "Could not auto-detect the board on {path} — specify board explicitly" + ) + })? + } + }; + + flash::start_flash_job( + &self.flash_job, + &self.mesh_service_arc(), + self.config.data_dir.clone(), + path, + board, + family, + ) + .await?; + + Ok(serde_json::json!({ "started": true })) + } + + /// mesh.flash-status — poll the current (or most recent) flash job. + pub(in crate::api::rpc) async fn handle_mesh_flash_status(&self) -> Result { + let job = self.flash_job.read().await; + match job.as_ref() { + Some(j) => { + let status: FlashJobStatus = j.snapshot().await; + let mut value = serde_json::to_value(&status)?; + if let Some(obj) = value.as_object_mut() { + obj.insert("active".into(), (!status.done).into()); + } + Ok(value) + } + None => Ok(serde_json::json!({ "active": false })), + } + } + + /// mesh.flash-cancel — best-effort; only honored before erase/write has + /// started (see `FlashJob::cancel`'s doc comment). + pub(in crate::api::rpc) async fn handle_mesh_flash_cancel(&self) -> Result { + let job = self.flash_job.read().await; + match job.as_ref() { + Some(j) => { + j.cancel().await?; + Ok(serde_json::json!({ "cancelled": true })) + } + None => anyhow::bail!("No flash job in progress"), + } + } +} diff --git a/core/archipelago/src/api/rpc/mesh/mod.rs b/core/archipelago/src/api/rpc/mesh/mod.rs index fe497876..51b53bc7 100644 --- a/core/archipelago/src/api/rpc/mesh/mod.rs +++ b/core/archipelago/src/api/rpc/mesh/mod.rs @@ -1,5 +1,6 @@ mod assistant; mod bitcoin_ops; +mod flash; mod messaging; mod safety; mod status; diff --git a/core/archipelago/src/api/rpc/mesh/status.rs b/core/archipelago/src/api/rpc/mesh/status.rs index 5dac2a2f..77087623 100644 --- a/core/archipelago/src/api/rpc/mesh/status.rs +++ b/core/archipelago/src/api/rpc/mesh/status.rs @@ -101,12 +101,36 @@ impl RpcHandler { detected.iter().any(|d| d == &path), "{path} is not a detected mesh-radio candidate port" ); - let service = self.mesh_service.read().await; - let probe = match service.as_ref() { - Some(svc) => svc.probe_device(&path).await?, - // No mesh service yet (radio never enabled) — probe directly. - None => mesh::listener::probe_device(&path).await?, - }; + // Refuse to probe while a firmware flash is in flight. Confirmed + // live 2026-07-23: esptool ("multiple access on port?") and + // rnodeconf (OSError Errno 71 Protocol error on an RTS ioctl) both + // failed with symptoms consistent with a second process holding the + // same serial fd — the flash subprocess runs for minutes outside + // our own async runtime, so nothing previously stopped a concurrent + // `mesh.probe-device` call (e.g. the hot-swap modal's own re-probe) + // from opening the identical port at the same time and corrupting + // both operations' handshakes. + if let Some(job) = self.flash_job.read().await.as_ref() { + anyhow::ensure!( + job.snapshot().await.done, + "A firmware flash is in progress — refusing to probe the serial port until it finishes" + ); + } + // Only hold the mesh_service lock long enough for the quick + // active-path guard check — NEVER across the actual probe, which + // can take 15-60s across its internal collision retries. Confirmed + // live 2026-07-23: holding this read lock for the full probe starved + // a concurrent firmware-flash job's MeshService::stop() (which needs + // the write lock) well past its own bounded timeout, surfacing as + // "Mesh listener did not release the serial port" even though + // stop() itself was fast. + { + let service = self.mesh_service.read().await; + if let Some(svc) = service.as_ref() { + svc.ensure_probe_allowed(&path).await?; + } + } + let probe = mesh::listener::probe_device(&path).await?; Ok(serde_json::to_value(probe)?) } diff --git a/core/archipelago/src/api/rpc/mod.rs b/core/archipelago/src/api/rpc/mod.rs index cc484c4d..83b0de23 100644 --- a/core/archipelago/src/api/rpc/mod.rs +++ b/core/archipelago/src/api/rpc/mod.rs @@ -89,6 +89,9 @@ pub struct RpcHandler { endpoint_rate_limiter: EndpointRateLimiter, response_cache: ResponseCache, mesh_service: Arc>>, + /// LoRa radio firmware-flash job state, sibling to `mesh_service` — one + /// job at a time, since flashing needs exclusive access to the port. + flash_job: crate::mesh::flash::FlashJobHandle, transport_router: Arc>>>, /// Shared content-addressed blob store. Set by ApiHandler after construction /// so mesh.send-content / mesh.fetch-content RPCs can reach it without a @@ -160,6 +163,7 @@ impl RpcHandler { endpoint_rate_limiter, response_cache: ResponseCache::new(5), mesh_service: Arc::new(tokio::sync::RwLock::new(None)), + flash_job: crate::mesh::flash::new_job_handle(), transport_router: Arc::new(tokio::sync::RwLock::new(None)), blob_store: Arc::new(tokio::sync::RwLock::new(None)), self_pubkey_hex: Arc::new(tokio::sync::RwLock::new(None)), diff --git a/core/archipelago/src/mesh/flash.rs b/core/archipelago/src/mesh/flash.rs new file mode 100644 index 00000000..33a0f089 --- /dev/null +++ b/core/archipelago/src/mesh/flash.rs @@ -0,0 +1,975 @@ +// WIP mesh/transport protocol — suppress dead code warnings +#![allow(dead_code)] +//! Firmware flashing for LoRa mesh radios — Heltec V3/V4 in v1, across all +//! three firmware families the mesh module already knows how to detect (see +//! `mesh::types::DeviceType`). Firmware is always fetched from upstream at +//! flash time (never bundled/pinned in the repo), and every flash defaults +//! to a full chip erase before write. +//! +//! MeshCore and Meshtastic are flashed the same way: download a released +//! image, `esptool erase_flash`, then `esptool write_flash 0x0 `. +//! Reticulum/RNode is different: `archy-rnodeconf --autoinstall` owns the +//! whole fetch+erase+flash+EEPROM-bootstrap sequence itself (confirmed live +//! via `archy-rnodeconf --help` — there is no raw esptool path exposed for +//! this family, so we deliberately don't resolve a firmware URL ourselves +//! for Reticulum; rnodeconf already knows how). + +use super::serial::DetectedDeviceInfo; +use super::types::DeviceType; +use super::MeshService; +use anyhow::{Context, Result}; +use regex::Regex; +use serde::Serialize; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::{Arc, OnceLock}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::process::Command; +use tokio::sync::RwLock; +use tracing::{info, warn}; + +/// Boards supported for v1. Both are ESP32-S3 (a single `--chip esp32s3` +/// esptool target covers both), but ship different USB identities and +/// different per-board firmware assets upstream. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum FlashBoard { + HeltecV3, + HeltecV4, +} + +impl FlashBoard { + /// Meshtastic's board id (matches the release manifest's `board` field + /// and its per-board asset naming, e.g. `firmware-heltec-v3-.factory.bin`). + fn meshtastic_id(self) -> &'static str { + match self { + Self::HeltecV3 => "heltec-v3", + Self::HeltecV4 => "heltec-v4", + } + } +} + +/// Map a detected USB vid:pid to a known flashable board, using the same +/// table as `image-recipe/configs/99-mesh-radio.rules`. CP2102 (10c4:ea60) +/// is confirmed there as Heltec V3's USB-UART bridge chip, and is safe to +/// auto-match since that vid:pid is bridge-chip-specific. +/// +/// Heltec V4 is NOT auto-matchable and deliberately has no entry here: it +/// was confirmed live (real hardware, 2026-07-23) to use the ESP32-S3's +/// built-in native-USB JTAG/serial peripheral, reporting vid:pid 303a:1001 +/// with product string "USB JTAG/serial debug unit" — that descriptor is +/// baked into the chip's ROM and is IDENTICAL across every ESP32-S3 board +/// with native USB enabled, not just Heltec V4. Adding `303a:1001 => +/// HeltecV4` here would silently misidentify any other native-USB ESP32-S3 +/// board (a T3-S3, a bare devkit, etc.) as a V4 and risk writing the wrong +/// board's image. Callers (the RPC layer / frontend) must let the user pick +/// the board manually whenever this returns `None`. +pub fn resolve_flash_board(info: &DetectedDeviceInfo) -> Option { + match (info.vid.as_deref(), info.pid.as_deref()) { + (Some("10c4"), Some("ea60")) => Some(FlashBoard::HeltecV3), + _ => None, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum FlashStage { + Downloading, + Erasing, + Writing, + Autoinstalling, + Done, + Failed, +} + +#[derive(Debug, Clone, Serialize)] +pub struct FlashJobStatus { + pub board: FlashBoard, + pub family: DeviceType, + pub path: String, + pub stage: FlashStage, + pub percent: Option, + pub log_tail: Vec, + pub done: bool, + pub error: Option, +} + +const LOG_TAIL_MAX: usize = 200; + +/// How long to wait after a successful flash before resuming the mesh +/// listener, so the board finishes its own post-flash boot/reset before we +/// start opening the port (which itself toggles DTR/RTS) again. +const POST_FLASH_SETTLE_DELAY: std::time::Duration = std::time::Duration::from_secs(5); + +/// Absolute ceiling on a whole flash job (download + erase + write, or +/// autoinstall), regardless of what it's doing internally. Last-resort +/// safety net so a hang anywhere can't wedge the single-flash-job guard +/// forever — generous enough to never trigger on a legitimately slow +/// multi-hundred-MB transfer. +const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15 * 60); + +/// How long to wait for MeshService::stop() to release the serial port +/// before giving up. Confirmed live 2026-07-23: the listener's own +/// reconnect/multi-candidate-probe loop doesn't check its shutdown signal +/// between candidates, so stop() can take a while (or, if the loop is +/// wedged, never return) — 20s comfortably covers a normal handshake-probe +/// cycle without leaving a flash request hanging indefinitely if the +/// listener genuinely won't let go. +const STOP_LISTENER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + +/// How long to keep retrying the port-free check before giving up. +const PORT_FREE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +/// Confirm nothing else has `path` open by actually opening (and immediately +/// closing) it ourselves. Retries across the timeout since a just-stopped +/// listener's fd can take a moment to actually release even after `stop()` +/// returns (task abort is a request, not an instant guarantee the OS-level +/// resource is gone yet). +async fn wait_for_port_free(path: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + PORT_FREE_TIMEOUT; + let mut last_err = None; + loop { + match serial2_tokio::SerialPort::open(path, 115200) { + Ok(_) => return Ok(()), + Err(e) => last_err = Some(e), + } + if tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + Err(anyhow::anyhow!( + "{path} is still held open by something else after {}s (last error: {}) — refusing to start the flasher against a contended port", + PORT_FREE_TIMEOUT.as_secs(), + last_err.map(|e| e.to_string()).unwrap_or_default() + )) +} + +/// Live state for the one flash job that can run at a time. A single global +/// slot is sufficient because flashing needs exclusive serial access to the +/// one port being flashed — there is no meaningful concept of two concurrent +/// flash jobs on this node. +pub struct FlashJob { + status: RwLock, + /// Set once the background task is spawned. Only used while `stage` is + /// still `Downloading` — an interrupted erase/write can leave the chip + /// in a worse state than either finished or unstarted, so cancellation + /// is refused once erase begins (see `cancel()`). + abort_handle: RwLock>, +} + +impl FlashJob { + fn new(board: FlashBoard, family: DeviceType, path: String) -> Arc { + Arc::new(Self { + abort_handle: RwLock::new(None), + status: RwLock::new(FlashJobStatus { + board, + family, + path, + stage: FlashStage::Downloading, + percent: None, + log_tail: Vec::new(), + done: false, + error: None, + }), + }) + } + + pub async fn snapshot(&self) -> FlashJobStatus { + self.status.read().await.clone() + } + + async fn set_stage(&self, stage: FlashStage) { + let mut s = self.status.write().await; + s.stage = stage; + s.percent = None; + } + + async fn set_percent(&self, percent: u8) { + self.status.write().await.percent = Some(percent.min(100)); + } + + async fn push_log(&self, line: impl Into) { + let mut s = self.status.write().await; + s.log_tail.push(line.into()); + let overflow = s.log_tail.len().saturating_sub(LOG_TAIL_MAX); + if overflow > 0 { + s.log_tail.drain(0..overflow); + } + } + + async fn fail(&self, err: &anyhow::Error) { + let mut s = self.status.write().await; + s.stage = FlashStage::Failed; + s.error = Some(format!("{err:#}")); + s.done = true; + } + + async fn finish(&self) { + let mut s = self.status.write().await; + s.stage = FlashStage::Done; + s.done = true; + } + + /// Best-effort cancel: only honored before erase/write/autoinstall has + /// started (i.e. still in `Downloading`). Once a stage that touches the + /// chip begins, this refuses — interrupting an erase or write can leave + /// the flash in a state worse than either finished or unstarted. + pub async fn cancel(&self) -> Result<()> { + let mut s = self.status.write().await; + if s.done { + anyhow::bail!("Flash job already finished"); + } + if s.stage != FlashStage::Downloading { + anyhow::bail!( + "Cannot cancel once {:?} has started — let it finish or fail on its own", + s.stage + ); + } + if let Some(handle) = self.abort_handle.write().await.take() { + handle.abort(); + } + s.stage = FlashStage::Failed; + s.error = Some("Cancelled by user".to_string()); + s.done = true; + Ok(()) + } +} + +/// Shared handle held by `RpcHandler`, sibling to `mesh_service`. +pub type FlashJobHandle = Arc>>>; + +pub fn new_job_handle() -> FlashJobHandle { + Arc::new(RwLock::new(None)) +} + +fn firmware_cache_dir(data_dir: &Path) -> PathBuf { + data_dir.join("mesh").join("firmware-cache") +} + +/// No blanket `.timeout()` here on purpose: reqwest's request timeout covers +/// the *entire* request including streaming the response body, which would +/// kill a legitimate large download partway through (Meshtastic's esp32s3 +/// zip is ~170MB) — not just a hung connection. `download_to_file` instead +/// applies a per-chunk stall timeout, and metadata calls (small JSON +/// responses) get their own short timeout at the call site. +fn github_client() -> Result { + reqwest::Client::builder() + .user_agent("archipelago-mesh-flash") + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .context("Failed to build HTTP client") +} + +/// Applied per-chunk while streaming a firmware download — if the transfer +/// stalls (no bytes for this long) it's treated as a failure, but a slow +/// download that's still making progress is never killed just for taking a +/// while. +const DOWNLOAD_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Applied to metadata calls (GitHub release JSON) — these are small +/// responses with no reason to ever take this long. +const METADATA_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + +/// Resolve what firmware is available for a board+family. v1 only ever +/// offers "latest" — MeshCore/Meshtastic latest GitHub release, or, for +/// Reticulum, "latest" meaning "whatever archy-rnodeconf --autoinstall +/// resolves on its own" (it does its own version checking upstream). +pub async fn list_firmware(family: DeviceType) -> Result> { + match family { + DeviceType::Reticulum => Ok(vec!["latest".to_string()]), + DeviceType::Meshtastic => { + let client = github_client()?; + let release: GithubRelease = client + .get("https://api.github.com/repos/meshtastic/firmware/releases/latest") + .send() + .await + .context("Fetching Meshtastic release list")? + .error_for_status() + .context("Meshtastic releases API error")? + .json() + .await + .context("Parsing Meshtastic release JSON")?; + Ok(vec![release.tag_name]) + } + DeviceType::Meshcore => { + let client = github_client()?; + let release: GithubRelease = client + .get("https://api.github.com/repos/meshcore-dev/MeshCore/releases/latest") + .send() + .await + .context("Fetching MeshCore release list")? + .error_for_status() + .context("MeshCore releases API error")? + .json() + .await + .context("Parsing MeshCore release JSON")?; + Ok(vec![release.tag_name]) + } + DeviceType::Unknown => anyhow::bail!("Pick a firmware family before listing versions"), + } +} + +#[derive(serde::Deserialize)] +struct GithubAsset { + name: String, + browser_download_url: String, +} + +#[derive(serde::Deserialize)] +struct GithubRelease { + tag_name: String, + assets: Vec, +} + +/// Start a flash job in the background. Returns as soon as the job has been +/// registered and the listener released — callers poll `FlashJobHandle` via +/// `mesh.flash-status` for progress. Only one job may be in flight at a time. +pub async fn start_flash_job( + handle: &FlashJobHandle, + mesh_service: &Arc>>, + data_dir: PathBuf, + path: String, + board: FlashBoard, + family: DeviceType, +) -> Result<()> { + { + let existing = handle.read().await; + if let Some(job) = existing.as_ref() { + if !job.snapshot().await.done { + anyhow::bail!("A firmware flash is already in progress on this node"); + } + } + } + + let job = FlashJob::new(board, family, path.clone()); + *handle.write().await = Some(Arc::clone(&job)); + + let bg_job = Arc::clone(&job); + let bg_service = Arc::clone(mesh_service); + let task = tokio::spawn(async move { + // esptool/archy-rnodeconf need exclusive serial access — release + // the listener's hold on the port before touching it. This USED + // TO run synchronously in start_flash_job before the job was even + // spawned, blocking the RPC call itself on s.stop().await — a real + // 2026-07-23 incident: the mesh listener was mid a multi-candidate + // reconnect/probe sequence that doesn't check its shutdown signal + // between candidates, so stop() never returned. The HTTP request + // timed out client-side ("Operation failed"), while the job + // (already inserted into `handle`) was permanently wedged — nothing + // had been spawned yet to ever mark it done, so every later flash + // attempt failed with "already in progress" until a full restart. + // Now this runs inside the spawned task with its own bounded + // timeout, so the RPC call always returns immediately regardless, + // and a slow-to-stop listener fails the job cleanly instead of + // hanging everything downstream of it forever. + let stop_result = tokio::time::timeout(STOP_LISTENER_TIMEOUT, async { + let mut svc = bg_service.write().await; + if let Some(s) = svc.as_mut() { + s.stop().await; + } + }) + .await; + if stop_result.is_err() { + let err = anyhow::anyhow!( + "Mesh listener did not release the serial port within {}s — it may still be mid a reconnect attempt. Try again once mesh.status shows the device idle, or restart the archipelago service if this persists.", + STOP_LISTENER_TIMEOUT.as_secs() + ); + bg_job.push_log(format!("ERROR: {err:#}")).await; + bg_job.fail(&err).await; + return; + } + + // Belt-and-suspenders port-free check. `stop()` above should have + // fully released the port, but esptool/rnodeconf run as external + // subprocesses for minutes outside our own async runtime — if + // ANYTHING else still has it open (a racing probe, a not-yet-dropped + // fd from an aborted task, anything we haven't anticipated), handing + // the port to the flasher anyway risks exactly the corruption + // confirmed live 2026-07-23: esptool's "device disconnected or + // multiple access on port?" and rnodeconf's raw `OSError: [Errno 71] + // Protocol error` on an RTS ioctl are both textbook two-openers-on- + // one-fd symptoms. Verify by actually opening it ourselves — cheap, + // and definitive — before ever starting the flasher. + if let Err(e) = wait_for_port_free(&path).await { + bg_job.push_log(format!("ERROR: {e:#}")).await; + bg_job.fail(&e).await; + return; + } + + // Outer ceiling on top of run_flash's own internal timeouts — + // belt-and-suspenders so that no future hang (network, subprocess, + // anything) can ever wedge the single-flash-job guard permanently + // again the way a stuck download did on 2026-07-23 (every + // subsequent mesh.flash-device call failed with "already in + // progress" until the service was restarted). Generous enough that + // a legitimately slow multi-hundred-MB transfer still completes. + let result = match tokio::time::timeout( + MAX_JOB_DURATION, + run_flash(board, family, &data_dir, &path, &bg_job), + ) + .await + { + Ok(inner) => inner, + Err(_) => Err(anyhow::anyhow!( + "Flash job exceeded the {}-minute ceiling — aborted", + MAX_JOB_DURATION.as_secs() / 60 + )), + }; + let succeeded = result.is_ok(); + + match &result { + Ok(()) => { + bg_job.push_log("Flash completed successfully".to_string()).await; + bg_job.finish().await; + info!(path = %path, board = ?board, family = %family, "LoRa firmware flash succeeded"); + } + Err(e) => { + // {:#} (alternate Display) walks the full anyhow context + // chain — plain {} / %e only prints the outermost .context() + // message, which made a real 2026-07-23 esptool failure + // undiagnosable from journalctl alone (just "esptool + // erase_flash failed", no actual esptool stderr). + warn!(path = %path, error = %format!("{e:#}"), "LoRa firmware flash failed"); + bg_job.push_log(format!("ERROR: {e:#}")).await; + bg_job.fail(e).await; + } + } + + // The board's firmware may now differ from whatever was pinned + // before — clear the pin either way so a later reconnect's strict + // auto-detect order picks up reality instead of getting wedged + // trying the old protocol first. + if let Ok(mut config) = super::load_config(&data_dir).await { + config.device_kind = None; + if let Err(e) = super::save_config(&data_dir, &config).await { + warn!(error = %e, "Failed to clear device_kind pin after flash"); + } + } + + if !succeeded { + // Deliberately do NOT auto-restart the listener here. A failed + // flash means we can't vouch for the board's state — reopening + // the port immediately (esptool/rnodeconf's own reset sequence + // plus our open() toggling DTR/RTS again right after) risks + // hammering a marginal device with reconnect attempts. Confirmed + // live 2026-07-23: exactly this sequence left a real Heltec V3 + // boot-looping for 5+ minutes after a failed flash. Leave mesh + // stopped; the user reconnects explicitly via the hot-swap + // modal/Mesh page once they've confirmed the board is alive. + warn!( + path = %path, + "Leaving mesh listener stopped after failed flash — reconnect manually once the board is confirmed responsive" + ); + return; + } + + // On success, give the board a moment to finish booting after the + // flash tool's own reset sequence before we start hammering it with + // connection attempts — same reasoning as above, just the + // lower-risk (successful-flash) side of it. + tokio::time::sleep(POST_FLASH_SETTLE_DELAY).await; + + let mut svc = bg_service.write().await; + if let Some(s) = svc.as_mut() { + match super::load_config(&data_dir).await { + Ok(config) => { + // Only resume if mesh is actually still enabled per the + // CURRENT persisted config — confirmed live 2026-07-23: + // unconditionally forcing a restart here, regardless of + // `enabled`, overrode a user's own concurrent "disable + // mesh" toggle and left the listener running while + // config said disabled. That inconsistent state is what + // made a later legitimate "Keep As Is" click (which + // correctly tries to start on a false→true transition) + // fail with "already running" — the listener had already + // been force-started behind the config's back. + let should_run = config.enabled; + if let Err(e) = s.configure(config).await { + warn!(error = %e, "Failed to resume mesh listener after flash"); + } + if should_run { + if let Err(e) = s.start() { + warn!(error = %e, "Failed to restart mesh listener after flash"); + } + } + } + Err(e) => warn!(error = %e, "Failed to load mesh config after flash"), + } + } + }); + *job.abort_handle.write().await = Some(task.abort_handle()); + + Ok(()) +} + +async fn run_flash( + board: FlashBoard, + family: DeviceType, + data_dir: &Path, + path: &str, + job: &Arc, +) -> Result<()> { + match family { + DeviceType::Meshtastic | DeviceType::Meshcore => { + let image = fetch_esptool_image(board, family, data_dir, job).await?; + esptool_erase_and_write(path, &image, job).await + } + DeviceType::Reticulum => { + let lora_region = super::load_config(data_dir) + .await + .ok() + .and_then(|c| c.lora_region); + rnodeconf_autoinstall(path, board, lora_region.as_deref(), job).await + } + DeviceType::Unknown => anyhow::bail!("Pick a firmware family before flashing"), + } +} + +// ─── MeshCore / Meshtastic: esptool ───────────────────────────────────── + +async fn fetch_esptool_image( + board: FlashBoard, + family: DeviceType, + data_dir: &Path, + job: &Arc, +) -> Result { + let cache = firmware_cache_dir(data_dir); + tokio::fs::create_dir_all(&cache) + .await + .context("Creating firmware cache dir")?; + let client = github_client()?; + + match family { + DeviceType::Meshtastic => fetch_meshtastic_image(&client, board, &cache, job).await, + DeviceType::Meshcore => fetch_meshcore_image(&client, board, &cache, job).await, + _ => anyhow::bail!("{family} is not flashed via esptool"), + } +} + +async fn fetch_meshtastic_image( + client: &reqwest::Client, + board: FlashBoard, + cache: &Path, + job: &Arc, +) -> Result { + let release: GithubRelease = client + .get("https://api.github.com/repos/meshtastic/firmware/releases/latest") + .timeout(METADATA_TIMEOUT) + .send() + .await + .context("Fetching Meshtastic release list")? + .error_for_status() + .context("Meshtastic releases API error")? + .json() + .await + .context("Parsing Meshtastic release JSON")?; + + // Meshtastic bundles all esp32s3 boards' images inside one per-platform + // zip rather than shipping per-board top-level assets — both Heltec V3 + // and V4 are esp32s3, so this is the right zip for both (confirmed live + // against v2.7.26.54e0d8d). + let zip_asset = release + .assets + .iter() + .find(|a| a.name.starts_with("firmware-esp32s3-") && a.name.ends_with(".zip")) + .ok_or_else(|| anyhow::anyhow!("No esp32s3 firmware zip in latest Meshtastic release"))?; + + let version = zip_asset + .name + .strip_prefix("firmware-esp32s3-") + .and_then(|s| s.strip_suffix(".zip")) + .ok_or_else(|| anyhow::anyhow!("Unexpected Meshtastic asset name: {}", zip_asset.name))? + .to_string(); + + let zip_path = cache.join(&zip_asset.name); + if tokio::fs::metadata(&zip_path).await.is_err() { + download_to_file(client, &zip_asset.browser_download_url, &zip_path, job).await?; + } else { + job.push_log(format!("Using cached {}", zip_asset.name)).await; + } + + // "*.factory.bin" is Meshtastic's full merged image (bootloader + + // partition table + app) meant to be written at offset 0x0 on a freshly + // erased chip — confirmed by inspecting the real zip's contents, as + // opposed to the plain "*.bin" OTA-update image which assumes an + // existing bootloader/partition table already on the chip. + let entry_name = format!( + "firmware-{}-{}.factory.bin", + board.meshtastic_id(), + version + ); + let out_path = cache.join(&entry_name); + if tokio::fs::metadata(&out_path).await.is_ok() { + return Ok(out_path); + } + + job.push_log(format!( + "Extracting {entry_name} from {}", + zip_asset.name + )) + .await; + let zip_path_owned = zip_path.clone(); + let entry_name_owned = entry_name.clone(); + let out_path_owned = out_path.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + let file = std::fs::File::open(&zip_path_owned).context("Opening downloaded firmware zip")?; + let mut archive = zip::ZipArchive::new(file).context("Reading firmware zip")?; + let mut entry = archive + .by_name(&entry_name_owned) + .with_context(|| format!("{entry_name_owned} not found in firmware zip"))?; + let mut out = + std::fs::File::create(&out_path_owned).context("Creating extracted firmware file")?; + std::io::copy(&mut entry, &mut out).context("Extracting firmware image")?; + Ok(()) + }) + .await + .context("Firmware extraction task panicked")??; + + Ok(out_path) +} + +async fn fetch_meshcore_image( + client: &reqwest::Client, + board: FlashBoard, + cache: &Path, + job: &Arc, +) -> Result { + let release: GithubRelease = client + .get("https://api.github.com/repos/meshcore-dev/MeshCore/releases/latest") + .timeout(METADATA_TIMEOUT) + .send() + .await + .context("Fetching MeshCore release list")? + .error_for_status() + .context("MeshCore releases API error")? + .json() + .await + .context("Parsing MeshCore release JSON")?; + + // Upstream's casing differs between boards (Heltec_v3_... vs + // heltec_v4_...) — match case-insensitively on the exact per-board + // substring so V4 isn't accidentally matched by "heltec_v4_tft_..." + // variants (there's a "_tft_" in between, so a straight substring match + // on "heltec_v4_companion_radio_usb" is already safe). + let needle = match board { + FlashBoard::HeltecV3 => "heltec_v3_companion_radio_usb", + FlashBoard::HeltecV4 => "heltec_v4_companion_radio_usb", + }; + let asset = release + .assets + .iter() + .find(|a| { + let lower = a.name.to_lowercase(); + lower.contains(needle) && lower.ends_with("-merged.bin") + }) + .ok_or_else(|| { + anyhow::anyhow!("No matching MeshCore image in release {}", release.tag_name) + })?; + + let out_path = cache.join(&asset.name); + if tokio::fs::metadata(&out_path).await.is_ok() { + job.push_log(format!("Using cached {}", asset.name)).await; + return Ok(out_path); + } + download_to_file(client, &asset.browser_download_url, &out_path, job).await?; + Ok(out_path) +} + +async fn download_to_file( + client: &reqwest::Client, + url: &str, + dest: &Path, + job: &Arc, +) -> Result<()> { + job.set_stage(FlashStage::Downloading).await; + // Bound only the wait for the response to *start* (headers) — NOT a + // request-level `.timeout()`, which would cap the whole body transfer + // again (the bug this replaced: a blanket 30s client timeout killed + // large downloads mid-stream). If the server never responds at all, + // this is what stops the job from hanging forever; the per-chunk stall + // timeout below is what guards the body once streaming starts. Without + // this, a server that accepts the TCP connection but never sends + // headers back hangs this call indefinitely — confirmed live + // 2026-07-23: a stuck `.send()` here wedged the single-flash-job guard + // for good, permanently blocking every subsequent flash attempt with + // "already in progress" until the service was restarted. + let resp = tokio::time::timeout(METADATA_TIMEOUT, client.get(url).send()) + .await + .context("Firmware download server did not respond")? + .context("Starting firmware download")? + .error_for_status() + .context("Firmware download returned an error status")?; + let total = resp.content_length(); + let tmp = dest.with_extension("part"); + let mut file = tokio::fs::File::create(&tmp) + .await + .context("Creating firmware download file")?; + let mut stream = resp.bytes_stream(); + let mut downloaded: u64 = 0; + use futures_util::StreamExt; + loop { + let next = tokio::time::timeout(DOWNLOAD_STALL_TIMEOUT, stream.next()) + .await + .context("Firmware download stalled")?; + let Some(chunk) = next else { break }; + let chunk = chunk.context("Reading firmware download stream")?; + file.write_all(&chunk) + .await + .context("Writing firmware download")?; + downloaded += chunk.len() as u64; + if let Some(total) = total { + if total > 0 { + job.set_percent(((downloaded.saturating_mul(100)) / total) as u8) + .await; + } + } + } + file.flush().await.ok(); + tokio::fs::rename(&tmp, dest) + .await + .context("Finalizing firmware download")?; + job.push_log(format!( + "Downloaded {} ({downloaded} bytes)", + dest.display() + )) + .await; + Ok(()) +} + +/// Both Heltec V3 and V4 are ESP32-S3 boards. +const ESPTOOL_CHIP: &str = "esp32s3"; + +/// esptool's auto-reset-into-bootloader handshake (toggling DTR/RTS in a +/// specific timed pattern) is well-known to be flaky on some CP2102/CH340 +/// board+adapter combinations — esptool's own docs recommend retrying at a +/// lower baud rate when this happens. Rather than fail the whole job on the +/// first hiccup, retry once at a conservative baud before giving up. +const ESPTOOL_FALLBACK_BAUD: &str = "115200"; + +/// `write_flash --erase-all` erases the whole chip before writing, in one +/// esptool invocation. This needs the esp32s3 stub flasher loaded (see +/// esptool_global_args' doc comment) — without it, --erase-all hits the +/// exact same ROM limitation a standalone `erase_flash` does ("ESP32-S3 ROM +/// does not support function erase_flash", confirmed live 2026-07-23), since +/// esptool's --erase-all is implemented as the same full-chip-erase command, +/// not a per-sector loop. +async fn esptool_erase_and_write(path: &str, image: &Path, job: &Arc) -> Result<()> { + job.set_stage(FlashStage::Writing).await; + let image_str = image.to_string_lossy().to_string(); + esptool_with_retry( + path, + &["write_flash", "--erase-all", "0x0", &image_str], + job, + ) + .await + .context("esptool write_flash failed")?; + + Ok(()) +} + +/// esptool's global flags (--chip/--port/--baud) MUST precede the subcommand +/// token (erase_flash/write_flash/...) — confirmed live 2026-07-23: +/// appending `--baud 115200` after the subcommand on the retry path +/// produced "esptool: error: unrecognized arguments: --baud 115200" every +/// time, so the fallback-baud retry never actually got a chance to run. +/// Building global args separately from subcommand args keeps this correct +/// by construction instead of relying on call-site ordering. +/// +/// Normal stub-loader mode (no --no-stub) needs the esp32s3 stub flasher +/// blob at /usr/lib/python3/dist-packages/esptool/targets/stub_flasher/ +/// stub_flasher_32s3.json — Debian's `esptool` package (4.7.0+dfsg-0.1) +/// ships without it (stripped for DFSG compliance: the prebuilt blob has no +/// buildable-from-source path Debian could verify), so scripts/self-update.sh +/// fetches the exact same file from the matching upstream esptool release +/// tag and installs it alongside the apt package (see the esptool install +/// step there). --no-stub (talk directly to the ROM bootloader, skip the +/// stub) was tried first and works for connecting, but the ROM bootloader +/// doesn't implement a full-chip-erase opcode at all — only the stub does — +/// so --no-stub broke our "always erase before write" default outright +/// rather than just being slower. Restoring the real stub file is the +/// correct fix, not routing around its absence. +fn esptool_global_args<'a>(path: &'a str, baud: Option<&'a str>) -> Vec<&'a str> { + let mut args = vec!["--chip", ESPTOOL_CHIP, "--port", path]; + if let Some(b) = baud { + args.push("--baud"); + args.push(b); + } + args +} + +async fn esptool_with_retry(path: &str, subcommand: &[&str], job: &Arc) -> Result<()> { + let mut cmd = Command::new("esptool"); + cmd.args(esptool_global_args(path, None)); + cmd.args(subcommand); + match run_streamed(cmd, None, job).await { + Ok(()) => Ok(()), + Err(first_err) => { + job.push_log(format!( + "First attempt failed ({first_err:#}); retrying once at {ESPTOOL_FALLBACK_BAUD} baud" + )) + .await; + let mut retry = Command::new("esptool"); + retry.args(esptool_global_args(path, Some(ESPTOOL_FALLBACK_BAUD))); + retry.args(subcommand); + run_streamed(retry, None, job) + .await + .context(format!("retry also failed (first attempt: {first_err:#})")) + } + } +} + +// ─── Reticulum/RNode: archy-rnodeconf ─────────────────────────────────── + +fn rnodeconf_bin() -> String { + std::env::var("ARCHY_RNODECONF_BIN") + .unwrap_or_else(|_| "/usr/local/bin/archy-rnodeconf".to_string()) +} + +/// `--autoinstall`'s "which board is this" step is interactive by design — +/// confirmed live against a real Heltec V4 (2026-07-23): even with a board +/// given on the command line, rnodeconf can't always tell V3 from V4 apart +/// (their bootstrap-time USB identity is often generic, same root cause as +/// `resolve_flash_board`'s doc comment), so it always asks. The full prompt +/// sequence observed for a Heltec board that already has *some* RNode +/// firmware installed (the common case — a truly blank chip likely skips +/// straight to the same "Device Selection" menu): +/// 1. numbered device-type menu → answer with the menu number +/// 2. "Hit enter to continue" → answer with a blank line +/// 3. numbered band menu → answer with the menu number +/// 4. "Is the above correct? [y/N]" → answer "y" +/// Feeding all four answers up front (rather than watching stdout for each +/// prompt text) works because the menu is always asked in this fixed order +/// for every board that needs (re)provisioning — verified by driving it +/// through an unprovisioned real V4 end-to-end (erase → flash → EEPROM +/// bootstrap → "Device signature validated" on the next probe). +fn rnodeconf_device_menu_number(board: FlashBoard) -> &'static str { + match board { + FlashBoard::HeltecV3 => "8", + FlashBoard::HeltecV4 => "9", + } +} + +/// rnodeconf's band choice is a coarse RF-frontend bootstrap parameter +/// (868/915/923 MHz), not the final operating frequency — that's still +/// configured later via the daemon's interface config, same as today. This +/// is a best-effort mapping from the node's persisted Meshtastic-style +/// region code (see `mesh::meshtastic::region_name_to_code`) down to +/// rnodeconf's 3-way menu; regions with no exact 868/923 match fall back to +/// 915 MHz as the broadest-compatibility default. +fn rnodeconf_band_menu_number(lora_region: Option<&str>) -> &'static str { + match lora_region.map(|s| s.trim().to_uppercase()) { + Some(r) if r.contains("868") => "1", + Some(r) if r.contains("923") => "3", + _ => "2", + } +} + +/// `--autoinstall` fetches, erases, flashes, and bootstraps the EEPROM for +/// a detected board as one atomic step (confirmed via `archy-rnodeconf +/// --help` AND a real end-to-end flash on real hardware) — this is the +/// RNode-side equivalent of our "always erase before write" default, since +/// autoinstall doesn't try to preserve any existing on-device state. +async fn rnodeconf_autoinstall( + path: &str, + board: FlashBoard, + lora_region: Option<&str>, + job: &Arc, +) -> Result<()> { + job.set_stage(FlashStage::Autoinstalling).await; + let bin = rnodeconf_bin(); + let mut cmd = if Path::new(&bin).exists() { + Command::new(bin) + } else { + // Dev fallback if only a plain venv/system rnodeconf is on PATH. + Command::new("rnodeconf") + }; + cmd.args(["--autoinstall", path]); + let stdin = format!( + "{}\n\n{}\ny\n", + rnodeconf_device_menu_number(board), + rnodeconf_band_menu_number(lora_region) + ); + run_streamed(cmd, Some(stdin.into_bytes()), job) + .await + .context("archy-rnodeconf --autoinstall failed") +} + +// ─── Subprocess streaming ──────────────────────────────────────────────── + +fn percent_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"\((\d{1,3})\s*%\)").expect("valid regex")) +} + +async fn run_streamed(mut cmd: Command, stdin: Option>, job: &Arc) -> Result<()> { + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + if stdin.is_some() { + cmd.stdin(Stdio::piped()); + } + // Deliberately NOT kill_on_drop: an interrupted erase/write can leave + // the chip in a worse state than either finished or unstarted (see the + // cancellation-safety note in mesh flashing docs). The job is expected + // to run to completion or fail on its own. + let mut child = cmd.spawn().context("Failed to start subprocess")?; + + if let Some(bytes) = stdin { + if let Some(mut child_stdin) = child.stdin.take() { + child_stdin + .write_all(&bytes) + .await + .context("Writing to subprocess stdin")?; + } + } + + let mut tasks = Vec::new(); + if let Some(stdout) = child.stdout.take() { + let job = Arc::clone(job); + tasks.push(tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if let Some(cap) = percent_regex().captures(&line) { + if let Ok(pct) = cap[1].parse::() { + job.set_percent(pct).await; + } + } + job.push_log(line).await; + } + })); + } + if let Some(stderr) = child.stderr.take() { + let job = Arc::clone(job); + tasks.push(tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + job.push_log(line).await; + } + })); + } + + let status = child.wait().await.context("Waiting for subprocess")?; + for t in tasks { + let _ = t.await; + } + if !status.success() { + // Exit status alone isn't diagnosable — the actual esptool/rnodeconf + // stderr (already captured into job.log_tail by the reader tasks + // above) is what actually explains a failure. Confirmed live + // 2026-07-23: a bare "Command exited with exit status: 1" told us + // nothing when esptool's real error was sitting in the log tail the + // whole time, only visible via the UI's live poll, not journald. + let tail: Vec = job + .snapshot() + .await + .log_tail + .iter() + .rev() + .take(10) + .rev() + .cloned() + .collect(); + anyhow::bail!("Command exited with {status}\n{}", tail.join("\n")); + } + Ok(()) +} diff --git a/core/archipelago/src/mesh/listener/mod.rs b/core/archipelago/src/mesh/listener/mod.rs index f486968d..55674d0d 100644 --- a/core/archipelago/src/mesh/listener/mod.rs +++ b/core/archipelago/src/mesh/listener/mod.rs @@ -557,6 +557,10 @@ pub fn spawn_mesh_listener( let mut shutdown = shutdown; let mut cmd_rx = cmd_rx; let mut reconnect_delay = RECONNECT_DELAY_INIT; + // Mutable so a successful auto-detect can pin the firmware kind for + // the rest of this listener's lifetime — see the pin-on-first-success + // block below for why. + let mut device_kind = device_kind; // Backlog #12 hot-swap re-binding: each run_mesh_session call already // builds a fresh device struct (contacts/current_region/etc. all // start empty), so per-device session state is naturally isolated @@ -628,6 +632,45 @@ pub fn spawn_mesh_listener( } } + // Pin the firmware kind after the first successful auto-detect. + // Confirmed live 2026-07-23: with device_kind left unpinned (e.g. + // after clearing a stale pin), EVERY reconnect re-runs the full + // Reticulum→Meshcore→Meshtastic auto-detect cascade — each + // candidate past the first does its own open() with the DTR/RTS + // reset both boards need, so a device correctly identified as + // Meshtastic still gets reset once for the failed Meshcore + // attempt before Meshtastic's own open() resets it again. That + // doubled the reset count on every single reconnect indefinitely, + // not just during initial detection. Once auto-detect has + // identified the device this listener is actually talking to, + // there's no reason to keep guessing on subsequent reconnects — + // pin it, both in this task's own loop (takes effect + // immediately) and on disk (survives a service restart). A + // genuine hot-swap to different firmware is still handled: the + // setup modal's `mesh.probe-device` always re-probes unpinned, + // and the flash flow already clears this pin on its own. + if device_kind.is_none() { + let detected = state.status.read().await.device_type; + if detected != super::types::DeviceType::Unknown { + device_kind = Some(detected); + match super::load_config(&data_dir).await { + Ok(mut cfg) if cfg.device_kind.is_none() => { + cfg.device_kind = Some(detected); + if let Err(e) = super::save_config(&data_dir, &cfg).await { + warn!("Failed to persist auto-detected device_kind: {}", e); + } else { + info!( + kind = %detected, + "Pinned auto-detected firmware kind to avoid repeated multi-protocol resets on reconnect" + ); + } + } + Ok(_) => {} + Err(e) => warn!("Failed to load mesh config to persist device_kind: {}", e), + } + } + } + // Update status to disconnected. device_type/firmware_version are // reset too — they were previously left holding the LAST radio's // identity, so after a hot-swap the UI showed the old firmware diff --git a/core/archipelago/src/mesh/listener/session.rs b/core/archipelago/src/mesh/listener/session.rs index d05f59b5..140e39e1 100644 --- a/core/archipelago/src/mesh/listener/session.rs +++ b/core/archipelago/src/mesh/listener/session.rs @@ -288,6 +288,7 @@ async fn auto_detect_and_open( None => "No serial devices found in /dev".to_string(), }); } + info!(candidates = ?paths, "Auto-detect candidate ports for this attempt"); for path in &paths { debug!(path = %path, "Probing for mesh radio device"); // Tried FIRST: `ReticulumLink::open()` gates its expensive daemon @@ -514,41 +515,21 @@ async fn open_preferred_path( }; } - // Reticulum first — see the matching comment on auto_detect_and_open: - // its cheap probe_rnode gate fails in ~1s for non-RNode firmware, while - // trying Meshcore/Meshtastic first was observed leaving a real RNode - // board unresponsive by the time Reticulum's turn came. - match ReticulumLink::open( - path, - data_dir, - Some(our_ed_pubkey_hex), - Some(our_x25519_pubkey_hex), - advert_name, - ) - .await - { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => return Ok((MeshRadioDevice::Reticulum(dev), info)), - Err(e) => { - debug!(path = %path, error = %e, "Preferred path is not a working Reticulum RNode") - } - }, - Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Reticulum"), - } - match MeshcoreDevice::open(path).await { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => return Ok((MeshRadioDevice::Meshcore(dev), info)), - Err(e) => debug!(path = %path, error = %e, "Preferred path is not Meshcore"), - }, - Err(e) => debug!(path = %path, error = %e, "Could not open preferred path as Meshcore"), - } - match MeshtasticDevice::open(path).await { - Ok(mut dev) => match dev.initialize().await { - Ok(info) => Ok((MeshRadioDevice::Meshtastic(dev), info)), - Err(e) => Err(e).context("Preferred path is not a working Meshtastic device"), - }, - Err(e) => Err(e).context("Could not open preferred path as Meshtastic"), - } + // Unpinned: don't probe this path ourselves at all. Confirmed live + // 2026-07-23 — this function used to run its own Reticulum→Meshcore→ + // Meshtastic sequence here, and the caller (run_mesh_session) falls + // back to `auto_detect_and_open` on any error, which scans every + // candidate path (this one included) with the exact same three-protocol + // sequence. With a single physical radio — the overwhelmingly common + // case — `path` here IS the one candidate `auto_detect_and_open` is + // about to try, so every unpinned reconnect was resetting the board via + // Reticulum/Meshcore/Meshtastic's DTR/RTS toggle TWICE: once here, once + // again moments later in auto-detect. Bailing immediately (no port + // access at all) means auto-detect's single pass is the only one that + // ever touches the port when nothing is pinned yet. (auto_detect_and_open + // carries the advert_name threading from main, so nothing is lost.) + let _ = advert_name; + anyhow::bail!("No device_kind pin — deferring to auto-detect for {path}") } /// Bring up a Reticulum daemon over plain TCP — no physical RNode, no diff --git a/core/archipelago/src/mesh/mod.rs b/core/archipelago/src/mesh/mod.rs index 325b32d1..a3933ee5 100644 --- a/core/archipelago/src/mesh/mod.rs +++ b/core/archipelago/src/mesh/mod.rs @@ -8,6 +8,7 @@ pub mod alerts; pub mod bitcoin_relay; pub mod crypto; +pub mod flash; pub mod listener; pub mod meshtastic; pub mod message_types; @@ -767,10 +768,18 @@ impl MeshService { self.server_name = name; } - /// Start the background mesh listener. + /// Start the background mesh listener. Idempotent: if the listener is + /// already running, this is a harmless no-op rather than an error — + /// confirmed live 2026-07-23, a real race between the flash job's own + /// post-flash restart and a concurrent user "Keep As Is" click (both + /// legitimately trying to ensure the listener is running) surfaced this + /// as a user-facing "Mesh listener already running" RPC error. Ensuring + /// the listener is running is the intent every caller actually has; + /// whichever caller's start() happens to win the race, the other + /// finding it already satisfied is success, not failure. pub fn start(&mut self) -> Result<()> { if self.listener_handle.is_some() { - anyhow::bail!("Mesh listener already running"); + return Ok(()); } let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -1084,13 +1093,12 @@ impl MeshService { self.state.peers.read().await.values().cloned().collect() } - /// Probe a serial port for a mesh radio without provisioning or keeping - /// it — powers the hot-swap "device detected" modal's current-details - /// view. Refuses to probe the port the live session currently occupies - /// (the probe would steal the serial port from under the session); a + /// Refuse to probe the port the live session currently occupies (the + /// probe would steal the serial port from under the session); a /// detected-but-not-connected port is fair game, accepting a benign race - /// with the reconnect loop (whichever loses just retries). - pub async fn probe_device(&self, path: &str) -> Result { + /// with the reconnect loop (whichever loses just retries). Split out from + /// the actual probe on purpose — see `probe_device`'s doc comment. + pub async fn ensure_probe_allowed(&self, path: &str) -> Result<()> { let status = self.state.status.read().await; if status.device_connected { if let Some(active) = status.device_path.as_deref() { @@ -1105,8 +1113,7 @@ impl MeshService { } } } - drop(status); - listener::probe_device(path).await + Ok(()) } /// Get message history. diff --git a/core/archipelago/src/mesh/serial.rs b/core/archipelago/src/mesh/serial.rs index 3645714f..59384363 100644 --- a/core/archipelago/src/mesh/serial.rs +++ b/core/archipelago/src/mesh/serial.rs @@ -557,29 +557,39 @@ fn likely_non_mesh_serial_device(path: &str) -> bool { /// Scan for serial devices that could be Meshcore radios. /// Returns paths to existing serial device files. /// -/// Candidates are deduplicated by canonical path: /dev/mesh-radio is a udev -/// symlink to a ttyUSB*/ttyACM* node that is ALSO in the candidate list, so -/// without this one physical board shows up (and gets probed, and gets its -/// DTR/RTS reset toggled) twice per cycle. The first candidate wins, which -/// keeps the stable /dev/mesh-radio name when the symlink exists. +/// Dedupes by canonical (symlink-resolved) target: `/dev/mesh-radio` is a +/// stable udev symlink to whatever `/dev/ttyUSB*`/`/dev/ttyACM*` node the +/// primary radio currently enumerates as, so both names always pointed at +/// the same candidate list entry and both passed this scan — confirmed live +/// 2026-07-23, this made an already-connected, working radio (connected via +/// its `/dev/mesh-radio` alias) simultaneously appear as a second, separate +/// "detected but unclaimed" device under its raw `/dev/ttyUSBn` name. The +/// hot-swap UI's active-session guard compares path strings, so it didn't +/// recognize the two aliases as the same port, showed the "device detected" +/// modal for a radio that was already set up, and probing it there opened +/// (and DTR/RTS-reset) the exact port the live session was mid-conversation +/// with — a continuous, UI-driven reset loop that only ran while that view +/// was open (matches the reported "stops when I leave, resumes when I come +/// back"). SERIAL_CANDIDATES lists `/dev/mesh-radio` first, so it wins the +/// dedup and is what's reported when both alias and target are present. +/// (Independently re-discovered and fixed on main 2026-07-26 — both sides +/// of the 2026-07-28 merge carried an equivalent implementation.) pub async fn detect_serial_devices() -> Vec { let mut devices = Vec::new(); - let mut seen_canonical: Vec = Vec::new(); + let mut seen_real_paths = std::collections::HashSet::new(); for path in SERIAL_CANDIDATES { if tokio::fs::metadata(path).await.is_ok() { if likely_non_mesh_serial_device(path) { debug!(path = %path, "Skipping known non-mesh serial device"); continue; } - let canonical = tokio::fs::canonicalize(path) + let real_path = tokio::fs::canonicalize(path) .await .unwrap_or_else(|_| std::path::PathBuf::from(path)); - if seen_canonical.contains(&canonical) { - debug!(path = %path, canonical = %canonical.display(), - "Skipping alias of an already-detected serial device"); + if !seen_real_paths.insert(real_path.clone()) { + debug!(path = %path, real_path = %real_path.display(), "Skipping duplicate alias for an already-listed device"); continue; } - seen_canonical.push(canonical); devices.push(path.to_string()); } } diff --git a/docs/combined-test-plan-2026-07-22.md b/docs/combined-test-plan-2026-07-22.md index 590e357b..cd0308ec 100644 --- a/docs/combined-test-plan-2026-07-22.md +++ b/docs/combined-test-plan-2026-07-22.md @@ -110,6 +110,61 @@ lands]. 2. ❑ Mobile Home: wallet card directly under My Apps (G5 from the voice epic). 3. ❑ Mesh RF settings panel (Mesh → Device) still loads and saves. +## H. LoRa radio firmware flashing (Heltec V3/V4, new — extends Section E) + +Full v1 scope is 3 firmware families × 2 boards (6 cells); mark each cell +tested on real hardware vs. code-reviewed only as this is run. + +1. ❑ From the hot-swap modal's step 1 (device already probed), press + **Flash Firmware…** → new step shows firmware-family + board pickers and + the erase-confirmation checkbox; "Erase & Flash Now" stays disabled until + family, board, AND the checkbox are all set. +2. ❑ Confirm what's currently on the test stick via the existing probe + BEFORE flashing it — don't flash the only known-good device without a + fallback board on hand. +3. ❑ Prefer a spare Heltec V3/V4 for the first destructive erase+flash run; + only exercise a primary/in-use stick once the flow is proven safe. +4. ❑ MeshCore → Heltec V3: erase + write completes, progress bar and log + tail update live, ends at "Flash complete". +5. ❑ Meshtastic → Heltec V3: same, using the extracted `*.factory.bin` from + the esp32s3 release zip. +6. ❑ Reticulum/RNode → Heltec V3: `archy-rnodeconf --autoinstall` path + completes (no raw esptool erase/write step for this family — see + `mesh/flash.rs` doc comment). +7. ❑ Repeat 4-6 against a Heltec V4. Confirmed 2026-07-23 on real hardware: + V4 uses the ESP32-S3's native-USB JTAG/serial peripheral (vid:pid + 303a:1001, generic to every native-USB ESP32-S3 board, not V4-specific) + — so unlike V3's CP2102 bridge chip, V4 is permanently NOT auto-matchable + by vid:pid. Board auto-detect should fail closed for it every time + (manual board selection required, "couldn't confirm automatically" + warning shown) — this is expected steady-state behavior, not a gap to + close later. +8. ❑ After a successful flash, the modal automatically re-probes and shows + the NEW firmware's badge/details — same as unplugging and replugging + (Section E item 3), but without physically touching the cable. +9. ❑ Deliberately test a failure path once (disconnect the board mid-write, + or point at a bad cached asset) — confirm the error surfaces in the + progress log AND that `docs/troubleshooting.md`'s "LoRa radio firmware + flash failed" recovery steps (BOOT+RST bootloader entry, manual esptool/ + rnodeconf command) actually get the board back to a flashable state. +10. ❑ Cancel button only appears (and only works) while still in the + "Downloading firmware…" stage — once erasing/writing starts, no cancel + affordance is offered. +11. ❑ **Boot-loop regression (2026-07-23 incident)**: after a *failed* flash + (e.g. kill network access mid-download to force a failure), confirm the + mesh listener does NOT auto-resume — `journalctl -u archipelago` should + show a single `Leaving mesh listener stopped after failed flash` line + and then go quiet for that device, not a repeating `mesh::serial: + Opened serial port... Starting Meshcore handshake` cycle every few + seconds. Reconnect manually via the hot-swap modal afterward and confirm + it connects normally (the board itself should be untouched — the + download fails before esptool/rnodeconf ever runs). +12. ❑ Separately, force a device to flap connected/disconnected a few times + in under 20s each (e.g. a marginal USB connection) and confirm + `reconnect_delay` in the logs actually escalates (5s → 10s → 20s → ...) + rather than resetting to 5s on every attempt — see + `STABLE_SESSION_THRESHOLD` in `mesh/listener/mod.rs`. + --- After this passes: fold the batch + other agent's work into the next release diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 10476296..70079225 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -443,6 +443,86 @@ free -h - Check Nginx WebSocket proxy config: `/etc/nginx/sites-available/archipelago` must include `proxy_set_header Upgrade $http_upgrade` - If on WiFi, try wired Ethernet for more stable connectivity +### 21. LoRa radio firmware flash failed / board unresponsive + +**Symptoms**: The "Erase & Flash Now" flow in the mesh hot-swap modal reports +an error, or the radio no longer enumerates as a serial device after a flash +attempt. + +**Diagnosis**: +```bash +# Poll the flash job's last-known stage/error directly +curl -s http://localhost:5678/rpc/v1 \ + -H 'Content-Type: application/json' \ + -d '{"method":"mesh.flash-status","params":{}}' + +# Confirm the board is still enumerating at all +ls -la /dev/ttyUSB* /dev/ttyACM* /dev/mesh-radio 2>&1 + +# esptool/rnodeconf binaries present? +which esptool; ls -la /usr/local/bin/archy-rnodeconf +``` + +**Solutions**: +- A failure during `erasing`/`writing` (MeshCore/Meshtastic) or + `autoinstalling` (Reticulum) can leave the chip erased or half-written — + this is expected risk of the "always erase first" default, not a bug. +- Heltec V3/V4 boards can be forced back into bootloader mode manually: hold + **BOOT**, tap **RST**, then release **BOOT** — this puts the chip in a + state esptool can always talk to, regardless of what firmware (if any) is + currently on it. +- With the board in bootloader mode, a manual recovery flash can be run + directly over SSH without the UI: + ```bash + esptool --chip esp32s3 --port /dev/ttyACM0 erase_flash + esptool --chip esp32s3 --port /dev/ttyACM0 write_flash 0x0 + ``` +- For Reticulum/RNode boards, the equivalent manual recovery is + `archy-rnodeconf /dev/ttyACM0 --autoinstall` (or `/usr/local/bin/archy-rnodeconf` + if it's not on `PATH`) — it re-runs the same fetch+erase+flash+bootstrap + sequence the UI triggers. +- If `esptool`/`archy-rnodeconf` are missing entirely, they should have been + installed by the last `self-update.sh` run — check + `sudo journalctl -u archipelago-update` for install failures, or install + `esptool` via `sudo apt-get install esptool` directly. +- Once a fresh image is confirmed written, unplug/replug the radio (or wait + for the next detection poll) — the hot-swap modal re-probes automatically + and shows whatever firmware is actually on the board now. + +**Known incident (2026-07-23) — reconnect storm / device boot-loop after a +failed flash**: a real Heltec V3 got stuck cycling "connect → partial +handshake → drop" every 5-15s for 5+ minutes after a `mesh.flash-device` +attempt failed with `Reading firmware download stream`. Root cause was two +compounding issues, both now fixed: +1. `spawn_mesh_listener`'s reconnect backoff (`core/archipelago/src/mesh/listener/mod.rs`) + reset to its 5s minimum any time the prior session had been `device_connected` + at all, even for under a second — so a device that connects-then-drops + repeatedly never actually backed off. Every retry's `open()` toggles + DTR/RTS, which resets many ESP32 boards' MCU (native-USB *and* + CP2102/CH340 auto-reset-circuit boards), so the aggressive retries were + themselves *causing* the boot loop, not just observing one. Fixed by only + resetting backoff when a session ran for at least `STABLE_SESSION_THRESHOLD` + (20s) — see that constant's doc comment. +2. `mesh::flash::start_flash_job`'s post-completion handler auto-resumed the + listener unconditionally, even after a *failed* flash, immediately + re-entering the reconnect loop above with no cooldown. Fixed: on failure + the listener is now deliberately left stopped (reconnect manually via the + UI once the board is confirmed alive); on success there's a 5s settle + delay before resuming, so the board finishes booting from the flash + tool's own reset before Archipelago starts probing it again. +3. Separately, the download itself was failing because `mesh::flash`'s HTTP + client had a blanket 30s request timeout that covered the *entire* + download (including streaming a 170MB Meshtastic zip), not just + connection setup — fixed with a per-chunk stall timeout instead of a + fixed total-transfer cap. + +If this symptom recurs (rapid repeating `mesh::serial: Opened serial +port... Starting Meshcore handshake` lines in `journalctl -u archipelago` +without a `LoRa firmware flash` job in progress), it's a NEW instance of the +same class of bug, not the one above — check whether backoff is actually +escalating (`Mesh session error: ... (retry in Xs)` — X should grow past 5s +within a few cycles) before assuming it's flashing-related. + --- ## General Maintenance diff --git a/image-recipe/_archived/build-auto-installer-iso.sh b/image-recipe/_archived/build-auto-installer-iso.sh index eaa14c79..ba5e321a 100755 --- a/image-recipe/_archived/build-auto-installer-iso.sh +++ b/image-recipe/_archived/build-auto-installer-iso.sh @@ -365,6 +365,11 @@ RUN apt-get update && apt-get -y full-upgrade && apt-get install -y --no-install ca-certificates \ openssl \ chrony \ + iputils-ping \ + esptool \ + python3-venv \ + binutils \ + libpython3.13 \ locales \ console-setup \ keyboard-configuration \ diff --git a/neode-ui/src/components/mesh/MeshDeviceSetupModal.vue b/neode-ui/src/components/mesh/MeshDeviceSetupModal.vue index c3bf54f9..4b4973bc 100644 --- a/neode-ui/src/components/mesh/MeshDeviceSetupModal.vue +++ b/neode-ui/src/components/mesh/MeshDeviceSetupModal.vue @@ -1,7 +1,7 @@