diff --git a/core/archipelago/src/mesh/reticulum.rs b/core/archipelago/src/mesh/reticulum.rs index 5af0b667..62944144 100644 --- a/core/archipelago/src/mesh/reticulum.rs +++ b/core/archipelago/src/mesh/reticulum.rs @@ -159,18 +159,41 @@ fn daemon_command( .arg("--archy-x25519-pubkey-hex") .arg(x); } - cmd.kill_on_drop(true) - // Run the daemon as its own process-group leader. The packaged binary is - // a PyInstaller one-file bootloader that forks the real Python process; - // making it a group leader lets Drop signal the WHOLE group so the - // forked child can't be orphaned and keep holding the serial port. - .process_group(0) + // Run the daemon as its own process-group leader. The packaged binary is + // a PyInstaller one-file bootloader that forks the real Python process; + // making it a group leader lets shutdown signal the WHOLE group so the + // forked child can't be orphaned and keep holding the serial port. + // Deliberately NOT `kill_on_drop`: that SIGKILLs the bootloader the instant + // the `Child` drops, before it can delete its `_MEI*` extraction dir (48M + // leaked into TMPDIR per daemon restart — filled .116's 12G /tmp tmpfs). + // Shutdown goes through `terminate_group` instead, on every path. + cmd.process_group(0) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); cmd } +/// Gracefully stop a daemon process group: SIGTERM now (the daemon's handler +/// releases the RNode + socket, and the PyInstaller bootloader gets to delete +/// its `_MEI*` extraction dir once the Python child exits), SIGKILL from a +/// detached backstop thread a few seconds later so a wedged daemon still can't +/// survive or keep holding the serial port. +fn terminate_group(child: &Child) { + if let Some(pid) = child.id() { + let pgid = -(pid as i32); + unsafe { + libc::kill(pgid, libc::SIGTERM); + } + std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(5)); + unsafe { + libc::kill(pgid, libc::SIGKILL); + } + }); + } +} + /// One peer learned via an RNS announce (LXMF delivery destination). #[derive(Clone)] struct ReticulumPeer { @@ -323,13 +346,22 @@ impl ReticulumLink { .await; } let label = iface.label(); - let socket_path = runtime_dir.join(format!( - "{}.sock", - label.replace(['/', ' ', ':', ','], "_") - )); + let iface_key = label.replace(['/', ' ', ':', ','], "_"); + let socket_path = runtime_dir.join(format!("{iface_key}.sock")); if socket_path.exists() { let _ = std::fs::remove_file(&socket_path); } + // Private TMPDIR for the PyInstaller one-file bootloader, wiped on every + // (re)spawn: even a hard-killed or power-lost daemon can never accumulate + // stale 48M `_MEI*` extraction dirs, and they land under the data dir + // instead of the small tmpfs /tmp. Per-interface so the live daemons of + // two radios never share a dir (the previous daemon for the SAME + // interface is guaranteed dead before we respawn it). + let tmp_dir = runtime_dir.join("tmp").join(&iface_key); + let _ = tokio::fs::remove_dir_all(&tmp_dir).await; + tokio::fs::create_dir_all(&tmp_dir) + .await + .context("Failed to create reticulum daemon tmp dir")?; let identity_key = data_dir.join("identity").join("node_key"); if !identity_key.exists() { anyhow::bail!( @@ -345,50 +377,62 @@ impl ReticulumLink { our_ed_pubkey_hex, our_x25519_pubkey_hex, ); - let mut child = cmd + cmd.env("TMPDIR", &tmp_dir); + let child = cmd .spawn() .context("Failed to spawn reticulum-daemon — is it installed/packaged?")?; // Wait for the socket to appear, then for the daemon's "ready" event. - let deadline = tokio::time::Instant::now() + Duration::from_secs(15); - let stream = loop { - if tokio::time::Instant::now() > deadline { - let _ = child.start_kill(); - anyhow::bail!("reticulum-daemon did not create its RPC socket in time"); + // Runs as a block so every failure path tears the just-spawned daemon + // group down via `terminate_group` (the child has no `kill_on_drop`). + let init = async { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let stream = loop { + if tokio::time::Instant::now() > deadline { + anyhow::bail!("reticulum-daemon did not create its RPC socket in time"); + } + match UnixStream::connect(&socket_path).await { + Ok(s) => break s, + Err(_) => tokio::time::sleep(Duration::from_millis(150)).await, + } + }; + let (read_half, write_half) = stream.into_split(); + let mut reader = BufReader::new(read_half); + + let mut line = String::new(); + tokio::time::timeout(Duration::from_secs(10), reader.read_line(&mut line)) + .await + .context("Timed out waiting for reticulum-daemon ready event")? + .context("reticulum-daemon RPC connection closed before ready")?; + let ready: Value = serde_json::from_str(line.trim()) + .context("reticulum-daemon sent a non-JSON ready line")?; + if ready.get("event").and_then(Value::as_str) != Some("ready") { + anyhow::bail!("reticulum-daemon's first message was not 'ready': {ready}"); } - match UnixStream::connect(&socket_path).await { - Ok(s) => break s, - Err(_) => tokio::time::sleep(Duration::from_millis(150)).await, + let dest_hash_hex = ready + .get("dest_hash") + .and_then(Value::as_str) + .context("ready event missing dest_hash")?; + let dest_hash = parse_hash16(dest_hash_hex)?; + let display_name = ready + .get("display_name") + .and_then(Value::as_str) + .map(str::to_string); + + info!( + iface = %label, + dest_hash = %dest_hash_hex, + "Reticulum daemon ready" + ); + Ok((write_half, reader, dest_hash, display_name)) + }; + let (write_half, reader, dest_hash, display_name) = match init.await { + Ok(parts) => parts, + Err(e) => { + terminate_group(&child); + return Err(e); } }; - let (read_half, write_half) = stream.into_split(); - let mut reader = BufReader::new(read_half); - - let mut line = String::new(); - tokio::time::timeout(Duration::from_secs(10), reader.read_line(&mut line)) - .await - .context("Timed out waiting for reticulum-daemon ready event")? - .context("reticulum-daemon RPC connection closed before ready")?; - let ready: Value = serde_json::from_str(line.trim()) - .context("reticulum-daemon sent a non-JSON ready line")?; - if ready.get("event").and_then(Value::as_str) != Some("ready") { - anyhow::bail!("reticulum-daemon's first message was not 'ready': {ready}"); - } - let dest_hash_hex = ready - .get("dest_hash") - .and_then(Value::as_str) - .context("ready event missing dest_hash")?; - let dest_hash = parse_hash16(dest_hash_hex)?; - let display_name = ready - .get("display_name") - .and_then(Value::as_str) - .map(str::to_string); - - info!( - iface = %label, - dest_hash = %dest_hash_hex, - "Reticulum daemon ready" - ); let mut link = Self { device_path: label, @@ -1078,22 +1122,12 @@ fn contains_detect_resp(buf: &[u8]) -> bool { impl Drop for ReticulumLink { fn drop(&mut self) { - // The packaged daemon is a PyInstaller one-file bootloader that forks the - // real Python process into the same (leader) group we spawned it in. - // SIGKILLing only the bootloader (what `start_kill`/`kill_on_drop` do) - // orphans that child, which keeps holding the serial port — the root - // cause of daemons piling up across reconnects and jamming the RNode. - // Signal the whole process group instead: SIGTERM is caught by the - // daemon's handler (clean RNode + socket release), and SIGKILL is the - // hard backstop so a wedged daemon can never survive. - if let Some(pid) = self.child.id() { - let pgid = -(pid as i32); - unsafe { - libc::kill(pgid, libc::SIGTERM); - libc::kill(pgid, libc::SIGKILL); - } - } - let _ = self.child.start_kill(); + // Group-wide SIGTERM with a delayed SIGKILL backstop (`terminate_group`). + // The old immediate SIGTERM+SIGKILL never let the PyInstaller bootloader + // run its exit cleanup, stranding a 48M `_MEI*` extraction dir on every + // reconnect/restart until /tmp filled; the grace period fixes that, and + // the per-spawn TMPDIR wipe in `spawn` covers any dir that still leaks. + terminate_group(&self.child); let _ = std::fs::remove_file(&self.socket_path); } }