fix(reticulum): stop leaking 48M _MEI dirs into /tmp on every daemon restart

The packaged reticulum-daemon is a PyInstaller one-file binary: the
bootloader extracts a ~48M _MEI* dir into TMPDIR and only deletes it on
clean exit. ReticulumLink's shutdown SIGKILLed the process group
immediately (kill_on_drop + instant SIGTERM+SIGKILL in Drop), so the
cleanup never ran and every reconnect/restart stranded another 48M in
/tmp — filling .116's 12G tmpfs in ~4h.

Two-pronged fix:
- terminate_group(): group-wide SIGTERM now, SIGKILL from a detached
  backstop thread 5s later, used on every shutdown path (Drop and all
  spawn-failure paths, which previously used bare start_kill and could
  orphan the forked Python child too).
- Per-interface private TMPDIR under the runtime dir, wiped on every
  (re)spawn — even a hard-killed or power-lost daemon can never
  accumulate stale extraction dirs, and they land on the data disk
  instead of the small tmpfs /tmp.

Verified: unit tests + the ignored mesh_service_connects_over_
reticulum_tcp_client integration test (spawns two real daemons over
loopback TCP through the changed spawn/terminate path), clean teardown
with no stray processes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-08 14:55:19 -04:00
parent c6b9e5d0c6
commit 9c02fbf164

View File

@ -159,18 +159,41 @@ fn daemon_command(
.arg("--archy-x25519-pubkey-hex")
.arg(x);
}
cmd.kill_on_drop(true)
// Run the daemon as its own process-group leader. The packaged binary is
// a PyInstaller one-file bootloader that forks the real Python process;
// making it a group leader lets Drop signal the WHOLE group so the
// making it a group leader lets shutdown signal the WHOLE group so the
// forked child can't be orphaned and keep holding the serial port.
.process_group(0)
// Deliberately NOT `kill_on_drop`: that SIGKILLs the bootloader the instant
// the `Child` drops, before it can delete its `_MEI*` extraction dir (48M
// leaked into TMPDIR per daemon restart — filled .116's 12G /tmp tmpfs).
// Shutdown goes through `terminate_group` instead, on every path.
cmd.process_group(0)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
cmd
}
/// Gracefully stop a daemon process group: SIGTERM now (the daemon's handler
/// releases the RNode + socket, and the PyInstaller bootloader gets to delete
/// its `_MEI*` extraction dir once the Python child exits), SIGKILL from a
/// detached backstop thread a few seconds later so a wedged daemon still can't
/// survive or keep holding the serial port.
fn terminate_group(child: &Child) {
if let Some(pid) = child.id() {
let pgid = -(pid as i32);
unsafe {
libc::kill(pgid, libc::SIGTERM);
}
std::thread::spawn(move || {
std::thread::sleep(Duration::from_secs(5));
unsafe {
libc::kill(pgid, libc::SIGKILL);
}
});
}
}
/// One peer learned via an RNS announce (LXMF delivery destination).
#[derive(Clone)]
struct ReticulumPeer {
@ -323,13 +346,22 @@ impl ReticulumLink {
.await;
}
let label = iface.label();
let socket_path = runtime_dir.join(format!(
"{}.sock",
label.replace(['/', ' ', ':', ','], "_")
));
let iface_key = label.replace(['/', ' ', ':', ','], "_");
let socket_path = runtime_dir.join(format!("{iface_key}.sock"));
if socket_path.exists() {
let _ = std::fs::remove_file(&socket_path);
}
// Private TMPDIR for the PyInstaller one-file bootloader, wiped on every
// (re)spawn: even a hard-killed or power-lost daemon can never accumulate
// stale 48M `_MEI*` extraction dirs, and they land under the data dir
// instead of the small tmpfs /tmp. Per-interface so the live daemons of
// two radios never share a dir (the previous daemon for the SAME
// interface is guaranteed dead before we respawn it).
let tmp_dir = runtime_dir.join("tmp").join(&iface_key);
let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
tokio::fs::create_dir_all(&tmp_dir)
.await
.context("Failed to create reticulum daemon tmp dir")?;
let identity_key = data_dir.join("identity").join("node_key");
if !identity_key.exists() {
anyhow::bail!(
@ -345,15 +377,18 @@ impl ReticulumLink {
our_ed_pubkey_hex,
our_x25519_pubkey_hex,
);
let mut child = cmd
cmd.env("TMPDIR", &tmp_dir);
let child = cmd
.spawn()
.context("Failed to spawn reticulum-daemon — is it installed/packaged?")?;
// Wait for the socket to appear, then for the daemon's "ready" event.
// Runs as a block so every failure path tears the just-spawned daemon
// group down via `terminate_group` (the child has no `kill_on_drop`).
let init = async {
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
let stream = loop {
if tokio::time::Instant::now() > deadline {
let _ = child.start_kill();
anyhow::bail!("reticulum-daemon did not create its RPC socket in time");
}
match UnixStream::connect(&socket_path).await {
@ -389,6 +424,15 @@ impl ReticulumLink {
dest_hash = %dest_hash_hex,
"Reticulum daemon ready"
);
Ok((write_half, reader, dest_hash, display_name))
};
let (write_half, reader, dest_hash, display_name) = match init.await {
Ok(parts) => parts,
Err(e) => {
terminate_group(&child);
return Err(e);
}
};
let mut link = Self {
device_path: label,
@ -1078,22 +1122,12 @@ fn contains_detect_resp(buf: &[u8]) -> bool {
impl Drop for ReticulumLink {
fn drop(&mut self) {
// The packaged daemon is a PyInstaller one-file bootloader that forks the
// real Python process into the same (leader) group we spawned it in.
// SIGKILLing only the bootloader (what `start_kill`/`kill_on_drop` do)
// orphans that child, which keeps holding the serial port — the root
// cause of daemons piling up across reconnects and jamming the RNode.
// Signal the whole process group instead: SIGTERM is caught by the
// daemon's handler (clean RNode + socket release), and SIGKILL is the
// hard backstop so a wedged daemon can never survive.
if let Some(pid) = self.child.id() {
let pgid = -(pid as i32);
unsafe {
libc::kill(pgid, libc::SIGTERM);
libc::kill(pgid, libc::SIGKILL);
}
}
let _ = self.child.start_kill();
// Group-wide SIGTERM with a delayed SIGKILL backstop (`terminate_group`).
// The old immediate SIGTERM+SIGKILL never let the PyInstaller bootloader
// run its exit cleanup, stranding a 48M `_MEI*` extraction dir on every
// reconnect/restart until /tmp filled; the grace period fixes that, and
// the per-spawn TMPDIR wipe in `spawn` covers any dir that still leaks.
terminate_group(&self.child);
let _ = std::fs::remove_file(&self.socket_path);
}
}