fix(reticulum): stop leaking 48M _MEI dirs into /tmp on every daemon restart
The packaged reticulum-daemon is a PyInstaller one-file binary: the bootloader extracts a ~48M _MEI* dir into TMPDIR and only deletes it on clean exit. ReticulumLink's shutdown SIGKILLed the process group immediately (kill_on_drop + instant SIGTERM+SIGKILL in Drop), so the cleanup never ran and every reconnect/restart stranded another 48M in /tmp — filling .116's 12G tmpfs in ~4h. Two-pronged fix: - terminate_group(): group-wide SIGTERM now, SIGKILL from a detached backstop thread 5s later, used on every shutdown path (Drop and all spawn-failure paths, which previously used bare start_kill and could orphan the forked Python child too). - Per-interface private TMPDIR under the runtime dir, wiped on every (re)spawn — even a hard-killed or power-lost daemon can never accumulate stale extraction dirs, and they land on the data disk instead of the small tmpfs /tmp. Verified: unit tests + the ignored mesh_service_connects_over_ reticulum_tcp_client integration test (spawns two real daemons over loopback TCP through the changed spawn/terminate path), clean teardown with no stray processes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
c6b9e5d0c6
commit
9c02fbf164
@ -159,18 +159,41 @@ fn daemon_command(
|
||||
.arg("--archy-x25519-pubkey-hex")
|
||||
.arg(x);
|
||||
}
|
||||
cmd.kill_on_drop(true)
|
||||
// Run the daemon as its own process-group leader. The packaged binary is
|
||||
// a PyInstaller one-file bootloader that forks the real Python process;
|
||||
// making it a group leader lets Drop signal the WHOLE group so the
|
||||
// making it a group leader lets shutdown signal the WHOLE group so the
|
||||
// forked child can't be orphaned and keep holding the serial port.
|
||||
.process_group(0)
|
||||
// Deliberately NOT `kill_on_drop`: that SIGKILLs the bootloader the instant
|
||||
// the `Child` drops, before it can delete its `_MEI*` extraction dir (48M
|
||||
// leaked into TMPDIR per daemon restart — filled .116's 12G /tmp tmpfs).
|
||||
// Shutdown goes through `terminate_group` instead, on every path.
|
||||
cmd.process_group(0)
|
||||
.stdin(std::process::Stdio::null())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped());
|
||||
cmd
|
||||
}
|
||||
|
||||
/// Gracefully stop a daemon process group: SIGTERM now (the daemon's handler
|
||||
/// releases the RNode + socket, and the PyInstaller bootloader gets to delete
|
||||
/// its `_MEI*` extraction dir once the Python child exits), SIGKILL from a
|
||||
/// detached backstop thread a few seconds later so a wedged daemon still can't
|
||||
/// survive or keep holding the serial port.
|
||||
fn terminate_group(child: &Child) {
|
||||
if let Some(pid) = child.id() {
|
||||
let pgid = -(pid as i32);
|
||||
unsafe {
|
||||
libc::kill(pgid, libc::SIGTERM);
|
||||
}
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_secs(5));
|
||||
unsafe {
|
||||
libc::kill(pgid, libc::SIGKILL);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// One peer learned via an RNS announce (LXMF delivery destination).
|
||||
#[derive(Clone)]
|
||||
struct ReticulumPeer {
|
||||
@ -323,13 +346,22 @@ impl ReticulumLink {
|
||||
.await;
|
||||
}
|
||||
let label = iface.label();
|
||||
let socket_path = runtime_dir.join(format!(
|
||||
"{}.sock",
|
||||
label.replace(['/', ' ', ':', ','], "_")
|
||||
));
|
||||
let iface_key = label.replace(['/', ' ', ':', ','], "_");
|
||||
let socket_path = runtime_dir.join(format!("{iface_key}.sock"));
|
||||
if socket_path.exists() {
|
||||
let _ = std::fs::remove_file(&socket_path);
|
||||
}
|
||||
// Private TMPDIR for the PyInstaller one-file bootloader, wiped on every
|
||||
// (re)spawn: even a hard-killed or power-lost daemon can never accumulate
|
||||
// stale 48M `_MEI*` extraction dirs, and they land under the data dir
|
||||
// instead of the small tmpfs /tmp. Per-interface so the live daemons of
|
||||
// two radios never share a dir (the previous daemon for the SAME
|
||||
// interface is guaranteed dead before we respawn it).
|
||||
let tmp_dir = runtime_dir.join("tmp").join(&iface_key);
|
||||
let _ = tokio::fs::remove_dir_all(&tmp_dir).await;
|
||||
tokio::fs::create_dir_all(&tmp_dir)
|
||||
.await
|
||||
.context("Failed to create reticulum daemon tmp dir")?;
|
||||
let identity_key = data_dir.join("identity").join("node_key");
|
||||
if !identity_key.exists() {
|
||||
anyhow::bail!(
|
||||
@ -345,15 +377,18 @@ impl ReticulumLink {
|
||||
our_ed_pubkey_hex,
|
||||
our_x25519_pubkey_hex,
|
||||
);
|
||||
let mut child = cmd
|
||||
cmd.env("TMPDIR", &tmp_dir);
|
||||
let child = cmd
|
||||
.spawn()
|
||||
.context("Failed to spawn reticulum-daemon — is it installed/packaged?")?;
|
||||
|
||||
// Wait for the socket to appear, then for the daemon's "ready" event.
|
||||
// Runs as a block so every failure path tears the just-spawned daemon
|
||||
// group down via `terminate_group` (the child has no `kill_on_drop`).
|
||||
let init = async {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
|
||||
let stream = loop {
|
||||
if tokio::time::Instant::now() > deadline {
|
||||
let _ = child.start_kill();
|
||||
anyhow::bail!("reticulum-daemon did not create its RPC socket in time");
|
||||
}
|
||||
match UnixStream::connect(&socket_path).await {
|
||||
@ -389,6 +424,15 @@ impl ReticulumLink {
|
||||
dest_hash = %dest_hash_hex,
|
||||
"Reticulum daemon ready"
|
||||
);
|
||||
Ok((write_half, reader, dest_hash, display_name))
|
||||
};
|
||||
let (write_half, reader, dest_hash, display_name) = match init.await {
|
||||
Ok(parts) => parts,
|
||||
Err(e) => {
|
||||
terminate_group(&child);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
let mut link = Self {
|
||||
device_path: label,
|
||||
@ -1078,22 +1122,12 @@ fn contains_detect_resp(buf: &[u8]) -> bool {
|
||||
|
||||
impl Drop for ReticulumLink {
|
||||
fn drop(&mut self) {
|
||||
// The packaged daemon is a PyInstaller one-file bootloader that forks the
|
||||
// real Python process into the same (leader) group we spawned it in.
|
||||
// SIGKILLing only the bootloader (what `start_kill`/`kill_on_drop` do)
|
||||
// orphans that child, which keeps holding the serial port — the root
|
||||
// cause of daemons piling up across reconnects and jamming the RNode.
|
||||
// Signal the whole process group instead: SIGTERM is caught by the
|
||||
// daemon's handler (clean RNode + socket release), and SIGKILL is the
|
||||
// hard backstop so a wedged daemon can never survive.
|
||||
if let Some(pid) = self.child.id() {
|
||||
let pgid = -(pid as i32);
|
||||
unsafe {
|
||||
libc::kill(pgid, libc::SIGTERM);
|
||||
libc::kill(pgid, libc::SIGKILL);
|
||||
}
|
||||
}
|
||||
let _ = self.child.start_kill();
|
||||
// Group-wide SIGTERM with a delayed SIGKILL backstop (`terminate_group`).
|
||||
// The old immediate SIGTERM+SIGKILL never let the PyInstaller bootloader
|
||||
// run its exit cleanup, stranding a 48M `_MEI*` extraction dir on every
|
||||
// reconnect/restart until /tmp filled; the grace period fixes that, and
|
||||
// the per-spawn TMPDIR wipe in `spawn` covers any dir that still leaks.
|
||||
terminate_group(&self.child);
|
||||
let _ = std::fs::remove_file(&self.socket_path);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user