lfg2025/archy
1
0
Fork 0
Code Issues 27 Pull Requests 4 Actions Packages Projects Releases 74 Wiki Activity

backend: harden rootless app lifecycle orchestration

Browse Source
This commit is contained in:
archipelago
2026-06-11 00:24:32 -04:00
parent 09ec64932f
commit c393b96da3
56 changed files with 7543 additions and 1994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/system/handlers.rs
+125 -22
View File
@@ -1,7 +1,7 @@
use super::*;
use crate::api::rpc::RpcHandler;
use anyhow::{Context, Result};
use tracing::{debug, info};
use tracing::{debug, info, warn};
impl RpcHandler {
/// server.set-name — Rename the server (persisted to data_dir/server-name)
@@ -32,6 +32,21 @@ impl RpcHandler {
data.server_info.name = Some(name.clone());
self.state_manager.update_data(data).await;
let hostname = hostname_from_server_name(&name);
let hostname_result = set_system_hostname(&hostname).await;
let (hostname_updated, hostname_error) = match hostname_result {
Ok(()) => (true, None),
Err(e) => {
warn!(
name = %name,
hostname = %hostname,
"Server name persisted but OS hostname update failed: {}",
e
);
(false, Some(e.to_string()))
}
};
info!("Server name updated to: {}", name);
// Push the new name to federation peers in background
@@ -43,7 +58,12 @@ impl RpcHandler {
}
});
Ok(serde_json::json!({ "name": name }))
Ok(serde_json::json!({
"name": name,
"hostname": hostname,
"hostname_updated": hostname_updated,
"hostname_error": hostname_error,
}))
}
/// system.stats — CPU usage, RAM used/total, disk used/total, uptime, load average
@@ -155,21 +175,7 @@ impl RpcHandler {
let mut freed_bytes: u64 = 0;
let mut actions: Vec<String> = Vec::new();
// 1. Prune dangling container images
match prune_container_images().await {
Ok(bytes) => {
if bytes > 0 {
freed_bytes += bytes;
actions.push(format!(
"Pruned dangling images: {} freed",
format_bytes(bytes)
));
}
}
Err(e) => actions.push(format!("Image prune failed: {}", e)),
}
// 2. Clean old log files (> 30 days)
// 1. Clean old log files (> 30 days)
match clean_old_logs(30).await {
Ok(bytes) => {
if bytes > 0 {
@@ -180,7 +186,20 @@ impl RpcHandler {
Err(e) => actions.push(format!("Log cleanup failed: {}", e)),
}
// 3. Remove stale temp files
match vacuum_journal_logs("200M").await {
Ok(bytes) => {
if bytes > 0 {
freed_bytes += bytes;
actions.push(format!(
"Vacuumed journal logs: {} freed",
format_bytes(bytes)
));
}
}
Err(e) => actions.push(format!("Journal cleanup failed: {}", e)),
}
// 2. Remove stale temp files
match clean_temp_files().await {
Ok(bytes) => {
if bytes > 0 {
@@ -191,17 +210,53 @@ impl RpcHandler {
Err(e) => actions.push(format!("Temp cleanup failed: {}", e)),
}
// 4. Prune container build cache
match prune_build_cache().await {
// 3. Keep only the most recent backend deploy backups. These are useful
// for rollback, but a long-lived alpha node can accumulate gigabytes of
// old binaries under /usr/local/bin.
match clean_backend_backups(3).await {
Ok(bytes) => {
if bytes > 0 {
freed_bytes += bytes;
actions.push(format!("Pruned build cache: {} freed", format_bytes(bytes)));
actions.push(format!(
"Removed old backend backups: {} freed",
format_bytes(bytes)
));
}
}
Err(e) => actions.push(format!("Build cache prune failed: {}", e)),
Err(e) => actions.push(format!("Backend backup cleanup failed: {}", e)),
}
match clean_legacy_backend_backups(3).await {
Ok(bytes) => {
if bytes > 0 {
freed_bytes += bytes;
actions.push(format!(
"Removed old legacy backend backups: {} freed",
format_bytes(bytes)
));
}
}
Err(e) => actions.push(format!("Legacy backend backup cleanup failed: {}", e)),
}
match clean_web_ui_backups(3).await {
Ok(bytes) => {
if bytes > 0 {
freed_bytes += bytes;
actions.push(format!(
"Removed old web UI backups: {} freed",
format_bytes(bytes)
));
}
}
Err(e) => actions.push(format!("Web UI backup cleanup failed: {}", e)),
}
actions.push(
"Skipped Podman image/volume prune: Podman store commands can block app health on busy nodes"
.to_string(),
);
tracing::info!(
"Disk cleanup complete: {} freed ({} actions)",
format_bytes(freed_bytes),
@@ -216,6 +271,54 @@ impl RpcHandler {
}
}
pub(super) fn hostname_from_server_name(name: &str) -> String {
let mut hostname = String::with_capacity(name.len());
let mut previous_dash = false;
for c in name.trim().chars().flat_map(char::to_lowercase) {
let valid = c.is_ascii_lowercase() || c.is_ascii_digit();
if valid {
hostname.push(c);
previous_dash = false;
} else if !previous_dash {
hostname.push('-');
previous_dash = true;
}
if hostname.len() >= 63 {
break;
}
}
let hostname = hostname.trim_matches('-').to_string();
if hostname.is_empty() {
"archipelago".to_string()
} else {
hostname
}
}
async fn set_system_hostname(hostname: &str) -> Result<()> {
let output = tokio::process::Command::new("/usr/bin/sudo")
.args(["-n", "/usr/bin/hostnamectl", "set-hostname", hostname])
.output()
.await
.context("Failed to run hostnamectl")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
anyhow::bail!(
"{}",
if stderr.is_empty() {
"hostnamectl failed".to_string()
} else {
stderr
}
);
}
Ok(())
}
impl RpcHandler {
/// system.factory-reset — Wipe all user data, remove containers, and restart.
/// Only preserves the data_dir itself (recreated empty on restart).
core/archipelago/src/api/rpc/system/mod.rs
+346 -47
View File
@@ -1,6 +1,9 @@
mod handlers;
use crate::update::host_sudo;
use anyhow::{Context, Result};
use std::path::{Path, PathBuf};
use std::time::SystemTime;
use tracing::{debug, info};
/// Push the server name to all federation peers by syncing state.
@@ -301,53 +304,12 @@ pub(super) async fn detect_usb_hardware_wallets() -> Result<Vec<serde_json::Valu
Ok(devices)
}
/// Prune dangling container images via `podman image prune -f`.
/// Returns estimated bytes freed.
pub(super) async fn prune_container_images() -> Result<u64> {
let output = tokio::process::Command::new("podman")
.args(["image", "prune", "-f"])
.output()
.await
.context("Failed to run podman image prune")?;
if !output.status.success() {
anyhow::bail!(
"podman image prune failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
// Podman outputs image IDs, estimate ~100MB per pruned image
let stdout = String::from_utf8_lossy(&output.stdout);
let pruned_count = stdout.lines().filter(|l| !l.trim().is_empty()).count();
Ok(pruned_count as u64 * 100_000_000) // rough estimate
}
/// Prune container build cache via `podman system prune -f`.
pub(super) async fn prune_build_cache() -> Result<u64> {
// Just prune volumes and build cache (not containers or images — those are handled above)
let output = tokio::process::Command::new("podman")
.args(["volume", "prune", "-f"])
.output()
.await
.context("Failed to run podman volume prune")?;
if !output.status.success() {
anyhow::bail!(
"podman volume prune failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let pruned_count = stdout.lines().filter(|l| !l.trim().is_empty()).count();
Ok(pruned_count as u64 * 10_000_000) // rough estimate per volume
}
/// Clean log files older than `max_age_days` from common log directories.
pub(super) async fn clean_old_logs(max_age_days: u64) -> Result<u64> {
let output = tokio::process::Command::new("sudo")
let output = tokio::process::Command::new("timeout")
.args([
"60s",
"sudo",
"find",
"/var/log",
"-type",
@@ -366,8 +328,10 @@ pub(super) async fn clean_old_logs(max_age_days: u64) -> Result<u64> {
let stdout = String::from_utf8_lossy(&output.stdout);
let deleted_count = stdout.lines().filter(|l| !l.trim().is_empty()).count();
// Also clean rotated/compressed logs
let _ = tokio::process::Command::new("sudo")
let _ = tokio::process::Command::new("timeout")
.args([
"60s",
"sudo",
"find",
"/var/log",
"-type",
@@ -384,14 +348,81 @@ pub(super) async fn clean_old_logs(max_age_days: u64) -> Result<u64> {
Ok(deleted_count as u64 * 500_000) // rough estimate per log file
}
/// Vacuum systemd journals to a bounded size. Returns measured bytes freed.
pub(super) async fn vacuum_journal_logs(max_size: &str) -> Result<u64> {
let before = journal_disk_usage().await.unwrap_or(0);
let output = tokio::process::Command::new("timeout")
.args(["60s", "sudo", "journalctl", "--vacuum-size", max_size])
.output()
.await
.context("Failed to run journal vacuum")?;
if !output.status.success() {
anyhow::bail!(
"journal vacuum failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
let after = journal_disk_usage().await.unwrap_or(before);
Ok(before.saturating_sub(after))
}
async fn journal_disk_usage() -> Result<u64> {
let output = tokio::process::Command::new("sudo")
.args(["-n", "journalctl", "--disk-usage"])
.output()
.await
.context("Failed to read journal disk usage")?;
if !output.status.success() {
anyhow::bail!(
"journalctl --disk-usage failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
parse_journal_disk_usage(&String::from_utf8_lossy(&output.stdout))
.ok_or_else(|| anyhow::anyhow!("could not parse journal disk usage"))
}
fn parse_journal_disk_usage(output: &str) -> Option<u64> {
let mut parts = output.split_whitespace();
while let Some(part) = parts.next() {
let (number, inline_unit) = split_number_unit(part);
let Ok(value) = number.parse::<f64>() else {
continue;
};
let unit = inline_unit.unwrap_or_else(|| parts.next().unwrap_or_default());
let multiplier = match unit {
"B" | "bytes" => 1.0,
"K" | "KB" | "KiB" => 1024.0,
"M" | "MB" | "MiB" => 1024.0 * 1024.0,
"G" | "GB" | "GiB" => 1024.0 * 1024.0 * 1024.0,
_ => continue,
};
return Some((value * multiplier) as u64);
}
None
}
fn split_number_unit(value: &str) -> (&str, Option<&str>) {
let split_at = value
.char_indices()
.find_map(|(idx, ch)| (!ch.is_ascii_digit() && ch != '.').then_some(idx))
.unwrap_or(value.len());
let (number, unit) = value.split_at(split_at);
(number, (!unit.is_empty()).then_some(unit))
}
/// Remove stale temp files from /tmp and /var/tmp.
pub(super) async fn clean_temp_files() -> Result<u64> {
let mut freed = 0u64;
for dir in &["/tmp", "/var/tmp"] {
let output = tokio::process::Command::new("sudo")
let output = tokio::process::Command::new("timeout")
.args([
"find", dir, "-type", "f", "-mtime", "+7", "-delete", "-print",
"45s", "sudo", "find", dir, "-type", "f", "-mtime", "+7", "-delete", "-print",
])
.output()
.await;
@@ -406,6 +437,177 @@ pub(super) async fn clean_temp_files() -> Result<u64> {
Ok(freed)
}
/// Keep the newest timestamped backend backups and remove older ones.
pub(super) async fn clean_backend_backups(keep: usize) -> Result<u64> {
clean_backend_backups_in(Path::new("/usr/local/bin"), keep).await
}
/// Keep the newest legacy backend backups and remove older alpha-era deploy artifacts.
pub(super) async fn clean_legacy_backend_backups(keep: usize) -> Result<u64> {
clean_named_backups_in(
Path::new("/usr/local/bin"),
keep,
|name| name.starts_with("archipelago.bak") || name.starts_with("archipelago.before-"),
false,
)
.await
}
/// Keep the newest web UI rollback backups and remove older copies.
pub(super) async fn clean_web_ui_backups(keep: usize) -> Result<u64> {
clean_named_backups_in(
Path::new("/opt/archipelago"),
keep,
|name| name.starts_with("web-ui.bak") || name == "web-ui.old",
true,
)
.await
}
async fn clean_backend_backups_in(dir: &Path, keep: usize) -> Result<u64> {
let mut backups = backend_backup_candidates(dir).await?;
remove_old_backups(&mut backups, keep, false).await
}
async fn clean_named_backups_in(
dir: &Path,
keep: usize,
matches_name: impl Fn(&str) -> bool,
allow_dirs: bool,
) -> Result<u64> {
let mut backups = named_backup_candidates(dir, matches_name, allow_dirs).await?;
remove_old_backups(&mut backups, keep, allow_dirs).await
}
async fn remove_old_backups(
backups: &mut Vec<BackupArtifact>,
keep: usize,
allow_dirs: bool,
) -> Result<u64> {
backups.sort_by(|a, b| {
b.modified
.cmp(&a.modified)
.then_with(|| b.name.cmp(&a.name))
});
let mut freed = 0u64;
for backup in backups.iter().skip(keep) {
let remove_result = if backup.is_dir && allow_dirs {
tokio::fs::remove_dir_all(&backup.path).await
} else {
tokio::fs::remove_file(&backup.path).await
};
match remove_result {
Ok(()) => freed += backup.size,
Err(_) => {
remove_path_with_sudo(&backup.path, backup.is_dir && allow_dirs).await?;
freed += backup.size;
}
}
}
Ok(freed)
}
async fn remove_path_with_sudo(path: &Path, recursive: bool) -> Result<()> {
let path = path.to_string_lossy();
let args = if recursive {
vec!["rm", "-rf", path.as_ref()]
} else {
vec!["rm", "-f", path.as_ref()]
};
let status = host_sudo(&args)
.await
.with_context(|| format!("removing {path} via sudo"))?;
if !status.success() {
anyhow::bail!(
"sudo rm {} {path} exited with {status}",
if recursive { "-rf" } else { "-f" }
);
}
Ok(())
}
#[derive(Debug)]
struct BackupArtifact {
path: PathBuf,
name: String,
modified: SystemTime,
size: u64,
is_dir: bool,
}
async fn backend_backup_candidates(dir: &Path) -> Result<Vec<BackupArtifact>> {
named_backup_candidates(
dir,
|name| {
name.strip_prefix("archipelago.backup-")
.is_some_and(|suffix| !suffix.is_empty() && !suffix.contains('/'))
},
false,
)
.await
}
async fn named_backup_candidates(
dir: &Path,
matches_name: impl Fn(&str) -> bool,
allow_dirs: bool,
) -> Result<Vec<BackupArtifact>> {
let mut backups = Vec::new();
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(entries) => entries,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(backups),
Err(e) => return Err(e).with_context(|| format!("reading {}", dir.display())),
};
while let Some(entry) = entries.next_entry().await? {
let file_name = entry.file_name();
let name = file_name.to_string_lossy();
if !matches_name(&name) {
continue;
}
let meta = entry.metadata().await?;
if !meta.is_file() && !(allow_dirs && meta.is_dir()) {
continue;
}
backups.push(BackupArtifact {
path: entry.path(),
name: name.to_string(),
modified: meta.modified().unwrap_or(SystemTime::UNIX_EPOCH),
size: path_size(&entry.path(), &meta).await.unwrap_or(meta.len()),
is_dir: meta.is_dir(),
});
}
Ok(backups)
}
async fn path_size(path: &Path, meta: &std::fs::Metadata) -> Result<u64> {
if meta.is_file() {
return Ok(meta.len());
}
if !meta.is_dir() {
return Ok(0);
}
let output = tokio::process::Command::new("du")
.args(["-sb", &path.to_string_lossy()])
.output()
.await
.with_context(|| format!("du -sb {}", path.display()))?;
if !output.status.success() {
anyhow::bail!("du -sb {} failed", path.display());
}
let stdout = String::from_utf8_lossy(&output.stdout);
stdout
.split_whitespace()
.next()
.ok_or_else(|| anyhow::anyhow!("du output missing size for {}", path.display()))?
.parse::<u64>()
.with_context(|| format!("parse du size for {}", path.display()))
}
pub(super) fn format_bytes(bytes: u64) -> String {
const KB: u64 = 1024;
const MB: u64 = KB * 1024;
@@ -422,6 +624,103 @@ pub(super) fn format_bytes(bytes: u64) -> String {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn backend_backup_cleanup_keeps_newest_files() {
let dir = tempfile::tempdir().unwrap();
for name in [
"archipelago.backup-20260501",
"archipelago.backup-20260502",
"archipelago.backup-20260503",
"archipelago.backup-20260504",
"archipelago.backup-20260505",
"archipelago.bak",
"archipelago",
] {
tokio::fs::write(dir.path().join(name), b"12345")
.await
.unwrap();
}
let freed = clean_backend_backups_in(dir.path(), 3).await.unwrap();
assert_eq!(freed, 10);
assert!(!dir.path().join("archipelago.backup-20260501").exists());
assert!(!dir.path().join("archipelago.backup-20260502").exists());
assert!(dir.path().join("archipelago.backup-20260503").exists());
assert!(dir.path().join("archipelago.backup-20260504").exists());
assert!(dir.path().join("archipelago.backup-20260505").exists());
assert!(dir.path().join("archipelago.bak").exists());
assert!(dir.path().join("archipelago").exists());
}
#[tokio::test]
async fn legacy_backend_backup_cleanup_keeps_newest_matching_files() {
let dir = tempfile::tempdir().unwrap();
for name in [
"archipelago.bak-1",
"archipelago.bak-2",
"archipelago.before-3",
"archipelago.backup-keep-separate",
"archipelago",
] {
tokio::fs::write(dir.path().join(name), b"12345")
.await
.unwrap();
}
let freed = clean_named_backups_in(
dir.path(),
1,
|name| name.starts_with("archipelago.bak") || name.starts_with("archipelago.before-"),
false,
)
.await
.unwrap();
assert_eq!(freed, 10);
assert_eq!(
[
"archipelago.bak-1",
"archipelago.bak-2",
"archipelago.before-3"
]
.into_iter()
.filter(|name| dir.path().join(name).exists())
.count(),
1
);
assert!(dir.path().join("archipelago.backup-keep-separate").exists());
assert!(dir.path().join("archipelago").exists());
}
#[test]
fn hostname_from_server_name_derives_linux_safe_hostname() {
assert_eq!(
handlers::hostname_from_server_name("My Archipelago Node"),
"my-archipelago-node"
);
assert_eq!(
handlers::hostname_from_server_name("Kitchen_Node!! 01"),
"kitchen-node-01"
);
assert_eq!(handlers::hostname_from_server_name("!!!"), "archipelago");
}
#[test]
fn parses_journal_disk_usage() {
assert_eq!(
parse_journal_disk_usage(
"Archived and active journals take up 463.9M in the file system."
),
Some(486_434_406)
);
}
}
/// Read temperatures from /sys/class/thermal/thermal_zone*/temp.
pub(super) async fn read_temperatures() -> Result<Vec<serde_json::Value>> {
let mut temps = Vec::new();