lfg2025/archy
1
0
Fork 0
Code Issues 26 Pull Requests 4 Actions Packages Projects Releases 74 Wiki Activity

backend: harden rootless app lifecycle orchestration

Browse Source
This commit is contained in:
archipelago
2026-06-11 00:24:32 -04:00
parent 09ec64932f
commit c393b96da3
56 changed files with 7543 additions and 1994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/server.rs
+190 -8
View File
@@ -17,6 +17,7 @@ use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tracing::{debug, error, info, warn};
@@ -331,6 +332,7 @@ impl Server {
// lifecycle op, and to break out if the spawned task dies
// without ever writing a final state.
let mut transitional_since: HashMap<String, Instant> = HashMap::new();
let mut scan_backoff_until: Option<Instant> = None;
if let Err(e) = scan_and_update_packages(
&scanner,
&state,
@@ -342,6 +344,10 @@ impl Server {
.await
{
error!("Failed to scan containers: {}", e);
if is_podman_scan_timeout(&e) {
scan_backoff_until = Some(Instant::now() + Duration::from_secs(30));
warn!("Podman container scan timed out; backing off scans for 30s");
}
}
// Bump the scan-completion counter so any caller waiting on a
// kicked scan (install/update success path) can proceed.
@@ -364,8 +370,16 @@ impl Server {
debug!("Scan kicked by install/update success — running immediately");
}
}
if let Some(until) = scan_backoff_until {
if Instant::now() < until {
debug!("Skipping container scan — Podman scan backoff active");
scan_tick.send_modify(|n| *n = n.wrapping_add(1));
continue;
}
}
if scanning.load(std::sync::atomic::Ordering::Relaxed) {
debug!("Skipping container scan — previous scan still in progress");
scan_tick.send_modify(|n| *n = n.wrapping_add(1));
continue;
}
scanning.store(true, std::sync::atomic::Ordering::Relaxed);
@@ -380,6 +394,12 @@ impl Server {
.await
{
error!("Failed to update containers: {}", e);
if is_podman_scan_timeout(&e) {
scan_backoff_until = Some(Instant::now() + Duration::from_secs(30));
warn!("Podman container scan timed out; backing off scans for 30s");
}
} else {
scan_backoff_until = None;
}
scan_tick.send_modify(|n| *n = n.wrapping_add(1));
scanning.store(false, std::sync::atomic::Ordering::Relaxed);
@@ -847,10 +867,10 @@ const TRANSITIONAL_STUCK_TIMEOUT: Duration = Duration::from_secs(120);
const INSTALLING_STUCK_TIMEOUT: Duration = Duration::from_secs(20 * 60);
fn transitional_stuck_timeout(state: &crate::data_model::PackageState) -> Duration {
if *state == crate::data_model::PackageState::Installing {
INSTALLING_STUCK_TIMEOUT
} else {
TRANSITIONAL_STUCK_TIMEOUT
use crate::data_model::PackageState::*;
match state {
Installing | Starting | Restarting => INSTALLING_STUCK_TIMEOUT,
_ => TRANSITIONAL_STUCK_TIMEOUT,
}
}
@@ -874,6 +894,18 @@ fn is_transitional(state: &crate::data_model::PackageState) -> bool {
)
}
fn absent_transitional_replacement(
state: &crate::data_model::PackageState,
) -> Option<crate::data_model::PackageState> {
match state {
// A stop operation is complete once the container record disappears.
// Do not leave the app card wedged in "Stopping..." just because the
// background task died or the backend restarted before it wrote back.
crate::data_model::PackageState::Stopping => Some(crate::data_model::PackageState::Stopped),
_ => None,
}
}
/// Merge a fresh scan entry `fresh` into `existing` while preserving
/// `existing.state` (which is transitional — the RPC spawn task owns it).
/// Non-state observability fields are taken from `fresh` so the UI still
@@ -881,8 +913,17 @@ fn is_transitional(state: &crate::data_model::PackageState) -> bool {
fn merge_preserving_transitional(
existing: &crate::data_model::PackageDataEntry,
fresh: &crate::data_model::PackageDataEntry,
user_stop_requested: bool,
) -> crate::data_model::PackageDataEntry {
let state = match (&existing.state, &fresh.state) {
// A user-initiated stop must keep showing Stopping while podman still
// reports Running. Repair/restart transitions do not have a user-stop
// marker, so a fresh Running scan means the app recovered.
(crate::data_model::PackageState::Stopping, crate::data_model::PackageState::Running)
if !user_stop_requested =>
{
fresh.state.clone()
}
// Removing with a live running container is stale: uninstall either
// failed or Archipelago restarted before the spawned task could revert
// state. Let the scanner recover the UI immediately instead of
@@ -909,6 +950,11 @@ fn merge_preserving_transitional(
}
}
fn is_podman_scan_timeout(error: &anyhow::Error) -> bool {
let msg = format!("{:#}", error);
msg.contains("podman ps") && msg.contains("timed out")
}
async fn scan_and_update_packages(
scanner: &DockerPackageScanner,
state: &StateManager,
@@ -925,6 +971,7 @@ async fn scan_and_update_packages(
pkg.exit_code = None;
}
}
normalize_reachable_package_health(&mut packages).await;
let (current_data, _) = state.get_snapshot().await;
let tor_addr = docker_packages::read_tor_address("archipelago").await;
@@ -992,7 +1039,11 @@ async fn scan_and_update_packages(
// observability fields (health, exit_code, lan_address
// via installed) from the fresh scan so the UI still
// sees live readings.
let merged_entry = merge_preserving_transitional(existing_entry, pkg);
let merged_entry = merge_preserving_transitional(
existing_entry,
pkg,
user_stopped.contains(id),
);
if existing.cloned() != Some(merged_entry.clone()) {
merged.insert(id.clone(), merged_entry);
changed = true;
@@ -1029,6 +1080,19 @@ async fn scan_and_update_packages(
// owner (spawn_task) is responsible for clearing state, not us.
if let Some(entry) = merged.get(&id) {
if is_transitional(&entry.state) {
if let Some(replacement) = absent_transitional_replacement(&entry.state) {
let mut updated = entry.clone();
updated.state = replacement;
updated.health = None;
updated.exit_code = None;
updated.install_progress = None;
updated.uninstall_stage = None;
merged.insert(id.clone(), updated);
transitional_since.remove(&id);
absence_tracker.remove(&id);
changed = true;
continue;
}
let entered = *transitional_since.entry(id.clone()).or_insert(now);
let timeout = transitional_stuck_timeout(&entry.state);
if now.duration_since(entered) > timeout {
@@ -1088,6 +1152,99 @@ async fn scan_and_update_packages(
Ok(())
}
async fn normalize_reachable_package_health(
packages: &mut HashMap<String, crate::data_model::PackageDataEntry>,
) {
for (id, pkg) in packages.iter_mut() {
if pkg.state != crate::data_model::PackageState::Running {
continue;
}
if !matches!(pkg.health.as_deref(), Some("starting" | "unhealthy" | "1")) {
continue;
}
let Some(port) = pkg
.installed
.as_ref()
.and_then(|i| i.interface_addresses.get("main"))
.and_then(|a| a.lan_address.as_deref())
.and_then(port_from_url)
.or_else(|| fallback_package_port(id))
else {
continue;
};
if frontend_port_http_ready(port).await {
debug!(app_id = %id, port, "normalizing reachable package health to healthy");
pkg.health = Some("healthy".to_string());
ensure_main_lan_address(pkg, port);
}
}
}
async fn frontend_port_http_ready(port: u16) -> bool {
let Ok(Ok(mut stream)) = tokio::time::timeout(
Duration::from_secs(2),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await
else {
return false;
};
let request = b"GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n";
if stream.write_all(request).await.is_err() {
return false;
}
let mut buf = [0u8; 64];
let Ok(Ok(n)) = tokio::time::timeout(Duration::from_secs(2), stream.read(&mut buf)).await
else {
return false;
};
if n == 0 {
return false;
}
let head = String::from_utf8_lossy(&buf[..n]);
head.starts_with("HTTP/1.1 2")
|| head.starts_with("HTTP/1.1 3")
|| head.starts_with("HTTP/1.0 2")
|| head.starts_with("HTTP/1.0 3")
}
fn ensure_main_lan_address(pkg: &mut crate::data_model::PackageDataEntry, port: u16) {
let Some(installed) = pkg.installed.as_mut() else {
return;
};
let main = installed
.interface_addresses
.entry("main".to_string())
.or_insert_with(|| crate::data_model::InterfaceAddress {
tor_address: String::new(),
lan_address: None,
});
if main.lan_address.is_none() {
main.lan_address = Some(format!("http://localhost:{port}"));
}
}
fn fallback_package_port(app_id: &str) -> Option<u16> {
match app_id {
"fedimint" | "fedimintd" => Some(8175),
"filebrowser" => Some(8083),
"indeedhub" => Some(7778),
"nginx-proxy-manager" => Some(8081),
"nostr-rs-relay" => Some(18081),
_ => None,
}
}
fn port_from_url(url: &str) -> Option<u16> {
let after_scheme = url.split_once("://").map(|(_, rest)| rest).unwrap_or(url);
let host_port = after_scheme.split('/').next().unwrap_or(after_scheme);
let port = host_port.rsplit_once(':')?.1;
port.parse::<u16>().ok()
}
/// Register Archipelago DWN protocols on startup.
async fn register_dwn_protocols(data_dir: &std::path::Path) -> Result<()> {
use crate::network::dwn_store::{DwnStore, ProtocolDefinition};
@@ -1211,10 +1368,19 @@ mod merge_tests {
// not clobber the transitional state owned by the RPC spawn task.
let existing = make_entry(PackageState::Stopping, Some("healthy"));
let fresh = make_entry(PackageState::Running, Some("starting"));
let merged = merge_preserving_transitional(&existing, &fresh);
let merged = merge_preserving_transitional(&existing, &fresh, true);
assert_eq!(merged.state, PackageState::Stopping);
}
#[test]
fn non_user_stopping_recovers_when_container_is_running() {
let existing = make_entry(PackageState::Stopping, Some("unknown"));
let fresh = make_entry(PackageState::Running, Some("healthy"));
let merged = merge_preserving_transitional(&existing, &fresh, false);
assert_eq!(merged.state, PackageState::Running);
assert_eq!(merged.health.as_deref(), Some("healthy"));
}
#[test]
fn merges_fresh_observability_fields() {
// Non-state observability fields (health, exit_code, installed)
@@ -1224,7 +1390,7 @@ mod merge_tests {
existing.exit_code = None;
let mut fresh = make_entry(PackageState::Running, Some("unhealthy"));
fresh.exit_code = Some(0);
let merged = merge_preserving_transitional(&existing, &fresh);
let merged = merge_preserving_transitional(&existing, &fresh, true);
assert_eq!(merged.state, PackageState::Stopping);
assert_eq!(merged.health.as_deref(), Some("unhealthy"));
assert_eq!(merged.exit_code, Some(0));
@@ -1234,7 +1400,7 @@ mod merge_tests {
fn stale_removing_recovers_when_container_is_running() {
let existing = make_entry(PackageState::Removing, Some("unknown"));
let fresh = make_entry(PackageState::Running, Some("healthy"));
let merged = merge_preserving_transitional(&existing, &fresh);
let merged = merge_preserving_transitional(&existing, &fresh, false);
assert_eq!(merged.state, PackageState::Running);
assert_eq!(merged.health.as_deref(), Some("healthy"));
}
@@ -1272,4 +1438,20 @@ mod merge_tests {
TRANSITIONAL_STUCK_TIMEOUT
);
}
#[test]
fn absent_stopping_transitions_to_stopped() {
assert_eq!(
absent_transitional_replacement(&PackageState::Stopping),
Some(PackageState::Stopped)
);
}
#[test]
fn absent_installing_still_waits_for_owner() {
assert_eq!(
absent_transitional_replacement(&PackageState::Installing),
None
);
}
}