lfg2025/archy
1
0
Fork 0
Code Issues 26 Pull Requests 4 Actions Packages Projects Releases 74 Wiki Activity

backend: harden rootless app lifecycle orchestration

Browse Source
This commit is contained in:
archipelago
2026-06-11 00:24:32 -04:00
parent 09ec64932f
commit c393b96da3
56 changed files with 7543 additions and 1994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/container.rs
+272 -51
View File
@@ -4,8 +4,9 @@ use super::RpcHandler;
use anyhow::{Context, Result};
use std::time::Duration;
const PODMAN_INSPECT_TIMEOUT: Duration = Duration::from_secs(10);
const PODMAN_PS_TIMEOUT: Duration = Duration::from_secs(10);
const PODMAN_INSPECT_TIMEOUT: Duration = Duration::from_secs(5);
const PODMAN_PS_TIMEOUT: Duration = Duration::from_secs(5);
const ORCHESTRATOR_HEALTH_TIMEOUT: Duration = Duration::from_secs(5);
impl RpcHandler {
pub(super) async fn handle_container_install(
@@ -171,46 +172,69 @@ impl RpcHandler {
// between "installed" and "not-installed" in the UI.
let (data, _) = self.state_manager.get_snapshot().await;
if data.server_info.status_info.containers_scanned && !data.package_data.is_empty() {
let containers: Vec<serde_json::Value> = data
.package_data
.iter()
.map(|(id, pkg)| {
// Keep this mapping in sync with the UI's
// ContainerStatus.state union in
// neode-ui/src/api/container-client.ts. The UI maps
// transitional variants to single-button labels
// (Stopping… / Starting… / Restarting…).
let state = match &pkg.state {
crate::data_model::PackageState::Running => "running",
crate::data_model::PackageState::Stopped => "stopped",
crate::data_model::PackageState::Exited => "exited",
crate::data_model::PackageState::Starting => "starting",
crate::data_model::PackageState::Stopping => "stopping",
crate::data_model::PackageState::Restarting => "restarting",
crate::data_model::PackageState::Installing => "installing",
crate::data_model::PackageState::Installed => "installed",
crate::data_model::PackageState::Updating => "updating",
crate::data_model::PackageState::Removing => "removing",
crate::data_model::PackageState::CreatingBackup => "creating-backup",
crate::data_model::PackageState::RestoringBackup => "restoring-backup",
crate::data_model::PackageState::BackingUp => "backing-up",
};
let lan = pkg
.installed
.as_ref()
.and_then(|i| i.interface_addresses.get("main"))
.and_then(|a| a.lan_address.as_deref());
serde_json::json!({
"id": id,
"name": id,
"state": state,
"image": "",
"created": "",
"ports": [],
"lan_address": lan,
})
})
.collect();
let mut containers = Vec::with_capacity(data.package_data.len());
for (id, pkg) in &data.package_data {
// Keep this mapping in sync with the UI's
// ContainerStatus.state union in
// neode-ui/src/api/container-client.ts. The UI maps
// transitional variants to single-button labels
// (Stopping… / Starting… / Restarting…).
let mut state = match &pkg.state {
crate::data_model::PackageState::Running => "running".to_string(),
crate::data_model::PackageState::Stopped => "stopped".to_string(),
crate::data_model::PackageState::Exited => "exited".to_string(),
crate::data_model::PackageState::Starting => "starting".to_string(),
crate::data_model::PackageState::Stopping => "stopping".to_string(),
crate::data_model::PackageState::Restarting => "restarting".to_string(),
crate::data_model::PackageState::Installing => "installing".to_string(),
crate::data_model::PackageState::Installed => "installed".to_string(),
crate::data_model::PackageState::Updating => "updating".to_string(),
crate::data_model::PackageState::Removing => "removing".to_string(),
crate::data_model::PackageState::CreatingBackup => {
"creating-backup".to_string()
}
crate::data_model::PackageState::RestoringBackup => {
"restoring-backup".to_string()
}
crate::data_model::PackageState::BackingUp => "backing-up".to_string(),
};
// Scanner backoff preserves cached package_data. Refresh stable
// states so callers do not see stale `running`/`exited` after
// health-monitor recovery or Quadlet --rm container removal.
if state == "running" && requires_launch_port_for_health(id) {
if !self.cached_reachable_health(id).await?.is_some() {
state = live_state_for_app(id)
.await
.unwrap_or("starting".to_string());
}
} else if should_refresh_cached_state(&state) {
if launch_port_reachable(id).await {
state = "running".to_string();
} else {
if let Some(live) = live_state_for_app(id).await {
state = live;
} else if quadlet_service_active(id).await {
state = "starting".to_string();
}
}
}
let lan = pkg
.installed
.as_ref()
.and_then(|i| i.interface_addresses.get("main"))
.and_then(|a| a.lan_address.as_deref());
containers.push(serde_json::json!({
"id": id,
"name": id,
"state": state,
"image": "",
"created": "",
"ports": [],
"lan_address": lan,
}));
}
return Ok(serde_json::json!(containers));
}
@@ -383,15 +407,33 @@ impl RpcHandler {
// If app_id is provided, get health for that app.
if let Some(params) = params {
if let Some(app_id) = params.get("app_id").and_then(|v| v.as_str()) {
if let Some(health) = self.cached_reachable_health(app_id).await? {
return Ok(serde_json::json!({ app_id: health }));
}
if let Some(health) = self.cached_state_health(app_id).await {
return Ok(serde_json::json!({ app_id: health }));
}
if requires_launch_port_for_health(app_id) {
return Ok(serde_json::json!({ app_id: "starting" }));
}
if let Some(health) = self.stack_health(app_id).await? {
return Ok(serde_json::json!({ app_id: health }));
}
let mut last_err: Option<anyhow::Error> = None;
for candidate in status_app_id_candidates(app_id) {
match orchestrator.health(&candidate).await {
Ok(health) => return Ok(serde_json::json!({ app_id: health })),
Err(e) => last_err = Some(e),
match tokio::time::timeout(
ORCHESTRATOR_HEALTH_TIMEOUT,
orchestrator.health(&candidate),
)
.await
{
Ok(Ok(health)) => return Ok(serde_json::json!({ app_id: health })),
Ok(Err(e)) => last_err = Some(e),
Err(_) => {}
}
}
for name in status_container_name_candidates(app_id) {
@@ -424,14 +466,19 @@ impl RpcHandler {
.and_then(|s| s.strip_suffix("-dev"))
.or_else(|| container.name.strip_prefix("archy-"))
.unwrap_or(container.name.as_str());
match orchestrator.health(app_id_candidate).await {
Ok(health) => {
match tokio::time::timeout(
ORCHESTRATOR_HEALTH_TIMEOUT,
orchestrator.health(app_id_candidate),
)
.await
{
Ok(Ok(health)) => {
health_map.insert(
app_id_candidate.to_string(),
serde_json::Value::String(health),
);
}
Err(_) => {
Ok(Err(_)) | Err(_) => {
health_map.insert(
app_id_candidate.to_string(),
serde_json::Value::String("unknown".to_string()),
@@ -443,6 +490,65 @@ impl RpcHandler {
Ok(serde_json::Value::Object(health_map))
}
async fn cached_state_health(&self, app_id: &str) -> Option<&'static str> {
let (data, _) = self.state_manager.get_snapshot().await;
let Some(pkg) = data.package_data.get(app_id) else {
if data.server_info.status_info.containers_scanned {
return Some("stopped");
}
return None;
};
match pkg.state {
crate::data_model::PackageState::Running => None,
crate::data_model::PackageState::Installing
| crate::data_model::PackageState::Installed
| crate::data_model::PackageState::Starting => Some("starting"),
crate::data_model::PackageState::Stopping
| crate::data_model::PackageState::Stopped
| crate::data_model::PackageState::Exited => Some("stopped"),
crate::data_model::PackageState::Removing => Some("removing"),
crate::data_model::PackageState::Restarting
| crate::data_model::PackageState::Updating
| crate::data_model::PackageState::CreatingBackup
| crate::data_model::PackageState::RestoringBackup
| crate::data_model::PackageState::BackingUp => Some("starting"),
}
}
async fn cached_reachable_health(&self, app_id: &str) -> Result<Option<String>> {
let (data, _) = self.state_manager.get_snapshot().await;
let pkg = data.package_data.get(app_id);
if matches!(
pkg.map(|pkg| &pkg.state),
Some(crate::data_model::PackageState::Removing)
) {
return Ok(None);
}
let url = pkg
.and_then(|pkg| pkg.installed.as_ref())
.and_then(|i| i.interface_addresses.get("main"))
.and_then(|a| a.lan_address.as_deref())
.map(ToOwned::to_owned)
.or_else(|| health_probe_url_for_app(app_id));
let Some(url) = url else {
return Ok(None);
};
if url.starts_with("http://") || url.starts_with("https://") {
return Ok(http_launch_url_reachable(&url)
.await
.then(|| "healthy".to_string()));
}
let Some(port) = port_from_url(&url) else {
return Ok(None);
};
Ok(launch_port_reachable_by_port(port)
.await
.then(|| "healthy".to_string()))
}
async fn stack_health(&self, app_id: &str) -> Result<Option<String>> {
let Some(members) = stack_health_members(app_id) else {
return Ok(None);
@@ -469,8 +575,14 @@ impl RpcHandler {
}
if saw_unknown {
if let Some(health) = self.cached_reachable_health(app_id).await? {
return Ok(Some(health));
}
Ok(Some("unknown".to_string()))
} else if saw_starting {
if let Some(health) = self.cached_reachable_health(app_id).await? {
return Ok(Some(health));
}
Ok(Some("starting".to_string()))
} else {
Ok(Some("healthy".to_string()))
@@ -482,7 +594,9 @@ async fn member_health(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
app_id: &str,
) -> Result<String> {
if let Ok(health) = orchestrator.health(app_id).await {
if let Ok(Ok(health)) =
tokio::time::timeout(ORCHESTRATOR_HEALTH_TIMEOUT, orchestrator.health(app_id)).await
{
return Ok(health);
}
for name in status_container_name_candidates(app_id) {
@@ -508,10 +622,8 @@ fn stack_health_members(app_id: &str) -> Option<&'static [&'static str]> {
"indeedhub-minio",
"indeedhub-relay",
"indeedhub-api",
"indeedhub-ffmpeg",
"indeedhub",
]),
"fedimint" => Some(&["fedimint"]),
_ => None,
}
}
@@ -583,6 +695,115 @@ fn status_container_name_candidates(app_id: &str) -> Vec<String> {
out
}
fn should_refresh_cached_state(state: &str) -> bool {
matches!(state, "exited" | "stopped" | "stopping")
}
async fn live_state_for_app(app_id: &str) -> Option<String> {
for name in status_container_name_candidates(app_id) {
if let Some(live) = inspect_container_state_value(&name).await {
if let Some(live_state) = live.get("state").and_then(|v| v.as_str()) {
return Some(live_state.to_string());
}
}
}
None
}
async fn quadlet_service_active(app_id: &str) -> bool {
for name in status_container_name_candidates(app_id) {
let service = format!("{name}.service");
let mut cmd = tokio::process::Command::new("systemctl");
cmd.args(["--user", "is-active", "--quiet", &service]);
cmd.kill_on_drop(true);
if matches!(
tokio::time::timeout(Duration::from_secs(2), cmd.status()).await,
Ok(Ok(status)) if status.success()
) {
return true;
}
}
false
}
fn health_probe_url_for_app(app_id: &str) -> Option<String> {
let port = match app_id {
"bitcoin-ui" => 8334,
"botfights" => 9100,
"btcpay-server" | "btcpay" | "btcpayserver" => 23000,
"dwn" => 3100,
"electrumx" | "electrs" | "mempool-electrs" | "electrs-ui" => 50002,
"fedimint" | "fedimintd" => 8175,
"filebrowser" => 8083,
"gitea" => 3001,
"grafana" => 3000,
"homeassistant" | "home-assistant" => 8123,
"immich" | "immich_server" => 2283,
"indeedhub" => 7778,
"jellyfin" => 8096,
"lnd" | "lnd-ui" => 18083,
"mempool" | "mempool-web" => 4080,
"nginx-proxy-manager" => 8081,
"ollama" => 11434,
"photoprism" => 2342,
"portainer" => 9000,
"searxng" => 8888,
"tailscale" => 8240,
"uptime-kuma" => 3002,
"vaultwarden" => 8082,
_ => return None,
};
Some(format!("http://localhost:{port}"))
}
fn requires_launch_port_for_health(app_id: &str) -> bool {
matches!(app_id, "fedimint" | "fedimintd" | "fedimint-gateway")
}
async fn launch_port_reachable(app_id: &str) -> bool {
let Some(port) = health_probe_url_for_app(app_id).and_then(|url| port_from_url(&url)) else {
return false;
};
launch_port_reachable_by_port(port).await
}
async fn launch_port_reachable_by_port(port: u16) -> bool {
matches!(
tokio::time::timeout(
Duration::from_secs(2),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await,
Ok(Ok(_))
)
}
async fn http_launch_url_reachable(url: &str) -> bool {
let Ok(client) = reqwest::Client::builder()
.timeout(Duration::from_secs(2))
.redirect(reqwest::redirect::Policy::none())
.build()
else {
return false;
};
match client.get(url).send().await {
Ok(response) => {
let status = response.status();
status.is_success() || status.is_redirection()
}
Err(_) => false,
}
}
fn port_from_url(url: &str) -> Option<u16> {
let after_colon = url.rsplit_once(':')?.1;
let port = after_colon
.chars()
.take_while(|c| c.is_ascii_digit())
.collect::<String>();
port.parse::<u16>().ok()
}
async fn inspect_container_state_value(name: &str) -> Option<serde_json::Value> {
if let Some(v) = ps_container_state_value(name).await {
return Some(v);