lfg2025/archy
1
0
Fork 0
Code Issues 27 Pull Requests 4 Actions Packages Projects Releases 74 Wiki Activity

backend: harden rootless app lifecycle orchestration

Browse Source
This commit is contained in:
archipelago
2026-06-11 00:24:32 -04:00
parent 09ec64932f
commit c393b96da3
56 changed files with 7543 additions and 1994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/container/bitcoin_ui_nginx.conf.template
+10
View File
@@ -23,5 +23,15 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header Cache-Control "no-store";
}
location /rpc/v1 {
proxy_pass http://127.0.0.1:5678/rpc/v1;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Cookie $http_cookie;
proxy_set_header X-CSRF-Token $http_x_csrf_token;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header Cache-Control "no-store";
}
location / { try_files $uri $uri/ /index.html; }
}
core/archipelago/src/container/boot_reconciler.rs
+58 -41
View File
@@ -34,6 +34,7 @@ pub struct BootReconciler {
/// `systemctl --user` and `podman`, which both block real time
/// and would race the paused-clock test fixtures.
companion_stage: bool,
wait_for_recovery: bool,
}
impl BootReconciler {
@@ -47,6 +48,7 @@ impl BootReconciler {
interval,
shutdown,
companion_stage: true,
wait_for_recovery: true,
}
}
@@ -56,6 +58,7 @@ impl BootReconciler {
#[cfg(test)]
pub fn without_companion_stage(mut self) -> Self {
self.companion_stage = false;
self.wait_for_recovery = false;
self
}
@@ -78,6 +81,21 @@ impl BootReconciler {
/// by the orchestrator, and companion failures are logged but never
/// propagated.
pub async fn run_forever(self) {
let wait_start = Instant::now();
while self.wait_for_recovery && !crate::crash_recovery::is_recovery_complete() {
if wait_start.elapsed() > Duration::from_secs(1800) {
tracing::warn!("boot reconciler: boot recovery did not complete within 30 minutes, starting anyway");
break;
}
tokio::select! {
_ = time::sleep(Duration::from_secs(5)) => {}
_ = self.shutdown.notified() => {
tracing::info!("boot reconciler: shutdown requested before recovery completed");
return;
}
}
}
// Initial pass: no delay.
self.tick().await;
@@ -244,58 +262,65 @@ mod tests {
ProdContainerOrchestrator::with_runtime(rt, PathBuf::from("/nonexistent-for-tests"));
let tmp = tempfile::tempdir().unwrap().keep();
orch.set_data_dir(tmp);
orch.set_disk_gb_for_test(2_000);
let orch = Arc::new(orch);
orch.insert_manifest_for_test(
pull_manifest("bitcoin-knots", "docker.io/bitcoin/knots:28"),
PathBuf::from("/tmp/bk"),
pull_manifest("test-app", "docker.io/example/test-app:1"),
PathBuf::from("/tmp/test-app"),
)
.await;
orch
}
#[tokio::test(start_paused = true)]
async fn wait_for_status_calls(rt: &CountingRuntime, expected: u32) -> u32 {
for _ in 0..100 {
let count = rt.status_call_count();
if count >= expected {
return count;
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(1)).await;
}
rt.status_call_count()
}
#[tokio::test]
async fn initial_pass_fires_immediately() {
let rt = Arc::new(CountingRuntime::new_with(&["bitcoin-knots"]));
let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
let orch = orch_with_one_running_manifest(rt.clone()).await;
let shutdown = Arc::new(Notify::new());
let reconciler =
BootReconciler::new(orch.clone(), Duration::from_secs(30), shutdown.clone())
BootReconciler::new(orch.clone(), Duration::from_millis(50), shutdown.clone())
.without_companion_stage();
let handle = tokio::spawn(reconciler.run_forever());
// Yield so the spawned task gets CPU to run its initial reconcile.
tokio::task::yield_now().await;
tokio::task::yield_now().await;
// We expect exactly one reconcile pass to have run by now (the initial),
// NOT a second one (the 30s sleep hasn't elapsed in paused time).
assert_eq!(rt.status_call_count(), 1, "initial pass should fire once");
assert_eq!(
wait_for_status_calls(&rt, 1).await,
1,
"initial pass should fire once"
);
shutdown.notify_one();
// Under paused clock the select! is blocked on sleep_until; the notify
// will unblock it. Advance wall-clock a hair so the notify gets polled.
tokio::task::yield_now().await;
let _ = tokio::time::timeout(Duration::from_secs(1), handle).await;
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn second_pass_fires_after_interval() {
let rt = Arc::new(CountingRuntime::new_with(&["bitcoin-knots"]));
let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
let orch = orch_with_one_running_manifest(rt.clone()).await;
let shutdown = Arc::new(Notify::new());
let reconciler =
BootReconciler::new(orch.clone(), Duration::from_secs(30), shutdown.clone())
BootReconciler::new(orch.clone(), Duration::from_millis(10), shutdown.clone())
.without_companion_stage();
let handle = tokio::spawn(reconciler.run_forever());
tokio::task::yield_now().await;
tokio::task::yield_now().await;
assert_eq!(rt.status_call_count(), 1);
assert_eq!(wait_for_status_calls(&rt, 1).await, 1);
// Fast-forward past one interval; the sleep_until should fire.
tokio::time::advance(Duration::from_secs(31)).await;
tokio::task::yield_now().await;
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(20)).await;
wait_for_status_calls(&rt, 2).await;
assert_eq!(
rt.status_call_count(),
@@ -308,27 +333,23 @@ mod tests {
let _ = tokio::time::timeout(Duration::from_secs(1), handle).await;
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn shutdown_terminates_loop() {
let rt = Arc::new(CountingRuntime::new_with(&["bitcoin-knots"]));
let rt = Arc::new(CountingRuntime::new_with(&["test-app"]));
let orch = orch_with_one_running_manifest(rt.clone()).await;
let shutdown = Arc::new(Notify::new());
let reconciler =
BootReconciler::new(orch.clone(), Duration::from_secs(30), shutdown.clone())
BootReconciler::new(orch.clone(), Duration::from_millis(50), shutdown.clone())
.without_companion_stage();
let handle = tokio::spawn(reconciler.run_forever());
tokio::task::yield_now().await;
tokio::task::yield_now().await;
wait_for_status_calls(&rt, 1).await;
shutdown.notify_one();
// The select! should wake on Notified and return. Use a real timeout
// with advancing the paused clock to make sure the task exits.
tokio::time::advance(Duration::from_millis(10)).await;
let result = tokio::time::timeout(Duration::from_secs(5), handle).await;
assert!(result.is_ok(), "reconciler did not exit after shutdown");
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn failure_in_one_pass_does_not_stop_loop() {
// Manifest references a container the runtime does not have AND
// cannot create (no install path — install_fresh will also fail to
@@ -344,26 +365,23 @@ mod tests {
);
let tmp = tempfile::tempdir().unwrap().keep();
orch.set_data_dir(tmp);
orch.set_disk_gb_for_test(2_000);
let orch = Arc::new(orch);
orch.insert_manifest_for_test(
pull_manifest("bitcoin-knots", "docker.io/bitcoin/knots:28"),
PathBuf::from("/tmp/bk"),
pull_manifest("test-app", "docker.io/example/test-app:1"),
PathBuf::from("/tmp/test-app"),
)
.await;
let shutdown = Arc::new(Notify::new());
let reconciler =
BootReconciler::new(orch.clone(), Duration::from_secs(30), shutdown.clone())
BootReconciler::new(orch.clone(), Duration::from_millis(10), shutdown.clone())
.without_companion_stage();
let handle = tokio::spawn(reconciler.run_forever());
tokio::task::yield_now().await;
tokio::task::yield_now().await;
let first = rt.status_call_count();
let first = wait_for_status_calls(&rt, 1).await;
assert!(first >= 1, "initial pass should have touched the runtime");
// Advance one interval — second pass should fire regardless of what
// the first pass did.
tokio::time::advance(Duration::from_secs(31)).await;
tokio::time::sleep(Duration::from_millis(20)).await;
tokio::task::yield_now().await;
tokio::task::yield_now().await;
let second = rt.status_call_count();
@@ -373,7 +391,6 @@ mod tests {
);
shutdown.notify_one();
tokio::time::advance(Duration::from_millis(10)).await;
let _ = tokio::time::timeout(Duration::from_secs(5), handle).await;
}
}
core/archipelago/src/container/companion.rs
+67 -10
View File
@@ -9,6 +9,7 @@
//! | bitcoin-core | archy-bitcoin-ui | RPC viewer |
//! | lnd | archy-lnd-ui | wallet/channel UI |
//! | electrumx | archy-electrs-ui | indexer status UI |
//! | fedimint | archy-fedimint-ui | wait/proxy Guardian UI |
//!
//! Lifecycle: `install` writes a Quadlet `.container` unit to
//! `~/.config/containers/systemd/`, daemon-reloads, then starts the
@@ -22,6 +23,7 @@
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use tokio::process::Command;
use tracing::{info, warn};
@@ -30,6 +32,9 @@ use crate::container::quadlet::{self, BindMount, NetworkMode, QuadletUnit};
use archipelago_container::image_uses_insecure_registry;
const COMPANION_REGISTRY: &str = "146.59.87.168:3000/lfg2025";
const COMPANION_IMAGE_CHECK_TIMEOUT: Duration = Duration::from_secs(15);
const COMPANION_BUILD_TIMEOUT: Duration = Duration::from_secs(900);
const COMPANION_PULL_TIMEOUT: Duration = Duration::from_secs(300);
/// Static description of one companion. The full list per backend
/// app_id lives in `companions_for`.
@@ -65,6 +70,7 @@ pub fn companions_for(package_id: &str) -> &'static [CompanionSpec] {
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => BITCOIN_UI,
"lnd" => LND_UI,
"electrumx" | "electrs" | "mempool-electrs" => ELECTRS_UI,
"fedimint" | "fedimintd" => FEDIMINT_UI,
_ => &[],
}
}
@@ -114,6 +120,20 @@ const ELECTRS_UI: &[CompanionSpec] = &[CompanionSpec {
host_network: true,
}];
const FEDIMINT_UI: &[CompanionSpec] = &[CompanionSpec {
name: "archy-fedimint-ui",
image_base: "fedimint-ui",
build_dir_candidates: &[
"/opt/archipelago/docker/fedimint-ui",
"/home/archipelago/archy/docker/fedimint-ui",
"/home/archipelago/Projects/archy/docker/fedimint-ui",
],
pre_start: None,
bind_mounts: &[],
ports: &[],
host_network: true,
}];
fn render_bitcoin_ui() -> futures_util::future::BoxFuture<'static, Result<()>> {
Box::pin(async {
let paths = crate::container::bitcoin_ui::RenderPaths::default();
@@ -201,11 +221,12 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
return Ok(local_image);
}
info!(companion = spec.name, "building locally from {dir}");
let out = Command::new("podman")
.args(["build", "-t", &local_image, dir])
.output()
.await
.context("spawn podman build")?;
let out = command_output_with_timeout(
Command::new("podman").args(["build", "-t", &local_image, dir]),
COMPANION_BUILD_TIMEOUT,
"podman build companion image",
)
.await?;
if out.status.success() {
return Ok(local_image);
}
@@ -226,7 +247,12 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
cmd.arg("--tls-verify=false");
}
cmd.arg(&registry_image);
let out = cmd.output().await.context("spawn podman pull")?;
let out = command_output_with_timeout(
&mut cmd,
COMPANION_PULL_TIMEOUT,
"podman pull companion image",
)
.await?;
if !out.status.success() {
anyhow::bail!(
"no local Dockerfile and registry pull failed for {}: {}",
@@ -238,11 +264,31 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
}
async fn image_exists(image: &str) -> bool {
Command::new("podman")
.args(["image", "exists", image])
.status()
let mut cmd = Command::new("podman");
cmd.args(["image", "inspect", image]);
match tokio::time::timeout(COMPANION_IMAGE_CHECK_TIMEOUT, cmd.status()).await {
Ok(Ok(status)) => status.success(),
Ok(Err(err)) => {
warn!(image = %image, error = %err, "companion image existence check failed");
false
}
Err(_) => {
warn!(image = %image, "companion image existence check timed out");
false
}
}
}
async fn command_output_with_timeout(
cmd: &mut Command,
timeout: Duration,
description: &str,
) -> Result<std::process::Output> {
cmd.kill_on_drop(true);
tokio::time::timeout(timeout, cmd.output())
.await
.is_ok_and(|status| status.success())
.with_context(|| format!("{description} timed out after {}s", timeout.as_secs()))?
.with_context(|| format!("spawn {description}"))
}
fn build_unit(spec: &CompanionSpec, image: &str) -> QuadletUnit {
@@ -368,6 +414,8 @@ mod tests {
assert_eq!(companions_for("electrumx").len(), 1);
assert_eq!(companions_for("electrs").len(), 1);
assert_eq!(companions_for("mempool-electrs").len(), 1);
assert_eq!(companions_for("fedimint").len(), 1);
assert_eq!(companions_for("fedimintd").len(), 1);
assert_eq!(companions_for("nextcloud").len(), 0);
assert_eq!(companions_for("not-a-real-app").len(), 0);
}
@@ -398,4 +446,13 @@ mod tests {
assert!(matches!(u.network, NetworkMode::Bridge(ref n) if n == "bridge"));
assert_eq!(u.ports, vec![(18083, 80, "tcp".into())]);
}
#[test]
fn fedimint_ui_uses_host_network_for_public_guardian_port() {
let spec = &FEDIMINT_UI[0];
let u = build_unit(spec, "localhost/fedimint-ui:latest");
assert_eq!(u.name, "archy-fedimint-ui");
assert!(matches!(u.network, NetworkMode::Host));
assert!(u.ports.is_empty());
}
}
core/archipelago/src/container/docker_packages.rs
+17 -35
View File
@@ -26,13 +26,7 @@ impl DockerPackageScanner {
/// Scan Docker containers and convert to package data
pub async fn scan_containers(&self) -> Result<HashMap<String, PackageDataEntry>> {
let containers = match self.runtime.list_containers().await {
Ok(c) => c,
Err(e) => {
debug!("Failed to list containers: {}", e);
return Ok(HashMap::new());
}
};
let containers = self.runtime.list_containers().await?;
debug!("Found {} containers", containers.len());
@@ -63,14 +57,6 @@ impl DockerPackageScanner {
"indeedhub-build_ffmpeg-worker_1",
"netbird-server",
"netbird-dashboard",
"saleor-api",
"saleor-worker",
"saleor-db",
"saleor-cache",
"saleor-jaeger",
"saleor-mailpit",
"saleor-storefront",
"saleor-storefront-app",
"buildx_buildkit_default",
];
@@ -298,7 +284,6 @@ fn get_app_tier(app_id: &str) -> &'static str {
"uptime-kuma" => "recommended",
"grafana" => "recommended",
"searxng" => "recommended",
"saleor" => "recommended",
"tailscale" | "netbird" => "recommended",
"portainer" => "recommended",
// Optional: everything else
@@ -519,13 +504,6 @@ fn get_app_metadata(app_id: &str) -> AppMetadata {
repo: "https://github.com/netbirdio/netbird".to_string(),
tier: "",
},
"saleor" => AppMetadata {
title: "Saleor".to_string(),
description: "Composable commerce platform with storefront, dashboard, and GraphQL API. The customer storefront opens on port 9011; admin dashboard is on 9010 with admin@example.com credentials stored on the node.".to_string(),
icon: "/assets/img/app-icons/saleor.svg".to_string(),
repo: "https://github.com/saleor/saleor".to_string(),
tier: "",
},
"gitea" => AppMetadata {
title: "Gitea".to_string(),
description: "Self-hosted Git service with repository and package hosting".to_string(),
@@ -732,20 +710,25 @@ async fn reachable_lan_address(app_id: &str, candidate: Option<String>) -> Optio
let Some(port) = url.rsplit(':').next().and_then(|p| p.parse::<u16>().ok()) else {
return None;
};
match tokio::time::timeout(
std::time::Duration::from_secs(2),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await
{
Ok(Ok(_)) => Some(url),
_ => {
debug!(app_id = %app_id, port, "suppressing unreachable launch URL");
None
}
if launch_port_reachable(port).await {
Some(url)
} else {
debug!(app_id = %app_id, port, "suppressing unreachable launch URL");
None
}
}
async fn launch_port_reachable(port: u16) -> bool {
matches!(
tokio::time::timeout(
std::time::Duration::from_secs(2),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await,
Ok(Ok(_))
)
}
fn requires_reachable_launch(app_id: &str) -> bool {
matches!(
app_id,
@@ -766,7 +749,6 @@ fn requires_reachable_launch(app_id: &str) -> bool {
| "tailscale"
| "immich"
| "searxng"
| "saleor"
)
}
core/archipelago/src/container/filebrowser.rs
+61 -25
View File
@@ -8,6 +8,8 @@ use anyhow::{Context, Result};
use std::path::PathBuf;
use tokio::fs;
use crate::update::host_sudo;
pub const DEFAULT_SRV_ROOT: &str = "/var/lib/archipelago/filebrowser";
pub const DEFAULT_DATA_DIR: &str = "/var/lib/archipelago/filebrowser-data";
pub const DEFAULT_CONFIG_PATH: &str = "/var/lib/archipelago/filebrowser-data/.filebrowser.json";
@@ -39,17 +41,11 @@ pub enum EnsureOutcome {
}
pub async fn ensure_config(paths: &EnsurePaths) -> Result<EnsureOutcome> {
fs::create_dir_all(&paths.srv_root)
.await
.with_context(|| format!("creating {}", paths.srv_root.display()))?;
fs::create_dir_all(&paths.data_dir)
.await
.with_context(|| format!("creating {}", paths.data_dir.display()))?;
create_dir_all_or_sudo(&paths.srv_root).await?;
create_dir_all_or_sudo(&paths.data_dir).await?;
for d in ["Documents", "Photos", "Music", "Downloads", "Builds"] {
fs::create_dir_all(paths.srv_root.join(d))
.await
.with_context(|| format!("creating {}/{}", paths.srv_root.display(), d))?;
create_dir_all_or_sudo(&paths.srv_root.join(d)).await?;
}
if paths.config_path.exists() {
@@ -60,27 +56,67 @@ pub async fn ensure_config(paths: &EnsurePaths) -> Result<EnsureOutcome> {
.config_path
.parent()
.ok_or_else(|| anyhow::anyhow!("config_path has no parent directory"))?;
fs::create_dir_all(parent)
.await
.with_context(|| format!("creating {}", parent.display()))?;
create_dir_all_or_sudo(parent).await?;
let tmp = paths.config_path.with_extension("tmp");
fs::write(&tmp, DEFAULT_CONFIG_JSON)
.await
.with_context(|| format!("writing tmp {}", tmp.display()))?;
fs::rename(&tmp, &paths.config_path)
.await
.with_context(|| {
format!(
"renaming {} -> {}",
tmp.display(),
paths.config_path.display()
)
})?;
write_config_atomically(paths).await?;
Ok(EnsureOutcome::Written)
}
async fn create_dir_all_or_sudo(path: &std::path::Path) -> Result<()> {
match fs::create_dir_all(path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
let path = path.to_string_lossy();
let status = host_sudo(&["mkdir", "-p", &path])
.await
.with_context(|| format!("creating {path} via sudo"))?;
if !status.success() {
anyhow::bail!("mkdir -p {path} via sudo exited with {status}");
}
Ok(())
}
Err(e) => Err(e).with_context(|| format!("creating {}", path.display())),
}
}
async fn write_config_atomically(paths: &EnsurePaths) -> Result<()> {
let tmp = paths.config_path.with_extension("tmp");
match fs::write(&tmp, DEFAULT_CONFIG_JSON).await {
Ok(()) => {
fs::rename(&tmp, &paths.config_path)
.await
.with_context(|| {
format!(
"renaming {} -> {}",
tmp.display(),
paths.config_path.display()
)
})?;
Ok(())
}
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
let script = format!(
"set -eu\ncat > '{}' <<'FILEBROWSERCONF'\n{}FILEBROWSERCONF\n",
shell_quote(&paths.config_path.to_string_lossy()),
DEFAULT_CONFIG_JSON
);
let status = host_sudo(&["sh", "-lc", &script])
.await
.context("writing .filebrowser.json via sudo")?;
if !status.success() {
anyhow::bail!("writing .filebrowser.json via sudo exited with {status}");
}
Ok(())
}
Err(e) => Err(e).with_context(|| format!("writing tmp {}", tmp.display())),
}
}
fn shell_quote(s: &str) -> String {
s.replace('\'', "'\\''")
}
#[cfg(test)]
mod tests {
use super::*;
core/archipelago/src/container/image_versions.rs
+26
View File
@@ -219,6 +219,10 @@ pub fn pinned_image_for_app(app_id: &str) -> Option<String> {
/// explicit versions we should advertise to users as available updates.
pub fn available_update_for_app(app_id: &str, running_image: &str) -> Option<String> {
let pinned = pinned_image_for_app(app_id)?;
available_update_for_images(&pinned, running_image)
}
fn available_update_for_images(pinned: &str, running_image: &str) -> Option<String> {
let pinned_version = extract_version_from_image(&pinned);
if is_floating_tag(&pinned_version) {
return None;
@@ -378,6 +382,28 @@ mod tests {
assert!(!is_floating_tag("v0.18.4-beta"));
}
#[test]
fn available_update_ignores_registry_only_changes() {
assert_eq!(
available_update_for_images(
"146.59.87.168:3000/lfg2025/nextcloud:29",
"git.tx1138.com/lfg2025/nextcloud:29",
),
None
);
}
#[test]
fn available_update_returns_pinned_version_for_same_repo_newer_tag() {
assert_eq!(
available_update_for_images(
"146.59.87.168:3000/lfg2025/nextcloud:29",
"146.59.87.168:3000/lfg2025/nextcloud:28",
),
Some("29".to_string())
);
}
#[test]
fn test_parse_image_versions() {
let content = r#"
core/archipelago/src/container/lnd.rs
+2 -1
View File
@@ -76,7 +76,7 @@ pub async fn ensure_wallet_initialized() -> Result<()> {
let admin_macaroon = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
let wallet_db = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/wallet.db";
if file_exists_as_root(wallet_db).await {
if file_exists_as_root(admin_macaroon).await && lnd_getinfo_ready(admin_macaroon).await {
if file_exists_as_root(admin_macaroon).await {
return Ok(());
}
unlock_existing_wallet().await?;
@@ -305,6 +305,7 @@ async fn decode_lnd_unlocker_response<T: for<'de> Deserialize<'de>>(
anyhow::bail!("LND REST {path} returned {status}: {text}")
}
#[allow(dead_code)]
async fn lnd_getinfo_ready(admin_macaroon: &str) -> bool {
let Ok(macaroon) = read_file_as_root(admin_macaroon).await else {
return false;
core/archipelago/src/container/prod_orchestrator.rs
+1127 -119
View File
File diff suppressed because it is too large Load Diff
core/archipelago/src/container/quadlet.rs
+258 -31
View File
@@ -34,9 +34,13 @@ use anyhow::{anyhow, Context, Result};
use archipelago_container::AppManifest;
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::fs;
use tokio::process::Command;
const QUADLET_START_TIMEOUT: Duration = Duration::from_secs(90);
const QUADLET_STOP_TIMEOUT: Duration = Duration::from_secs(45);
/// Default rootless quadlet directory. Resolved per-user at runtime via
/// `unit_dir()`. Tests pass an explicit dir.
pub const DEFAULT_REL_UNIT_DIR: &str = ".config/containers/systemd";
@@ -61,6 +65,12 @@ pub enum NetworkMode {
/// attached to it. The network must already exist (orchestrator's
/// `ensure_container_network` handles that on every reconcile tick).
Bridge(String),
/// Rootless slirp4netns networking. Podman rejects network aliases with
/// this mode, so render only Network=slirp4netns.
Slirp4netns,
/// Rootless pasta networking. This is more reliable than slirp4netns for
/// host port forwarding on long-running web apps.
Pasta,
}
/// systemd Restart= policy for the generated `.service` unit. Companions
@@ -181,6 +191,12 @@ impl QuadletUnit {
NetworkMode::Host => {
let _ = writeln!(s, "Network=host");
}
NetworkMode::Slirp4netns => {
let _ = writeln!(s, "Network=slirp4netns");
}
NetworkMode::Pasta => {
let _ = writeln!(s, "Network=pasta");
}
NetworkMode::Bridge(net) => {
let _ = writeln!(s, "Network={net}");
for alias in &self.network_aliases {
@@ -261,6 +277,13 @@ impl QuadletUnit {
}
let _ = writeln!(s);
let _ = writeln!(s, "[Service]");
// Dependency-gated apps may legitimately keep their container entrypoint
// in a wait loop before the actual daemon binds ports. Fedimint waits
// for Bitcoin IBD to finish before execing fedimintd; systemd's default
// start timeout otherwise kills the generated podman run job and leaves
// the unit stuck in deactivating. Health/status remains app-level state,
// not a systemd start gate.
let _ = writeln!(s, "TimeoutStartSec=0");
// Restart policy + 10s backoff. RestartSec keeps a crash-loop
// from saturating the journal. Companions: Always. Backends:
// OnFailure (clean stops stay stopped).
@@ -334,6 +357,8 @@ impl QuadletUnit {
// either form.
other if !other.is_empty() && other != "isolated" => NetworkMode::Bridge(other.into()),
_ => match app.container.network.as_deref() {
Some("slirp4netns") => NetworkMode::Slirp4netns,
Some("pasta") => NetworkMode::Pasta,
Some(n) if !n.is_empty() && n != "host" => NetworkMode::Bridge(n.into()),
_ => NetworkMode::Default,
},
@@ -382,7 +407,7 @@ impl QuadletUnit {
entrypoint: app.container.entrypoint.clone(),
command: app.container.custom_args.clone(),
read_only_root: app.security.readonly_root,
no_new_privileges: true,
no_new_privileges: app.security.no_new_privileges,
cpu_quota: app.resources.cpu_limit,
restart_policy: RestartPolicy::OnFailure,
}
@@ -436,13 +461,14 @@ fn translate_health_check(hc: &archipelago_container::HealthCheck) -> Option<Hea
let path = hc.path.as_deref().unwrap_or("/");
format!("{url}{path}")
};
let helper_timeout = health_timeout_seconds(&hc.timeout);
// Images vary wildly: SearXNG ships wget but no curl, while some
// Node images ship neither. Use whichever probe helper exists and
// skip Podman health if the image has none; host-side lifecycle
// probes still verify reachability.
format!(
"if command -v wget >/dev/null 2>&1; then wget -q -T 5 -O /dev/null {0}; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 5 {0}; else exit 0; fi",
final_url
"if command -v wget >/dev/null 2>&1; then wget -q -T {1} -O /dev/null {0}; elif command -v curl >/dev/null 2>&1; then curl -fsS -m {1} {0}; else exit 0; fi",
final_url, helper_timeout
)
}
"cmd" => hc.endpoint.as_deref()?.to_string(),
@@ -456,6 +482,29 @@ fn translate_health_check(hc: &archipelago_container::HealthCheck) -> Option<Hea
})
}
fn health_timeout_seconds(raw: &str) -> u64 {
let trimmed = raw.trim();
if trimmed.is_empty() {
return 5;
}
let (number, multiplier) = match trimmed.chars().last() {
Some('s') | Some('S') => (&trimmed[..trimmed.len() - 1], 1),
Some('m') | Some('M') => (&trimmed[..trimmed.len() - 1], 60),
Some('h') | Some('H') => (&trimmed[..trimmed.len() - 1], 3600),
Some(c) if c.is_ascii_digit() => (trimmed, 1),
_ => return 5,
};
number
.trim()
.parse::<u64>()
.ok()
.and_then(|n| n.checked_mul(multiplier))
.filter(|n| *n > 0)
.unwrap_or(5)
}
/// Parse the manifest's memory_limit string into MiB. Recognises the
/// forms our manifests actually use: "<n>", "<n>m"/"<n>M", "<n>g"/"<n>G".
/// Returns None for anything else; the caller treats None as unlimited.
@@ -532,12 +581,21 @@ pub async fn enable_now(service: &str) -> Result<()> {
// .service file lives under /run, not /etc — `enable` would refuse
// ("transient or generated"). The unit's `[Install] WantedBy` is
// honoured at daemon-reload, so we just start it.
let status = Command::new("systemctl")
.args(["--user", "start", service])
.status()
let status = systemctl_user_status(&["start", service], QUADLET_START_TIMEOUT)
.await
.with_context(|| format!("spawn systemctl --user start {service}"))?;
.with_context(|| format!("systemctl --user start {service}"))?;
if !status.success() {
if wait_not_deactivating(service, Duration::from_secs(30)).await {
let retry = systemctl_user_status(&["start", service], QUADLET_START_TIMEOUT)
.await
.with_context(|| format!("retry systemctl --user start {service}"))?;
if retry.success() {
return Ok(());
}
return Err(anyhow!(
"systemctl --user start {service} exited {status}; retry exited {retry}"
));
}
return Err(anyhow!("systemctl --user start {service} exited {status}"));
}
Ok(())
@@ -545,32 +603,112 @@ pub async fn enable_now(service: &str) -> Result<()> {
/// Restart a generated Quadlet service after rewriting a known-bad unit.
pub async fn restart_service(service: &str) -> Result<()> {
let status = Command::new("systemctl")
.args(["--user", "restart", service])
.status()
.await
.with_context(|| format!("spawn systemctl --user restart {service}"))?;
if !status.success() {
// `systemctl restart` hides the stop phase. On rootless Podman nodes a
// generated unit can sit in deactivating while `podman rm -f` hangs, which
// makes RPC/UI state look frozen. Split restart into bounded stop + start
// so stop timeouts can be recovered with an app-scoped kill/reset.
if let Err(err) = stop_service(service).await {
tracing::warn!(
service = %service,
error = %err,
"quadlet stop failed during restart; waiting for unit to settle before start"
);
}
if !wait_not_deactivating(service, Duration::from_secs(120)).await {
return Err(anyhow!(
"systemctl --user restart {service} exited {status}"
"systemctl --user restart {service} could not leave deactivating state"
));
}
Ok(())
enable_now(service).await
}
/// Stop a generated Quadlet service without removing its unit file.
pub async fn stop_service(service: &str) -> Result<()> {
let status = Command::new("systemctl")
.args(["--user", "stop", service])
.status()
.await
.with_context(|| format!("spawn systemctl --user stop {service}"))?;
if !status.success() {
return Err(anyhow!("systemctl --user stop {service} exited {status}"));
match systemctl_user_status(&["stop", service], QUADLET_STOP_TIMEOUT).await {
Ok(status) if status.success() => Ok(()),
Ok(status) => Err(anyhow!("systemctl --user stop {service} exited {status}")),
Err(err) => {
tracing::warn!(
service = %service,
error = %err,
"quadlet stop timed out/failed; killing app-scoped unit"
);
kill_and_reset_service(service).await?;
if !wait_not_deactivating(service, Duration::from_secs(60)).await {
return Err(anyhow!(
"systemctl --user stop {service} remained deactivating after app-scoped kill"
));
}
Ok(())
}
}
}
async fn systemctl_user_status(
args: &[&str],
timeout: Duration,
) -> Result<std::process::ExitStatus> {
let mut cmd = Command::new("systemctl");
cmd.arg("--user").args(args);
cmd.kill_on_drop(true);
tokio::time::timeout(timeout, cmd.status())
.await
.with_context(|| {
format!(
"systemctl --user {} timed out after {}s",
args.join(" "),
timeout.as_secs()
)
})?
.with_context(|| format!("spawn systemctl --user {}", args.join(" ")))
}
async fn kill_and_reset_service(service: &str) -> Result<()> {
let _ = systemctl_user_status(
&["kill", "--kill-whom=all", "-s", "SIGKILL", service],
Duration::from_secs(15),
)
.await;
tokio::time::sleep(Duration::from_secs(2)).await;
let _ = systemctl_user_status(&["reset-failed", service], Duration::from_secs(15)).await;
Ok(())
}
async fn wait_not_deactivating(service: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
let Ok(status) =
systemctl_user_output(&["is-active", service], Duration::from_secs(5)).await
else {
return true;
};
let state = String::from_utf8_lossy(&status.stdout).trim().to_string();
if state != "deactivating" && state != "activating" {
return true;
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
async fn systemctl_user_output(args: &[&str], timeout: Duration) -> Result<std::process::Output> {
let mut cmd = Command::new("systemctl");
cmd.arg("--user").args(args);
cmd.kill_on_drop(true);
tokio::time::timeout(timeout, cmd.output())
.await
.with_context(|| {
format!(
"systemctl --user {} timed out after {}s",
args.join(" "),
timeout.as_secs()
)
})?
.with_context(|| format!("spawn systemctl --user {}", args.join(" ")))
}
pub fn contains_stale_health_gate(unit_body: &str) -> bool {
unit_body.contains("Notify=healthy")
|| unit_body.contains("TimeoutStartSec=600")
@@ -579,6 +717,12 @@ pub fn contains_stale_health_gate(unit_body: &str) -> bool {
pub fn health_cmd_changed(old_body: &str, new_body: &str) -> bool {
directive_values(old_body, "HealthCmd=") != directive_values(new_body, "HealthCmd=")
|| directive_values(old_body, "HealthInterval=")
!= directive_values(new_body, "HealthInterval=")
|| directive_values(old_body, "HealthTimeout=")
!= directive_values(new_body, "HealthTimeout=")
|| directive_values(old_body, "HealthRetries=")
!= directive_values(new_body, "HealthRetries=")
}
pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
@@ -588,9 +732,11 @@ pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
}
pub fn network_aliases_changed(old_body: &str, new_body: &str) -> bool {
let old_network = directive_values(old_body, "Network=");
let new_network = directive_values(new_body, "Network=");
let old_aliases = directive_values(old_body, "NetworkAlias=");
let new_aliases = directive_values(new_body, "NetworkAlias=");
old_aliases != new_aliases
old_network != new_network || old_aliases != new_aliases
}
pub fn exec_changed(old_body: &str, new_body: &str) -> bool {
@@ -620,9 +766,11 @@ pub async fn disable_remove(unit_name: &str, dir: &Path) -> Result<()> {
.await;
let path = dir.join(format!("{unit_name}.container"));
if fs::try_exists(&path).await.unwrap_or(false) {
fs::remove_file(&path)
.await
.with_context(|| format!("remove {}", path.display()))?;
match fs::remove_file(&path).await {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err).with_context(|| format!("remove {}", path.display())),
}
}
daemon_reload_user().await.ok();
// Defensive: kill the actual container too, in case quadlet left it.
@@ -957,6 +1105,48 @@ app:
assert!(!s.contains("Network=host"));
}
#[test]
fn from_manifest_slirp4netns_omits_network_alias() {
let yaml = r#"
app:
id: vaultwarden
name: Vaultwarden
version: 1.0.0
container:
image: registry/vaultwarden:1
network: slirp4netns
security:
network_policy: isolated
"#;
let m = AppManifest::parse(yaml).expect("manifest must parse");
let s = QuadletUnit::from_manifest(&m, "vaultwarden").render();
assert!(s.contains("Network=slirp4netns"));
assert!(!s.contains("NetworkAlias="));
assert!(!s.contains("--network-alias"));
}
#[test]
fn from_manifest_pasta_omits_network_alias() {
let yaml = r#"
app:
id: nextcloud
name: Nextcloud
version: 1.0.0
container:
image: registry/nextcloud:1
network: pasta
security:
network_policy: isolated
"#;
let m = AppManifest::parse(yaml).expect("manifest must parse");
let s = QuadletUnit::from_manifest(&m, "nextcloud").render();
assert!(s.contains("Network=pasta"));
assert!(!s.contains("NetworkAlias="));
assert!(!s.contains("--network-alias"));
}
#[test]
fn from_manifest_preserves_grafana_data_uid_and_volume_shape() {
let yaml = r#"
@@ -1056,18 +1246,20 @@ app:
assert!(s.contains("HealthRetries=3"));
assert!(!s.contains("Notify=healthy"));
assert!(!s.contains("TimeoutStartSec=600"));
assert!(s.contains("TimeoutStartSec=0"));
}
#[test]
fn render_skips_health_directives_when_absent() {
// No health spec → no Notify=healthy, no HealthCmd, no TimeoutStartSec
// override. Companions rely on this so their rendered bytes stay
// unchanged.
// No health spec → no Notify=healthy and no HealthCmd. TimeoutStartSec=0
// is a service-level baseline so dependency-waiting apps are not killed
// by systemd before their app daemon binds.
let s = sample_unit().render();
assert!(!s.contains("HealthCmd="));
assert!(!s.contains("Notify=healthy"));
assert!(!s.contains("HealthRetries="));
assert!(!s.contains("TimeoutStartSec="));
assert!(s.contains("TimeoutStartSec=0"));
assert!(!s.contains("TimeoutStartSec=600"));
}
#[test]
@@ -1094,7 +1286,7 @@ app:
let h = translate_health_check(&http).expect("http must translate");
assert_eq!(
h.cmd,
"if command -v wget >/dev/null 2>&1; then wget -q -T 5 -O /dev/null http://localhost:8080/health; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 5 http://localhost:8080/health; else exit 0; fi"
"if command -v wget >/dev/null 2>&1; then wget -q -T 3 -O /dev/null http://localhost:8080/health; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 3 http://localhost:8080/health; else exit 0; fi"
);
let cmdck = HealthCheck {
@@ -1163,6 +1355,25 @@ app:
assert!(h.cmd.contains("https://example.local/health"));
}
#[test]
fn translate_health_check_http_uses_manifest_timeout_for_helpers() {
use archipelago_container::HealthCheck;
let http = HealthCheck {
check_type: "http".into(),
endpoint: Some("localhost:3000".into()),
path: Some("/api/health".into()),
interval: "30s".into(),
timeout: "30s".into(),
retries: 5,
};
let h = translate_health_check(&http).expect("http must translate");
assert!(h.cmd.contains("wget -q -T 30 "), "got: {}", h.cmd);
assert!(h.cmd.contains("curl -fsS -m 30 "), "got: {}", h.cmd);
assert_eq!(h.timeout, "30s");
assert_eq!(h.retries, 5);
}
#[test]
fn from_manifest_picks_up_health_check() {
let yaml = r#"
@@ -1201,6 +1412,14 @@ app:
assert!(!network_aliases_changed(new, new));
}
#[test]
fn network_aliases_changed_detects_network_mode_drift() {
let old = "[Container]\nNetwork=slirp4netns\n";
let new = "[Container]\n";
assert!(network_aliases_changed(old, new));
assert!(!network_aliases_changed(new, new));
}
#[test]
fn shell_join_escapes_dollars_for_container_runtime_expansion() {
let rendered = shell_join(&["sh".into(), "-lc".into(), "echo ${BITCOIN_RPC_PASS}".into()]);
@@ -1223,6 +1442,14 @@ app:
assert!(!health_cmd_changed(new, new));
}
#[test]
fn health_cmd_changed_detects_probe_timing_drift() {
let old = "[Container]\nHealthCmd=curl -fsS http://localhost:8080/\nHealthTimeout=5s\nHealthRetries=3\n";
let new = "[Container]\nHealthCmd=curl -fsS http://localhost:8080/\nHealthTimeout=30s\nHealthRetries=5\n";
assert!(health_cmd_changed(old, new));
assert!(!health_cmd_changed(new, new));
}
#[test]
fn from_manifest_renders_to_a_systemd_unit() {
// End-to-end: parse a real-shape manifest, build the unit, render