chore(release): stage v1.7.55-alpha
This commit is contained in:
@@ -252,7 +252,8 @@ impl DevContainerOrchestrator {
|
||||
match status.state {
|
||||
archipelago_container::ContainerState::Running => Ok("healthy".to_string()),
|
||||
archipelago_container::ContainerState::Stopped
|
||||
| archipelago_container::ContainerState::Exited => Ok("unhealthy".to_string()),
|
||||
| archipelago_container::ContainerState::Exited
|
||||
| archipelago_container::ContainerState::Stopping => Ok("unhealthy".to_string()),
|
||||
archipelago_container::ContainerState::Created => Ok("starting".to_string()),
|
||||
archipelago_container::ContainerState::Paused => Ok("paused".to_string()),
|
||||
archipelago_container::ContainerState::Unknown(_) => Ok("unknown".to_string()),
|
||||
|
||||
@@ -136,11 +136,13 @@ impl DockerPackageScanner {
|
||||
let lan_address = if let Some(ui_address) = ui_containers.get(&app_id) {
|
||||
// Apps with separate UI containers (e.g. archy-bitcoin-ui, archy-lnd-ui)
|
||||
debug!("Using UI container for {}: {}", app_id, ui_address);
|
||||
Some(ui_address.clone())
|
||||
reachable_lan_address(&app_id, Some(ui_address.clone())).await
|
||||
} else {
|
||||
// Dynamic: use actual port bindings from container, fall back to static map
|
||||
extract_lan_address(&container.ports)
|
||||
.or_else(|| PodmanClient::lan_address_for(&app_id))
|
||||
// Prefer the known web UI port over arbitrary first binding
|
||||
// (for example Gitea exposes SSH on 2222 before web on 3001).
|
||||
let candidate = PodmanClient::lan_address_for(&app_id)
|
||||
.or_else(|| extract_lan_address(&container.ports));
|
||||
reachable_lan_address(&app_id, candidate).await
|
||||
};
|
||||
|
||||
debug!(
|
||||
@@ -156,21 +158,8 @@ impl DockerPackageScanner {
|
||||
// Extract actual version from container image tag
|
||||
let running_version = image_versions::extract_version_from_image(&container.image);
|
||||
|
||||
// Check for available update by comparing running image vs pinned image
|
||||
let available_update =
|
||||
image_versions::pinned_image_for_app(&app_id).and_then(|pinned| {
|
||||
if pinned != container.image {
|
||||
let pinned_version = image_versions::extract_version_from_image(&pinned);
|
||||
// Don't flag if both are "latest" — no meaningful diff
|
||||
if pinned_version != "latest" || running_version != "latest" {
|
||||
Some(pinned_version)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
image_versions::available_update_for_app(&app_id, &container.image);
|
||||
|
||||
let package = PackageDataEntry {
|
||||
state: package_state.clone(),
|
||||
@@ -631,6 +620,51 @@ fn extract_lan_address(ports: &[String]) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
async fn reachable_lan_address(app_id: &str, candidate: Option<String>) -> Option<String> {
|
||||
let url = candidate?;
|
||||
if !requires_reachable_launch(app_id) {
|
||||
return Some(url);
|
||||
}
|
||||
let Some(port) = url.rsplit(':').next().and_then(|p| p.parse::<u16>().ok()) else {
|
||||
return None;
|
||||
};
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
tokio::net::TcpStream::connect(("127.0.0.1", port)),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) => Some(url),
|
||||
_ => {
|
||||
debug!(app_id = %app_id, port, "suppressing unreachable launch URL");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn requires_reachable_launch(app_id: &str) -> bool {
|
||||
matches!(
|
||||
app_id,
|
||||
"botfights"
|
||||
| "btcpay-server"
|
||||
| "fedimint"
|
||||
| "filebrowser"
|
||||
| "grafana"
|
||||
| "homeassistant"
|
||||
| "home-assistant"
|
||||
| "jellyfin"
|
||||
| "mempool"
|
||||
| "nginx-proxy-manager"
|
||||
| "uptime-kuma"
|
||||
| "gitea"
|
||||
| "nextcloud"
|
||||
| "portainer"
|
||||
| "tailscale"
|
||||
| "immich"
|
||||
| "searxng"
|
||||
)
|
||||
}
|
||||
|
||||
fn companion_lan_address(app_id: &str) -> Option<String> {
|
||||
match app_id {
|
||||
"bitcoin" | "bitcoin-knots" | "bitcoin-core" => Some("http://localhost:8334".to_string()),
|
||||
@@ -642,6 +676,7 @@ fn companion_lan_address(app_id: &str) -> Option<String> {
|
||||
fn convert_state(container_state: &ContainerState) -> (PackageState, ServiceStatus) {
|
||||
match container_state {
|
||||
ContainerState::Running => (PackageState::Running, ServiceStatus::Running),
|
||||
ContainerState::Stopping => (PackageState::Stopping, ServiceStatus::Stopped),
|
||||
ContainerState::Stopped => (PackageState::Stopped, ServiceStatus::Stopped),
|
||||
ContainerState::Exited => (PackageState::Exited, ServiceStatus::Stopped),
|
||||
ContainerState::Created => (PackageState::Stopped, ServiceStatus::Stopped),
|
||||
|
||||
@@ -205,6 +205,30 @@ pub fn pinned_image_for_app(app_id: &str) -> Option<String> {
|
||||
images.get(var).cloned()
|
||||
}
|
||||
|
||||
/// Return the pinned tag only when the running image is genuinely behind.
|
||||
/// Registry host changes alone are not app updates, and floating tags are not
|
||||
/// explicit versions we should advertise to users as available updates.
|
||||
pub fn available_update_for_app(app_id: &str, running_image: &str) -> Option<String> {
|
||||
let pinned = pinned_image_for_app(app_id)?;
|
||||
let pinned_version = extract_version_from_image(&pinned);
|
||||
if is_floating_tag(&pinned_version) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let running_version = extract_version_from_image(running_image);
|
||||
if pinned_version == running_version {
|
||||
return None;
|
||||
}
|
||||
|
||||
let pinned_repo = image_without_registry_or_tag(&pinned);
|
||||
let running_repo = image_without_registry_or_tag(running_image);
|
||||
if pinned_repo != running_repo {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(pinned_version)
|
||||
}
|
||||
|
||||
/// Extract version tag from a full image reference.
|
||||
/// e.g. "git.tx1138.com/lfg2025/lnd:v0.18.4-beta" → "v0.18.4-beta"
|
||||
/// Returns "latest" if no tag or tag is empty.
|
||||
@@ -223,6 +247,32 @@ pub fn extract_version_from_image(image: &str) -> String {
|
||||
"latest".to_string()
|
||||
}
|
||||
|
||||
fn is_floating_tag(tag: &str) -> bool {
|
||||
matches!(tag, "latest" | "stable" | "release" | "main")
|
||||
}
|
||||
|
||||
fn image_without_registry_or_tag(image: &str) -> &str {
|
||||
let without_tag = strip_tag(image);
|
||||
match without_tag.split_once('/') {
|
||||
Some((first, rest))
|
||||
if first.contains('.') || first.contains(':') || first == "localhost" =>
|
||||
{
|
||||
rest
|
||||
}
|
||||
_ => without_tag,
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_tag(image: &str) -> &str {
|
||||
if let Some(slash_pos) = image.rfind('/') {
|
||||
let after_slash = &image[slash_pos..];
|
||||
if let Some(colon_pos) = after_slash.rfind(':') {
|
||||
return &image[..slash_pos + colon_pos];
|
||||
}
|
||||
}
|
||||
image
|
||||
}
|
||||
|
||||
/// Container names and their image variable names for multi-container stacks.
|
||||
/// Returns empty vec for single-container apps.
|
||||
pub fn containers_for_stack(app_id: &str) -> Vec<(&'static str, &'static str)> {
|
||||
@@ -286,6 +336,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_registry_and_tag_for_image_identity() {
|
||||
assert_eq!(
|
||||
image_without_registry_or_tag("146.59.87.168:3000/lfg2025/lnd:v0.18.4-beta"),
|
||||
"lfg2025/lnd"
|
||||
);
|
||||
assert_eq!(
|
||||
image_without_registry_or_tag("git.tx1138.com/lfg2025/lnd:v0.18.4-beta"),
|
||||
"lfg2025/lnd"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floating_tags_are_not_explicit_updates() {
|
||||
assert!(is_floating_tag("latest"));
|
||||
assert!(is_floating_tag("stable"));
|
||||
assert!(!is_floating_tag("v0.18.4-beta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_image_versions() {
|
||||
let content = r#"
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::update::host_sudo;
|
||||
|
||||
pub const DEFAULT_DATA_DIR: &str = "/var/lib/archipelago/lnd";
|
||||
pub const DEFAULT_CONF_PATH: &str = "/var/lib/archipelago/lnd/lnd.conf";
|
||||
const LND_REST_BASE_URL: &str = "https://127.0.0.1:18080";
|
||||
pub const WALLET_PASSWORD: &str = "hellohello";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -42,7 +43,7 @@ pub async fn ensure_config(paths: &EnsurePaths, rpc_pass: &str) -> Result<Ensure
|
||||
let existing = fs::read_to_string(&paths.conf_path)
|
||||
.await
|
||||
.with_context(|| format!("reading {}", paths.conf_path.display()))?;
|
||||
if has_required_lnd_flags(&existing) {
|
||||
if has_required_lnd_flags(&existing, rpc_pass) {
|
||||
return Ok(EnsureOutcome::Unchanged);
|
||||
}
|
||||
}
|
||||
@@ -121,6 +122,31 @@ async fn read_file_as_root(path: &str) -> Result<Vec<u8>> {
|
||||
}
|
||||
|
||||
async fn unlock_existing_wallet() -> Result<()> {
|
||||
unlock_existing_wallet_via_rest().await
|
||||
}
|
||||
|
||||
async fn unlock_existing_wallet_via_rest() -> Result<()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(20))
|
||||
.danger_accept_invalid_certs(true)
|
||||
.build()
|
||||
.context("building LND REST client")?;
|
||||
|
||||
let wallet_password = base64::engine::general_purpose::STANDARD.encode(WALLET_PASSWORD);
|
||||
match post_lnd_unlocker_json::<serde_json::Value>(
|
||||
&client,
|
||||
"/v1/unlockwallet",
|
||||
serde_json::json!({ "wallet_password": wallet_password }),
|
||||
)
|
||||
.await
|
||||
.context("unlocking existing LND wallet")?
|
||||
{
|
||||
UnlockerResponse::Value(_) | UnlockerResponse::WalletAlreadyExists => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
async fn unlock_existing_wallet_via_lncli() -> Result<()> {
|
||||
let mut last_err = None;
|
||||
for _ in 0..60 {
|
||||
let mut cmd = tokio::process::Command::new("podman");
|
||||
@@ -221,7 +247,7 @@ async fn get_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
|
||||
client: &reqwest::Client,
|
||||
path: &str,
|
||||
) -> Result<UnlockerResponse<T>> {
|
||||
let url = format!("https://127.0.0.1:8080{path}");
|
||||
let url = format!("{LND_REST_BASE_URL}{path}");
|
||||
let mut last_err = None;
|
||||
for _ in 0..60 {
|
||||
match client.get(&url).send().await {
|
||||
@@ -244,7 +270,7 @@ async fn post_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
|
||||
path: &str,
|
||||
body: serde_json::Value,
|
||||
) -> Result<UnlockerResponse<T>> {
|
||||
let url = format!("https://127.0.0.1:8080{path}");
|
||||
let url = format!("{LND_REST_BASE_URL}{path}");
|
||||
let mut last_err = None;
|
||||
for _ in 0..60 {
|
||||
match client.post(&url).json(&body).send().await {
|
||||
@@ -291,7 +317,7 @@ async fn lnd_getinfo_ready(admin_macaroon: &str) -> bool {
|
||||
return false;
|
||||
};
|
||||
client
|
||||
.get("https://127.0.0.1:8080/v1/getinfo")
|
||||
.get(format!("{LND_REST_BASE_URL}/v1/getinfo"))
|
||||
.header("Grpc-Metadata-macaroon", hex::encode(macaroon))
|
||||
.send()
|
||||
.await
|
||||
@@ -344,12 +370,14 @@ fn shell_quote(s: &str) -> String {
|
||||
s.replace('\'', "'\\''")
|
||||
}
|
||||
|
||||
fn has_required_lnd_flags(conf: &str) -> bool {
|
||||
fn has_required_lnd_flags(conf: &str, rpc_pass: &str) -> bool {
|
||||
let rpc_pass_line = format!("bitcoind.rpcpass={rpc_pass}");
|
||||
[
|
||||
"bitcoin.active=true",
|
||||
"bitcoin.mainnet=true",
|
||||
"bitcoin.node=bitcoind",
|
||||
"bitcoind.rpchost=bitcoin-knots:8332",
|
||||
rpc_pass_line.as_str(),
|
||||
]
|
||||
.iter()
|
||||
.all(|needle| conf.lines().any(|line| line.trim() == *needle))
|
||||
@@ -378,7 +406,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ensure_config_is_idempotent() {
|
||||
async fn ensure_config_repairs_rpc_password_drift() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let paths = EnsurePaths {
|
||||
data_dir: tmp.path().join("lnd"),
|
||||
@@ -391,10 +419,10 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
ensure_config(&paths, "second").await.unwrap(),
|
||||
EnsureOutcome::Unchanged
|
||||
EnsureOutcome::Written
|
||||
);
|
||||
let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
|
||||
assert!(conf.contains("bitcoind.rpcpass=first"));
|
||||
assert!(conf.contains("bitcoind.rpcpass=second"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -47,6 +47,27 @@ use crate::update::host_sudo;
|
||||
/// Keep in sync with the running fixture on .116. Centralized as a constant
|
||||
/// so the rule is visible in one place and unit-testable.
|
||||
const UI_APP_IDS: &[&str] = &["bitcoin-ui", "electrs-ui", "lnd-ui"];
|
||||
const ARCHIVAL_BITCOIN_DISK_GB: u64 = 1000;
|
||||
|
||||
fn is_required_baseline_app(app_id: &str) -> bool {
|
||||
matches!(
|
||||
app_id,
|
||||
"bitcoin-knots"
|
||||
| "electrumx"
|
||||
| "lnd"
|
||||
| "mempool-api"
|
||||
| "mempool"
|
||||
| "archy-mempool-db"
|
||||
| "filebrowser"
|
||||
)
|
||||
}
|
||||
|
||||
fn requires_archival_bitcoin(app_id: &str) -> bool {
|
||||
matches!(
|
||||
app_id,
|
||||
"electrumx" | "mempool-api" | "mempool" | "archy-mempool-db"
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute the podman container name for a manifest.
|
||||
///
|
||||
@@ -129,6 +150,20 @@ async fn wait_for_host_port(port: u16, timeout_secs: u64) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_manifest_host_ports(manifest: &AppManifest, timeout_secs: u64) -> Result<()> {
|
||||
for port in manifest.app.ports.iter().map(|p| p.host) {
|
||||
if !wait_for_host_port(port, timeout_secs).await {
|
||||
return Err(anyhow::anyhow!(
|
||||
"{} host port {} did not become reachable within {}s",
|
||||
manifest.app.id,
|
||||
port,
|
||||
timeout_secs
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn patch_indeedhub_nostr_provider() {
|
||||
let _ = tokio::process::Command::new("podman")
|
||||
.args([
|
||||
@@ -316,6 +351,8 @@ pub struct ProdContainerOrchestrator {
|
||||
/// false so the legacy path remains the production path until the
|
||||
/// 20× lifecycle harness goes green against the new path.
|
||||
use_quadlet_backends: bool,
|
||||
#[cfg(test)]
|
||||
test_disk_gb: Option<u64>,
|
||||
}
|
||||
|
||||
struct FileSecretsProvider {
|
||||
@@ -363,6 +400,8 @@ impl ProdContainerOrchestrator {
|
||||
lnd_paths: lnd::EnsurePaths::default(),
|
||||
secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"),
|
||||
use_quadlet_backends: config.use_quadlet_backends,
|
||||
#[cfg(test)]
|
||||
test_disk_gb: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -380,6 +419,7 @@ impl ProdContainerOrchestrator {
|
||||
lnd_paths: lnd::EnsurePaths::default(),
|
||||
secrets_dir: PathBuf::from("/var/lib/archipelago/secrets"),
|
||||
use_quadlet_backends: false,
|
||||
test_disk_gb: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,6 +451,11 @@ impl ProdContainerOrchestrator {
|
||||
self.lnd_paths = paths;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn set_disk_gb_for_test(&mut self, disk_gb: u64) {
|
||||
self.test_disk_gb = Some(disk_gb);
|
||||
}
|
||||
|
||||
/// Walk `manifests_dir` looking for `*/manifest.yml` files. Parses each,
|
||||
/// validates it, and stores it in the in-memory state.
|
||||
///
|
||||
@@ -587,8 +632,19 @@ impl ProdContainerOrchestrator {
|
||||
.collect()
|
||||
};
|
||||
let mut report = ReconcileReport::default();
|
||||
let disk_gb = self.disk_gb();
|
||||
for lm in manifests {
|
||||
let app_id = lm.manifest.app.id.clone();
|
||||
if mode == ReconcileMode::ExistingOnly
|
||||
&& requires_archival_bitcoin(&app_id)
|
||||
&& disk_gb < ARCHIVAL_BITCOIN_DISK_GB
|
||||
{
|
||||
report.record(
|
||||
&app_id,
|
||||
ReconcileAction::Left("requires-archival-bitcoin".into()),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match self.ensure_running_with_mode(&lm, mode).await {
|
||||
Ok(action) => report.record(&app_id, action),
|
||||
Err(e) => {
|
||||
@@ -691,8 +747,20 @@ impl ProdContainerOrchestrator {
|
||||
return Ok(ReconcileAction::Installed);
|
||||
}
|
||||
self.run_post_start_hooks(&app_id).await?;
|
||||
wait_for_manifest_host_ports(&resolved_manifest, 60).await?;
|
||||
Ok(ReconcileAction::Started)
|
||||
}
|
||||
ContainerState::Stopping => {
|
||||
tracing::warn!(
|
||||
app_id = %app_id,
|
||||
container = %name,
|
||||
"container stuck in stopping state; force-recreating container record"
|
||||
);
|
||||
self.prepare_for_start(&resolved_manifest).await?;
|
||||
let _ = self.runtime.remove_container(&name).await;
|
||||
self.install_fresh(lm).await?;
|
||||
Ok(ReconcileAction::Installed)
|
||||
}
|
||||
ContainerState::Created => {
|
||||
self.prepare_for_start(&resolved_manifest).await?;
|
||||
if self.container_env_drifted(&name, &resolved_manifest).await {
|
||||
@@ -714,6 +782,7 @@ impl ProdContainerOrchestrator {
|
||||
return Ok(ReconcileAction::Installed);
|
||||
}
|
||||
self.run_post_start_hooks(&app_id).await?;
|
||||
wait_for_manifest_host_ports(&resolved_manifest, 60).await?;
|
||||
Ok(ReconcileAction::Started)
|
||||
}
|
||||
ContainerState::Paused => Ok(ReconcileAction::Left("paused".to_string())),
|
||||
@@ -721,7 +790,36 @@ impl ProdContainerOrchestrator {
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Container missing entirely → install fresh.
|
||||
// Container missing entirely. With Quadlet backends enabled, an
|
||||
// existing .container file is installed state even if Podman
|
||||
// lost the container record after a crash/reboot. Sync the unit
|
||||
// bytes first (clears stale Notify=healthy/nc probes), then ask
|
||||
// user systemd to start the generated service.
|
||||
if self.use_quadlet_backends && self.quadlet_unit_exists(&name).await? {
|
||||
self.prepare_for_start(&resolved_manifest).await?;
|
||||
self.sync_quadlet_unit(lm, &name).await?;
|
||||
quadlet::enable_now(&format!("{name}.service"))
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!("start existing quadlet service {name}.service")
|
||||
})?;
|
||||
self.run_post_start_hooks(&app_id).await?;
|
||||
wait_for_manifest_host_ports(&resolved_manifest, 60).await?;
|
||||
return Ok(ReconcileAction::Started);
|
||||
}
|
||||
|
||||
// Required baseline services must self-heal even if both the
|
||||
// podman record and Quadlet unit are gone. These are installed
|
||||
// by first boot and are prerequisites for dependent apps; an
|
||||
// "absent" result leaves the node permanently degraded after
|
||||
// crash cleanup.
|
||||
if mode == ReconcileMode::ExistingOnly && is_required_baseline_app(&app_id) {
|
||||
self.install_fresh(lm).await?;
|
||||
return Ok(ReconcileAction::Installed);
|
||||
}
|
||||
|
||||
// Optional container missing entirely → leave absent during
|
||||
// boot reconcile; explicit install/start can create it.
|
||||
if mode == ReconcileMode::ExistingOnly {
|
||||
return Ok(ReconcileAction::Left("absent".to_string()));
|
||||
}
|
||||
@@ -806,6 +904,7 @@ impl ProdContainerOrchestrator {
|
||||
.with_context(|| format!("start_container {name}"))?;
|
||||
}
|
||||
self.run_post_start_hooks(&lm.manifest.app.id).await?;
|
||||
wait_for_manifest_host_ports(&resolved_manifest, 60).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -942,6 +1041,16 @@ impl ProdContainerOrchestrator {
|
||||
Ok(Some(ReconcileAction::Installed))
|
||||
}
|
||||
|
||||
async fn quadlet_unit_exists(&self, name: &str) -> Result<bool> {
|
||||
let unit_dir = quadlet::unit_dir()
|
||||
.await
|
||||
.context("locate user quadlet unit dir for existing unit check")?;
|
||||
let unit_path = unit_dir.join(format!("{name}.container"));
|
||||
tokio::fs::try_exists(&unit_path)
|
||||
.await
|
||||
.with_context(|| format!("check existing quadlet unit {}", unit_path.display()))
|
||||
}
|
||||
|
||||
/// Drift-sync an existing Quadlet unit file's bytes against what the
|
||||
/// current renderer produces. No-op when the flag is off, when the
|
||||
/// app is a companion (companion.rs owns those units), or when no
|
||||
@@ -971,9 +1080,20 @@ impl ProdContainerOrchestrator {
|
||||
if !tokio::fs::try_exists(&unit_path).await.unwrap_or(false) {
|
||||
return Ok(());
|
||||
}
|
||||
let old_body = tokio::fs::read_to_string(&unit_path)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let restart_required = quadlet::contains_stale_health_gate(&old_body);
|
||||
|
||||
let mut resolved = lm.manifest.clone();
|
||||
self.resolve_dynamic_env(&mut resolved)?;
|
||||
let unit = quadlet::QuadletUnit::from_manifest(&resolved, name);
|
||||
let new_body = unit.render();
|
||||
let restart_for_port_change = quadlet::publish_ports_changed(&old_body, &new_body);
|
||||
let restart_for_network_alias_change =
|
||||
quadlet::network_aliases_changed(&old_body, &new_body);
|
||||
let restart_for_exec_change = quadlet::exec_changed(&old_body, &new_body);
|
||||
let restart_for_health_change = quadlet::health_cmd_changed(&old_body, &new_body);
|
||||
let changed = quadlet::write_if_changed(&unit, &unit_dir)
|
||||
.await
|
||||
.with_context(|| format!("drift-sync quadlet unit for {name}"))?;
|
||||
@@ -987,6 +1107,36 @@ impl ProdContainerOrchestrator {
|
||||
"Quadlet unit drift-synced — file rewritten, .service NOT restarted (operator restart picks up new config)"
|
||||
);
|
||||
}
|
||||
if changed
|
||||
&& (restart_required
|
||||
|| restart_for_port_change
|
||||
|| restart_for_network_alias_change
|
||||
|| restart_for_exec_change
|
||||
|| restart_for_health_change)
|
||||
{
|
||||
let service = unit.service_name();
|
||||
let reason = if restart_required {
|
||||
"stale health gate"
|
||||
} else if restart_for_port_change {
|
||||
"port binding drift"
|
||||
} else if restart_for_network_alias_change {
|
||||
"network alias drift"
|
||||
} else if restart_for_health_change {
|
||||
"health command drift"
|
||||
} else {
|
||||
"exec drift"
|
||||
};
|
||||
tracing::info!(
|
||||
app_id = %lm.manifest.app.id,
|
||||
container = %name,
|
||||
service = %service,
|
||||
reason = reason,
|
||||
"Quadlet unit rewrite requires service restart"
|
||||
);
|
||||
quadlet::restart_service(&service)
|
||||
.await
|
||||
.with_context(|| format!("restart drifted quadlet service {service}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1283,7 +1433,10 @@ impl ProdContainerOrchestrator {
|
||||
let mut started = false;
|
||||
match frontend_status.state {
|
||||
ContainerState::Running => {}
|
||||
ContainerState::Stopped | ContainerState::Exited | ContainerState::Created => {
|
||||
ContainerState::Stopped
|
||||
| ContainerState::Exited
|
||||
| ContainerState::Created
|
||||
| ContainerState::Stopping => {
|
||||
self.runtime
|
||||
.start_container("indeedhub")
|
||||
.await
|
||||
@@ -1366,7 +1519,7 @@ impl ProdContainerOrchestrator {
|
||||
fn detect_host_facts(&self) -> HostFacts {
|
||||
let host_ip = Self::detect_host_ip().unwrap_or_else(|| "127.0.0.1".to_string());
|
||||
let host_mdns = Self::detect_host_mdns();
|
||||
let disk_gb = Self::detect_disk_gb();
|
||||
let disk_gb = self.disk_gb();
|
||||
HostFacts {
|
||||
host_ip,
|
||||
host_mdns,
|
||||
@@ -1429,6 +1582,14 @@ impl ProdContainerOrchestrator {
|
||||
kb / 1_000_000
|
||||
}
|
||||
|
||||
fn disk_gb(&self) -> u64 {
|
||||
#[cfg(test)]
|
||||
if let Some(disk_gb) = self.test_disk_gb {
|
||||
return disk_gb;
|
||||
}
|
||||
Self::detect_disk_gb()
|
||||
}
|
||||
|
||||
fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
|
||||
let facts = self.detect_host_facts();
|
||||
let mut env = manifest.app.environment.clone();
|
||||
@@ -1555,13 +1716,17 @@ impl ProdContainerOrchestrator {
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
|
||||
let expected_entry = manifest
|
||||
.app
|
||||
.container
|
||||
.entrypoint
|
||||
.clone()
|
||||
.unwrap_or_default();
|
||||
current_entry != expected_entry || current_cmd != manifest.app.container.custom_args
|
||||
if let Some(expected_entry) = &manifest.app.container.entrypoint {
|
||||
if current_entry != *expected_entry {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if !manifest.app.container.custom_args.is_empty()
|
||||
&& current_cmd != manifest.app.container.custom_args
|
||||
{
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn apply_data_uid(&self, manifest: &AppManifest) -> Result<()> {
|
||||
@@ -1691,10 +1856,20 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
let action = self.ensure_running(&lm).await?;
|
||||
match action {
|
||||
ReconcileAction::NoOp | ReconcileAction::Started | ReconcileAction::Installed => Ok(()),
|
||||
ReconcileAction::Left(state) => Err(anyhow::anyhow!(
|
||||
"container {} left in {state}",
|
||||
compute_container_name(&lm.manifest)
|
||||
)),
|
||||
ReconcileAction::Left(state) => {
|
||||
let name = compute_container_name(&lm.manifest);
|
||||
tracing::warn!(
|
||||
app_id = %app_id,
|
||||
container = %name,
|
||||
state = %state,
|
||||
"start: container in wedged state, force-recreating"
|
||||
);
|
||||
let lock = self.app_lock(app_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let _ = self.runtime.stop_container(&name).await;
|
||||
let _ = self.runtime.remove_container(&name).await;
|
||||
self.install_fresh(&lm).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1707,6 +1882,12 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
let lock = self.app_lock(app_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let name = compute_container_name(&lm.manifest);
|
||||
// Quadlet-owned containers are restarted by systemd if only `podman stop`
|
||||
// is used. Stop the user service first, then stop the container as a
|
||||
// defensive fallback for legacy/non-Quadlet installs.
|
||||
if let Err(err) = quadlet::stop_service(&format!("{name}.service")).await {
|
||||
tracing::debug!(container = %name, error = %err, "quadlet stop skipped/failed");
|
||||
}
|
||||
self.runtime
|
||||
.stop_container(&name)
|
||||
.await
|
||||
@@ -1718,6 +1899,24 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
let lock = self.app_lock(app_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let name = compute_container_name(&lm.manifest);
|
||||
|
||||
let service = format!("{name}.service");
|
||||
if self.quadlet_unit_exists(&name).await? {
|
||||
let mut resolved_manifest = lm.manifest.clone();
|
||||
self.resolve_dynamic_env(&mut resolved_manifest)?;
|
||||
self.prepare_for_start(&resolved_manifest).await?;
|
||||
self.sync_quadlet_unit(&lm, &name).await?;
|
||||
if let Err(err) = quadlet::restart_service(&service).await {
|
||||
tracing::warn!(container = %name, error = %err, "quadlet restart failed; trying start");
|
||||
quadlet::enable_now(&service)
|
||||
.await
|
||||
.with_context(|| format!("restart start quadlet service {service}"))?;
|
||||
}
|
||||
self.run_post_start_hooks(app_id).await?;
|
||||
wait_for_manifest_host_ports(&resolved_manifest, 60).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Best-effort stop (ignored if already stopped), then start.
|
||||
let _ = self.runtime.stop_container(&name).await;
|
||||
self.prepare_for_start(&lm.manifest).await?;
|
||||
@@ -1819,8 +2018,10 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
||||
async fn health(&self, app_id: &str) -> Result<String> {
|
||||
let status = <Self as ContainerOrchestrator>::status(self, app_id).await?;
|
||||
Ok(match status.state {
|
||||
ContainerState::Running => "healthy".to_string(),
|
||||
ContainerState::Stopped | ContainerState::Exited => "unhealthy".to_string(),
|
||||
ContainerState::Running => status.health.unwrap_or_else(|| "healthy".to_string()),
|
||||
ContainerState::Stopped | ContainerState::Exited | ContainerState::Stopping => {
|
||||
"unhealthy".to_string()
|
||||
}
|
||||
ContainerState::Created => "starting".to_string(),
|
||||
ContainerState::Paused => "paused".to_string(),
|
||||
ContainerState::Unknown(s) => format!("unknown:{s}"),
|
||||
@@ -1846,6 +2047,8 @@ mod tests {
|
||||
calls: StdMutex<Vec<String>>,
|
||||
/// container_name -> ContainerState. Absence = "doesn't exist".
|
||||
containers: StdMutex<HashMap<String, ContainerState>>,
|
||||
/// container_name -> Podman health status.
|
||||
health: StdMutex<HashMap<String, String>>,
|
||||
/// image_ref -> present. Absence = "not present in local storage".
|
||||
images: StdMutex<HashMap<String, bool>>,
|
||||
/// container_name -> env that create_container received.
|
||||
@@ -1869,6 +2072,12 @@ mod tests {
|
||||
.unwrap()
|
||||
.insert(name.to_string(), state);
|
||||
}
|
||||
fn set_health(&self, name: &str, health: &str) {
|
||||
self.health
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(name.to_string(), health.to_string());
|
||||
}
|
||||
fn mark_image_present(&self, tag: &str) {
|
||||
self.images.lock().unwrap().insert(tag.to_string(), true);
|
||||
}
|
||||
@@ -1929,11 +2138,12 @@ mod tests {
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| anyhow::anyhow!("not found: {name}"))?;
|
||||
let health = self.health.lock().unwrap().get(name).cloned();
|
||||
Ok(ContainerStatus {
|
||||
id: format!("id-{name}"),
|
||||
name: name.to_string(),
|
||||
state,
|
||||
health: None,
|
||||
health,
|
||||
exit_code: None,
|
||||
started_at: None,
|
||||
image: "test-image".to_string(),
|
||||
@@ -2436,6 +2646,82 @@ app:
|
||||
assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_existing_installs_missing_required_baseline_app() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let mut orch = orch_with(rt.clone()).await;
|
||||
orch.set_disk_gb_for_test(500);
|
||||
orch.insert_manifest_for_test(
|
||||
pull_manifest("filebrowser", "docker.io/filebrowser/filebrowser:latest"),
|
||||
PathBuf::from("/tmp/filebrowser"),
|
||||
)
|
||||
.await;
|
||||
|
||||
let report = orch.reconcile_existing().await;
|
||||
|
||||
assert_eq!(
|
||||
report.actions,
|
||||
vec![("filebrowser".to_string(), ReconcileAction::Installed)]
|
||||
);
|
||||
assert!(report.failures.is_empty());
|
||||
let calls = rt.calls();
|
||||
assert!(calls.iter().any(|c| c.starts_with("pull_image:")));
|
||||
assert!(calls
|
||||
.iter()
|
||||
.any(|c| c == "create_container:filebrowser:offset=0"));
|
||||
assert!(calls.iter().any(|c| c == "start_container:filebrowser"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_existing_skips_archival_baseline_apps_on_pruned_hosts() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let mut orch = orch_with(rt.clone()).await;
|
||||
orch.set_disk_gb_for_test(500);
|
||||
orch.insert_manifest_for_test(
|
||||
pull_manifest("electrumx", "docker.io/spesmilo/electrumx:latest"),
|
||||
PathBuf::from("/tmp/electrumx"),
|
||||
)
|
||||
.await;
|
||||
|
||||
let report = orch.reconcile_existing().await;
|
||||
|
||||
assert_eq!(
|
||||
report.actions,
|
||||
vec![(
|
||||
"electrumx".to_string(),
|
||||
ReconcileAction::Left("requires-archival-bitcoin".into())
|
||||
)]
|
||||
);
|
||||
assert!(report.failures.is_empty());
|
||||
let calls = rt.calls();
|
||||
assert!(!calls.iter().any(|c| c.starts_with("pull_image:")));
|
||||
assert!(!calls.iter().any(|c| c.starts_with("create_container:")));
|
||||
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_existing_leaves_missing_optional_app_absent() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let orch = orch_with(rt.clone()).await;
|
||||
orch.insert_manifest_for_test(
|
||||
pull_manifest("gitea", "docker.io/gitea/gitea:latest"),
|
||||
PathBuf::from("/tmp/gitea"),
|
||||
)
|
||||
.await;
|
||||
|
||||
let report = orch.reconcile_existing().await;
|
||||
|
||||
assert_eq!(
|
||||
report.actions,
|
||||
vec![("gitea".to_string(), ReconcileAction::Left("absent".into()))]
|
||||
);
|
||||
assert!(report.failures.is_empty());
|
||||
let calls = rt.calls();
|
||||
assert!(!calls.iter().any(|c| c.starts_with("pull_image:")));
|
||||
assert!(!calls.iter().any(|c| c.starts_with("create_container:")));
|
||||
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_collects_per_app_failures_without_short_circuiting() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
@@ -2611,6 +2897,32 @@ app:
|
||||
assert!(calls.iter().any(|c| c == "start_container:bitcoin-knots"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_force_recreates_stopping_container() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
rt.set_state("nostr-rs-relay", ContainerState::Stopping);
|
||||
let orch = orch_with(rt.clone()).await;
|
||||
orch.insert_manifest_for_test(
|
||||
pull_manifest("nostr-rs-relay", "docker.io/scsibug/nostr-rs-relay:0.8.9"),
|
||||
PathBuf::from("/tmp/nostr-rs-relay"),
|
||||
)
|
||||
.await;
|
||||
|
||||
let report = orch.reconcile_all().await;
|
||||
|
||||
assert!(report.failures.is_empty(), "{:?}", report.failures);
|
||||
assert_eq!(
|
||||
report.actions,
|
||||
vec![("nostr-rs-relay".to_string(), ReconcileAction::Installed)]
|
||||
);
|
||||
let calls = rt.calls();
|
||||
assert!(calls.iter().any(|c| c == "remove_container:nostr-rs-relay"));
|
||||
assert!(calls
|
||||
.iter()
|
||||
.any(|c| c == "create_container:nostr-rs-relay:offset=0"));
|
||||
assert!(calls.iter().any(|c| c == "start_container:nostr-rs-relay"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_maps_states_to_strings() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
@@ -2620,6 +2932,9 @@ app:
|
||||
.await;
|
||||
assert_eq!(orch.health("lnd").await.unwrap(), "healthy");
|
||||
|
||||
rt.set_health("lnd", "unhealthy");
|
||||
assert_eq!(orch.health("lnd").await.unwrap(), "unhealthy");
|
||||
|
||||
rt.set_state("lnd", ContainerState::Exited);
|
||||
assert_eq!(orch.health("lnd").await.unwrap(), "unhealthy");
|
||||
|
||||
@@ -2628,6 +2943,9 @@ app:
|
||||
|
||||
rt.set_state("lnd", ContainerState::Created);
|
||||
assert_eq!(orch.health("lnd").await.unwrap(), "starting");
|
||||
|
||||
rt.set_state("lnd", ContainerState::Stopping);
|
||||
assert_eq!(orch.health("lnd").await.unwrap(), "unhealthy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -52,6 +52,10 @@ pub struct BindMount {
|
||||
#[allow(dead_code)] // Bridge is reserved for Phase 5 per-app network isolation.
|
||||
pub enum NetworkMode {
|
||||
#[default]
|
||||
Default,
|
||||
/// Host networking is only for companion/proxy containers that need to
|
||||
/// reach node-local daemons directly. It cannot be combined with
|
||||
/// PublishPort because Podman discards port mappings in host mode.
|
||||
Host,
|
||||
/// A user-defined podman network — quadlet creates the container
|
||||
/// attached to it. The network must already exist (orchestrator's
|
||||
@@ -86,11 +90,11 @@ impl RestartPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Container healthcheck wired through to systemd via `Notify=healthy`.
|
||||
/// When set, `systemctl start <name>.service` blocks until the container's
|
||||
/// own healthcheck reports green — eliminating the "container up but RPC
|
||||
/// not ready" race that the orchestrator currently papers over with
|
||||
/// post-start polling.
|
||||
/// Container healthcheck wired through to Podman.
|
||||
/// Systemd should consider the unit started once the container process is
|
||||
/// running; health probes are app status, not boot ordering. Blocking
|
||||
/// `systemctl start` on health made boot reconciliation hang when an image
|
||||
/// lacked the probe helper binary, even though the service itself was live.
|
||||
///
|
||||
/// Ranges roughly mirror the manifest's HealthCheck struct: `cmd` is the
|
||||
/// shell form (`/usr/bin/curl -fsS http://localhost:8332/health` etc.),
|
||||
@@ -120,8 +124,8 @@ pub struct QuadletUnit {
|
||||
pub extra_podman_args: Vec<String>,
|
||||
pub depends_on: Vec<String>,
|
||||
/// Phase 3.4: when present the rendered unit emits HealthCmd=,
|
||||
/// HealthInterval=, HealthTimeout=, HealthRetries=, AND Notify=healthy
|
||||
/// so systemctl start blocks on a green health probe.
|
||||
/// HealthInterval=, HealthTimeout=, and HealthRetries= for Podman's
|
||||
/// health state without blocking systemd's start job.
|
||||
pub health: Option<HealthSpec>,
|
||||
// Backend-manifest extensions (Phase 3.1). Companion units leave
|
||||
// these defaulted; the renderer skips empty/false directives so a
|
||||
@@ -130,6 +134,7 @@ pub struct QuadletUnit {
|
||||
pub environment: Vec<String>,
|
||||
pub devices: Vec<String>,
|
||||
pub add_hosts: Vec<(String, String)>,
|
||||
pub network_aliases: Vec<String>,
|
||||
pub entrypoint: Option<Vec<String>>,
|
||||
pub command: Vec<String>,
|
||||
pub read_only_root: bool,
|
||||
@@ -172,11 +177,16 @@ impl QuadletUnit {
|
||||
// must surface as a unit start failure, not a silent retry storm.
|
||||
let _ = writeln!(s, "Pull=never");
|
||||
match &self.network {
|
||||
NetworkMode::Default => {}
|
||||
NetworkMode::Host => {
|
||||
let _ = writeln!(s, "Network=host");
|
||||
}
|
||||
NetworkMode::Bridge(net) => {
|
||||
let _ = writeln!(s, "Network={net}");
|
||||
for alias in &self.network_aliases {
|
||||
let _ = writeln!(s, "NetworkAlias={alias}");
|
||||
let _ = writeln!(s, "PodmanArgs=--network-alias={alias}");
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(user) = &self.user {
|
||||
@@ -234,11 +244,6 @@ impl QuadletUnit {
|
||||
let _ = writeln!(s, "HealthInterval={}", h.interval);
|
||||
let _ = writeln!(s, "HealthTimeout={}", h.timeout);
|
||||
let _ = writeln!(s, "HealthRetries={}", h.retries);
|
||||
// Notify=healthy: systemd treats the unit as "started" only
|
||||
// after the first green health probe. Start ordering
|
||||
// (Requires=/After=) downstream of this unit therefore
|
||||
// doesn't fire until the app is actually serving requests.
|
||||
let _ = writeln!(s, "Notify=healthy");
|
||||
}
|
||||
if let Some(ep) = &self.entrypoint {
|
||||
// Quadlet's Exec= replaces the image entrypoint+cmd. When
|
||||
@@ -261,20 +266,6 @@ impl QuadletUnit {
|
||||
// OnFailure (clean stops stay stopped).
|
||||
let _ = writeln!(s, "Restart={}", self.restart_policy.as_systemd());
|
||||
let _ = writeln!(s, "RestartSec=10");
|
||||
if self.health.is_some() {
|
||||
// Notify=healthy makes systemd block the unit's "started"
|
||||
// state on the first green health probe. systemd's default
|
||||
// TimeoutStartSec is 90s — but `HealthInterval=30s` ×
|
||||
// `HealthRetries=3` is itself 90s, so the timeout fires the
|
||||
// moment the third probe MIGHT succeed. On .228 every backend
|
||||
// (lnd, electrumx, fedimint, btcpay-server, mempool-api,
|
||||
// bitcoin-knots) timed out at 90s and systemd terminated the
|
||||
// container while it was still warming up. Bump to 600s — long
|
||||
// enough for slow-starting backends (electrumx replays its
|
||||
// index, lnd unlocks its wallet) without being so long that a
|
||||
// truly stuck unit hangs forever.
|
||||
let _ = writeln!(s, "TimeoutStartSec=600");
|
||||
}
|
||||
let _ = writeln!(s);
|
||||
let _ = writeln!(s, "[Install]");
|
||||
let _ = writeln!(s, "WantedBy=default.target");
|
||||
@@ -290,11 +281,15 @@ fn shell_join(parts: &[String]) -> String {
|
||||
parts
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let p = p.replace(['\r', '\n'], " ");
|
||||
if p.is_empty() || p.chars().any(|c| c.is_whitespace() || "\"\\$`".contains(c)) {
|
||||
let escaped = p.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
let escaped = p
|
||||
.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('$', "$$");
|
||||
format!("\"{escaped}\"")
|
||||
} else {
|
||||
p.clone()
|
||||
p
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -323,7 +318,7 @@ impl QuadletUnit {
|
||||
other if !other.is_empty() && other != "isolated" => NetworkMode::Bridge(other.into()),
|
||||
_ => match app.container.network.as_deref() {
|
||||
Some(n) if !n.is_empty() && n != "host" => NetworkMode::Bridge(n.into()),
|
||||
_ => NetworkMode::Host,
|
||||
_ => NetworkMode::Default,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -366,6 +361,7 @@ impl QuadletUnit {
|
||||
environment: app.environment.clone(),
|
||||
devices: app.devices.clone(),
|
||||
add_hosts: vec![("host.archipelago".into(), "10.89.0.1".into())],
|
||||
network_aliases: vec![name.to_string()],
|
||||
entrypoint: app.container.entrypoint.clone(),
|
||||
command: app.container.custom_args.clone(),
|
||||
read_only_root: app.security.readonly_root,
|
||||
@@ -378,12 +374,11 @@ impl QuadletUnit {
|
||||
|
||||
/// Translate the manifest's HealthCheck shape into a HealthSpec the
|
||||
/// renderer understands. Returns None when the manifest's health spec
|
||||
/// is malformed or unsupported — we'd rather skip Notify=healthy than
|
||||
/// emit a broken HealthCmd that fails the unit start forever.
|
||||
/// is malformed or unsupported rather than emitting a broken HealthCmd.
|
||||
///
|
||||
/// Supported shapes:
|
||||
/// - type: tcp, endpoint: "host:port" → `nc -z host port`
|
||||
/// - type: http, endpoint: "host:port" or "http(s)://host:port", path → curl
|
||||
/// - type: tcp, endpoint: "host:port" → skipped for Quadlet units
|
||||
/// - type: http, endpoint: "host:port" or "http(s)://host:port", path → wget/curl
|
||||
/// - type: cmd, endpoint: "<shell command>" → `<shell command>` verbatim
|
||||
///
|
||||
/// For type=http we accept the endpoint with or without scheme; manifests
|
||||
@@ -393,13 +388,11 @@ impl QuadletUnit {
|
||||
/// that pasted on .228 2026-05-02 and failed every probe.
|
||||
fn translate_health_check(hc: &archipelago_container::HealthCheck) -> Option<HealthSpec> {
|
||||
let cmd = match hc.check_type.as_str() {
|
||||
"tcp" => {
|
||||
let endpoint = hc.endpoint.as_deref()?;
|
||||
let (host, port) = endpoint.rsplit_once(':')?;
|
||||
// nc is in busybox/coreutils on every base image we ship.
|
||||
// The -z flag does a "scan" that exits 0 on connect, 1 otherwise.
|
||||
format!("nc -z {host} {port}")
|
||||
}
|
||||
// A generic TCP probe inside arbitrary app images is not reliable:
|
||||
// some images lack nc, some lack bash /dev/tcp, and failures leave
|
||||
// Podman/systemd health in a false-negative state. Keep TCP readiness
|
||||
// checks in the host-side lifecycle/status layer instead.
|
||||
"tcp" => return None,
|
||||
"http" => {
|
||||
let endpoint = hc.endpoint.as_deref()?.trim();
|
||||
// Accept either bare host:port or a full URL. If endpoint
|
||||
@@ -426,9 +419,14 @@ fn translate_health_check(hc: &archipelago_container::HealthCheck) -> Option<Hea
|
||||
let path = hc.path.as_deref().unwrap_or("/");
|
||||
format!("{url}{path}")
|
||||
};
|
||||
// -fsS: fail on non-2xx, silent except on error, show errors.
|
||||
// -m 5: per-request timeout matches the default manifest timeout.
|
||||
format!("curl -fsS -m 5 {final_url}")
|
||||
// Images vary wildly: SearXNG ships wget but no curl, while some
|
||||
// Node images ship neither. Use whichever probe helper exists and
|
||||
// skip Podman health if the image has none; host-side lifecycle
|
||||
// probes still verify reachability.
|
||||
format!(
|
||||
"if command -v wget >/dev/null 2>&1; then wget -q -T 5 -O /dev/null {0}; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 5 {0}; else exit 0; fi",
|
||||
final_url
|
||||
)
|
||||
}
|
||||
"cmd" => hc.endpoint.as_deref()?.to_string(),
|
||||
_ => return None,
|
||||
@@ -528,6 +526,70 @@ pub async fn enable_now(service: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Restart a generated Quadlet service after rewriting a known-bad unit.
|
||||
pub async fn restart_service(service: &str) -> Result<()> {
|
||||
let status = Command::new("systemctl")
|
||||
.args(["--user", "restart", service])
|
||||
.status()
|
||||
.await
|
||||
.with_context(|| format!("spawn systemctl --user restart {service}"))?;
|
||||
if !status.success() {
|
||||
return Err(anyhow!(
|
||||
"systemctl --user restart {service} exited {status}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop a generated Quadlet service without removing its unit file.
|
||||
pub async fn stop_service(service: &str) -> Result<()> {
|
||||
let status = Command::new("systemctl")
|
||||
.args(["--user", "stop", service])
|
||||
.status()
|
||||
.await
|
||||
.with_context(|| format!("spawn systemctl --user stop {service}"))?;
|
||||
if !status.success() {
|
||||
return Err(anyhow!("systemctl --user stop {service} exited {status}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn contains_stale_health_gate(unit_body: &str) -> bool {
|
||||
unit_body.contains("Notify=healthy")
|
||||
|| unit_body.contains("TimeoutStartSec=600")
|
||||
|| unit_body.contains("HealthCmd=nc -z")
|
||||
}
|
||||
|
||||
pub fn health_cmd_changed(old_body: &str, new_body: &str) -> bool {
|
||||
directive_values(old_body, "HealthCmd=") != directive_values(new_body, "HealthCmd=")
|
||||
}
|
||||
|
||||
pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
|
||||
let old_ports = directive_values(old_body, "PublishPort=");
|
||||
let new_ports = directive_values(new_body, "PublishPort=");
|
||||
old_ports != new_ports
|
||||
}
|
||||
|
||||
pub fn network_aliases_changed(old_body: &str, new_body: &str) -> bool {
|
||||
let old_aliases = directive_values(old_body, "NetworkAlias=");
|
||||
let new_aliases = directive_values(new_body, "NetworkAlias=");
|
||||
old_aliases != new_aliases
|
||||
}
|
||||
|
||||
pub fn exec_changed(old_body: &str, new_body: &str) -> bool {
|
||||
let old_exec = directive_values(old_body, "Exec=");
|
||||
let new_exec = directive_values(new_body, "Exec=");
|
||||
old_exec != new_exec
|
||||
}
|
||||
|
||||
fn directive_values(unit_body: &str, prefix: &str) -> Vec<String> {
|
||||
unit_body
|
||||
.lines()
|
||||
.filter_map(|line| line.trim().strip_prefix(prefix))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Stop + remove a quadlet unit and its on-disk file. Best-effort:
|
||||
/// errors stop only the destructive write at the failing step so a
|
||||
/// partial removal doesn't leave a quadlet file pointing at a service
|
||||
@@ -706,6 +768,14 @@ mod tests {
|
||||
);
|
||||
// Embedded quotes must escape:
|
||||
assert_eq!(shell_join(&[r#"say "hi""#.into()]), r#""say \"hi\"""#);
|
||||
assert_eq!(
|
||||
shell_join(&[
|
||||
"sh".into(),
|
||||
"-lc".into(),
|
||||
"if true; then\n exec app;\nfi".into()
|
||||
]),
|
||||
"sh -lc \"if true; then exec app; fi\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -829,6 +899,29 @@ app:
|
||||
assert_eq!(u.restart_policy, RestartPolicy::OnFailure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_manifest_uses_default_network_for_isolated_ports() {
|
||||
let yaml = r#"
|
||||
app:
|
||||
id: searxng
|
||||
name: SearXNG
|
||||
version: 1.0.0
|
||||
container:
|
||||
image: searxng:latest
|
||||
ports:
|
||||
- host: 8888
|
||||
container: 8080
|
||||
protocol: tcp
|
||||
security:
|
||||
network_policy: isolated
|
||||
"#;
|
||||
let m = AppManifest::parse(yaml).expect("manifest must parse");
|
||||
let s = QuadletUnit::from_manifest(&m, "searxng").render();
|
||||
|
||||
assert!(s.contains("PublishPort=8888:8080/tcp"));
|
||||
assert!(!s.contains("Network=host"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_manifest_preserves_grafana_data_uid_and_volume_shape() {
|
||||
let yaml = r#"
|
||||
@@ -916,28 +1009,25 @@ app:
|
||||
u.name = "lnd".into();
|
||||
u.image = "x:1".into();
|
||||
u.health = Some(HealthSpec {
|
||||
cmd: "nc -z localhost 10009".into(),
|
||||
cmd: "probe-ready".into(),
|
||||
interval: "30s".into(),
|
||||
timeout: "5s".into(),
|
||||
retries: 3,
|
||||
});
|
||||
let s = u.render();
|
||||
assert!(s.contains("HealthCmd=nc -z localhost 10009"));
|
||||
assert!(s.contains("HealthCmd=probe-ready"));
|
||||
assert!(s.contains("HealthInterval=30s"));
|
||||
assert!(s.contains("HealthTimeout=5s"));
|
||||
assert!(s.contains("HealthRetries=3"));
|
||||
assert!(s.contains("Notify=healthy"));
|
||||
// Notify=healthy needs a long-enough TimeoutStartSec or systemd
|
||||
// kills the unit before the first probe can pass — observed live
|
||||
// on .228 2026-05-02 across all six backends.
|
||||
assert!(s.contains("TimeoutStartSec=600"), "got: {s}");
|
||||
assert!(!s.contains("Notify=healthy"));
|
||||
assert!(!s.contains("TimeoutStartSec=600"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_skips_health_directives_when_absent() {
|
||||
// No health spec → no Notify=healthy, no HealthCmd, no
|
||||
// TimeoutStartSec override (default 90s applies). Companions rely
|
||||
// on this so their rendered bytes stay unchanged.
|
||||
// No health spec → no Notify=healthy, no HealthCmd, no TimeoutStartSec
|
||||
// override. Companions rely on this so their rendered bytes stay
|
||||
// unchanged.
|
||||
let s = sample_unit().render();
|
||||
assert!(!s.contains("HealthCmd="));
|
||||
assert!(!s.contains("Notify=healthy"));
|
||||
@@ -956,9 +1046,7 @@ app:
|
||||
timeout: "5s".into(),
|
||||
retries: 3,
|
||||
};
|
||||
let h = translate_health_check(&tcp).expect("tcp must translate");
|
||||
assert_eq!(h.cmd, "nc -z localhost 10009");
|
||||
assert_eq!(h.retries, 3);
|
||||
assert!(translate_health_check(&tcp).is_none());
|
||||
|
||||
let http = HealthCheck {
|
||||
check_type: "http".into(),
|
||||
@@ -969,7 +1057,10 @@ app:
|
||||
retries: 5,
|
||||
};
|
||||
let h = translate_health_check(&http).expect("http must translate");
|
||||
assert_eq!(h.cmd, "curl -fsS -m 5 http://localhost:8080/health");
|
||||
assert_eq!(
|
||||
h.cmd,
|
||||
"if command -v wget >/dev/null 2>&1; then wget -q -T 5 -O /dev/null http://localhost:8080/health; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 5 http://localhost:8080/health; else exit 0; fi"
|
||||
);
|
||||
|
||||
let cmdck = HealthCheck {
|
||||
check_type: "cmd".into(),
|
||||
@@ -982,8 +1073,7 @@ app:
|
||||
let h = translate_health_check(&cmdck).expect("cmd must translate");
|
||||
assert_eq!(h.cmd, "/usr/local/bin/probe.sh");
|
||||
|
||||
// Unknown type → None (renderer skips Notify=healthy entirely
|
||||
// rather than emit a broken HealthCmd that hangs the unit start).
|
||||
// Unknown type → None rather than emit a broken HealthCmd.
|
||||
let bad = HealthCheck {
|
||||
check_type: "exec".into(),
|
||||
endpoint: Some("foo".into()),
|
||||
@@ -994,7 +1084,7 @@ app:
|
||||
};
|
||||
assert!(translate_health_check(&bad).is_none());
|
||||
|
||||
// Malformed tcp endpoint → None (no port separator).
|
||||
// TCP is skipped entirely for Quadlet units.
|
||||
let badtcp = HealthCheck {
|
||||
check_type: "tcp".into(),
|
||||
endpoint: Some("hostonly".into()),
|
||||
@@ -1022,7 +1112,7 @@ app:
|
||||
retries: 3,
|
||||
};
|
||||
let h = translate_health_check(&with_scheme).expect("with-scheme must translate");
|
||||
assert_eq!(h.cmd, "curl -fsS -m 5 http://localhost:8175/");
|
||||
assert!(h.cmd.contains("http://localhost:8175/"));
|
||||
assert!(!h.cmd.contains("http://http://"), "got: {}", h.cmd);
|
||||
|
||||
let with_https = HealthCheck {
|
||||
@@ -1035,7 +1125,7 @@ app:
|
||||
};
|
||||
let h = translate_health_check(&with_https).expect("https must translate");
|
||||
// Endpoint already has /health → don't append the default "/".
|
||||
assert_eq!(h.cmd, "curl -fsS -m 5 https://example.local/health");
|
||||
assert!(h.cmd.contains("https://example.local/health"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1056,12 +1146,46 @@ app:
|
||||
"#;
|
||||
let m = AppManifest::parse(yaml).unwrap();
|
||||
let u = QuadletUnit::from_manifest(&m, "lnd");
|
||||
let h = u.health.as_ref().expect("health should be populated");
|
||||
assert_eq!(h.cmd, "nc -z localhost 10009");
|
||||
assert_eq!(h.interval, "15s");
|
||||
assert_eq!(h.timeout, "4s");
|
||||
assert_eq!(h.retries, 5);
|
||||
assert!(u.render().contains("Notify=healthy"));
|
||||
assert!(u.health.is_none());
|
||||
assert!(!u.render().contains("Notify=healthy"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publish_ports_changed_detects_port_binding_drift() {
|
||||
let old = "[Container]\nPublishPort=9735:9735/tcp\nPublishPort=8080:8080/tcp\n";
|
||||
let new = "[Container]\nPublishPort=9735:9735/tcp\nPublishPort=18080:8080/tcp\n";
|
||||
assert!(publish_ports_changed(old, new));
|
||||
assert!(!publish_ports_changed(new, new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_aliases_changed_detects_service_discovery_drift() {
|
||||
let old = "[Container]\nNetwork=archy-net\n";
|
||||
let new = "[Container]\nNetwork=archy-net\nNetworkAlias=bitcoin-knots\n";
|
||||
assert!(network_aliases_changed(old, new));
|
||||
assert!(!network_aliases_changed(new, new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shell_join_escapes_dollars_for_container_runtime_expansion() {
|
||||
let rendered = shell_join(&["sh".into(), "-lc".into(), "echo ${BITCOIN_RPC_PASS}".into()]);
|
||||
assert!(rendered.contains("$${BITCOIN_RPC_PASS}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exec_changed_detects_command_drift() {
|
||||
let old = "[Container]\nExec=sh -lc \"echo ${BITCOIN_RPC_PASS}\"\n";
|
||||
let new = "[Container]\nExec=sh -lc \"echo $${BITCOIN_RPC_PASS}\"\n";
|
||||
assert!(exec_changed(old, new));
|
||||
assert!(!exec_changed(new, new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_cmd_changed_detects_probe_drift() {
|
||||
let old = "[Container]\nHealthCmd=curl -fsS http://localhost:8080/\n";
|
||||
let new = "[Container]\nHealthCmd=if command -v wget >/dev/null 2>&1; then wget -q -T 5 -O /dev/null http://localhost:8080/; elif command -v curl >/dev/null 2>&1; then curl -fsS -m 5 http://localhost:8080/; else exit 0; fi\n";
|
||||
assert!(health_cmd_changed(old, new));
|
||||
assert!(!health_cmd_changed(new, new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1098,6 +1222,8 @@ app:
|
||||
assert!(body.contains("ContainerName=lnd"));
|
||||
assert!(body.contains("Image=registry/lnd:latest"));
|
||||
assert!(body.contains("Network=archy-net"));
|
||||
assert!(body.contains("NetworkAlias=lnd"));
|
||||
assert!(body.contains("PodmanArgs=--network-alias=lnd"));
|
||||
assert!(body.contains("PublishPort=10009:10009/tcp"));
|
||||
assert!(body.contains("Volume=/var/lib/archipelago/lnd:/root/.lnd:Z"));
|
||||
assert!(body.contains("Environment=LND_NETWORK=mainnet"));
|
||||
|
||||
Reference in New Issue
Block a user