chore(release): stage v1.7.52-alpha

This commit is contained in:
archipelago
2026-05-05 11:29:18 -04:00
parent 10fbb8f87c
commit 745cb1c626
86 changed files with 4084 additions and 966 deletions
+1 -1
View File
@@ -80,7 +80,7 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
[[package]]
name = "archipelago"
version = "1.7.51-alpha"
version = "1.7.52-alpha"
dependencies = [
"anyhow",
"archipelago-container",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "archipelago"
version = "1.7.51-alpha"
version = "1.7.52-alpha"
edition = "2021"
description = "Archipelago Bitcoin Node OS - Native backend"
authors = ["Archipelago Team"]
+37 -5
View File
@@ -379,11 +379,22 @@ impl RpcHandler {
// If app_id is provided, get health for that app.
if let Some(params) = params {
if let Some(app_id) = params.get("app_id").and_then(|v| v.as_str()) {
let health = orchestrator
.health(app_id)
.await
.context("Failed to get container health")?;
return Ok(serde_json::json!({ app_id: health }));
let mut last_err: Option<anyhow::Error> = None;
for candidate in status_app_id_candidates(app_id) {
match orchestrator.health(&candidate).await {
Ok(health) => return Ok(serde_json::json!({ app_id: health })),
Err(e) => last_err = Some(e),
}
}
for name in status_container_name_candidates(app_id) {
if let Some(health) = inspect_container_health_value(&name).await {
return Ok(serde_json::json!({ app_id: health }));
}
}
if let Some(e) = last_err {
return Err(e.context("Failed to get container health"));
}
return Err(anyhow::anyhow!("Failed to get container health"));
}
}
@@ -449,6 +460,14 @@ fn status_app_id_candidates(app_id: &str) -> Vec<String> {
push("mempool-electrs");
push("electrumx");
}
"mempool" | "mempool-web" => {
push("mempool");
push("archy-mempool-web");
}
"immich" => {
push("immich");
push("immich_server");
}
_ => push(app_id),
}
@@ -469,6 +488,8 @@ fn status_container_name_candidates(app_id: &str) -> Vec<String> {
"lnd-ui" => push("archy-lnd-ui"),
"electrs-ui" => push("archy-electrs-ui"),
"electrs" | "mempool-electrs" => push("electrumx"),
"mempool" | "mempool-web" | "archy-mempool-web" => push("mempool"),
"immich" => push("immich_server"),
_ => {}
}
@@ -511,3 +532,14 @@ async fn inspect_container_state_value(name: &str) -> Option<serde_json::Value>
"running": running,
}))
}
async fn inspect_container_health_value(name: &str) -> Option<String> {
let v = inspect_container_state_value(name).await?;
match v.get("state").and_then(|s| s.as_str()).unwrap_or("unknown") {
"running" => Some("healthy".to_string()),
"created" => Some("starting".to_string()),
"paused" => Some("paused".to_string()),
"exited" | "stopped" => Some("unhealthy".to_string()),
other => Some(format!("unknown:{other}")),
}
}
@@ -13,6 +13,7 @@ impl RpcHandler {
match method {
"echo" => self.handle_echo(params).await,
"server.echo" => self.handle_echo(params).await,
"server.get-state" => self.handle_server_get_state().await,
"health" => self.handle_health().await,
"auth.login" => self.handle_auth_login(params).await,
"auth.logout" => self.handle_auth_logout().await,
@@ -530,6 +531,11 @@ impl RpcHandler {
Ok(serde_json::json!({ "message": "Hello from Archipelago!" }))
}
async fn handle_server_get_state(&self) -> Result<serde_json::Value> {
let (data, rev) = self.state_manager.get_snapshot().await;
Ok(serde_json::json!({ "data": data, "rev": rev }))
}
pub(super) async fn handle_health(&self) -> Result<serde_json::Value> {
let recovery_complete = crate::crash_recovery::is_recovery_complete();
let uptime = crate::crash_recovery::uptime_seconds();
+43 -7
View File
@@ -309,16 +309,23 @@ pub(super) fn all_container_names(package_id: &str) -> Vec<String> {
let archy = format!("archy-{}", package_id);
match package_id {
// Bitcoin: multiple historical names
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => vec![
// Bitcoin variants share the UI but not the backend process. Keep
// backend names precise so stopping one implementation does not clear
// stop markers or issue podman operations for the other.
"bitcoin" | "bitcoin-knots" => vec![
"bitcoin-knots".into(),
"bitcoin".into(),
"bitcoin-core".into(),
"archy-bitcoin-knots".into(),
"archy-bitcoin".into(),
"bitcoin-ui".into(),
"archy-bitcoin-ui".into(),
],
"bitcoin-core" => vec![
"bitcoin-core".into(),
"archy-bitcoin-core".into(),
"bitcoin-ui".into(),
"archy-bitcoin-ui".into(),
],
// LND + UI
"lnd" => vec!["lnd".into(), "archy-lnd".into(), "archy-lnd-ui".into()],
// Electrumx: multiple aliases
@@ -377,6 +384,15 @@ pub(super) fn all_container_names(package_id: &str) -> Vec<String> {
"penpot-exporter".into(),
"penpot-frontend".into(),
],
"indeedhub" => vec![
"indeedhub-postgres".into(),
"indeedhub-redis".into(),
"indeedhub-minio".into(),
"indeedhub-relay".into(),
"indeedhub-api".into(),
"indeedhub-ffmpeg".into(),
"indeedhub".into(),
],
"nostr-vpn" => vec![
"nostr-vpn".into(),
"archy-nostr-vpn".into(),
@@ -411,6 +427,22 @@ pub(super) async fn get_containers_for_app(package_id: &str) -> Result<Vec<Strin
Ok(result)
}
#[cfg(test)]
mod tests {
use super::all_container_names;
#[test]
fn bitcoin_variant_container_names_are_precise() {
let core = all_container_names("bitcoin-core");
assert!(core.contains(&"bitcoin-core".to_string()));
assert!(!core.contains(&"bitcoin-knots".to_string()));
let knots = all_container_names("bitcoin-knots");
assert!(knots.contains(&"bitcoin-knots".to_string()));
assert!(!knots.contains(&"bitcoin-core".to_string()));
}
}
/// Get data directories to clean for an app.
/// Caller must validate package_id before calling.
pub(super) fn get_data_dirs_for_app(package_id: &str) -> Vec<String> {
@@ -802,7 +834,11 @@ pub(super) async fn get_app_config(
vec!["/var/lib/archipelago/uptime-kuma:/app/data".to_string()],
vec!["TZ=UTC".to_string()],
None,
None,
Some(vec![
"--".to_string(),
"node".to_string(),
"server/server.js".to_string(),
]),
),
"tailscale" => (
vec!["8240:8240".to_string()],
@@ -817,7 +853,7 @@ pub(super) async fn get_app_config(
Some(vec![
"sh".to_string(),
"-c".to_string(),
"tailscale web --listen 0.0.0.0:8240 & exec tailscaled".to_string(),
"tailscaled --tun=userspace-networking & sleep 2; tailscale web --listen 0.0.0.0:8240 & wait".to_string(),
]),
),
"fedimint" => (
@@ -978,8 +1014,8 @@ pub(super) async fn get_app_config(
None,
)
}
// Gitea binds to 3001 internally. Nginx on port 3000 strips X-Frame-Options
// so Gitea works in Archipelago's iframe. See nginx-gitea-iframe.conf.
// Gitea listens on container port 3000 and is launched directly on
// host port 3001 because it blocks iframe embedding.
"gitea" => (
vec!["3001:3000".to_string(), "2222:22".to_string()],
vec![
@@ -1,5 +1,7 @@
use super::config::get_containers_for_app;
use anyhow::Result;
use crate::data_model::{PackageDataEntry, PackageState};
use anyhow::{Context, Result};
use std::collections::HashMap;
use tracing::info;
/// Names of container variants that represent a running Bitcoin node
@@ -8,6 +10,13 @@ const BITCOIN_NAMES: &[&str] = &["bitcoin-knots", "bitcoin-core", "bitcoin"];
/// Names of container variants that represent a running Electrum indexer
const ELECTRUM_NAMES: &[&str] = &["electrumx", "mempool-electrs", "electrs"];
fn requires_unpruned_bitcoin(package_id: &str) -> bool {
matches!(
package_id,
"electrumx" | "mempool-electrs" | "electrs" | "mempool" | "mempool-web"
)
}
/// Snapshot of which dependency services are currently running.
pub(super) struct RunningDeps {
pub has_bitcoin: bool,
@@ -15,13 +24,43 @@ pub(super) struct RunningDeps {
pub has_lnd: bool,
}
pub(super) fn detect_running_deps_from_package_data(
packages: &HashMap<String, PackageDataEntry>,
) -> RunningDeps {
let is_running = |names: &[&str]| {
names.iter().any(|name| {
packages
.get(*name)
.map(|pkg| pkg.state == PackageState::Running)
.unwrap_or(false)
})
};
RunningDeps {
has_bitcoin: is_running(BITCOIN_NAMES),
has_electrumx: is_running(ELECTRUM_NAMES),
has_lnd: is_running(&["lnd"]),
}
}
/// Query podman for currently running containers and return dependency status.
pub(super) async fn detect_running_deps() -> Result<RunningDeps> {
let dep_check = tokio::process::Command::new("podman")
.args(["ps", "--format", "{{.Names}}"])
.output()
.await
.map_err(|e| anyhow::anyhow!("Failed to check running containers: {}", e))?;
let dep_check = tokio::time::timeout(
std::time::Duration::from_secs(30),
tokio::process::Command::new("podman")
.args(["ps", "--format", "{{.Names}}"])
.output(),
)
.await
.map_err(|_| anyhow::anyhow!("Timed out checking running containers"))?
.map_err(|e| anyhow::anyhow!("Failed to check running containers: {}", e))?;
if !dep_check.status.success() {
anyhow::bail!(
"Failed to check running containers: {}",
String::from_utf8_lossy(&dep_check.stderr).trim()
);
}
let running = String::from_utf8_lossy(&dep_check.stdout);
let is_running = |names: &[&str]| {
@@ -76,6 +115,65 @@ pub(super) fn check_install_deps(package_id: &str, deps: &RunningDeps) -> Result
}
}
/// ElectrumX and Mempool's Electrum backend need historical blocks from an
/// unpruned node while building their indexes. A pruned Bitcoin node can be
/// running and RPC-reachable but still leave them stuck with closed ports.
pub(super) async fn check_bitcoin_pruning_compatibility(package_id: &str) -> Result<()> {
if !requires_unpruned_bitcoin(package_id) {
return Ok(());
}
let (rpc_user, rpc_pass) = crate::bitcoin_rpc::bitcoin_rpc_credentials().await;
let body = serde_json::json!({
"jsonrpc": "1.0",
"id": "package-install-prune-check",
"method": "getblockchaininfo",
"params": [],
});
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.build()
.context("building Bitcoin RPC client")?;
let resp = client
.post(crate::constants::BITCOIN_RPC_URL)
.basic_auth(rpc_user, Some(rpc_pass))
.header("Content-Type", "application/json")
.json(&body)
.send()
.await
.context("checking Bitcoin pruning status")?;
let status = resp.status();
let json: serde_json::Value = resp.json().await.context("decode Bitcoin RPC response")?;
if !status.is_success() {
anyhow::bail!(
"Bitcoin RPC returned {} while checking pruning status",
status
);
}
if let Some(error) = json.get("error").filter(|e| !e.is_null()) {
anyhow::bail!("Bitcoin RPC error while checking pruning status: {}", error);
}
let Some(result) = json.get("result") else {
anyhow::bail!("Bitcoin RPC response missing result while checking pruning status");
};
if result
.get("pruned")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
anyhow::bail!(
"{} requires an unpruned Bitcoin node while indexing. Current Bitcoin is pruned; use a full node with enough disk for txindex/full block history, then reinstall/restart {}.",
package_id,
package_id
);
}
Ok(())
}
/// Log informational messages about optional dependencies.
pub(super) fn log_optional_dep_info(package_id: &str, deps: &RunningDeps) {
if matches!(package_id, "btcpay-server" | "btcpayserver") && !deps.has_lnd {
@@ -129,6 +227,18 @@ pub(super) fn startup_order(package_id: &str) -> &'static [&'static str] {
"mempool",
],
"immich" => &["immich_postgres", "immich_redis", "immich_server"],
"indeedhub" => &[
"indeedhub-postgres",
"indeedhub-redis",
"indeedhub-minio",
"indeedhub-relay",
"indeedhub-api",
"indeedhub-ffmpeg",
"indeedhub",
],
"btcpay-server" | "btcpayserver" | "btcpay" => {
&["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"]
}
"penpot" | "penpot-frontend" => &[
"penpot-postgres",
"penpot-valkey",
@@ -211,3 +321,24 @@ pub(super) fn configure_fedimint_lnd(
]);
}
}
#[cfg(test)]
mod tests {
use super::requires_unpruned_bitcoin;
#[test]
fn unpruned_bitcoin_required_for_electrum_indexers_and_mempool() {
for package_id in [
"electrumx",
"mempool-electrs",
"electrs",
"mempool",
"mempool-web",
] {
assert!(requires_unpruned_bitcoin(package_id), "{package_id}");
}
for package_id in ["bitcoin-knots", "btcpay-server", "lnd", "fedimint"] {
assert!(!requires_unpruned_bitcoin(package_id), "{package_id}");
}
}
}
+419 -166
View File
@@ -3,8 +3,9 @@ use super::config::{
is_readonly_compatible, is_valid_docker_image,
};
use super::dependencies::{
check_install_deps, configure_fedimint_lnd, detect_running_deps, log_optional_dep_info,
needs_archy_net,
check_bitcoin_pruning_compatibility, check_install_deps, configure_fedimint_lnd,
detect_running_deps, detect_running_deps_from_package_data, log_optional_dep_info,
needs_archy_net, RunningDeps,
};
use super::progress::parse_pull_progress;
use super::validation::validate_app_id;
@@ -32,6 +33,130 @@ pub(in crate::api::rpc) async fn install_log(msg: &str) {
}
}
pub(super) async fn patch_indeedhub_nostr_provider() {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"sed",
"-i",
"/X-Frame-Options/d",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let provider_src = "/opt/archipelago/web-ui/nostr-provider.js";
if tokio::fs::metadata(provider_src).await.is_ok() {
let _ = tokio::process::Command::new("podman")
.args([
"cp",
provider_src,
"indeedhub:/usr/share/nginx/html/nostr-provider.js",
])
.output()
.await;
}
let check = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"grep",
"-q",
"nostr-provider",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let already_patched = check.map(|o| o.status.success()).unwrap_or(false);
if !already_patched {
let cat_out = tokio::process::Command::new("podman")
.args(["exec", "indeedhub", "cat", "/etc/nginx/conf.d/default.conf"])
.output()
.await;
if let Ok(out) = cat_out {
if out.status.success() {
let conf = String::from_utf8_lossy(&out.stdout).to_string();
let conf = conf.replace(
"location = /sw.js {",
"location = /nostr-provider.js {\n\
add_header Cache-Control \"no-cache, no-store, must-revalidate\";\n\
expires off;\n\
}\n\n\
location = /sw.js {",
);
let conf = if conf.contains("try_files") && !conf.contains("sub_filter") {
conf.replacen(
"try_files $uri $uri/ /index.html;",
"try_files $uri $uri/ /index.html;\n\
sub_filter_once on;\n\
sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';",
1,
)
} else {
conf
};
let tmp_path = "/tmp/indeedhub-nginx-patch.conf";
if tokio::fs::write(tmp_path, &conf).await.is_ok() {
let _ = tokio::process::Command::new("podman")
.args(["cp", tmp_path, "indeedhub:/etc/nginx/conf.d/default.conf"])
.output()
.await;
let _ = tokio::fs::remove_file(tmp_path).await;
}
}
}
}
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"sed",
"-i",
"s|proxy_set_header X-Forwarded-Prefix /api;|proxy_set_header X-Forwarded-Prefix $http_x_forwarded_prefix/api;|",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let reload = tokio::process::Command::new("podman")
.args(["exec", "indeedhub", "nginx", "-s", "reload"])
.output()
.await;
match reload {
Ok(o) if o.status.success() => {
info!("IndeeHub: NIP-07 provider injected, nginx patched and reloaded");
}
Ok(o) => {
tracing::warn!(
"IndeeHub nginx reload failed: {}",
String::from_utf8_lossy(&o.stderr)
);
}
Err(e) => {
tracing::warn!("IndeeHub nginx reload error: {}", e);
}
}
}
fn dependency_cache_satisfies(package_id: &str, deps: &RunningDeps) -> bool {
match package_id {
"electrumx" | "mempool-electrs" | "electrs" | "lnd" | "btcpay-server" | "btcpayserver" => {
deps.has_bitcoin
}
"mempool" | "mempool-web" => deps.has_bitcoin && deps.has_electrumx,
"fedimint" => true,
_ => true,
}
}
impl RpcHandler {
/// Install a package from a Docker image.
/// Security: Image verification, resource limits, network isolation.
@@ -62,6 +187,8 @@ impl RpcHandler {
package_id, docker_image
);
cleanup_stale_package_ports(package_id).await;
if !is_valid_docker_image(docker_image) {
install_log(&format!(
"INSTALL FAIL: {} — invalid image format",
@@ -108,11 +235,22 @@ impl RpcHandler {
return self.install_indeedhub_stack().await;
}
// Dependency checks
let deps = detect_running_deps().await?;
// Dependency checks. Prefer the scanner's cached package state so a
// congested Podman API does not turn an already-running dependency into
// a false install failure. Fall back to a bounded direct Podman probe
// only when the cache does not show the dependency.
let deps = {
let (data, _) = self.state_manager.get_snapshot().await;
let cached = detect_running_deps_from_package_data(&data.package_data);
if dependency_cache_satisfies(package_id, &cached) {
cached
} else {
detect_running_deps().await?
}
};
check_install_deps(package_id, &deps)?;
check_bitcoin_pruning_compatibility(package_id).await?;
log_optional_dep_info(package_id, &deps);
check_bitcoin_implementation_conflict(package_id).await?;
let repaired_bitcoin_conf =
if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") {
// Materialise the RPC password file before any install path
@@ -243,6 +381,7 @@ impl RpcHandler {
package_id
))
.await;
ensure_host_port_listener(package_id, package_id).await?;
return Ok(serde_json::json!({
"success": true,
"package_id": package_id,
@@ -268,6 +407,8 @@ impl RpcHandler {
Ok(container_name) => {
self.set_install_phase(package_id, InstallPhase::WaitingHealthy)
.await;
crate::api::rpc::package::runtime::reconcile_companions_for(package_id)
.await;
install_log(&format!(
"INSTALL ORCH OK: {} (app={}) — container={}",
package_id, orchestrator_app_id, container_name
@@ -368,17 +509,15 @@ impl RpcHandler {
"--restart=unless-stopped",
];
let is_tailscale = package_id == "tailscale";
// Explicit DNS alias for aardvark-dns (must outlive run_args)
let network_alias_flag = format!("--network-alias={}", container_name);
// Network mode
if is_tailscale {
run_args.push("--network=host");
run_args.push("--privileged");
run_args.push("--cap-add=NET_ADMIN");
run_args.push("--cap-add=NET_RAW");
run_args.push("--device=/dev/net/tun");
if package_id == "uptime-kuma" || package_id == "gitea" || package_id == "tailscale" {
// These standalone web UIs have repeatedly lost host listeners
// under Podman's rootless pasta backend while staying healthy internally.
// Use slirp4netns/rootlessport for this standalone web UI.
run_args.push("--network=slirp4netns:allow_host_loopback=true");
} else if needs_archy_net(package_id) {
// Create archy-net if it doesn't exist (idempotent — "already exists" is fine)
match tokio::process::Command::new("podman")
@@ -420,30 +559,24 @@ impl RpcHandler {
run_args.push(&host_gateway_flag);
// Security hardening (skip for privileged containers)
let security_caps: Vec<String> = if !is_tailscale {
get_app_capabilities(package_id)
} else {
vec![]
};
let readonly_compatible = !is_tailscale && is_readonly_compatible(package_id);
let security_caps: Vec<String> = get_app_capabilities(package_id);
let readonly_compatible = is_readonly_compatible(package_id);
if !is_tailscale {
run_args.push("--cap-drop=ALL");
run_args.push("--security-opt=no-new-privileges:true");
run_args.push("--pids-limit=4096");
for cap in &security_caps {
run_args.push(cap);
}
if readonly_compatible {
run_args.push("--read-only");
run_args.push("--tmpfs=/tmp:rw,noexec,nosuid,size=256m");
run_args.push("--tmpfs=/run:rw,noexec,nosuid,size=64m");
}
run_args.push("--cap-drop=ALL");
run_args.push("--security-opt=no-new-privileges:true");
run_args.push("--pids-limit=4096");
for cap in &security_caps {
run_args.push(cap);
}
if readonly_compatible {
run_args.push("--read-only");
run_args.push("--tmpfs=/tmp:rw,noexec,nosuid,size=256m");
run_args.push("--tmpfs=/run:rw,noexec,nosuid,size=64m");
}
// Jellyfin: .NET CoreCLR needs exec-enabled /tmp for JIT compilation
if package_id == "jellyfin" {
run_args.push("--tmpfs=/tmp:rw,exec,size=256m");
}
// Jellyfin: .NET CoreCLR needs exec-enabled /tmp for JIT compilation
if package_id == "jellyfin" {
run_args.push("--tmpfs=/tmp:rw,exec,size=256m");
}
// Create data directories (mkdir only — chown happens AFTER config files are written)
@@ -490,12 +623,9 @@ impl RpcHandler {
// NOW chown data directories to container UID (after all config files are written)
self.create_data_dirs(package_id, &volumes).await;
// Port mappings (skip for host-network containers)
if !is_tailscale {
for port in &ports {
run_args.push("-p");
run_args.push(port);
}
for port in &ports {
run_args.push("-p");
run_args.push(port);
}
// Volume mounts
@@ -570,7 +700,14 @@ impl RpcHandler {
cmd.args(args);
}
let run_output = cmd.output().await.context("Failed to run container")?;
let mut run_output = cmd.output().await.context("Failed to run container")?;
if !run_output.status.success() {
let stderr = String::from_utf8_lossy(&run_output.stderr).to_string();
if cleanup_start_conflict(package_id, &stderr).await {
run_output = cmd.output().await.context("Failed to rerun container")?;
}
}
if !run_output.status.success() {
let stderr = String::from_utf8_lossy(&run_output.stderr);
@@ -680,6 +817,12 @@ impl RpcHandler {
// Post-install hooks — await completion before returning success
self.run_post_install_hooks(package_id).await;
if package_id == "nextcloud" {
repair_nextcloud_permissions().await;
}
ensure_host_port_listener(package_id, container_name).await?;
install_log(&format!(
"INSTALL OK: {} (container: {})",
package_id,
@@ -744,36 +887,16 @@ impl RpcHandler {
Ok(has_local_fallback)
}
/// Pull image with retry and exponential backoff (3 attempts: 5s, 15s, 45s).
/// Pull image once through the configured registry list. Each registry URL
/// already has a bounded timeout, so retrying the full list can leave the UI
/// in Installing for close to an hour when a large image is unavailable or
/// a registry stalls.
async fn pull_image_with_progress(&self, package_id: &str, docker_image: &str) -> Result<()> {
const MAX_ATTEMPTS: u32 = 3;
const BACKOFF_SECS: [u64; 3] = [5, 15, 45];
for attempt in 1..=MAX_ATTEMPTS {
match self.do_pull_image(package_id, docker_image).await {
Ok(()) => return Ok(()),
Err(e) if attempt < MAX_ATTEMPTS => {
let delay = BACKOFF_SECS[(attempt - 1) as usize];
tracing::warn!(
"Image pull failed for {} (attempt {}/{}): {}. Retrying in {}s...",
docker_image,
attempt,
MAX_ATTEMPTS,
e,
delay
);
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
}
Err(e) => {
self.clear_install_progress(package_id).await;
return Err(e.context(format!(
"Failed to pull {} after {} attempts",
docker_image, MAX_ATTEMPTS
)));
}
}
if let Err(e) = self.do_pull_image(package_id, docker_image).await {
self.clear_install_progress(package_id).await;
return Err(e.context(format!("Failed to pull {}", docker_image)));
}
unreachable!()
Ok(())
}
/// Pull one image URL with live progress streamed through
@@ -799,24 +922,29 @@ impl RpcHandler {
.spawn()
.context("Failed to start image pull")?;
// 10-minute per-URL budget — large layers (Minio, Postgres,
// ffmpeg) regularly take several minutes and we'd rather wait
// than bounce to the next mirror mid-download.
let pull_result = tokio::time::timeout(std::time::Duration::from_secs(600), async {
if let Some(stderr) = child.stderr.take() {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
let pkg_id = package_id.to_string();
let state_mgr = self.state_manager.clone();
// 5-minute per-URL budget. A full install tries each configured mirror
// once, so a two-registry setup fails visibly in roughly 10 minutes
// instead of staying in Installing for up to an hour.
const PULL_URL_TIMEOUT_SECS: u64 = 300;
let pull_result = tokio::time::timeout(
std::time::Duration::from_secs(PULL_URL_TIMEOUT_SECS),
async {
if let Some(stderr) = child.stderr.take() {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
let pkg_id = package_id.to_string();
let state_mgr = self.state_manager.clone();
while let Ok(Some(line)) = lines.next_line().await {
if let Some((downloaded, total)) = parse_pull_progress(&line) {
Self::update_install_progress(&state_mgr, &pkg_id, downloaded, total).await;
while let Ok(Some(line)) = lines.next_line().await {
if let Some((downloaded, total)) = parse_pull_progress(&line) {
Self::update_install_progress(&state_mgr, &pkg_id, downloaded, total)
.await;
}
}
}
}
child.wait().await
})
child.wait().await
},
)
.await;
match pull_result {
@@ -826,7 +954,11 @@ impl RpcHandler {
Ok(false)
}
Err(_) => {
tracing::warn!("Image pull timed out after 600s: {}", url);
tracing::warn!(
"Image pull timed out after {}s: {}",
PULL_URL_TIMEOUT_SECS,
url
);
let _ = child.kill().await;
let _ = child.wait().await; // reap zombie
Ok(false)
@@ -963,7 +1095,7 @@ impl RpcHandler {
// Current BTCPay Postgres image runs as uid 999 inside the
// container, so its rootless host-mapped uid is 100998.
"btcpay-postgres" | "archy-btcpay-db" => 999,
"electrumx" | "electrs" => 1000,
"electrumx" | "electrs" => 0,
_ => 0, // Most containers run as root (UID 0)
};
if container_uid == 0 {
@@ -1392,18 +1524,18 @@ autopilot.active=false\n",
}
}
// Gitea: keep it on its native host port (3001) and serve it under
// /app/gitea/ via the main Archipelago nginx config. Avoids colliding
// with Grafana, which also uses host port 3000.
// Gitea: keep it on its native host port (3001). The UI opens Gitea
// in a new tab on that direct port so absolute asset URLs must be
// rooted at the host port rather than Archipelago's /app/gitea/ path.
if package_id == "gitea" {
let _ = tokio::fs::remove_file("/etc/nginx/conf.d/gitea-iframe.conf").await;
// Set ROOT_URL to the UI path-based route so links/assets stay
// anchored under Archipelago's app proxy endpoint.
// Set ROOT_URL to the direct launch route so links/assets stay
// anchored under the same origin Gitea is launched from.
let host_ip = &self.config.host_ip;
let _ = tokio::process::Command::new("podman")
.args(["exec", "gitea", "sh", "-c",
&format!("grep -q ROOT_URL /data/gitea/conf/app.ini && sed -i 's|ROOT_URL.*|ROOT_URL = http://{}/app/gitea/|' /data/gitea/conf/app.ini || true", host_ip)])
&format!("grep -q ROOT_URL /data/gitea/conf/app.ini && sed -i 's|ROOT_URL.*|ROOT_URL = http://{}:3001/|' /data/gitea/conf/app.ini || true", host_ip)])
.output()
.await;
// Also ensure X_FRAME_OPTIONS is empty so Gitea doesn't send the header
@@ -1413,14 +1545,8 @@ autopilot.active=false\n",
.output()
.await;
// Reload main nginx so /app/gitea/ routing changes take effect.
let _ = tokio::process::Command::new("nginx")
.args(["-s", "reload"])
.output()
.await;
info!(
"Gitea: ROOT_URL set to http://{}/app/gitea/, X_FRAME_OPTIONS cleared",
"Gitea: ROOT_URL set to http://{}:3001/, X_FRAME_OPTIONS cleared",
host_ip
);
}
@@ -1661,6 +1787,159 @@ autopilot.active=false\n",
}
}
async fn cleanup_stale_package_ports(package_id: &str) {
match package_id {
"grafana" => cleanup_stale_pasta_port("3000").await,
"searxng" => cleanup_stale_pasta_port("8888").await,
"uptime-kuma" => cleanup_stale_pasta_port("3002").await,
"gitea" => {
cleanup_stale_pasta_port("3001").await;
cleanup_stale_pasta_port("2222").await;
cleanup_stale_pasta_port("3000").await;
}
_ => {}
}
}
async fn cleanup_start_conflict(package_id: &str, stderr: &str) -> bool {
match package_id {
"grafana"
if stderr.contains("pasta failed") || stderr.contains("address already in use") =>
{
cleanup_stale_pasta_port("3000").await;
true
}
"searxng"
if stderr.contains("pasta failed") || stderr.contains("address already in use") =>
{
cleanup_stale_pasta_port("8888").await;
true
}
"uptime-kuma"
if stderr.contains("pasta failed") || stderr.contains("address already in use") =>
{
cleanup_stale_pasta_port("3002").await;
true
}
"gitea" if stderr.contains("pasta failed") || stderr.contains("address already in use") => {
cleanup_stale_pasta_port("3001").await;
cleanup_stale_pasta_port("2222").await;
cleanup_stale_pasta_port("3000").await;
true
}
_ => false,
}
}
async fn cleanup_stale_pasta_port(port: &str) {
let kill_listener = format!(
"ss -ltnp 'sport = :{}' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | xargs -r kill 2>/dev/null || true",
port
);
let _ = tokio::process::Command::new("sh")
.args(["-c", &kill_listener])
.output()
.await;
let pattern = format!("pasta.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
async fn repair_nextcloud_permissions() {
let script = "chmod 755 /var/www/html /var/www/html/config /var/www/html/data 2>/dev/null || true; chmod 644 /var/www/html/.htaccess /var/www/html/index.php /var/www/html/status.php /var/www/html/config/.htaccess 2>/dev/null || true; chmod 640 /var/www/html/config/config.php 2>/dev/null || true";
let output = tokio::process::Command::new("podman")
.args(["exec", "nextcloud", "sh", "-lc", script])
.output()
.await;
match output {
Ok(out) if out.status.success() => {}
Ok(out) => {
let stderr = String::from_utf8_lossy(&out.stderr);
tracing::warn!("Nextcloud permission repair failed: {}", stderr.trim());
}
Err(err) => tracing::warn!("Failed to run Nextcloud permission repair: {}", err),
}
}
async fn ensure_host_port_listener(package_id: &str, container_name: &str) -> Result<()> {
let Some(port) = required_host_port(package_id) else {
return Ok(());
};
if wait_for_host_port(port, 10).await {
return Ok(());
}
install_log(&format!(
"INSTALL REPAIR: {} — host port {} missing after start; restarting container",
package_id, port
))
.await;
cleanup_stale_package_ports(package_id).await;
let output = tokio::process::Command::new("podman")
.args(["restart", container_name])
.output()
.await
.context("failed to restart container after missing host port")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow::anyhow!(
"Container {} host port {} was not listening and restart failed: {}",
container_name,
port,
stderr.trim()
));
}
if wait_for_host_port(port, 60).await {
install_log(&format!(
"INSTALL REPAIR OK: {} — host port {} is listening after restart",
package_id, port
))
.await;
return Ok(());
}
Err(anyhow::anyhow!(
"Container {} is running but host port {} is not listening",
container_name,
port
))
}
fn required_host_port(package_id: &str) -> Option<u16> {
match package_id {
"grafana" => Some(3000),
"searxng" => Some(8888),
"uptime-kuma" => Some(3002),
"gitea" => Some(3001),
_ => None,
}
}
async fn wait_for_host_port(port: u16, timeout_secs: u64) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
loop {
if tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok()
{
return true;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
/// Resolve the host gateway IP for --add-host flag.
/// Resolve the default gateway IP from the routing table for --add-host flag.
/// Explicit IP avoids issues with "host-gateway" in rootless Podman.
@@ -1792,73 +2071,6 @@ fn should_try_orchestrator_install(package_id: &str, orchestrator_available: boo
orchestrator_available && uses_orchestrator_install_flow(package_id)
}
async fn check_bitcoin_implementation_conflict(package_id: &str) -> Result<()> {
let other = match package_id {
"bitcoin-core" => "bitcoin-knots",
"bitcoin-knots" => "bitcoin-core",
_ => return Ok(()),
};
// Three cases for the OTHER variant:
// - missing → no conflict, continue
// - running → real conflict, refuse install
// - any other state (created/exited/configured/...) → stuck from a
// prior failed install. Auto-remove so reinstall is reachable
// without a manual `podman rm`. This is what unblocks the .198
// "bitcoin-core stuck in created, port 8332 held by bitcoin-knots"
// deadlock that no UI path could exit.
let inspect = tokio::process::Command::new("podman")
.args(["inspect", other, "--format", "{{.State.Status}}"])
.output()
.await
.context("Failed to inspect conflicting Bitcoin container")?;
if !inspect.status.success() {
return Ok(());
}
let state = String::from_utf8_lossy(&inspect.stdout).trim().to_string();
if state == "running" {
let current = pretty_bitcoin_name(other);
let requested = pretty_bitcoin_name(package_id);
return Err(anyhow::anyhow!(
"{} is currently running. Stop and uninstall {} before installing {}; both implementations use the same Bitcoin data directory and ports.",
current, current, requested
));
}
info!(
"Removing stuck {} container (state={}) before installing {}",
other, state, package_id
);
install_log(&format!(
"INSTALL UNSTUCK: removing {} (state={}) before installing {}",
other, state, package_id
))
.await;
let rm = tokio::process::Command::new("podman")
.args(["rm", "-f", other])
.output()
.await
.context("Failed to remove stuck Bitcoin container")?;
if !rm.status.success() {
let stderr = String::from_utf8_lossy(&rm.stderr);
return Err(anyhow::anyhow!(
"Failed to remove stuck {} container: {}",
other,
stderr.trim()
));
}
Ok(())
}
fn pretty_bitcoin_name(id: &str) -> &'static str {
match id {
"bitcoin-core" => "Bitcoin Core",
"bitcoin-knots" => "Bitcoin Knots",
_ => "another Bitcoin node",
}
}
fn orchestrator_install_app_id(package_id: &str) -> &str {
match package_id {
"electrs" | "mempool-electrs" => "electrumx",
@@ -1903,6 +2115,7 @@ mod tests {
orchestrator_install_app_id, should_try_orchestrator_install,
uses_orchestrator_install_flow,
};
use crate::api::rpc::package::runtime::orchestrator_uninstall_app_ids;
#[test]
fn orchestrator_install_allowlist_includes_ported_backends() {
@@ -1955,4 +2168,44 @@ mod tests {
assert_eq!(orchestrator_install_app_id("mempool-electrs"), "electrumx");
assert_eq!(orchestrator_install_app_id("lnd"), "lnd");
}
#[test]
fn uninstall_aliases_map_to_exact_manifest_app_ids() {
assert_eq!(
orchestrator_uninstall_app_ids("bitcoin-knots"),
vec!["bitcoin-knots", "bitcoin-ui"]
);
assert_eq!(
orchestrator_uninstall_app_ids("electrs"),
vec!["electrumx", "electrs-ui"]
);
assert_eq!(
orchestrator_uninstall_app_ids("btcpay-server"),
vec!["btcpay-server", "archy-nbxplorer", "archy-btcpay-db"]
);
}
#[tokio::test]
async fn companion_reconcile_aliases_include_ui_app_ids() {
use crate::api::rpc::package::runtime::reconcile_companions_for;
// Smoke only: unknown/non-companion apps are a no-op. Full companion
// behavior is covered in container::companion tests; this guards that
// the helper remains callable from install/start/restart paths.
reconcile_companions_for("filebrowser").await;
}
#[test]
fn missing_companion_is_ok_only_for_known_ui_companions() {
use crate::api::rpc::package::runtime::is_missing_companion_ok;
assert!(is_missing_companion_ok(
"archy-bitcoin-ui",
"Error: no container with name or ID \"archy-bitcoin-ui\" found"
));
assert!(!is_missing_companion_ok(
"bitcoin-knots",
"Error: no container with name or ID \"bitcoin-knots\" found"
));
}
}
+401 -3
View File
@@ -59,6 +59,8 @@ impl RpcHandler {
}
let package_id_owned = package_id.to_string();
let companion_app_id = package_id_owned.clone();
let orchestrator = self.orchestrator.clone();
let state_manager = Arc::clone(&self.state_manager);
let pre_state =
flip_package_state(&state_manager, &package_id_owned, PackageState::Starting).await;
@@ -70,8 +72,14 @@ impl RpcHandler {
.await;
tokio::spawn(async move {
match do_package_start(&to_start).await {
let result = if let Some(orchestrator) = orchestrator.as_ref() {
do_orchestrator_package_start(orchestrator.as_ref(), &to_start).await
} else {
do_package_start(&to_start).await
};
match result {
Ok(()) => {
reconcile_companions_for(&companion_app_id).await;
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
.await;
}
@@ -123,6 +131,8 @@ impl RpcHandler {
}
let package_id_owned = package_id.to_string();
let to_stop = containers.clone();
let orchestrator = self.orchestrator.clone();
let state_manager = Arc::clone(&self.state_manager);
let pre_state =
flip_package_state(&state_manager, &package_id_owned, PackageState::Stopping).await;
@@ -134,7 +144,12 @@ impl RpcHandler {
.await;
tokio::spawn(async move {
match do_package_stop(&containers).await {
let result = if let Some(orchestrator) = orchestrator.as_ref() {
do_orchestrator_package_stop(orchestrator.as_ref(), &to_stop).await
} else {
do_package_stop(&containers).await
};
match result {
Ok(()) => {
set_package_state(&state_manager, &package_id_owned, PackageState::Stopped)
.await;
@@ -182,7 +197,10 @@ impl RpcHandler {
}
let package_id_owned = package_id.to_string();
let companion_app_id = package_id_owned.clone();
let to_restart = ordered_containers_for_start(package_id).await?;
let state_manager = Arc::clone(&self.state_manager);
let orchestrator = self.orchestrator.clone();
let pre_state =
flip_package_state(&state_manager, &package_id_owned, PackageState::Restarting).await;
@@ -193,8 +211,14 @@ impl RpcHandler {
.await;
tokio::spawn(async move {
match do_package_restart(&containers).await {
let result = if let Some(orchestrator) = orchestrator.as_ref() {
do_orchestrator_package_restart(orchestrator.as_ref(), &to_restart).await
} else {
do_package_restart(&containers).await
};
match result {
Ok(()) => {
reconcile_companions_for(&companion_app_id).await;
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
.await;
}
@@ -232,6 +256,15 @@ impl RpcHandler {
// within ~10s of `podman rm`, leaving them orphaned post-uninstall.
crate::container::companion::remove_for(package_id).await;
// Keep the production reconciler from recreating an app immediately
// after uninstall. The reconciler owns a manifest map independent of
// podman state, so a raw `podman rm` alone is not enough.
if let Some(orchestrator) = &self.orchestrator {
for app_id in orchestrator_uninstall_app_ids(package_id) {
let _ = orchestrator.remove(&app_id, preserve_data).await;
}
}
let containers_to_remove = get_containers_for_app(package_id).await?;
if containers_to_remove.is_empty() {
tracing::warn!("Uninstall {}: no containers found", package_id);
@@ -576,6 +609,7 @@ async fn do_package_start(to_start: &[String]) -> Result<()> {
if i > 0 {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
repair_before_package_start(name).await;
tracing::info!("Starting container: {}", name);
let out = tokio::process::Command::new("podman")
.args(["start", name])
@@ -585,6 +619,7 @@ async fn do_package_start(to_start: &[String]) -> Result<()> {
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
tracing::error!("Failed to start {}: {}", name, stderr);
cleanup_start_conflict(name, &stderr).await;
install_log(&format!("START FAIL: {}{}", name, stderr)).await;
errors.push(format!("{}: {}", name, stderr));
}
@@ -630,9 +665,86 @@ async fn do_package_start(to_start: &[String]) -> Result<()> {
errors.join("; ")
));
}
for name in to_start {
ensure_runtime_host_port_listener(name).await?;
}
Ok(())
}
async fn do_orchestrator_package_start(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
to_start: &[String],
) -> Result<()> {
let mut errors = Vec::new();
for (i, name) in to_start.iter().enumerate() {
if i > 0 {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
match orchestrator.start(name).await {
Ok(()) => wait_after_orchestrator_start(name).await,
Err(e) if is_unknown_app_id_error(&e) => {
do_package_start(&[name.clone()]).await?;
}
Err(e) => {
tracing::error!(container = %name, error = %e, "orchestrator start failed");
install_log(&format!("START FAIL: {}{:#}", name, e)).await;
errors.push(format!("{}: {:#}", name, e));
}
}
}
if errors.is_empty() {
Ok(())
} else {
Err(anyhow::anyhow!("Start failed: {}", errors.join("; ")))
}
}
async fn wait_after_orchestrator_start(name: &str) {
let delay = match name {
"archy-btcpay-db" => 5,
"archy-nbxplorer" => 8,
_ => 0,
};
if delay > 0 {
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
}
}
async fn do_orchestrator_package_stop(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
containers: &[String],
) -> Result<()> {
let mut errors = Vec::new();
for name in containers.iter().rev() {
match orchestrator.stop(name).await {
Ok(()) => {}
Err(e) if is_unknown_app_id_error(&e) => {
if let Err(e) = do_package_stop(&[name.clone()]).await {
errors.push(format!("{}: {:#}", name, e));
}
}
Err(e) => {
tracing::error!(container = %name, error = %e, "orchestrator stop failed");
errors.push(format!("{}: {:#}", name, e));
}
}
}
if errors.is_empty() {
Ok(())
} else {
Err(anyhow::anyhow!("Stop failed: {}", errors.join("; ")))
}
}
async fn do_orchestrator_package_restart(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
to_restart: &[String],
) -> Result<()> {
do_orchestrator_package_stop(orchestrator, to_restart).await?;
do_orchestrator_package_start(orchestrator, to_restart).await
}
/// Stop all containers with their per-container graceful-shutdown timeout.
async fn do_package_stop(containers: &[String]) -> Result<()> {
let mut errors = Vec::new();
@@ -649,6 +761,10 @@ async fn do_package_stop(containers: &[String]) -> Result<()> {
.context(format!("Failed to exec podman stop {}", name))?;
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
if is_missing_companion_ok(name, &stderr) {
tracing::debug!(container = %name, "companion already absent during stop");
continue;
}
tracing::error!("Failed to stop {}: {}", name, stderr);
errors.push(format!("{}: {}", name, stderr));
}
@@ -665,6 +781,7 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
let mut errors = Vec::new();
for name in containers {
tracing::info!("Restarting container: {}", name);
repair_before_package_start(name).await;
let out = tokio::process::Command::new("podman")
.args(["restart", "-t", stop_timeout_secs(name), name])
.output()
@@ -673,6 +790,10 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
if is_missing_companion_ok(name, &stderr) {
tracing::debug!(container = %name, "companion absent during restart; reconcile will recreate it");
continue;
}
tracing::warn!(
"podman restart {} failed: {}, trying stop+start",
name,
@@ -692,12 +813,18 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
let start_err = String::from_utf8_lossy(&start_out.stderr)
.trim()
.to_string();
cleanup_start_conflict(name, &start_err).await;
if is_missing_companion_ok(name, &start_err) {
tracing::debug!(container = %name, "companion absent during restart fallback; reconcile will recreate it");
continue;
}
tracing::error!("stop+start {} also failed: {}", name, start_err);
errors.push(format!("{}: {}", name, start_err));
} else {
tracing::info!("Restarted {} via stop+start fallback", name);
}
}
ensure_runtime_host_port_listener(name).await?;
}
if !errors.is_empty() {
return Err(anyhow::anyhow!("Restart failed: {}", errors.join("; ")));
@@ -705,6 +832,239 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
Ok(())
}
fn is_unknown_app_id_error(err: &anyhow::Error) -> bool {
err.chain()
.any(|cause| cause.to_string().contains("unknown app_id"))
}
async fn repair_before_package_start(container_name: &str) {
match container_name {
"btcpay-server" | "archy-nbxplorer" => repair_btcpay_dirs().await,
"indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" | "indeedhub-relay"
| "indeedhub-api" | "indeedhub-ffmpeg" | "indeedhub" => repair_indeedhub_network().await,
"grafana" => cleanup_stale_pasta_port("3000").await,
"gitea" => cleanup_gitea_stale_ports().await,
_ => {}
}
}
async fn ensure_runtime_host_port_listener(container_name: &str) -> Result<()> {
let Some(port) = runtime_required_host_port(container_name) else {
return Ok(());
};
if wait_for_runtime_host_port(port, 10).await {
return Ok(());
}
install_log(&format!(
"START REPAIR: {} — host port {} missing after start; restarting container",
container_name, port
))
.await;
let output = tokio::process::Command::new("podman")
.args(["restart", container_name])
.output()
.await
.context("failed to restart container after missing host port")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow::anyhow!(
"Container {} host port {} was not listening and restart failed: {}",
container_name,
port,
stderr.trim()
));
}
if wait_for_runtime_host_port(port, 60).await {
install_log(&format!(
"START REPAIR OK: {} — host port {} is listening after restart",
container_name, port
))
.await;
return Ok(());
}
Err(anyhow::anyhow!(
"Container {} is running but host port {} is not listening",
container_name,
port
))
}
fn runtime_required_host_port(container_name: &str) -> Option<u16> {
match container_name {
"grafana" => Some(3000),
"searxng" => Some(8888),
"uptime-kuma" => Some(3002),
"gitea" => Some(3001),
_ => None,
}
}
async fn wait_for_runtime_host_port(port: u16, timeout_secs: u64) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
loop {
if tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok()
{
return true;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
async fn repair_btcpay_dirs() {
let _ = tokio::process::Command::new("sudo")
.args([
"mkdir",
"-p",
"/var/lib/archipelago/btcpay/Main",
"/var/lib/archipelago/nbxplorer/Main",
])
.output()
.await;
for dir in [
"/var/lib/archipelago/btcpay",
"/var/lib/archipelago/nbxplorer",
] {
let _ = tokio::process::Command::new("sudo")
.args(["chown", "-R", "1000:1000", dir])
.output()
.await;
}
repair_btcpay_database_password().await;
}
async fn repair_btcpay_database_password() {
let Ok(db_pass) =
tokio::fs::read_to_string("/var/lib/archipelago/secrets/btcpay-db-password").await
else {
return;
};
let db_pass = db_pass.trim();
if db_pass.is_empty() {
return;
}
let _ = tokio::process::Command::new("podman")
.args(["start", "archy-btcpay-db"])
.output()
.await;
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
let escaped = db_pass.replace('\'', "''");
let sql = format!("ALTER USER btcpay WITH PASSWORD '{}';", escaped);
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"archy-btcpay-db",
"psql",
"-U",
"btcpay",
"-d",
"btcpay",
"-c",
&sql,
])
.output()
.await;
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"archy-btcpay-db",
"createdb",
"-U",
"btcpay",
"nbxplorer",
])
.output()
.await;
}
async fn repair_indeedhub_network() {
super::stacks::repair_indeedhub_network_aliases().await;
}
async fn cleanup_start_conflict(container_name: &str, stderr: &str) {
if !stderr.contains("address already in use") && !stderr.contains("pasta failed") {
return;
}
if container_name == "gitea" {
cleanup_gitea_stale_ports().await;
return;
}
if container_name != "grafana" {
return;
}
cleanup_stale_pasta_port("3000").await;
}
async fn cleanup_stale_pasta_port(port: &str) {
let kill_listener = format!(
"ss -ltnp 'sport = :{}' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | xargs -r kill 2>/dev/null || true",
port
);
let _ = tokio::process::Command::new("sh")
.args(["-c", &kill_listener])
.output()
.await;
let pattern = format!("pasta.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
let pattern = format!("rootlessport.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
async fn cleanup_gitea_stale_ports() {
for port in ["3001", "2222", "3000"] {
let kill_listener = format!(
"ss -ltnp 'sport = :{}' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | xargs -r kill 2>/dev/null || true",
port
);
let _ = tokio::process::Command::new("sh")
.args(["-c", &kill_listener])
.output()
.await;
let pattern = format!("pasta.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
let pattern = format!("rootlessport.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
pub(super) fn is_missing_companion_ok(name: &str, stderr: &str) -> bool {
matches!(
name,
"archy-bitcoin-ui" | "archy-lnd-ui" | "archy-electrs-ui"
) && stderr.contains("no container with name or ID")
}
/// Flip the primary package entry's state and return the pre-transition
/// state for revert on error. Mirrors `transitional::flip_to_transitional`
/// but lives here because the package path keys by `package_id` (which may
@@ -738,3 +1098,41 @@ async fn set_package_state(
}
}
}
pub(super) async fn reconcile_companions_for(package_id: &str) {
let app_ids = match package_id {
"bitcoin" | "bitcoin-core" => vec!["bitcoin-core".to_string(), "bitcoin-ui".to_string()],
"bitcoin-knots" => vec!["bitcoin-knots".to_string(), "bitcoin-ui".to_string()],
"lnd" => vec!["lnd".to_string(), "lnd-ui".to_string()],
"electrumx" | "electrs" | "mempool-electrs" => {
vec!["electrumx".to_string(), "electrs-ui".to_string()]
}
_ => return,
};
for (companion, err) in crate::container::companion::reconcile(&app_ids).await {
tracing::warn!(companion = %companion, error = %err, "companion reconcile failed");
}
}
pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec<String> {
match package_id {
"bitcoin" | "bitcoin-core" => vec!["bitcoin-core".into(), "bitcoin-ui".into()],
"bitcoin-knots" => vec!["bitcoin-knots".into(), "bitcoin-ui".into()],
"lnd" => vec!["lnd".into(), "lnd-ui".into()],
"electrumx" | "electrs" | "mempool-electrs" => {
vec!["electrumx".into(), "electrs-ui".into()]
}
"mempool" | "mempool-web" => vec![
"mempool-api".into(),
"archy-mempool-web".into(),
"archy-mempool-db".into(),
],
"btcpay-server" | "btcpayserver" | "btcpay" => vec![
"btcpay-server".into(),
"archy-nbxplorer".into(),
"archy-btcpay-db".into(),
],
"fedimint" => vec!["fedimint".into(), "fedimint-gateway".into()],
_ => vec![package_id.to_string()],
}
}
+89 -7
View File
@@ -8,7 +8,7 @@ use crate::data_model::InstallPhase;
use anyhow::{Context, Result};
use tracing::info;
use super::install::install_log;
use super::install::{install_log, patch_indeedhub_nostr_provider};
/// Adopt an existing container stack: start all named containers and return success.
/// Returns `Ok(Some(json))` if the primary container was found (adopted),
@@ -40,6 +40,8 @@ async fn adopt_stack_if_exists(
))
.await;
repair_stack_before_adopt(stack_name).await;
for container in all_containers {
if names.iter().any(|n| n == container) {
let _ = tokio::process::Command::new("podman")
@@ -55,6 +57,10 @@ async fn adopt_stack_if_exists(
.collect();
wait_for_stack_containers(stack_name, &existing, 60).await?;
if stack_name == "indeedhub" {
patch_indeedhub_nostr_provider().await;
}
install_log(&format!(
"INSTALL ADOPT OK: {} — started existing containers",
stack_name
@@ -67,6 +73,76 @@ async fn adopt_stack_if_exists(
})))
}
async fn repair_stack_before_adopt(stack_name: &str) {
match stack_name {
"btcpay" | "btcpay-server" => {
let _ = tokio::process::Command::new("sudo")
.args([
"mkdir",
"-p",
"/var/lib/archipelago/btcpay/Main",
"/var/lib/archipelago/nbxplorer/Main",
])
.output()
.await;
let user = std::env::var("USER").unwrap_or_else(|_| "archipelago".to_string());
for dir in [
"/var/lib/archipelago/btcpay",
"/var/lib/archipelago/nbxplorer",
] {
let _ = tokio::process::Command::new("sudo")
.args(["chown", "-R", &format!("{}:{}", user, user), dir])
.output()
.await;
}
}
"indeedhub" => repair_indeedhub_network_aliases().await,
_ => {}
}
}
pub(in crate::api::rpc::package) async fn repair_indeedhub_network_aliases() {
let _ = tokio::process::Command::new("podman")
.args(["network", "create", "indeedhub-net"])
.output()
.await;
for (container, alias) in [
("indeedhub-postgres", "postgres"),
("indeedhub-redis", "redis"),
("indeedhub-minio", "minio"),
("indeedhub-relay", "relay"),
("indeedhub-api", "api"),
("indeedhub", "indeedhub"),
] {
let exists = tokio::process::Command::new("podman")
.args(["container", "exists", container])
.status()
.await
.map(|s| s.success())
.unwrap_or(false);
if !exists {
continue;
}
let _ = tokio::process::Command::new("podman")
.args(["network", "disconnect", "-f", "indeedhub-net", container])
.output()
.await;
let _ = tokio::process::Command::new("podman")
.args([
"network",
"connect",
"--alias",
alias,
"indeedhub-net",
container,
])
.output()
.await;
}
}
async fn run_required_stack_command(
stack_name: &str,
label: &str,
@@ -480,6 +556,12 @@ impl RpcHandler {
/// Install BTCPay stack (postgres + nbxplorer + btcpay-server).
pub(super) async fn install_btcpay_stack(&self) -> Result<serde_json::Value> {
if let Some(orchestrated) =
install_stack_via_orchestrator(self, "btcpay-server", btcpay_stack_app_ids()).await?
{
return Ok(orchestrated);
}
if let Some(adopted) = adopt_stack_if_exists(
"btcpay-server",
"btcpay",
@@ -490,12 +572,6 @@ impl RpcHandler {
return Ok(adopted);
}
if let Some(orchestrated) =
install_stack_via_orchestrator(self, "btcpay-server", btcpay_stack_app_ids()).await?
{
return Ok(orchestrated);
}
// Dependency check: Bitcoin must be running
let deps = super::dependencies::detect_running_deps().await?;
super::dependencies::check_install_deps("btcpay-server", &deps)?;
@@ -1231,6 +1307,10 @@ impl RpcHandler {
"indeedhub-net",
"--restart",
"unless-stopped",
"--tmpfs",
"/run:rw,nosuid,nodev,size=16m",
"--tmpfs",
"/var/cache/nginx:rw,nosuid,nodev,size=32m",
"-p",
"7778:7777",
&format!("{}/indeedhub:1.0.0", registry),
@@ -1265,6 +1345,8 @@ impl RpcHandler {
.await;
self.clear_install_progress("indeedhub").await;
patch_indeedhub_nostr_provider().await;
install_log("INSTALL OK: indeedhub stack").await;
info!("IndeedHub stack installed");
Ok(serde_json::json!({
+22 -105
View File
@@ -71,88 +71,35 @@ pub async fn ensure_doctor_installed() {
Ok(_) => debug!("Secrets directory already at expected mode"),
Err(e) => warn!("Secrets dir tightening failed (non-fatal): {:#}", e),
}
// Podman self-heal MUST be the last bootstrap stage. If podman's
// runtime state is wedged, the orchestrator's first reconcile tick
// (which fires seconds after bootstrap returns) will hang or error
// on every container. Cleaning the runroot here gives the rest of
// the process a healthy podman to talk to.
// Podman probing MUST be the last bootstrap stage. We used to delete
// transient runroot state here when `podman info` failed, but live nodes
// can still have rootlessport/conmon processes holding that state. Removing
// it automatically makes failures worse: containers lose `.containerenv`,
// ports stay bound, and later starts fail. Report the fault instead; repair
// must be deliberate/operator-driven.
match heal_podman_state().await {
Ok(PodmanHealOutcome::Healthy) => debug!("podman runtime state healthy"),
Ok(PodmanHealOutcome::Cleaned) => warn!(
"podman runtime state was wedged at startup — cleaned runroot and re-probed (CRITICAL)"
Ok(PodmanHealOutcome::Unhealthy) => warn!(
"podman runtime state is unhealthy at startup — skipping automatic runroot cleanup"
),
Err(e) => warn!(
"podman self-heal failed (non-fatal, will retry next boot): {:#}",
e
),
Err(e) => warn!("podman self-heal failed (non-fatal, will retry next boot): {:#}", e),
}
}
#[derive(Debug, PartialEq, Eq)]
enum PodmanHealOutcome {
Healthy,
Cleaned,
Unhealthy,
}
/// Probe `podman info`. If it succeeds the daemon's runtime state is
/// fine and we return `Healthy` immediately. If it times out, fails to
/// spawn, or returns an "invalid internal status" / "database state"
/// error, the runtime state under `$XDG_RUNTIME_DIR/{containers,libpod}`
/// is likely wedged. We delete those two dirs and re-probe — podman
/// rebuilds runtime state from persistent storage under
/// `$HOME/.local/share/containers/storage/`.
///
/// `$XDG_RUNTIME_DIR/podman/` is **deliberately not touched**: that's
/// where systemd's socket-activated `podman.sock` listener lives. If we
/// removed it, every libpod HTTP call from the orchestrator would fail
/// with "connection refused" until `systemctl --user restart
/// podman.socket` ran — far worse than the wedge we'd be trying to fix.
///
/// Why this is safe at startup:
/// - We run BEFORE the orchestrator starts its reconcile loop, so no
/// archipelago code is currently calling podman.
/// - Persistent container metadata lives under
/// `~/.local/share/containers/`, which we never touch.
/// - `unless-stopped` containers and Quadlet-supervised services are
/// parented under user.slice, not archipelago.service, so they keep
/// running even while we clean podman's runtime view of them. After
/// the cleanup + re-probe podman re-discovers them.
///
/// What this does NOT cover:
/// - Storage corruption under `~/.local/share/containers/storage/`.
/// That requires a destructive `podman system reset`, which we will
/// never do automatically — operator must intervene.
/// - Networking corruption (netavark cache). Currently `podman info`
/// doesn't diagnose that; if cleanup doesn't fix it, the operator
/// will see the warning in the journal.
/// Subdirectories of `$XDG_RUNTIME_DIR` that hold podman's transient
/// state and are safe to remove when `podman info` is wedged. The
/// `podman/` subdir is **deliberately absent** — that's where systemd's
/// socket-activated `podman.sock` listener lives. Removing it would
/// silently break every libpod HTTP call from the orchestrator until
/// `systemctl --user restart podman.socket`. See
/// `heal_podman_state` docstring for the full rationale and the
/// `heal_podman_state_does_not_clean_socket_dir` regression test.
const HEAL_RUNTIME_SUBDIRS: &[&str] = &["containers", "libpod"];
async fn heal_podman_state() -> Result<PodmanHealOutcome> {
if probe_podman_ok().await {
return Ok(PodmanHealOutcome::Healthy);
}
let xdg = std::env::var("XDG_RUNTIME_DIR")
.context("XDG_RUNTIME_DIR not set; can't locate podman runtime state to clean")?;
for sub in HEAL_RUNTIME_SUBDIRS {
let path = PathBuf::from(&xdg).join(sub);
match fs::remove_dir_all(&path).await {
Ok(()) => debug!(path = %path.display(), "removed podman runtime state dir"),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => warn!(path = %path.display(), "remove failed: {}", e),
}
}
if probe_podman_ok().await {
Ok(PodmanHealOutcome::Cleaned)
} else {
Err(anyhow::anyhow!(
"podman info still failing after runtime cleanup; storage may be corrupt — operator must intervene"
))
}
Ok(PodmanHealOutcome::Unhealthy)
}
/// True iff `podman info` returns 0 within 5s. Any timeout, spawn
@@ -298,7 +245,6 @@ async fn run_runtime_assets() -> Result<bool> {
if changed {
let _ = host_sudo(&["systemctl", "daemon-reload"]).await;
let _ = host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await;
}
Ok(changed)
}
@@ -453,21 +399,14 @@ async fn run() -> Result<bool> {
let timer_changed = write_root_if_needed(DOCTOR_TIMER_PATH, DOCTOR_TIMER).await?;
changed = changed || service_changed || timer_changed;
// 3. Reload + enable. Only when we actually touched units, or when the
// timer isn't enabled yet (catches fresh upgrades of boxes that predate
// the doctor entirely).
let timer_enabled = is_timer_enabled().await;
if service_changed || timer_changed || !timer_enabled {
// 3. Reload if units changed. Do not enable/start the timer here: lifecycle
// qualification and explicit app operations need deterministic Podman
// ownership, and the doctor can race those flows. Operators can enable it
// separately when they want periodic host repair.
if service_changed || timer_changed {
if let Err(e) = host_sudo(&["systemctl", "daemon-reload"]).await {
warn!("daemon-reload failed: {:#}", e);
}
if let Err(e) =
host_sudo(&["systemctl", "enable", "--now", "archipelago-doctor.timer"]).await
{
warn!("enable archipelago-doctor.timer failed: {:#}", e);
} else if !timer_enabled {
info!("Enabled archipelago-doctor.timer");
}
}
Ok(changed)
@@ -508,15 +447,6 @@ async fn write_root_if_needed(path: &str, content: &str) -> Result<bool> {
Ok(true)
}
async fn is_timer_enabled() -> bool {
tokio::process::Command::new("systemctl")
.args(["is-enabled", "--quiet", "archipelago-doctor.timer"])
.status()
.await
.map(|s| s.success())
.unwrap_or(false)
}
/// Patch the nginx site config to add a `/api/app-catalog` proxy block if
/// it's missing. The original ISO shipped individual per-endpoint `location`
/// blocks and no catch-all `/api/`, so `/api/app-catalog` silently fell
@@ -615,22 +545,9 @@ async fn run_nginx() -> Result<bool> {
mod tests {
use super::*;
/// Regression gate for the 2026-05-01 bootstrap bug: heal_podman_state
/// was removing $XDG_RUNTIME_DIR/podman/ alongside containers/ and
/// libpod/, which silently broke the systemd-bound podman.sock and
/// every libpod HTTP call from the orchestrator. If anyone re-adds
/// "podman" to HEAL_RUNTIME_SUBDIRS this test fires before we ship.
#[test]
fn heal_podman_state_does_not_clean_socket_dir() {
assert!(
!HEAL_RUNTIME_SUBDIRS.contains(&"podman"),
"HEAL_RUNTIME_SUBDIRS must not include 'podman' — that dir holds \
systemd's podman.sock listener; removing it breaks every libpod \
HTTP call from the orchestrator. See bootstrap.rs commit bb421803."
);
// Sanity: the actually-runtime-state dirs are still in the list so
// we don't accidentally turn the heal into a no-op.
assert!(HEAL_RUNTIME_SUBDIRS.contains(&"containers"));
assert!(HEAL_RUNTIME_SUBDIRS.contains(&"libpod"));
fn podman_heal_outcome_no_longer_has_cleanup_variant() {
let outcome = PodmanHealOutcome::Unhealthy;
assert_ne!(outcome, PodmanHealOutcome::Healthy);
}
}
@@ -3,8 +3,8 @@
//!
//! Step 5 of the rust-orchestrator migration. Spawned once from `main.rs`
//! (Step 6) after the initial `adopt_existing()` pass. Every `interval` it
//! calls `ProdContainerOrchestrator::reconcile_all()`, which ensures every
//! loaded manifest has a running container, installing fresh ones as needed.
//! calls `ProdContainerOrchestrator::reconcile_existing()`, which repairs
//! containers that already exist without installing every catalog manifest.
//!
//! Per answered design Q3, `interval` defaults to 30 seconds.
//!
@@ -96,7 +96,7 @@ impl BootReconciler {
}
async fn tick(&self) {
let report = self.orchestrator.reconcile_all().await;
let report = self.orchestrator.reconcile_existing().await;
Self::log_report(&report);
if !self.companion_stage {
@@ -240,10 +240,11 @@ mod tests {
async fn orch_with_one_running_manifest(
rt: Arc<CountingRuntime>,
) -> Arc<ProdContainerOrchestrator> {
let orch = Arc::new(ProdContainerOrchestrator::with_runtime(
rt,
PathBuf::from("/nonexistent-for-tests"),
));
let mut orch =
ProdContainerOrchestrator::with_runtime(rt, PathBuf::from("/nonexistent-for-tests"));
let tmp = tempfile::tempdir().unwrap().keep();
orch.set_data_dir(tmp);
let orch = Arc::new(orch);
orch.insert_manifest_for_test(
pull_manifest("bitcoin-knots", "docker.io/bitcoin/knots:28"),
PathBuf::from("/tmp/bk"),
@@ -337,10 +338,13 @@ mod tests {
// will run, and the next pass will see a new state. We care about
// "loop keeps ticking even when the report has actions".
let rt = Arc::new(CountingRuntime::default());
let orch = Arc::new(ProdContainerOrchestrator::with_runtime(
let mut orch = ProdContainerOrchestrator::with_runtime(
rt.clone(),
PathBuf::from("/nonexistent-for-tests"),
));
);
let tmp = tempfile::tempdir().unwrap().keep();
orch.set_data_dir(tmp);
let orch = Arc::new(orch);
orch.insert_manifest_for_test(
pull_manifest("bitcoin-knots", "docker.io/bitcoin/knots:28"),
PathBuf::from("/tmp/bk"),
+41 -5
View File
@@ -50,6 +50,10 @@ pub struct CompanionSpec {
/// Bind mounts. Always read-only — companions don't write to
/// host paths.
pub bind_mounts: &'static [(&'static str, &'static str)],
/// Host-to-container TCP ports for non-host-network companions.
pub ports: &'static [(u16, u16)],
/// Whether the companion must share the host network namespace.
pub host_network: bool,
}
pub type PreStartHook = fn() -> futures_util::future::BoxFuture<'static, Result<()>>;
@@ -78,6 +82,8 @@ const BITCOIN_UI: &[CompanionSpec] = &[CompanionSpec {
"/var/lib/archipelago/bitcoin-ui/nginx.conf",
"/etc/nginx/conf.d/default.conf",
)],
ports: &[],
host_network: true,
}];
const LND_UI: &[CompanionSpec] = &[CompanionSpec {
@@ -90,6 +96,8 @@ const LND_UI: &[CompanionSpec] = &[CompanionSpec {
],
pre_start: None,
bind_mounts: &[],
ports: &[(18083, 80)],
host_network: false,
}];
const ELECTRS_UI: &[CompanionSpec] = &[CompanionSpec {
@@ -102,6 +110,8 @@ const ELECTRS_UI: &[CompanionSpec] = &[CompanionSpec {
],
pre_start: None,
bind_mounts: &[],
ports: &[],
host_network: true,
}];
fn render_bitcoin_ui() -> futures_util::future::BoxFuture<'static, Result<()>> {
@@ -183,9 +193,12 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
for dir in spec.build_dir_candidates {
let dockerfile = PathBuf::from(dir).join("Dockerfile");
if fs::try_exists(&dockerfile).await.unwrap_or(false) {
if image_exists(&local_image).await {
return Ok(local_image);
}
info!(companion = spec.name, "building locally from {dir}");
let out = Command::new("podman")
.args(["build", "--no-cache", "-t", &local_image, dir])
.args(["build", "-t", &local_image, dir])
.output()
.await
.context("spawn podman build")?;
@@ -220,15 +233,24 @@ async fn ensure_image_present(spec: &CompanionSpec) -> Result<String> {
Ok(registry_image)
}
async fn image_exists(image: &str) -> bool {
Command::new("podman")
.args(["image", "exists", image])
.status()
.await
.is_ok_and(|status| status.success())
}
fn build_unit(spec: &CompanionSpec, image: &str) -> QuadletUnit {
QuadletUnit {
name: spec.name.into(),
description: format!("Archipelago companion UI: {}", spec.name),
image: image.into(),
// Companions proxy to localhost — backend is on :5678, bitcoin
// RPC on :8332. Host network is the simplest way to reach them
// without per-app gateway plumbing.
network: NetworkMode::Host,
network: if spec.host_network {
NetworkMode::Host
} else {
NetworkMode::Bridge("bridge".into())
},
// Run as root inside the container so nginx can chown its
// worker dirs. Rootless podman maps this to a high host UID,
// so it is unprivileged on the host.
@@ -251,6 +273,11 @@ fn build_unit(spec: &CompanionSpec, image: &str) -> QuadletUnit {
read_only: true,
})
.collect(),
ports: spec
.ports
.iter()
.map(|(host, container)| (*host, *container, "tcp".into()))
.collect(),
extra_podman_args: vec![],
depends_on: vec![],
// Companions don't use the backend-manifest extension fields;
@@ -353,4 +380,13 @@ mod tests {
);
assert!(u.bind_mounts[0].read_only);
}
#[test]
fn lnd_ui_uses_port_mapping_not_host_port_80() {
let spec = &LND_UI[0];
let u = build_unit(spec, "localhost/lnd-ui:latest");
assert_eq!(u.name, "archy-lnd-ui");
assert!(matches!(u.network, NetworkMode::Bridge(ref n) if n == "bridge"));
assert_eq!(u.ports, vec![(18083, 80, "tcp".into())]);
}
}
@@ -63,10 +63,14 @@ impl DockerPackageScanner {
"indeedhub-build_ffmpeg-worker_1",
];
// First pass: collect UI containers
// First pass: collect running UI containers. Custom UI-backed apps must
// not advertise a launch URL unless their companion is actually alive.
let mut ui_containers: HashMap<String, String> = HashMap::new();
for container in &containers {
if container.name.ends_with("-ui") {
if !matches!(container.state, ContainerState::Running) {
continue;
}
// Map fedimint-ui -> fedimint, lnd-ui -> lnd (normalize archy- prefix for lookup)
let parent_app = container
.name
@@ -76,10 +80,10 @@ impl DockerPackageScanner {
.strip_prefix("archy-")
.unwrap_or(parent_app)
.to_string();
if !container.ports.is_empty() {
if let Some(ui_address) = extract_lan_address(&container.ports) {
ui_containers.insert(canonical_id, ui_address);
}
let ui_address = extract_lan_address(&container.ports)
.or_else(|| companion_lan_address(&canonical_id));
if let Some(ui_address) = ui_address {
ui_containers.insert(canonical_id, ui_address);
}
}
}
@@ -133,12 +137,6 @@ impl DockerPackageScanner {
// Apps with separate UI containers (e.g. archy-bitcoin-ui, archy-lnd-ui)
debug!("Using UI container for {}: {}", app_id, ui_address);
Some(ui_address.clone())
} else if app_id == "bitcoin-knots" {
Some("http://localhost:8334".to_string())
} else if app_id == "lnd" {
Some("http://localhost:8081".to_string())
} else if app_id == "electrumx" || app_id == "mempool-electrs" || app_id == "electrs" {
Some("http://localhost:50002".to_string())
} else {
// Dynamic: use actual port bindings from container, fall back to static map
extract_lan_address(&container.ports)
@@ -633,6 +631,14 @@ fn extract_lan_address(ports: &[String]) -> Option<String> {
None
}
fn companion_lan_address(app_id: &str) -> Option<String> {
match app_id {
"bitcoin" | "bitcoin-knots" | "bitcoin-core" => Some("http://localhost:8334".to_string()),
"electrumx" | "mempool-electrs" | "electrs" => Some("http://localhost:50002".to_string()),
_ => None,
}
}
fn convert_state(container_state: &ContainerState) -> (PackageState, ServiceStatus) {
match container_state {
ContainerState::Running => (PackageState::Running, ServiceStatus::Running),
+425
View File
@@ -0,0 +1,425 @@
//! lnd config bootstrap helper.
use anyhow::{Context, Result};
use base64::Engine;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use tokio::fs;
use crate::update::host_sudo;
pub const DEFAULT_DATA_DIR: &str = "/var/lib/archipelago/lnd";
pub const DEFAULT_CONF_PATH: &str = "/var/lib/archipelago/lnd/lnd.conf";
pub const WALLET_PASSWORD: &str = "hellohello";
#[derive(Debug, Clone)]
pub struct EnsurePaths {
pub data_dir: PathBuf,
pub conf_path: PathBuf,
}
impl Default for EnsurePaths {
fn default() -> Self {
Self {
data_dir: PathBuf::from(DEFAULT_DATA_DIR),
conf_path: PathBuf::from(DEFAULT_CONF_PATH),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EnsureOutcome {
Written,
Unchanged,
}
pub async fn ensure_config(paths: &EnsurePaths, rpc_pass: &str) -> Result<EnsureOutcome> {
fs::create_dir_all(&paths.data_dir)
.await
.with_context(|| format!("creating {}", paths.data_dir.display()))?;
if paths.conf_path.exists() {
let existing = fs::read_to_string(&paths.conf_path)
.await
.with_context(|| format!("reading {}", paths.conf_path.display()))?;
if has_required_lnd_flags(&existing) {
return Ok(EnsureOutcome::Unchanged);
}
}
let conf = format!(
"debuglevel=info\n\
maxpendingchannels=10\n\
alias=Archipelago Node\n\
color=#f7931a\n\
listen=0.0.0.0:9735\n\
rpclisten=0.0.0.0:10009\n\
restlisten=0.0.0.0:8080\n\
bitcoin.active=true\n\
bitcoin.mainnet=true\n\
bitcoin.node=bitcoind\n\
bitcoind.rpchost=bitcoin-knots:8332\n\
bitcoind.rpcuser=archipelago\n\
bitcoind.rpcpass={}\n\
bitcoind.rpcpolling=true\n\
bitcoind.estimatemode=ECONOMICAL\n",
rpc_pass
);
write_config_atomically(paths, &conf).await?;
Ok(EnsureOutcome::Written)
}
pub async fn ensure_wallet_initialized() -> Result<()> {
let admin_macaroon = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
let wallet_db = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/wallet.db";
if file_exists_as_root(wallet_db).await {
if file_exists_as_root(admin_macaroon).await && lnd_getinfo_ready(admin_macaroon).await {
return Ok(());
}
unlock_existing_wallet().await?;
wait_for_admin_macaroon(admin_macaroon).await?;
return Ok(());
}
init_wallet_via_rest().await?;
wait_for_admin_macaroon(admin_macaroon).await
}
async fn file_exists_as_root(path: &str) -> bool {
if std::path::Path::new(path).exists() {
return true;
}
tokio::process::Command::new("sudo")
.args(["test", "-f", path])
.status()
.await
.map(|status| status.success())
.unwrap_or(false)
}
async fn read_file_as_root(path: &str) -> Result<Vec<u8>> {
match fs::read(path).await {
Ok(bytes) => Ok(bytes),
Err(direct_err) => {
let out = tokio::process::Command::new("sudo")
.args(["cat", path])
.output()
.await
.with_context(|| format!("reading {path} via sudo"))?;
if out.status.success() {
Ok(out.stdout)
} else {
anyhow::bail!(
"reading {path} failed (direct: {direct_err}; sudo: {})",
String::from_utf8_lossy(&out.stderr).trim()
)
}
}
}
}
async fn unlock_existing_wallet() -> Result<()> {
let mut last_err = None;
for _ in 0..60 {
let mut cmd = tokio::process::Command::new("podman");
cmd.args(["exec", "-i", "lnd", "lncli", "unlock", "--stdin"]);
cmd.stdin(std::process::Stdio::piped());
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd.spawn().context("spawning lncli wallet unlock")?;
if let Some(mut stdin) = child.stdin.take() {
use tokio::io::AsyncWriteExt;
stdin
.write_all(format!("{}\n", WALLET_PASSWORD).as_bytes())
.await
.context("writing lncli password")?;
}
let out = child
.wait_with_output()
.await
.context("waiting for lncli")?;
if out.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&out.stderr);
let stdout = String::from_utf8_lossy(&out.stdout);
let msg = format!("{stderr}{stdout}");
if msg.contains("wallet already unlocked") || msg.contains("already unlocked") {
return Ok(());
}
last_err = Some(msg);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!(
"lncli wallet unlock failed: {}",
last_err.unwrap_or_else(|| "unknown error".to_string())
)
}
#[derive(Debug, Deserialize)]
struct GenSeedResponse {
cipher_seed_mnemonic: Vec<String>,
}
#[derive(Debug)]
enum UnlockerResponse<T> {
Value(T),
WalletAlreadyExists,
}
#[derive(Debug, Serialize)]
struct InitWalletRequest {
wallet_password: String,
cipher_seed_mnemonic: Vec<String>,
}
async fn init_wallet_via_rest() -> Result<()> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(20))
.danger_accept_invalid_certs(true)
.build()
.context("building LND REST client")?;
let seed: GenSeedResponse = match get_lnd_unlocker_json(&client, "/v1/genseed")
.await
.context("generating LND wallet seed")?
{
UnlockerResponse::Value(seed) => seed,
UnlockerResponse::WalletAlreadyExists => {
unlock_existing_wallet().await?;
return Ok(());
}
};
if seed.cipher_seed_mnemonic.is_empty() {
anyhow::bail!("LND genseed returned no seed words");
}
let wallet_password = base64::engine::general_purpose::STANDARD.encode(WALLET_PASSWORD);
let req = InitWalletRequest {
wallet_password,
cipher_seed_mnemonic: seed.cipher_seed_mnemonic,
};
match post_lnd_unlocker_json::<serde_json::Value>(
&client,
"/v1/initwallet",
serde_json::to_value(req)?,
)
.await
.context("initializing LND wallet")?
{
UnlockerResponse::Value(_) => {}
UnlockerResponse::WalletAlreadyExists => unlock_existing_wallet().await?,
}
Ok(())
}
async fn get_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
client: &reqwest::Client,
path: &str,
) -> Result<UnlockerResponse<T>> {
let url = format!("https://127.0.0.1:8080{path}");
let mut last_err = None;
for _ in 0..60 {
match client.get(&url).send().await {
Ok(resp) => match decode_lnd_unlocker_response(resp, path).await {
Ok(value) => return Ok(value),
Err(e) => last_err = Some(e.to_string()),
},
Err(e) => last_err = Some(e.to_string()),
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!(
"LND REST {path} unavailable: {}",
last_err.unwrap_or_else(|| "unknown error".to_string())
)
}
async fn post_lnd_unlocker_json<T: for<'de> Deserialize<'de>>(
client: &reqwest::Client,
path: &str,
body: serde_json::Value,
) -> Result<UnlockerResponse<T>> {
let url = format!("https://127.0.0.1:8080{path}");
let mut last_err = None;
for _ in 0..60 {
match client.post(&url).json(&body).send().await {
Ok(resp) => match decode_lnd_unlocker_response(resp, path).await {
Ok(value) => return Ok(value),
Err(e) => last_err = Some(e.to_string()),
},
Err(e) => last_err = Some(e.to_string()),
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!(
"LND REST {path} unavailable: {}",
last_err.unwrap_or_else(|| "unknown error".to_string())
)
}
async fn decode_lnd_unlocker_response<T: for<'de> Deserialize<'de>>(
resp: reqwest::Response,
path: &str,
) -> Result<UnlockerResponse<T>> {
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
if status.is_success() {
let value = serde_json::from_str(&text)
.with_context(|| format!("parsing LND REST response from {path}"))?;
return Ok(UnlockerResponse::Value(value));
}
if text.contains("wallet already exists") {
return Ok(UnlockerResponse::WalletAlreadyExists);
}
anyhow::bail!("LND REST {path} returned {status}: {text}")
}
async fn lnd_getinfo_ready(admin_macaroon: &str) -> bool {
let Ok(macaroon) = read_file_as_root(admin_macaroon).await else {
return false;
};
let Ok(client) = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.danger_accept_invalid_certs(true)
.build()
else {
return false;
};
client
.get("https://127.0.0.1:8080/v1/getinfo")
.header("Grpc-Metadata-macaroon", hex::encode(macaroon))
.send()
.await
.map(|resp| resp.status().is_success())
.unwrap_or(false)
}
async fn wait_for_admin_macaroon(admin_macaroon: &str) -> Result<()> {
for _ in 0..60 {
if file_exists_as_root(admin_macaroon).await {
return Ok(());
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
anyhow::bail!("LND admin macaroon not created after wallet init")
}
async fn write_config_atomically(paths: &EnsurePaths, conf: &str) -> Result<()> {
let tmp = paths.conf_path.with_extension("tmp");
match fs::write(&tmp, conf).await {
Ok(()) => {
fs::rename(&tmp, &paths.conf_path).await.with_context(|| {
format!(
"renaming {} -> {}",
tmp.display(),
paths.conf_path.display()
)
})?;
Ok(())
}
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
let script = format!(
"set -eu\ncat > '{}' <<'LNDCONF'\n{}LNDCONF\n",
shell_quote(&paths.conf_path.to_string_lossy()),
conf
);
let status = host_sudo(&["sh", "-lc", &script])
.await
.context("writing lnd.conf via sudo")?;
if !status.success() {
anyhow::bail!("writing lnd.conf via sudo exited with {status}");
}
Ok(())
}
Err(e) => Err(e).with_context(|| format!("writing tmp {}", tmp.display())),
}
}
fn shell_quote(s: &str) -> String {
s.replace('\'', "'\\''")
}
fn has_required_lnd_flags(conf: &str) -> bool {
[
"bitcoin.active=true",
"bitcoin.mainnet=true",
"bitcoin.node=bitcoind",
"bitcoind.rpchost=bitcoin-knots:8332",
]
.iter()
.all(|needle| conf.lines().any(|line| line.trim() == *needle))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn ensure_config_writes_required_bitcoin_network_flags() {
let tmp = tempfile::TempDir::new().unwrap();
let paths = EnsurePaths {
data_dir: tmp.path().join("lnd"),
conf_path: tmp.path().join("lnd/lnd.conf"),
};
let out = ensure_config(&paths, "secret").await.unwrap();
assert_eq!(out, EnsureOutcome::Written);
let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
assert!(conf.contains("bitcoin.active=true"));
assert!(conf.contains("bitcoin.mainnet=true"));
assert!(conf.contains("bitcoin.node=bitcoind"));
assert!(conf.contains("bitcoind.rpchost=bitcoin-knots:8332"));
assert!(conf.contains("bitcoind.rpcpass=secret"));
}
#[tokio::test]
async fn ensure_config_is_idempotent() {
let tmp = tempfile::TempDir::new().unwrap();
let paths = EnsurePaths {
data_dir: tmp.path().join("lnd"),
conf_path: tmp.path().join("lnd/lnd.conf"),
};
assert_eq!(
ensure_config(&paths, "first").await.unwrap(),
EnsureOutcome::Written
);
assert_eq!(
ensure_config(&paths, "second").await.unwrap(),
EnsureOutcome::Unchanged
);
let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
assert!(conf.contains("bitcoind.rpcpass=first"));
}
#[tokio::test]
async fn ensure_config_repairs_incomplete_existing_config() {
let tmp = tempfile::TempDir::new().unwrap();
let paths = EnsurePaths {
data_dir: tmp.path().join("lnd"),
conf_path: tmp.path().join("lnd/lnd.conf"),
};
fs::create_dir_all(&paths.data_dir).await.unwrap();
fs::write(&paths.conf_path, "debuglevel=info\n")
.await
.unwrap();
assert_eq!(
ensure_config(&paths, "repaired").await.unwrap(),
EnsureOutcome::Written
);
let conf = fs::read_to_string(&paths.conf_path).await.unwrap();
assert!(conf.contains("bitcoin.mainnet=true"));
assert!(conf.contains("bitcoind.rpcpass=repaired"));
}
#[test]
fn wallet_password_is_valid_for_lncli() {
assert!(WALLET_PASSWORD.len() > 8);
}
}
+1
View File
@@ -6,6 +6,7 @@ pub mod dev_orchestrator;
pub mod docker_packages;
pub mod filebrowser;
pub mod image_versions;
pub mod lnd;
pub mod prod_orchestrator;
pub mod quadlet;
pub mod registry;
File diff suppressed because it is too large Load Diff
+11 -9
View File
@@ -202,7 +202,11 @@ impl QuadletUnit {
);
}
for (host, container, proto) in &self.ports {
let p = if proto.is_empty() { "tcp" } else { proto.as_str() };
let p = if proto.is_empty() {
"tcp"
} else {
proto.as_str()
};
let _ = writeln!(s, "PublishPort={host}:{container}/{p}");
}
for env in &self.environment {
@@ -387,9 +391,7 @@ impl QuadletUnit {
/// `http://localhost:8175/`). Earlier we blindly prepended `http://` even
/// when one was already there, producing `http://http://...` HealthCmds
/// that pasted on .228 2026-05-02 and failed every probe.
fn translate_health_check(
hc: &archipelago_container::HealthCheck,
) -> Option<HealthSpec> {
fn translate_health_check(hc: &archipelago_container::HealthCheck) -> Option<HealthSpec> {
let cmd = match hc.check_type.as_str() {
"tcp" => {
let endpoint = hc.endpoint.as_deref()?;
@@ -703,10 +705,7 @@ mod tests {
"bash -c \"echo hi\""
);
// Embedded quotes must escape:
assert_eq!(
shell_join(&[r#"say "hi""#.into()]),
r#""say \"hi\"""#
);
assert_eq!(shell_join(&[r#"say "hi""#.into()]), r#""say \"hi\"""#);
}
#[test]
@@ -823,7 +822,10 @@ app:
assert!(!u.bind_mounts[0].read_only);
assert_eq!(u.entrypoint, Some(vec!["/usr/local/bin/bitcoind".into()]));
assert_eq!(u.command, vec!["-server=1", "-rpcbind=0.0.0.0"]);
assert!(u.add_hosts.iter().any(|(n, ip)| n == "host.archipelago" && ip == "10.89.0.1"));
assert!(u
.add_hosts
.iter()
.any(|(n, ip)| n == "host.archipelago" && ip == "10.89.0.1"));
assert_eq!(u.restart_policy, RestartPolicy::OnFailure);
}
+29
View File
@@ -412,6 +412,19 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
};
}
let names: Vec<String> = names
.into_iter()
.filter(|n| should_auto_start_stopped_container(n))
.collect();
if names.is_empty() {
return RecoveryReport {
total: 0,
recovered: 0,
failed: Vec::new(),
};
}
// Sort by startup tier: databases first, then core, then dependent services, then apps
let mut records: Vec<RunningContainerRecord> = names
.iter()
@@ -430,6 +443,13 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
recover_containers(&records).await
}
fn should_auto_start_stopped_container(name: &str) -> bool {
// Keep generic boot recovery narrow. The Rust manifest reconciler owns
// managed app stacks; starting every exited Podman container here races
// it and resurrects legacy/orphan helper containers.
matches!(name, "filebrowser" | "nostr-rs-relay")
}
/// Simple tier ordering for boot recovery (mirrors health_monitor tiers).
fn container_boot_tier(name: &str) -> u8 {
let id = name.strip_prefix("archy-").unwrap_or(name);
@@ -603,4 +623,13 @@ mod tests {
let result = check_for_crash(tmp.path()).await.unwrap();
assert!(result.is_none());
}
#[test]
fn generic_boot_recovery_skips_manifest_owned_and_legacy_stacks() {
assert!(should_auto_start_stopped_container("filebrowser"));
assert!(should_auto_start_stopped_container("nostr-rs-relay"));
assert!(!should_auto_start_stopped_container("bitcoin-knots"));
assert!(!should_auto_start_stopped_container("lnd"));
assert!(!should_auto_start_stopped_container("indeedhub-postgres"));
}
}
+107 -8
View File
@@ -4,7 +4,7 @@
// handles "created" state containers, resets dependent counters when deps recover,
// and sends WebSocket notifications to the UI on failure.
use crate::data_model::{Notification, NotificationLevel};
use crate::data_model::{Notification, NotificationLevel, PackageState};
use crate::state::StateManager;
use crate::webhooks::{self, WebhookEvent};
use serde::{Deserialize, Serialize};
@@ -67,14 +67,14 @@ fn container_dependencies(name: &str) -> &'static [&'static str] {
let id = name.strip_prefix("archy-").unwrap_or(name);
match id {
// Bitcoin-dependent chain
"lnd" => &["bitcoin-knots"],
"electrumx" | "mempool-electrs" | "electrs" => &["bitcoin-knots"],
"nbxplorer" => &["bitcoin-knots"],
"lnd" => &["bitcoin"],
"electrumx" | "mempool-electrs" | "electrs" => &["bitcoin"],
"nbxplorer" => &["bitcoin"],
"btcpay-server" => &["btcpay-db", "nbxplorer"],
"mempool-api" => &["mempool-db", "electrumx"],
"mempool-web" => &["mempool-api"],
"fedimint" => &["bitcoin-knots"],
"fedimint-gateway" => &["bitcoin-knots", "fedimint"],
"fedimint" => &["bitcoin"],
"fedimint-gateway" => &["bitcoin", "fedimint"],
// IndeedHub stack
"indeedhub-api" => &["indeedhub-postgres", "indeedhub-redis"],
@@ -88,7 +88,7 @@ fn container_dependencies(name: &str) -> &'static [&'static str] {
"penpot-frontend" => &["penpot-backend"],
// UI containers
"bitcoin-ui" => &["bitcoin-knots"],
"bitcoin-ui" => &["bitcoin"],
"lnd-ui" => &["lnd"],
"electrs-ui" => &["electrumx"],
@@ -103,6 +103,16 @@ fn deps_are_running(name: &str, containers: &[ContainerHealth]) -> bool {
return true;
}
for dep in deps {
if *dep == "bitcoin" {
let bitcoin_running = containers.iter().any(|c| {
let c_id = c.name.strip_prefix("archy-").unwrap_or(&c.name);
matches!(c_id, "bitcoin" | "bitcoin-knots" | "bitcoin-core") && c.state == "running"
});
if !bitcoin_running {
return false;
}
continue;
}
// Check both plain name and archy- prefixed name
let dep_running = containers.iter().any(|c| {
let c_id = c.name.strip_prefix("archy-").unwrap_or(&c.name);
@@ -115,6 +125,24 @@ fn deps_are_running(name: &str, containers: &[ContainerHealth]) -> bool {
true
}
fn conflicting_bitcoin_variant(name: &str) -> Option<&'static str> {
match name.strip_prefix("archy-").unwrap_or(name) {
"bitcoin-core" => Some("bitcoin-knots"),
"bitcoin-knots" | "bitcoin" => Some("bitcoin-core"),
_ => None,
}
}
fn has_running_bitcoin_conflict(name: &str, containers: &[ContainerHealth]) -> bool {
let Some(conflict) = conflicting_bitcoin_variant(name) else {
return false;
};
containers.iter().any(|c| {
let id = c.name.strip_prefix("archy-").unwrap_or(&c.name);
id == conflict && c.state == "running"
})
}
/// Track restart attempts per container with exponential backoff and stability reset.
struct RestartTracker {
attempts: HashMap<String, u32>,
@@ -539,6 +567,16 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
debug!("Skipping uninstalled container: {}", container.name);
continue;
}
if matches!(
pkg.state,
PackageState::Starting | PackageState::Stopping | PackageState::Restarting
) {
debug!(
"Skipping container during package lifecycle transition: {} ({:?})",
container.name, pkg.state
);
continue;
}
} else {
// Orphan: container exists in podman but archipelago has
// no package_data entry for it. Common after a variant
@@ -650,6 +688,14 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
continue;
}
if has_running_bitcoin_conflict(&container.name, &containers) {
debug!(
"Skipping auto-restart for {} because the other Bitcoin implementation is running",
container.name
);
continue;
}
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
if let Some(prev) = prev_tier {
if tier > prev {
@@ -916,7 +962,7 @@ mod tests {
#[test]
fn test_container_dependencies() {
assert!(container_dependencies("lnd").contains(&"bitcoin-knots"));
assert!(container_dependencies("lnd").contains(&"bitcoin"));
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-postgres"));
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-redis"));
assert!(container_dependencies("mempool-api").contains(&"mempool-db"));
@@ -957,6 +1003,59 @@ mod tests {
assert!(!deps_are_running("indeedhub-api", &partial));
}
#[test]
fn test_bitcoin_dependency_accepts_core_or_knots() {
let core = vec![ContainerHealth {
name: "bitcoin-core".into(),
app_id: "bitcoin-core".into(),
state: "running".into(),
healthy: true,
}];
assert!(deps_are_running("lnd", &core));
let knots = vec![ContainerHealth {
name: "bitcoin-knots".into(),
app_id: "bitcoin-knots".into(),
state: "running".into(),
healthy: true,
}];
assert!(deps_are_running("fedimint", &knots));
let stopped = vec![ContainerHealth {
name: "bitcoin-core".into(),
app_id: "bitcoin-core".into(),
state: "stopped".into(),
healthy: false,
}];
assert!(!deps_are_running("electrumx", &stopped));
}
#[test]
fn test_bitcoin_conflict_detection() {
let containers = vec![ContainerHealth {
name: "bitcoin-core".into(),
app_id: "bitcoin-core".into(),
state: "running".into(),
healthy: true,
}];
assert!(has_running_bitcoin_conflict("bitcoin-knots", &containers));
assert!(!has_running_bitcoin_conflict("bitcoin-core", &containers));
assert!(!has_running_bitcoin_conflict("lnd", &containers));
}
#[test]
fn test_bitcoin_conflict_ignores_stopped_sibling() {
let containers = vec![ContainerHealth {
name: "bitcoin-core".into(),
app_id: "bitcoin-core".into(),
state: "stopped".into(),
healthy: false,
}];
assert!(!has_running_bitcoin_conflict("bitcoin-knots", &containers));
}
#[test]
fn test_container_tier_core() {
assert_eq!(container_tier("bitcoin-knots"), StartupTier::CoreInfra);
+31 -36
View File
@@ -137,44 +137,39 @@ async fn main() -> Result<()> {
// Write PID marker early so we can detect crashes on next startup
crash_recovery::write_pid_marker(&config.data_dir).await?;
// Crash recovery runs in background so health endpoint is available immediately
{
let data_dir = config.data_dir.clone();
tokio::spawn(async move {
// Check if previous instance shut down cleanly
match crash_recovery::check_for_crash(&data_dir).await {
Ok(Some(containers)) => {
info!(
"🔧 Recovering {} containers from previous crash...",
containers.len()
);
let report = crash_recovery::recover_containers(&containers).await;
info!(
"🔧 Recovery complete: {}/{} containers restarted (failed: {:?})",
report.recovered, report.total, report.failed
);
}
Ok(None) => {}
Err(e) => {
tracing::warn!("Crash recovery check failed: {}", e);
}
}
// Start any stopped containers (handles clean reboot)
// Skips user-stopped containers, uses tier ordering
let boot_report = crash_recovery::start_stopped_containers(&data_dir).await;
if boot_report.total > 0 {
info!(
"🔄 Boot startup: {}/{} containers started (failed: {:?})",
boot_report.recovered, boot_report.total, boot_report.failed
);
}
// Signal to health monitor that boot recovery is done
crash_recovery::mark_recovery_complete();
});
// Run crash recovery before starting the manifest reconciler. Both paths
// mutate Podman; running them concurrently can corrupt transient runtime
// state and leave netavark/conmon unable to start containers.
match crash_recovery::check_for_crash(&config.data_dir).await {
Ok(Some(containers)) => {
info!(
"🔧 Recovering {} containers from previous crash...",
containers.len()
);
let report = crash_recovery::recover_containers(&containers).await;
info!(
"🔧 Recovery complete: {}/{} containers restarted (failed: {:?})",
report.recovered, report.total, report.failed
);
}
Ok(None) => {}
Err(e) => {
tracing::warn!("Crash recovery check failed: {}", e);
}
}
// Start any stopped containers (handles clean reboot). This remains
// synchronous for the same reason: no concurrent reconciler during Podman
// startup/recovery operations.
let boot_report = crash_recovery::start_stopped_containers(&config.data_dir).await;
if boot_report.total > 0 {
info!(
"🔄 Boot startup: {}/{} containers started (failed: {:?})",
boot_report.recovered, boot_report.total, boot_report.failed
);
}
crash_recovery::mark_recovery_complete();
// Construct the container orchestrator once. In prod mode we load the
// on-disk app manifests, do an initial adoption pass, and spawn the
// BootReconciler loop (Step 5/6 of the rust-orchestrator migration).
+2 -2
View File
@@ -13,8 +13,8 @@ use std::path::Path;
const RESERVED_PORTS: &[u16] = &[
80, 443, 81, // HTTP/HTTPS
8332, 8333, 8334, // Bitcoin RPC/P2P
9735, 10009, 8080, // LND P2P, gRPC, REST
8081, // LND UI (archy-lnd-ui)
9735, 10009, 8080, // LND P2P, gRPC, REST
18083, // LND UI (archy-lnd-ui)
4080, 8999, 50001, // Mempool stack
23000, // BTCPay
8173, 8174, 8175, // Fedimint
+45 -2
View File
@@ -313,6 +313,7 @@ impl Server {
let scanner = create_docker_scanner(&config).await?;
let state = state_manager.clone();
let identity_clone = identity.clone();
let data_dir = config.data_dir.clone();
let scan_kick = api_handler.rpc_handler().scan_kick();
let scan_tick = api_handler.rpc_handler().scan_tick();
@@ -334,6 +335,7 @@ impl Server {
&scanner,
&state,
identity_clone.as_ref(),
&data_dir,
&mut absence_tracker,
&mut transitional_since,
)
@@ -371,6 +373,7 @@ impl Server {
&scanner,
&state,
identity_clone.as_ref(),
&data_dir,
&mut absence_tracker,
&mut transitional_since,
)
@@ -865,8 +868,19 @@ fn merge_preserving_transitional(
existing: &crate::data_model::PackageDataEntry,
fresh: &crate::data_model::PackageDataEntry,
) -> crate::data_model::PackageDataEntry {
let state = match (&existing.state, &fresh.state) {
// Removing with a live running container is stale: uninstall either
// failed or Archipelago restarted before the spawned task could revert
// state. Let the scanner recover the UI immediately instead of
// keeping the app wedged in Removing for 20 minutes.
(crate::data_model::PackageState::Removing, crate::data_model::PackageState::Running) => {
fresh.state.clone()
}
_ => existing.state.clone(),
};
crate::data_model::PackageDataEntry {
state: existing.state.clone(),
state,
// install_progress and uninstall_stage are also owned by the
// initiating op (same reason as state) — keep them.
install_progress: existing.install_progress.clone(),
@@ -885,10 +899,18 @@ async fn scan_and_update_packages(
scanner: &DockerPackageScanner,
state: &StateManager,
identity: &NodeIdentity,
data_dir: &std::path::Path,
absence_tracker: &mut HashMap<String, u32>,
transitional_since: &mut HashMap<String, Instant>,
) -> Result<()> {
let packages = scanner.scan_containers().await?;
let mut packages = scanner.scan_containers().await?;
let user_stopped = crate::crash_recovery::load_user_stopped(data_dir).await;
for (id, pkg) in packages.iter_mut() {
if pkg.state == crate::data_model::PackageState::Exited && user_stopped.contains(id) {
pkg.state = crate::data_model::PackageState::Stopped;
pkg.exit_code = None;
}
}
let (current_data, _) = state.get_snapshot().await;
let tor_addr = docker_packages::read_tor_address("archipelago").await;
@@ -992,6 +1014,18 @@ async fn scan_and_update_packages(
// owner (spawn_task) is responsible for clearing state, not us.
if let Some(entry) = merged.get(&id) {
if is_transitional(&entry.state) {
let entered = *transitional_since.entry(id.clone()).or_insert(now);
if now.duration_since(entered) > TRANSITIONAL_STUCK_TIMEOUT {
warn!(
"Container {} stuck in {:?} and absent for >{}s; removing stale transitional state",
id,
entry.state,
TRANSITIONAL_STUCK_TIMEOUT.as_secs()
);
merged.remove(&id);
transitional_since.remove(&id);
changed = true;
}
absence_tracker.remove(&id);
continue;
}
@@ -1170,6 +1204,15 @@ mod merge_tests {
assert_eq!(merged.exit_code, Some(0));
}
#[test]
fn stale_removing_recovers_when_container_is_running() {
let existing = make_entry(PackageState::Removing, Some("unknown"));
let fresh = make_entry(PackageState::Running, Some("healthy"));
let merged = merge_preserving_transitional(&existing, &fresh);
assert_eq!(merged.state, PackageState::Running);
assert_eq!(merged.health.as_deref(), Some("healthy"));
}
#[test]
fn is_transitional_covers_all_variants() {
for s in [
+9 -3
View File
@@ -109,7 +109,7 @@ impl PodmanClient {
pub fn lan_address_for(name: &str) -> Option<String> {
let url = match name {
"bitcoin-knots" | "bitcoin-ui" => "http://localhost:8334",
"lnd" | "archy-lnd-ui" => "http://localhost:8081",
"lnd" | "archy-lnd-ui" => "http://localhost:18083",
"homeassistant" => "http://localhost:8123",
"archy-mempool-web" | "mempool" => "http://localhost:4080",
"btcpay-server" => "http://localhost:23000",
@@ -374,7 +374,10 @@ impl PodmanClient {
"env": env_map,
"entrypoint": manifest.app.container.entrypoint.clone(),
"command": manifest.app.container.custom_args.clone(),
"hostadd": ["host.containers.internal:host-gateway"],
"hostadd": [
"host.containers.internal:host-gateway",
"host.archipelago:10.89.0.1",
],
"devices": manifest.app.devices.iter().map(|d| {
serde_json::json!({"path": d})
}).collect::<Vec<_>>(),
@@ -392,7 +395,10 @@ impl PodmanClient {
if let Some(network) = custom_network {
body.as_object_mut()
.expect("container create body is a JSON object")
.insert("networks".to_string(), serde_json::json!({ network: {} }));
.insert(
"networks".to_string(),
serde_json::json!({ network: { "aliases": [name] } }),
);
}
let result = self
+123 -1
View File
@@ -104,7 +104,20 @@ impl ContainerRuntime for PodmanRuntime {
}
async fn list_containers(&self) -> Result<Vec<ContainerStatus>> {
self.client.list_containers().await
match self.client.list_containers().await {
Ok(containers) => Ok(containers),
Err(api_err) => {
let output = self.podman_cli(&["ps", "-a", "--format", "json"]).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(
api_err.context(format!("podman ps fallback failed: {}", stderr.trim()))
);
}
parse_podman_ps_json(&output.stdout)
.with_context(|| format!("podman API list failed: {api_err}"))
}
}
}
async fn image_exists(&self, image_ref: &str) -> Result<bool> {
@@ -147,6 +160,83 @@ impl ContainerRuntime for PodmanRuntime {
}
}
fn parse_podman_ps_json(stdout: &[u8]) -> Result<Vec<ContainerStatus>> {
let text = String::from_utf8_lossy(stdout);
if text.trim().is_empty() {
return Ok(Vec::new());
}
let containers: Vec<serde_json::Value> = serde_json::from_str(&text)?;
Ok(containers
.into_iter()
.map(|c| {
let name = c
.get("Names")
.and_then(|v| v.as_array())
.and_then(|a| a.first())
.and_then(|v| v.as_str())
.or_else(|| c.get("Names").and_then(|v| v.as_str()))
.unwrap_or("")
.to_string();
let status = c.get("Status").and_then(|v| v.as_str()).unwrap_or("");
let state = c.get("State").and_then(|v| v.as_str()).unwrap_or("unknown");
ContainerStatus {
id: c
.get("Id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
name: name.clone(),
state: ContainerState::from(state),
health: parse_health_from_status(status),
exit_code: c.get("ExitCode").and_then(|v| v.as_i64()).map(|c| c as i32),
started_at: c
.get("StartedAt")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
image: c
.get("Image")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
created: c
.get("Created")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
ports: parse_podman_ps_ports(c.get("Ports")),
lan_address: PodmanClient::lan_address_for(&name),
}
})
.collect())
}
fn parse_podman_ps_ports(ports: Option<&serde_json::Value>) -> Vec<String> {
ports
.and_then(|v| v.as_array())
.map(|ports| {
ports
.iter()
.filter_map(|port| {
let host = port.get("host_port").and_then(|v| v.as_u64())?;
let container = port.get("container_port").and_then(|v| v.as_u64())?;
let proto = port
.get("protocol")
.and_then(|v| v.as_str())
.unwrap_or("tcp");
Some(format!("0.0.0.0:{host}->{container}/{proto}"))
})
.collect()
})
.unwrap_or_default()
}
fn parse_health_from_status(status: &str) -> Option<String> {
let start = status.rfind('(')?;
let end = status.rfind(')')?;
(start < end).then(|| status[start + 1..end].to_string())
}
/// Build the argv for `podman build` from a BuildConfig.
///
/// Extracted so it can be unit-tested without actually invoking podman.
@@ -646,4 +736,36 @@ mod tests {
let args = build_args_for_podman(&c);
assert_eq!(args.last().unwrap(), "/final/context");
}
#[test]
fn parse_podman_ps_json_handles_cli_output() {
let stdout = br#"[
{
"Id": "abc123",
"Names": ["mempool"],
"Image": "docker.io/mempool/frontend:latest",
"State": "running",
"Status": "Up 2 minutes (healthy)",
"Created": "2026-05-03T00:00:00Z",
"StartedAt": "2026-05-03T00:01:00Z",
"ExitCode": 0,
"Ports": [
{
"host_port": 4080,
"container_port": 8080,
"protocol": "tcp"
}
]
}
]"#;
let containers = parse_podman_ps_json(stdout).unwrap();
assert_eq!(containers.len(), 1);
assert_eq!(containers[0].id, "abc123");
assert_eq!(containers[0].name, "mempool");
assert_eq!(containers[0].state, ContainerState::Running);
assert_eq!(containers[0].health.as_deref(), Some("healthy"));
assert_eq!(containers[0].exit_code, Some(0));
assert_eq!(containers[0].ports, vec!["0.0.0.0:4080->8080/tcp"]);
}
}