fix: overhaul container lifecycle — recovery, health, uninstall, UI state

Container recovery:
- Health monitor: MAX_RESTART_ATTEMPTS 3→10, interval 60s→120s
- Dependency-aware restarts: won't restart services before their deps
- Reset dependent counters when a dependency recovers
- Handle "created" state containers (were invisible to health monitor)
- Added IndeedHub, mempool-api, mysql to tier system
- Crash recovery: podman start timeout 30s→120s with retry
- Podman client: socket timeout 5s→30s, added restart policy

UI state representation:
- Exit code 0 shows "stopped" (gray), not "crashed" (red)
- Exit code 137 shows "killed (OOM)"
- Non-zero exit shows "crashed" (red)
- Added exit_code field to PackageDataEntry

Install/uninstall fixes:
- Install returns error when container doesn't start (was silent success)
- Post-install hooks awaited instead of fire-and-forget tokio::spawn
- Uninstall: graceful rm before force, volume prune, network cleanup
- Uninstall returns error on partial failure (was 200 OK)

Config consistency:
- DB passwords read from /var/lib/archipelago/secrets/ (was hardcoded)
- Bitcoin: added ZMQ ports 28332/28333 for LND block notifications
- IndeedHub port 7777→8190 (was conflicting with strfry)
- Marketplace versions: LND 0.17.4→0.18.4, Mempool 2.5.0→3.0.0

Performance:
- Metrics collector interval 60s→300s (was duplicating health monitor)
- Podman client: proper error propagation instead of unwrap_or_default

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-31 07:03:57 +01:00
co-authored by Claude Opus 4.6
parent cdff10a8bc
commit 64b57dca7d
65 changed files with 3950 additions and 298 deletions
+53 -30
View File
@@ -262,33 +262,47 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
}
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
tokio::process::Command::new("podman")
.args(["start", &record.name])
.output(),
)
.await;
// Try up to 2 attempts with increasing timeout (120s first, 180s retry)
let mut started = false;
for attempt in 0..2u32 {
let timeout_secs = if attempt == 0 { 120 } else { 180 };
if attempt > 0 {
info!("Retrying container {} (attempt {})", record.name, attempt + 1);
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
}
let result = tokio::time::timeout(
std::time::Duration::from_secs(timeout_secs),
tokio::process::Command::new("podman")
.args(["start", &record.name])
.output(),
)
.await;
match result {
Ok(Ok(output)) if output.status.success() => {
info!("Successfully restarted container: {}", record.name);
report.recovered += 1;
}
Ok(Ok(output)) => {
let stderr = String::from_utf8_lossy(&output.stderr);
warn!("Failed to restart container {}: {}", record.name, stderr.trim());
report.failed.push(record.name.clone());
}
Ok(Err(e)) => {
warn!("Failed to execute podman start for {}: {}", record.name, e);
report.failed.push(record.name.clone());
}
Err(_) => {
warn!("Timeout starting container {} (30s)", record.name);
report.failed.push(record.name.clone());
match result {
Ok(Ok(output)) if output.status.success() => {
info!("Successfully restarted container: {}", record.name);
report.recovered += 1;
started = true;
break;
}
Ok(Ok(output)) => {
let stderr = String::from_utf8_lossy(&output.stderr);
warn!("Failed to restart container {} (attempt {}): {}",
record.name, attempt + 1, stderr.trim());
}
Ok(Err(e)) => {
warn!("Failed to execute podman start for {} (attempt {}): {}",
record.name, attempt + 1, e);
}
Err(_) => {
warn!("Timeout starting container {} ({}s, attempt {})",
record.name, timeout_secs, attempt + 1);
}
}
}
if !started {
report.failed.push(record.name.clone());
}
}
report
@@ -313,7 +327,7 @@ fn is_process_running(pid: u32) -> bool {
/// Skips containers that the user intentionally stopped via the UI.
pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
let output = match tokio::time::timeout(
std::time::Duration::from_secs(30),
std::time::Duration::from_secs(60),
tokio::process::Command::new("podman")
.args(["ps", "-a", "--filter", "status=exited", "--filter", "status=created", "--format", "{{.Names}}"])
.output(),
@@ -322,7 +336,7 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
{
Ok(result) => result,
Err(_) => {
warn!("Timeout listing stopped containers (30s)");
warn!("Timeout listing stopped containers (60s)");
return RecoveryReport { total: 0, recovered: 0, failed: Vec::new() };
}
};
@@ -374,12 +388,21 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
fn container_boot_tier(name: &str) -> u8 {
let id = name.strip_prefix("archy-").unwrap_or(name);
match id {
"btcpay-db" | "mempool-db" | "penpot-postgres" | "immich_postgres"
| "immich_redis" | "penpot-valkey" => 0,
// Tier 0: Databases and data stores
"btcpay-db" | "mempool-db" | "mysql-mempool" | "penpot-postgres"
| "immich_postgres" | "immich_redis" | "penpot-valkey"
| "endurain-db" | "nextcloud-db"
| "indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" => 0,
// Tier 1: Core infrastructure
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => 1,
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer" => 2,
// Tier 2: Dependent services
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer"
| "mempool-api" | "indeedhub-api" => 2,
// Tier 4: Frontend/UI
"mempool-web" | "bitcoin-ui" | "lnd-ui" | "electrs-ui"
| "penpot-frontend" | "penpot-exporter" => 4,
| "penpot-frontend" | "penpot-exporter"
| "indeedhub" => 4,
// Tier 3: Everything else
_ => 3,
}
}