fix: overhaul container lifecycle — recovery, health, uninstall, UI state
Container recovery: - Health monitor: MAX_RESTART_ATTEMPTS 3→10, interval 60s→120s - Dependency-aware restarts: won't restart services before their deps - Reset dependent counters when a dependency recovers - Handle "created" state containers (were invisible to health monitor) - Added IndeedHub, mempool-api, mysql to tier system - Crash recovery: podman start timeout 30s→120s with retry - Podman client: socket timeout 5s→30s, added restart policy UI state representation: - Exit code 0 shows "stopped" (gray), not "crashed" (red) - Exit code 137 shows "killed (OOM)" - Non-zero exit shows "crashed" (red) - Added exit_code field to PackageDataEntry Install/uninstall fixes: - Install returns error when container doesn't start (was silent success) - Post-install hooks awaited instead of fire-and-forget tokio::spawn - Uninstall: graceful rm before force, volume prune, network cleanup - Uninstall returns error on partial failure (was 200 OK) Config consistency: - DB passwords read from /var/lib/archipelago/secrets/ (was hardcoded) - Bitcoin: added ZMQ ports 28332/28333 for LND block notifications - IndeedHub port 7777→8190 (was conflicting with strfry) - Marketplace versions: LND 0.17.4→0.18.4, Mempool 2.5.0→3.0.0 Performance: - Metrics collector interval 60s→300s (was duplicating health monitor) - Podman client: proper error propagation instead of unwrap_or_default Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
cdff10a8bc
commit
64b57dca7d
@@ -262,33 +262,47 @@ pub async fn recover_containers(containers: &[RunningContainerRecord]) -> Recove
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
|
||||
}
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
tokio::process::Command::new("podman")
|
||||
.args(["start", &record.name])
|
||||
.output(),
|
||||
)
|
||||
.await;
|
||||
// Try up to 2 attempts with increasing timeout (120s first, 180s retry)
|
||||
let mut started = false;
|
||||
for attempt in 0..2u32 {
|
||||
let timeout_secs = if attempt == 0 { 120 } else { 180 };
|
||||
if attempt > 0 {
|
||||
info!("Retrying container {} (attempt {})", record.name, attempt + 1);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
|
||||
}
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(timeout_secs),
|
||||
tokio::process::Command::new("podman")
|
||||
.args(["start", &record.name])
|
||||
.output(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Ok(output)) if output.status.success() => {
|
||||
info!("Successfully restarted container: {}", record.name);
|
||||
report.recovered += 1;
|
||||
}
|
||||
Ok(Ok(output)) => {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
warn!("Failed to restart container {}: {}", record.name, stderr.trim());
|
||||
report.failed.push(record.name.clone());
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!("Failed to execute podman start for {}: {}", record.name, e);
|
||||
report.failed.push(record.name.clone());
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("Timeout starting container {} (30s)", record.name);
|
||||
report.failed.push(record.name.clone());
|
||||
match result {
|
||||
Ok(Ok(output)) if output.status.success() => {
|
||||
info!("Successfully restarted container: {}", record.name);
|
||||
report.recovered += 1;
|
||||
started = true;
|
||||
break;
|
||||
}
|
||||
Ok(Ok(output)) => {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
warn!("Failed to restart container {} (attempt {}): {}",
|
||||
record.name, attempt + 1, stderr.trim());
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!("Failed to execute podman start for {} (attempt {}): {}",
|
||||
record.name, attempt + 1, e);
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("Timeout starting container {} ({}s, attempt {})",
|
||||
record.name, timeout_secs, attempt + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !started {
|
||||
report.failed.push(record.name.clone());
|
||||
}
|
||||
}
|
||||
|
||||
report
|
||||
@@ -313,7 +327,7 @@ fn is_process_running(pid: u32) -> bool {
|
||||
/// Skips containers that the user intentionally stopped via the UI.
|
||||
pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
|
||||
let output = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
std::time::Duration::from_secs(60),
|
||||
tokio::process::Command::new("podman")
|
||||
.args(["ps", "-a", "--filter", "status=exited", "--filter", "status=created", "--format", "{{.Names}}"])
|
||||
.output(),
|
||||
@@ -322,7 +336,7 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
warn!("Timeout listing stopped containers (30s)");
|
||||
warn!("Timeout listing stopped containers (60s)");
|
||||
return RecoveryReport { total: 0, recovered: 0, failed: Vec::new() };
|
||||
}
|
||||
};
|
||||
@@ -374,12 +388,21 @@ pub async fn start_stopped_containers(data_dir: &Path) -> RecoveryReport {
|
||||
fn container_boot_tier(name: &str) -> u8 {
|
||||
let id = name.strip_prefix("archy-").unwrap_or(name);
|
||||
match id {
|
||||
"btcpay-db" | "mempool-db" | "penpot-postgres" | "immich_postgres"
|
||||
| "immich_redis" | "penpot-valkey" => 0,
|
||||
// Tier 0: Databases and data stores
|
||||
"btcpay-db" | "mempool-db" | "mysql-mempool" | "penpot-postgres"
|
||||
| "immich_postgres" | "immich_redis" | "penpot-valkey"
|
||||
| "endurain-db" | "nextcloud-db"
|
||||
| "indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" => 0,
|
||||
// Tier 1: Core infrastructure
|
||||
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => 1,
|
||||
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer" => 2,
|
||||
// Tier 2: Dependent services
|
||||
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer"
|
||||
| "mempool-api" | "indeedhub-api" => 2,
|
||||
// Tier 4: Frontend/UI
|
||||
"mempool-web" | "bitcoin-ui" | "lnd-ui" | "electrs-ui"
|
||||
| "penpot-frontend" | "penpot-exporter" => 4,
|
||||
| "penpot-frontend" | "penpot-exporter"
|
||||
| "indeedhub" => 4,
|
||||
// Tier 3: Everything else
|
||||
_ => 3,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user