fix: overhaul container lifecycle — recovery, health, uninstall, UI state
Container recovery: - Health monitor: MAX_RESTART_ATTEMPTS 3→10, interval 60s→120s - Dependency-aware restarts: won't restart services before their deps - Reset dependent counters when a dependency recovers - Handle "created" state containers (were invisible to health monitor) - Added IndeedHub, mempool-api, mysql to tier system - Crash recovery: podman start timeout 30s→120s with retry - Podman client: socket timeout 5s→30s, added restart policy UI state representation: - Exit code 0 shows "stopped" (gray), not "crashed" (red) - Exit code 137 shows "killed (OOM)" - Non-zero exit shows "crashed" (red) - Added exit_code field to PackageDataEntry Install/uninstall fixes: - Install returns error when container doesn't start (was silent success) - Post-install hooks awaited instead of fire-and-forget tokio::spawn - Uninstall: graceful rm before force, volume prune, network cleanup - Uninstall returns error on partial failure (was 200 OK) Config consistency: - DB passwords read from /var/lib/archipelago/secrets/ (was hardcoded) - Bitcoin: added ZMQ ports 28332/28333 for LND block notifications - IndeedHub port 7777→8190 (was conflicting with strfry) - Marketplace versions: LND 0.17.4→0.18.4, Mempool 2.5.0→3.0.0 Performance: - Metrics collector interval 60s→300s (was duplicating health monitor) - Podman client: proper error propagation instead of unwrap_or_default Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dorian
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
cdff10a8bc
commit
64b57dca7d
@@ -1,6 +1,7 @@
|
||||
// Container Health Monitor
|
||||
// Checks container health every 60s, auto-restarts unhealthy containers (max 3 times)
|
||||
// with exponential backoff (10s, 30s, 90s), dependency-aware startup ordering,
|
||||
// Checks container health every 120s, auto-restarts unhealthy containers (max 10 times)
|
||||
// with exponential backoff (10s..120s), dependency-aware restart ordering (deps first),
|
||||
// handles "created" state containers, resets dependent counters when deps recover,
|
||||
// and sends WebSocket notifications to the UI on failure.
|
||||
|
||||
use crate::data_model::{Notification, NotificationLevel};
|
||||
@@ -13,10 +14,10 @@ use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const MAX_RESTART_ATTEMPTS: u32 = 3;
|
||||
const CHECK_INTERVAL_SECS: u64 = 60;
|
||||
/// Backoff delays per attempt: 10s, 30s, 90s
|
||||
const BACKOFF_DELAYS_SECS: [u64; 3] = [10, 30, 90];
|
||||
const MAX_RESTART_ATTEMPTS: u32 = 10;
|
||||
const CHECK_INTERVAL_SECS: u64 = 120;
|
||||
/// Backoff delays per attempt — escalating from 10s to 120s
|
||||
const BACKOFF_DELAYS_SECS: [u64; 10] = [10, 15, 20, 30, 30, 45, 60, 60, 90, 120];
|
||||
/// Reset restart counter after 1 hour of stability
|
||||
const STABILITY_RESET_SECS: u64 = 3600;
|
||||
|
||||
@@ -39,25 +40,83 @@ enum StartupTier {
|
||||
fn container_tier(name: &str) -> StartupTier {
|
||||
let id = name.strip_prefix("archy-").unwrap_or(name);
|
||||
match id {
|
||||
// Tier 0: Databases
|
||||
"btcpay-db" | "mempool-db" | "penpot-postgres" | "immich_postgres"
|
||||
| "immich_redis" | "penpot-valkey" | "endurain-db" | "nextcloud-db" => StartupTier::Database,
|
||||
// Tier 0: Databases and data stores
|
||||
"btcpay-db" | "mempool-db" | "mysql-mempool" | "penpot-postgres"
|
||||
| "immich_postgres" | "immich_redis" | "penpot-valkey"
|
||||
| "endurain-db" | "nextcloud-db"
|
||||
| "indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" => StartupTier::Database,
|
||||
|
||||
// Tier 1: Core infrastructure
|
||||
"bitcoin-knots" | "bitcoin-core" | "bitcoin" => StartupTier::CoreInfra,
|
||||
|
||||
// Tier 2: Dependent services
|
||||
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer" => StartupTier::DependentService,
|
||||
// Tier 2: Dependent services (need databases or bitcoin)
|
||||
"lnd" | "electrumx" | "mempool-electrs" | "electrs" | "nbxplorer"
|
||||
| "mempool-api" | "indeedhub-api" => StartupTier::DependentService,
|
||||
|
||||
// Tier 4: Frontend/UI
|
||||
"mempool-web" | "bitcoin-ui" | "lnd-ui" | "electrs-ui"
|
||||
| "penpot-frontend" | "penpot-exporter" => StartupTier::Frontend,
|
||||
| "penpot-frontend" | "penpot-exporter"
|
||||
| "indeedhub" => StartupTier::Frontend,
|
||||
|
||||
// Tier 3: Everything else
|
||||
// Tier 3: Application layer (everything else)
|
||||
_ => StartupTier::Application,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map containers to their required dependencies.
|
||||
/// When a dependent fails, check and restart its dependencies first.
|
||||
fn container_dependencies(name: &str) -> &'static [&'static str] {
|
||||
let id = name.strip_prefix("archy-").unwrap_or(name);
|
||||
match id {
|
||||
// Bitcoin-dependent chain
|
||||
"lnd" => &["bitcoin-knots"],
|
||||
"electrumx" | "mempool-electrs" | "electrs" => &["bitcoin-knots"],
|
||||
"nbxplorer" => &["bitcoin-knots"],
|
||||
"btcpay-server" => &["btcpay-db", "nbxplorer"],
|
||||
"mempool-api" => &["mempool-db", "electrumx"],
|
||||
"mempool-web" => &["mempool-api"],
|
||||
"fedimint" => &["bitcoin-knots"],
|
||||
"fedimint-gateway" => &["lnd"],
|
||||
|
||||
// IndeedHub stack
|
||||
"indeedhub-api" => &["indeedhub-postgres", "indeedhub-redis"],
|
||||
"indeedhub" => &["indeedhub-api"],
|
||||
"indeedhub-relay" => &["indeedhub-postgres"],
|
||||
"indeedhub-ffmpeg" => &["indeedhub-api"],
|
||||
|
||||
// Multi-container stacks
|
||||
"immich_server" => &["immich_postgres", "immich_redis"],
|
||||
"penpot-backend" => &["penpot-postgres", "penpot-valkey"],
|
||||
"penpot-frontend" => &["penpot-backend"],
|
||||
|
||||
// UI containers
|
||||
"bitcoin-ui" => &["bitcoin-knots"],
|
||||
"lnd-ui" => &["lnd"],
|
||||
"electrs-ui" => &["electrumx"],
|
||||
|
||||
_ => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if all of a container's dependencies are currently running.
|
||||
fn deps_are_running(name: &str, containers: &[ContainerHealth]) -> bool {
|
||||
let deps = container_dependencies(name);
|
||||
if deps.is_empty() {
|
||||
return true;
|
||||
}
|
||||
for dep in deps {
|
||||
// Check both plain name and archy- prefixed name
|
||||
let dep_running = containers.iter().any(|c| {
|
||||
let c_id = c.name.strip_prefix("archy-").unwrap_or(&c.name);
|
||||
(c_id == *dep || c.name == *dep) && c.state == "running"
|
||||
});
|
||||
if !dep_running {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Track restart attempts per container with exponential backoff and stability reset.
|
||||
struct RestartTracker {
|
||||
attempts: HashMap<String, u32>,
|
||||
@@ -372,7 +431,7 @@ async fn check_containers() -> Vec<ContainerHealth> {
|
||||
async fn restart_container(name: &str) -> bool {
|
||||
info!("Auto-restarting unhealthy container: {}", name);
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
std::time::Duration::from_secs(120),
|
||||
tokio::process::Command::new("podman")
|
||||
.args(["start", name])
|
||||
.output(),
|
||||
@@ -394,7 +453,7 @@ async fn restart_container(name: &str) -> bool {
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("Timeout starting container {} (30s)", name);
|
||||
warn!("Timeout starting container {} (120s)", name);
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -466,13 +525,33 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
if container.healthy {
|
||||
if tracker.attempt_count(&container.name) > 0 {
|
||||
info!("Container {} is healthy again after restart", container.name);
|
||||
// Reset attempt counters for containers that depend on this one,
|
||||
// since their previous failures may have been caused by this
|
||||
// dependency being down
|
||||
let recovered_id = container.name.strip_prefix("archy-")
|
||||
.unwrap_or(&container.name).to_string();
|
||||
for other in &containers {
|
||||
let deps = container_dependencies(&other.name);
|
||||
if deps.iter().any(|d| *d == recovered_id || *d == container.name) {
|
||||
if tracker.attempt_count(&other.name) > 0 {
|
||||
info!("Resetting restart counter for {} (dependency {} recovered)",
|
||||
other.name, container.name);
|
||||
tracker.clear(&other.name);
|
||||
restart_history.clear(&other.name);
|
||||
history_dirty = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
tracker.clear(&container.name);
|
||||
restart_history.clear(&container.name);
|
||||
history_dirty = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if container.state == "exited" || container.state == "stopped" {
|
||||
// Handle exited, stopped, AND created state containers
|
||||
if container.state == "exited" || container.state == "stopped"
|
||||
|| container.state == "created"
|
||||
{
|
||||
// Skip user-stopped containers
|
||||
if user_stopped.contains(&container.name) {
|
||||
debug!("Skipping user-stopped container: {}", container.name);
|
||||
@@ -509,6 +588,13 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip if dependencies aren't running — they need to start first
|
||||
if !deps_are_running(&container.name, &containers) {
|
||||
let deps = container_dependencies(&container.name);
|
||||
debug!("Container {} waiting for dependencies {:?}", container.name, deps);
|
||||
continue;
|
||||
}
|
||||
|
||||
// When transitioning to a higher tier, wait briefly for previous tier to stabilize
|
||||
if let Some(prev) = prev_tier {
|
||||
if tier > prev {
|
||||
@@ -695,13 +781,13 @@ mod tests {
|
||||
#[test]
|
||||
fn test_max_restart_attempts_constant() {
|
||||
assert!(MAX_RESTART_ATTEMPTS >= 1);
|
||||
assert!(MAX_RESTART_ATTEMPTS <= 10);
|
||||
assert_eq!(MAX_RESTART_ATTEMPTS, 3);
|
||||
assert!(MAX_RESTART_ATTEMPTS <= 20);
|
||||
assert_eq!(MAX_RESTART_ATTEMPTS, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_interval_constant() {
|
||||
assert_eq!(CHECK_INTERVAL_SECS, 60);
|
||||
assert_eq!(CHECK_INTERVAL_SECS, 120);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -740,6 +826,44 @@ mod tests {
|
||||
assert_eq!(container_tier("archy-btcpay-db"), StartupTier::Database);
|
||||
assert_eq!(container_tier("immich_postgres"), StartupTier::Database);
|
||||
assert_eq!(container_tier("penpot-valkey"), StartupTier::Database);
|
||||
assert_eq!(container_tier("indeedhub-postgres"), StartupTier::Database);
|
||||
assert_eq!(container_tier("indeedhub-redis"), StartupTier::Database);
|
||||
assert_eq!(container_tier("indeedhub-minio"), StartupTier::Database);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_container_tier_indeedhub_api() {
|
||||
assert_eq!(container_tier("indeedhub-api"), StartupTier::DependentService);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_container_tier_mempool_api() {
|
||||
assert_eq!(container_tier("mempool-api"), StartupTier::DependentService);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_container_dependencies() {
|
||||
assert!(container_dependencies("lnd").contains(&"bitcoin-knots"));
|
||||
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-postgres"));
|
||||
assert!(container_dependencies("indeedhub-api").contains(&"indeedhub-redis"));
|
||||
assert!(container_dependencies("mempool-api").contains(&"mempool-db"));
|
||||
assert!(container_dependencies("mempool-api").contains(&"electrumx"));
|
||||
assert!(container_dependencies("nextcloud").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deps_are_running() {
|
||||
let containers = vec![
|
||||
ContainerHealth { name: "indeedhub-postgres".into(), app_id: "indeedhub-postgres".into(), state: "running".into(), healthy: true },
|
||||
ContainerHealth { name: "indeedhub-redis".into(), app_id: "indeedhub-redis".into(), state: "running".into(), healthy: true },
|
||||
ContainerHealth { name: "indeedhub-api".into(), app_id: "indeedhub-api".into(), state: "exited".into(), healthy: false },
|
||||
];
|
||||
assert!(deps_are_running("indeedhub-api", &containers));
|
||||
// Missing postgres
|
||||
let partial = vec![
|
||||
ContainerHealth { name: "indeedhub-redis".into(), app_id: "indeedhub-redis".into(), state: "running".into(), healthy: true },
|
||||
];
|
||||
assert!(!deps_are_running("indeedhub-api", &partial));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user