feat: container orchestration, branding overhaul, onboarding logging
Container orchestration: - Health monitor with crash recovery and auto-restart - Doctor service (periodic health checks via systemd timer) - Reconcile service (desired-state convergence) - Stack-aware install/uninstall with dependency tracking Branding: - Custom GRUB background (designer artwork, 1024x768) - ISOLINUX boot menu: centered, orange accents, clean labels - Terminal banners: adaptive width, basic ANSI colors, fits 80-col - Removed auto-generated splash scripts (designer provides assets) - GRUB theme: lowercase branding Frontend: - 401 handler clears localStorage immediately (prevents cascade) Backend: - Onboarding/auth logging ([onboarding] tag in journalctl) - Cookie Secure flag logging for debugging HTTP/HTTPS issues ISO fixes: - Install log saved before unmount (was silently failing) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
9d38989048
commit
9db55b0b34
@@ -6,8 +6,9 @@
|
||||
use crate::data_model::{Notification, NotificationLevel};
|
||||
use crate::state::StateManager;
|
||||
use crate::webhooks::{self, WebhookEvent};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -177,6 +178,69 @@ impl MemoryTracker {
|
||||
|
||||
}
|
||||
|
||||
// ── Persistent restart tracking ────────────────────────────────────────
|
||||
// Survives process restarts so a container can't loop infinitely by
|
||||
// crashing 3 times → triggering process restart → resetting counter → repeat.
|
||||
|
||||
const RESTART_HISTORY_FILE: &str = "restart-tracker.json";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
struct RestartHistory {
|
||||
containers: HashMap<String, ContainerRestartRecord>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
struct ContainerRestartRecord {
|
||||
attempts: u32,
|
||||
last_failure_epoch: i64,
|
||||
}
|
||||
|
||||
impl RestartHistory {
|
||||
async fn load(data_dir: &Path) -> Self {
|
||||
let path = data_dir.join(RESTART_HISTORY_FILE);
|
||||
match tokio::fs::read_to_string(&path).await {
|
||||
Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
|
||||
Err(_) => Self::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn save(&self, data_dir: &Path) {
|
||||
let path = data_dir.join(RESTART_HISTORY_FILE);
|
||||
if let Ok(json) = serde_json::to_string(self) {
|
||||
let _ = tokio::fs::write(&path, json).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed the in-memory RestartTracker from persisted history.
|
||||
fn seed_tracker(&self, tracker: &mut RestartTracker) {
|
||||
let now_epoch = chrono::Utc::now().timestamp();
|
||||
for (name, record) in &self.containers {
|
||||
// Only seed if last failure was within the stability window
|
||||
let secs_since_failure = now_epoch - record.last_failure_epoch;
|
||||
if secs_since_failure < STABILITY_RESET_SECS as i64 && record.attempts > 0 {
|
||||
tracker.attempts.insert(name.clone(), record.attempts);
|
||||
info!(
|
||||
"Restored restart counter for {}: {} attempts ({}s ago)",
|
||||
name, record.attempts, secs_since_failure
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_attempt(&mut self, name: &str) {
|
||||
let entry = self.containers.entry(name.to_string()).or_insert(ContainerRestartRecord {
|
||||
attempts: 0,
|
||||
last_failure_epoch: 0,
|
||||
});
|
||||
entry.attempts += 1;
|
||||
entry.last_failure_epoch = chrono::Utc::now().timestamp();
|
||||
}
|
||||
|
||||
fn clear(&mut self, name: &str) {
|
||||
self.containers.remove(name);
|
||||
}
|
||||
}
|
||||
|
||||
/// Query container memory stats from podman.
|
||||
async fn check_container_memory() -> HashMap<String, u64> {
|
||||
let output = match tokio::time::timeout(
|
||||
@@ -373,6 +437,11 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
let mut mem_check_counter: u32 = 0;
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(CHECK_INTERVAL_SECS));
|
||||
|
||||
// Load persistent restart history and seed the in-memory tracker
|
||||
let mut restart_history = RestartHistory::load(&data_dir).await;
|
||||
restart_history.seed_tracker(&mut tracker);
|
||||
let mut history_dirty = false;
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
mem_check_counter += 1;
|
||||
@@ -406,6 +475,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
if tracker.attempt_count(&container.name) > 0 {
|
||||
info!("Container {} is healthy again after restart", container.name);
|
||||
tracker.clear(&container.name);
|
||||
restart_history.clear(&container.name);
|
||||
history_dirty = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -430,6 +501,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
if tracker.should_reset_failed(&container.name) {
|
||||
info!("Resetting restart counter for {} after {}s stability window", container.name, STABILITY_RESET_SECS);
|
||||
tracker.clear(&container.name);
|
||||
restart_history.clear(&container.name);
|
||||
history_dirty = true;
|
||||
}
|
||||
|
||||
if tracker.attempt_count(&container.name) >= MAX_RESTART_ATTEMPTS {
|
||||
@@ -453,6 +526,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
prev_tier = Some(tier);
|
||||
|
||||
if tracker.record_attempt(&container.name) {
|
||||
restart_history.record_attempt(&container.name);
|
||||
history_dirty = true;
|
||||
let attempt = tracker.attempt_count(&container.name);
|
||||
info!("Restarting {} (tier {:?}, attempt {}/{}, backoff {}s)",
|
||||
container.name, tier, attempt, MAX_RESTART_ATTEMPTS,
|
||||
@@ -509,6 +584,12 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
|
||||
state.update_data(data).await;
|
||||
debug!("Health monitor: state updated with notifications");
|
||||
}
|
||||
|
||||
// Persist restart history to disk (debounced: once per check cycle)
|
||||
if history_dirty {
|
||||
restart_history.save(&data_dir).await;
|
||||
history_dirty = false;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user