feat: container orchestration, branding overhaul, onboarding logging

Container orchestration:
- Health monitor with crash recovery and auto-restart
- Doctor service (periodic health checks via systemd timer)
- Reconcile service (desired-state convergence)
- Stack-aware install/uninstall with dependency tracking

Branding:
- Custom GRUB background (designer artwork, 1024x768)
- ISOLINUX boot menu: centered, orange accents, clean labels
- Terminal banners: adaptive width, basic ANSI colors, fits 80-col
- Removed auto-generated splash scripts (designer provides assets)
- GRUB theme: lowercase branding

Frontend:
- 401 handler clears localStorage immediately (prevents cascade)

Backend:
- Onboarding/auth logging ([onboarding] tag in journalctl)
- Cookie Secure flag logging for debugging HTTP/HTTPS issues

ISO fixes:
- Install log saved before unmount (was silently failing)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-28 11:34:29 +00:00
co-authored by Claude Opus 4.6
parent 9d38989048
commit 9db55b0b34
22 changed files with 626 additions and 763 deletions
+82 -1
View File
@@ -6,8 +6,9 @@
use crate::data_model::{Notification, NotificationLevel};
use crate::state::StateManager;
use crate::webhooks::{self, WebhookEvent};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use tracing::{debug, info, warn};
@@ -177,6 +178,69 @@ impl MemoryTracker {
}
// ── Persistent restart tracking ────────────────────────────────────────
// Survives process restarts so a container can't loop infinitely by
// crashing 3 times → triggering process restart → resetting counter → repeat.
const RESTART_HISTORY_FILE: &str = "restart-tracker.json";
#[derive(Serialize, Deserialize, Default)]
struct RestartHistory {
containers: HashMap<String, ContainerRestartRecord>,
}
#[derive(Serialize, Deserialize, Clone)]
struct ContainerRestartRecord {
attempts: u32,
last_failure_epoch: i64,
}
impl RestartHistory {
async fn load(data_dir: &Path) -> Self {
let path = data_dir.join(RESTART_HISTORY_FILE);
match tokio::fs::read_to_string(&path).await {
Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
Err(_) => Self::default(),
}
}
async fn save(&self, data_dir: &Path) {
let path = data_dir.join(RESTART_HISTORY_FILE);
if let Ok(json) = serde_json::to_string(self) {
let _ = tokio::fs::write(&path, json).await;
}
}
/// Seed the in-memory RestartTracker from persisted history.
fn seed_tracker(&self, tracker: &mut RestartTracker) {
let now_epoch = chrono::Utc::now().timestamp();
for (name, record) in &self.containers {
// Only seed if last failure was within the stability window
let secs_since_failure = now_epoch - record.last_failure_epoch;
if secs_since_failure < STABILITY_RESET_SECS as i64 && record.attempts > 0 {
tracker.attempts.insert(name.clone(), record.attempts);
info!(
"Restored restart counter for {}: {} attempts ({}s ago)",
name, record.attempts, secs_since_failure
);
}
}
}
fn record_attempt(&mut self, name: &str) {
let entry = self.containers.entry(name.to_string()).or_insert(ContainerRestartRecord {
attempts: 0,
last_failure_epoch: 0,
});
entry.attempts += 1;
entry.last_failure_epoch = chrono::Utc::now().timestamp();
}
fn clear(&mut self, name: &str) {
self.containers.remove(name);
}
}
/// Query container memory stats from podman.
async fn check_container_memory() -> HashMap<String, u64> {
let output = match tokio::time::timeout(
@@ -373,6 +437,11 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
let mut mem_check_counter: u32 = 0;
let mut interval = tokio::time::interval(std::time::Duration::from_secs(CHECK_INTERVAL_SECS));
// Load persistent restart history and seed the in-memory tracker
let mut restart_history = RestartHistory::load(&data_dir).await;
restart_history.seed_tracker(&mut tracker);
let mut history_dirty = false;
loop {
interval.tick().await;
mem_check_counter += 1;
@@ -406,6 +475,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
if tracker.attempt_count(&container.name) > 0 {
info!("Container {} is healthy again after restart", container.name);
tracker.clear(&container.name);
restart_history.clear(&container.name);
history_dirty = true;
}
continue;
}
@@ -430,6 +501,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
if tracker.should_reset_failed(&container.name) {
info!("Resetting restart counter for {} after {}s stability window", container.name, STABILITY_RESET_SECS);
tracker.clear(&container.name);
restart_history.clear(&container.name);
history_dirty = true;
}
if tracker.attempt_count(&container.name) >= MAX_RESTART_ATTEMPTS {
@@ -453,6 +526,8 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
prev_tier = Some(tier);
if tracker.record_attempt(&container.name) {
restart_history.record_attempt(&container.name);
history_dirty = true;
let attempt = tracker.attempt_count(&container.name);
info!("Restarting {} (tier {:?}, attempt {}/{}, backoff {}s)",
container.name, tier, attempt, MAX_RESTART_ATTEMPTS,
@@ -509,6 +584,12 @@ pub fn spawn_health_monitor(state: Arc<StateManager>, data_dir: PathBuf) {
state.update_data(data).await;
debug!("Health monitor: state updated with notifications");
}
// Persist restart history to disk (debounced: once per check cycle)
if history_dirty {
restart_history.save(&data_dir).await;
history_dirty = false;
}
}
});
}