From cbd463e980691ec9b6bb2cf1c97e6512b16f4b1e Mon Sep 17 00:00:00 2001 From: archipelago Date: Mon, 31 Aug 2026 07:23:44 -0400 Subject: [PATCH] feat(host): crash/hardware-error capture, delivered by a new host-fixup OTA channel (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kdump + rasdaemon on every node, per docs/kdump-rasdaemon-design.md with the approved decisions: hang capture ON (a wedged kiosk dumps and reboots itself instead of sitting dead), crashkernel=256M, backfill ships with this release, phase-2 UI surfacing deferred. Host fixups (docs/system-level-ota-design.md) are the general answer to 'deliver system-level updates OTA': curated OS packages, sysctl drop-ins, service enablement and the GRUB crashkernel line, carried by the signed binary and applied idempotently at startup — non-fatal by construction (offline/locked-dpkg nodes converge on a later boot), skipped on dev boxes and non-Debian hosts. This formalizes the polkit/audio repair precedents into a channel with a stated policy: pinned packages and parameter intent only, never dist-upgrade automation; the ISO bakes the identical end state into fresh installs (next commit). The one runtime limitation is honest: crashkernel memory can only be reserved at boot, so the fixup writes GRUB, runs update-grub, and logs that it takes effect on the next reboot. tests/lifecycle/os-audit.sh gains section D — a graded baseline check: FAIL if capture never landed, WARN if written but awaiting reboot, PASS when reserved, policy live and rasdaemon recording. Section D runs independently of RPC health: a wedged backend must not mask that the node also stopped capturing evidence. Verification: host_fixups unit tests 4/4; cargo fmt clean; full suite runs in the release gate (create-release) and the archi-dev-box lifecycle gate before the tag. --- core/archipelago/src/host_fixups.rs | 344 ++++++++++++++++++++++++++++ core/archipelago/src/main.rs | 7 + docs/README.md | 1 + docs/kdump-rasdaemon-design.md | 6 +- docs/system-level-ota-design.md | 82 +++++++ tests/lifecycle/os-audit.sh | 42 ++++ 6 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 core/archipelago/src/host_fixups.rs create mode 100644 docs/system-level-ota-design.md diff --git a/core/archipelago/src/host_fixups.rs b/core/archipelago/src/host_fixups.rs new file mode 100644 index 00000000..d781a7c7 --- /dev/null +++ b/core/archipelago/src/host_fixups.rs @@ -0,0 +1,344 @@ +//! Host-level fixups: OS packages, kernel parameters and system services the +//! node needs, delivered by the same signed-binary OTA that ships everything +//! else (docs/system-level-ota-design.md). +//! +//! Scope and posture — read before adding anything here: +//! +//! * **Idempotent + non-fatal.** Every step is a no-op when the host already +//! has the desired state, and a failure (offline box, locked dpkg, missing +//! package in the release's Debian suite) logs a warning and moves on. A +//! host fixup must never be able to stop the node from starting. +//! * **Curated, pinned intent — not dist-upgrade automation.** We deliver the +//! specific packages and settings a release deliberately adds (crash +//! capture, hardware-error logging, later: unattended-upgrades posture, host +//! firewall). Regular Debian upgrades stay with the operator; this channel +//! never silently swaps a kernel or a libc. +//! * **Fresh installs converge too.** The ISO bakes the same end state in +//! (Dockerfile.rootfs, auto-install.sh cmdline), so the fixup is a no-op on +//! new machines and only does real work on already-deployed nodes. +//! * **Kernel cmdline can't move at runtime.** `crashkernel=` reserves memory +//! at boot; the fixup writes GRUB and update-grub so the change lands on the +//! next reboot, and says so in the log. Everything else (packages, sysctls, +//! services) applies immediately. +//! +//! First payload (#144, docs/kdump-rasdaemon-design.md): kdump + rasdaemon — +//! post-mortem and hardware-error capture: +//! * kdump-tools/kexec-tools/rasdaemon installed +//! * /etc/default/kdump-tools: USE_KDUMP=1, dumps to /var/crash, compressed +//! core collector +//! * /etc/sysctl.d/99-archipelago-kdump.conf: a wedged node dumps and +//! reboots rather than sitting dead until power-cycled +//! * crashkernel=256M appended to the installed GRUB cmdline (next reboot) +//! * /var/crash pruned to the two newest dumps +//! +//! The module is skipped on dev boxes (same guard bootstrap::run uses) and on +//! hosts without dpkg. + +use anyhow::{Context, Result}; +use tracing::{debug, info, warn}; + +use crate::update::host_sudo; + +/// Packages the node's host must have. Keep this list short and justified — +/// every entry is state we now own on the fleet's OS images. +const HOST_PACKAGES: &[&str] = &["kdump-tools", "kexec-tools", "rasdaemon"]; + +/// Crash-kernel reservation. 256M covers the capture kernel plus makedumpfile +/// on the fleet's 16–64GB amd64 machines (~1–2% of RAM, permanently reserved). +/// The arm image (RPi) is out of scope for phase 1 — see the design doc. +const CRASHKERNEL_PARAM: &str = "crashkernel=256M"; + +const KDUMP_SYSDROPIN_PATH: &str = "/etc/sysctl.d/99-archipelago-kdump.conf"; +const KDUMP_SYSDROPIN: &str = "\ +# Archipelago kdump policy (#144). A wedged kiosk is useless until someone +# power-cycles it — capture the evidence, then reboot by itself. Dumps land in +# /var/crash (see docs/kdump-rasdaemon-design.md); keep-2 pruning is done by +# the host fixup pass, not a timer. +kernel.panic = 10 +kernel.panic_on_oops = 1 +kernel.hung_task_panic = 1 +kernel.hardlockup_panic = 1 +"; + +/// How many dumps to keep in /var/crash. Two ≈ 4 GiB worst case on the 30 GiB +/// unencrypted root — the partition usage itself is tracked by disk_monitor. +const KEEP_DUMPS: usize = 2; + +/// Entry point, spawned from main.rs at startup like the other ensure_* heals. +pub async fn ensure_host_fixups() { + // Dev-box guard (same rationale as bootstrap::run): on contributor + // machines /home/archipelago/archy is a symlink into a git checkout and + // the host is the contributor's own OS — never touch it. + let home_archy = std::path::Path::new("/home/archipelago/archy"); + if tokio::fs::symlink_metadata(home_archy) + .await + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + debug!("/home/archipelago/archy is a symlink — skipping host fixups (dev box)"); + return; + } + // Non-Debian hosts: nothing we manage here applies. + if tokio::fs::symlink_metadata("/usr/bin/dpkg").await.is_err() { + debug!("no dpkg on this host — skipping host fixups"); + return; + } + + if let Err(e) = run_host_fixups().await { + warn!("host fixups failed (non-fatal): {:#}", e); + } +} + +async fn run_host_fixups() -> Result<()> { + // 1. Packages — install only what's missing; a locked/offline apt must + // never block anything downstream (steps below degrade to no-ops). + match ensure_packages().await { + Ok(true) => info!("host fixups: installed missing packages"), + Ok(false) => debug!("host fixups: all packages present"), + Err(e) => warn!("host fixups: package install failed (non-fatal): {:#}", e), + } + + // 2. kdump config + sysctl drop-in + GRUB cmdline + services. One helper + // per concern so a failure in one logs and leaves the others running. + if let Err(e) = ensure_kdump_sysdropin().await { + warn!( + "host fixups: kdump sysctl drop-in failed (non-fatal): {:#}", + e + ); + } + if let Err(e) = ensure_kdump_defaults().await { + warn!( + "host fixups: kdump-tools config failed (non-fatal): {:#}", + e + ); + } + match ensure_crashkernel_cmdline().await? { + true => { + warn!("host fixups: crashkernel= written to GRUB — takes effect on the NEXT reboot") + } + false => debug!("host fixups: crashkernel already in GRUB cmdline"), + } + if let Err(e) = ensure_rasdaemon_enabled().await { + warn!("host fixups: rasdaemon enable failed (non-fatal): {:#}", e); + } + if let Err(e) = prune_crash_dumps().await { + debug!("host fixups: /var/crash prune skipped: {:#}", e); + } + Ok(()) +} + +/// True if any package was installed. Mirrors the polkit repair's apt posture: +/// install without `apt-get update` first; only if that fails (fresh suite, +/// stale index), update once and retry. Both under timeout, both non-fatal. +async fn ensure_packages() -> Result { + let wanted = HOST_PACKAGES + .iter() + .map(|p| format!("'{p}'")) + .collect::>() + .join(" "); + let script = format!( + r#" +set -u +WANTED="{wanted}" +MISSING="" +for p in $WANTED; do + dpkg-query -W -f='${{Status}}' "$p" 2>/dev/null | grep -q 'install ok installed' || MISSING="$MISSING $p" +done +[ -z "$MISSING" ] && exit 0 +timeout 240 apt-get install -y --no-install-recommends $MISSING >/dev/null 2>&1 \ + || timeout 240 sh -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends $MISSING >/dev/null 2>&1' \ + || exit 3 +exit 2 +"# + ); + let status = host_sudo(&["sh", "-lc", &script]) + .await + .context("install host packages")?; + match status.code() { + Some(0) => Ok(false), + Some(2) => Ok(true), + code => anyhow::bail!("host package install exited with {code:?}"), + } +} + +/// Write the sysctl drop-in and apply it live (these four keys are all +/// runtime-settable, so the hang/panic policy takes effect without a reboot). +async fn ensure_kdump_sysdropin() -> Result<()> { + let script = format!( + r#" +set -u +PATH_FILE='{KDUMP_SYSDROPIN_PATH}' +CONTENT_FILE=/tmp/archy-kdump-sysctl.$$.tmp +cat > "$CONTENT_FILE" <<'SYSEOF' +{KDUMP_SYSDROPIN}SYSEOF +if [ -f "$PATH_FILE" ] && cmp -s "$CONTENT_FILE" "$PATH_FILE"; then + rm -f "$CONTENT_FILE" + exit 0 +fi +mv "$CONTENT_FILE" "$PATH_FILE" +chmod 644 "$PATH_FILE" +sysctl --system >/dev/null 2>&1 || true +exit 2 +"# + ); + let status = host_sudo(&["sh", "-lc", &script]) + .await + .context("write kdump sysctl drop-in")?; + match status.code() { + Some(0) => Ok(()), + Some(2) => { + info!("host fixups: installed {KDUMP_SYSDROPIN_PATH} (hang/panic policy)"); + Ok(()) + } + code => anyhow::bail!("kdump sysctl drop-in exited with {code:?}"), + } +} + +/// Point kdump-tools at /var/crash with a compressed core collector. Works on +/// the package's shipped defaults file (USE_KDUMP=0, commented KDUMP_COREDIR) +/// and on any state we already wrote — pure line surgery, idempotent. +async fn ensure_kdump_defaults() -> Result<()> { + let script = r#" +set -u +CONF=/etc/default/kdump-tools +[ -f "$CONF" ] || exit 3 +CHANGED=0 +set_kv() { + # set_kv KEY VALUE — replace any (possibly commented) KEY= line with + # KEY='VALUE', appending at the end when absent. + KEY="$1"; VAL="$2" + if grep -qE "^${KEY}=" "$CONF" 2>/dev/null; then + if ! grep -qE "^${KEY}='?${VAL}'?$" "$CONF"; then + sed -i "s|^${KEY}=.*|${KEY}=\"${VAL}\"|" "$CONF" + CHANGED=1 + fi + else + printf '\n%s="%s"\n' "$KEY" "$VAL" >> "$CONF" + CHANGED=1 + fi +} +set_kv USE_KDUMP 1 +set_kv KDUMP_COREDIR /var/crash +set_kv CORE_COLLECTOR 'makedumpfile -l --message-level 1 -d 31' +[ "$CHANGED" -eq 1 ] || exit 0 +systemctl enable kdump-tools >/dev/null 2>&1 || true +exit 2 +"#; + let status = host_sudo(&["sh", "-lc", script]) + .await + .context("configure kdump-tools")?; + match status.code() { + Some(0) => Ok(()), + Some(2) => { + info!("host fixups: kdump-tools configured (USE_KDUMP=1, /var/crash)"); + Ok(()) + } + code => anyhow::bail!("kdump-tools config exited with {code:?}"), + } +} + +/// Append `crashkernel=` to the installed GRUB cmdline and run update-grub. +/// The reservation itself only exists after the next reboot — memory cannot +/// be set aside at runtime — so the caller must log the reboot caveat. +/// Returns true if the cmdline changed. +async fn ensure_crashkernel_cmdline() -> Result { + let script = format!( + r#" +set -u +GRUB=/etc/default/grub +PARAM='{CRASHKERNEL_PARAM}' +[ -f "$GRUB" ] || exit 3 +LINE=$(grep -E '^GRUB_CMDLINE_LINUX_DEFAULT=' "$GRUB" | head -1) +[ -n "$LINE" ] || exit 3 +case "$LINE" in + *"$PARAM"*) exit 0 ;; +esac +NEWLINE=$(printf '%s' "$LINE" | sed "s/\"$/ $PARAM\"/") +sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|$NEWLINE|" "$GRUB" +timeout 120 update-grub >/dev/null 2>&1 || true +exit 2 +"# + ); + let status = host_sudo(&["sh", "-lc", &script]) + .await + .context("set crashkernel= in GRUB")?; + match status.code() { + Some(0) => Ok(false), + Some(2) => Ok(true), + code => anyhow::bail!("crashkernel cmdline fixup exited with {code:?}"), + } +} + +async fn ensure_rasdaemon_enabled() -> Result<()> { + let status = host_sudo(&["systemctl", "enable", "--now", "rasdaemon"]) + .await + .context("enable rasdaemon")?; + if status.success() { + Ok(()) + } else { + anyhow::bail!("systemctl enable --now rasdaemon exited with {status}") + } +} + +/// Keep only the newest [`KEEP_DUMPS`] dumps in /var/crash. Called on every +/// fixup pass rather than by a timer: the pass runs at every startup, which is +/// exactly the cadence at which new dumps appear (a dump ends in a reboot). +async fn prune_crash_dumps() -> Result<()> { + let script = format!( + r#" +set -u +DIR=/var/crash +[ -d "$DIR" ] || exit 0 +KEEP={KEEP_DUMPS} +COUNT=$(ls -1 "$DIR" 2>/dev/null | wc -l) +[ "$COUNT" -gt "$KEEP" ] || exit 0 +ls -1dt "$DIR"/* 2>/dev/null | tail -n +"$((KEEP + 1))" | while IFS= read -r victim; do + rm -rf -- "$victim" +done +exit 2 +"# + ); + let status = host_sudo(&["sh", "-lc", &script]) + .await + .context("prune /var/crash")?; + match status.code() { + Some(0) => Ok(()), + Some(2) => { + info!("host fixups: pruned old dumps in /var/crash (keep {KEEP_DUMPS})"); + Ok(()) + } + code => anyhow::bail!("/var/crash prune exited with {code:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sysctl_dropin_carries_the_full_hang_capture_policy() { + for key in [ + "kernel.panic = 10", + "kernel.panic_on_oops = 1", + "kernel.hung_task_panic = 1", + "kernel.hardlockup_panic = 1", + ] { + assert!(KDUMP_SYSDROPIN.contains(key), "drop-in missing {key}"); + } + } + + #[test] + fn package_list_is_exactly_the_kdump_rasdaemon_set() { + assert_eq!(HOST_PACKAGES, &["kdump-tools", "kexec-tools", "rasdaemon"]); + } + + #[test] + fn crashkernel_param_is_sized_and_unprefixed() { + assert_eq!(CRASHKERNEL_PARAM, "crashkernel=256M"); + } + + #[test] + fn keep_dumps_is_two() { + assert_eq!(KEEP_DUMPS, 2); + } +} diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index 46aa522d..b6c02a39 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -55,6 +55,7 @@ mod entropy; mod federation; mod fips; mod health_monitor; +mod host_fixups; mod host_ip; mod identity; mod identity_manager; @@ -435,6 +436,12 @@ async fn main() -> Result<()> { // iframe on kiosk nodes (docs/tv-input-iframe-apps.md). tokio::spawn(bootstrap::ensure_gamepad_keys()); + // Host-level fixups (#144 + docs/system-level-ota-design.md): kdump + + // rasdaemon — crash/hardware-error capture delivered to already-deployed + // nodes over the signed binary OTA. Idempotent, non-fatal, background; + // the crashkernel= GRUB edit lands on the next reboot. + tokio::spawn(host_fixups::ensure_host_fixups()); + // Mesh access: mirror IPv4-published app ports onto [::] so direct-port // app URLs (http://[]:) work from the companion. tokio::spawn(mesh_ports::run_mesh_port_mirror()); diff --git a/docs/README.md b/docs/README.md index 2ba0c625..0ff22c46 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,7 @@ step-by-step guides, and some predate the current implementation. - [Manifest Hooks](manifest-hooks-design.md) - [Peering & Federation Trust](peering-trust-model.md) — naming/semantics of trust levels vs discovery (#134) - [kdump + rasdaemon Troubleshooting](kdump-rasdaemon-design.md) — post-mortem and hardware-error capture on nodes (#144) +- [System-Level OTA](system-level-ota-design.md) — how host-level packages/config reach already-deployed nodes - [Meshroller Integration](meshroller-integration-design.md) - [Nostr Git Source Hosting](nostr-git-source-hosting.md) - [Nostr Identity Import](nostr-identity-import-plan.md) · [Nostr Signer Login (research)](nostr-signer-login-research.md) diff --git a/docs/kdump-rasdaemon-design.md b/docs/kdump-rasdaemon-design.md index 76f90e9b..1cf2445e 100644 --- a/docs/kdump-rasdaemon-design.md +++ b/docs/kdump-rasdaemon-design.md @@ -1,6 +1,10 @@ # kdump + rasdaemon — post-mortem and hardware-error capture (#144) -Status: DRAFT for review +Status: IMPLEMENTED (phase 1) — decisions approved 2026-08-30: hang capture ON, +crashkernel=256M, ship the backfill with this release, phase-2 UI deferred. +Delivery: image-recipe (Dockerfile.rootfs, auto-install.sh cmdline) + +`core/archipelago/src/host_fixups.rs` (existing nodes, see +docs/system-level-ota-design.md) + `tests/lifecycle/os-audit.sh` section D. Owner: node image (image-recipe) + lifecycle gate Issue: #144 — "Configure kdump and rasdaemon for troubleshooting" diff --git a/docs/system-level-ota-design.md b/docs/system-level-ota-design.md new file mode 100644 index 00000000..71164666 --- /dev/null +++ b/docs/system-level-ota-design.md @@ -0,0 +1,82 @@ +# System-Level OTA — host fixups + +Status: Implemented (first payload shipped alongside this doc) +Owner: `core/archipelago/src/host_fixups.rs` +Related: docs/kdump-rasdaemon-design.md (first payload), CLAUDE.md invariants + +## The problem + +The binary OTA updates the node's own software, and the signed app catalog +updates apps. But the **host OS** — Debian packages, kernel parameters, +system services — previously moved only through ISO re-installs. A node +deployed a year ago can be running today's node software on a host that +never gained anything the image learned since. Issue #99 (missing polkit +rule on old nodes) and the audio-stack heal were each hand-carved +one-off bootstrap repairs; there was no general channel and no stated +policy for touching the host from the node. + +## The mechanism + +`host_fixups::ensure_host_fixups()` — spawned from `main.rs` at startup +alongside the other `ensure_*` heals, in the background, best-effort: + +1. **Dev-box guard** — skip when `/home/archipelago/archy` is a symlink + (contributor checkout) and when there's no dpkg (non-Debian host). +2. **Packages** — install only what's missing, from a curated, in-code + list (`HOST_PACKAGES`), `apt-get install` first, one `apt-get update` + retry, both under timeout, never fatal (offline/locked-dpkg nodes + converge on a later boot). +3. **Configuration** — idempotent per-concern helpers writing root-owned + config (via the existing `host_sudo` path): sysctl drop-ins, service + defaults, GRUB cmdline, service enablement. +4. **Reporting** — every step logs what it did; failures log warnings and + move on. A host fixup must never be able to stop the node from starting. + +### Why embedded-in-the-binary rather than fetched + +Same reasoning as the tor-helper (`bootstrap.rs`): the signed binary OTA +is the only authenticated delivery channel every node already trusts and +pulls on schedule. Fixups compiled into the binary travel with a version, +are reviewable in git, and can't be served to a subset of the fleet. + +## Policy — what may travel this channel + +| May | May not | +|---|---| +| Specific, pinned packages the node needs (kdump-tools, rasdaemon, …) | `dist-upgrade` or silent kernel/libc swaps — regular Debian upgrades stay with the operator | +| Kernel *parameters* via GRUB/sysctl — with the next-reboot caveat logged loudly | Anything requiring a secret, or touching LUKS key material | +| Service enablement + config the image also bakes in | Divergence: the ISO must converge to the SAME end state so fresh installs are a no-op | +| Small, reviewable, per-concern Rust functions with tests | Shell-script-of-things payloads beyond a single concern | + +The rule: **the ISO and the fixup must express the same intent twice, +in reviewable places** — Dockerfile.rootfs/auto-install.sh for fresh +installs, `host_fixups.rs` for the deployed fleet. A change that lands in +one and not the other is a bug. + +## Kernel cmdline caveat + +`crashkernel=` (and any future `hugepages=`-style reservation) only takes +effect at boot: the fixup writes `/etc/default/grub` + `update-grub` and +logs `takes effect on the NEXT reboot`. Operators reboot nodes when +applying releases; no special ceremony is required beyond that, but the +lifecycle gate grades this state honestly (WARN for written-but-not-yet- +rebooted, FAIL for never-written — see `tests/lifecycle/os-audit.sh` +section D). + +## Verification story + +- Unit tests pin the policy constants and script shapes + (`host_fixups` tests in `core/archipelago`). +- `tests/lifecycle/os-audit.sh` section D asserts the end state on a real + node (config present, crashkernel reserved or pending reboot, hang + policy live, rasdaemon active). +- The lifecycle gate runs on archi-dev-box per release; the QEMU ISO + smoke covers fresh installs. + +## Future payloads (candidates, not commitments) + +- `unattended-upgrades` posture + a default-deny host nftables ruleset + (the §F hardening-plan item — needs its own design first). +- Host firewall rules for mesh/WG ports. +- Chronic: anything the image learns post-deploy that old nodes must + converge on (the polkit and audio precedents, formalized). diff --git a/tests/lifecycle/os-audit.sh b/tests/lifecycle/os-audit.sh index 72b3cc8b..4ca451cf 100755 --- a/tests/lifecycle/os-audit.sh +++ b/tests/lifecycle/os-audit.sh @@ -9,6 +9,8 @@ # C. FM-guards — the concrete failure modes that have bitten the # fleet: port-drift (FM8), secret-completeness (FM2), # orphaned container states (FM9), OTA wedge (FM12) +# D. Host capture (#144) — kdump + rasdaemon baseline: crash dumps configured +# and reserved, hang policy live, ECC recording running # # Everything here is READ-ONLY: no install/stop/start/uninstall, no service bounce. # Safe to run against a live production node. It is the per-boot building block the @@ -226,6 +228,43 @@ section_c() { fi } +# ══ Section D — host capture (#144): kdump + rasdaemon ═══════════════════════ +section_d() { + echo + echo "== D. Host capture — crash + hardware-error evidence (#144) ==" + if [[ "$ARCHY_LOCAL" != "1" ]]; then + record WARN "kdump + rasdaemon baseline" "remote node — host checks skipped" + return + fi + # D1. kdump enabled in config (image bakes it in; OTA host fixups converge) + if grep -qE '^USE_KDUMP=.?1' /etc/default/kdump-tools 2>/dev/null; then + record PASS "kdump-tools configured" "USE_KDUMP=1, dumps to /var/crash" + else + record FAIL "kdump-tools configured" "/etc/default/kdump-tools missing USE_KDUMP=1 — host fixup didn't land" + fi + # D2. crashkernel reservation — memory is reserved at BOOT, so a node that + # took the OTA fixup but hasn't rebooted yet is WARN, not FAIL. + if grep -q 'crashkernel=' /proc/cmdline 2>/dev/null; then + record PASS "crashkernel reserved" "$(grep -oE 'crashkernel=[^ ]+' /proc/cmdline | head -1)" + elif grep -q 'crashkernel=' /etc/default/grub 2>/dev/null; then + record WARN "crashkernel reserved" "written to GRUB — applies on next reboot" + else + record FAIL "crashkernel reserved" "absent from /proc/cmdline AND /etc/default/grub" + fi + # D3. hang/panic capture policy — runtime-settable, expected immediately + if [[ "$(cat /proc/sys/kernel/hung_task_panic 2>/dev/null)" == "1" ]]; then + record PASS "hang-capture policy live" "kernel.hung_task_panic=1" + else + record FAIL "hang-capture policy live" "kernel.hung_task_panic!=1 — sysctl drop-in not applied" + fi + # D4. rasdaemon recording hardware errors (ECC/AER events → sqlite) + if systemctl is-active --quiet rasdaemon 2>/dev/null; then + record PASS "rasdaemon active" "hardware-error events recorded to /var/lib/rasdaemon" + else + record FAIL "rasdaemon active" "service not running — package missing or host fixup failed" + fi +} + # ── run ──────────────────────────────────────────────────────────────────────── echo "==============================================================" echo " OS-wide audit — ${BASE_URL} ($(date '+%Y-%m-%d %H:%M:%S'))" @@ -237,6 +276,9 @@ if (( FAIL == 0 )) || [[ -n "$SESSION" ]]; then section_b section_c fi +# Host-capture baseline is independent of RPC health: a wedged backend must +# not mask that the node also stopped capturing evidence. +section_d echo echo "=============================================================="