diff --git a/.gitea/workflows/build-iso.yml b/.gitea/workflows/build-iso.yml new file mode 100644 index 00000000..d1abdc54 --- /dev/null +++ b/.gitea/workflows/build-iso.yml @@ -0,0 +1,62 @@ +name: Build Archipelago release ISO (gated) + +# Resurrected from image-recipe/_archived/.gitea-workflows/build-iso-dev.yml. +# Dispatch-only on purpose: the ISO is cut per release, not per push, and +# the iso-builder runner is a live node — builds are deliberate events. +on: + workflow_dispatch: + +jobs: + build-iso: + runs-on: iso-builder + timeout-minutes: 180 + steps: + - name: Sync source to workspace + run: | + # Direct fetch + sync (actions/checkout token is broken on this Gitea) + REPO_DIR="$HOME/Projects/archy" + [ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy" + cd "$REPO_DIR" && git fetch origin main && git reset --hard origin/main + echo "=== Source at commit: $(git log --oneline -1) ===" + + - name: Install ISO build dependencies + run: | + if dpkg -s debootstrap squashfs-tools xorriso isolinux syslinux-common mtools \ + grub-efi-amd64-bin grub-pc-bin grub-common >/dev/null 2>&1; then + echo "ISO build deps already installed, skipping apt" + else + sudo apt-get update -qq + sudo apt-get install -y -qq \ + debootstrap squashfs-tools xorriso \ + isolinux syslinux-common mtools \ + grub-efi-amd64-bin grub-pc-bin grub-common + fi + + - name: Build backend + frontend if stale + run: | + REPO_DIR="$HOME/Projects/archy" + [ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy" + cd "$REPO_DIR" + . "$HOME/.cargo/env" 2>/dev/null || true + VERSION=$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + if ! strings core/target/release/archipelago 2>/dev/null | grep -qF "$VERSION"; then + cargo build --release --manifest-path core/Cargo.toml -p archipelago + fi + if ! grep -rqoF "$VERSION" web/dist/neode-ui/assets/*.js 2>/dev/null; then + (cd neode-ui && npm ci && npm run build) + fi + + - name: Gated ISO build (gates + build + smoke + qemu) + run: | + REPO_DIR="$HOME/Projects/archy" + [ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy" + cd "$REPO_DIR" + . "$HOME/.cargo/env" 2>/dev/null || true + bash scripts/build-iso-release.sh + + - name: Report artifacts + if: always() + run: | + REPO_DIR="$HOME/Projects/archy" + [ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy" + ls -lh "$REPO_DIR"/image-recipe/results/*.iso 2>/dev/null | tail -3 || echo "no ISO produced" diff --git a/image-recipe/_archived/test-iso-qemu.sh b/image-recipe/_archived/test-iso-qemu.sh index 70efe9d5..f44521e1 100755 --- a/image-recipe/_archived/test-iso-qemu.sh +++ b/image-recipe/_archived/test-iso-qemu.sh @@ -83,7 +83,7 @@ QEMU_ARGS=( # Display mode if [ "$NOGRAPHIC" = true ]; then - QEMU_ARGS+=(-nographic -append "console=ttyS0") + QEMU_ARGS+=(-display none) else QEMU_ARGS+=(-vga virtio -display default) fi diff --git a/scripts/build-iso-release.sh b/scripts/build-iso-release.sh new file mode 100755 index 00000000..913673f8 --- /dev/null +++ b/scripts/build-iso-release.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Gated ISO release build — the single command that turns a signed release +# on `main` into a tested installer ISO. +# +# Stages (fail-fast, each logged with timing): +# 0. preflight — Linux, clean tree on main, version parity across +# Cargo.toml / package.json / releases/manifest.json / +# CHANGELOG / git tag, manifest signature present +# 1. gates — tests/release/run.sh (static + frontend + backend +# slice), strict catalog drift, FULL cargo test suite +# 2. artifacts — release binary embeds the version, frontend dist +# matches, AIUI present (OTA-strip regression guard) +# 3. build — image-recipe/build-debian-iso.sh (unbundled by default) +# 4. smoke — scripts/iso-smoke-test.sh (mount-level, version-checked) +# 5. qemu — headless boot test (skippable with --no-qemu) +# +# Usage: +# scripts/build-iso-release.sh [--skip-gates] [--no-qemu] [--bundled] [--rc N] +# +# The ISO is NOT signed here — run scripts/sign-iso-checksums.sh with the +# offline RELEASE_MASTER_MNEMONIC afterwards (publisher only). + +set -u + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" + +SKIP_GATES=0 NO_QEMU=0 UNBUNDLED=1 RC_OVERRIDE="" +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-gates) SKIP_GATES=1 ;; + --no-qemu) NO_QEMU=1 ;; + --bundled) UNBUNDLED=0 ;; + --rc) RC_OVERRIDE="${2:?--rc needs a number}"; shift ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac + shift +done + +[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env" + +PASS=() FAIL=() +stage() { # stage + local name="$1"; shift + local t0=$SECONDS + echo + echo "═══ [$name] $*" + if "$@"; then + echo "═══ [$name] PASS ($((SECONDS - t0))s)" + PASS+=("$name") + else + local rc=$? + echo "═══ [$name] FAIL exit=$rc ($((SECONDS - t0))s)" + FAIL+=("$name") + summary 1 + fi +} +summary() { + echo + echo "──────── ISO release build summary ────────" + printf 'PASS: %s\n' "${PASS[@]:-none}" + [[ ${#FAIL[@]} -gt 0 ]] && printf 'FAIL: %s\n' "${FAIL[@]}" + exit "${1:-0}" +} + +# ── Stage 0: preflight ─────────────────────────────────────────────── +preflight() { + [ "$(uname -s)" = "Linux" ] || { echo "ISO builds run on Linux only"; return 1; } + + local branch; branch="$(git rev-parse --abbrev-ref HEAD)" + [ "$branch" = "main" ] || { echo "must build from main (on: $branch)"; return 1; } + + if [ -n "$(git status --porcelain)" ]; then + echo "working tree is not clean — release ISOs build from committed state only:" + git status --porcelain | head -20 + return 1 + fi + + VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/')" + local ui_ver manifest_ver + ui_ver="$(python3 -c 'import json;print(json.load(open("neode-ui/package.json"))["version"])')" + manifest_ver="$(python3 -c 'import json;print(json.load(open("releases/manifest.json"))["version"])')" + echo " Cargo.toml: $VERSION" + echo " package.json: $ui_ver" + echo " releases/manifest: $manifest_ver" + [ "$VERSION" = "$ui_ver" ] || { echo "version mismatch Cargo vs package.json"; return 1; } + [ "$VERSION" = "$manifest_ver" ] || { echo "version mismatch Cargo vs releases/manifest.json"; return 1; } + + head -5 CHANGELOG.md | grep -qF "v$VERSION" \ + || { echo "CHANGELOG.md top entry is not v$VERSION"; return 1; } + + git rev-parse -q --verify "refs/tags/v$VERSION" >/dev/null \ + || { echo "tag v$VERSION does not exist — cut the release first (scripts/create-release.sh)"; return 1; } + + # The ISO must only ever be cut from a ceremony-signed manifest. + python3 - <<'EOF' || return 1 +import json, sys +m = json.load(open("releases/manifest.json")) +sig, by = m.get("signature"), m.get("signed_by", "") +if not sig or not by.startswith("did:key:"): + print("releases/manifest.json is UNSIGNED — run the signing ceremony first") + sys.exit(1) +print(f" manifest signed by {by[:32]}…") +EOF + + echo " version: $VERSION @ $(git rev-parse --short HEAD), tree clean, manifest signed" +} +stage "preflight" preflight +VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/')" + +# ── Stage 1: gates ─────────────────────────────────────────────────── +if [ "$SKIP_GATES" = "0" ]; then + stage "release-gate-harness" bash tests/release/run.sh + stage "catalog-drift-strict" python3 scripts/check-app-catalog-drift.py --release --strict + # Full Rust suite — the release harness only runs a 6-module slice; + # ~1000 tests otherwise go unverified at ISO time (hardening plan §H). + stage "cargo-test-full" timeout 5400 env CARGO_INCREMENTAL=0 \ + nice -n 10 cargo test --manifest-path core/Cargo.toml -p archipelago --bin archipelago +else + echo; echo "═══ [gates] SKIPPED (--skip-gates)" +fi + +# ── Stage 2: artifact verification ─────────────────────────────────── +verify_artifacts() { + local bin="core/target/release/archipelago" + [ -x "$bin" ] || { echo "missing release binary $bin — build it first"; return 1; } + strings "$bin" | grep -qF "$VERSION" \ + || { echo "release binary does not embed $VERSION — stale build"; return 1; } + echo " backend binary embeds $VERSION ($(du -h "$bin" | cut -f1))" + + [ -f web/dist/neode-ui/index.html ] || { echo "missing frontend dist"; return 1; } + grep -rqoF "$VERSION" web/dist/neode-ui/assets/*.js \ + || { echo "frontend dist does not contain $VERSION — stale build"; return 1; } + echo " frontend dist contains $VERSION" + + # AIUI must ride inside the dist BEFORE packaging or OTA upgrades + # silently strip it from nodes in the field. + [ -f web/dist/neode-ui/aiui/index.html ] \ + || { echo "AIUI missing from web/dist/neode-ui/aiui — fold it in before building"; return 1; } + echo " AIUI present in frontend dist" +} +stage "verify-artifacts" verify_artifacts + +# ── Stage 3: build the ISO ─────────────────────────────────────────── +build_iso() { + local env_args=( + UNBUNDLED="$UNBUNDLED" + BUILD_FROM_SOURCE=0 + DEV_SERVER=localhost + ARCHIPELAGO_BIN="$REPO/core/target/release/archipelago" + ) + [ -n "$RC_OVERRIDE" ] && env_args+=(RC="$RC_OVERRIDE") + sudo -E env "${env_args[@]}" nice -n 5 bash image-recipe/build-debian-iso.sh +} +stage "build-iso" build_iso + +find_iso() { + ls -t "$REPO"/image-recipe/results/archipelago-installer-"$VERSION"*-x86_64_RC*.iso 2>/dev/null | head -1 +} +ISO="$(find_iso)" +[ -n "$ISO" ] || { echo "FAIL: no ISO produced for $VERSION in image-recipe/results/"; FAIL+=("locate-iso"); summary 1; } + +# ── Stage 4: mount-level smoke test ────────────────────────────────── +stage "iso-smoke" bash scripts/iso-smoke-test.sh "$ISO" "$VERSION" + +# ── Stage 5: QEMU boot test (best-effort) ──────────────────────────── +# The ISO's kernel cmdline has no serial console, so the serial-log +# sanity grep can miss a perfectly healthy boot. Run it, report it, +# but don't fail an otherwise-green build on it. +if [ "$NO_QEMU" = "0" ] && command -v qemu-system-x86_64 >/dev/null 2>&1; then + echo + echo "═══ [qemu-boot] (best-effort) test-iso-qemu.sh $ISO 180" + if bash image-recipe/_archived/test-iso-qemu.sh "$ISO" 180; then + echo "═══ [qemu-boot] PASS" + PASS+=("qemu-boot") + else + echo "═══ [qemu-boot] INCONCLUSIVE (not gating — verify on real hardware)" + PASS+=("qemu-boot(inconclusive)") + fi +else + echo; echo "═══ [qemu-boot] SKIPPED" +fi + +# ── Done ───────────────────────────────────────────────────────────── +SHA_FILE="$ISO.sha256" +[ -f "$SHA_FILE" ] || (cd "$(dirname "$ISO")" && sha256sum "$(basename "$ISO")" > "$SHA_FILE") + +echo +echo "════════════════════════════════════════════════════" +echo " ISO RELEASE BUILD COMPLETE — v$VERSION" +echo "════════════════════════════════════════════════════" +echo " ISO: $ISO ($(du -h "$ISO" | cut -f1))" +echo " SHA256: $(cut -d' ' -f1 "$SHA_FILE")" +echo +echo " Next steps (publisher, offline mnemonic required):" +echo " 1. scripts/sign-iso-checksums.sh $ISO" +echo " 2. upload ISO + .sha256 + signed checksum JSON alongside the" +echo " v$VERSION Gitea release assets" +summary 0 diff --git a/scripts/iso-smoke-test.sh b/scripts/iso-smoke-test.sh new file mode 100755 index 00000000..760e13d8 --- /dev/null +++ b/scripts/iso-smoke-test.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Mount-level smoke test for an Archipelago installer ISO. +# +# Verifies boot plumbing (BIOS + UEFI + live-boot), the auto-installer +# payload, and — the check that has bitten before — that the backend +# binary inside the ISO actually embeds the version the filename claims. +# +# Usage: +# scripts/iso-smoke-test.sh [expected-version] +# +# expected-version defaults to core/archipelago/Cargo.toml. Needs sudo +# (loop mount). Exits non-zero on the first hard failure; prints a +# PASS/FAIL table either way. + +set -u + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ISO="${1:-}" +EXPECTED_VERSION="${2:-$(grep -m1 '^version' "$REPO/core/archipelago/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/')}" + +if [ -z "$ISO" ] || [ ! -f "$ISO" ]; then + echo "usage: $0 [expected-version]" >&2 + exit 2 +fi + +FAIL=0 +ok() { echo " OK: $*"; } +bad() { echo " FAIL: $*"; FAIL=1; } +warn() { echo " WARN: $*"; } + +echo "ISO smoke test" +echo " ISO: $ISO ($(du -h "$ISO" | cut -f1))" +echo " Version: $EXPECTED_VERSION (expected)" + +# ── Filename ↔ version parity (gap: ISO version can silently drift) ── +case "$(basename "$ISO")" in + *"$EXPECTED_VERSION"*) ok "filename contains $EXPECTED_VERSION" ;; + *) bad "filename does not contain expected version $EXPECTED_VERSION" ;; +esac + +MNT="$(mktemp -d)" +INITRD_DIR="" +cleanup() { + sudo umount "$MNT" 2>/dev/null || true + rmdir "$MNT" 2>/dev/null || true + [ -n "$INITRD_DIR" ] && sudo rm -rf "$INITRD_DIR" 2>/dev/null +} +trap cleanup EXIT + +if ! sudo mount -o loop,ro "$ISO" "$MNT"; then + echo " FAIL: could not loop-mount ISO" >&2 + exit 1 +fi + +# ── Required boot + installer files ────────────────────────────────── +for f in live/vmlinuz live/initrd.img live/filesystem.squashfs \ + isolinux/isolinux.bin isolinux/isolinux.cfg \ + boot/grub/grub.cfg EFI/BOOT/BOOTX64.EFI \ + archipelago/auto-install.sh archipelago/rootfs.tar; do + if [ -e "$MNT/$f" ]; then + ok "$f ($(sudo du -h "$MNT/$f" 2>/dev/null | cut -f1))" + else + bad "missing $f" + fi +done + +# ── GRUB must boot the live system ─────────────────────────────────── +if grep -q "boot=live" "$MNT/boot/grub/grub.cfg" 2>/dev/null; then + ok "grub.cfg has boot=live" +else + bad "grub.cfg missing boot=live" +fi + +# ── initrd must contain live-boot scripts ──────────────────────────── +if command -v unmkinitramfs >/dev/null 2>&1; then + INITRD_DIR="$(mktemp -d)" + sudo unmkinitramfs "$MNT/live/initrd.img" "$INITRD_DIR" 2>/dev/null + if [ -e "$INITRD_DIR/scripts/live" ] || [ -e "$INITRD_DIR/main/scripts/live" ]; then + ok "initrd has live-boot scripts" + else + bad "initrd missing live-boot scripts" + fi +else + warn "unmkinitramfs not installed — skipping initrd live-boot check" +fi + +# ── Backend binary inside the ISO embeds the expected version ──────── +# (the v1.4.0-binary-in-a-v1.5-ISO incident: a stale captured binary +# shipped and the fleet rejected its fips.yaml on Activate) +BIN_IN_ISO="" +if [ -f "$MNT/archipelago/bin/archipelago" ]; then + BIN_IN_ISO="$MNT/archipelago/bin/archipelago" + if sudo strings "$BIN_IN_ISO" 2>/dev/null | grep -qF "$EXPECTED_VERSION"; then + ok "payload backend binary embeds $EXPECTED_VERSION" + else + bad "payload backend binary does NOT embed $EXPECTED_VERSION (stale binary)" + fi +else + # Fall back to the copy inside rootfs.tar + TMPBIN="$(mktemp -d)" + if sudo tar -xf "$MNT/archipelago/rootfs.tar" -C "$TMPBIN" \ + usr/local/bin/archipelago 2>/dev/null; then + if sudo strings "$TMPBIN/usr/local/bin/archipelago" | grep -qF "$EXPECTED_VERSION"; then + ok "rootfs backend binary embeds $EXPECTED_VERSION" + else + bad "rootfs backend binary does NOT embed $EXPECTED_VERSION (stale binary)" + fi + else + bad "no backend binary found at archipelago/bin/ or in rootfs.tar" + fi + sudo rm -rf "$TMPBIN" +fi + +# ── Frontend payload present ───────────────────────────────────────── +if [ -f "$MNT/archipelago/web-ui/index.html" ]; then + ok "frontend payload (archipelago/web-ui/index.html)" + if [ -f "$MNT/archipelago/web-ui/aiui/index.html" ]; then + ok "AIUI included in frontend payload" + else + warn "AIUI missing from archipelago/web-ui (verify rootfs copy before shipping)" + fi +else + warn "no archipelago/web-ui payload on ISO (frontend may live in rootfs.tar only)" +fi + +echo +if [ "$FAIL" = "1" ]; then + echo "ISO SMOKE TEST: FAILED" + exit 1 +fi +echo "ISO SMOKE TEST: PASSED" diff --git a/tests/release/run.sh b/tests/release/run.sh index cf79e0b6..5c577d33 100755 --- a/tests/release/run.sh +++ b/tests/release/run.sh @@ -62,7 +62,7 @@ summary() { # ── Stage 1: static ────────────────────────────────────────────────── stage "git-diff-check" git diff --check stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml --all --check -stage "catalog-drift" python3 scripts/check-app-catalog-drift.py +stage "catalog-drift" python3 scripts/check-app-catalog-drift.py --release --strict # Every release must surface its CHANGELOG entry in the Settings "What's New" # modal. The modal hardcodes a block per version and has drifted behind before # (sat at v1.7.84 while the fleet shipped to v1.7.92). Fail if any CHANGELOG