feat(release): gated ISO build pipeline
- scripts/build-iso-release.sh: single gated command from signed release to tested ISO (preflight version/signature parity, release gate harness, strict catalog drift, full Rust suite, artifact version checks, build, mount smoke, best-effort QEMU boot) - scripts/iso-smoke-test.sh: standalone mount-level ISO verification incl. stale-binary version assertion inside the payload - .gitea/workflows/build-iso.yml: resurrected ISO CI as dispatch-only job calling the gated orchestrator (old one was stranded in _archived/) - tests/release/run.sh: catalog drift now runs --strict (was silently always-pass) - test-iso-qemu.sh: headless mode used -append without -kernel, which QEMU rejects; use -display none so -serial file: capture works Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
parent
c4c558954b
commit
2609f60e6c
62
.gitea/workflows/build-iso.yml
Normal file
62
.gitea/workflows/build-iso.yml
Normal file
@ -0,0 +1,62 @@
|
||||
name: Build Archipelago release ISO (gated)
|
||||
|
||||
# Resurrected from image-recipe/_archived/.gitea-workflows/build-iso-dev.yml.
|
||||
# Dispatch-only on purpose: the ISO is cut per release, not per push, and
|
||||
# the iso-builder runner is a live node — builds are deliberate events.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-iso:
|
||||
runs-on: iso-builder
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Sync source to workspace
|
||||
run: |
|
||||
# Direct fetch + sync (actions/checkout token is broken on this Gitea)
|
||||
REPO_DIR="$HOME/Projects/archy"
|
||||
[ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy"
|
||||
cd "$REPO_DIR" && git fetch origin main && git reset --hard origin/main
|
||||
echo "=== Source at commit: $(git log --oneline -1) ==="
|
||||
|
||||
- name: Install ISO build dependencies
|
||||
run: |
|
||||
if dpkg -s debootstrap squashfs-tools xorriso isolinux syslinux-common mtools \
|
||||
grub-efi-amd64-bin grub-pc-bin grub-common >/dev/null 2>&1; then
|
||||
echo "ISO build deps already installed, skipping apt"
|
||||
else
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq \
|
||||
debootstrap squashfs-tools xorriso \
|
||||
isolinux syslinux-common mtools \
|
||||
grub-efi-amd64-bin grub-pc-bin grub-common
|
||||
fi
|
||||
|
||||
- name: Build backend + frontend if stale
|
||||
run: |
|
||||
REPO_DIR="$HOME/Projects/archy"
|
||||
[ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy"
|
||||
cd "$REPO_DIR"
|
||||
. "$HOME/.cargo/env" 2>/dev/null || true
|
||||
VERSION=$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/')
|
||||
if ! strings core/target/release/archipelago 2>/dev/null | grep -qF "$VERSION"; then
|
||||
cargo build --release --manifest-path core/Cargo.toml -p archipelago
|
||||
fi
|
||||
if ! grep -rqoF "$VERSION" web/dist/neode-ui/assets/*.js 2>/dev/null; then
|
||||
(cd neode-ui && npm ci && npm run build)
|
||||
fi
|
||||
|
||||
- name: Gated ISO build (gates + build + smoke + qemu)
|
||||
run: |
|
||||
REPO_DIR="$HOME/Projects/archy"
|
||||
[ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy"
|
||||
cd "$REPO_DIR"
|
||||
. "$HOME/.cargo/env" 2>/dev/null || true
|
||||
bash scripts/build-iso-release.sh
|
||||
|
||||
- name: Report artifacts
|
||||
if: always()
|
||||
run: |
|
||||
REPO_DIR="$HOME/Projects/archy"
|
||||
[ -d "$REPO_DIR" ] || REPO_DIR="$HOME/archy"
|
||||
ls -lh "$REPO_DIR"/image-recipe/results/*.iso 2>/dev/null | tail -3 || echo "no ISO produced"
|
||||
@ -83,7 +83,7 @@ QEMU_ARGS=(
|
||||
|
||||
# Display mode
|
||||
if [ "$NOGRAPHIC" = true ]; then
|
||||
QEMU_ARGS+=(-nographic -append "console=ttyS0")
|
||||
QEMU_ARGS+=(-display none)
|
||||
else
|
||||
QEMU_ARGS+=(-vga virtio -display default)
|
||||
fi
|
||||
|
||||
199
scripts/build-iso-release.sh
Executable file
199
scripts/build-iso-release.sh
Executable file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env bash
|
||||
# Gated ISO release build — the single command that turns a signed release
|
||||
# on `main` into a tested installer ISO.
|
||||
#
|
||||
# Stages (fail-fast, each logged with timing):
|
||||
# 0. preflight — Linux, clean tree on main, version parity across
|
||||
# Cargo.toml / package.json / releases/manifest.json /
|
||||
# CHANGELOG / git tag, manifest signature present
|
||||
# 1. gates — tests/release/run.sh (static + frontend + backend
|
||||
# slice), strict catalog drift, FULL cargo test suite
|
||||
# 2. artifacts — release binary embeds the version, frontend dist
|
||||
# matches, AIUI present (OTA-strip regression guard)
|
||||
# 3. build — image-recipe/build-debian-iso.sh (unbundled by default)
|
||||
# 4. smoke — scripts/iso-smoke-test.sh (mount-level, version-checked)
|
||||
# 5. qemu — headless boot test (skippable with --no-qemu)
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-iso-release.sh [--skip-gates] [--no-qemu] [--bundled] [--rc N]
|
||||
#
|
||||
# The ISO is NOT signed here — run scripts/sign-iso-checksums.sh with the
|
||||
# offline RELEASE_MASTER_MNEMONIC afterwards (publisher only).
|
||||
|
||||
set -u
|
||||
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO"
|
||||
|
||||
SKIP_GATES=0 NO_QEMU=0 UNBUNDLED=1 RC_OVERRIDE=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--skip-gates) SKIP_GATES=1 ;;
|
||||
--no-qemu) NO_QEMU=1 ;;
|
||||
--bundled) UNBUNDLED=0 ;;
|
||||
--rc) RC_OVERRIDE="${2:?--rc needs a number}"; shift ;;
|
||||
*) echo "unknown flag: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env"
|
||||
|
||||
PASS=() FAIL=()
|
||||
stage() { # stage <name> <cmd...>
|
||||
local name="$1"; shift
|
||||
local t0=$SECONDS
|
||||
echo
|
||||
echo "═══ [$name] $*"
|
||||
if "$@"; then
|
||||
echo "═══ [$name] PASS ($((SECONDS - t0))s)"
|
||||
PASS+=("$name")
|
||||
else
|
||||
local rc=$?
|
||||
echo "═══ [$name] FAIL exit=$rc ($((SECONDS - t0))s)"
|
||||
FAIL+=("$name")
|
||||
summary 1
|
||||
fi
|
||||
}
|
||||
summary() {
|
||||
echo
|
||||
echo "──────── ISO release build summary ────────"
|
||||
printf 'PASS: %s\n' "${PASS[@]:-none}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && printf 'FAIL: %s\n' "${FAIL[@]}"
|
||||
exit "${1:-0}"
|
||||
}
|
||||
|
||||
# ── Stage 0: preflight ───────────────────────────────────────────────
|
||||
preflight() {
|
||||
[ "$(uname -s)" = "Linux" ] || { echo "ISO builds run on Linux only"; return 1; }
|
||||
|
||||
local branch; branch="$(git rev-parse --abbrev-ref HEAD)"
|
||||
[ "$branch" = "main" ] || { echo "must build from main (on: $branch)"; return 1; }
|
||||
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo "working tree is not clean — release ISOs build from committed state only:"
|
||||
git status --porcelain | head -20
|
||||
return 1
|
||||
fi
|
||||
|
||||
VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/')"
|
||||
local ui_ver manifest_ver
|
||||
ui_ver="$(python3 -c 'import json;print(json.load(open("neode-ui/package.json"))["version"])')"
|
||||
manifest_ver="$(python3 -c 'import json;print(json.load(open("releases/manifest.json"))["version"])')"
|
||||
echo " Cargo.toml: $VERSION"
|
||||
echo " package.json: $ui_ver"
|
||||
echo " releases/manifest: $manifest_ver"
|
||||
[ "$VERSION" = "$ui_ver" ] || { echo "version mismatch Cargo vs package.json"; return 1; }
|
||||
[ "$VERSION" = "$manifest_ver" ] || { echo "version mismatch Cargo vs releases/manifest.json"; return 1; }
|
||||
|
||||
head -5 CHANGELOG.md | grep -qF "v$VERSION" \
|
||||
|| { echo "CHANGELOG.md top entry is not v$VERSION"; return 1; }
|
||||
|
||||
git rev-parse -q --verify "refs/tags/v$VERSION" >/dev/null \
|
||||
|| { echo "tag v$VERSION does not exist — cut the release first (scripts/create-release.sh)"; return 1; }
|
||||
|
||||
# The ISO must only ever be cut from a ceremony-signed manifest.
|
||||
python3 - <<'EOF' || return 1
|
||||
import json, sys
|
||||
m = json.load(open("releases/manifest.json"))
|
||||
sig, by = m.get("signature"), m.get("signed_by", "")
|
||||
if not sig or not by.startswith("did:key:"):
|
||||
print("releases/manifest.json is UNSIGNED — run the signing ceremony first")
|
||||
sys.exit(1)
|
||||
print(f" manifest signed by {by[:32]}…")
|
||||
EOF
|
||||
|
||||
echo " version: $VERSION @ $(git rev-parse --short HEAD), tree clean, manifest signed"
|
||||
}
|
||||
stage "preflight" preflight
|
||||
VERSION="$(grep -m1 '^version' core/archipelago/Cargo.toml | sed 's/.*"\(.*\)".*/\1/')"
|
||||
|
||||
# ── Stage 1: gates ───────────────────────────────────────────────────
|
||||
if [ "$SKIP_GATES" = "0" ]; then
|
||||
stage "release-gate-harness" bash tests/release/run.sh
|
||||
stage "catalog-drift-strict" python3 scripts/check-app-catalog-drift.py --release --strict
|
||||
# Full Rust suite — the release harness only runs a 6-module slice;
|
||||
# ~1000 tests otherwise go unverified at ISO time (hardening plan §H).
|
||||
stage "cargo-test-full" timeout 5400 env CARGO_INCREMENTAL=0 \
|
||||
nice -n 10 cargo test --manifest-path core/Cargo.toml -p archipelago --bin archipelago
|
||||
else
|
||||
echo; echo "═══ [gates] SKIPPED (--skip-gates)"
|
||||
fi
|
||||
|
||||
# ── Stage 2: artifact verification ───────────────────────────────────
|
||||
verify_artifacts() {
|
||||
local bin="core/target/release/archipelago"
|
||||
[ -x "$bin" ] || { echo "missing release binary $bin — build it first"; return 1; }
|
||||
strings "$bin" | grep -qF "$VERSION" \
|
||||
|| { echo "release binary does not embed $VERSION — stale build"; return 1; }
|
||||
echo " backend binary embeds $VERSION ($(du -h "$bin" | cut -f1))"
|
||||
|
||||
[ -f web/dist/neode-ui/index.html ] || { echo "missing frontend dist"; return 1; }
|
||||
grep -rqoF "$VERSION" web/dist/neode-ui/assets/*.js \
|
||||
|| { echo "frontend dist does not contain $VERSION — stale build"; return 1; }
|
||||
echo " frontend dist contains $VERSION"
|
||||
|
||||
# AIUI must ride inside the dist BEFORE packaging or OTA upgrades
|
||||
# silently strip it from nodes in the field.
|
||||
[ -f web/dist/neode-ui/aiui/index.html ] \
|
||||
|| { echo "AIUI missing from web/dist/neode-ui/aiui — fold it in before building"; return 1; }
|
||||
echo " AIUI present in frontend dist"
|
||||
}
|
||||
stage "verify-artifacts" verify_artifacts
|
||||
|
||||
# ── Stage 3: build the ISO ───────────────────────────────────────────
|
||||
build_iso() {
|
||||
local env_args=(
|
||||
UNBUNDLED="$UNBUNDLED"
|
||||
BUILD_FROM_SOURCE=0
|
||||
DEV_SERVER=localhost
|
||||
ARCHIPELAGO_BIN="$REPO/core/target/release/archipelago"
|
||||
)
|
||||
[ -n "$RC_OVERRIDE" ] && env_args+=(RC="$RC_OVERRIDE")
|
||||
sudo -E env "${env_args[@]}" nice -n 5 bash image-recipe/build-debian-iso.sh
|
||||
}
|
||||
stage "build-iso" build_iso
|
||||
|
||||
find_iso() {
|
||||
ls -t "$REPO"/image-recipe/results/archipelago-installer-"$VERSION"*-x86_64_RC*.iso 2>/dev/null | head -1
|
||||
}
|
||||
ISO="$(find_iso)"
|
||||
[ -n "$ISO" ] || { echo "FAIL: no ISO produced for $VERSION in image-recipe/results/"; FAIL+=("locate-iso"); summary 1; }
|
||||
|
||||
# ── Stage 4: mount-level smoke test ──────────────────────────────────
|
||||
stage "iso-smoke" bash scripts/iso-smoke-test.sh "$ISO" "$VERSION"
|
||||
|
||||
# ── Stage 5: QEMU boot test (best-effort) ────────────────────────────
|
||||
# The ISO's kernel cmdline has no serial console, so the serial-log
|
||||
# sanity grep can miss a perfectly healthy boot. Run it, report it,
|
||||
# but don't fail an otherwise-green build on it.
|
||||
if [ "$NO_QEMU" = "0" ] && command -v qemu-system-x86_64 >/dev/null 2>&1; then
|
||||
echo
|
||||
echo "═══ [qemu-boot] (best-effort) test-iso-qemu.sh $ISO 180"
|
||||
if bash image-recipe/_archived/test-iso-qemu.sh "$ISO" 180; then
|
||||
echo "═══ [qemu-boot] PASS"
|
||||
PASS+=("qemu-boot")
|
||||
else
|
||||
echo "═══ [qemu-boot] INCONCLUSIVE (not gating — verify on real hardware)"
|
||||
PASS+=("qemu-boot(inconclusive)")
|
||||
fi
|
||||
else
|
||||
echo; echo "═══ [qemu-boot] SKIPPED"
|
||||
fi
|
||||
|
||||
# ── Done ─────────────────────────────────────────────────────────────
|
||||
SHA_FILE="$ISO.sha256"
|
||||
[ -f "$SHA_FILE" ] || (cd "$(dirname "$ISO")" && sha256sum "$(basename "$ISO")" > "$SHA_FILE")
|
||||
|
||||
echo
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " ISO RELEASE BUILD COMPLETE — v$VERSION"
|
||||
echo "════════════════════════════════════════════════════"
|
||||
echo " ISO: $ISO ($(du -h "$ISO" | cut -f1))"
|
||||
echo " SHA256: $(cut -d' ' -f1 "$SHA_FILE")"
|
||||
echo
|
||||
echo " Next steps (publisher, offline mnemonic required):"
|
||||
echo " 1. scripts/sign-iso-checksums.sh $ISO"
|
||||
echo " 2. upload ISO + .sha256 + signed checksum JSON alongside the"
|
||||
echo " v$VERSION Gitea release assets"
|
||||
summary 0
|
||||
131
scripts/iso-smoke-test.sh
Executable file
131
scripts/iso-smoke-test.sh
Executable file
@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mount-level smoke test for an Archipelago installer ISO.
|
||||
#
|
||||
# Verifies boot plumbing (BIOS + UEFI + live-boot), the auto-installer
|
||||
# payload, and — the check that has bitten before — that the backend
|
||||
# binary inside the ISO actually embeds the version the filename claims.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/iso-smoke-test.sh <path-to-iso> [expected-version]
|
||||
#
|
||||
# expected-version defaults to core/archipelago/Cargo.toml. Needs sudo
|
||||
# (loop mount). Exits non-zero on the first hard failure; prints a
|
||||
# PASS/FAIL table either way.
|
||||
|
||||
set -u
|
||||
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
ISO="${1:-}"
|
||||
EXPECTED_VERSION="${2:-$(grep -m1 '^version' "$REPO/core/archipelago/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/')}"
|
||||
|
||||
if [ -z "$ISO" ] || [ ! -f "$ISO" ]; then
|
||||
echo "usage: $0 <path-to-iso> [expected-version]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
ok() { echo " OK: $*"; }
|
||||
bad() { echo " FAIL: $*"; FAIL=1; }
|
||||
warn() { echo " WARN: $*"; }
|
||||
|
||||
echo "ISO smoke test"
|
||||
echo " ISO: $ISO ($(du -h "$ISO" | cut -f1))"
|
||||
echo " Version: $EXPECTED_VERSION (expected)"
|
||||
|
||||
# ── Filename ↔ version parity (gap: ISO version can silently drift) ──
|
||||
case "$(basename "$ISO")" in
|
||||
*"$EXPECTED_VERSION"*) ok "filename contains $EXPECTED_VERSION" ;;
|
||||
*) bad "filename does not contain expected version $EXPECTED_VERSION" ;;
|
||||
esac
|
||||
|
||||
MNT="$(mktemp -d)"
|
||||
INITRD_DIR=""
|
||||
cleanup() {
|
||||
sudo umount "$MNT" 2>/dev/null || true
|
||||
rmdir "$MNT" 2>/dev/null || true
|
||||
[ -n "$INITRD_DIR" ] && sudo rm -rf "$INITRD_DIR" 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if ! sudo mount -o loop,ro "$ISO" "$MNT"; then
|
||||
echo " FAIL: could not loop-mount ISO" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Required boot + installer files ──────────────────────────────────
|
||||
for f in live/vmlinuz live/initrd.img live/filesystem.squashfs \
|
||||
isolinux/isolinux.bin isolinux/isolinux.cfg \
|
||||
boot/grub/grub.cfg EFI/BOOT/BOOTX64.EFI \
|
||||
archipelago/auto-install.sh archipelago/rootfs.tar; do
|
||||
if [ -e "$MNT/$f" ]; then
|
||||
ok "$f ($(sudo du -h "$MNT/$f" 2>/dev/null | cut -f1))"
|
||||
else
|
||||
bad "missing $f"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── GRUB must boot the live system ───────────────────────────────────
|
||||
if grep -q "boot=live" "$MNT/boot/grub/grub.cfg" 2>/dev/null; then
|
||||
ok "grub.cfg has boot=live"
|
||||
else
|
||||
bad "grub.cfg missing boot=live"
|
||||
fi
|
||||
|
||||
# ── initrd must contain live-boot scripts ────────────────────────────
|
||||
if command -v unmkinitramfs >/dev/null 2>&1; then
|
||||
INITRD_DIR="$(mktemp -d)"
|
||||
sudo unmkinitramfs "$MNT/live/initrd.img" "$INITRD_DIR" 2>/dev/null
|
||||
if [ -e "$INITRD_DIR/scripts/live" ] || [ -e "$INITRD_DIR/main/scripts/live" ]; then
|
||||
ok "initrd has live-boot scripts"
|
||||
else
|
||||
bad "initrd missing live-boot scripts"
|
||||
fi
|
||||
else
|
||||
warn "unmkinitramfs not installed — skipping initrd live-boot check"
|
||||
fi
|
||||
|
||||
# ── Backend binary inside the ISO embeds the expected version ────────
|
||||
# (the v1.4.0-binary-in-a-v1.5-ISO incident: a stale captured binary
|
||||
# shipped and the fleet rejected its fips.yaml on Activate)
|
||||
BIN_IN_ISO=""
|
||||
if [ -f "$MNT/archipelago/bin/archipelago" ]; then
|
||||
BIN_IN_ISO="$MNT/archipelago/bin/archipelago"
|
||||
if sudo strings "$BIN_IN_ISO" 2>/dev/null | grep -qF "$EXPECTED_VERSION"; then
|
||||
ok "payload backend binary embeds $EXPECTED_VERSION"
|
||||
else
|
||||
bad "payload backend binary does NOT embed $EXPECTED_VERSION (stale binary)"
|
||||
fi
|
||||
else
|
||||
# Fall back to the copy inside rootfs.tar
|
||||
TMPBIN="$(mktemp -d)"
|
||||
if sudo tar -xf "$MNT/archipelago/rootfs.tar" -C "$TMPBIN" \
|
||||
usr/local/bin/archipelago 2>/dev/null; then
|
||||
if sudo strings "$TMPBIN/usr/local/bin/archipelago" | grep -qF "$EXPECTED_VERSION"; then
|
||||
ok "rootfs backend binary embeds $EXPECTED_VERSION"
|
||||
else
|
||||
bad "rootfs backend binary does NOT embed $EXPECTED_VERSION (stale binary)"
|
||||
fi
|
||||
else
|
||||
bad "no backend binary found at archipelago/bin/ or in rootfs.tar"
|
||||
fi
|
||||
sudo rm -rf "$TMPBIN"
|
||||
fi
|
||||
|
||||
# ── Frontend payload present ─────────────────────────────────────────
|
||||
if [ -f "$MNT/archipelago/web-ui/index.html" ]; then
|
||||
ok "frontend payload (archipelago/web-ui/index.html)"
|
||||
if [ -f "$MNT/archipelago/web-ui/aiui/index.html" ]; then
|
||||
ok "AIUI included in frontend payload"
|
||||
else
|
||||
warn "AIUI missing from archipelago/web-ui (verify rootfs copy before shipping)"
|
||||
fi
|
||||
else
|
||||
warn "no archipelago/web-ui payload on ISO (frontend may live in rootfs.tar only)"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$FAIL" = "1" ]; then
|
||||
echo "ISO SMOKE TEST: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "ISO SMOKE TEST: PASSED"
|
||||
@ -62,7 +62,7 @@ summary() {
|
||||
# ── Stage 1: static ──────────────────────────────────────────────────
|
||||
stage "git-diff-check" git diff --check
|
||||
stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml --all --check
|
||||
stage "catalog-drift" python3 scripts/check-app-catalog-drift.py
|
||||
stage "catalog-drift" python3 scripts/check-app-catalog-drift.py --release --strict
|
||||
# Every release must surface its CHANGELOG entry in the Settings "What's New"
|
||||
# modal. The modal hardcodes a block per version and has drifted behind before
|
||||
# (sat at v1.7.84 while the fleet shipped to v1.7.92). Fail if any CHANGELOG
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user