fix(release): publish the manifest only after assets are proven fetchable
Today's outage window came from ordering, and the ordering was baked into the
publish script itself: it pushed main — the branch nodes read the manifest
from — together with the tag, up front, then uploaded and verified assets
afterward. So the manifest advertised the new version for the entire
upload+verify window. When an upload failed inside that window, every polling
node briefly saw a v1.7.126-alpha update whose binary 500'd and whose tarball
did not yet exist.
Reordered so the manifest goes live last:
1. push the TAG only (the Gitea release and asset URLs hang off it; the tag
alone changes nothing for nodes)
2. upload assets
3. verify every asset downloads in full and matches the manifest sha256/size
4. only then push main — the step that actually triggers nodes
Also fixes a way a bad asset could slip through unnoticed: the inline
verification ran in a `while read` pipe subshell, where its `fail` (exit 1)
terminated only the subshell and let the script continue to "published and
verified". Verification now runs in the main shell via a new
check-release-assets.sh, which fails hard on the first bad asset. The same
script is the reusable by-hand verifier used to recover today's release
(both assets confirmed 200 + sha256-match before the manifest was re-published).
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
co-authored by
Claude Opus 5
Claude Opus 5
parent
35f992fdb4
commit
308f3cbd84
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# check-release-assets.sh — prove a release's artifacts are actually fetchable
|
||||
# BEFORE its manifest goes live on main.
|
||||
#
|
||||
# The manifest is the trigger: nodes read releases/manifest.json from branch
|
||||
# main, and the moment it names a new version they try to download it. So the
|
||||
# assets must resolve before the manifest lands, not after. On 2026-08-07 the
|
||||
# order was reversed — the v1.7.126-alpha manifest went live while its binary
|
||||
# 500'd and its frontend tarball had never uploaded — and every polling node
|
||||
# would have advertised an update it could not fetch.
|
||||
#
|
||||
# For each component in the manifest this checks:
|
||||
# 1. the download URL returns HTTP 200
|
||||
# 2. the downloaded bytes match the manifest's sha256 and size
|
||||
#
|
||||
# It downloads each asset in full, because a HEAD 200 is not proof the body is
|
||||
# intact — the corrupt binary that day passed HEAD-shaped checks and still
|
||||
# served a broken stream. Slower, but this is the last gate before publish.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/check-release-assets.sh # check releases/manifest.json
|
||||
# scripts/check-release-assets.sh path/to/manifest.json
|
||||
#
|
||||
# Exit 0 = every asset is downloadable and matches. Non-zero = do NOT publish.
|
||||
set -euo pipefail
|
||||
|
||||
MANIFEST="${1:-releases/manifest.json}"
|
||||
if [[ ! -f "$MANIFEST" ]]; then
|
||||
echo "ERROR: manifest not found: $MANIFEST" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
command -v python3 >/dev/null 2>&1 || { echo "ERROR: python3 required" >&2; exit 2; }
|
||||
command -v sha256sum >/dev/null 2>&1 || { echo "ERROR: sha256sum required" >&2; exit 2; }
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# Emit "url<TAB>sha256<TAB>size<TAB>name" per component, tolerating the field
|
||||
# name variations the manifest has used (download_url/url, size_bytes/size).
|
||||
rows="$(python3 - "$MANIFEST" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
comps = d.get("components") or []
|
||||
if not comps:
|
||||
sys.exit("manifest has no components")
|
||||
for c in comps:
|
||||
url = c.get("download_url") or c.get("url") or ""
|
||||
sha = c.get("sha256") or ""
|
||||
size = c.get("size_bytes") or c.get("size") or ""
|
||||
name = c.get("name") or "(unnamed)"
|
||||
if not url or not sha:
|
||||
sys.exit(f"component {name!r} missing url or sha256")
|
||||
print(f"{url}\t{sha}\t{size}\t{name}")
|
||||
PY
|
||||
)"
|
||||
|
||||
version="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("version","?"))' "$MANIFEST")"
|
||||
echo "Checking release assets for v${version} ($MANIFEST)"
|
||||
echo ""
|
||||
|
||||
fail=0
|
||||
n=0
|
||||
while IFS=$'\t' read -r url sha size name; do
|
||||
[ -z "$url" ] && continue
|
||||
n=$((n + 1))
|
||||
out="$TMP/asset.$n"
|
||||
echo " [$name]"
|
||||
echo " $url"
|
||||
|
||||
code="$(curl -sL -o "$out" -w '%{http_code}' "$url" || echo "000")"
|
||||
if [ "$code" != "200" ]; then
|
||||
echo " FAIL: HTTP $code (asset not served)"
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
|
||||
got_size="$(stat -c%s "$out")"
|
||||
if [ -n "$size" ] && [ "$size" != "$got_size" ]; then
|
||||
echo " FAIL: size $got_size, manifest says $size"
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
|
||||
got_sha="$(sha256sum "$out" | awk '{print $1}')"
|
||||
if [ "$got_sha" != "$sha" ]; then
|
||||
echo " FAIL: sha256 mismatch"
|
||||
echo " served: $got_sha"
|
||||
echo " manifest: $sha"
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo " OK: HTTP 200, ${got_size} bytes, sha256 matches"
|
||||
done <<< "$rows"
|
||||
|
||||
echo ""
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo "REFUSING: one or more assets are not fetchable or do not match the manifest."
|
||||
echo "Do NOT publish the manifest — nodes would advertise an update they cannot"
|
||||
echo "apply. Upload/repair the assets, re-run this, and only then flip the"
|
||||
echo "manifest live on main."
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: all $n asset(s) for v${version} download and match the manifest."
|
||||
@@ -62,8 +62,19 @@ repo_path=${repo_path%.git}
|
||||
api="$scheme://$host/api/v1/repos/$repo_path"
|
||||
release_url="$api/releases/tags/v${VERSION}"
|
||||
|
||||
echo "Pushing main and v${VERSION} to $REMOTE..."
|
||||
git -C "$PROJECT_ROOT" push "$REMOTE" main "refs/tags/v${VERSION}"
|
||||
# ORDER MATTERS. The manifest is the trigger — nodes read releases/manifest.json
|
||||
# from branch main and try to download the named version the moment it appears.
|
||||
# So main (which carries the live manifest) must be pushed LAST, only after the
|
||||
# assets are uploaded and their bytes verified against the manifest. The tag is
|
||||
# pushed first because the Gitea release and its asset download URLs hang off it,
|
||||
# but the tag alone changes nothing for nodes.
|
||||
#
|
||||
# This used to push main and the tag together, up front, then upload assets. That
|
||||
# left the manifest live for the entire upload+verify window — and on 2026-08-07
|
||||
# an upload failed inside that window, so every polling node briefly advertised a
|
||||
# v1.7.126-alpha update whose binary 500'd and whose tarball did not exist.
|
||||
echo "Pushing tag v${VERSION} to $REMOTE (not main yet)..."
|
||||
git -C "$PROJECT_ROOT" push "$REMOTE" "refs/tags/v${VERSION}"
|
||||
|
||||
release_json=$(curl -fsS -u "$auth" "$release_url" || true)
|
||||
if [ -z "$release_json" ]; then
|
||||
@@ -107,24 +118,16 @@ upload_asset() {
|
||||
upload_asset "$BACKEND" "archipelago"
|
||||
upload_asset "$FRONTEND" "archipelago-frontend-${VERSION}.tar.gz"
|
||||
|
||||
echo "Verifying public download URLs from manifest (size + sha256)..."
|
||||
python3 - "$PROJECT_ROOT/releases/manifest.json" <<'PY' | while read -r url size sha; do
|
||||
import json
|
||||
import sys
|
||||
echo "Verifying public download URLs (full GET + size + sha256)..."
|
||||
# Delegated to check-release-assets.sh so the same verifier is used here and by
|
||||
# hand during recovery. It fails hard on the first bad asset — the previous
|
||||
# inline `while read` ran in a pipe subshell, where a `fail` (exit) killed only
|
||||
# the subshell and let this script march on to "published and verified".
|
||||
"$PROJECT_ROOT/scripts/check-release-assets.sh" "$PROJECT_ROOT/releases/manifest.json" \
|
||||
|| fail "asset verification failed — NOT pushing main. The manifest stays off the branch nodes read, so no node sees a version it cannot fetch. Repair the assets and re-run."
|
||||
|
||||
manifest = json.load(open(sys.argv[1]))
|
||||
for component in manifest["components"]:
|
||||
print(component["download_url"], component["size_bytes"], component["sha256"])
|
||||
PY
|
||||
# Full GET, not HEAD: a size-correct/content-wrong mirror asset must fail
|
||||
# the publish gate, so compare the actual bytes against the manifest sha256.
|
||||
tmp=$(mktemp)
|
||||
curl -fsSL --max-time 900 -o "$tmp" "$url" || { rm -f "$tmp"; fail "download URL failed: $url"; }
|
||||
actual_size=$(stat -c %s "$tmp")
|
||||
actual_sha=$(sha256sum "$tmp" | awk '{print $1}')
|
||||
rm -f "$tmp"
|
||||
[ "$actual_size" = "$size" ] || fail "download size mismatch for $url (expected $size, got $actual_size)"
|
||||
[ "$actual_sha" = "$sha" ] || fail "download sha256 mismatch for $url (expected $sha, got $actual_sha)"
|
||||
done
|
||||
# Assets are proven fetchable — only now does the manifest become live.
|
||||
echo "Assets verified. Pushing main to $REMOTE (this makes v${VERSION} live)..."
|
||||
git -C "$PROJECT_ROOT" push "$REMOTE" main
|
||||
|
||||
echo "Release v${VERSION} published and verified on $REMOTE."
|
||||
|
||||
Reference in New Issue
Block a user