lfg2025/archy
1
0
Fork 0
Code Issues 27 Pull Requests 5 Actions Packages Projects Releases 3 Wiki Activity

Archipelago v1.7.129-alpha

Browse Source
This commit is contained in:
Archipelago
2026-08-12 10:55:50 +00:00
commit 1595a02a7a
2058 changed files with 470069 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/first-boot-secrets/rotation-tests.sh
Executable
+458
View File
@@ -0,0 +1,458 @@
#!/bin/bash
# Regression harness for scripts/security/host-secrets-audit.sh
# (audit finding F-03, phase 10 / KEY-02, deployed half — decision D-06).
#
# Sibling of run-tests.sh, which covers the ISO-build half. That one proves a
# node never SERVES on a key it did not generate. This one proves a node can
# TELL you whether it is already doing so, and can be fixed without losing the
# operator's session in the middle.
#
# The properties under test are mostly negative or ordering properties, and
# neither kind is assertable against real key material on a real node:
#
# - "--apply without --yes touches nothing" needs a tree to diff
# - "old fingerprints are recorded BEFORE the swap" needs the swap observed
# - "a failed generation leaves the live keys byte-identical" needs failure
# to be forcible
# - "a node with no anchor is reported unknown, never per-node" needs a node
# with no anchor to exist
#
# So the generators are stubbed and the script is driven against temp roots
# through its HOST_SECRETS_ROOT seam — the same move 10-03's harness makes with
# FIRST_BOOT_SECRETS_ROOT, and the same reason.
#
# Usage: bash tests/first-boot-secrets/rotation-tests.sh
# Exit 0 only if all seven cases PASS.
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPT="$REPO/scripts/security/host-secrets-audit.sh"
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
PASS_COUNT=0
FAIL_COUNT=0
ok() { echo "PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); }
bad() { echo "FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); }
[ -f "$SCRIPT" ] || { echo "FAIL: script not found at $SCRIPT"; exit 1; }
[ -x "$SCRIPT" ] || { echo "FAIL: $SCRIPT is not executable"; exit 1; }
if bash -n "$SCRIPT"; then
echo "host-secrets-audit.sh: $(wc -l < "$SCRIPT") lines; bash -n clean"
else
echo "FAIL: host-secrets-audit.sh does not parse"; exit 1
fi
# A rotation script that restarts sshd instead of reloading it disconnects the
# operator on a remote node with no console. Checked here rather than left to
# review, because it is a one-word edit away at all times.
if grep -qn 'systemctl restart ssh' "$SCRIPT"; then
echo "FAIL: host-secrets-audit.sh contains 'systemctl restart ssh' — a restart kills the operator's own session"
exit 1
fi
grep -q 'systemctl reload ssh' "$SCRIPT" || { echo "FAIL: no 'systemctl reload ssh' in the script"; exit 1; }
echo "sshd handling: reload present, restart absent"
# ── Stubs ─────────────────────────────────────────────────────────────────
# Prepended to PATH so openssl / ssh-keygen / systemctl calls land here.
# STUB_OPENSSL_MODE ok | fail (affects `req` only, so a failed
# generation is never mistaken for a
# failed validation)
# STUB_SSHKEYGEN_MODE ok | fail (affects `-A` only, so `-lf` keeps
# working and old fingerprints can still
# be read on the abort path)
# STUB_COUNTER_DIR where generation counters live
# STUB_SYSTEMCTL_LOG file the systemctl stub appends to
make_stubs() {
local dir="$1"
mkdir -p "$dir"
cat > "$dir/openssl" <<'STUB'
#!/bin/bash
sub="${1:-}"
case "$sub" in
pkey)
f=""; pubout=0
while [ $# -gt 0 ]; do
case "$1" in
-in) f="$2"; shift 2 ;;
-pubout) pubout=1; shift ;;
*) shift ;;
esac
done
[ -n "$f" ] && [ -s "$f" ] || exit 1
p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1
[ "$pubout" = 1 ] && printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p"
exit 0
;;
x509)
f=""; want_pub=0; want_fp=0
while [ $# -gt 0 ]; do
case "$1" in
-in) f="$2"; shift 2 ;;
-pubkey) want_pub=1; shift ;;
-fingerprint) want_fp=1; shift ;;
*) shift ;;
esac
done
[ -n "$f" ] && [ -s "$f" ] || exit 1
if [ "$want_pub" = 1 ]; then
p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1
printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p"
fi
if [ "$want_fp" = 1 ]; then
# Content-derived, so a rotated cert has a different digest and the
# old/new comparison in case 7 means something.
printf 'sha256 Fingerprint=%s\n' "$(sha256sum "$f" | cut -c1-32)"
fi
exit 0
;;
esac
[ "$sub" = "req" ] || exit 0
c="${STUB_COUNTER_DIR:-/tmp}/openssl-req.count"
n=$(cat "$c" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$c"
[ "${STUB_OPENSSL_MODE:-ok}" = "fail" ] && exit 1
keyout=""; out=""
while [ $# -gt 0 ]; do
case "$1" in
-keyout) keyout="$2"; shift 2 ;;
-out) out="$2"; shift 2 ;;
*) shift ;;
esac
done
# Both halves carry the same generation id, so the script's pair check passes
# for a real generation and would fail for a mismatched pair.
[ -n "$keyout" ] && printf -- '-----BEGIN PRIVATE KEY-----\nrotated\nSTUB_PUB=%s\n-----END PRIVATE KEY-----\n' "$n" > "$keyout"
[ -n "$out" ] && printf -- '-----BEGIN CERTIFICATE-----\nrotated\nSTUB_PUB=%s\n-----END CERTIFICATE-----\n' "$n" > "$out"
exit 0
STUB
cat > "$dir/ssh-keygen" <<'STUB'
#!/bin/bash
# -lf <pub> -> a fingerprint derived from the file's contents, so a rotated
# key necessarily fingerprints differently.
# -A -f <dir> -> a fresh host-key set, each generation distinct.
if [ "${1:-}" = "-lf" ]; then
f="${2:-}"
[ -s "$f" ] || exit 1
printf '256 SHA256:%s %s (ED25519)\n' "$(sha256sum "$f" | cut -c1-24)" "stub@archipelago"
exit 0
fi
c="${STUB_COUNTER_DIR:-/tmp}/ssh-keygen.count"
n=$(cat "$c" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$c"
[ "${STUB_SSHKEYGEN_MODE:-ok}" = "fail" ] && exit 1
root=""
while [ $# -gt 0 ]; do
case "$1" in
-f) root="$2"; shift 2 ;;
*) shift ;;
esac
done
[ -n "$root" ] || exit 1
mkdir -p "$root/etc/ssh"
for t in rsa ecdsa ed25519; do
printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nrotated-gen%s-%s\n' "$n" "$t" > "$root/etc/ssh/ssh_host_${t}_key"
printf -- 'ssh-%s AAAArotated-gen%s stub@archipelago\n' "$t" "$n" > "$root/etc/ssh/ssh_host_${t}_key.pub"
done
exit 0
STUB
cat > "$dir/systemctl" <<'STUB'
#!/bin/bash
# Records each call ALONGSIDE whether the rotation record already exists at
# that moment. That is how "old fingerprints were written BEFORE the swap"
# becomes an observable ordering fact rather than an inference from content:
# the first reload happens after the first swap, so the record must already be
# on disk by then.
if [ -n "${STUB_SYSTEMCTL_LOG:-}" ]; then
rj="no"
[ -f "${HOST_SECRETS_ROOT:-}/var/lib/archipelago/host-key-rotation.json" ] && rj="yes"
echo "$* rotjson=$rj" >> "$STUB_SYSTEMCTL_LOG"
fi
exit 0
STUB
chmod +x "$dir"/openssl "$dir"/ssh-keygen "$dir"/systemctl
}
STUBS="$WORK/stubs"
make_stubs "$STUBS"
# ── Tree builders ─────────────────────────────────────────────────────────
# T0 is a fixed "this node's first boot" instant. Everything is dated relative
# to it so the cases read as timelines rather than as magic numbers.
T0=$(date -u -d '2026-06-01 12:00:00' +%s)
at() { date -u -d "@$1" '+%Y-%m-%d %H:%M:%S'; }
new_root() {
local name="$1"
local r="$WORK/root-$name"
rm -rf "$r"
mkdir -p "$r/var/lib/archipelago" "$r/var/log" "$r/etc/ssh" \
"$r/etc/archipelago/ssl" "$r/opt/archipelago" "$r/root" "$r/dev"
printf '%s' "$r"
}
# Host keys + TLS material dated at <epoch>.
put_material() {
local r="$1" when="$2" tag="${3:-baked}"
local t
for t in rsa ecdsa ed25519; do
printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\n%s-%s\n' "$tag" "$t" > "$r/etc/ssh/ssh_host_${t}_key"
printf -- 'ssh-%s AAAA%s stub@archipelago\n' "$t" "$tag" > "$r/etc/ssh/ssh_host_${t}_key.pub"
done
printf -- '-----BEGIN PRIVATE KEY-----\n%s\nSTUB_PUB=0\n-----END PRIVATE KEY-----\n' "$tag" > "$r/etc/archipelago/ssl/archipelago.key"
printf -- '-----BEGIN CERTIFICATE-----\n%s\nSTUB_PUB=0\n-----END CERTIFICATE-----\n' "$tag" > "$r/etc/archipelago/ssl/archipelago.crt"
touch -d "$(at "$when")" "$r"/etc/ssh/ssh_host_* \
"$r/etc/archipelago/ssl/archipelago.key" "$r/etc/archipelago/ssl/archipelago.crt"
}
put_anchor() {
local r="$1" when="$2"
: > "$r/var/lib/archipelago/.secrets-regenerated"
touch -d "$(at "$when")" "$r/var/lib/archipelago/.secrets-regenerated"
}
CASE_RC=0
CASE_OUT=""
CASE_ERR=""
run_script() {
local root="$1" name="$2"; shift 2
CASE_OUT="$WORK/$name.out"; CASE_ERR="$WORK/$name.err"
mkdir -p "$WORK/counters-$name"
set +e
env PATH="$STUBS:$PATH" \
HOST_SECRETS_ROOT="$root" \
STUB_OPENSSL_MODE="${STUB_OPENSSL_MODE:-ok}" \
STUB_SSHKEYGEN_MODE="${STUB_SSHKEYGEN_MODE:-ok}" \
STUB_COUNTER_DIR="$WORK/counters-$name" \
STUB_SYSTEMCTL_LOG="$WORK/$name.systemctl" \
bash "$SCRIPT" "$@" > "$CASE_OUT" 2> "$CASE_ERR"
CASE_RC=$?
set -e
}
verdict_of() { sed -n 's/.*"verdict": "\([^"]*\)".*/\1/p' "$1" | head -1; }
# A content+mtime+mode snapshot of everything except the script's own outputs,
# so "touched nothing" can be asserted as a whole-tree fact.
snapshot_tree() {
local r="$1"
( cd "$r" && find . -path ./var/lib/archipelago -prune -o \( -type f -o -type l \) -print0 \
| sort -z | xargs -0 -r stat -c '%n %s %Y %a' ) 2>/dev/null
( cd "$r" && find . -path ./var/lib/archipelago -prune -o -type f -print0 \
| sort -z | xargs -0 -r sha256sum ) 2>/dev/null
}
# ── Case 1: keys newer than the anchor -> per-node ────────────────────────
R=$(new_root per-node)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 + 5))" fresh
BEFORE=$(snapshot_tree "$R")
run_script "$R" per-node --detect
c=""
[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero"
J="$R/var/lib/archipelago/host-secrets-audit.json"
[ -f "$J" ] || c="$c no-json"
[ "$(verdict_of "$J" 2>/dev/null)" = "per-node" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)"
grep -q '"checked_at"' "$J" 2>/dev/null || c="$c no-checked-at"
grep -q '"ssh_host_key_fingerprints"' "$J" 2>/dev/null || c="$c no-fingerprints"
[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c detect-modified-the-tree"
if [ -z "$c" ]; then
ok "host keys newer than the anchor -> per-node, JSON written, nothing else changed"
else
bad "host keys newer than the anchor ->$c"; echo " root=$R rc=$CASE_RC"
fi
# ── Case 2: keys 30 days older than the anchor -> shared ─────────────────
R=$(new_root shared-mtime)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 - 30 * 86400))" baked
run_script "$R" shared-mtime --detect
c=""
J="$R/var/lib/archipelago/host-secrets-audit.json"
[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero"
[ "$(verdict_of "$J" 2>/dev/null)" = "shared" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)"
grep -q 'ssh_host_rsa_key mtime is' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-ssh-key"
grep -q 'archipelago.key mtime is' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-tls-key"
grep -qi 'SHARED' "$WORK/shared-mtime.out" || c="$c no-human-verdict-on-stdout"
if [ -z "$c" ]; then
ok "host keys 30 days older than the anchor -> shared, evidence names both key classes"
else
bad "host keys 30 days older than the anchor ->$c"; echo " root=$R rc=$CASE_RC"
fi
# ── Case 3: the fail-open fingerprint -> shared on direct evidence ───────
# Deliberately dated so the mtime signal says per-node. If this case passes it
# is because signal 2 fired, not because the timestamps happened to agree.
R=$(new_root fail-open)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 + 5))" kept-baked
{
echo "Mon Jun 1 12:00:00 UTC 2026: regenerating per-device secrets"
echo "Mon Jun 1 12:00:01 UTC 2026: WARNING: TLS regeneration failed, keeping baked key"
echo "Mon Jun 1 12:00:02 UTC 2026: WARNING: ssh-keygen -A failed, keeping baked host keys"
} > "$R/var/log/archipelago-first-boot-secrets.log"
run_script "$R" fail-open --detect
c=""
J="$R/var/lib/archipelago/host-secrets-audit.json"
[ "$(verdict_of "$J" 2>/dev/null)" = "shared" ] || c="$c verdict=$(verdict_of "$J" 2>/dev/null)"
grep -q '/var/lib/archipelago/.secrets-regenerated' "$J" 2>/dev/null || c="$c evidence-missing-marker-signal"
grep -q '/var/log/archipelago-first-boot-secrets.log' "$J" 2>/dev/null || c="$c evidence-missing-log-signal"
grep -q 'fail-open fingerprint' "$J" 2>/dev/null || c="$c evidence-does-not-name-the-combination"
if [ -z "$c" ]; then
ok "marker plus a WARNING: line -> shared, with both signals in evidence, despite per-node mtimes"
else
bad "marker plus a WARNING: line ->$c"; echo " root=$R rc=$CASE_RC"
fi
# ── Case 4: stripped rootfs, no host keys -> fail-closed-missing ─────────
# Not `shared`. The distinction is the whole reason signal 4 exists: on a
# 10-03-or-later node an absent key means generation never succeeded, which is
# fail-closed working, and rotating is not the remedy.
R=$(new_root stripped)
put_anchor "$R" "$T0"
printf 'F-03 identity strip\n' > "$R/opt/archipelago/rootfs-identity-stripped"
run_script "$R" stripped --detect
c=""
J="$R/var/lib/archipelago/host-secrets-audit.json"
[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero"
v=$(verdict_of "$J" 2>/dev/null)
[ "$v" = "fail-closed-missing" ] || c="$c verdict=$v"
[ "$v" = "shared" ] && c="$c CALLED-MISSING-MATERIAL-SHARED"
grep -q 'rootfs-identity-stripped' "$J" 2>/dev/null || c="$c evidence-missing-provenance"
if [ -z "$c" ]; then
ok "identity-stripped rootfs with no host keys -> fail-closed-missing, not shared"
else
bad "identity-stripped rootfs with no host keys ->$c"; echo " root=$R rc=$CASE_RC"
fi
# ── Case 5: no anchor at all -> unknown ──────────────────────────────────
# The signal that must never be guessed. An absent anchor is absence of
# evidence, and reporting per-node here would leave an exposed node looking
# clean (T-10-37).
R=$(new_root no-anchor)
put_material "$R" "$T0" whatever
: > "$R/etc/machine-id" # present but empty, as on a stripped rootfs
run_script "$R" no-anchor --detect
c=""
J="$R/var/lib/archipelago/host-secrets-audit.json"
v=$(verdict_of "$J" 2>/dev/null)
[ "$v" = "unknown" ] || c="$c verdict=$v"
[ "$v" = "per-node" ] && c="$c CLAIMED-PER-NODE-WITHOUT-EVIDENCE"
grep -q 'no anchor' "$J" 2>/dev/null || c="$c evidence-does-not-explain-why"
if [ -z "$c" ]; then
ok "no first-boot anchor -> unknown, never per-node"
else
bad "no first-boot anchor ->$c"; echo " root=$R rc=$CASE_RC"
fi
# ── Case 6: --apply without --yes is inert ───────────────────────────────
R=$(new_root dry-run)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 - 30 * 86400))" baked
BEFORE=$(snapshot_tree "$R")
BEFORE_STATE=$(ls -A "$R/var/lib/archipelago")
run_script "$R" dry-run --apply
c=""
[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero"
[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c TREE-CHANGED"
[ "$(ls -A "$R/var/lib/archipelago")" = "$BEFORE_STATE" ] || c="$c STATE-DIR-CHANGED"
[ -f "$R/var/lib/archipelago/host-key-rotation.json" ] && c="$c rotation-record-written"
ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover"
grep -qi 'DRY RUN' "$WORK/dry-run.out" || c="$c no-dry-run-notice"
grep -qi 'one-way' "$WORK/dry-run.out" || c="$c does-not-warn-that-it-is-one-way"
if [ -z "$c" ]; then
ok "--apply without --yes -> exits 0 and not one byte of the tree changes"
else
bad "--apply without --yes ->$c"; echo " root=$R rc=$CASE_RC"
# `diff` exits 1 when it finds differences, which under `set -o pipefail`
# would abort the run before the summary — i.e. a failing case would hide the
# other cases. Report and carry on.
diff <(echo "$BEFORE") <(snapshot_tree "$R") | head -10 || true
fi
# ── Case 7a: --apply --yes rotates, recording old fingerprints first ─────
R=$(new_root rotate)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 - 30 * 86400))" baked
OLD_SSH_SHA=$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key" | cut -d' ' -f1)
OLD_TLS_SHA=$(sha256sum "$R/etc/archipelago/ssl/archipelago.key" | cut -d' ' -f1)
# The fingerprint the old key WOULD produce, computed independently of the
# script, so the "old" half of the record is checked against an outside source.
OLD_FP_EXPECT=$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key.pub" | cut -c1-24)
run_script "$R" rotate --apply --yes
c=""
ROT="$R/var/lib/archipelago/host-key-rotation.json"
J="$R/var/lib/archipelago/host-secrets-audit.json"
[ "$CASE_RC" -eq 0 ] || c="$c exit-nonzero"
[ -f "$ROT" ] || c="$c no-rotation-record"
grep -q '"old_ssh_fingerprints"' "$ROT" 2>/dev/null || c="$c no-old-ssh-fingerprints"
grep -q '"new_ssh_fingerprints"' "$ROT" 2>/dev/null || c="$c no-new-ssh-fingerprints"
grep -q '"old_tls_sha256"' "$ROT" 2>/dev/null || c="$c no-old-tls"
grep -q '"new_tls_sha256"' "$ROT" 2>/dev/null || c="$c no-new-tls"
grep -q "$OLD_FP_EXPECT" "$ROT" 2>/dev/null || c="$c old-fingerprint-does-not-match-the-pre-rotation-key"
# ORDERING: the first systemctl call happens after the first swap, so the
# record must already exist by then.
FIRST_SYSTEMCTL=$(head -1 "$WORK/rotate.systemctl" 2>/dev/null || echo "")
case "$FIRST_SYSTEMCTL" in
*rotjson=yes) ;;
"") c="$c no-service-reload-happened" ;;
*) c="$c OLD-FINGERPRINTS-NOT-RECORDED-BEFORE-THE-SWAP[$FIRST_SYSTEMCTL]" ;;
esac
grep -q 'reload ssh' "$WORK/rotate.systemctl" 2>/dev/null || c="$c sshd-not-reloaded"
grep -q 'restart ssh' "$WORK/rotate.systemctl" 2>/dev/null && c="$c SSHD-RESTARTED"
grep -q 'reload nginx' "$WORK/rotate.systemctl" 2>/dev/null || c="$c nginx-not-reloaded"
# Material actually replaced.
[ "$(sha256sum "$R/etc/ssh/ssh_host_ed25519_key" | cut -d' ' -f1)" = "$OLD_SSH_SHA" ] && c="$c ssh-key-not-replaced"
[ "$(sha256sum "$R/etc/archipelago/ssl/archipelago.key" | cut -d' ' -f1)" = "$OLD_TLS_SHA" ] && c="$c tls-key-not-replaced"
ls "$R"/etc/ssh/ssh_host_*_key >/dev/null 2>&1 || c="$c NO-HOST-KEYS-LEFT"
ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover"
# The verdict file must reflect the post-rotation state, not the pre-rotation one.
[ "$(verdict_of "$J" 2>/dev/null)" = "per-node" ] || c="$c post-rotation-verdict=$(verdict_of "$J" 2>/dev/null)"
if [ -z "$c" ]; then
ok "--apply --yes -> old fingerprints recorded BEFORE the swap, keys replaced, sshd reloaded not restarted, verdict re-derived"
else
bad "--apply --yes ->$c"; echo " root=$R rc=$CASE_RC"
echo " stderr: $(head -c 300 "$WORK/rotate.err" 2>/dev/null)"
fi
# ── Case 7b: a failed generation aborts before touching anything live ────
# The failure mode that loses a remote node forever is a rotation that gets
# halfway. Force the SSH generator to fail after the TLS generator succeeded —
# the exact interleaving in which a naive implementation has already swapped
# the TLS pair — and require the live material to be byte-identical.
R=$(new_root abort)
put_anchor "$R" "$T0"
put_material "$R" "$((T0 - 30 * 86400))" baked
BEFORE=$(snapshot_tree "$R")
STUB_SSHKEYGEN_MODE=fail run_script "$R" abort --apply --yes
c=""
[ "$CASE_RC" -ne 0 ] || c="$c exit-zero-on-aborted-rotation"
[ "$(snapshot_tree "$R")" = "$BEFORE" ] || c="$c LIVE-MATERIAL-CHANGED-ON-AN-ABORTED-ROTATION"
[ -f "$R/var/lib/archipelago/host-key-rotation.json" ] && c="$c rotation-record-written-for-a-rotation-that-never-happened"
ls "$R"/etc/ssh/ssh_host_*_key >/dev/null 2>&1 || c="$c NO-HOST-KEYS-LEFT"
ls "$R"/etc/archipelago/ssl/*.rotnew >/dev/null 2>&1 && c="$c staging-leftover"
grep -qi 'ABORTED' "$WORK/abort.err" || c="$c no-loud-abort-on-stderr"
[ -s "$WORK/abort.systemctl" ] && c="$c reloaded-a-service-during-an-aborted-rotation"
if [ -z "$c" ]; then
ok "generation failure -> aborts before any swap; live keys byte-identical, no service reloaded"
else
bad "generation failure ->$c"; echo " root=$R rc=$CASE_RC"
# `diff` exits 1 when it finds differences, which under `set -o pipefail`
# would abort the run before the summary — i.e. a failing case would hide the
# other cases. Report and carry on.
diff <(echo "$BEFORE") <(snapshot_tree "$R") | head -10 || true
echo " stderr: $(head -c 300 "$WORK/abort.err" 2>/dev/null)"
fi
echo ""
echo "──────── host-secrets-audit summary ────────"
echo "passed: $PASS_COUNT failed: $FAIL_COUNT"
[ "$FAIL_COUNT" -eq 0 ]
tests/first-boot-secrets/run-tests.sh
Executable
+610
View File
@@ -0,0 +1,610 @@
#!/bin/bash
# Regression harness for the first-boot per-device secret regeneration script
# (audit finding F-03, phase 10 / KEY-02).
#
# What this pins, and why it exists at all: the script used to `touch` its
# completion marker unconditionally, outside both success branches, so one
# transient failure at first boot left the node running the ISO-wide shared
# SSH host key and TLS private key forever, silently. The property that must
# never regress is therefore negative — "on failure the marker is NOT created"
# — and a negative property is only assertable if the failure can be forced.
# So the generators are stubbed and the script is driven against a temp root
# through the FIRST_BOOT_SECRETS_ROOT seam.
#
# The script under test is not a file in this repo: it is a heredoc inside
# image-recipe/_archived/build-auto-installer-iso.sh (which is LIVE —
# image-recipe/build-debian-iso.sh execs it). The harness extracts the heredoc
# body between the SECRETSSCRIPT delimiters so it is testing the bytes that
# actually ship, not a copy that can drift.
#
# Usage: bash tests/first-boot-secrets/run-tests.sh
# Exit 0 only if all three cases PASS.
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
BUILDER="$REPO/image-recipe/_archived/build-auto-installer-iso.sh"
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
PASS_COUNT=0
FAIL_COUNT=0
ok() { echo "PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); }
bad() { echo "FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); }
# ── Step 0: extract the script under test and syntax-check it ────────────
[ -f "$BUILDER" ] || { echo "FAIL: builder not found at $BUILDER"; exit 1; }
SCRIPT="$WORK/first-boot-secrets.sh"
awk '/^cat > "\$WORK_DIR\/first-boot-secrets.sh" <<.SECRETSSCRIPT.$/ { f = 1; next }
f && /^SECRETSSCRIPT$/ { f = 0 }
f { print }' \
"$BUILDER" > "$SCRIPT"
if [ ! -s "$SCRIPT" ]; then
echo "FAIL: could not extract the first-boot-secrets.sh heredoc from the builder"
echo " (did the SECRETSSCRIPT delimiter or the cat> line change?)"
exit 1
fi
chmod +x "$SCRIPT"
if bash -n "$SCRIPT"; then
echo "extracted $(wc -l < "$SCRIPT") lines from the builder; bash -n clean"
else
echo "FAIL: extracted script does not parse"
exit 1
fi
# ── Stubs ─────────────────────────────────────────────────────────────────
# A stub dir is prepended to PATH so the script's openssl / ssh-keygen /
# systemctl / logger calls hit these instead of the real tools. Behaviour is
# driven by env vars the stubs read at call time.
#
# STUB_OPENSSL_MODE ok | fail (fail affects `req` only —
# `pkey`/`x509` validation still
# works, so a failed generation
# cannot be mistaken for a failed
# validation)
# STUB_SSHKEYGEN_MODE ok | fail | fail-twice (uses a counter file)
# STUB_OPENSSL_MISMATCH yes | no — emit a cert whose public key is
# from a DIFFERENT generation than
# the key beside it. Both halves
# still parse individually; only a
# pair check catches it.
# STUB_COUNTER_DIR where the counter file lives
# STUB_SYSTEMCTL_FAILED_UNITS units `systemctl is-failed` should report failed
# STUB_SYSTEMCTL_LOG file the systemctl stub appends its args to
make_stubs() {
local dir="$1"
mkdir -p "$dir"
cat > "$dir/openssl" <<'STUB'
#!/bin/bash
# Stub openssl. Honours -keyout/-out so the script's staging-then-swap and its
# non-empty checks are exercised for real, and implements the `pkey`/`x509`
# parse-back validation the generator does before it swaps.
sub="${1:-}"
# Capability probe. The script asks `openssl req -help` whether it can backdate.
if [ "$sub" = "req" ] && [ "${2:-}" = "-help" ]; then
[ "${STUB_OPENSSL_NOT_BEFORE:-yes}" = "yes" ] && echo " -not_before val stub"
exit 0
fi
case "$sub" in
pkey)
f=""; pubout=0
while [ $# -gt 0 ]; do
case "$1" in
-in) f="$2"; shift 2 ;;
-pubout) pubout=1; shift ;;
*) shift ;;
esac
done
[ -n "$f" ] && [ -s "$f" ] || exit 1
if [ "$pubout" = 1 ]; then
# The stub keypair carries the generation it came from; printing it
# as the "public key" is what lets the harness express a mismatched
# key/cert pair at all.
p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1
printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p"
fi
exit 0
;;
x509)
# Validation (-noout -in f) plus date readback. The stub cert carries
# the epochs it was minted with, so the harness can drive the script's
# date arithmetic without a real certificate.
f=""; want_start=0; want_end=0; want_pub=0
while [ $# -gt 0 ]; do
case "$1" in
-in) f="$2"; shift 2 ;;
-startdate) want_start=1; shift ;;
-enddate) want_end=1; shift ;;
-pubkey) want_pub=1; shift ;;
*) shift ;;
esac
done
[ -n "$f" ] && [ -s "$f" ] || exit 1
if [ "$want_pub" = 1 ]; then
p=$(sed -n 's/^STUB_PUB=//p' "$f"); [ -n "$p" ] || exit 1
printf -- '-----BEGIN PUBLIC KEY-----\nstubpub-%s\n-----END PUBLIC KEY-----\n' "$p"
fi
if [ "$want_start" = 1 ] || [ "$want_end" = 1 ]; then
nb=$(sed -n 's/^STUB_NOTBEFORE=//p' "$f"); na=$(sed -n 's/^STUB_NOTAFTER=//p' "$f")
[ -n "$nb" ] && [ -n "$na" ] || exit 1
[ "$want_start" = 1 ] && echo "notBefore=$(date -u -d "@$nb" '+%b %e %H:%M:%S %Y GMT')"
[ "$want_end" = 1 ] && echo "notAfter=$(date -u -d "@$na" '+%b %e %H:%M:%S %Y GMT')"
fi
exit 0
;;
esac
# Count real mints. Comparing certificate dates cannot detect a re-mint when
# the clock is frozen — the second cert carries the same notBefore — so the
# anti-spin assertions count invocations instead.
reqcount="${STUB_COUNTER_DIR:-/tmp}/openssl-req.count"
rn=$(cat "$reqcount" 2>/dev/null || echo 0)
echo $((rn + 1)) > "$reqcount"
[ "${STUB_OPENSSL_MODE:-ok}" = "fail" ] && exit 1
keyout="" out="" nb="" na="" days=""
while [ $# -gt 0 ]; do
case "$1" in
-keyout) keyout="$2"; shift 2 ;;
-out) out="$2"; shift 2 ;;
-not_before) nb="$2"; shift 2 ;;
-not_after) na="$2"; shift 2 ;;
-days) days="$2"; shift 2 ;;
*) shift ;;
esac
done
# Mirror openssl: -not_before/-not_after win; otherwise notBefore is "now" and
# notAfter is now + days. "now" honours the harness's fake clock.
now="${FIRST_BOOT_SECRETS_NOW:-$(date -u +%s)}"
# YYYYMMDDHHMMSSZ -> something GNU date can parse
asn1() { printf '%s' "$1" | sed -E 's/^([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})Z?$/\1-\2-\3 \4:\5:\6 UTC/'; }
if [ -n "$nb" ]; then nb_epoch=$(date -u -d "$(asn1 "$nb")" +%s 2>/dev/null || echo "$now"); else nb_epoch="$now"; fi
if [ -n "$na" ]; then na_epoch=$(date -u -d "$(asn1 "$na")" +%s 2>/dev/null || echo $((now + 315360000))); else na_epoch=$((now + ${days:-3650} * 86400)); fi
# Each generation gets its own public-key identity. STUB_OPENSSL_MISMATCH makes
# the cert carry a different one, i.e. a cert from another generation beside
# this key — the pair that passes both individual parse checks and still breaks
# nginx.
gen_id=$((rn + 1))
key_pub="$gen_id"
crt_pub="$gen_id"
[ "${STUB_OPENSSL_MISMATCH:-no}" = "yes" ] && crt_pub="$((gen_id + 1000))"
[ -n "$keyout" ] && printf -- '-----BEGIN PRIVATE KEY-----\nstub\nSTUB_PUB=%s\n-----END PRIVATE KEY-----\n' "$key_pub" > "$keyout"
[ -n "$out" ] && printf -- '-----BEGIN CERTIFICATE-----\nstub\nSTUB_PUB=%s\nSTUB_NOTBEFORE=%s\nSTUB_NOTAFTER=%s\n-----END CERTIFICATE-----\n' "$crt_pub" "$nb_epoch" "$na_epoch" > "$out"
exit 0
STUB
cat > "$dir/ssh-keygen" <<'STUB'
#!/bin/bash
# Stub ssh-keygen -A: writes a host-key set into <-f dir>/etc/ssh, matching
# the real tool's layout, which is what the script globs for.
mode="${STUB_SSHKEYGEN_MODE:-ok}"
counter="${STUB_COUNTER_DIR:-/tmp}/ssh-keygen.count"
n=$(cat "$counter" 2>/dev/null || echo 0)
n=$((n + 1))
echo "$n" > "$counter"
case "$mode" in
fail) exit 1 ;;
fail-twice) [ "$n" -le 2 ] && exit 1 ;;
esac
root=""
while [ $# -gt 0 ]; do
case "$1" in
-f) root="$2"; shift 2 ;;
*) shift ;;
esac
done
[ -n "$root" ] || exit 1
mkdir -p "$root/etc/ssh"
for t in rsa ecdsa ed25519; do
printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nstub-%s\n' "$t" > "$root/etc/ssh/ssh_host_${t}_key"
printf -- 'ssh-%s AAAAstub stub@archipelago\n' "$t" > "$root/etc/ssh/ssh_host_${t}_key.pub"
done
exit 0
STUB
cat > "$dir/systemctl" <<'STUB'
#!/bin/bash
# Stub systemctl. Records every invocation so the harness can prove the
# self-heal path actually restarts a unit that failed for want of a key, and
# reports is-failed honestly so first-boot and self-heal take different paths.
[ -n "${STUB_SYSTEMCTL_LOG:-}" ] && echo "$*" >> "$STUB_SYSTEMCTL_LOG"
if [ "${1:-}" = "is-failed" ]; then
unit="${!#}"
for u in ${STUB_SYSTEMCTL_FAILED_UNITS:-}; do
[ "$u" = "$unit" ] && exit 0
done
exit 1
fi
exit 0
STUB
# Must not be allowed to touch the host journal during a test run.
printf '#!/bin/bash\nexit 0\n' > "$dir/logger"
chmod +x "$dir"/openssl "$dir"/ssh-keygen "$dir"/systemctl "$dir"/logger
}
STUBS="$WORK/stubs"
make_stubs "$STUBS"
# ── Runner ────────────────────────────────────────────────────────────────
# run_case <name> <openssl_mode> <sshkeygen_mode> [prestage] [reuse]
#
# prestage baked — root already holds the shared keys, i.e. a pre-strip
# rootfs. Assertions can then prove the swap replaced them.
# stripped — root holds no key material at all, i.e. the rootfs this
# build actually ships. This is the state in which "no key
# may appear from anywhere but the generator" is testable.
# reuse 1 — do not wipe the root or the counters; continue from the
# previous run against the same node. Models a reboot or a
# timer-triggered retry.
CASE_ROOT=""
CASE_RC=0
CASE_SYSTEMCTL_LOG=""
run_case() {
local name="$1" openssl_mode="$2" sshkeygen_mode="$3"
local prestage="${4:-baked}" reuse="${5:-0}"
CASE_ROOT="$WORK/root-$name"
CASE_SYSTEMCTL_LOG="$WORK/$name.systemctl"
if [ "$reuse" != "1" ]; then
rm -rf "$CASE_ROOT"
mkdir -p "$CASE_ROOT/var/lib/archipelago" "$CASE_ROOT/var/log" \
"$CASE_ROOT/etc/ssh" "$CASE_ROOT/etc/archipelago/ssl"
if [ "$prestage" = "baked" ]; then
echo "BAKED-SHARED-HOST-KEY" > "$CASE_ROOT/etc/ssh/ssh_host_rsa_key"
echo "BAKED-SHARED-TLS-KEY" > "$CASE_ROOT/etc/archipelago/ssl/archipelago.key"
fi
rm -f "$WORK/counters-$name/ssh-keygen.count"
mkdir -p "$WORK/counters-$name"
: > "$CASE_SYSTEMCTL_LOG"
fi
set +e
env PATH="$STUBS:$PATH" \
FIRST_BOOT_SECRETS_ROOT="$CASE_ROOT" \
FIRST_BOOT_SECRETS_BACKOFF="0 0 0" \
STUB_OPENSSL_MODE="$openssl_mode" \
STUB_SSHKEYGEN_MODE="$sshkeygen_mode" \
STUB_COUNTER_DIR="$WORK/counters-$name" \
STUB_SYSTEMCTL_LOG="$CASE_SYSTEMCTL_LOG" \
STUB_SYSTEMCTL_FAILED_UNITS="${STUB_SYSTEMCTL_FAILED_UNITS:-}" \
STUB_OPENSSL_NOT_BEFORE="${STUB_OPENSSL_NOT_BEFORE:-yes}" \
STUB_OPENSSL_MISMATCH="${STUB_OPENSSL_MISMATCH:-no}" \
FIRST_BOOT_SECRETS_NOW="${FIRST_BOOT_SECRETS_NOW:-}" \
bash "$SCRIPT" > "$WORK/$name.out" 2> "$WORK/$name.err"
CASE_RC=$?
set -e
}
fail_detail() {
echo " exit=$CASE_RC root=$CASE_ROOT"
echo " stderr: $(head -c 300 "$WORK/$1.err" 2>/dev/null)"
}
# ── Case 1: both generators succeed ──────────────────────────────────────
run_case both-ok ok ok
c1=""
[ "$CASE_RC" -eq 0 ] || c1="$c1 exit-nonzero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c1="$c1 marker-missing"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c1="$c1 stale-failure-record"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] || c1="$c1 tls-key-missing"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c1="$c1 tls-crt-missing"
grep -q BAKED-SHARED-TLS-KEY "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" && c1="$c1 tls-key-not-replaced"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c1="$c1 ssh-host-key-missing"
grep -q BAKED-SHARED-HOST-KEY "$CASE_ROOT/etc/ssh/ssh_host_rsa_key" && c1="$c1 ssh-key-not-replaced"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c1="$c1 dotnew-leftover"
if [ -z "$c1" ]; then
ok "both generators succeed -> exit 0, marker set, keys swapped in"
else
bad "both generators succeed ->$c1"; fail_detail both-ok
fi
# ── Case 2: openssl fails every attempt -> FAIL CLOSED ───────────────────
# This is the case that would have passed against the old script and is the
# whole reason this harness exists: the old code logged a warning and set the
# marker anyway.
run_case tls-fail fail ok
c2=""
[ "$CASE_RC" -ne 0 ] || c2="$c2 exit-zero-on-failure"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c2="$c2 MARKER-SET-ON-FAILURE"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c2="$c2 no-failure-record"
grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \
|| c2="$c2 failure-record-does-not-name-TLS"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c2="$c2 dotnew-leftover"
grep -qi 'FAILED' "$WORK/tls-fail.err" || c2="$c2 no-loud-stderr"
if [ -z "$c2" ]; then
ok "openssl fails every attempt -> exit non-zero, NO marker, failure record names TLS"
else
bad "openssl fails every attempt ->$c2"; fail_detail tls-fail
fi
# ── Case 3: ssh-keygen fails twice then succeeds -> backoff recovers ─────
run_case ssh-flaky ok fail-twice
c3=""
[ "$CASE_RC" -eq 0 ] || c3="$c3 exit-nonzero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c3="$c3 marker-missing"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c3="$c3 failure-record-present"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c3="$c3 ssh-host-key-missing"
attempts=$(cat "$WORK/counters-ssh-flaky/ssh-keygen.count" 2>/dev/null || echo 0)
[ "$attempts" -eq 3 ] || c3="$c3 expected-3-attempts-got-$attempts"
if [ -z "$c3" ]; then
ok "ssh-keygen fails twice then succeeds -> backoff recovers within one boot (3 attempts)"
else
bad "ssh-keygen fails twice then succeeds ->$c3"; fail_detail ssh-flaky
fi
# ── Case 4: TLS fails every attempt on a STRIPPED root -> no key at all ──
# Case 2 proves the marker is not set. This proves the stronger property that
# replaced the installer's TLS fallback: on the rootfs we actually ship, a
# failed generation leaves NO key, from any source. If anything ever mints a
# key outside gen_tls — an install-time fallback, a placeholder, a zero-length
# touch to keep nginx happy — this is the case that goes red.
run_case tls-fail-stripped fail ok stripped
c4=""
[ "$CASE_RC" -ne 0 ] || c4="$c4 exit-zero-on-failure"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c4="$c4 MARKER-SET-ON-FAILURE"
[ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] && c4="$c4 TLS-KEY-EXISTS-AFTER-FAILURE"
[ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] && c4="$c4 TLS-CRT-EXISTS-AFTER-FAILURE"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c4="$c4 no-failure-record"
grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \
|| c4="$c4 failure-record-does-not-name-TLS"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c4="$c4 dotnew-leftover"
if [ -z "$c4" ]; then
ok "TLS fails every attempt on a stripped root -> NO key, NO marker, non-zero exit, record names TLS"
else
bad "TLS fails every attempt on a stripped root ->$c4"; fail_detail tls-fail-stripped
fi
# ── Case 5: self-heal — a failed run, then a later run that succeeds ─────
# The case that proves a node is not permanently dead. Run 1 is a machine whose
# generator fails every retry; run 2 is the same machine minutes later, once the
# transient cause cleared, driven by archipelago-first-boot-secrets.timer. It
# must end with the key present and the marker set, with no human at a console.
# Run 2 also declares nginx/ssh already `failed` — they tried to start without a
# key — so the run must actively restart them, not just reload. A "recovery"
# that leaves the services down is not a recovery.
#
# Run 1 asserts only enough to establish the precondition (it really did fail,
# and it left the node eligible to retry). Whether a key exists after a failure
# is case 4's job — asserting it here too would make a single defect light up
# two cases and blunt the signal.
run_case self-heal fail ok stripped
c5=""
[ "$CASE_RC" -ne 0 ] || c5="$c5 run1-exit-zero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c5="$c5 run1-marker-set"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c5="$c5 run1-no-failure-record"
STUB_SYSTEMCTL_FAILED_UNITS="nginx ssh" run_case self-heal ok ok stripped 1
[ "$CASE_RC" -eq 0 ] || c5="$c5 run2-exit-nonzero"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c5="$c5 run2-marker-missing"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] || c5="$c5 run2-key-missing"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c5="$c5 run2-crt-missing"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c5="$c5 run2-ssh-key-missing"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c5="$c5 run2-stale-failure-record"
grep -q 'restart nginx' "$CASE_SYSTEMCTL_LOG" 2>/dev/null || c5="$c5 run2-did-not-restart-failed-nginx"
if [ -z "$c5" ]; then
ok "self-heal: failed run then a later successful run -> key present, marker set, failed units restarted"
else
bad "self-heal ->$c5"; fail_detail self-heal
fi
# ── Case 6: single-producer invariant ────────────────────────────────────
# The regression that would silently recreate F-03 is not a broken assertion —
# it is somebody adding a second, well-meaning place that mints a key. A second
# producer brings its own idea of success, its own absent retry policy and its
# own absent failure record, and that is what made F-03 silent.
#
# So: every executable key-creating invocation in the builder must live inside
# the first-boot-secrets.sh heredoc, i.e. inside gen_tls/gen_ssh. Comments are
# exempt (they discuss the history); binary-existence checks are not matched
# because they do not carry a key-creating subcommand.
c6=""
SS_START=$(grep -n '^cat > "\$WORK_DIR/first-boot-secrets.sh" <<.SECRETSSCRIPT.$' "$BUILDER" | cut -d: -f1)
SS_END=$(awk -v s="$SS_START" 'NR>s && /^SECRETSSCRIPT$/ { print NR; exit }' "$BUILDER")
if [ -z "$SS_START" ] || [ -z "$SS_END" ]; then
c6="$c6 could-not-locate-generator-heredoc"
else
PRODUCERS=$(grep -nE 'openssl[[:space:]]+req|ssh-keygen[[:space:]]+-A|ssh-keygen[[:space:]]+-t' "$BUILDER" \
| grep -vE '^[0-9]+:[[:space:]]*#' || true)
while IFS= read -r line; do
[ -z "$line" ] && continue
ln=${line%%:*}
if [ "$ln" -lt "$SS_START" ] || [ "$ln" -gt "$SS_END" ]; then
c6="$c6 SECOND-PRODUCER-at-line-$ln"
fi
done <<< "$PRODUCERS"
# Sanity: the one producer we expect must actually be in there, otherwise an
# empty result would pass this case vacuously.
echo "$PRODUCERS" | grep -q 'openssl[[:space:]]*req' || c6="$c6 no-tls-producer-found-at-all"
echo "$PRODUCERS" | grep -q 'ssh-keygen' || c6="$c6 no-ssh-producer-found-at-all"
fi
if [ -z "$c6" ]; then
ok "single-producer invariant: every key-creating invocation is inside gen_tls/gen_ssh"
else
bad "single-producer invariant ->$c6"
echo " generator heredoc spans lines $SS_START-$SS_END of $BUILDER"
fi
# ── Case 7: the Dockerfile heredoc delimiter must be quoted ──────────────
# Lives in this harness rather than a sibling because it guards the same file
# and the same failure mode the rest of these cases exist for: a build-side
# defect that is invisible to `bash -n` and only shows up as damage on a build
# host. Splitting it into its own runner would mean two commands to remember
# and one of them getting skipped.
#
# The bug: `cat > ... <<DOCKERFILE` (unquoted) makes the build shell perform
# command substitution on the Dockerfile body, so a backtick inside a COMMENT
# is executed on the build host and its output spliced into the Dockerfile.
# Six comments did exactly that, and one of them ran `systemctl start
# archipelago-fips.service` against the build machine on every ISO build. The
# comment text was silently deleted from the generated Dockerfile too.
#
# The assertion is on the DELIMITER, not on backticks. With a quoted delimiter
# a backticked comment is inert and perfectly legal — six of them are back in
# the body on purpose. Flagging backticks would be flagging a non-bug, and
# would fail on the very comments this fix restored. Quoting is the fix;
# vigilance about backticks is not.
c7=""
DF_HEREDOCS=$(grep -nE 'cat >>? "\$WORK_DIR/Dockerfile\.rootfs" <<' "$BUILDER" || true)
if [ -z "$DF_HEREDOCS" ]; then
c7="$c7 no-dockerfile-heredoc-found"
else
while IFS= read -r hd; do
[ -z "$hd" ] && continue
ln=${hd%%:*}
delim=$(printf '%s' "$hd" | sed -E 's/.*<<-?[[:space:]]*//')
case "$delim" in
\'*\'|\"*\")
: ;; # quoted — the body is emitted verbatim, nothing executes
*)
c7="$c7 UNQUOTED-DELIMITER-at-line-$ln"
# Only meaningful when unquoted: report what would actually run.
bare=$(printf '%s' "$delim" | tr -d "\"'")
endln=$(awk -v s="$ln" -v d="$bare" 'NR>s && $0==d { print NR; exit }' "$BUILDER")
if [ -n "$endln" ]; then
subs=$(awk -v s="$ln" -v e="$endln" 'NR>s && NR<e && (/`/ || /\$\(/) { print NR }' "$BUILDER" | tr '\n' ',')
[ -n "$subs" ] && c7="$c7 would-execute-at-lines:${subs%,}"
fi
;;
esac
done <<< "$DF_HEREDOCS"
fi
if [ -z "$c7" ]; then
ok "Dockerfile heredoc delimiters are quoted — a backticked comment cannot execute"
else
bad "Dockerfile heredoc quoting ->$c7"
fi
# ── Case 8: a cert minted under a wrong clock is detected and repaired ───
# The failure fail-closed cannot catch, because generation SUCCEEDS. This unit
# runs before chrony has corrected the clock; on a node with a dead RTC,
# `openssl req -x509` stamps a notBefore years out. Clock ahead -> clients
# reject the cert as "not yet valid"; clock behind -> notAfter is already in
# the past once time syncs. The old code would have marked the node done and
# never revisited it.
#
# Run 1 mints under a clock set to 2013 (a classic dead-RTC value). The node
# must still be usable — keys installed, marker set, exit 0 — but the bad dates
# must be recorded, not blessed.
# Run 2 is the same node after chrony fixes the clock: the cert must be
# regenerated with sane dates and the record cleared, with nobody at a console.
# Run 3 proves the anti-spin guard: a third run changes nothing.
BAD_CLOCK=1370000000 # 2013-06-01, i.e. a dead RTC
GOOD_CLOCK=1785000000 # 2026-07-25, inside the plausible window
c8=""
FIRST_BOOT_SECRETS_NOW="$BAD_CLOCK" run_case clock ok ok stripped
[ "$CASE_RC" -eq 0 ] || c8="$c8 run1-exit-nonzero"
[ -s "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] || c8="$c8 run1-no-cert-node-unusable"
[ -s "$CASE_ROOT/etc/ssh/ssh_host_ed25519_key" ] || c8="$c8 run1-no-ssh-key"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] || c8="$c8 run1-marker-missing"
grep -q 'failed=cert-dates' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \
|| c8="$c8 run1-BAD-DATES-NOT-RECORDED"
run1_nb=$(sed -n 's/^STUB_NOTBEFORE=//p' "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" 2>/dev/null)
FIRST_BOOT_SECRETS_NOW="$GOOD_CLOCK" run_case clock ok ok stripped 1
[ "$CASE_RC" -eq 0 ] || c8="$c8 run2-exit-nonzero"
run2_nb=$(sed -n 's/^STUB_NOTBEFORE=//p' "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" 2>/dev/null)
[ -n "$run2_nb" ] || c8="$c8 run2-cert-unreadable"
[ "$run2_nb" != "$run1_nb" ] || c8="$c8 CERT-NOT-REGENERATED-AFTER-CLOCK-FIX"
if [ -n "$run2_nb" ]; then
[ "$run2_nb" -ge 1767225600 ] || c8="$c8 run2-notBefore-still-below-floor"
# backdated, but not into the implausible past
[ "$run2_nb" -le "$GOOD_CLOCK" ] || c8="$c8 run2-notBefore-in-the-future"
[ "$run2_nb" -lt "$GOOD_CLOCK" ] || c8="$c8 run2-notBefore-not-backdated"
fi
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] && c8="$c8 run2-stale-bad-date-record"
# Anti-spin. Counted, not date-compared: with a frozen clock a re-mint produces
# a byte-identical notBefore, so dates cannot tell "left alone" from
# "regenerated again". Counting mints is the only assertion that distinguishes
# them — the first version of this check compared dates and sailed straight
# past a deliberately broken anti-spin guard.
mints() { cat "$WORK/counters-$1/openssl-req.count" 2>/dev/null || echo 0; }
# A further run with a good clock and a good cert must NOT mint again.
before3=$(mints clock)
FIRST_BOOT_SECRETS_NOW="$GOOD_CLOCK" run_case clock ok ok stripped 1
[ "$(mints clock)" -eq "$before3" ] || c8="$c8 SPINNING-reminted-a-good-cert"
# And a node whose clock stays wrong must not mint a fresh bad cert on every
# timer tick — the loop the fix must not introduce.
FIRST_BOOT_SECRETS_NOW="$BAD_CLOCK" run_case clockstuck ok ok stripped
stuck1=$(mints clockstuck)
FIRST_BOOT_SECRETS_NOW="$BAD_CLOCK" run_case clockstuck ok ok stripped 1
stuck2=$(mints clockstuck)
[ "$stuck2" -eq "$stuck1" ] || c8="$c8 SPINNING-reminted-while-clock-still-wrong($stuck1->$stuck2)"
if [ -z "$c8" ]; then
ok "wrong clock: cert flagged not blessed, regenerated once time syncs, and no spin either way"
else
bad "wrong clock ->$c8"; fail_detail clock
fi
# ── Case 9: a key and a cert that are not a pair ─────────────────────────
# The second failure that generation-succeeded hides. Parsing each half back
# proves each is well-formed, never that they belong together; a key from one
# generation beside a cert from another passes both individual parse checks,
# gets blessed, and then nginx refuses to start at the exact moment the marker
# claims first-boot succeeded. Only comparing the two public keys catches it.
c9=""
# 9a — the mismatch arises during generation: fail closed, exactly like any
# other TLS failure. Nothing installed, nothing blessed, no .new left behind.
STUB_OPENSSL_MISMATCH=yes run_case mismatch-gen ok ok stripped
[ "$CASE_RC" -ne 0 ] || c9="$c9 gen-exit-zero-on-mismatched-pair"
[ -f "$CASE_ROOT/var/lib/archipelago/.secrets-regenerated" ] && c9="$c9 gen-MARKER-SET-ON-MISMATCH"
[ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" ] && c9="$c9 gen-MISMATCHED-KEY-INSTALLED"
[ -e "$CASE_ROOT/etc/archipelago/ssl/archipelago.crt" ] && c9="$c9 gen-MISMATCHED-CRT-INSTALLED"
[ -f "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" ] || c9="$c9 gen-no-failure-record"
grep -q 'failed=.*TLS' "$CASE_ROOT/var/lib/archipelago/first-boot-secrets.failed" 2>/dev/null \
|| c9="$c9 gen-failure-record-does-not-name-TLS"
ls "$CASE_ROOT"/etc/archipelago/ssl/*.new >/dev/null 2>&1 && c9="$c9 gen-dotnew-leftover"
# 9b — the mismatch is already on disk from somewhere else: an older build, a
# half-finished manual edit. The node is "done" by every marker, so only the
# needs_tls pair check can notice. One regeneration must repair it.
run_case mismatch-disk ok ok stripped
[ "$CASE_RC" -eq 0 ] || c9="$c9 disk-setup-run-failed"
disk_crt="$CASE_ROOT/etc/archipelago/ssl/archipelago.crt"
# Swap in a cert from a different generation, leaving the dates untouched so
# this can only trip the pair check and not the clock check.
sed -i 's/^STUB_PUB=.*/STUB_PUB=999999/' "$disk_crt"
before9=$(mints mismatch-disk)
run_case mismatch-disk ok ok stripped 1
[ "$(mints mismatch-disk)" -gt "$before9" ] || c9="$c9 MISMATCHED-PAIR-ON-DISK-NOT-REPAIRED"
[ "$CASE_RC" -eq 0 ] || c9="$c9 disk-repair-exit-nonzero"
key_pub=$(sed -n 's/^STUB_PUB=//p' "$CASE_ROOT/etc/archipelago/ssl/archipelago.key" 2>/dev/null)
crt_pub=$(sed -n 's/^STUB_PUB=//p' "$disk_crt" 2>/dev/null)
[ -n "$key_pub" ] && [ "$key_pub" = "$crt_pub" ] || c9="$c9 pair-still-mismatched-after-repair($key_pub/$crt_pub)"
# Anti-spin: the repaired pair matches, so a further run must mint nothing.
before9b=$(mints mismatch-disk)
run_case mismatch-disk ok ok stripped 1
[ "$(mints mismatch-disk)" -eq "$before9b" ] || c9="$c9 SPINNING-reminted-a-matching-pair"
if [ -z "$c9" ]; then
ok "mismatched key/cert: fails closed when generated, repaired once when found on disk, no spin"
else
bad "mismatched key/cert ->$c9"; fail_detail mismatch-gen
fi
# ── Summary ───────────────────────────────────────────────────────────────
echo
echo "──────── first-boot-secrets summary ────────"
echo "passed: $PASS_COUNT failed: $FAIL_COUNT"
[ "$FAIL_COUNT" -eq 0 ] || exit 1
exit 0
tests/lifecycle/TESTING.md
+263
View File
@@ -0,0 +1,263 @@
# Container subsystem testing — scorecard and roadmap
The bar (verbatim from the v1.7.52 owner):
> "best performant, minimal code, tested containers possible in the world.
> No bloated code, no problems installing a single one, no problems
> uninstalling, every one needs to be tested 20+ times in every state
> before we make another update, not a single container failure outside
> of hardware or internet failure is allowed."
This document is the live tracker for whether we're meeting that bar.
Every PR that touches the container subsystem updates the scoreboard
below. **If you can't honestly tick the box, the change isn't ready.**
---
## Production-quality pass — 2026-06-21 (current, v1.7.99-alpha)
The migration's aim, restated as **five pillars** (every app must satisfy all five):
1. **Quadlet-everywhere** — every container is a declarative systemd Quadlet
unit under `user.slice`, never inside `archipelago.service`'s cgroup. Kills
FM3 (restarting/updating archipelago SIGKILLs every container in its cgroup);
systemd becomes the per-app supervisor.
2. **Level-triggered reconciler** — a 30s idempotent reconcile loop drives
desired→current from manifests + secrets. Self-healing, not edge-triggered.
3. **Lifecycle bulletproof** — every app passes the full matrix
(install / UI reachable / stop / start / restart / reinstall / reboot-survive
/ archipelago-restart-survive / uninstall) **5× green on .228** — run ON the node
(`ARCHY_ITERATIONS=5`).
(Multinode / fleet testing is tracked separately.)
before any release.
4. **Data-driven apps** — install/uninstall needs only the app's manifest +
catalog entry. **No host OS changes** (no apt, no /etc, no host units) and
**no archipelago binary code per app**. Only *core* apps (bitcoin, lnd,
electrumx, fedimint + gateway/clientd) may carry bespoke handling if truly
unavoidable.
5. **Rootless + security-first (non-negotiable)** — containers run in the
unprivileged `archipelago` user namespace; never root, no `--privileged`,
drop-all-caps + add-back only what a manifest declares. Secrets are `0600`,
owned by the service user. Security is king.
**Per-app definition of done:** all five pillars hold → lifecycle matrix 5×
green on .228 (run ON the node) → catalog/registry updated (`app-catalog/catalog.json`
+ `releases/app-catalog.json`, rebuilt image pushed to the mirror) → tracker
cell ticked. Only then move to the next app. (Fleet/multinode verification is a
separate pass, tracked internally.)
**.228 testing constraint:** do NOT touch `bitcoin-knots`, `electrumx`, or
`lnd` on .228 — they are synced and healthy; destructive cycles there would
cost hours of resync.
### Session work log (resolved)
The 2026-06-21 mid-session resume block that lived here (generated-secrets
rollout for fedimint-gateway/-clientd, icon/naming fixes) is **done and
shipped**: the generated-secrets system is a platform primitive
(`container.generated_secrets`, see `docs/app-manifest-spec.md`), the
manifests declare it, and the single-node gate went green on .228 on
2026-06-23. Day-to-day open items live in the issue tracker —
don't add session logs here.
---
## Test layers
| Layer | What it asserts | Toolchain | Latency / iteration |
|---|---|---|---|
| L0 — Rust unit | Pure-function behaviour (manifest parsing, secret resolution, structural invariants) | `cargo test --workspace --bins` | ~5s |
| L1 — RPC API | The JSON-RPC API responds correctly per app (`container-list`, `package.{install,start,stop,restart,uninstall}`, `bitcoin.getinfo`, etc.) | bats + lib/rpc.bash | ~30s per suite |
| L2 — UI surface | The URLs a user actually clicks (dashboard, `/app/<id>/`, direct-port iframes) return 200 with non-empty bodies | bats + lib/ui-probes.bash | ~10s per suite |
| L3 — Lifecycle survival | Containers survive operational events (archipelago restart, host reboot, kill -9 mid-install, OOM) | bats (gated) | ~60s per scenario |
| L4 — Browser journey | Real DOM-level user flow (login → install → wait → click → use) | playwright (TBD) | ~30-120s per journey |
| L5 — Chaos / failure-path | Failure modes recover gracefully (corrupt config, deleted bolt DB, network partition) | bats (chaos-gated) | ~120s per scenario |
| L6 — Performance | Cold install latency, reconcile-tick cost, podman call count per lifecycle event | timed bats + Prometheus (TBD) | ~60s per benchmark |
Release gate: **L0+L1+L2+L3 green × 20 iterations** on .228 (run ON the node; 5× for
now). Multinode/fleet testing is a separate pass. L4+L5+L6 are quality gates
we add as they mature; not blocking the v1.7.52 tag.
## Coverage matrix — current state
Legend: ● fully covered, ◐ partial, ○ missing
### Per-app × per-state matrix (L1 + L2)
| App | Container present | Valid state | RPC reachable | UI URL 200 | Stop | Start | Restart | Reinstall | Reboot survives | Archipelago-restart survives |
|---|---|---|---|---|---|---|---|---|---|---|
| bitcoin-knots | ● | ● | ● | ● (port 8334) | ● | ● | ● | ● | ○ | ◐ regression-gate only |
| bitcoin-core | ◐ shares with knots | ◐ | ○ | ◐ | ○ | ○ | ○ | ○ | ○ | ◐ regression-gate |
| lnd | ● | ● | ● (lncli) | ● (`/app/lnd/`) | ● | ● | ● | ● | ○ | ◐ regression-gate |
| electrumx | ● | ● | ● (TCP 50001) | ● (`/app/electrumx/`) | ● | ● | ● | ● | ○ | ◐ regression-gate |
| btcpay-server | ● | ● | ◐ frontend-port | ● (`/app/btcpay/`) | ● | ● | ● | ● | ○ | ○ |
| mempool | ● | ● | ● (`/api/v1/backend-info`) | ● (`/app/mempool/`) | ● | ● | ● | ● | ○ | ○ |
| fedimint | ● | ● | ◐ container-only | ● (`/app/fedimint/`) | ● | ● | ● | ● | ○ | ○ |
| filebrowser | ○ | ○ | ○ | ● probe-only | ○ | ○ | ○ | ○ | ○ | ◐ via companions |
| archy-bitcoin-ui | ◐ via companions | ◐ | n/a | ● (port 8334) | ○ | ○ | ○ | n/a | ◐ via companions | ● |
| archy-lnd-ui | ◐ via companions | ◐ | n/a | ● (`/app/lnd/`) | ○ | ○ | ○ | n/a | ◐ via companions | ● |
| archy-electrs-ui | ◐ via companions | ◐ | n/a | ● (`/app/electrumx/`) | ○ | ○ | ○ | n/a | ◐ via companions | ● |
Done: 50 of 110 cells. Goal: 110/110 ● for the listed apps before
v1.7.52 tags.
### Layer-by-layer status
| Layer | Tests | Suites | Status |
|---|---:|---:|---|
| L0 unit | 631 | n/a | ● green |
| L1 RPC | 70 | bitcoin-knots, lnd, electrumx, btcpay, mempool, fedimint, required-stack, package-update-smoke | ● for the 6 core apps |
| L2 UI | 9 | ui-coverage | ● for dashboard + 7 proxy paths + bitcoin-ui:8334 |
| L3 lifecycle survival | 14 | companion-survives-archipelago-restart, backend-survives-archipelago-restart, required-stack-destructive, use-quadlet-backends-install | ◐ companions ● ; backends ◐ regression-gate (will fail until Phase 3 Quadlet ships); quadlet post-condition gate ✅ skip-clean today, hard gate when flag flipped |
| L1 wallet-receive / drift / secrets | 5 | bitcoin-receive, port-drift, secret-completeness | ● guards the v1.7.9x wallet fleet failures |
| L4 browser journey | 0 | none | ○ not started |
| L5 chaos | 0 | none | ○ not started |
| L6 performance | 0 | none | ○ not started |
### Wallet / Bitcoin fleet-failure regression suites (added after v1.7.90-alpha)
Three production failures shipped on v1.7.90-alpha despite the existing harness,
because nothing exercised the receive path, port-mapping drift, or secret
completeness on a live node. New suites close those gaps (all run on the archy
host, read-only, so they join `run.sh`/`run-gate.sh` automatically):
| Suite | Failure it guards | Asserts |
|---|---|---|
| `bitcoin-receive.bats` | .116 ("Operation failed" on receive) and .228 (false "wallet is locked") | LND REST reachable on the **manifest** host port; `lnd.newaddress` returns a `bc1…` address on a running node; receive errors are specific, never the generic catch-all |
| `port-drift.bats` | .116 (lnd REST stuck on host 8080 vs manifest 18080) | every installed backend's live `podman inspect` PortBindings match its manifest `ports:` (the external mirror of the orchestrator's `host_port_bindings_drifted`) |
| `secret-completeness.bats` | .198 (bitcoin-knots needs `bitcoin-rpc-txrelay-rpcauth`, never generated → stack cascade) | every `secret_file` referenced by an installed backend manifest exists in the secrets dir |
Backed by L0 unit tests (`cargo test … drift missing_secret lnd`) and a vitest
for the frontend reason-code mapping (`bitcoinReceive.test.ts`). The release
gate `scripts/create-release.sh` now runs `tests/release/run.sh` (which includes
these) and **aborts the release on failure** — previously it ran no tests at all.
## Run commands
```bash
# L0 unit:
cd core && cargo test --workspace --bins
# Single bats suite:
ARCHY_PASSWORD=password123 tests/lifecycle/run.sh bitcoin-knots
# Full bats suite (read-only):
ARCHY_PASSWORD=password123 tests/lifecycle/run.sh
# Full + destructive (for the verification fleet):
ARCHY_PASSWORD=password123 ARCHY_ALLOW_DESTRUCTIVE=1 tests/lifecycle/run.sh
# 5× release-gate run:
ARCHY_PASSWORD=password123 ARCHY_ALLOW_DESTRUCTIVE=1 ARCHY_ITERATIONS=5 \
tests/lifecycle/run-gate.sh
# CASCADE tier (uninstall → no-ghost → reinstall) — opt-in, NOT in the canonical
# gate. Installs/uninstalls a THROWAWAY app (default grafana; skips if already
# installed). Run on-node to also assert data-dir removal:
ARCHY_PASSWORD=password123 ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1 \
tests/lifecycle/run.sh cascade-uninstall
```
### CASCADE tier — uninstall/reinstall regression guard (Workstream F)
The 5× gate is DESTRUCTIVE-only (stop/start/restart/survive); it never exercised
uninstall/reinstall, where the worst lifecycle bugs lived. `cascade-uninstall.bats`
closes that gap and encodes the fixes for two field bugs:
| Suite | Failure it guards | Asserts |
|---|---|---|
| `cascade-uninstall.bats` | **#13 uninstall ghost** (immich/grafana stayed in My Apps after uninstall) and **#14 reinstall stops** (stalled on stale state/data) | fresh install reaches `running` via a truthful (non-silent) progression; uninstall makes the entry **disappear from `server.get-state` package-data** (no ghost, no stuck uninstall stage) + removes the container + (on-node) the data dir; reinstall returns to `running`; node left as found |
Throwaway-app + precondition-skip (won't touch an app that's already installed),
so it's safe on a populated node. Override the app via `ARCHY_CASCADE_APP` /
`ARCHY_CASCADE_IMAGE` / `ARCHY_CASCADE_CONFIG` / `ARCHY_CASCADE_DATA_DIR`.
Gated on `ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1`. Verified 7/7 on .228 (2026-06-24).
### All-apps lifecycle matrix (Workstream F)
The per-app suites cover ~8 core apps in depth; `all-apps-matrix.bats` covers
**every installed app in breadth, automatically** — it derives the app set from
`server.get-state` package-data (no hardcoded list) and grows coverage as nodes
install more apps. **Read-only**, so it joins `run.sh`/`run-gate.sh` on every node.
| Suite | Guards (fleet-wide) | Asserts (per installed app) |
|---|---|---|
| `all-apps-matrix.bats` | apps STUCK transitional (the #13/#14 ghost generalized), error/failed apps, unreachable UI apps (port-drift generalized) | settles to a non-transitional state within a window; not error/failed; recognized (non-garbage) state; every **running UI app** (manifest `ui=="true"`) exposes a non-null lan-address |
Tunables: `ARCHY_MATRIX_SETTLE_SECS` (45), `ARCHY_MATRIX_UI_SECS` (30),
`ARCHY_MATRIX_ALLOW_STOPPED` (ids allowed non-running). Verified 5/5 on .228
(17 apps) and .116 (20 apps incl. grafana/nextcloud/photoprism/gitea), 2026-06-24.
To exercise the Phase 3.2 Quadlet-backend path on a target node without
editing config.json (which would require an archipelago restart and
trigger FM3 until 3.5 ships), set the env var on `archipelago.service`:
```bash
sudo systemctl edit archipelago # add: [Service]\nEnvironment=ARCHIPELAGO_USE_QUADLET_BACKENDS=1
sudo systemctl restart archipelago # one cgroup-cascade hit; survivable on a debug node
```
After the restart, `package.install` for any orchestrator-managed backend
will route through `install_via_quadlet`, and the
`use-quadlet-backends-install.bats` suite turns from skip → hard gate.
## LoC budget
Goal: minimum-viable container subsystem.
| Module | LoC today | Target | Δ | Status |
|---|---:|---:|---:|---|
| `core/container/src/dependency_resolver.rs` | — | — | -270 | ● deleted |
| `core/container/src/health_monitor.rs` | 196 | 0 | -196 | ◐ pending health migration into reconciler (Phase 3.5) |
| `core/container/src/podman_client.rs::create/start/stop` | ~400 | ~150 | -250 | ◐ pending Quadlet migration (Phase 3.5) |
| `core/archipelago/src/container/dev_orchestrator.rs` | 410 | 0 | -410 | ○ pending dev_mode strategy decision |
| `core/archipelago/src/container/data_manager.rs` | 96 | 0 | -96 | ○ couples with dev_orchestrator |
| `core/container/src/bitcoin_simulator.rs` | 219 | 0 | -219 | ○ couples with dev_orchestrator |
| `core/container/src/port_manager.rs` | 175 | 0 | -175 | ○ couples with dev_orchestrator |
| `core/archipelago/src/api/rpc/package/install.rs::install_bitcoincoin_rpc_repair` | ~150 | 0 | -150 | ◐ pending fold into orchestrator pre-start |
| imperative `install_fresh` in prod_orchestrator | ~120 | 0 | -120 | ◐ Phase 3.2 wired behind `use_quadlet_backends` flag (default off); 3.3 in-place migration ✅; 3.4 health-gated startup (`Notify=healthy`) ✅ + `TimeoutStartSec=600` race fix ✅; 3.4a unit drift-sync each reconcile ✅; flip default after 5× green |
**Today: -270 LoC committed. Outstanding deletes possible: ~1,616 LoC** (if Phase 3 ships fully + dev_mode resolved).
Net target for v1.7.52: container subsystem ≈ **half** of today's LoC.
## Performance KPIs (TBD — measure first, then target)
We don't have a performance harness yet. Add as L6 lands:
| KPI | Today | Target | Notes |
|---|---|---|---|
| cold install: bitcoin-knots manifest → `running` healthcheck | unknown | < 30s once image is local | excludes the ~1GB image pull |
| cold install: lnd | unknown | < 60s once image is local | wallet unlock dominates |
| reconcile-tick wall time (no-op pass over all installed apps) | unknown | < 250ms | the current orchestrator does many `podman inspect` calls |
| podman shell-outs per package.install (orchestrator path) | 7-10 | 1-2 (Quadlet) | post-Phase-3 |
| daemon startup (boot → port 5678 listening) | unknown | < 5s | reconcile is async after this |
## Release gates
1.8.0 ships only when ALL of (see the issue tracker for the live
priority-ordered list of what's still open across these):
1. ☑ Bitcoin-stops fix verified live on a fresh node (`tests/lifecycle/bats/bitcoin-knots.bats`
stop/restart tier, part of the green single-node gate)
2. ☑ `ARCHY_ITERATIONS=5 tests/lifecycle/run-gate.sh` returns 0 **run ON .228** — GREEN 2026-06-23, 5/5, 0 failures
3. ☐ Multinode/fleet — tracked separately,
the actual next exit criterion, NOT satisfied yet
4. ☐ The L3 `backend-survives-archipelago-restart` suite passes fleet-wide default-on
(Phase 3 Quadlet is merged + validated but still opt-in via `ARCHIPELAGO_USE_QUADLET_BACKENDS`
on .228/.198 only — not the default)
5. ☑ Cargo: 0 warnings, 0 unused (confirmed 2026-07-01 release build); full test suite green
per last confirmed run
6. ☑ LoC: Phase 3 Quadlet merged (opt-in) — satisfies the "at least one of" bar; default-flip
itself is tracked as its own item in the unified tracker
7. ☑ Layman-readable changelog — `CHANGELOG.md` backfilled through v1.8.00-alpha
(per `feedback_changelog_layman.md`)
8. ☐ Tag pushed to origin + gitea-local + gitea-vps2 (per `feedback_ship_ritual.md`) —
version decided 2026-07-08 (`1.8.0-alpha`); tag once the pre-tag items above close
## How to update this document
When you land a change that materially moves any cell of the matrix or
any LoC row, update this file in the same commit. Reviewers checking
the PR can read the diff to TESTING.md as the answer to "what did
this commit improve?". Without the update, the change is half-shipped.
tests/lifecycle/bats/all-apps-lifecycle.bats
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/all-apps-lifecycle.bats
#
# DESTRUCTIVE per-app lifecycle matrix across EVERY installed app (breadth) —
# the active counterpart to the read-only all-apps-matrix.bats and the ~8 deep
# per-app suites. For each installed, NON-protected app it drives:
# stop → verify stopped → start → verify running → restart → verify running
# and, when ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1, a FULL TEARDOWN:
# uninstall (full, removes data) → verify GONE from My Apps (no #13 ghost) →
# reinstall from the node catalog → verify running.
#
# Reinstall spec source: the node catalog (default /opt/archipelago/web-ui/
# catalog.json), whose `.apps[]` entries carry {dockerImage, containerConfig} —
# exactly what package.install needs. Multi-container stacks (immich, mempool,
# netbird, btcpay, indeedhub) ignore dockerImage internally but still require it,
# and route to their orchestrator/stack handler; the catalog entry is enough to
# trigger the reinstall. An app with no catalog entry is skipped (logged), not
# failed — there's no spec to reinstall it from.
#
# ── PROTECTED apps (NEVER touched — neither cycled nor torn down) ────────────
# - chain state, expensive to resync: bitcoin*, electrumx/electrs
# - WALLET / financial state, teardown = IRREVERSIBLE fund/credential loss:
# lnd, btcpay*, fedimint*
# The user asked to protect only bitcoin + electrum; the wallet-bearing apps
# are protected by DEFAULT here for safety (a full uninstall destroys their
# seed/channel/guardian state). Override the entire set with
# ARCHY_MATRIX_PROTECT="space separated ids" to tear them down too — you WILL
# lose their data.
#
# ── Gating ──────────────────────────────────────────────────────────────────
# lifecycle tier → ARCHY_ALLOW_DESTRUCTIVE=1
# teardown tier → ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1
# Both skip otherwise, so this file is inert in a normal run. ON-NODE ONLY
# (reads catalog.json on disk + drives the local package lifecycle).
#
# This is a HEAVY suite: a full teardown of ~15-20 apps re-pulls images and can
# run for a long time. Intended as an explicit, supervised coverage pass, not a
# per-iteration gate step.
load '../lib/rpc.bash'
CATALOG="${ARCHY_CATALOG:-/opt/archipelago/web-ui/catalog.json}"
# Protected — see header. Override with ARCHY_MATRIX_PROTECT to change the set.
PROTECT="${ARCHY_MATRIX_PROTECT:-bitcoin-knots bitcoin-core bitcoin electrumx electrs mempool-electrs lnd btcpay-server btcpayserver btcpay fedimint fedimint-clientd fedimint-gateway}"
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
is_protected() {
local id="$1" p
for p in $PROTECT; do [[ "$p" == "$id" ]] && return 0; done
return 1
}
get_package_data() {
rpc_result server.get-state '{}' 2>/dev/null | jq -c '.data["package-data"] // {}'
}
# Canonical app ids the catalog can (re)install.
catalog_ids() {
jq -r '(.apps // [])[].id' "$CATALOG" 2>/dev/null
}
# Installed primary apps we will exercise: catalog ids present in My Apps,
# minus the protected set. (Catalog-scoped so we skip sub-containers like
# immich_postgres that surface as their own package-data entries.)
target_apps() {
local pd; pd=$(get_package_data)
local id
for id in $(catalog_ids); do
echo "$pd" | jq -e --arg i "$id" 'has($i)' >/dev/null 2>&1 || continue
is_protected "$id" && continue
echo "$id"
done
}
# Top-level state of an app in My Apps, or "absent" when the entry is gone.
app_state() {
get_package_data | jq -r --arg i "$1" '.[$i].state // "absent"'
}
# Poll My Apps until app $1 reaches state $2 (or "absent"); $3 = timeout secs.
wait_state() {
local id="$1" target="$2" timeout="${3:-180}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
[[ "$(app_state "$id")" == "$target" ]] && return 0
sleep 3
done
echo "wait_state: $id never reached '$target' (last='$(app_state "$id")') within ${timeout}s" >&2
return 1
}
# Let the node drain before cycling the NEXT app.
#
# Without this the suite competes with itself. Cycling ten apps back-to-back
# drove this 4-core node from load 10.8 at preflight to 17.6 mid-loop, and the
# contention then landed on whatever ran next: btcpay's recovery stretched from
# 52s measured on a quiet box to 216s and then 512s, blowing through waits of
# 180s and 300s. Widening those waits is a losing game, because the run itself
# sets the load they have to survive — so pace the source instead.
#
# Waits for load1 to fall below the ceiling, capped so a genuinely busy node
# can't stall the suite forever. ARCHY_APP_SETTLE_SECS=0 disables it.
settle_between_apps() {
local cap="${ARCHY_APP_SETTLE_SECS:-90}"
(( cap > 0 )) || return 0
local cores ceiling
cores=$(nproc 2>/dev/null || echo 4)
ceiling="${ARCHY_APP_SETTLE_LOAD:-$(( cores * 2 ))}"
local deadline=$(( $(date +%s) + cap )) load1
while (( $(date +%s) < deadline )); do
load1=$(awk '{print $1}' /proc/loadavg)
awk -v l="$load1" -v m="$ceiling" 'BEGIN { exit !(l < m) }' && return 0
sleep 5
done
echo "# settle: load1 $load1 still >= $ceiling after ${cap}s — continuing anyway" >&3
}
# Build a package.install payload for $1 from the catalog, or fail (no spec).
catalog_install_payload() {
local id="$1" img cfg
img=$(jq -r --arg i "$id" '(.apps // [])[] | select(.id==$i) | .dockerImage // empty' "$CATALOG")
[[ -n "$img" ]] || return 1
cfg=$(jq -c --arg i "$id" '(.apps // [])[] | select(.id==$i) | .containerConfig // null' "$CATALOG")
if [[ "$cfg" == "null" ]]; then
jq -nc --arg id "$id" --arg img "$img" '{id:$id, dockerImage:$img}'
else
jq -nc --arg id "$id" --arg img "$img" --argjson cfg "$cfg" '{id:$id, dockerImage:$img, containerConfig:$cfg}'
fi
}
# ────────────────────────────────────────────────────────────────────
@test "prerequisites: catalog present and at least one target app" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
[[ -f "$CATALOG" ]] || { echo "# catalog not found: $CATALOG" >&3; false; }
run target_apps
[ "$status" -eq 0 ]
[ -n "$output" ] || { echo "# no non-protected installed apps to exercise" >&3; false; }
echo "# protected (skipped): $PROTECT" >&3
echo "# targets ($(echo "$output" | wc -w)): $(echo $output)" >&3
}
@test "lifecycle: stop → start → restart every non-protected app" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
local fails="" id first=1
for id in $(target_apps); do
[[ "$(app_state "$id")" == "running" ]] || continue # only cycle running apps
# Drain before each app after the first. Placed at the TOP of the body so it
# still runs when a previous iteration bailed out via `continue`.
(( first )) && first=0 || settle_between_apps
# Each rpc_result must be allowed to fail without aborting the loop.
# rpc_result returns non-zero whenever the response carries .error, and
# under bats' errexit a bare call ends the test right there — so a single
# transient RPC hiccup killed the run BEFORE the $fails summary below could
# name the app. That is exactly what happened on 2026-08-08: the whole test
# died at package.stop with no indication of which of the ten targets it was
# (it was mempool, and the same call succeeded by hand moments later).
rpc_result package.stop "{\"id\":\"$id\"}" >/dev/null 2>&1 \
|| { fails+="$id:stop-rpc "; continue; }
wait_state "$id" stopped 120 || { fails+="$id:stop "; }
rpc_result package.start "{\"id\":\"$id\"}" >/dev/null 2>&1 \
|| { fails+="$id:start-rpc "; continue; }
wait_state "$id" running 240 || { fails+="$id:start "; continue; }
rpc_result package.restart "{\"id\":\"$id\"}" >/dev/null 2>&1 \
|| { fails+="$id:restart-rpc "; continue; }
wait_state "$id" running 240 || { fails+="$id:restart "; }
done
[[ -z "$fails" ]] || { echo "# lifecycle failures: $fails" >&3; false; }
}
@test "teardown: full uninstall (no ghost) → reinstall every non-protected app" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
local fails="" skipped="" id payload
for id in $(target_apps); do
if ! payload=$(catalog_install_payload "$id"); then
skipped+="$id "
continue
fi
rpc_result package.uninstall "{\"id\":\"$id\"}" >/dev/null 2>&1
# No ghost: the entry must leave My Apps (the #13 class). 71cc9ac4 bounds the
# teardown so this can no longer hang indefinitely.
if ! wait_state "$id" absent 300; then
fails+="$id:ghost "
continue
fi
rpc_result package.install "$payload" >/dev/null 2>&1
wait_state "$id" running 420 || fails+="$id:reinstall "
done
[[ -n "$skipped" ]] && echo "# skipped (no catalog spec to reinstall from): $skipped" >&3
[[ -z "$fails" ]] || { echo "# teardown failures: $fails" >&3; false; }
}
tests/lifecycle/bats/all-apps-matrix.bats
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/all-apps-matrix.bats
#
# Manifest-driven, fleet-wide lifecycle health matrix. The per-app suites
# (bitcoin-knots, lnd, mempool, immich, …) cover ~8 core apps in depth; this
# covers EVERY installed app in breadth, automatically — no hardcoded list.
#
# It derives the app set from server.get-state's package-data (the My Apps map)
# and asserts baseline health across all of them. Read-only (no destructive env
# needed), so it joins run.sh / run-gate.sh on every node and grows coverage as
# nodes install more apps.
#
# Catches, fleet-wide, the bug classes the narrow gate missed:
# - apps STUCK in a transitional state (the #13/#14 ghost: installing/removing
# that never settles)
# - apps sitting in error/failed
# - running UI apps with no reachable lan-address (generalized port-drift)
load '../lib/rpc.bash'
# Transitional states are legitimate momentarily but must not PERSIST. Steady:
# running/stopped/exited/created/paused/installed/not-installed.
TRANSITIONAL_RE='^(installing|pulling-image|pulling|downloading|removing|uninstalling|updating|starting|stopping|restarting)$'
BAD_RE='^(error|failed)$'
# Apps whose state is allowed to be non-running at rest (no UI/health expectation
# beyond "settled"). Empty by default; override via ARCHY_MATRIX_ALLOW_STOPPED
# (space-separated ids) on nodes where an app is intentionally left stopped.
ALLOW_STOPPED="${ARCHY_MATRIX_ALLOW_STOPPED:-}"
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
# Echo the package-data object (the My Apps map) once.
get_package_data() {
rpc_result server.get-state '{}' 2>/dev/null | jq -c '.data["package-data"] // {}'
}
# Space-separated list of installed app ids.
app_ids() {
get_package_data | jq -r 'keys[]'
}
# ────────────────────────────────────────────────────────────────────
@test "matrix has apps to check (get-state returns a non-empty My Apps map)" {
run app_ids
[ "$status" -eq 0 ]
[ -n "$output" ]
echo "# matrix covers $(echo "$output" | wc -w) apps: $(echo $output)" >&3
}
@test "no installed app is STUCK in a transitional state (settles within window)" {
local settle="${ARCHY_MATRIX_SETTLE_SECS:-45}"
local deadline=$(( $(date +%s) + settle ))
local stuck=""
# Re-poll: a transitional state right now may just be a genuine in-progress op,
# so only fail apps that are STILL transitional after the settle window.
while :; do
stuck=""
local pd; pd=$(get_package_data)
for id in $(echo "$pd" | jq -r 'keys[]'); do
local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
[[ "$st" =~ $TRANSITIONAL_RE ]] && stuck+="${id}=${st} "
done
[[ -z "$stuck" ]] && break
(( $(date +%s) >= deadline )) && break
sleep 5
done
[[ -z "$stuck" ]] || { echo "# STUCK transitional after ${settle}s: $stuck" >&3; false; }
}
@test "no installed app is in an error/failed state" {
local pd; pd=$(get_package_data)
local bad=""
for id in $(echo "$pd" | jq -r 'keys[]'); do
local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
[[ "$st" =~ $BAD_RE ]] && bad+="${id}=${st} "
done
[[ -z "$bad" ]] || { echo "# error/failed apps: $bad" >&3; false; }
}
@test "every running app reports a recognized state (no empty/garbage state)" {
local pd; pd=$(get_package_data)
local junk=""
for id in $(echo "$pd" | jq -r 'keys[]'); do
local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
case "$st" in
running|stopped|exited|created|paused|installed|not-installed|\
installing|pulling-image|pulling|downloading|removing|uninstalling|updating|starting|stopping|restarting|\
error|failed|degraded) : ;;
*) junk+="${id}='${st}' " ;;
esac
done
[[ -z "$junk" ]] || { echo "# unrecognized state values: $junk" >&3; false; }
}
@test "every running UI app exposes a lan-address (generalized port-drift)" {
# A running app whose manifest declares a UI interface (ui=="true") must have a
# non-null lan-address on that interface — otherwise its UI is unreachable
# (the immich/port-drift failure mode, asserted across ALL UI apps). Poll
# briefly to absorb the transient null seen while a container is mid-recreate.
local deadline=$(( $(date +%s) + ${ARCHY_MATRIX_UI_SECS:-30} ))
local missing=""
while :; do
missing=""
local pd; pd=$(get_package_data)
for id in $(echo "$pd" | jq -r 'keys[]'); do
local st; st=$(echo "$pd" | jq -r --arg i "$id" '.[$i].state // "unknown"')
[[ "$st" == "running" ]] || continue
# interface keys whose manifest marks ui=="true"
local ui_ifaces
ui_ifaces=$(echo "$pd" | jq -r --arg i "$id" \
'.[$i].manifest.interfaces // {} | to_entries[] | select(.value.ui=="true") | .key')
for k in $ui_ifaces; do
local addr
addr=$(echo "$pd" | jq -r --arg i "$id" --arg k "$k" \
'.[$i].installed["interface-addresses"][$k]["lan-address"] // "null"')
[[ "$addr" == "null" || -z "$addr" ]] && missing+="${id}:${k} "
done
done
[[ -z "$missing" ]] && break
(( $(date +%s) >= deadline )) && break
sleep 3
done
[[ -z "$missing" ]] || { echo "# running UI apps missing lan-address: $missing" >&3; false; }
}
tests/lifecycle/bats/backend-survives-archipelago-restart.bats
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/backend-survives-archipelago-restart.bats
#
# Quadlet-everywhere promise (Phase 3 of v1.7.52): backend containers
# (bitcoin-knots / lnd / electrumx) are managed by systemd via Quadlet
# units, NOT parented under archipelago.service's cgroup. Restarting the
# archipelago service must NOT take them down.
#
# This is the regression gate for FM3 (cgroup cascade SIGKILL — observed
# live on .198 on 2026-05-01: stopping archipelago.service killed every
# container in its cgroup, leaving the box in a multi-hour recovery
# loop). Until v1.7.52 Phase 3 ships, this suite is EXPECTED TO FAIL on
# fleet boxes — it serves as the executable definition of "Phase 3
# complete". Do not gate the release on it passing pre-Phase-3.
#
# Sister to companion-survives-archipelago-restart.bats which tests the
# same property for UI companions (already shipping via Quadlet since
# commit 6e716f68).
#
# Gated by ARCHY_ALLOW_DESTRUCTIVE=1 because it bounces archipelago.
# bats-core ships no `fail`; bats-assert isn't installed on the alpha fleet.
# Define the same minimal helper the other suites use (see mempool.bats) so a
# tripped assertion reports as a real test failure, not a status-127 crash.
fail() { echo "$@" >&2; return 1; }
backend_units=(
"bitcoin-knots"
"bitcoin-core"
"lnd"
"electrumx"
)
container_running() {
local name="$1"
[[ "$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" == "true" ]]
}
wait_archipelago_back() {
local timeout="${1:-60}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if curl -fsS -o /dev/null "http://127.0.0.1:5678/health" 2>/dev/null; then
return 0
fi
sleep 2
done
return 1
}
@test "destructive gate enabled" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
}
@test "at least one backend container is running before restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
local up=0
for c in "${backend_units[@]}"; do
if container_running "$c"; then
up=$(( up + 1 ))
fi
done
(( up > 0 )) || skip "No backends installed on this node"
}
@test "backends survive archipelago restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# Snapshot: which backends were up before we touched anything.
local before=()
for c in "${backend_units[@]}"; do
if container_running "$c"; then
before+=("$c")
fi
done
(( ${#before[@]} > 0 )) || skip "No backends installed on this node"
# Capture pre-restart container IDs so we can verify the SAME process
# survives — not "the orchestrator started a fresh container after the
# cascade SIGKILL'd the original" (which would also be a fail; FM3 is
# specifically about losing the running container, even if the
# orchestrator can recreate one minutes later).
declare -A pre_id
for c in "${before[@]}"; do
pre_id["$c"]=$(podman inspect --format '{{.Id}}' "$c" 2>/dev/null || echo "")
done
# Bounce archipelago. Same approach as companion-survives-* for parity.
if systemctl --user list-units --no-legend archipelago.service | grep -q archipelago; then
systemctl --user restart archipelago.service
else
sudo systemctl restart archipelago.service
fi
run wait_archipelago_back 60
[ "$status" -eq 0 ]
# Every backend that was up before must still be up after, AND it must
# be the SAME container instance (same .Id). A different .Id means the
# original was killed and a fresh one was created — that's the FM3
# failure we're catching.
for c in "${before[@]}"; do
run container_running "$c"
[ "$status" -eq 0 ] || fail "backend $c died across archipelago restart (FM3 cgroup cascade)"
local post_id
post_id=$(podman inspect --format '{{.Id}}' "$c" 2>/dev/null || echo "")
[[ -n "$post_id" ]] || fail "backend $c has no container id after restart"
[[ "$post_id" == "${pre_id[$c]}" ]] \
|| fail "backend $c was recreated across archipelago restart (FM3): pre=${pre_id[$c]:0:12} post=${post_id:0:12}"
done
}
tests/lifecycle/bats/bitcoin-knots.bats
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/bitcoin-knots.bats
#
# Lifecycle tests for the bitcoin-knots package.
#
# Tiers:
# - Read-only (always runs): presence, status, state-reporting consistency
# - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart on this very container
# - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
# — this breaks LND/ElectrumX/BTCPay/mempool, so never enabled on a node serving real users.
#
# Pre-req: bitcoin-knots is installed. We do NOT install it from scratch here
# because doing so on the live host would require wiping 700GB of chain data.
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1 # make sure setup_file gets a fresh token
rpc_login
unset ARCHY_FORCE_LOGIN # subsequent test subshells reuse the session file
}
teardown_file() {
rpc_logout_local
}
# ────────────────────────────────────────────────────────────────────
# Read-only tier
# ────────────────────────────────────────────────────────────────────
@test "container-list includes bitcoin-knots" {
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "bitcoin-knots")' >/dev/null
}
@test "container-list reports a valid state for bitcoin-knots" {
# Poll briefly: a container caught mid-reconcile can momentarily report a
# transient state ("restarting"/"configured"/"removing") or no state at all.
# A genuinely-stuck container never settles, so this still catches real
# breakage; it only absorbs churn (e.g. another container bouncing right
# before the read-only tier runs).
local state="" deadline=$(( $(date +%s) + 30 ))
while (( $(date +%s) < deadline )); do
run rpc_result container-list
[ "$status" -eq 0 ]
state=$(echo "$output" | jq -r '.[] | select(.name == "bitcoin-knots") | .state')
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] && return 0
sleep 3
done
echo "bitcoin-knots never reported a settled valid state within 30s (last: '$state')" >&2
return 1
}
@test "container-status returns a valid status object for bitcoin-knots" {
# During orchestrator alias migration, container-status can fail for some
# app_id aliases even while container-list/state is correct. Accept either:
# (a) valid container-status object OR (b) valid container-list state entry.
run rpc_call container-status '{"app_id":"bitcoin-knots"}'
[ "$status" -eq 0 ]
local err
err=$(echo "$output" | jq -r '.error.message // empty')
if [[ -z "$err" ]]; then
echo "$output" | jq -e '.result | has("status") or has("state") or has("running")' >/dev/null
return 0
fi
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "bitcoin-knots") | has("state")' >/dev/null
}
@test "bitcoin.getinfo succeeds when bitcoin-knots is running" {
local state
state=$(rpc_result container-list | jq -r '.[] | select(.name == "bitcoin-knots") | .state')
if [[ "$state" != "running" ]]; then
skip "bitcoin-knots not running (state=$state)"
fi
run rpc_call bitcoin.getinfo
[ "$status" -eq 0 ]
echo "$output" | jq -e '.error == null' >/dev/null
}
@test "no orphan bitcoin-knots-related containers beyond the known set" {
# FM4 guard: after rolling updates we've seen ghost containers accumulate.
# Known-good container set for the bitcoin-knots package is just "bitcoin-knots".
# Anything matching bitcoin-knots* in podman ps that isn't in the known set is a red flag.
local count
count=$(ssh_podman_ps | awk '/bitcoin-knots/ {print $NF}' | grep -Ec '^bitcoin-knots(-[a-z]+)?$' || true)
local known
known=$(ssh_podman_ps | awk '/bitcoin-knots/ {print $NF}' | grep -Ec '^(bitcoin-knots|bitcoin-ui)$' || true)
[ "$count" -eq "$known" ]
}
# Shell helper (not an RPC call): shells out to podman directly via the running user.
# Only works when bats is run on the archy host itself (which is the plan).
ssh_podman_ps() {
podman ps -a --format '{{.ID}} {{.State}} {{.Names}}'
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier (stop → start → restart on the same container)
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions bitcoin-knots to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.stop '{"id":"bitcoin-knots"}'
[ "$status" -eq 0 ]
run wait_for_container_status bitcoin-knots stopped 60
[ "$status" -eq 0 ]
}
@test "package.start brings bitcoin-knots back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.start '{"id":"bitcoin-knots"}'
[ "$status" -eq 0 ]
run wait_for_container_status bitcoin-knots running 120
[ "$status" -eq 0 ]
}
@test "package.restart leaves bitcoin-knots in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.restart '{"id":"bitcoin-knots"}'
[ "$status" -eq 0 ]
run wait_for_container_status bitcoin-knots running 120
[ "$status" -eq 0 ]
}
@test "bitcoin.getinfo succeeds after restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# Give bitcoind up to 120s to accept RPC after a cold restart — reloading the
# block index + chainstate can take a while even on a synced node.
local deadline=$(( $(date +%s) + 120 ))
while (( $(date +%s) < deadline )); do
if rpc_call bitcoin.getinfo | jq -e '.error == null' >/dev/null 2>&1; then
return 0
fi
sleep 3
done
# NB: bats-assert's `fail` is not loaded in this file (only ../lib/rpc.bash),
# so emit + return non-zero directly rather than calling an undefined helper
# (which fails with "fail: command not found" / status 127 and hides the real
# reason). A node mid-IBD legitimately can't serve getinfo here — that's an
# environmental precondition (see required-stack "synced archival"), not a
# product regression.
echo "bitcoin.getinfo never recovered after restart within 120s" >&2
return 1
}
# ────────────────────────────────────────────────────────────────────
# Cascade-destructive tier (uninstall + reinstall)
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes bitcoin-knots" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.uninstall '{"id":"bitcoin-knots","preserve_data":true}'
[ "$status" -eq 0 ]
run wait_for_container_status bitcoin-knots absent 120
[ "$status" -eq 0 ]
}
@test "package.install bitcoin-knots returns to running" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
# manifest_path is relative to data_dir/apps/
run rpc_result package.install '{"manifest_path":"bitcoin-knots/manifest.yaml"}'
[ "$status" -eq 0 ]
run wait_for_container_status bitcoin-knots running 180
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/bitcoin-receive.bats
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/bitcoin-receive.bats
#
# Regression coverage for the Bitcoin "Receive" flow. Receive addresses come
# from LND's hot wallet via the `lnd.newaddress` RPC, so this exercises the
# exact path that broke on the fleet:
# - .116: LND REST published on the wrong host port (8080 vs the manifest's
# 18080) -> connection refused -> receive failed with the generic
# "Operation failed. Check server logs." message.
# - .228: the same family surfaced to the UI as a *false* "wallet is locked".
#
# These tests run on the archy host (they shell into podman / curl localhost).
#
# Tiers: read-only only — generating a receive address is non-destructive.
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
# Resolve the LND REST host port from the manifest (single source of truth) so
# this test follows the manifest rather than hard-coding 18080.
_lnd_rest_host_port() {
local mf
for mf in \
"${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yml" \
"${ARCHIPELAGO_APPS_DIR:-/opt/archipelago/apps}/lnd/manifest.yaml" \
"$BATS_TEST_DIRNAME/../../../apps/lnd/manifest.yml"; do
[[ -r "$mf" ]] || continue
# The REST mapping is the `- host: <N>` whose following `container:` is 8080.
awk '
/- host:/ { host=$3 }
/container:/ { if ($2 == 8080 && host != "") { print host; exit } }
' "$mf"
return 0
done
}
_lnd_running() {
rpc_result container-list 2>/dev/null \
| jq -e '.[] | select(.name == "lnd" and .state == "running")' >/dev/null 2>&1
}
# ────────────────────────────────────────────────────────────────────
# Read-only tier
# ────────────────────────────────────────────────────────────────────
@test "LND REST is reachable on the manifest host port (catches port drift)" {
_lnd_running || skip "lnd not running"
local port
port=$(_lnd_rest_host_port)
[[ -n "$port" ]] || skip "could not resolve LND REST host port from manifest"
# A TCP connect is enough: drift (container published on a different host
# port) shows up as connection-refused here, exactly as on .116.
run curl -sk -o /dev/null --max-time 8 "https://127.0.0.1:${port}/v1/getinfo"
if [ "$status" -ne 0 ]; then
echo "LND REST not reachable on host port ${port} (curl exit $status) — likely published-port drift" >&2
return 1
fi
}
@test "lnd.newaddress returns a bech32 address when lnd is running" {
_lnd_running || skip "lnd not running"
# The bitcoin bounce in bitcoin-knots.bats cascade-restarts lnd (24fd97ed).
# Depending on where the probe lands in lnd's startup it sees a different
# transient code — REST unreachable, gRPC "waiting to start" (mapped to
# LND_ERROR), wallet locked until the auto-unlocker gets through, or a
# post-unlock sync phase. All of those are the node settling, not broken —
# retry until the deadline below (run A caught WALLET_LOCKED, run B caught LND_ERROR
# while lnd was seconds into its restart; both self-healed within a couple
# of minutes). Only LND_WALLET_UNINITIALIZED (no wallet — never self-heals)
# fails immediately, and anything still erroring after the window fails
# loudly below.
# 420s, not 180s: the window starts when this TEST starts, but the lnd
# restart that locks the wallet can land partway into it. On 2026-08-08 the
# restart hit 65s in and the wallet unlocked at 2m25s (journal: lnd.service
# started 20:11:05, "wallet has been unlocked without a time limit"
# 20:13:48) — 48s after this deadline expired, so the test reported
# LND_WALLET_LOCKED on a node that was fine. The daemon's own unlock budget
# is ~10 min (UNLOCK_NOT_READY_ATTEMPTS=600) because opening the channel and
# graph dbs takes minutes on a loaded box, so anything under that is the test
# being stricter than the product it is testing.
local deadline=$((SECONDS + ${ARCHY_LND_UNLOCK_SECS:-420})) err addr
while :; do
run rpc_call lnd.newaddress
[ "$status" -eq 0 ]
err=$(echo "$output" | jq -r '.error.message // .error // empty')
addr=$(echo "$output" | jq -r '.result.address // empty')
[[ -n "$err" && "$err" != *LND_WALLET_UNINITIALIZED* && $SECONDS -lt $deadline ]] || break
sleep 10
done
# The whole point of the fix: a running lnd must hand back a real address.
if [[ -n "$err" ]]; then
echo "lnd.newaddress errored on a running node: $err" >&2
return 1
fi
if [[ "$addr" != bc1* ]]; then
echo "expected a bech32 (bc1…) address, got: '$addr'" >&2
return 1
fi
}
@test "receive errors are specific, never the generic catch-all" {
# Even when receive legitimately can't produce an address, the message must be
# actionable (start with 'Bitcoin address' and/or carry a [CODE] token) — the
# generic 'Operation failed' is what hid the real cause on .116.
run rpc_call lnd.newaddress
[ "$status" -eq 0 ]
local err
err=$(echo "$output" | jq -r '.error.message // .error // empty')
if [[ "$err" == "Operation failed. Check server logs for details." ]]; then
echo "receive returned the generic catch-all instead of a specific reason" >&2
return 1
fi
}
tests/lifecycle/bats/btcpay.bats
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/btcpay.bats
#
# Lifecycle tests for the btcpay-server multi-container stack:
# - btcpay-server (the main app)
# - archy-btcpay-db (postgres)
# - archy-nbxplorer (Bitcoin watcher)
#
# Multi-container variant of bitcoin-knots.bats / lnd.bats / electrumx.bats.
# UI URL coverage is in ui-coverage.bats; this suite is L1 (RPC API) + L3
# (lifecycle survival).
#
# Pre-req: btcpay-server installed, bitcoin-knots running.
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
btcpay_components=(
"btcpay-server"
"archy-btcpay-db"
"archy-nbxplorer"
)
@test "container-list includes every btcpay-stack component" {
run rpc_result container-list
[ "$status" -eq 0 ]
for c in "${btcpay_components[@]}"; do
echo "$output" | jq -e --arg n "$c" '.[] | select(.name == $n)' >/dev/null \
|| skip "btcpay component $c not present (stack not installed)"
done
}
@test "container-list reports valid states for every btcpay component" {
run rpc_result container-list
[ "$status" -eq 0 ]
local present=0
for c in "${btcpay_components[@]}"; do
local state
state=$(echo "$output" | jq -r --arg n "$c" '.[] | select(.name == $n) | .state')
[[ -n "$state" ]] || continue
present=$((present + 1))
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] \
|| fail "invalid state for $c: $state"
done
(( present > 0 )) || skip "btcpay stack not installed"
}
@test "no orphan btcpay-related containers beyond the known set" {
local total known
total=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(btcpay|archy-btcpay|archy-nbxplorer)' || true)
known=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(btcpay-server|archy-btcpay-db|archy-nbxplorer)$' || true)
[ "$total" -eq "$known" ]
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions btcpay-server to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "btcpay-server not installed"
run rpc_result package.stop '{"id":"btcpay-server"}'
[ "$status" -eq 0 ]
run wait_for_container_status btcpay-server stopped 60
[ "$status" -eq 0 ]
}
@test "package.start brings btcpay-server back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "btcpay-server not installed"
run rpc_result package.start '{"id":"btcpay-server"}'
[ "$status" -eq 0 ]
# 300s, not 180s: stopping btcpay DELETES the container (quadlet renders
# --rm), so this is a full recreate of a dotnet image, not a container start.
# Measured 52s on a quiet box; it blew past 180s during a gate run on the
# same node at load ~11. That is the app being slow under contention, not a
# lifecycle fault.
run wait_for_container_status btcpay-server running "${ARCHY_BTCPAY_START_SECS:-300}"
[ "$status" -eq 0 ]
}
@test "package.restart leaves btcpay-server in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "btcpay-server not installed"
run rpc_result package.restart '{"id":"btcpay-server"}'
[ "$status" -eq 0 ]
run wait_for_container_status btcpay-server running "${ARCHY_BTCPAY_START_SECS:-300}"
[ "$status" -eq 0 ]
}
@test "db + nbxplorer remain running across btcpay-server restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
for c in archy-btcpay-db archy-nbxplorer; do
podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "btcpay supporting container $c not installed"
done
for c in archy-btcpay-db archy-nbxplorer; do
local state
state=$(podman inspect --format '{{.State.Status}}' "$c" 2>/dev/null)
[[ "$state" == "running" ]] \
|| fail "supporting btcpay container $c is not running (state=$state) — package.restart cascaded into it"
done
}
@test "package.uninstall removes the whole btcpay stack" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
podman inspect btcpay-server --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "btcpay-server not installed"
run rpc_result package.uninstall '{"id":"btcpay-server","preserve_data":true}'
[ "$status" -eq 0 ]
for c in "${btcpay_components[@]}"; do
run wait_for_container_status "$c" absent 120
[ "$status" -eq 0 ] || fail "btcpay component $c not removed by uninstall"
done
}
@test "package.install restores the whole btcpay stack" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install '{"manifest_path":"btcpay-server/manifest.yaml"}'
[ "$status" -eq 0 ]
for c in "${btcpay_components[@]}"; do
run wait_for_container_status "$c" running 240
[ "$status" -eq 0 ] || fail "btcpay component $c never reached running after reinstall"
done
}
tests/lifecycle/bats/cascade-uninstall.bats
+220
View File
@@ -0,0 +1,220 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/cascade-uninstall.bats
#
# CASCADE-tier regression guard for the uninstall → reinstall lifecycle — the
# exact bug class the gate's DESTRUCTIVE tier never exercised:
# #13 "uninstall ghost" — app stayed in My Apps after uninstall because the
# package state entry wasn't cleared when teardown hit
# cleanup residue (returned Err before removing it).
# #14 "reinstall stops" — a reinstall stalled partway on the stale state/data
# left behind by the broken uninstall.
#
# Uses a THROWAWAY app (default grafana — not installed on prod/test nodes, no
# user data) so it can drive the FULL teardown path (no preserve_data), which is
# where #13 actually bit. Precondition-skips if the app is already installed, so
# it can NEVER destroy real data on a populated node.
#
# "No ghost" is asserted against server.get-state's package-data (literally the
# My Apps map) — the entry must disappear, not linger with a stale state /
# stuck uninstall stage.
#
# Gated on ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1. RPC-based, so it works on-node or
# against a remote ARCHY_HOST (the data-dir residue check is on-node only).
load '../lib/rpc.bash'
CASCADE_APP="${ARCHY_CASCADE_APP:-grafana}"
CASCADE_IMAGE="${ARCHY_CASCADE_IMAGE:-docker.io/grafana/grafana:10.2.0}"
CASCADE_CONFIG="${ARCHY_CASCADE_CONFIG:-{\"ports\":[\"3000:3000\"],\"volumes\":[\"/var/lib/archipelago/grafana:/var/lib/grafana\"],\"env\":[\"GF_PATHS_DATA=/var/lib/grafana\",\"GF_USERS_ALLOW_SIGN_UP=false\"]}}"
CASCADE_DATA_DIR="${ARCHY_CASCADE_DATA_DIR:-/var/lib/archipelago/${CASCADE_APP}}"
setup_file() {
cascade_enabled || return 0
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
cascade_enabled() {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]]
}
# True when CASCADE_APP has an entry in My Apps (server.get-state package-data).
app_in_my_apps() {
rpc_result server.get-state '{}' 2>/dev/null \
| jq -e --arg id "$CASCADE_APP" '.data["package-data"] | has($id)' >/dev/null 2>&1
}
# Top-level state of CASCADE_APP in My Apps, or "absent" when the entry is gone.
app_state() {
rpc_result server.get-state '{}' 2>/dev/null \
| jq -r --arg id "$CASCADE_APP" '.data["package-data"][$id].state // "absent"'
}
# Live uninstall stage shown by My Apps, or an empty string when the entry is gone
# or the backend has not emitted a stage yet.
app_uninstall_stage() {
rpc_result server.get-state '{}' 2>/dev/null \
| jq -r --arg id "$CASCADE_APP" '.data["package-data"][$id]["uninstall-stage"] // ""'
}
# Mirror the frontend's AppCard.vue mapping so the gate proves the UI has
# backend data that can render as a monotonic, non-fake progress bar.
uninstall_stage_percent() {
local stage="$1"
if [[ "$stage" =~ \(([0-9]+)[[:space:]]*/[[:space:]]*([0-9]+)\) ]]; then
local done="${BASH_REMATCH[1]}" total="${BASH_REMATCH[2]}"
if (( total > 0 )); then
(( done > total )) && done="$total"
echo $(( 10 + (done * 40 / total) ))
return 0
fi
fi
if [[ "$stage" =~ [Vv]olume ]]; then echo 70; return 0; fi
if [[ "$stage" =~ [Dd]ata ]]; then echo 90; return 0; fi
return 1
}
# Poll until CASCADE_APP disappears while enforcing the progress contract:
# stages must be parseable, monotonic, below 100 before terminal absence, and
# the operation must emit at least one visible stage instead of silently hanging.
wait_absent_with_truthful_uninstall_progress() {
local timeout="${1:-180}"
local deadline=$(( $(date +%s) + timeout ))
local saw_stage=0 last_percent=0
while (( $(date +%s) < deadline )); do
local state stage percent
state="$(app_state)"
[[ "$state" == "absent" ]] && {
(( saw_stage == 1 )) || {
echo "uninstall progress: no uninstall-stage observed before terminal absence" >&2
return 1
}
return 0
}
stage="$(app_uninstall_stage)"
if [[ -n "$stage" ]]; then
if ! percent="$(uninstall_stage_percent "$stage")"; then
echo "uninstall progress: unparseable stage '$stage'" >&2
return 1
fi
(( percent >= last_percent )) || {
echo "uninstall progress regressed: ${percent}% after ${last_percent}% (stage '$stage')" >&2
return 1
}
(( percent < 100 )) || {
echo "uninstall progress reached ${percent}% before terminal absence (stage '$stage')" >&2
return 1
}
saw_stage=1
last_percent="$percent"
fi
sleep 2
done
echo "wait_absent_with_truthful_uninstall_progress: $CASCADE_APP did not disappear within ${timeout}s (last='$(app_state)', stage='$(app_uninstall_stage)')" >&2
return 1
}
# Poll My Apps until CASCADE_APP reaches $1 (a state, or "absent").
wait_app_state() {
local target="$1" timeout="${2:-180}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
[[ "$(app_state)" == "$target" ]] && return 0
sleep 3
done
echo "wait_app_state: $CASCADE_APP never reached '$target' (last='$(app_state)') within ${timeout}s" >&2
return 1
}
# ────────────────────────────────────────────────────────────────────
@test "cascade gate enabled" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
}
@test "precondition: ${CASCADE_APP} is not already installed (protects real data)" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
if app_in_my_apps; then
skip "${CASCADE_APP} already installed here — refusing to uninstall (would destroy data); set ARCHY_CASCADE_APP to an uninstalled throwaway"
fi
}
@test "install ${CASCADE_APP} (fresh) reaches running with a truthful, non-silent progression" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
app_in_my_apps && skip "already installed (precondition skip)"
run rpc_result package.install "{\"id\":\"${CASCADE_APP}\",\"dockerImage\":\"${CASCADE_IMAGE}\",\"containerConfig\":${CASCADE_CONFIG}}"
[ "$status" -eq 0 ]
# Progress truthfulness: must pass through a transitional install state (not a
# silent no-op) and land on running. A warm image cache can blow through the
# transitional states between polls, so a missed transitional is a warn, not a
# failure; reaching running is the hard assertion.
local saw_transitional=0 deadline=$(( $(date +%s) + 300 ))
while (( $(date +%s) < deadline )); do
case "$(app_state)" in
installing|pulling-image|pulling|downloading|starting|created) saw_transitional=1 ;;
running) break ;;
esac
sleep 2
done
[ "$(app_state)" == "running" ]
[ "$saw_transitional" -eq 1 ] || echo "# note: no transitional install state observed (image likely cached)" >&3
}
@test "uninstall ${CASCADE_APP} reports truthful progress and clears My Apps — NO ghost (#13)" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
app_in_my_apps || skip "${CASCADE_APP} not installed (install step must have failed)"
run rpc_result package.uninstall "{\"id\":\"${CASCADE_APP}\"}"
[ "$status" -eq 0 ]
# The container must go away…
run wait_for_container_status "$CASCADE_APP" absent 180
[ "$status" -eq 0 ]
# …AND the My Apps entry must be GONE — the #13 ghost was the entry lingering
# with a stale state / stuck uninstall stage. While polling, prove the backend
# emits stage data the UI can render as monotonic, non-full progress.
run wait_absent_with_truthful_uninstall_progress 120
[ "$status" -eq 0 ]
# Belt-and-suspenders: the key is truly absent from package-data.
run app_in_my_apps
[ "$status" -ne 0 ]
}
@test "uninstall removed the data dir (full teardown, no residue)" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
# Needs the local filesystem — on-node runs only.
case "${ARCHY_HOST:-127.0.0.1}" in
127.0.0.1|localhost) : ;;
*) skip "data-dir residue check is on-node only (ARCHY_HOST=${ARCHY_HOST})" ;;
esac
[[ ! -e "$CASCADE_DATA_DIR" ]]
}
@test "reinstall ${CASCADE_APP} returns to running (#14)" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install "{\"id\":\"${CASCADE_APP}\",\"dockerImage\":\"${CASCADE_IMAGE}\",\"containerConfig\":${CASCADE_CONFIG}}"
[ "$status" -eq 0 ]
run wait_app_state running 300
[ "$status" -eq 0 ]
}
@test "cleanup: uninstall ${CASCADE_APP} to leave the node as found" {
cascade_enabled || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.uninstall "{\"id\":\"${CASCADE_APP}\"}"
[ "$status" -eq 0 ]
run wait_for_container_status "$CASCADE_APP" absent 180
[ "$status" -eq 0 ]
run wait_absent_with_truthful_uninstall_progress 120
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/companion-survives-archipelago-restart.bats
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/companion-survives-archipelago-restart.bats
#
# Quadlet promise: companion UIs (archy-bitcoin-ui, archy-lnd-ui,
# archy-electrs-ui) are managed by systemd, not archipelago. Restarting
# the archipelago user service must NOT take them down.
#
# This is the regression gate for the .228 incident in
# feedback_container_lifecycle_failure_modes.md (FM1: companions vanished
# from `podman ps -a` after archipelago crash-loop).
#
# Gated by ARCHY_ALLOW_DESTRUCTIVE=1 because it bounces archipelago.
companion_units=(
"archy-bitcoin-ui"
"archy-lnd-ui"
"archy-electrs-ui"
)
unit_dir="$HOME/.config/containers/systemd"
unit_file_present() {
local name="$1"
[[ -f "$unit_dir/$name.container" ]]
}
service_active() {
local name="$1"
systemctl --user is-active --quiet "$name.service"
}
container_running() {
local name="$1"
[[ "$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" == "true" ]]
}
wait_service_active() {
local name="$1"
local timeout="${2:-60}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if service_active "$name"; then
return 0
fi
sleep 2
done
return 1
}
wait_archipelago_back() {
local timeout="${1:-60}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if curl -fsS -o /dev/null "http://127.0.0.1:5678/health" 2>/dev/null; then
return 0
fi
sleep 2
done
return 1
}
@test "destructive gate enabled" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
}
@test "every installed companion has a quadlet unit on disk" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
local present=0
for c in "${companion_units[@]}"; do
if container_running "$c"; then
run unit_file_present "$c"
[ "$status" -eq 0 ]
present=$(( present + 1 ))
fi
done
(( present > 0 )) || skip "No companions installed on this node"
}
@test "every installed companion service is active before restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
for c in "${companion_units[@]}"; do
if container_running "$c"; then
run service_active "$c"
[ "$status" -eq 0 ]
fi
done
}
@test "companions survive archipelago restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# Snapshot: which companions were up before we touched anything.
local before=()
for c in "${companion_units[@]}"; do
if container_running "$c"; then
before+=("$c")
fi
done
(( ${#before[@]} > 0 )) || skip "No companions installed on this node"
# Bounce archipelago. The user service is the production canonical name;
# fall back to the system service for older nodes.
if systemctl --user list-units --no-legend archipelago.service | grep -q archipelago; then
systemctl --user restart archipelago.service
else
sudo systemctl restart archipelago.service
fi
run wait_archipelago_back 60
[ "$status" -eq 0 ]
# Every companion that was up before must still be up + healthy after.
for c in "${before[@]}"; do
run service_active "$c"
[ "$status" -eq 0 ]
run container_running "$c"
[ "$status" -eq 0 ]
done
}
@test "deleted unit file is recreated within one reconcile tick" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# Pick a companion that's currently running.
local target=""
for c in "${companion_units[@]}"; do
if container_running "$c"; then
target="$c"
break
fi
done
[[ -n "$target" ]] || skip "No companions installed on this node"
# Delete the unit file behind systemd's back. The reconciler should
# notice and rewrite it within one 30s tick, then start the service.
rm -f "$unit_dir/$target.container"
systemctl --user daemon-reload >/dev/null 2>&1 || true
systemctl --user stop "$target.service" >/dev/null 2>&1 || true
# Allow up to two reconcile ticks (60s + grace).
run wait_service_active "$target" 90
[ "$status" -eq 0 ]
run unit_file_present "$target"
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/electrumx.bats
+204
View File
@@ -0,0 +1,204 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/electrumx.bats
#
# Lifecycle tests for the electrumx package (containers are named
# `electrumx` + `archy-electrs-ui`). Mirrors bitcoin-knots.bats /
# lnd.bats so the 5× release-gate run exercises electrumx through
# the same state matrix.
#
# Tiers:
# - Read-only (always runs): presence, valid state, TCP reachable
# - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
# - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
#
# Pre-req: electrumx is installed and bitcoin-knots is running (electrumx
# depends on bitcoind RPC for headers).
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
# How far electrumx is behind its bitcoin daemon, from its own recent log
# ("our height: N ... daemon: M"). Prints the gap; prints nothing when no
# fresh (<30 min) sync line exists — callers must treat that as "unknown",
# never as "synced".
#
# Why this exists: ElectrumX serves NO sessions until its initial sync has
# caught up, and it flushes its DB cache at 1GB — i.e. rarely — so every
# restart discards all unflushed progress back to the last flush. On
# 2026-08-09 this node had spent 8d14h syncing largely because gate runs and
# reboots kept taking its progress away. A gate that stop/start/restarts a
# mid-sync electrumx therefore (a) honestly fails the serving probe and
# (b) actively destroys hours of sync work. Skipping WITH THE GAP NAMED is
# the truthful behaviour — this is a positively-detected syncing state, not
# the container-absent skip trap fixed earlier in this file's history.
electrumx_sync_gap() {
local line ours daemon
line=$(podman logs --tail 400 --since 30m electrumx 2>/dev/null \
| grep -E 'our height: [0-9,]+ daemon: [0-9,]+' | tail -1)
[[ -z "$line" ]] && return 0
ours=$(echo "$line" | grep -oE 'our height: [0-9,]+' | tr -dc '0-9')
daemon=$(echo "$line" | grep -oE 'daemon: [0-9,]+' | tr -dc '0-9')
[[ -n "$ours" && -n "$daemon" ]] && echo $((daemon - ours))
}
skip_if_initial_sync() {
local gap
gap=$(electrumx_sync_gap)
if [[ -n "$gap" ]] && (( gap > 10 )); then
skip "electrumx initial sync in progress ($gap blocks behind) — $1"
fi
}
# ────────────────────────────────────────────────────────────────────
# Read-only tier
# ────────────────────────────────────────────────────────────────────
@test "container-list includes electrumx" {
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "electrumx")' >/dev/null
}
@test "container-list reports a valid state for electrumx" {
run rpc_result container-list
[ "$status" -eq 0 ]
local state
state=$(echo "$output" | jq -r '.[] | select(.name == "electrumx") | .state')
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
}
@test "electrumx TCP port accepts connections when running" {
local state
state=$(rpc_result container-list | jq -r '.[] | select(.name == "electrumx") | .state')
if [[ "$state" != "running" ]]; then
skip "electrumx not running (state=$state)"
fi
# ElectrumX serves no sessions until initial sync completes, so probing a
# mid-sync instance can only fail — skip with the gap named instead.
skip_if_initial_sync "sessions are not served until it catches up"
# Same probe required-stack.bats uses — divergence flags a real regression.
# It is a real Electrum round-trip, not a bare connect(): podman's port
# forwarder accepts the TCP handshake on the host-published port even when
# nothing inside the container is serving, which kept this test green for
# days while mempool-api could not reach electrumx at all. See the longer
# note in required-stack.bats.
run python3 - <<'PY'
import json, socket
s = socket.create_connection(("127.0.0.1", 50001), 5)
s.settimeout(10)
s.sendall((json.dumps({"id": 0, "method": "server.version",
"params": ["archy-gate", "1.4"]}) + "\n").encode())
buf = b""
while b"\n" not in buf:
chunk = s.recv(4096)
if not chunk:
raise SystemExit("electrumx closed the connection without replying "
"— listening but not serving (still syncing?)")
buf += chunk
s.close()
resp = json.loads(buf.split(b"\n")[0])
if "result" not in resp:
raise SystemExit(f"electrumx returned no result: {resp}")
print("ok", resp["result"])
PY
[ "$status" -eq 0 ]
}
@test "no orphan electrumx-related containers beyond the known set" {
# FM4 guard: known-good electrumx-package set is {electrumx, archy-electrs-ui}.
local total known
total=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(electrumx|electrs|archy-electrs(-[a-z]+)?)$' || true)
known=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(electrumx|archy-electrs-ui)$' || true)
[ "$total" -eq "$known" ]
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier (stop → start → restart on the same container)
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions electrumx to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
skip_if_initial_sync "restarting discards unflushed sync progress (1GB flush cache)"
run rpc_result package.stop '{"id":"electrumx"}'
[ "$status" -eq 0 ]
run wait_for_container_status electrumx stopped 60
[ "$status" -eq 0 ]
}
@test "package.start brings electrumx back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
skip_if_initial_sync "restarting discards unflushed sync progress (1GB flush cache)"
run rpc_result package.start '{"id":"electrumx"}'
[ "$status" -eq 0 ]
run wait_for_container_status electrumx running 120
[ "$status" -eq 0 ]
}
@test "package.restart leaves electrumx in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
skip_if_initial_sync "restarting discards unflushed sync progress (1GB flush cache)"
run rpc_result package.restart '{"id":"electrumx"}'
[ "$status" -eq 0 ]
run wait_for_container_status electrumx running 120
[ "$status" -eq 0 ]
}
@test "electrumx TCP port recovers after restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
skip_if_initial_sync "restarting discards unflushed sync progress (1GB flush cache)"
# electrumx replays its index against bitcoind on cold start; allow 120s.
local deadline=$(( $(date +%s) + 120 ))
while (( $(date +%s) < deadline )); do
if python3 -c 'import socket; socket.create_connection(("127.0.0.1", 50001), 3).close()' \
>/dev/null 2>&1; then
return 0
fi
sleep 3
done
fail "electrumx TCP port never reopened after restart"
}
# ────────────────────────────────────────────────────────────────────
# Cascade-destructive tier (uninstall + reinstall)
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes electrumx" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.uninstall '{"id":"electrumx","preserve_data":true}'
[ "$status" -eq 0 ]
run wait_for_container_status electrumx absent 120
[ "$status" -eq 0 ]
}
@test "package.install electrumx returns to running" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install '{"manifest_path":"electrumx/manifest.yaml"}'
[ "$status" -eq 0 ]
run wait_for_container_status electrumx running 240
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/fedimint.bats
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/fedimint.bats
#
# Lifecycle tests for the fedimint package. The fedimint federation
# daemon runs as a single container; the gateway is its own package
# (fedimint-gateway). Mirrors the single-container pattern of
# lnd.bats / electrumx.bats for L1 (RPC API) + L3 (lifecycle survival).
# UI URL coverage is in ui-coverage.bats.
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
fedimint_skip_if_absent() {
podman inspect fedimint --format '{{.State.Status}}' >/dev/null 2>&1 \
|| skip "fedimint not installed"
}
@test "container-list includes fedimint" {
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "fedimint")' >/dev/null \
|| skip "fedimint not installed"
}
@test "container-list reports a valid state for fedimint" {
fedimint_skip_if_absent
run rpc_result container-list
[ "$status" -eq 0 ]
local state
state=$(echo "$output" | jq -r '.[] | select(.name == "fedimint") | .state')
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
}
@test "no orphan fedimint-related containers beyond the known set" {
local total known
total=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(fedimint|fedimintd|fedimint-gateway)' || true)
# `fedimint-clientd` (the dual-ecash HTTP bridge) is a legitimate, known
# container — and the unanchored `total` regex above counts it (it starts
# with "fedimint"). It must therefore be in the known set too, or every node
# running fedimint-clientd false-fails this orphan check.
known=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(fedimint|fedimint-clientd|fedimint-gateway)$' || true)
[ "$total" -eq "$known" ]
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions fedimint to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
fedimint_skip_if_absent
run rpc_result package.stop '{"id":"fedimint"}'
[ "$status" -eq 0 ]
run wait_for_container_status fedimint stopped 60
[ "$status" -eq 0 ]
}
@test "package.start brings fedimint back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
fedimint_skip_if_absent
run rpc_result package.start '{"id":"fedimint"}'
[ "$status" -eq 0 ]
run wait_for_container_status fedimint running 180
[ "$status" -eq 0 ]
}
@test "package.restart leaves fedimint in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
fedimint_skip_if_absent
run rpc_result package.restart '{"id":"fedimint"}'
[ "$status" -eq 0 ]
run wait_for_container_status fedimint running 180
[ "$status" -eq 0 ]
}
# ────────────────────────────────────────────────────────────────────
# Cascade-destructive tier
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes fedimint" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
fedimint_skip_if_absent
run rpc_result package.uninstall '{"id":"fedimint","preserve_data":true}'
[ "$status" -eq 0 ]
run wait_for_container_status fedimint absent 120
[ "$status" -eq 0 ]
}
@test "package.install fedimint returns to running" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install '{"manifest_path":"fedimint/manifest.yaml"}'
[ "$status" -eq 0 ]
run wait_for_container_status fedimint running 240
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/immich.bats
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/immich.bats
#
# Lifecycle tests for the manifest-driven immich stack. The user-facing package is
# "immich" (catalog title + icon); container-list reports it package-level as
# "immich". Its containers are named immich_server / immich_postgres /
# immich_redis (underscore) to match the runtime's per-app lifecycle references.
#
# Tiers:
# - Read-only (always): presence + valid state
# - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
# - Cascade (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall (preserve_data)
#
# RPC-based, so correct whether run on the host or against a remote ARCHY_HOST.
load '../lib/rpc.bash'
IMMICH_IMAGE="source.archipelago-foundation.org/lfg2025/immich-server:release"
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
# ────────────────────────────────────────────────────────────────────
# Read-only tier
# ────────────────────────────────────────────────────────────────────
@test "container-list includes immich" {
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "immich")' >/dev/null
}
@test "container-list reports a valid state for immich" {
run rpc_result container-list
[ "$status" -eq 0 ]
local state
state=$(echo "$output" | jq -r '.[] | select(.name == "immich") | .state')
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
}
@test "immich exposes its web UI lan-address (port 2283)" {
# Poll briefly: lan_address is derived from the published host port, which is
# momentarily absent (null) while immich_server is mid-recreate (e.g. a
# health-monitor bounce during the read-only tier). A genuinely unexposed
# immich never publishes 2283, so this still catches real port drift; it only
# absorbs the transient null seen under churn.
# 90s (not 30s): the immich stack (postgres→redis→server with DB migrations on
# boot) can take >30s to publish its host port after a churn-induced recreate,
# and the destructive-tier immich tests already allow 180–240s for the same
# stack. A genuinely unexposed immich still never publishes 2283, so this keeps
# catching real port drift while tolerating slow-but-healthy boots.
local deadline=$(( $(date +%s) + 90 ))
while (( $(date +%s) < deadline )); do
run rpc_result container-list
[ "$status" -eq 0 ]
if echo "$output" \
| jq -e '.[] | select(.name == "immich") | .lan_address // "" | test("2283")' >/dev/null; then
return 0
fi
sleep 3
done
echo "immich never reported a lan_address containing 2283 within 90s" >&2
return 1
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier (stop → start → restart)
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions immich to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# package.stop is async ({"status":"stopping"}) and a stack stop can race a
# still-settling prior op, so the end state — not the immediate RPC return — is
# the assertion.
rpc_call package.stop '{"id":"immich"}' >/dev/null 2>&1 || true
run wait_for_container_status immich stopped 90
[ "$status" -eq 0 ]
}
@test "package.start brings immich back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# Async start; the server comes up only after postgres is ready (~30s+), so wait.
rpc_call package.start '{"id":"immich"}' >/dev/null 2>&1 || true
run wait_for_container_status immich running 180
[ "$status" -eq 0 ]
}
@test "package.restart leaves immich in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.restart '{"id":"immich"}'
[ "$status" -eq 0 ]
# Restart = ordered stop+start of the whole 3-container stack (postgres→redis→
# server, with the server doing DB-readiness + migrations on boot), so it needs
# at least as long as `start` (180s) — more, since it stops first. The old 120s
# was inconsistent with the start test and false-failed on heavily-loaded nodes.
run wait_for_container_status immich running 240
[ "$status" -eq 0 ]
}
# ────────────────────────────────────────────────────────────────────
# Cascade tier (uninstall + reinstall the stack)
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes immich (data preserved)" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.uninstall '{"id":"immich","preserve_data":true}'
[ "$status" -eq 0 ]
run wait_for_container_status immich absent 120
[ "$status" -eq 0 ]
}
@test "package.install immich returns to running" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install "{\"id\":\"immich\",\"dockerImage\":\"${IMMICH_IMAGE}\"}"
[ "$status" -eq 0 ]
run wait_for_container_status immich running 180
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/lnd.bats
+154
View File
@@ -0,0 +1,154 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/lnd.bats
#
# Lifecycle tests for the lnd package. Mirrors bitcoin-knots.bats so the
# 5× release-gate run exercises lnd through the same state matrix.
#
# Tiers:
# - Read-only (always runs): presence, state-reporting consistency, RPC reachable
# - Destructive (ARCHY_ALLOW_DESTRUCTIVE=1): stop → start → restart
# - Cascade-destructive (ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1): uninstall → reinstall
#
# Pre-req: lnd is installed. Reinstall path is gated separately because it
# wipes the wallet macaroons and forces re-onboarding.
load '../lib/rpc.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
# ────────────────────────────────────────────────────────────────────
# Read-only tier
# ────────────────────────────────────────────────────────────────────
@test "container-list includes lnd" {
run rpc_result container-list
[ "$status" -eq 0 ]
echo "$output" | jq -e '.[] | select(.name == "lnd")' >/dev/null
}
@test "container-list reports a valid state for lnd" {
run rpc_result container-list
[ "$status" -eq 0 ]
local state
state=$(echo "$output" | jq -r '.[] | select(.name == "lnd") | .state')
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]]
}
@test "lnd cli getinfo succeeds when lnd is running" {
local state
state=$(rpc_result container-list | jq -r '.[] | select(.name == "lnd") | .state')
if [[ "$state" != "running" ]]; then
skip "lnd not running (state=$state)"
fi
# lnd's RPC readiness LAGS the container "running" state: after a (re)start the
# wallet must auto-unlock before lncli answers, so a single-shot getinfo races
# that window and false-fails. Retry until ready (~90s), like a health probe.
# `timeout 10` per attempt: a wedged lnd RPC (e.g. chain-blind after its
# bitcoin backend was recreated under it, .228 2026-07-08) otherwise hangs
# a single exec — and with it the whole suite — indefinitely.
run sh -lc 'for i in $(seq 1 80); do
timeout 10 podman exec lnd lncli \
--tlscertpath /root/.lnd/tls.cert \
--macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon \
--rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
sleep 3
done; exit 1'
[ "$status" -eq 0 ]
}
@test "no orphan lnd-related containers beyond the known set" {
# FM4 guard: rolling updates have left ghost containers behind in the past.
# Known-good lnd-package container set is {lnd, archy-lnd-ui}.
local total known
total=$(podman ps -a --format '{{.Names}}' | grep -Ec '^(archy-)?lnd(-[a-z]+)?$' || true)
known=$(podman ps -a --format '{{.Names}}' | grep -Ec '^(lnd|archy-lnd-ui)$' || true)
[ "$total" -eq "$known" ]
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier (stop → start → restart on the same container)
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions lnd to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.stop '{"id":"lnd"}'
[ "$status" -eq 0 ]
run wait_for_container_status lnd stopped 60
[ "$status" -eq 0 ]
}
@test "package.start brings lnd back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.start '{"id":"lnd"}'
[ "$status" -eq 0 ]
run wait_for_container_status lnd running 240
[ "$status" -eq 0 ]
}
@test "package.restart leaves lnd in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
run rpc_result package.restart '{"id":"lnd"}'
[ "$status" -eq 0 ]
run wait_for_container_status lnd running 240
[ "$status" -eq 0 ]
}
@test "lncli getinfo recovers after restart" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
# lnd takes longer than bitcoind to accept RPC after cold restart because
# the wallet has to be unlocked first, then it reconnects to bitcoind and
# re-syncs the graph. On a loaded node this exceeds 90s (observed ~2min on
# .228, then synced_to_chain:true). Give it 240s.
local deadline=$(( $(date +%s) + 240 ))
while (( $(date +%s) < deadline )); do
if sh -lc 'podman exec lnd lncli \
--tlscertpath /root/.lnd/tls.cert \
--macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon \
--rpcserver localhost:10009 getinfo >/dev/null' 2>/dev/null; then
return 0
fi
sleep 3
done
fail "lncli getinfo never recovered after restart"
}
# ────────────────────────────────────────────────────────────────────
# Cascade-destructive tier (uninstall + reinstall)
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes lnd" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.uninstall '{"id":"lnd","preserve_data":true}'
[ "$status" -eq 0 ]
run wait_for_container_status lnd absent 120
[ "$status" -eq 0 ]
}
@test "package.install lnd returns to running" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install '{"manifest_path":"lnd/manifest.yaml"}'
[ "$status" -eq 0 ]
run wait_for_container_status lnd running 180
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/mempool.bats
+197
View File
@@ -0,0 +1,197 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/mempool.bats
#
# Lifecycle tests for the mempool stack:
# - mempool (legacy install path; the frontend container)
# - mempool-api (orchestrator-managed; the backend api)
# - archy-mempool-db (orchestrator-managed; the mariadb)
# - archy-mempool-web (orchestrator-managed; the proxy/static layer)
#
# The mempool stack is split between the legacy install path (mempool itself)
# and orchestrator-managed sub-containers — see uses_orchestrator_install_flow
# in install.rs. Tests here treat them as one stack at the package.install/stop
# level, addressed by id "mempool". UI URL coverage is in ui-coverage.bats.
load '../lib/rpc.bash'
# bats-assert is not loaded in this suite (only rpc.bash), so provide a minimal
# `fail` so the `|| fail "..."` guards below report a real assertion failure
# instead of an undefined-command status 127 that masks the actual reason.
fail() { echo "$@" >&2; return 1; }
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
mempool_components=(
"mempool-api"
"archy-mempool-db"
)
mempool_optional_components=(
"mempool"
"archy-mempool-web"
)
mempool_skip_if_absent() {
for c in "${mempool_components[@]}"; do
podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1 && return 0
done
skip "mempool stack not installed"
}
@test "container-list includes the core mempool components" {
run rpc_result container-list
[ "$status" -eq 0 ]
local found=0
for c in "${mempool_components[@]}"; do
if echo "$output" | jq -e --arg n "$c" '.[] | select(.name == $n)' >/dev/null; then
found=$((found + 1))
fi
done
(( found > 0 )) || skip "mempool stack not installed"
}
@test "every present mempool component reports a valid state" {
run rpc_result container-list
[ "$status" -eq 0 ]
local present=0
for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
local state
state=$(echo "$output" | jq -r --arg n "$c" '.[] | select(.name == $n) | .state')
[[ -n "$state" ]] || continue
present=$((present + 1))
[[ "$state" =~ ^(running|stopped|exited|created|paused)$ ]] \
|| fail "invalid state for $c: $state"
done
(( present > 0 )) || skip "mempool stack not installed"
}
@test "no orphan mempool-related containers beyond the known set" {
# Poll for steady state (don't single-shot): a stack restart in a prior tier
# briefly leaves a recreated member visible alongside its replacement, so a
# one-shot count can momentarily see total>known even though the reconciler
# converges within seconds. A genuine orphan never clears, so this still
# catches it — it just tolerates the transient recreate window.
local total known deadline=$(( $(date +%s) + 30 ))
while (( $(date +%s) < deadline )); do
total=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(mempool|archy-mempool)' || true)
known=$(podman ps -a --format '{{.Names}}' \
| grep -Ec '^(mempool|mempool-api|archy-mempool-db|archy-mempool-web)$' || true)
[ "$total" -eq "$known" ] && return 0
sleep 3
done
echo "orphan mempool container persisted >30s (total=$total known=$known):" >&2
podman ps -a --format '{{.Names}}' | grep -E '^(mempool|archy-mempool)' \
| grep -vE '^(mempool|mempool-api|archy-mempool-db|archy-mempool-web)$' >&2 || true
return 1
}
# ────────────────────────────────────────────────────────────────────
# Destructive tier — operate on the package id "mempool" which the
# legacy install path treats as the whole stack
# ────────────────────────────────────────────────────────────────────
@test "package.stop transitions mempool stack to stopped" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
mempool_skip_if_absent
run rpc_result package.stop '{"id":"mempool"}'
[ "$status" -eq 0 ]
# The frontend container is the user-visible target; supporting
# services may stay running depending on orchestrator policy.
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
run wait_for_container_status mempool stopped 60
[ "$status" -eq 0 ]
fi
}
@test "package.start brings mempool stack back to running" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
mempool_skip_if_absent
run rpc_result package.start '{"id":"mempool"}'
[ "$status" -eq 0 ]
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
run wait_for_container_status mempool running 180
[ "$status" -eq 0 ]
fi
}
@test "package.restart leaves mempool stack in running state" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
mempool_skip_if_absent
run rpc_result package.restart '{"id":"mempool"}'
[ "$status" -eq 0 ]
if podman inspect mempool --format '{{.State.Status}}' >/dev/null 2>&1; then
run wait_for_container_status mempool running 180
[ "$status" -eq 0 ]
fi
}
@test "mempool api backend remains queryable when stack is up" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
mempool_skip_if_absent
# mempool-api on :8999 — same probe required-stack.bats uses for parity.
# This case runs immediately after package.restart, so mempool-api has just
# dropped + must re-establish its electrs/bitcoin connection (it reports
# "offline" in the frontend during this window). Give it the same recovery
# budget the passing parity probes use (required-stack-destructive: 240s,
# package-update-smoke: 300s) — 180s was too tight for the post-restart path.
local deadline=$(( $(date +%s) + 300 ))
while (( $(date +%s) < deadline )); do
if curl -fsS -m 5 "http://127.0.0.1:8999/api/v1/backend-info" >/dev/null 2>&1; then
return 0
fi
sleep 3
done
# NB: bats-assert's `fail` is not loaded in this file (only ../lib/rpc.bash),
# so emit + return non-zero directly rather than calling an undefined helper.
echo "mempool-api never responded on :8999 within 300s" >&2
return 1
}
# ────────────────────────────────────────────────────────────────────
# Cascade-destructive tier
# ────────────────────────────────────────────────────────────────────
@test "package.uninstall removes the mempool stack" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
mempool_skip_if_absent
run rpc_result package.uninstall '{"id":"mempool","preserve_data":true}'
[ "$status" -eq 0 ]
for c in "${mempool_components[@]}" "${mempool_optional_components[@]}"; do
if podman inspect "$c" --format '{{.State.Status}}' >/dev/null 2>&1; then
run wait_for_container_status "$c" absent 120
[ "$status" -eq 0 ] || fail "mempool component $c not removed by uninstall"
fi
done
}
@test "package.install restores the mempool stack" {
[[ "${ARCHY_ALLOW_CASCADE_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_CASCADE_DESTRUCTIVE not set"
run rpc_result package.install '{"manifest_path":"mempool/manifest.yaml"}'
[ "$status" -eq 0 ]
# At minimum the core orchestrator-managed components must come back.
for c in "${mempool_components[@]}"; do
run wait_for_container_status "$c" running 240
[ "$status" -eq 0 ] || fail "mempool component $c never reached running after reinstall"
done
}
tests/lifecycle/bats/package-update-smoke.bats
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/package-update-smoke.bats
#
# Destructive update smoke checks.
# Requires RPC auth (ARCHY_PASSWORD) and ARCHY_ALLOW_DESTRUCTIVE=1.
load '../lib/rpc.bash'
require_destructive() {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
}
require_auth() {
[[ -n "${ARCHY_PASSWORD:-}" ]] || skip "ARCHY_PASSWORD not set"
}
wait_http_ok() {
local url="$1"
local timeout="${2:-240}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if curl -fsS "$url" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
wait_started_at_change() {
local name="$1"
local old_started_at="$2"
local timeout="${3:-300}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
local started_at running
started_at=$(podman inspect --format '{{.State.StartedAt}}' "$name" 2>/dev/null || true)
running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
if [[ -n "$started_at" && "$started_at" != "$old_started_at" && "$running" == "true" ]]; then
return 0
fi
sleep 3
done
return 1
}
wait_running() {
local name="$1"
local timeout="${2:-240}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
local running
running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
if [[ "$running" == "true" ]]; then
return 0
fi
sleep 2
done
return 1
}
setup_file() {
require_auth
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
}
teardown_file() {
rpc_logout_local
}
@test "package.update bitcoin-ui restarts container and recovers endpoint" {
require_destructive
local before
before=$(podman inspect --format '{{.State.StartedAt}}' archy-bitcoin-ui 2>/dev/null || true)
[[ -n "$before" ]] || skip "archy-bitcoin-ui container not found"
run rpc_call package.update '{"id":"bitcoin-ui"}'
[ "$status" -eq 0 ]
local err
err=$(echo "$output" | jq -r '.error.message // empty')
if [[ -z "$err" ]]; then
echo "$output" | jq -e '.result.status == "updating"' >/dev/null
run wait_started_at_change archy-bitcoin-ui "$before" 360
if [[ "$status" -ne 0 ]]; then
run wait_running archy-bitcoin-ui 120
[ "$status" -eq 0 ]
fi
elif [[ "$err" == *"already updating"* ]]; then
:
else
echo "unexpected package.update error: $err" >&2
return 1
fi
run wait_http_ok "http://127.0.0.1:8334/" 180
[ "$status" -eq 0 ]
}
@test "package.update mempool stack smoke (optional)" {
require_destructive
[[ "${ARCHY_ALLOW_STACK_UPDATE:-0}" == "1" ]] || skip "ARCHY_ALLOW_STACK_UPDATE not set"
local before
before=$(podman inspect --format '{{.State.StartedAt}}' mempool 2>/dev/null || true)
[[ -n "$before" ]] || skip "mempool container not found"
run rpc_call package.update '{"id":"mempool"}'
[ "$status" -eq 0 ]
local err
err=$(echo "$output" | jq -r '.error.message // empty')
if [[ -z "$err" ]]; then
echo "$output" | jq -e '.result.status == "updating"' >/dev/null
run wait_started_at_change mempool "$before" 420
if [[ "$status" -ne 0 ]]; then
run wait_running mempool 120
[ "$status" -eq 0 ]
fi
elif [[ "$err" == *"already updating"* ]]; then
:
else
echo "unexpected package.update error: $err" >&2
return 1
fi
run wait_http_ok "http://127.0.0.1:4080/" 240
[ "$status" -eq 0 ]
run wait_http_ok "http://127.0.0.1:8999/api/v1/backend-info" 300
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/port-drift.bats
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/port-drift.bats
#
# Regression guard for the .116 failure class: a backend container that is
# "Up" but publishes its ports to the WRONG host ports because the manifest
# changed after the container was created (e.g. lnd REST stuck on host 8080
# while the manifest — and every in-process client — expects 18080).
#
# This mirrors the orchestrator's `host_port_bindings_drifted` check, but from
# the outside: it compares the live `podman inspect` PortBindings against the
# manifest `ports:` for each installed backend. Runs on the archy host.
#
# Tiers: read-only.
_apps_dir() {
local d
for d in "${ARCHIPELAGO_APPS_DIR:-}" /opt/archipelago/apps \
"$BATS_TEST_DIRNAME/../../../apps"; do
[[ -n "$d" && -d "$d" ]] && { echo "$d"; return 0; }
done
return 1
}
_manifest_for() {
local app="$1" dir
dir=$(_apps_dir) || return 1
local mf
for mf in "$dir/$app/manifest.yml" "$dir/$app/manifest.yaml"; do
[[ -r "$mf" ]] && { echo "$mf"; return 0; }
done
return 1
}
# Emit "host container" pairs from a manifest's ports: block.
_manifest_ports() {
awk '
/^[[:space:]]*ports:/ { inports=1; next }
inports && /^[[:space:]]*[a-z_]+:[[:space:]]*$/ && !/protocol:|host:|container:/ { inports=0 }
inports && /- host:/ { host=$3 }
inports && /container:/ { print host, $2 }
' "$1"
}
# For a given container + (host,container) port, emit a "DRIFT: …" line on
# mismatch (and nothing otherwise). Stays silent for unpublished / host-net
# ports — those are handled elsewhere and must never be treated as drift.
_drift_line() {
local cname="$1" want_host="$2" cport="$3"
local bindings actual
bindings=$(podman inspect "$cname" --format '{{json .HostConfig.PortBindings}}' 2>/dev/null) || return 0
actual=$(echo "$bindings" | jq -r --arg k "${cport}/tcp" '.[$k][]?.HostPort // empty' 2>/dev/null)
[[ -n "$actual" ]] || return 0
echo "$actual" | grep -qx "$want_host" && return 0
echo "DRIFT: $cname container-port $cport published on host [$actual] but manifest wants $want_host"
}
@test "backend containers publish ports that match their manifest" {
command -v podman >/dev/null 2>&1 || skip "podman not available"
local checked=0 violations="" app cname mf line
# container-name : manifest-app-id
for pair in "lnd:lnd" "bitcoin-knots:bitcoin-knots" "electrumx:electrumx"; do
cname="${pair%%:*}"; app="${pair##*:}"
podman container exists "$cname" 2>/dev/null || continue
mf=$(_manifest_for "$app") || continue
while read -r host cport; do
[[ -n "$host" && -n "$cport" ]] || continue
checked=$((checked + 1))
line=$(_drift_line "$cname" "$host" "$cport")
[[ -n "$line" ]] && violations+="${line}"$'\n'
done < <(_manifest_ports "$mf")
done
[[ "$checked" -gt 0 ]] || skip "no installed backend containers with published ports to check"
if [[ -n "$violations" ]]; then
echo "published-port drift detected:" >&2
echo "$violations" >&2
return 1
fi
}
tests/lifecycle/bats/required-stack-destructive.bats
Executable
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/required-stack-destructive.bats
#
# Controlled destructive lifecycle checks for required stack containers.
# Runs only when ARCHY_ALLOW_DESTRUCTIVE=1.
required_containers=(
"archy-bitcoin-ui"
"archy-lnd-ui"
"archy-electrs-ui"
"mempool"
"mempool-api"
)
container_installed() {
podman ps -a --format '{{.Names}}' | grep -Fx "$1" >/dev/null
}
# Only the subset of required_containers actually installed on this node —
# a node without the mempool stack (or another optional app) shouldn't
# hard-fail restarting/probing something it was never meant to have.
installed_required_containers() {
local c
for c in "${required_containers[@]}"; do
container_installed "$c" && echo "$c"
done
# Always succeed — under `set -e`, the function's own exit code is that of
# its last statement, so if the last array entry happens to be a container
# NOT installed on this node, the whole function (and any bare
# `x="$(installed_required_containers)"` caller) would spuriously fail even
# though earlier entries matched fine.
return 0
}
wait_running() {
local name="$1"
local timeout="${2:-120}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
local running
running=$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)
if [[ "$running" == "true" ]]; then
return 0
fi
sleep 2
done
return 1
}
wait_http_ok() {
local url="$1"
local timeout="${2:-180}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if curl -fsS "$url" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
restart_with_retry() {
local name="$1"
local attempts="${2:-3}"
local i
for ((i=1; i<=attempts; i++)); do
if podman restart "$name" >/dev/null 2>&1; then
return 0
fi
sleep 3
done
return 1
}
@test "required-stack destructive gate enabled" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
}
@test "restart each required service container and verify it recovers" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
local targets; targets="$(installed_required_containers)"
[[ -n "$targets" ]] || skip "none of required_containers installed on this node"
while IFS= read -r c; do
run restart_with_retry "$c" 4
[ "$status" -eq 0 ]
run wait_running "$c" 180
[ "$status" -eq 0 ]
done <<< "$targets"
}
@test "required endpoints still respond after restarts" {
[[ "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || skip "ARCHY_ALLOW_DESTRUCTIVE not set"
if container_installed archy-bitcoin-ui; then
run wait_http_ok "http://127.0.0.1:8334/" 180
[ "$status" -eq 0 ]
fi
# :8081 is nginx-proxy-manager — an OPTIONAL app (not in required_containers).
# Only assert it when NPM is actually installed on this node; otherwise the
# required-endpoints check false-fails on nodes that don't run NPM.
if podman ps --format '{{.Names}}' | grep -q '^nginx-proxy-manager$'; then
run wait_http_ok "http://127.0.0.1:8081/" 180
[ "$status" -eq 0 ]
fi
if container_installed mempool; then
run wait_http_ok "http://127.0.0.1:4080/" 180
[ "$status" -eq 0 ]
fi
if container_installed mempool-api; then
run wait_http_ok "http://127.0.0.1:8999/api/v1/backend-info" 240
[ "$status" -eq 0 ]
fi
if container_installed lnd; then
# lnd RPC readiness lags container 'running' (wallet unlock + graph sync) —
# retry rather than single-shot. See lnd.bats.
run sh -lc 'for i in $(seq 1 60); do
podman exec lnd lncli --tlscertpath /root/.lnd/tls.cert --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon --rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
sleep 3
done; exit 1'
[ "$status" -eq 0 ]
fi
}
tests/lifecycle/bats/required-stack.bats
+241
View File
@@ -0,0 +1,241 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/required-stack.bats
#
# Read-only release-gate checks for the Bitcoin/electrum/lnd/mempool stack.
# Originally written against .116's fixed app roster; the "present"/"running"
# checks below now only require containers actually installed on THIS node
# (podman_all_names — present in `podman ps -a` even if stopped), so a node
# with a different app subset (e.g. no mempool stack) doesn't hard-fail on
# apps it was never meant to have. Per-app checks further down (mempool,
# filebrowser, ...) skip individually if that app isn't installed, matching
# the mempool_skip_if_absent idiom in mempool.bats.
#
# This suite is intentionally non-destructive and does not use RPC auth;
# it can run anytime as a health gate during long sync/reindex windows.
required_containers=(
"bitcoin-knots"
"electrumx"
"lnd"
"archy-mempool-db"
"mempool-api"
"mempool"
"filebrowser"
"archy-bitcoin-ui"
"archy-lnd-ui"
"archy-electrs-ui"
)
fail() { echo "$@" >&2; return 1; }
podman_names() {
podman ps --format '{{.Names}}'
}
podman_all_names() {
podman ps -a --format '{{.Names}}'
}
container_running() {
local name="$1"
podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null
}
container_installed() {
local name="$1"
podman_all_names | grep -Fx "$name" >/dev/null
}
skip_if_not_installed() {
container_installed "$1" || skip "$1 not installed on this node"
}
# ElectrumX serves no sessions until its initial sync completes, so the
# protocol probe below can only fail mid-sync. Skip with the gap NAMED (a
# positively-detected state from a fresh log line, never inferred from
# absence). Canonical rationale lives in electrumx.bats next to its twin.
skip_if_electrumx_initial_sync() {
local line ours daemon
line=$(podman logs --tail 400 --since 30m electrumx 2>/dev/null \
| grep -E 'our height: [0-9,]+ daemon: [0-9,]+' | tail -1)
[[ -z "$line" ]] && return 0
ours=$(echo "$line" | grep -oE 'our height: [0-9,]+' | tr -dc '0-9')
daemon=$(echo "$line" | grep -oE 'daemon: [0-9,]+' | tr -dc '0-9')
[[ -n "$ours" && -n "$daemon" ]] || return 0
local gap=$((daemon - ours))
if (( gap > 10 )); then
skip "electrumx initial sync in progress ($gap blocks behind) — sessions are not served until it catches up"
fi
}
# The subset of required_containers actually installed on this node.
installed_required_containers() {
local c
for c in "${required_containers[@]}"; do
container_installed "$c" && echo "$c"
done
# Always succeed — see the identical comment in required-stack-destructive.bats.
return 0
}
bitcoin_rpc() {
curl -fsS --max-time 60 \
--user "archipelago:$(cat /var/lib/archipelago/secrets/bitcoin-rpc-password)" \
--data-binary '{"jsonrpc":"1.0","id":"required-stack","method":"getblockchaininfo","params":[]}' \
-H 'content-type: text/plain;' \
http://127.0.0.1:8332/
}
bitcoin_json() {
python3 -c 'import json,sys; r=json.load(sys.stdin)["result"]; print(r[sys.argv[1]])' "$1"
}
@test "required containers are present" {
# Under sustained 5× churn an app may still be mid-restart when this runs;
# wait for the whole required set rather than single-shot. Only checks
# containers actually installed on this node (see installed_required_containers).
local targets; targets="$(installed_required_containers)"
[[ -n "$targets" ]] || skip "none of required_containers installed on this node"
local deadline=$(( $(date +%s) + 180 )) names missing
while (( $(date +%s) < deadline )); do
names="$(podman_names)"; missing=""
while IFS= read -r c; do
echo "$names" | grep -Fx "$c" >/dev/null || missing="$missing $c"
done <<< "$targets"
[[ -z "$missing" ]] && return 0
sleep 3
done
fail "required containers never all present; missing:$missing"
}
@test "required containers are running" {
local targets; targets="$(installed_required_containers)"
[[ -n "$targets" ]] || skip "none of required_containers installed on this node"
local deadline=$(( $(date +%s) + 180 )) notrunning
while (( $(date +%s) < deadline )); do
notrunning=""
while IFS= read -r c; do
[[ "$(container_running "$c" 2>/dev/null)" == "true" ]] || notrunning="$notrunning $c"
done <<< "$targets"
[[ -z "$notrunning" ]] && return 0
sleep 3
done
fail "required containers never all running; not-running:$notrunning"
}
@test "bitcoin-knots RPC responds" {
skip_if_not_installed bitcoin-knots
run bitcoin_rpc
[ "$status" -eq 0 ]
echo "$output" | python3 -c 'import json,sys; r=json.load(sys.stdin)["result"]; assert r["chain"] == "main" and r["blocks"] >= 0'
}
@test "bitcoin backend is synced archival for electrumx/lnd gate" {
skip_if_not_installed bitcoin-knots
run bitcoin_rpc
[ "$status" -eq 0 ]
local pruned ibd blocks headers
pruned="$(echo "$output" | bitcoin_json pruned)"
ibd="$(echo "$output" | bitcoin_json initialblockdownload)"
blocks="$(echo "$output" | bitcoin_json blocks)"
headers="$(echo "$output" | bitcoin_json headers)"
if [ "$pruned" = "True" ] || [ "$pruned" = "true" ]; then
echo "bitcoin is pruned (blocks=$blocks headers=$headers); electrumx cannot index pruned historical blocks"
return 1
fi
if [ "$ibd" = "True" ] || [ "$ibd" = "true" ]; then
echo "bitcoin is still in initial block download (blocks=$blocks headers=$headers)"
return 1
fi
}
@test "electrumx answers the Electrum protocol (not just an open socket)" {
skip_if_not_installed electrumx
skip_if_electrumx_initial_sync
# A bare connect() to the HOST-published port proves nothing: podman's port
# forwarder accepts the TCP handshake even when nothing inside the container
# is listening. On 2026-08-09 this test was green while mempool-api was in a
# hard ECONNREFUSED loop against electrumx:50001 from inside archy-net —
# electrumx was still doing its initial sync (DB height 959,774 vs chain tip
# 961,706) and ElectrumX does not serve sessions until it has caught up.
# So do a real protocol round-trip: the forwarder cannot fake a reply.
run python3 - <<'PY'
import json, socket
s = socket.create_connection(("127.0.0.1", 50001), 5)
s.settimeout(10)
req = json.dumps({"id": 0, "method": "server.version",
"params": ["archy-gate", "1.4"]}) + "\n"
s.sendall(req.encode())
buf = b""
while b"\n" not in buf:
chunk = s.recv(4096)
if not chunk:
raise SystemExit("electrumx closed the connection without replying "
"— it is listening but not serving (still syncing?)")
buf += chunk
s.close()
resp = json.loads(buf.split(b"\n")[0])
if "result" not in resp:
raise SystemExit(f"electrumx returned no result: {resp}")
print("ok", resp["result"])
PY
[ "$status" -eq 0 ]
}
@test "lnd CLI getinfo succeeds" {
skip_if_not_installed lnd
# lnd RPC readiness lags the container "running" state (wallet auto-unlock on
# start), so retry until ready rather than single-shot. See lnd.bats note.
run sh -lc 'for i in $(seq 1 30); do
timeout 20 podman exec lnd lncli --tlscertpath /root/.lnd/tls.cert --macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon --rpcserver localhost:10009 getinfo >/dev/null 2>&1 && exit 0
sleep 3
done; exit 1'
[ "$status" -eq 0 ]
}
@test "lnd REST port accepts connections" {
skip_if_not_installed lnd
run python3 - <<'PY'
import socket
s = socket.create_connection(("127.0.0.1", 18080), 3)
s.close()
print("ok")
PY
[ "$status" -eq 0 ]
}
@test "mempool api endpoint responds" {
skip_if_not_installed mempool-api
# mempool-api reconnects to electrumx after a stack restart — retry ~180s.
run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:8999/api/v1/backend-info" && exit 0; sleep 3; done; exit 1'
[ "$status" -eq 0 ]
}
@test "mempool frontend responds" {
skip_if_not_installed mempool
run sh -lc 'for i in $(seq 1 60); do curl -fsS -m 5 -o /dev/null "http://127.0.0.1:4080/" && exit 0; sleep 3; done; exit 1'
[ "$status" -eq 0 ]
}
@test "bitcoin ui responds" {
skip_if_not_installed archy-bitcoin-ui
# The companion (archy-bitcoin-ui) may have just been recreated by an earlier
# companion-survives test; its nginx takes a moment to serve. Retry ~120s
# rather than single-shot.
run sh -lc 'for i in $(seq 1 40); do curl -fsS -o /dev/null "http://127.0.0.1:8334/" && exit 0; sleep 3; done; exit 1'
[ "$status" -eq 0 ]
}
@test "lnd ui responds" {
skip_if_not_installed archy-lnd-ui
run curl -fsS "http://127.0.0.1:18083/"
[ "$status" -eq 0 ]
}
@test "filebrowser responds" {
skip_if_not_installed filebrowser
run curl -fsS "http://127.0.0.1:8083/"
[ "$status" -eq 0 ]
}
tests/lifecycle/bats/secret-completeness.bats
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/secret-completeness.bats
#
# Regression guard for the .198 failure class: a manifest references a
# `secret_env.secret_file` that was never generated on the node, so secret
# resolution hard-fails and the container won't start — cascading the whole
# Bitcoin stack. (bitcoin-knots gained `bitcoin-rpc-txrelay-rpcauth`, which old
# nodes lacked, so bitcoind never came up and reinstall "just stopped".)
#
# For every installed backend, assert every secret_file it references exists in
# the secrets dir. Runs on the archy host.
#
# Tiers: read-only.
SECRETS_DIR="${ARCHY_SECRETS_DIR:-/var/lib/archipelago/secrets}"
_apps_dir() {
local d
for d in "${ARCHIPELAGO_APPS_DIR:-}" /opt/archipelago/apps \
"$BATS_TEST_DIRNAME/../../../apps"; do
[[ -n "$d" && -d "$d" ]] && { echo "$d"; return 0; }
done
return 1
}
_manifest_for() {
local app="$1" dir mf
dir=$(_apps_dir) || return 1
for mf in "$dir/$app/manifest.yml" "$dir/$app/manifest.yaml"; do
[[ -r "$mf" ]] && { echo "$mf"; return 0; }
done
return 1
}
_secret_files_in() {
# Emit each `secret_file:` value referenced by the manifest.
grep -E '^[[:space:]]*secret_file:' "$1" 2>/dev/null | awk '{print $2}'
}
_secret_exists() {
local f="$SECRETS_DIR/$1"
[[ -e "$f" ]] && return 0
sudo -n test -f "$f" 2>/dev/null
}
@test "every installed backend's referenced secrets exist on disk" {
command -v podman >/dev/null 2>&1 || skip "podman not available"
[[ -d "$SECRETS_DIR" ]] || sudo -n test -d "$SECRETS_DIR" 2>/dev/null || skip "secrets dir not present"
local checked=0 missing="" app cname mf sf
# container-name : manifest-app-id (the bitcoin stack that cascades)
for pair in \
"bitcoin-knots:bitcoin-knots" "lnd:lnd" "electrumx:electrumx" \
"mempool-api:mempool-api" "btcpay-server:btcpay-server" \
"archy-nbxplorer:archy-nbxplorer" "fedimint:fedimint" \
"fedimint-gateway:fedimint-gateway"; do
cname="${pair%%:*}"; app="${pair##*:}"
podman container exists "$cname" 2>/dev/null || continue
mf=$(_manifest_for "$app") || continue
while read -r sf; do
[[ -n "$sf" ]] || continue
checked=$((checked + 1))
_secret_exists "$sf" || missing+="${app} -> ${sf}\n"
done < <(_secret_files_in "$mf")
done
[[ "$checked" -gt 0 ]] || skip "no installed backends with secret references to check"
if [[ -n "$missing" ]]; then
echo "installed apps reference missing secrets:" >&2
echo -e "$missing" >&2
return 1
fi
}
tests/lifecycle/bats/ui-coverage.bats
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/ui-coverage.bats
#
# UI surface tests — exercises the URLs a real user actually clicks
# through, not just the JSON-RPC API. Fills the coverage gap where the
# previous bats suites would report "container is up" while the iframe
# behind /app/<id>/ was returning 502 because nginx had a stale upstream
# or the proxy port was wrong.
#
# URL map sourced from neode-ui/src/views/appSession/appSessionConfig.ts
# (the frontend's own resolveAppUrl). Tests here MUST stay in sync with
# that file — divergence is the whole bug class we're guarding against.
#
# Each app probe is gated on its container being running:
# - container down → skip (clean dependency report, no false-fail)
# - container up → URL MUST return 200 with non-empty body
#
# Looped 5× via tests/lifecycle/run-gate.sh.
load '../lib/rpc.bash'
load '../lib/ui-probes.bash'
setup_file() {
: "${ARCHY_PASSWORD:?Set ARCHY_PASSWORD env var to the UI password}"
export ARCHY_FORCE_LOGIN=1
rpc_login
unset ARCHY_FORCE_LOGIN
HOST="${ARCHY_HOST:-127.0.0.1}"
export HOST
# Honour ARCHY_SCHEME like lib/rpc.bash does, defaulting to https so nodes
# that already pass keep testing the TLS path. These probes used to hardcode
# https, which made the whole suite unrunnable on a node that serves the
# dashboard over http: on archi-dev-box :443 is bound to the Tailscale /
# WireGuard / LAN interface addresses but NOT to loopback, while :80 is bound
# on 0.0.0.0 — so every proxy probe failed "curl failed (network/timeout)"
# against http-reachable endpoints that were in fact serving 200.
UI_BASE="${ARCHY_SCHEME:-https}://$HOST"
export UI_BASE
}
teardown_file() {
rpc_logout_local
}
# ────────────────────────────────────────────────────────────────────
# Dashboard shell + catalog (always required)
# ────────────────────────────────────────────────────────────────────
@test "dashboard / returns the Vue SPA shell" {
run probe_dashboard_shell
[ "$status" -eq 0 ]
}
@test "dashboard catalog endpoint responds with apps" {
run probe_dashboard_catalog
[ "$status" -eq 0 ]
}
# ────────────────────────────────────────────────────────────────────
# Bitcoin UI — direct host port (8334), companion container
# ────────────────────────────────────────────────────────────────────
@test "bitcoin-ui is reachable on :8334 when archy-bitcoin-ui is running" {
probe_app_url archy-bitcoin-ui "http://$HOST:8334/" "bitcoin-ui (direct port 8334)"
}
# ────────────────────────────────────────────────────────────────────
# HTTPS proxy paths — match HTTPS_PROXY_PATHS in appSessionConfig.ts
# ────────────────────────────────────────────────────────────────────
@test "lnd proxy /app/lnd/ responds when lnd is running" {
probe_app_url lnd "$UI_BASE/app/lnd/" "lnd (proxy /app/lnd/)"
}
@test "electrumx proxy /app/electrumx/ responds when electrumx is running" {
# electrumx companion (archy-electrs-ui) is what serves the iframe HTML;
# the electrumx daemon is just the TCP backend.
probe_app_url archy-electrs-ui "$UI_BASE/app/electrumx/" "electrumx (proxy /app/electrumx/)"
}
@test "mempool proxy /app/mempool/ responds when mempool is running" {
probe_app_url mempool "$UI_BASE/app/mempool/" "mempool (proxy /app/mempool/)"
}
@test "fedimint proxy /app/fedimint/ responds when fedimint is running" {
probe_app_url fedimint "$UI_BASE/app/fedimint/" "fedimint (proxy /app/fedimint/)"
}
@test "btcpay proxy /app/btcpay/ responds when btcpay-server is running" {
probe_app_url btcpay-server "$UI_BASE/app/btcpay/" "btcpay (proxy /app/btcpay/)"
}
@test "filebrowser proxy /app/filebrowser/ responds when filebrowser is running" {
probe_app_url filebrowser "$UI_BASE/app/filebrowser/" "filebrowser (proxy /app/filebrowser/)"
}
# ────────────────────────────────────────────────────────────────────
# Companion-served URLs that aren't in HTTPS_PROXY_PATHS but show up
# in the dashboard. archy-lnd-ui shares lnd's iframe path; archy-electrs-ui
# shares electrumx's. The earlier test already covers those — leaving
# this section for future companion-direct probes (none today).
# ────────────────────────────────────────────────────────────────────
tests/lifecycle/bats/use-quadlet-backends-install.bats
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env bats
# tests/lifecycle/bats/use-quadlet-backends-install.bats
#
# Validates the post-condition of Phase 3.2's `use_quadlet_backends`
# install path. When the orchestrator routed at least one backend
# install through `install_via_quadlet`, this suite asserts that the
# resulting state has the four properties the Phase 3 design promises:
#
# 1. A `.container` unit file exists in ~/.config/containers/systemd/
# and is well-formed (required sections + directives).
# 2. The corresponding `.service` is active under `systemctl --user`.
# 3. The container is in `podman ps` (running).
# 4. The container's cgroup is under `user.slice/...`, NOT under
# `archipelago.service` — proving FM3 (cgroup cascade SIGKILL on
# archipelago restart) is structurally fixed for that container.
#
# Auto-skips if no Quadlet-managed backend exists yet — so it runs as a
# no-op on nodes where `use_quadlet_backends` is still false (today's
# default), and turns into a hard regression gate as soon as anyone
# flips the flag and reinstalls.
#
# Run on a node with rootless podman + systemd-user (every alpha-fleet
# box). No env vars required for the read-only checks. The cleanup
# section at the bottom is gated by ARCHY_ALLOW_DESTRUCTIVE=1.
# bats-core ships no `fail`; bats-assert isn't installed on the alpha fleet.
# Define the same minimal helper the other suites use (see mempool.bats) so a
# tripped assertion reports as a real test failure, not a status-127 crash.
fail() { echo "$@" >&2; return 1; }
quadlet_dir() {
echo "${XDG_CONFIG_HOME:-$HOME/.config}/containers/systemd"
}
# List Quadlet `.container` units that correspond to backend containers
# (i.e., NOT companions like archy-*-ui, which already shipped via Quadlet
# in v1.7.41 and have their own coverage in companion-survives-archipelago-
# restart.bats). Echoes one container name per line; empty if none found.
backend_quadlet_units() {
local d
d="$(quadlet_dir)"
[[ -d "$d" ]] || return 0
# Strip the .container extension; filter out archy-*-ui companions.
# wyoming-* (piper/whisper voice services) are mid-integration and not yet
# part of the platform contract — exclude until their packaging lands.
for f in "$d"/*.container; do
[[ -e "$f" ]] || continue
local name
name="$(basename "$f" .container)"
[[ "$name" =~ ^archy-.*-ui$ ]] && continue
[[ "$name" =~ ^wyoming- ]] && continue
echo "$name"
done
}
# A unit file on disk does NOT imply the app should be running: an
# explicitly user-stopped app keeps its .container file (e.g. the inactive
# half of the bitcoin-core/bitcoin-knots multi-version pair), and its
# .service being inactive / container absent is the CORRECT state. The
# orchestrator persists that intent in user-stopped.json; honour it here so
# the active-state assertions below don't false-fail on stopped-on-purpose
# apps (gate tests 123/124, .228 2026-07-09).
USER_STOPPED_FILE="${ARCHY_DATA_DIR:-/var/lib/archipelago}/user-stopped.json"
is_user_stopped() {
local name="$1"
[[ -r "$USER_STOPPED_FILE" ]] || return 1
jq -e --arg n "$name" --arg s "${name#archy-}" \
'index($n) != null or index($s) != null' "$USER_STOPPED_FILE" >/dev/null 2>&1
}
# Read the cgroup path of a running container's main process. For
# rootless podman the conmon-run target lands the container's pid1 in
# the cgroup that owns its supervising .service.
container_cgroup_path() {
local name="$1"
local pid
pid="$(podman inspect --format '{{.State.Pid}}' "$name" 2>/dev/null)"
[[ -n "$pid" && "$pid" != "0" ]] || return 1
# cgroup v2 line: "0::/path/to/cgroup"
awk -F: '$1=="0"{print $3}' "/proc/$pid/cgroup" 2>/dev/null
}
# Per-test gate. Each @test calls this so the suite is a clean no-op on
# nodes where use_quadlet_backends is still false (today's default) —
# bats doesn't propagate setup-level skip semantics across @test blocks.
require_quadlet_backends() {
local count
count="$(backend_quadlet_units | wc -l)"
(( count > 0 )) || skip "no backend .container units in $(quadlet_dir) — use_quadlet_backends not enabled or no backends installed"
}
@test "Quadlet unit dir exists or is plausibly creatable" {
local d
d="$(quadlet_dir)"
# Either it already exists, or its parent does (so quadlet can mkdir it).
[[ -d "$d" ]] || [[ -d "$(dirname "$d")" ]] \
|| skip "no XDG_CONFIG_HOME and no \$HOME/.config — not a desktop-style host"
}
@test "each backend Quadlet unit has the required sections + directives" {
require_quadlet_backends
local d
d="$(quadlet_dir)"
while read -r name; do
[[ -z "$name" ]] && continue
local body
body="$(<"$d/$name.container")"
# [Container] section + Image=
[[ "$body" == *"[Container]"* ]] || fail "$name: missing [Container] section"
[[ "$body" == *"Image="* ]] || fail "$name: missing Image= directive"
# [Service] section with Restart=always (backends AND companions alike).
#
# This asserted Restart=on-failure until 8908fb4f, on the rationale that
# backends needed it "so an operator-issued `systemctl stop` actually stays
# stopped". That rationale was wrong on two counts. systemd never applies
# Restart= to a unit stopped via `systemctl stop`, which is how archipelago
# stops apps — so on-failure bought nothing there. And because quadlet
# renders --rm, a container that exits CLEANLY is deleted and on-failure
# will not bring it back: bitcoind exits 0 on SIGTERM, so backends were
# vanishing after a clean exit.
#
# Verified on-device before the change landed: `podman stop bitcoin-knots`
# -> back in 12s, while a dashboard-issued stop stayed stopped for 90s.
# Operator approved it explicitly, gated on exactly that verification.
[[ "$body" == *"[Service]"* ]] || fail "$name: missing [Service] section"
[[ "$body" == *"Restart=always"* ]] \
|| fail "$name: backend unit must use Restart=always (--rm deletes a cleanly-exited container, and on-failure never restarts it)"
# [Install] section so `systemctl --user enable` is well-defined.
[[ "$body" == *"[Install]"* ]] || fail "$name: missing [Install] section"
[[ "$body" == *"WantedBy="* ]] || fail "$name: missing WantedBy= in [Install]"
done < <(backend_quadlet_units)
}
@test "health is app-level state, NOT a systemd start gate (no Notify=healthy)" {
require_quadlet_backends
# Phase 3.4 originally emitted Notify=healthy so `systemctl start` blocked
# until the healthcheck passed. That was deliberately reverted: gating start
# on health hung boot reconciliation for dependency-waiting apps (fedimint
# idles its entrypoint until Bitcoin IBD finishes; lnd until the macaroon
# unlocks), leaving units stuck in "deactivating". The renderer now emits
# HealthCmd= for Podman's health state but TimeoutStartSec=0 and NO
# Notify=healthy (see quadlet.rs render() + contains_stale_health_gate()).
# This asserts the current invariant: no backend unit gates start on health.
local d
d="$(quadlet_dir)"
while read -r name; do
[[ -z "$name" ]] && continue
local body
body="$(<"$d/$name.container")"
[[ "$body" != *"Notify=healthy"* ]] \
|| fail "$name: emits Notify=healthy — stale health gate; start would block on health and can hang boot reconcile"
done < <(backend_quadlet_units)
}
@test "every backend Quadlet unit's .service is active in systemctl --user" {
require_quadlet_backends
while read -r name; do
[[ -z "$name" ]] && continue
is_user_stopped "$name" && continue
# Converges-to-active, not instantly-active: a dependency-degraded app
# (mempool-api while electrumx catches up to the daemon) exits at startup
# and flaps through 'activating' for a couple of minutes after a lifecycle
# cycle; systemd's Restart=on-failure heals it. A genuine crash-loop still
# fails after the settle window (gate 2026-07-09, .228 iterations 1+2).
local state="" deadline=$((SECONDS + 180))
while (( SECONDS < deadline )); do
state="$(systemctl --user is-active "$name.service" 2>&1)" && break
sleep 5
done
[[ "$state" == "active" ]] \
|| fail "$name.service is '$state' — did not reach 'active' within 180s"
done < <(backend_quadlet_units)
}
@test "every backend Quadlet unit has a running podman container" {
require_quadlet_backends
while read -r name; do
[[ -z "$name" ]] && continue
is_user_stopped "$name" && continue
# Same settle window as the active-state assert above: a quadlet --rm
# container is absent for a few seconds around each systemd retry.
local state="" deadline=$((SECONDS + 180))
while (( SECONDS < deadline )); do
state="$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" \
&& [[ "$state" == "true" ]] && break
sleep 5
done
[[ "$state" == "true" ]] \
|| fail "$name has no running container within 180s (state=${state:-absent})"
done < <(backend_quadlet_units)
}
@test "FM3 fix: backend cgroup is under user.slice, not archipelago.service" {
require_quadlet_backends
# The whole point of Phase 3 — verify the kernel-level invariant.
while read -r name; do
[[ -z "$name" ]] && continue
local cg
cg="$(container_cgroup_path "$name")" || skip "$name has no readable PID; container may have crashed mid-test"
[[ -n "$cg" ]] || fail "$name: empty cgroup path"
# Acceptable: anything under user.slice (rootless podman lands here when
# quadlet-managed). Forbidden: anything under archipelago.service's tree.
[[ "$cg" == *"user.slice"* ]] \
|| fail "$name: cgroup '$cg' is not under user.slice — FM3 cascade still possible"
[[ "$cg" != *"archipelago.service"* ]] \
|| fail "$name: cgroup '$cg' is under archipelago.service — Phase 3 promise broken"
done < <(backend_quadlet_units)
}
tests/lifecycle/lib/rpc.bash
Executable
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env bash
# tests/lifecycle/lib/rpc.bash
#
# Shared JSON-RPC client for archipelago lifecycle tests.
# Handles login, session cookie + CSRF token management, and request plumbing.
#
# Environment variables honored:
# ARCHY_HOST — default: 127.0.0.1
# ARCHY_SCHEME — default: https
# ARCHY_PASSWORD — REQUIRED. The UI password.
#
# After sourcing, call `rpc_login` once per test file in setup_file or setup.
# Then call `rpc_call METHOD [JSON_PARAMS]` to invoke methods.
# rpc_call prints the raw JSON response to stdout.
set -euo pipefail
ARCHY_HOST="${ARCHY_HOST:-127.0.0.1}"
ARCHY_SCHEME="${ARCHY_SCHEME:-https}"
ARCHY_BASE_URL="${ARCHY_SCHEME}://${ARCHY_HOST}"
# Session file lives in a stable per-user location so every bats subshell
# (setup_file, setup, each @test) sees the same cookies. File format:
# line 1: session cookie value
# line 2: csrf cookie value
RPC_SESSION_FILE="${RPC_SESSION_FILE:-${TMPDIR:-/tmp}/archy-rpc-session-${UID:-$(id -u)}}"
RPC_SESSION=""
RPC_CSRF=""
# Load cookies from $RPC_SESSION_FILE into RPC_SESSION/RPC_CSRF.
# Returns 1 if the file is missing or malformed.
_rpc_load_session() {
[[ -r "$RPC_SESSION_FILE" ]] || return 1
local lines
mapfile -t lines < "$RPC_SESSION_FILE"
RPC_SESSION="${lines[0]:-}"
RPC_CSRF="${lines[1]:-}"
[[ -n "$RPC_SESSION" && -n "$RPC_CSRF" ]]
}
# Log in with $ARCHY_PASSWORD and persist session + csrf cookies to $RPC_SESSION_FILE.
# Idempotent-ish: if a valid session file already exists and ARCHY_FORCE_LOGIN
# is not set, we reuse it (saves a round-trip per test file).
rpc_login() {
if _rpc_load_session && [[ -z "${ARCHY_FORCE_LOGIN:-}" ]]; then
return 0
fi
if [[ -z "${ARCHY_PASSWORD:-}" ]]; then
echo "rpc_login: ARCHY_PASSWORD env var not set" >&2
return 1
fi
local headers body
headers=$(mktemp)
body=$(curl -sk -D "$headers" -X POST "${ARCHY_BASE_URL}/rpc/v1" \
-H 'Content-Type: application/json' \
--data-raw "{\"jsonrpc\":\"2.0\",\"method\":\"auth.login\",\"params\":{\"password\":\"${ARCHY_PASSWORD}\"},\"id\":1}")
local err
err=$(echo "$body" | jq -r '.error // empty')
if [[ -n "$err" && "$err" != "null" ]]; then
echo "rpc_login failed: $err" >&2
rm -f "$headers"
return 1
fi
RPC_SESSION=$(grep -i '^set-cookie: session=' "$headers" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r')
RPC_CSRF=$(grep -i '^set-cookie: csrf_token=' "$headers" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r')
rm -f "$headers"
if [[ -z "$RPC_SESSION" || -z "$RPC_CSRF" ]]; then
echo "rpc_login: missing session or csrf cookie in response" >&2
return 1
fi
# Persist for subsequent subshells.
umask 077
printf '%s\n%s\n' "$RPC_SESSION" "$RPC_CSRF" > "$RPC_SESSION_FILE"
return 0
}
# Forget persisted session (e.g., at end of a test run).
rpc_logout_local() {
rm -f "$RPC_SESSION_FILE"
RPC_SESSION=""
RPC_CSRF=""
}
# Call an RPC method.
# Usage: rpc_call METHOD [PARAMS_JSON]
# Prints the full JSON-RPC response object to stdout.
# Returns 0 on successful HTTP call (regardless of RPC-level error).
rpc_call() {
local method="$1"
local params="${2:-null}"
local id="${3:-$RANDOM}"
if [[ -z "$RPC_SESSION" || -z "$RPC_CSRF" ]]; then
_rpc_load_session || {
echo "rpc_call: not logged in (call rpc_login first)" >&2
return 1
}
fi
local payload
if [[ "$params" == "null" ]]; then
payload=$(jq -nc --arg m "$method" --argjson id "$id" '{jsonrpc:"2.0",method:$m,id:$id}')
else
payload=$(jq -nc --arg m "$method" --argjson id "$id" --argjson p "$params" '{jsonrpc:"2.0",method:$m,params:$p,id:$id}')
fi
curl -sk -X POST "${ARCHY_BASE_URL}/rpc/v1" \
-H 'Content-Type: application/json' \
-H "Cookie: session=${RPC_SESSION}; csrf_token=${RPC_CSRF}" \
-H "X-CSRF-Token: ${RPC_CSRF}" \
--data-raw "$payload"
}
# Convenience: call rpc and return only the .result field (or fail if .error is set).
rpc_result() {
local resp
resp=$(rpc_call "$@")
local err
err=$(echo "$resp" | jq -r '.error // empty')
if [[ -n "$err" && "$err" != "null" ]]; then
echo "rpc_result: $1 failed: $err" >&2
echo "full response: $resp" >&2
return 1
fi
echo "$resp" | jq '.result'
}
# Wait for a container to reach a given status ("running" or "stopped" or "absent").
# Usage: wait_for_container_status NAME STATUS [TIMEOUT_SECONDS]
wait_for_container_status() {
local name="$1"
local target="$2"
local timeout="${3:-60}"
local deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
local list state status
list=$(rpc_result container-list 2>/dev/null || echo '[]')
if [[ "$target" == "absent" ]]; then
if ! echo "$list" | jq -e --arg n "$name" '.[] | select(.name == $n)' >/dev/null 2>&1; then
return 0
fi
else
# Primary source: container-list state keyed by container name.
state=$(echo "$list" | jq -r --arg n "$name" '.[] | select(.name == $n) | .state // "unknown"')
if [[ "$state" == "$target" ]]; then
return 0
fi
# Fallback: container-status RPC accepts app_id. For common UI-prefixed
# names, strip archy- prefix before querying.
local app_id="$name"
if [[ $app_id == bitcoin-knots ]]; then
app_id=bitcoin-core
elif [[ $app_id == electrs || $app_id == mempool-electrs ]]; then
app_id=electrumx
elif [[ $app_id == archy-* ]]; then
app_id=${app_id#archy-}
fi
status=$(rpc_result container-status "{\"app_id\":\"$app_id\"}" 2>/dev/null | jq -r '.status // .state // "unknown"')
if [[ "$status" == "$target" ]]; then
return 0
fi
fi
sleep 2
done
echo "wait_for_container_status: $name did not reach '$target' within ${timeout}s" >&2
return 1
}
tests/lifecycle/lib/ui-probes.bash
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env bash
# tests/lifecycle/lib/ui-probes.bash
#
# HTTPS proxy + iframe URL probes. Sourced from bats files. Pairs with
# lib/rpc.bash but tests the URL surface a real user actually clicks
# (dashboard, /app/<id>/ proxy paths, direct-port iframes), not just the
# JSON-RPC API.
#
# Pattern: every probe is a skip-or-assert pair:
# - if the container that backs the URL is not running → skip
# (cleanly reports the dependency, doesn't false-fail)
# - if it IS running → the URL MUST return 200
# That catches the "container up but UI broken" failure mode that the
# RPC-only tests miss (.198 today: archy-bitcoin-ui Up 12 minutes,
# but is the iframe actually serving usable HTML? this layer answers).
# Curl options for a probe: short timeout, follow redirects, ignore self-
# signed cert (the alpha fleet uses one), no proxy environment leak.
PROBE_CURL_OPTS=(-skfL -m 8 --noproxy "*")
# ────────────────────────────────────────────────────────────────────
# Container-state oracle
# ────────────────────────────────────────────────────────────────────
# True iff `name` is currently in the running state per podman.
probe_container_running() {
local name="$1"
[[ "$(podman inspect --format '{{.State.Running}}' "$name" 2>/dev/null)" == "true" ]]
}
# ────────────────────────────────────────────────────────────────────
# URL probes
# ────────────────────────────────────────────────────────────────────
# Probe an HTTPS URL — assert 200 and non-empty body.
# Usage: probe_https_200 URL "human description"
probe_https_200() {
local url="$1"
local label="${2:-$url}"
local body status
body=$(curl "${PROBE_CURL_OPTS[@]}" -w '%{http_code}' "$url" 2>/dev/null) || {
echo "probe_https_200: $label ($url) — curl failed (network/timeout)" >&2
return 1
}
status="${body: -3}"
body="${body:0:-3}"
if [[ "$status" != "200" ]]; then
echo "probe_https_200: $label ($url) returned $status (want 200)" >&2
return 1
fi
if [[ -z "$body" ]]; then
echo "probe_https_200: $label ($url) returned empty body" >&2
return 1
fi
return 0
}
# Probe a URL backed by a container — skip if container is not running,
# assert 200 if it is. This is the standard shape for app UI tests.
# Usage: probe_app_url CONTAINER URL "human description"
probe_app_url() {
local container="$1"
local url="$2"
local label="${3:-$url}"
if ! probe_container_running "$container"; then
skip "$label: backing container '$container' is not running"
fi
# An app's proxy/UI takes time to serve 200 after a (re)start — the backend
# may still be unlocking/syncing (lnd) and the companion nginx reloading.
# Retry up to ~90s rather than single-shot, so a readiness race isn't a fail.
local deadline=$(( $(date +%s) + 90 ))
while (( $(date +%s) < deadline )); do
if probe_https_200 "$url" "$label"; then
return 0
fi
sleep 3
done
run probe_https_200 "$url" "$label"
[ "$status" -eq 0 ]
}
# Probe the archipelago dashboard itself (the SPA shell at https://node/).
# Asserts 200 and that the body looks like the Vue index, not an nginx
# default page. Catches "frontend tarball was extracted with the wrong
# layout" — see feedback_release_tarball_layout.md.
probe_dashboard_shell() {
local host="${ARCHY_HOST:-127.0.0.1}"
local url="${ARCHY_SCHEME:-https}://$host/"
local body
body=$(curl "${PROBE_CURL_OPTS[@]}" "$url" 2>/dev/null) || {
echo "probe_dashboard_shell: $url — curl failed" >&2
return 1
}
# Vue shell carries one of these markers: <div id="app">, the SPA bundle
# tag, or the manifest link. Nginx default does not.
if echo "$body" | grep -qE 'id="app"|<script.*\.js"|manifest\.webmanifest'; then
return 0
fi
echo "probe_dashboard_shell: $url returned 200 but body doesn't look like the Vue shell" >&2
echo "first 200 bytes: ${body:0:200}" >&2
return 1
}
# Probe the catalog endpoint that the dashboard uses to populate tiles.
# Returns 0 if catalog is reachable AND has at least one entry.
probe_dashboard_catalog() {
local host="${ARCHY_HOST:-127.0.0.1}"
local body
body=$(curl "${PROBE_CURL_OPTS[@]}" "${ARCHY_SCHEME:-https}://$host/catalog.json" 2>/dev/null) || {
echo "probe_dashboard_catalog: /catalog.json fetch failed" >&2
return 1
}
if ! echo "$body" | jq -e 'length > 0' >/dev/null 2>&1; then
echo "probe_dashboard_catalog: /catalog.json is not a non-empty array/object" >&2
return 1
fi
return 0
}
tests/lifecycle/os-audit.sh
Executable
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env bash
# tests/lifecycle/os-audit.sh — one non-destructive OS-wide health gate.
#
# Ties together, in a single pass with one scorecard + exit code:
# A. Backend / RPC health — node is up, not wedged mid-OTA, core daemons answer
# B. All-apps lifecycle audit — every catalog app: valid state, real health,
# reachable launch URL, populated launch metadata
# (delegates to remote-lifecycle.sh, audit-only)
# C. FM-guards — the concrete failure modes that have bitten the
# fleet: port-drift (FM8), secret-completeness (FM2),
# orphaned container states (FM9), OTA wedge (FM12)
#
# Everything here is READ-ONLY: no install/stop/start/uninstall, no service bounce.
# Safe to run against a live production node. It is the per-boot building block the
# reboot-survival harness (L3) calls after each reboot.
#
# Env:
# ARCHY_HOST (default 127.0.0.1)
# ARCHY_SCHEME (default https; use http for .116 / nginx-:80-only nodes)
# ARCHY_PASSWORD (required)
# ARCHY_LOCAL (auto: 1 when ARCHY_HOST is loopback) — gates host-only podman checks
#
# Usage:
# ARCHY_HOST=127.0.0.1 ARCHY_SCHEME=http ARCHY_PASSWORD=... tests/lifecycle/os-audit.sh
#
# Exit: 0 = every section green; 1 = one or more checks failed; 2 = setup/usage error.
set -uo pipefail
HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ARCHY_HOST="${ARCHY_HOST:-127.0.0.1}"
ARCHY_SCHEME="${ARCHY_SCHEME:-https}"
ARCHY_PASSWORD="${ARCHY_PASSWORD:-}"
BASE_URL="${ARCHY_SCHEME}://${ARCHY_HOST}"
# Host-only checks (podman sweeps) make sense only when this script runs ON the node.
if [[ -z "${ARCHY_LOCAL:-}" ]]; then
case "$ARCHY_HOST" in
127.0.0.1|localhost|::1) ARCHY_LOCAL=1 ;;
*) ARCHY_LOCAL=0 ;;
esac
fi
if [[ -z "$ARCHY_PASSWORD" ]]; then
echo "ARCHY_PASSWORD env var must be set." >&2
exit 2
fi
for tool in curl jq; do
command -v "$tool" >/dev/null 2>&1 || { echo "missing required tool: $tool" >&2; exit 2; }
done
# ── scorecard state ───────────────────────────────────────────────────────────
PASS=0; FAIL=0; WARN=0
declare -a RESULTS=()
record() { # record <PASS|FAIL|WARN> <label> [detail]
local status="$1" label="$2" detail="${3:-}"
case "$status" in
PASS) PASS=$((PASS+1)) ;;
FAIL) FAIL=$((FAIL+1)) ;;
WARN) WARN=$((WARN+1)) ;;
esac
RESULTS+=("$(printf '%-4s %-38s %s' "$status" "$label" "$detail")")
printf ' [%s] %s %s\n' "$status" "$label" "$detail"
}
# ── minimal RPC client (session + CSRF) ────────────────────────────────────────
SESSION=""; CSRF=""
rpc_login() {
local hdr; hdr=$(mktemp)
curl -sk -D "$hdr" -X POST "${BASE_URL}/rpc/v1" -H 'Content-Type: application/json' \
-d "$(jq -nc --arg p "$ARCHY_PASSWORD" '{jsonrpc:"2.0",id:1,method:"auth.login",params:{password:$p}}')" \
-o /dev/null 2>/dev/null
SESSION=$(grep -i '^set-cookie: session=' "$hdr" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r')
CSRF=$(grep -i '^set-cookie: csrf_token=' "$hdr" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r')
rm -f "$hdr"
[[ -n "$SESSION" && -n "$CSRF" ]]
}
# rpc <method> [params-json] -> prints raw JSON response
rpc() {
local method="$1" params="${2:-{\}}"
curl -sk -X POST "${BASE_URL}/rpc/v1" -H 'Content-Type: application/json' \
-H "Cookie: session=${SESSION}; csrf_token=${CSRF}" -H "X-CSRF-Token: ${CSRF}" \
-d "$(jq -nc --arg m "$method" --argjson p "$params" '{jsonrpc:"2.0",id:2,method:$m,params:$p}')" 2>/dev/null
}
# rpc_ok <method> [params] -> 0 if a result came back with no error
rpc_ok() {
local resp; resp=$(rpc "$@")
[[ -n "$resp" ]] && [[ "$(jq -r '.error // empty' <<<"$resp" 2>/dev/null)" == "" ]] \
&& [[ "$(jq -r 'has("result")' <<<"$resp" 2>/dev/null)" == "true" ]]
}
# ══ Section A — Backend / RPC health ═══════════════════════════════════════════
section_a() {
echo
echo "== A. Backend / RPC health =="
# unauth health probe first (doesn't need a session)
local health; health=$(curl -sk -X POST "${BASE_URL}/rpc/v1" -H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","id":1,"method":"health","params":{}}' 2>/dev/null)
if [[ "$(jq -r '.result.status // empty' <<<"$health" 2>/dev/null)" =~ ^(ok|degraded)$ ]]; then
record PASS "node responds (health)" "status=$(jq -r '.result.status' <<<"$health")"
else
record FAIL "node responds (health)" "no/invalid health response — node down?"
return
fi
if ! rpc_login; then
record FAIL "auth.login" "could not establish session (wrong password or rate-limited)"
return
fi
record PASS "auth.login" "session established"
# FM12 — OTA must not be wedged mid-apply.
# NB: must use has() not `//` — jq's `//` treats a legit `false` as empty and
# would fall through to "unknown" on a perfectly healthy node.
local us; us=$(rpc update.status)
local inprog; inprog=$(jq -r '
if (.result|type=="object") and (.result|has("update_in_progress")) then .result.update_in_progress
elif (.result|type=="object") and (.result|has("in_progress")) then .result.in_progress
else "unknown" end' <<<"$us" 2>/dev/null)
if [[ "$inprog" == "false" ]]; then
record PASS "OTA not wedged (update.status)" "update_in_progress=false"
elif [[ "$inprog" == "unknown" ]]; then
record WARN "OTA not wedged (update.status)" "could not read update_in_progress"
else
record FAIL "OTA not wedged (update.status)" "update_in_progress=$inprog (FM12 wedge)"
fi
# Core daemons answer (only assert for ones present on this node)
if rpc_ok bitcoin.getinfo || rpc_ok bitcoin.relay-status; then
record PASS "bitcoin RPC reachable" ""
else
record WARN "bitcoin RPC reachable" "bitcoin.getinfo/relay-status did not answer (not installed?)"
fi
# LND wallet must be UNLOCKED. NB: lnd.getinfo masks a locked wallet (it
# returns an all-zero success, error:null), so it can't detect the lock. Probe
# the actual receive path (lnd.newaddress) instead: a LOCKED wallet returns the
# LND_WALLET_LOCKED reason code — the exact fleet-wide receive breakage. A
# locked wallet is a hard FAIL; "not installed" is a WARN. (newaddress derives
# a fresh address — harmless; LND tolerates address gaps.)
if rpc_ok lnd.getinfo; then
local na; na=$(rpc lnd.newaddress)
if grep -qE "LND_WALLET_LOCKED|wallet is locked|WALLET_LOCKED" <<<"$na"; then
record FAIL "lnd wallet unlocked (lnd.newaddress)" "wallet LOCKED — auto-unlock failed (Bitcoin-receive broken)"
elif [[ "$(jq -r '(has("result") and (.result!=null))' <<<"$na" 2>/dev/null)" == "true" ]]; then
record PASS "lnd wallet unlocked (lnd.newaddress)" ""
else
record WARN "lnd wallet unlocked (lnd.newaddress)" "newaddress: $(jq -rc '.error.message // "no address"' <<<"$na" 2>/dev/null | head -c 60)"
fi
else
record WARN "lnd RPC reachable" "lnd.getinfo did not answer (not installed?)"
fi
if rpc_ok system.stats || rpc_ok system.get-metrics; then
record PASS "system metrics reachable" ""
else
record WARN "system metrics reachable" "system.stats/get-metrics did not answer"
fi
# FM13 — disk pressure early-warning (best-effort; field names vary by version)
local ds; ds=$(rpc system.disk-status)
local usep; usep=$(jq -r '[.result.use_percent,.result.used_percent,.result.percent]|map(select(.!=null))|first // empty' <<<"$ds" 2>/dev/null)
if [[ -n "$usep" ]]; then
if (( ${usep%.*} >= 90 )); then
record FAIL "disk pressure (system.disk-status)" "${usep}% used (FM13 risk)"
else
record PASS "disk pressure (system.disk-status)" "${usep}% used"
fi
fi
}
# ══ Section B — All-apps lifecycle audit (delegates to remote-lifecycle.sh) ═════
section_b() {
echo
echo "== B. All-apps lifecycle audit (non-destructive, all catalog apps) =="
local out rc
# No ARCHY_APPS + no ARCHY_FULL_LIFECYCLE => audit every catalog app (audit_app).
out=$(ARCHY_HOST="$ARCHY_HOST" ARCHY_SCHEME="$ARCHY_SCHEME" ARCHY_PASSWORD="$ARCHY_PASSWORD" \
ARCHY_APPS="" ARCHY_FULL_LIFECYCLE=0 \
"$HERE/remote-lifecycle.sh" 2>&1)
rc=$?
# Surface the per-app lines but drop the noisy optional-probe jq parse errors.
echo "$out" | grep -vE '^jq: (parse )?error' | sed 's/^/ /'
if (( rc == 0 )); then
record PASS "broad all-apps audit" "remote-lifecycle.sh exit 0"
else
local n; n=$(echo "$out" | grep -oE 'FAILED checks: [0-9]+' | grep -oE '[0-9]+' | tail -1)
record FAIL "broad all-apps audit" "remote-lifecycle.sh exit $rc (${n:-?} app checks failed)"
fi
}
# ══ Section C — FM-guards ══════════════════════════════════════════════════════
run_bats_guard() { # run_bats_guard <suite> <label> <fm>
local suite="$1" label="$2" fm="$3" out rc
if ! command -v bats >/dev/null 2>&1; then
record WARN "$label" "bats not installed — $fm guard skipped"
return
fi
out=$(ARCHY_HOST="$ARCHY_HOST" ARCHY_SCHEME="$ARCHY_SCHEME" ARCHY_PASSWORD="$ARCHY_PASSWORD" \
"$HERE/run.sh" "$suite" 2>&1); rc=$?
if (( rc == 0 )); then
record PASS "$label" "$fm guard green"
else
record FAIL "$label" "$fm — $(echo "$out" | grep -E '^not ok' | head -1)"
fi
}
section_c() {
echo
echo "== C. FM-guards (the concrete fleet failure modes) =="
run_bats_guard port-drift "port bindings match manifest" "FM8"
run_bats_guard secret-completeness "all referenced secrets exist" "FM2"
# FM9 — orphaned container states (host-only: needs local podman)
if [[ "$ARCHY_LOCAL" == "1" ]] && command -v podman >/dev/null 2>&1; then
local orphans
orphans=$(podman ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null \
| grep -iE '(^| )(stopping|removing|created)( |$)' || true)
if [[ -z "$orphans" ]]; then
record PASS "no orphaned container states" "no stopping/removing/created"
else
record FAIL "no orphaned container states" "FM9: $(echo "$orphans" | tr '\n' ';')"
fi
else
record WARN "no orphaned container states" "remote node — host podman sweep skipped"
fi
}
# ── run ────────────────────────────────────────────────────────────────────────
echo "=============================================================="
echo " OS-wide audit — ${BASE_URL} ($(date '+%Y-%m-%d %H:%M:%S'))"
echo " local=${ARCHY_LOCAL}"
echo "=============================================================="
section_a
# Only proceed to apps/FM-guards if the node itself answered.
if (( FAIL == 0 )) || [[ -n "$SESSION" ]]; then
section_b
section_c
fi
echo
echo "=============================================================="
echo " SCORECARD: ${PASS} pass / ${FAIL} fail / ${WARN} warn"
echo "=============================================================="
printf '%s\n' "${RESULTS[@]}"
echo
if (( FAIL > 0 )); then
echo "RESULT: FAIL ($FAIL critical checks failed)"
exit 1
fi
echo "RESULT: PASS"
exit 0
tests/lifecycle/remote-lifecycle.sh
Executable
+630
View File
@@ -0,0 +1,630 @@
#!/usr/bin/env bash
# Remote app lifecycle runner for Archipelago nodes.
#
# Exercises the same public surface the UI uses:
# - JSON-RPC package.install/start/stop/restart/uninstall
# - HTTPS/direct-port launch probes from appSessionConfig.ts
#
# Default mode is audit-only. Use ARCHY_FULL_LIFECYCLE=1 for destructive
# preserve-data cycles: install -> launch -> stop -> start -> restart ->
# uninstall(preserve_data=true) -> reinstall -> launch.
set -euo pipefail
ARCHY_HOST="${ARCHY_HOST:-}"
ARCHY_SCHEME="${ARCHY_SCHEME:-https}"
ARCHY_PASSWORD="${ARCHY_PASSWORD:-}"
ARCHY_ITERATIONS="${ARCHY_ITERATIONS:-1}"
ARCHY_FULL_LIFECYCLE="${ARCHY_FULL_LIFECYCLE:-0}"
ARCHY_APPS="${ARCHY_APPS:-}"
ARCHY_TIMEOUT="${ARCHY_TIMEOUT:-900}"
ARCHY_STABILITY_SECONDS="${ARCHY_STABILITY_SECONDS:-5}"
ARCHY_ALLOW_BITCOIN_SWAP="${ARCHY_ALLOW_BITCOIN_SWAP:-0}"
ARCHY_APP_CATALOG="${ARCHY_APP_CATALOG:-}"
ARCHY_PRUNED_NODE="${ARCHY_PRUNED_NODE:-auto}"
if [[ -z "$ARCHY_HOST" || -z "$ARCHY_PASSWORD" ]]; then
echo "ARCHY_HOST and ARCHY_PASSWORD are required" >&2
exit 2
fi
if ! [[ "$ARCHY_ITERATIONS" =~ ^[1-9][0-9]*$ ]]; then
echo "ARCHY_ITERATIONS must be a positive integer" >&2
exit 2
fi
if ! [[ "$ARCHY_STABILITY_SECONDS" =~ ^[0-9]+$ ]]; then
echo "ARCHY_STABILITY_SECONDS must be a non-negative integer" >&2
exit 2
fi
BASE_URL="${ARCHY_SCHEME}://${ARCHY_HOST}"
SESSION=""
CSRF=""
CATALOG_FILE=""
ALL_APPS=(
bitcoin-knots
btcpay-server
lnd
mempool
homeassistant
grafana
searxng
ollama
nextcloud
vaultwarden
jellyfin
photoprism
immich
filebrowser
nginx-proxy-manager
portainer
tailscale
uptime-kuma
electrumx
fedimint
indeedhub
dwn
botfights
gitea
)
ARCHIVAL_ONLY_APPS=(
electrumx
mempool
)
app_in_list() {
local needle="$1"
shift
local item
for item in "$@"; do
[[ "$item" == "$needle" ]] && return 0
done
return 1
}
fetch_catalog() {
CATALOG_FILE=$(mktemp)
if [[ -n "$ARCHY_APP_CATALOG" ]]; then
cp "$ARCHY_APP_CATALOG" "$CATALOG_FILE"
return 0
fi
if curl -skfL --connect-timeout 8 -m 30 "${BASE_URL}/api/app-catalog" -o "$CATALOG_FILE" \
&& jq -e '.apps | length > 0' "$CATALOG_FILE" >/dev/null; then
return 0
fi
curl -skfL --connect-timeout 8 -m 30 "${BASE_URL}/catalog.json" -o "$CATALOG_FILE"
jq -e '.apps | length > 0' "$CATALOG_FILE" >/dev/null
}
catalog_app_ids() {
jq -r '.apps[] | select((.dockerImage // "") != "") | .id' "$CATALOG_FILE"
}
catalog_app_json() {
local app="$1"
[[ -n "$CATALOG_FILE" && -r "$CATALOG_FILE" ]] || return 1
jq -c --arg app "$app" '
.registry as $registry
| .apps[]
| select(.id == $app)
| .dockerImage = (if ((.dockerImage // "") | contains("/")) then .dockerImage else ($registry + "/" + .dockerImage) end)
' "$CATALOG_FILE" | head -n 1
}
is_pruned_node() {
case "$ARCHY_PRUNED_NODE" in
1|true|yes) return 0 ;;
0|false|no) return 1 ;;
esac
local pass body
pass=$(ssh "${ARCHY_HOST}" 'sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null || cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null' 2>/dev/null || true)
[[ -n "$pass" ]] || return 1
body=$(curl -fsS --max-time 20 \
--user "archipelago:${pass}" \
--data-binary '{"jsonrpc":"1.0","id":"remote-lifecycle","method":"getblockchaininfo","params":[]}' \
-H 'content-type: text/plain;' \
"http://${ARCHY_HOST}:8332/" 2>/dev/null || true)
printf '%s' "$body" | jq -e '.result.pruned == true' >/dev/null 2>&1
}
image_for() {
case "$1" in
bitcoin-knots) echo "source.archipelago-foundation.org/lfg2025/bitcoin-knots:latest" ;;
bitcoin-core) echo "docker.io/bitcoin/bitcoin:28.4" ;;
btcpay-server) echo "docker.io/btcpayserver/btcpayserver:2.4.2" ;;
lnd) echo "source.archipelago-foundation.org/lfg2025/lnd:v0.18.4-beta" ;;
mempool) echo "source.archipelago-foundation.org/lfg2025/mempool-frontend:v3.0.0" ;;
homeassistant) echo "source.archipelago-foundation.org/lfg2025/home-assistant:2024.1" ;;
grafana) echo "source.archipelago-foundation.org/lfg2025/grafana:10.2.0" ;;
searxng) echo "source.archipelago-foundation.org/lfg2025/searxng:latest" ;;
ollama) echo "source.archipelago-foundation.org/lfg2025/ollama:latest" ;;
nextcloud) echo "source.archipelago-foundation.org/lfg2025/nextcloud:28" ;;
vaultwarden) echo "source.archipelago-foundation.org/lfg2025/vaultwarden:1.30.0-alpine" ;;
jellyfin) echo "source.archipelago-foundation.org/lfg2025/jellyfin:10.8.13" ;;
photoprism) echo "source.archipelago-foundation.org/lfg2025/photoprism:240915" ;;
immich) echo "source.archipelago-foundation.org/lfg2025/immich-server:release" ;;
filebrowser) echo "source.archipelago-foundation.org/lfg2025/filebrowser:v2.27.0" ;;
nginx-proxy-manager) echo "source.archipelago-foundation.org/lfg2025/nginx-proxy-manager:latest" ;;
portainer) echo "source.archipelago-foundation.org/lfg2025/portainer:latest" ;;
uptime-kuma) echo "source.archipelago-foundation.org/lfg2025/uptime-kuma:1" ;;
tailscale) echo "source.archipelago-foundation.org/lfg2025/tailscale:stable" ;;
electrumx) echo "source.archipelago-foundation.org/lfg2025/electrumx:v1.18.0" ;;
fedimint) echo "source.archipelago-foundation.org/lfg2025/fedimintd:v0.10.0" ;;
indeedhub) echo "source.archipelago-foundation.org/lfg2025/indeedhub:1.0.0" ;;
botfights) echo "source.archipelago-foundation.org/lfg2025/botfights:1.1.0" ;;
gitea) echo "docker.io/gitea/gitea:1.23" ;;
*) return 1 ;;
esac
}
launch_url_for() {
case "$1" in
bitcoin-knots|bitcoin-core|bitcoin-ui) echo "http://${ARCHY_HOST}:8334/" ;;
lnd|archy-lnd-ui) echo "http://${ARCHY_HOST}:18083/" ;;
electrumx|electrs|mempool-electrs|archy-electrs-ui) echo "http://${ARCHY_HOST}:50002/" ;;
mempool|mempool-web|archy-mempool-web) echo "http://${ARCHY_HOST}:4080/" ;;
fedimint|fedimintd) echo "http://${ARCHY_HOST}:8175/" ;;
fedimint-gateway) echo "http://${ARCHY_HOST}:8176/" ;;
filebrowser) echo "http://${ARCHY_HOST}:8083/" ;;
grafana) echo "http://${ARCHY_HOST}:3000/" ;;
btcpay-server) echo "http://${ARCHY_HOST}:23000/" ;;
jellyfin) echo "http://${ARCHY_HOST}:8096/" ;;
searxng) echo "http://${ARCHY_HOST}:8888/" ;;
ollama) echo "http://${ARCHY_HOST}:11434/" ;;
immich|immich_server) echo "http://${ARCHY_HOST}:2283/" ;;
portainer) echo "http://${ARCHY_HOST}:9000/" ;;
nginx-proxy-manager) echo "http://${ARCHY_HOST}:8081/" ;;
tailscale) echo "http://${ARCHY_HOST}:8240/" ;;
uptime-kuma) echo "http://${ARCHY_HOST}:3002/" ;;
homeassistant) echo "http://${ARCHY_HOST}:8123/" ;;
vaultwarden) echo "http://${ARCHY_HOST}:8082/" ;;
photoprism) echo "http://${ARCHY_HOST}:2342/" ;;
dwn) echo "http://${ARCHY_HOST}:3100/" ;;
botfights) echo "http://${ARCHY_HOST}:9100/" ;;
gitea) echo "http://${ARCHY_HOST}:3001/" ;;
indeedhub) echo "http://${ARCHY_HOST}:7778/" ;;
*) return 1 ;;
esac
}
rpc_login() {
local headers body err
headers=$(mktemp)
body=$(curl -sk -D "$headers" -X POST "${BASE_URL}/rpc/v1" \
-H 'Content-Type: application/json' \
--data-raw "$(jq -nc --arg p "$ARCHY_PASSWORD" '{jsonrpc:"2.0",method:"auth.login",params:{password:$p},id:1}')")
err=$(printf '%s' "$body" | jq -r '.error.message // empty')
if [[ -n "$err" ]]; then
rm -f "$headers"
echo "login failed on $ARCHY_HOST: $err" >&2
return 1
fi
SESSION=$(grep -i '^set-cookie: session=' "$headers" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r')
CSRF=$(grep -i '^set-cookie: csrf_token=' "$headers" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r')
rm -f "$headers"
[[ -n "$SESSION" && -n "$CSRF" ]]
}
rpc_call() {
local method="$1" params="${2:-null}" id="${3:-2}"
local payload
if [[ "$params" == "null" ]]; then
payload=$(jq -nc --arg m "$method" --argjson id "$id" '{jsonrpc:"2.0",method:$m,id:$id}')
else
payload=$(jq -nc --arg m "$method" --argjson p "$params" --argjson id "$id" '{jsonrpc:"2.0",method:$m,params:$p,id:$id}')
fi
curl -sk -X POST "${BASE_URL}/rpc/v1" \
--connect-timeout 8 \
-m "${ARCHY_RPC_TIMEOUT:-60}" \
-H 'Content-Type: application/json' \
-H "Cookie: session=${SESSION}; csrf_token=${CSRF}" \
-H "X-CSRF-Token: ${CSRF}" \
--data-raw "$payload"
}
rpc_result() {
local resp err
resp=$(rpc_call "$@")
err=$(printf '%s' "$resp" | jq -r '.error.message // empty')
if [[ -n "$err" ]]; then
echo "$err" >&2
return 1
fi
printf '%s' "$resp" | jq '.result'
}
container_state() {
local app="$1"
rpc_result container-list | jq -r --arg app "$app" '
(map(select(.name == $app or .id == $app)) | first | .state // "absent") | ascii_downcase
'
}
container_health() {
local app="$1" health
health=$(
ARCHY_RPC_TIMEOUT="${ARCHY_HEALTH_RPC_TIMEOUT:-20}" \
rpc_result container-health "$(jq -nc --arg app "$app" '{app_id:$app}')" \
| jq -r --arg app "$app" '(.[$app] // "") | if . == "" then "unknown" else ascii_downcase end'
) || health=unknown
if [[ "$app" == "indeedhub" && "$health" != "healthy" ]] && probe_launch "$app" >/dev/null 2>&1; then
health=healthy
fi
printf '%s\n' "$health"
}
assert_container_healthy() {
local app="$1" health
health=$(container_health "$app" 2>/dev/null || echo unknown)
case "$health" in
healthy) return 0 ;;
*) echo "bad health: $app is $health" >&2; return 1 ;;
esac
}
wait_container_healthy() {
local app="$1" timeout="${2:-$ARCHY_TIMEOUT}" deadline health
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
health=$(container_health "$app" 2>/dev/null || echo unknown)
if [[ "$health" == "healthy" ]]; then return 0; fi
sleep 5
done
echo "bad health: $app is ${health:-unknown}" >&2
return 1
}
observe_stable() {
local app="$1" seconds="${2:-$ARCHY_STABILITY_SECONDS}" deadline state
(( seconds == 0 )) && return 0
deadline=$(( $(date +%s) + seconds ))
while (( $(date +%s) < deadline )); do
state=$(container_state "$app" 2>/dev/null || echo unknown)
if [[ "$state" != "running" ]]; then
if [[ "$app" == "indeedhub" ]] && probe_launch "$app" >/dev/null 2>&1; then
sleep 5
continue
fi
echo "stability failed: $app left running state (last=$state)" >&2
return 1
fi
assert_container_healthy "$app" || return 1
sleep 5
done
}
wait_state() {
local app="$1" target="$2" timeout="${3:-$ARCHY_TIMEOUT}"
local deadline state
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
state=$(container_state "$app" 2>/dev/null || echo unknown)
if [[ "$target" == "absent" && "$state" == "absent" ]]; then return 0; fi
if [[ "$target" == "stopped" && "$state" == "absent" ]]; then return 0; fi
if [[ "$target" != "absent" && "$state" == "$target" ]]; then return 0; fi
if [[ "$app" == "indeedhub" && "$target" == "running" ]] && probe_launch "$app" >/dev/null 2>&1; then return 0; fi
sleep 5
done
echo "$app did not reach $target within ${timeout}s (last=$state)" >&2
return 1
}
wait_absent_settled() {
local app="$1" timeout="${2:-$ARCHY_TIMEOUT}"
local deadline state seen_absent=0
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
state=$(container_state "$app" 2>/dev/null || echo unknown)
if [[ "$state" == "absent" ]]; then
if (( seen_absent == 1 )); then return 0; fi
seen_absent=1
else
seen_absent=0
fi
sleep 5
done
echo "$app did not settle absent within ${timeout}s (last=$state)" >&2
return 1
}
wait_not_installing() {
local app="$1" timeout="${2:-$ARCHY_TIMEOUT}"
local deadline state
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
state=$(container_state "$app" 2>/dev/null || echo unknown)
case "$state" in
installing|starting|restarting|updating) sleep 5 ;;
*) return 0 ;;
esac
done
echo "$app did not settle from install transition within ${timeout}s (last=$state)" >&2
return 1
}
probe_launch() {
local app="$1" url code bytes body
url=$(launch_url_for "$app") || return 0
body=$(mktemp)
code=$(curl -skL --connect-timeout 8 -m 20 -o "$body" -w '%{http_code}' "$url" || true)
bytes=$(wc -c < "$body" 2>/dev/null || printf 0)
if [[ "$code" != "200" || "$bytes" -eq 0 ]]; then
echo "launch failed: $app $url status=$code bytes=$bytes" >&2
rm -f "$body"
return 1
fi
case "$app" in
lnd) probe_lnd_wallet_connect "$body" || { rm -f "$body"; return 1; } ;;
electrumx|electrs|mempool-electrs) probe_electrum_wallet_connect "$body" || { rm -f "$body"; return 1; } ;;
indeedhub) probe_indeedhub_nostr_signer "$body" || { rm -f "$body"; return 1; } ;;
tailscale) probe_tailscale_login_ui "$body" || { rm -f "$body"; return 1; } ;;
esac
rm -f "$body"
}
wait_launch() {
local app="$1" timeout="${2:-$ARCHY_TIMEOUT}" deadline
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
if probe_launch "$app" >/dev/null 2>&1; then return 0; fi
sleep 5
done
probe_launch "$app"
}
assert_launch_metadata() {
local app="$1" timeout="${2:-$ARCHY_TIMEOUT}" deadline lan
launch_url_for "$app" >/dev/null 2>&1 || return 0
deadline=$(( $(date +%s) + timeout ))
while (( $(date +%s) < deadline )); do
lan=$(rpc_result container-list | jq -r --arg app "$app" '
(map(select(.name == $app or .id == $app)) | first | .lan_address // "")
')
if [[ -n "$lan" && "$lan" != "null" ]]; then return 0; fi
sleep 5
done
if [[ -z "${lan:-}" || "$lan" == "null" ]]; then
echo "launch metadata missing: $app has no lan_address" >&2
return 1
fi
}
require_body() {
local body="$1" needle="$2" label="$3"
if ! grep -Fq "$needle" "$body"; then
echo "launch missing $label: $needle" >&2
return 1
fi
}
probe_lnd_wallet_connect() {
local body="$1" info err
require_body "$body" 'Connect Your Wallet' 'LND wallet heading' || return 1
require_body "$body" 'id="lndQrBox"' 'LND QR container' || return 1
require_body "$body" 'id="connHost"' 'LND host field' || return 1
require_body "$body" 'value="rest-tor"' 'LND REST Tor mode' || return 1
require_body "$body" 'value="grpc-tor"' 'LND gRPC Tor mode' || return 1
require_body "$body" 'value="rest-local"' 'LND REST local mode' || return 1
require_body "$body" 'value="grpc-local"' 'LND gRPC local mode' || return 1
require_body "$body" 'Copy lndconnect URI' 'LND connect URI button' || return 1
info=$(curl -skL --connect-timeout 8 -m 20 \
-H "Cookie: session=${SESSION}; csrf_token=${CSRF}" \
-H "X-CSRF-Token: ${CSRF}" \
"${BASE_URL}/lnd-connect-info" || true)
err=$(printf '%s' "$info" | jq -r '.error // empty' 2>/dev/null || true)
if [[ -n "$err" ]]; then
echo "lnd connect info error: $err" >&2
return 1
fi
printf '%s' "$info" | jq -e '
(.cert_base64url | type == "string" and length > 100) and
(.macaroon_base64url | type == "string" and length > 50) and
(.tor_onion | type == "string" and test("^[a-z2-7]+\\.onion$")) and
(.rest_port == 18080) and
(.grpc_port == 10009)
' >/dev/null || {
echo "lnd connect info incomplete: $info" >&2
return 1
}
}
probe_electrum_wallet_connect() {
local body="$1"
require_body "$body" 'Connect Your Wallet' 'Electrum wallet heading' || return 1
require_body "$body" 'id="qrLocalBox"' 'Electrum local QR container' || return 1
require_body "$body" 'id="qrTorBox"' 'Electrum Tor QR container' || return 1
require_body "$body" 'id="localAddress"' 'Electrum local address field' || return 1
require_body "$body" 'id="torAddress"' 'Electrum Tor address field' || return 1
require_body "$body" '50001' 'Electrum wallet port' || return 1
require_body "$body" 'renderQR' 'Electrum QR renderer' || return 1
curl -skL --connect-timeout 8 -m 20 -f "http://${ARCHY_HOST}:50002/qrcode.js" >/dev/null || {
echo "electrum qrcode.js unavailable" >&2
return 1
}
local status
status=$(curl -skL --connect-timeout 8 -m 20 "${BASE_URL}/electrs-status" || true)
printf '%s' "$status" | jq -e '(.tor_onion | type == "string" and test("^[a-z2-7]+\\.onion$"))' >/dev/null || {
echo "electrum tor connection info incomplete: $status" >&2
return 1
}
}
probe_indeedhub_nostr_signer() {
local body="$1" provider pubkey signed now
require_body "$body" '/nostr-provider.js' 'IndeedHub Nostr provider injection' || return 1
provider=$(curl -skL --connect-timeout 8 -m 20 "http://${ARCHY_HOST}:7778/nostr-provider.js" || true)
if [[ -z "$provider" ]]; then
echo "indeedhub nostr-provider.js unavailable" >&2
return 1
fi
printf '%s' "$provider" | grep -Eq 'window\.nostr|nostr' || {
echo "indeedhub nostr-provider.js does not look like a Nostr signer bridge" >&2
return 1
}
pubkey=$(rpc_result node.nostr-pubkey | jq -r '.nostr_pubkey // empty')
if ! [[ "$pubkey" =~ ^[0-9a-fA-F]{64}$ ]]; then
echo "indeedhub Nostr signer pubkey unavailable: $pubkey" >&2
return 1
fi
now=$(date +%s)
signed=$(rpc_result node.nostr-sign "$(jq -nc --argjson created_at "$now" '{event:{kind:1,created_at:$created_at,tags:[],content:"archy lifecycle indeedhub signer probe"}}')")
printf '%s' "$signed" | jq -e --arg pubkey "$pubkey" '
.pubkey == $pubkey and
(.id | type == "string" and test("^[0-9a-f]{64}$")) and
(.sig | type == "string" and test("^[0-9a-f]{128}$")) and
.content == "archy lifecycle indeedhub signer probe"
' >/dev/null || {
echo "indeedhub Nostr signer did not return a valid signed event: $signed" >&2
return 1
}
}
probe_tailscale_login_ui() {
local body="$1"
if grep -Eiq 'tailscale|login|log in|sign in|authenticate|authorize|auth key|connect' "$body"; then
return 0
fi
echo "tailscale launch did not present login/auth UI content" >&2
return 1
}
install_app() {
local app="$1" app_json image params
app_json=$(catalog_app_json "$app" || true)
if [[ -n "$app_json" ]]; then
params=$(printf '%s' "$app_json" | jq -c '{id, dockerImage, version, containerConfig} | with_entries(select(.value != null))')
else
image=$(image_for "$app")
params=$(jq -nc --arg id "$app" --arg img "$image" '{id:$id,dockerImage:$img,version:"latest"}')
fi
rpc_result package.install "$params" >/dev/null
}
expect_archival_blocked_install() {
local app="$1" app_json resp err params
app_json=$(catalog_app_json "$app")
params=$(printf '%s' "$app_json" | jq -c '{id, dockerImage, version, containerConfig} | with_entries(select(.value != null))')
resp=$(rpc_call package.install "$params")
err=$(printf '%s' "$resp" | jq -r '.error.message // empty')
if [[ "$err" != *"Requires an archival Bitcoin node"* && "$err" != *"requires an archival Bitcoin node"* && "$err" != *"running pruned Bitcoin"* ]]; then
echo "expected archival Bitcoin block for $app, got: $resp" >&2
return 1
fi
}
start_app() { rpc_result package.start "$(jq -nc --arg id "$1" '{id:$id}')" >/dev/null; }
stop_app() { rpc_result package.stop "$(jq -nc --arg id "$1" '{id:$id}')" >/dev/null; }
restart_app() { rpc_result package.restart "$(jq -nc --arg id "$1" '{id:$id}')" >/dev/null; }
uninstall_app() { rpc_result package.uninstall "$(jq -nc --arg id "$1" '{id:$id,preserve_data:true}')" >/dev/null; }
audit_app() {
local app="$1" state rc=0
state=$(container_state "$app" || echo unknown)
printf '%-22s state=%s\n' "$app" "$state"
case "$state" in
absent) ;;
running)
wait_container_healthy "$app" || rc=1
wait_launch "$app" || rc=1
assert_launch_metadata "$app" || rc=1
observe_stable "$app" || rc=1
;;
*) echo "bad state: $app is $state" >&2; rc=1 ;;
esac
return "$rc"
}
full_lifecycle_app() {
local app="$1"
if [[ "$app" == "bitcoin-core" && "$ARCHY_ALLOW_BITCOIN_SWAP" != "1" ]]; then
echo "skip bitcoin-core: set ARCHY_ALLOW_BITCOIN_SWAP=1 to test mutually-exclusive Bitcoin implementation"
return 0
fi
if app_in_list "$app" "${ARCHIVAL_ONLY_APPS[@]}" && is_pruned_node; then
echo "== $app: expect archival Bitcoin block =="
expect_archival_blocked_install "$app"
return $?
fi
echo "== $app: install =="
install_app "$app" || return 1
wait_not_installing "$app" || return 1
wait_state "$app" running || return 1
wait_container_healthy "$app" || return 1
wait_launch "$app" || return 1
assert_launch_metadata "$app" || return 1
observe_stable "$app" || return 1
echo "== $app: stop =="
stop_app "$app" || return 1
wait_state "$app" stopped 300 || return 1
echo "== $app: start =="
start_app "$app" || return 1
wait_state "$app" running || return 1
wait_container_healthy "$app" || return 1
wait_launch "$app" || return 1
assert_launch_metadata "$app" || return 1
observe_stable "$app" || return 1
echo "== $app: restart =="
restart_app "$app" || return 1
wait_state "$app" running || return 1
wait_container_healthy "$app" || return 1
wait_launch "$app" || return 1
assert_launch_metadata "$app" || return 1
observe_stable "$app" || return 1
echo "== $app: uninstall preserve_data =="
uninstall_app "$app" || return 1
wait_absent_settled "$app" 600 || return 1
echo "== $app: reinstall =="
install_app "$app" || return 1
wait_not_installing "$app" || return 1
wait_state "$app" running || return 1
wait_container_healthy "$app" || return 1
wait_launch "$app" || return 1
assert_launch_metadata "$app" || return 1
observe_stable "$app" || return 1
}
apps=()
if [[ -n "$ARCHY_APPS" ]]; then
IFS=',' read -r -a apps <<< "$ARCHY_APPS"
fetch_catalog || true
elif [[ "$ARCHY_FULL_LIFECYCLE" == "1" ]]; then
fetch_catalog
mapfile -t apps < <(catalog_app_ids)
else
if fetch_catalog; then
mapfile -t apps < <(catalog_app_ids)
else
apps=("${ALL_APPS[@]}")
fi
fi
rpc_login
failed=0
for i in $(seq 1 "$ARCHY_ITERATIONS"); do
echo "### $ARCHY_HOST iteration $i / $ARCHY_ITERATIONS ###"
for app in "${apps[@]}"; do
if [[ "$ARCHY_FULL_LIFECYCLE" == "1" ]]; then
full_lifecycle_app "$app" || failed=$((failed + 1))
else
audit_app "$app" || failed=$((failed + 1))
fi
done
done
if (( failed > 0 )); then
echo "FAILED checks: $failed" >&2
exit 1
fi
echo "all checks passed"
tests/lifecycle/run-gate.sh
Executable
+232
View File
@@ -0,0 +1,232 @@
#!/usr/bin/env bash
# tests/lifecycle/run-gate.sh — loop the lifecycle harness N times (default 5×, the release gate).
#
# Each iteration: setup-teardown → run.sh (with the same args you'd pass
# to run.sh) → setup-teardown. Tallies pass/fail per iteration and prints a
# summary at the end. Returns non-zero if any iteration failed.
#
# Env:
# ARCHY_ITERATIONS (default: 5)
# ARCHY_FAIL_FAST=1 stop on first failed iteration
# ARCHY_GATE_CASCADE=1 after the 5× loop, run ONE cascade pass
# (uninstall→no-ghost→reinstall a throwaway
# app); requires ARCHY_ALLOW_DESTRUCTIVE=1
# ARCHY_PREFLIGHT=0 skip the host-readiness preflight
# ARCHY_MAX_LOAD load1 ceiling (default: nproc + 1)
# ARCHY_PREFLIGHT_SECS how long to wait for load to fall
# (default: 900)
# plus everything run.sh / lib/rpc.bash respects
# (ARCHY_PASSWORD, ARCHY_HOST, ARCHY_SCHEME, ARCHY_ALLOW_DESTRUCTIVE,
# ARCHY_ALLOW_CASCADE_DESTRUCTIVE, ARCHY_ALLOW_NOAUTH)
#
# Usage:
# tests/lifecycle/run-gate.sh # 5× full bats/ suite
# ARCHY_ITERATIONS=5 tests/lifecycle/run-gate.sh # 5× full suite
# tests/lifecycle/run-gate.sh bitcoin-knots # 5× a single suite
#
# Suggested release-gate invocation:
# ARCHY_PASSWORD=password123 ARCHY_ALLOW_DESTRUCTIVE=1 \
# tests/lifecycle/run-gate.sh
#
# Release-gate WITH the cascade tier (uninstall/reinstall regression guard):
# ARCHY_PASSWORD=password123 ARCHY_ALLOW_DESTRUCTIVE=1 ARCHY_GATE_CASCADE=1 \
# tests/lifecycle/run-gate.sh
set -euo pipefail
HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "$HERE"
ITER="${ARCHY_ITERATIONS:-5}"
if ! [[ "$ITER" =~ ^[1-9][0-9]*$ ]]; then
echo "ARCHY_ITERATIONS must be a positive integer, got: $ITER" >&2
exit 2
fi
passed=0
failed=0
failures=()
start=$(date +%s)
# Best-effort settle: wait for the backend stack to be healthy before an
# iteration starts, so back-to-back destructive iterations don't compound
# restart churn (lnd wallet-unlock + the 4-container mempool stack reconnect
# need time to recover). On-node gate only (localhost probes); never fails the
# run — just delays up to the deadline. Disable with ARCHY_SETTLE=0.
settle_stack() {
[[ "${ARCHY_SETTLE:-1}" == "1" && "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]] || return 0
# 300s (not 180s): on heavy nodes the immich stack's recovery after the prior
# iteration's archipelago-restart test (crash_recovery retries on a ~120s
# cadence) can take several minutes, and the next iteration's read-only
# lan_address probe false-fails if immich is still mid-boot. The settle is a
# cap, not a fixed wait — it returns the instant every probe is green.
local deadline=$(( $(date +%s) + ${ARCHY_SETTLE_SECS:-300} ))
while (( $(date +%s) < deadline )); do
local ok=1
# mempool-api + frontend + bitcoin-ui = good proxies for "stack reconnected"
curl -fsS -m 4 -o /dev/null "http://127.0.0.1:8999/api/v1/backend-info" 2>/dev/null || ok=0
curl -fsS -m 4 -o /dev/null "http://127.0.0.1:4080/" 2>/dev/null || ok=0
podman exec lnd lncli --tlscertpath /root/.lnd/tls.cert \
--macaroonpath /root/.lnd/data/chain/bitcoin/mainnet/readonly.macaroon \
--rpcserver localhost:10009 getinfo >/dev/null 2>&1 || ok=0
# Only gate on immich where it's actually installed (heavy nodes). Its web
# port is the same signal test 64 checks, so settling here keeps the next
# iteration's read-only immich probe from racing a still-recovering stack.
if podman container exists immich_server 2>/dev/null; then
curl -fsS -m 4 -o /dev/null "http://127.0.0.1:2283/" 2>/dev/null || ok=0
fi
(( ok == 1 )) && { echo " (stack settled)"; return 0; }
sleep 4
done
echo " (stack settle deadline reached — proceeding anyway)"
}
# Host readiness, checked ONCE before iteration 1.
#
# Why this exists: on 2026-08-08 a gate run on a box at load ~14 failed five
# times over, every failure reading "could not create a container" (searxng:start,
# package.start btcpay-server, 3× electrumx) and never a lifecycle fault. That
# sent two separate sessions hunting a phantom host-wide cgroup failure. It was
# load. Measured on that 4-core box: at load ~14 podman runs 9-16 processes deep
# and healthchecks time out 3-8/min; at load ~3.7, podman ~1 and zero timeouts.
#
# So: refuse to start on a loaded host rather than emit misleading failures.
# Note what is deliberately NOT checked — the count of "Failed to create
# container" in the journal. Those lines are emitted by healthcheck exec churn
# and by settling after a boot; they never reach 0 on a busy node, and gating on
# them blocks the gate forever. Prove container creation POSITIVELY instead.
preflight_host() {
[[ "${ARCHY_PREFLIGHT:-1}" == "1" ]] || return 0
command -v podman >/dev/null 2>&1 || return 0 # remote/off-node run
local cores max_load
cores=$(nproc 2>/dev/null || echo 4)
max_load="${ARCHY_MAX_LOAD:-$((cores + 1))}"
echo "── preflight: host readiness ──"
# 1. aardvark-dns must be singular. Two of them serve divergent state and make
# container-name resolution flaky, which then looks like a lifecycle bug.
local dns
dns=$(pgrep -c aardvark-dns 2>/dev/null || echo 0)
if (( dns > 1 )); then
echo " FAIL: $dns aardvark-dns processes running (expected 1)." >&2
echo " Duplicate DNS servers desync container-name resolution." >&2
return 1
fi
echo " aardvark-dns: $dns"
# 2. Wait for load to fall. It oscillates on nodes doing IBD or media
# indexing, so a brief spike is not fatal — a sustained one is.
local deadline=$(( $(date +%s) + ${ARCHY_PREFLIGHT_SECS:-900} ))
local load1
while :; do
load1=$(awk '{print $1}' /proc/loadavg)
awk -v l="$load1" -v m="$max_load" 'BEGIN { exit !(l < m) }' && break
if (( $(date +%s) >= deadline )); then
echo " FAIL: load1 $load1 still above $max_load after ${ARCHY_PREFLIGHT_SECS:-900}s." >&2
echo " Quiesce the node (bitcoind IBD, electrumx indexing, CI runners)" >&2
echo " or override with ARCHY_MAX_LOAD=. Running now yields failures" >&2
echo " that look like lifecycle bugs but are contention." >&2
return 1
fi
echo " load1 $load1 > $max_load — waiting…"
sleep 20
done
echo " load1: $load1 (ceiling $max_load, $cores cores)"
# 3. Prove the host can actually create a container, 3× — the positive test
# that the journal grep only ever approximated.
local img
img=$(podman images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null \
| grep -v '<none>' | head -1)
if [[ -z "$img" ]]; then
echo " (no local image — skipping container-create probe)"
else
local n
for n in 1 2 3; do
if ! timeout 60 podman run --rm "$img" /bin/true >/dev/null 2>&1; then
echo " FAIL: container-create probe $n/3 failed using $img." >&2
echo " The host genuinely cannot create containers; fix that first." >&2
return 1
fi
done
echo " container-create probe: 3/3 via $img"
fi
echo "── preflight: OK ──"
}
if ! preflight_host; then
echo "Preflight failed — refusing to start the gate. (ARCHY_PREFLIGHT=0 to skip.)" >&2
exit 3
fi
# One initial teardown so a previous run's cookies don't poison iteration 1.
./setup-teardown.sh
for i in $(seq 1 "$ITER"); do
echo
echo "═══ iteration $i / $ITER ═══"
iter_start=$(date +%s)
settle_stack
if ./run.sh "$@"; then
iter_end=$(date +%s)
passed=$((passed + 1))
echo "── iteration $i: PASS ($((iter_end - iter_start))s) ──"
else
rc=$?
iter_end=$(date +%s)
failed=$((failed + 1))
failures+=("$i")
echo "── iteration $i: FAIL (exit=$rc, $((iter_end - iter_start))s) ──"
if [[ "${ARCHY_FAIL_FAST:-0}" == "1" ]]; then
echo "ARCHY_FAIL_FAST=1, stopping early"
break
fi
fi
# Teardown between iterations so iteration N+1 starts with a clean
# session-cookie state regardless of what iteration N did.
./setup-teardown.sh
done
# Optional CASCADE pass — uninstall → no-ghost → reinstall of a throwaway app
# (default grafana, via cascade-uninstall.bats). Run ONCE, not folded into the
# 5× loop on purpose: uninstall/reinstall every iteration would balloon runtime
# and re-pull images. One pass gates the #13 ghost / #14 reinstall-stop /
# uninstall-hang class (the bug fixed in 71cc9ac4). Opt-in so default gate
# behavior is unchanged; counts into the pass/fail tally.
if [[ "${ARCHY_GATE_CASCADE:-0}" == "1" && "${ARCHY_ALLOW_DESTRUCTIVE:-0}" == "1" ]]; then
echo
echo "═══ CASCADE pass (1×) ═══"
settle_stack
if ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1 ./run.sh cascade-uninstall; then
passed=$((passed + 1))
echo "── CASCADE: PASS ──"
else
failed=$((failed + 1))
failures+=("cascade")
echo "── CASCADE: FAIL ──"
fi
./setup-teardown.sh
fi
end=$(date +%s)
echo
echo "════════════════════════════════════════"
echo " RESULTS"
echo " iterations: $((passed + failed)) / $ITER"
echo " passed: $passed"
echo " failed: $failed"
if (( failed > 0 )); then
echo " failed at: ${failures[*]}"
fi
echo " wall time: $((end - start))s"
echo "════════════════════════════════════════"
if (( failed > 0 )); then
exit 1
fi
tests/lifecycle/run.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# tests/lifecycle/run.sh — entrypoint for lifecycle tests.
#
# Must be run on an archy host. Requires bats + jq + curl.
#
# Env:
# ARCHY_PASSWORD (required unless ARCHY_ALLOW_NOAUTH=1)
# ARCHY_HOST (default: 127.0.0.1)
# ARCHY_SCHEME (default: https)
# ARCHY_ALLOW_DESTRUCTIVE=1 enable stop/start/restart tests
# ARCHY_ALLOW_CASCADE_DESTRUCTIVE=1 enable uninstall/reinstall tests (rarely used)
# ARCHY_ALLOW_NOAUTH=1 allow running read-only suites that don't use RPC auth
#
# Usage:
# tests/lifecycle/run.sh # all .bats files
# tests/lifecycle/run.sh bitcoin-knots # single file (no extension)
# tests/lifecycle/run.sh required-stack required-stack-destructive
# tests/lifecycle/run.sh package-update-smoke
set -euo pipefail
HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "$HERE"
if ! command -v bats >/dev/null 2>&1; then
echo "bats not installed. On Debian: sudo apt-get install -y bats" >&2
exit 2
fi
if [[ -z "${ARCHY_PASSWORD:-}" && "${ARCHY_ALLOW_NOAUTH:-0}" != "1" ]]; then
echo "ARCHY_PASSWORD env var must be set (or ARCHY_ALLOW_NOAUTH=1 for no-auth suites)." >&2
exit 2
fi
if (( $# == 0 )); then
exec bats bats/
fi
targets=()
for arg in "$@"; do
if [[ -f "bats/${arg}.bats" ]]; then
targets+=("bats/${arg}.bats")
elif [[ -f "$arg" ]]; then
targets+=("$arg")
else
echo "unknown test target: $arg" >&2
exit 2
fi
done
exec bats "${targets[@]}"
tests/lifecycle/setup-teardown.sh
Executable
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# tests/lifecycle/setup-teardown.sh
#
# Cleanup helper used between lifecycle test iterations. Run before AND after
# a full bats pass (run-gate.sh handles this). Idempotent — safe to run any
# time, on any host.
#
# Removes:
# - /tmp/archy-rpc-session-* — stale RPC session cookies from earlier runs.
# If absent we'd reuse a session that was logged out by an auth.logout
# test, then the next iteration would silently 401.
# - Per-bats-run scratch files our tests may leave behind (none yet, but
# this is the place when we add them).
#
# Does NOT touch:
# - Real archipelago state (state.json, secrets, packages.json).
# - Running containers — destructive container teardown is the test's
# responsibility, not the harness's. We only clean the harness's own
# transient state.
# - SSH known_hosts, archipelago configs, etc.
set -euo pipefail
cleaned=0
# Match the pattern from lib/rpc.bash:26
session_glob="/tmp/archy-rpc-session-*"
# shellcheck disable=SC2086 # we want word-splitting on the glob
for f in $session_glob; do
if [[ -f "$f" ]]; then
rm -f "$f"
cleaned=$((cleaned + 1))
fi
done
if [[ "${ARCHY_TEARDOWN_VERBOSE:-0}" == "1" ]]; then
echo "setup-teardown: cleaned $cleaned stale session file(s)"
fi
tests/mesh/run-mesh-tests.sh
Executable
+85
View File
@@ -0,0 +1,85 @@
#!/bin/bash
# Mesh / Reticulum test suite — the "is the mesh stack healthy" gate.
#
# Three layers, cheapest first:
# 1. Rust unit tests (no hardware, ~2s once built)
# 2. Daemon selftest (full RNS+LXMF bring-up, no radio; also verifies
# the announce app_data wire contract + set_name)
# 3. Live-node assertions (optional; needs a running archipelago with a
# radio — set MESH_TEST_LIVE=1 MESH_TEST_PW=...)
#
# Usage:
# tests/mesh/run-mesh-tests.sh # layers 1+2
# MESH_TEST_LIVE=1 MESH_TEST_PW='...' tests/mesh/run-mesh-tests.sh
# MESH_TEST_HOST=100.64.0.55 ... # live-test a remote node
set -u
cd "$(dirname "$0")/../.."
FAIL=0
ok() { echo "ok - $1"; }
bad() { echo "not ok - $1"; FAIL=1; }
# ── 1. Rust unit tests ────────────────────────────────────────────────
RUST_RESULTS=$(cd core && cargo test -p archipelago --bin archipelago mesh 2>&1 | grep "^test result:")
if [ -n "$RUST_RESULTS" ] && ! echo "$RUST_RESULTS" | grep -vq " 0 failed"; then
ok "rust mesh unit tests ($(echo "$RUST_RESULTS" | grep -o '[0-9]* passed' | head -1))"
else
bad "rust mesh unit tests"
fi
# ── 2. Reticulum daemon selftest (no radio) ───────────────────────────
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
head -c 32 /dev/urandom > "$TMP/key"
DAEMON=reticulum-daemon/.venv/bin/python
if [ -x "$DAEMON" ]; then
if "$DAEMON" reticulum-daemon/reticulum_daemon.py \
--identity-key "$TMP/key" --rns-config "$TMP/rns" \
--socket "$TMP/sock" --display-name "SelftestNode" --selftest 2>/dev/null \
| grep -q "announce_app_data=verified set_name=verified"; then
ok "daemon selftest (announce wire contract + set_name)"
else
bad "daemon selftest"
fi
else
echo "skip - daemon selftest (no venv at $DAEMON)"
fi
# ── 3. Live node assertions (opt-in) ──────────────────────────────────
if [ "${MESH_TEST_LIVE:-0}" = "1" ]; then
HOST="${MESH_TEST_HOST:-127.0.0.1}"
PW="${MESH_TEST_PW:?set MESH_TEST_PW}"
# Nodes differ: dev boxes serve plain http on :80, ISO installs https.
RPC=""
for base in "http://$HOST" "https://$HOST" "http://$HOST:5678"; do
code=$(curl -ksS -o /dev/null -w '%{http_code}' -m 5 -X POST "$base/rpc/v1" 2>/dev/null || true)
case "$code" in 000|"") continue ;; *) RPC="$base/rpc/v1"; break ;; esac
done
[ -n "$RPC" ] || { bad "live: no RPC endpoint reachable on $HOST"; echo FAIL; exit 1; }
JAR="$TMP/jar"
curl -ksS -c "$JAR" -H "Content-Type: application/json" \
-d "{\"jsonrpc\":\"2.0\",\"method\":\"auth.login\",\"params\":{\"password\":\"$PW\"},\"id\":1}" \
"$RPC" > "$TMP/login"
if grep -q '"error":null' "$TMP/login"; then ok "live: rpc login"; else bad "live: rpc login"; fi
call() {
local csrf; csrf=$(awk '/^[^#]/ && /csrf_token/ {print $7; exit}' "$JAR")
curl -ksS -b "$JAR" -c "$JAR" -H "Content-Type: application/json" \
-H "X-CSRF-Token: $csrf" \
-d "{\"jsonrpc\":\"2.0\",\"method\":\"$1\",\"params\":${2:-{\}},\"id\":2}" \
--max-time 60 "$RPC"
}
ST=$(call mesh.status)
echo "$ST" | grep -q '"device_connected":true' \
&& ok "live: radio connected ($(echo "$ST" | grep -o '"device_type":"[a-z]*"'))" \
|| bad "live: radio connected"
echo "$ST" | grep -q '"self_advert_name":"[^"]' \
&& ok "live: node has a mesh name" || bad "live: node has a mesh name"
call mesh.refresh | grep -q '"refreshed":true' \
&& ok "live: mesh.refresh" || bad "live: mesh.refresh"
call mesh.broadcast | grep -q '"broadcast":true' \
&& ok "live: mesh.broadcast" || bad "live: mesh.broadcast"
# No peer may ever display a raw identity blob as its name.
call mesh.peers | grep -q '"advert_name":"ARCHY:' \
&& bad "live: no ARCHY-blob peer names" || ok "live: no ARCHY-blob peer names"
fi
[ "$FAIL" = 0 ] && echo "PASS" || { echo "FAIL"; exit 1; }
tests/multinode/.env.example
+15
View File
@@ -0,0 +1,15 @@
# Local credentials for the multinode test suites — copy to tests/multinode/.env
# (git-ignored) and fill in. Sourced automatically by lib/multinode.bash.
# NEVER commit real node passwords.
# smoke.sh / repro-federation-sync.sh
A_PW=changeme # node A (default URL http://192.0.2.12)
B_PW=changeme # node B (default URL https://192.0.2.10)
#C_URL=https://x.x.x.x # optional third node
#C_PW=changeme
# meshtastic.sh
MA_PW=changeme
MB_PW=changeme
#MC_URL=https://x.x.x.x
#MC_PW=changeme
tests/multinode/lib/multinode.bash
Executable
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
# Multi-node RPC harness library.
#
# Unlike tests/lifecycle/lib/rpc.bash (which targets a single ARCHY_HOST),
# this drives N independent archipelago nodes in one run so we can exercise
# real node-to-node paths: federation sync over Tor, FIPS anchoring, etc.
#
# A "node handle" is a short label (e.g. A, B, alice). For each handle you
# register a base URL + UI password; the lib logs in and keeps that node's
# session/CSRF cookies in its own state file so calls never cross wires.
#
# Usage:
# source tests/multinode/lib/multinode.bash
# node_register A https://192.0.2.10 "$A_PW"
# node_register B http://192.0.2.12 "$B_PW"
# node_login A; node_login B
# node_rpc A node.tor-address
# node_result B federation.list-nodes
#
# Requires: curl, jq.
#
# Credentials: node passwords are NEVER committed. Export *_PW env vars, or
# put them in tests/multinode/.env (git-ignored; see .env.example) — sourced
# automatically below so every suite picks them up.
#
# Note: this is a library — it does NOT set shell options (set -u/-e), since
# that would leak into the sourcing script. Each function guards its own vars
# with ${var:-} defaults. Callers set their own options.
# Auto-load git-ignored local credentials (tests/multinode/.env), if present.
_MN_ENV_FILE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)/.env"
# shellcheck disable=SC1090
[[ -f "$_MN_ENV_FILE" ]] && source "$_MN_ENV_FILE"
# Where per-node session state lives (one file per handle).
MULTINODE_STATE_DIR="${MULTINODE_STATE_DIR:-/tmp/archy-multinode}"
mkdir -p "$MULTINODE_STATE_DIR"
# handle -> base url / password, kept in associative arrays.
declare -gA _MN_URL
declare -gA _MN_PW
declare -gA _MN_SESSION
declare -gA _MN_CSRF
# node_register HANDLE BASE_URL PASSWORD
node_register() {
local h="$1" url="$2" pw="$3"
_MN_URL[$h]="${url%/}"
_MN_PW[$h]="$pw"
}
_mn_session_file() { echo "$MULTINODE_STATE_DIR/session-$1"; }
# node_login HANDLE — authenticate and capture session + csrf cookies.
node_login() {
local h="$1"
local url="${_MN_URL[$h]:-}" pw="${_MN_PW[$h]:-}"
if [[ -z "$url" || -z "$pw" ]]; then
echo "node_login: handle '$h' not registered" >&2
return 1
fi
local headers; headers=$(mktemp)
local body
body=$(curl -sk --connect-timeout 10 --max-time "${MULTINODE_RPC_TIMEOUT:-120}" \
-D "$headers" -X POST "${url}/rpc/v1" \
-H 'Content-Type: application/json' \
--data-raw "{\"jsonrpc\":\"2.0\",\"method\":\"auth.login\",\"params\":{\"password\":\"${pw}\"},\"id\":1}")
local err; err=$(echo "$body" | jq -r '.error.message // empty' 2>/dev/null)
if [[ -n "$err" ]]; then
echo "node_login[$h] failed: $err" >&2
rm -f "$headers"
return 1
fi
local session csrf
session=$(grep -i '^set-cookie: session=' "$headers" | head -1 | sed -E 's/.*session=([^;]+).*/\1/' | tr -d '\r')
csrf=$(grep -i '^set-cookie: csrf_token=' "$headers" | head -1 | sed -E 's/.*csrf_token=([^;]+).*/\1/' | tr -d '\r')
rm -f "$headers"
if [[ -z "$session" || -z "$csrf" ]]; then
echo "node_login[$h]: missing session/csrf cookie" >&2
return 1
fi
_MN_SESSION[$h]="$session"
_MN_CSRF[$h]="$csrf"
printf '%s\n%s\n' "$session" "$csrf" > "$(_mn_session_file "$h")"
}
# node_rpc HANDLE METHOD [PARAMS_JSON] — raw JSON-RPC response on stdout.
node_rpc() {
local h="$1" method="$2" params="${3:-}"
local url="${_MN_URL[$h]:-}"
local session="${_MN_SESSION[$h]:-}" csrf="${_MN_CSRF[$h]:-}"
if [[ -z "$session" || -z "$csrf" ]] && [[ -f "$(_mn_session_file "$h")" ]]; then
mapfile -t lines < "$(_mn_session_file "$h")"
session="${lines[0]:-}"; csrf="${lines[1]:-}"
_MN_SESSION[$h]="$session"; _MN_CSRF[$h]="$csrf"
fi
local payload
if [[ -z "$params" ]]; then
payload=$(jq -nc --arg m "$method" '{jsonrpc:"2.0",method:$m,id:1}')
else
payload=$(jq -nc --arg m "$method" --argjson p "$params" '{jsonrpc:"2.0",method:$m,params:$p,id:1}')
fi
# Bounded so one slow/hung server-side RPC can't hang the whole suite;
# override per-run with MULTINODE_RPC_TIMEOUT (seconds).
curl -sk --connect-timeout 10 --max-time "${MULTINODE_RPC_TIMEOUT:-120}" \
-X POST "${url}/rpc/v1" \
-H 'Content-Type: application/json' \
-H "Cookie: session=${session}; csrf_token=${csrf}" \
-H "X-CSRF-Token: ${csrf}" \
--data-raw "$payload"
}
# node_result HANDLE METHOD [PARAMS_JSON] — .result on success; prints error to
# stderr and returns non-zero on RPC error.
node_result() {
local resp; resp=$(node_rpc "$@")
local err; err=$(echo "$resp" | jq -r '.error.message // empty' 2>/dev/null)
if [[ -n "$err" ]]; then
echo "node_result[$1 $2] error: $err" >&2
return 1
fi
echo "$resp" | jq '.result'
}
# node_onion HANDLE — echo this node's own .onion address (empty if none).
node_onion() {
node_result "$1" node.tor-address 2>/dev/null | jq -r '. // empty | if type=="object" then (.onion // .address // .tor_address // empty) else . end' 2>/dev/null
}
tests/multinode/meshtastic.sh
Executable
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env bash
# tests/multinode/meshtastic.sh — two-/three-radio Meshtastic parity harness.
#
# Validates that Meshtastic radios have the SAME mesh-tab features Meshcore got,
# done over the real wire. It drives 2 (optionally 3) archipelago nodes, each
# with a Meshtastic radio attached, and exercises the full message pipeline:
#
# 1. detect — each node reports a connected meshtastic device
# 2. discover — A sees B as a peer (NodeInfo discovery), and vice-versa
# 3. dm — A → B direct message round-trips (native unicast)
# 4. privacy — a third listener C does NOT see the A→B DM (proves the
# directed-unicast fix: DMs are not broadcast on the channel)
# 5. channel — A's channel broadcast IS seen by both B and C
# 6. typed — a typed envelope (reaction) round-trips with message_type set
# 7. assistant — (optional) an !ai query gets a PRIVATE reply, not a channel
# blast (gated on ASSIST=1 + assistant enabled on B)
# 8. reachable — reports each peer's `reachable`/`last_advert` so the ambiguous
# Meshtastic reachability semantics can be eyeballed on-air
# before anyone "fixes" them
#
# The privacy test (4) is the on-air proof of the meshtastic.rs send_text_msg
# unicast change. Without it, A→B DMs land on every node's channel feed.
#
# Nodes override via env (each must have a Meshtastic radio on the SAME LoRa
# channel/region so they can actually hear each other):
# MA_URL MA_PW node A (sender) default .116 http / <FLEET_PW>
# MB_URL MB_PW node B (receiver) default .228 https / password123
# MC_URL MC_PW node C (eavesdrop) OPTIONAL — enables privacy test (4)
#
# MB_NAME B's mesh node name, if A's peer list is ambiguous (>1 peer)
# PROP_WAIT seconds to wait for LoRa propagation per step (default 45)
# ASSIST set =1 to run the assistant private-reply test (7)
#
# Usage:
# tests/multinode/meshtastic.sh
# MA_URL=http://192.0.2.12 MB_URL=https://192.0.2.10 \
# MC_URL=https://192.0.2.11 tests/multinode/meshtastic.sh
#
# Requires: curl, jq. Exit code = number of failed assertions (0 = all green).
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/multinode.bash
source "$HERE/lib/multinode.bash"
# ── node registration ──────────────────────────────────────────────────────
MA_URL="${MA_URL:-http://192.0.2.12}"; MA_PW="${MA_PW:?MA_PW required — export it or set tests/multinode/.env (see .env.example)}"
MB_URL="${MB_URL:-https://192.0.2.10}"; MB_PW="${MB_PW:?MB_PW required — export it or set tests/multinode/.env (see .env.example)}"
MC_URL="${MC_URL:-}"; MC_PW="${MC_PW:-}"
PROP_WAIT="${PROP_WAIT:-45}"
MB_NAME="${MB_NAME:-}"
ASSIST="${ASSIST:-0}"
node_register A "$MA_URL" "$MA_PW"
node_register B "$MB_URL" "$MB_PW"
HAVE_C=0
if [[ -n "$MC_URL" ]]; then node_register C "$MC_URL" "$MC_PW"; HAVE_C=1; fi
# ── tiny assert framework (mirrors smoke.sh) ───────────────────────────────
if [[ -t 1 ]]; then
green() { printf '\033[32m%s\033[0m' "$*"; }
red() { printf '\033[31m%s\033[0m' "$*"; }
yellow() { printf '\033[33m%s\033[0m' "$*"; }
else
green() { printf '%s' "$*"; }; red() { printf '%s' "$*"; }; yellow() { printf '%s' "$*"; }
fi
PASS=0; FAIL=0; SKIP=0; declare -a FAILED_NAMES
ok() { printf ' %s %s\n' "$(green ✓)" "$1"; PASS=$((PASS+1)); }
no() { printf ' %s %s\n' "$(red ✗)" "$1"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); }
skip() { printf ' %s %s (%s)\n' "$(yellow —)" "$1" "${2:-skipped}"; SKIP=$((SKIP+1)); }
assert_true() { [[ "$2" == "true" ]] && ok "$1" || no "$1 (got '$2')"; }
section() { printf '\n%s\n' "$(yellow "── $* ──")"; }
# nonce for this run so message matches can't collide with stale history
NONCE="mtparity-$$-${RANDOM}"
# ── helpers ────────────────────────────────────────────────────────────────
# mesh_connected HANDLE -> "true" if a meshtastic device is connected
mesh_connected() {
local s; s=$(node_result "$1" mesh.status 2>/dev/null) || { echo false; return; }
local conn type
conn=$(echo "$s" | jq -r '.device_connected // false')
type=$(echo "$s" | jq -r '.device_type // "unknown"')
[[ "$conn" == "true" && "$type" == "meshtastic" ]] && echo true || echo false
}
# self_name HANDLE -> this node's meshtastic long-name (from firmware_version)
self_name() {
node_result "$1" mesh.status 2>/dev/null | jq -r '.firmware_version // empty'
}
# contact_id_for HANDLE NAME -> the contact_id of the peer whose advert_name
# matches NAME (case-insensitive substring); empty if not found / ambiguous.
contact_id_for() {
local h="$1" want="$2"
node_result "$h" mesh.peers 2>/dev/null | jq -r --arg w "$want" '
[.peers[] | select((.advert_name // "" | ascii_downcase)
| contains($w | ascii_downcase))] as $m
| if ($m|length)==1 then ($m[0].contact_id|tostring) else "" end'
}
# peer_count_excl_self HANDLE -> number of peers
peer_count() { node_result "$1" mesh.peers 2>/dev/null | jq -r '.count // 0'; }
# saw_text HANDLE NEEDLE [direction] -> "true" if a message whose plaintext
# contains NEEDLE exists (optionally filtered to a direction: sent/received)
saw_text() {
local h="$1" needle="$2" dir="${3:-}"
node_result "$h" mesh.messages '{"limit":200}' 2>/dev/null | jq -r --arg n "$needle" --arg d "$dir" '
[.messages[] | select((.plaintext // "") | contains($n))
| select($d=="" or (.direction==$d))] | length > 0'
}
# wait_text HANDLE NEEDLE — poll up to PROP_WAIT for a received message
wait_text() {
local h="$1" needle="$2" waited=0
while (( waited < PROP_WAIT )); do
[[ "$(saw_text "$h" "$needle" received)" == "true" ]] && return 0
sleep 3; waited=$((waited+3))
done
return 1
}
# ── login ──────────────────────────────────────────────────────────────────
section "login"
node_login A && ok "A login ($MA_URL)" || { no "A unreachable ($MA_URL)"; echo; exit 1; }
node_login B && ok "B login ($MB_URL)" || { no "B unreachable ($MB_URL)"; echo; exit 1; }
if (( HAVE_C )); then
node_login C && ok "C login ($MC_URL)" || { skip "C login" "unreachable — privacy test disabled"; HAVE_C=0; }
fi
# ── 1. detect ──────────────────────────────────────────────────────────────
section "1. device detection"
A_CONN=$(mesh_connected A); B_CONN=$(mesh_connected B)
assert_true "A has a connected meshtastic radio" "$A_CONN"
assert_true "B has a connected meshtastic radio" "$B_CONN"
if [[ "$A_CONN" != "true" || "$B_CONN" != "true" ]]; then
printf '\n%s\n' "$(yellow 'Both A and B need a Meshtastic radio attached & mesh enabled.')"
printf '%s\n' "$(yellow 'Aborting on-air tests; see mesh.status output above.')"
echo; printf 'PASS=%d FAIL=%d SKIP=%d\n' "$PASS" "$FAIL" "$SKIP"; exit "$FAIL"
fi
A_NAME=$(self_name A); B_NAME=$(self_name B)
printf ' A=%s B=%s\n' "${A_NAME:-?}" "${B_NAME:-?}"
[[ -n "$MB_NAME" ]] && B_NAME="$MB_NAME"
# ── 2. peer discovery ──────────────────────────────────────────────────────
section "2. peer discovery (NodeInfo)"
DISCO=0; waited=0
while (( waited < PROP_WAIT )); do
CID=$(contact_id_for A "${B_NAME:-Meshtastic}")
[[ -n "$CID" ]] && { DISCO=1; break; }
# fall back: any single non-channel peer
if [[ -z "$MB_NAME" && "$(peer_count A)" == "1" ]]; then
CID=$(node_result A mesh.peers | jq -r '.peers[0].contact_id'); DISCO=1; break
fi
sleep 3; waited=$((waited+3))
done
if (( DISCO )); then ok "A discovered B as a peer (contact_id=$CID)"
else
no "A did not discover B within ${PROP_WAIT}s"
printf ' A peers: %s\n' "$(node_result A mesh.peers | jq -c '.peers[]? | {contact_id,advert_name}')"
fi
# ── 3. direct message round-trip ───────────────────────────────────────────
section "3. direct message (native unicast)"
if (( DISCO )); then
DM="$NONCE-dm hello-from-A"
if node_result A mesh.send "$(jq -nc --argjson c "$CID" --arg m "$DM" '{contact_id:$c,message:$m}')" >/dev/null; then
ok "A sent DM to B (contact_id=$CID)"
if wait_text B "$NONCE-dm"; then ok "B received the DM"
else no "B did not receive the DM within ${PROP_WAIT}s"; fi
else no "mesh.send failed on A"; fi
else skip "DM round-trip" "B not discovered"; fi
# ── 4. privacy: third node must NOT see the DM ─────────────────────────────
section "4. DM privacy (directed, not broadcast)"
if (( HAVE_C )) && (( DISCO )); then
C_CONN=$(mesh_connected C)
if [[ "$C_CONN" != "true" ]]; then
skip "DM privacy" "C has no meshtastic radio"
else
# Give C the same window the DM had to propagate, then assert absence.
sleep "$PROP_WAIT"
if [[ "$(saw_text C "$NONCE-dm")" == "true" ]]; then
no "C (eavesdropper) saw the A→B DM — it is being BROADCAST, not unicast"
else
ok "C did NOT see the A→B DM (directed unicast confirmed)"
fi
fi
else
skip "DM privacy" "needs MC_URL (third radio) + discovered peer"
fi
# ── 5. channel broadcast reaches everyone ──────────────────────────────────
section "5. channel broadcast"
CH="$NONCE-chan broadcast-to-all"
if node_result A mesh.send-channel "$(jq -nc --arg m "$CH" '{channel:0,message:$m}')" >/dev/null; then
ok "A sent a channel broadcast"
if wait_text B "$NONCE-chan"; then ok "B received the broadcast"; else no "B missed the broadcast"; fi
if (( HAVE_C )) && [[ "$(mesh_connected C)" == "true" ]]; then
if [[ "$(saw_text C "$NONCE-chan")" == "true" ]]; then ok "C also received the broadcast"
else no "C missed the broadcast (it should reach all channel members)"; fi
fi
else no "mesh.send-channel failed on A"; fi
# ── 6. typed envelope round-trip ───────────────────────────────────────────
section "6. typed message (reaction envelope)"
if (( DISCO )); then
# A reaction is the smallest typed envelope; it should arrive with a
# non-"text" message_type, proving the typed pipeline works over Meshtastic.
REACT_PARAMS=$(jq -nc --argjson c "$CID" --arg n "$NONCE" \
'{contact_id:$c, emoji:"👍", target_seq:0, note:$n}')
if node_result A mesh.send-reaction "$REACT_PARAMS" >/dev/null 2>&1; then
ok "A sent a reaction (typed envelope)"
sleep "$PROP_WAIT"
TYPED=$(node_result B mesh.messages '{"limit":200}' 2>/dev/null \
| jq -r '[.messages[] | select(.message_type != null and .message_type != "text")] | length > 0')
assert_true "B received a non-text typed message" "$TYPED"
else
skip "typed message" "mesh.send-reaction rejected params (check handler signature)"
fi
else skip "typed message" "B not discovered"; fi
# ── 7. assistant private reply (optional) ──────────────────────────────────
section "7. AI assistant private reply (optional)"
if [[ "$ASSIST" == "1" ]] && (( DISCO )); then
AST=$(node_result B mesh.assistant-status 2>/dev/null | jq -r '.enabled // false')
if [[ "$AST" != "true" ]]; then
skip "assistant reply" "assistant not enabled on B"
else
Q="$NONCE-ai !ai are you there"
node_result A mesh.send-channel "$(jq -nc --arg m "$Q" '{channel:0,message:$m}')" >/dev/null
sleep "$PROP_WAIT"
# A should get a private DM reply; C (if present) should NOT.
if [[ "$(saw_text A "$NONCE-ai-reply")" == "true" || "$(node_result A mesh.messages '{"limit":50}' | jq -r '[.messages[]|select(.direction=="received")]|length>0')" == "true" ]]; then
ok "A received an assistant reply"
else
no "A did not receive an assistant reply within ${PROP_WAIT}s"
fi
if (( HAVE_C )) && [[ "$(mesh_connected C)" == "true" ]]; then
# heuristic: the reply text shouldn't be on C's channel feed
skip "assistant reply privacy" "eyeball C's feed — automated check is heuristic"
fi
fi
else
skip "assistant reply" "set ASSIST=1 and enable the assistant on B to run"
fi
# ── 8. reachability snapshot (report-only) ─────────────────────────────────
section "8. reachability snapshot (report-only)"
node_result A mesh.peers 2>/dev/null | jq -r '.peers[]?
| " \(.advert_name // "?") reachable=\(.reachable) last_advert=\(.last_advert // 0)"'
printf '%s\n' "$(yellow ' NOTE: Meshtastic flood-routes; path_len is always 0xff, so `reachable`')"
printf '%s\n' "$(yellow ' may read true even for stale nodes. Confirm desired semantics here')"
printf '%s\n' "$(yellow ' before changing the refresh_contacts reachability rule.')"
# ── summary ────────────────────────────────────────────────────────────────
section "summary"
printf 'PASS=%s FAIL=%s SKIP=%s\n' "$(green "$PASS")" "$( ((FAIL)) && red "$FAIL" || green 0 )" "$(yellow "$SKIP")"
if (( FAIL )); then
printf 'failed:\n'; for n in "${FAILED_NAMES[@]}"; do printf ' - %s\n' "$n"; done
fi
exit "$FAIL"
tests/multinode/repro-federation-sync.sh
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Controlled two-node reproduction of node-to-node federation sync.
#
# Pairs two real nodes via federation.invite/join, triggers federation.sync-state
# in both directions, and reports which transport actually carried the call and
# any per-peer error. This is the controlled repro for the reported
# "Tor connection cloud->node not working" symptom: raw Tor transport is known
# good (see README), so this isolates whether the APP-level sync path works and,
# if it fails, surfaces the exact error string.
#
# Env (override as needed):
# A_URL A_PW node A base url + UI password (default .116 http)
# B_URL B_PW node B base url + UI password (default .228 https)
# FORCE_TOR=1 set both nodes' federation transport preference to Tor first
#
# Usage: tests/multinode/repro-federation-sync.sh
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$HERE/lib/multinode.bash"
A_URL="${A_URL:-http://192.0.2.12}"; A_PW="${A_PW:?A_PW required — export it or set tests/multinode/.env (see .env.example)}"
B_URL="${B_URL:-https://192.0.2.10}"; B_PW="${B_PW:?B_PW required — export it or set tests/multinode/.env (see .env.example)}"
bar() { printf '\n=== %s ===\n' "$*"; }
node_register A "$A_URL" "$A_PW"
node_register B "$B_URL" "$B_PW"
bar "login"
node_login A || { echo "A login failed"; exit 1; }
node_login B || { echo "B login failed"; exit 1; }
echo "A=$A_URL B=$B_URL logged in"
bar "onions"
A_ONION=$(node_onion A); B_ONION=$(node_onion B)
echo "A onion: ${A_ONION:-<none>}"
echo "B onion: ${B_ONION:-<none>}"
if [[ "${FORCE_TOR:-0}" == "1" ]]; then
bar "force federation transport = tor on both"
node_rpc A transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error'
node_rpc B transport.set-preference '{"service":"federation","pref":"tor"}' | jq -c '.result // .error'
fi
bar "federation state BEFORE"
echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)"
echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") did=\(.did[0:24])… last_seen=\(.last_seen // "never")"' 2>/dev/null || echo " (none/err)"
bar "pair: A invites, B joins"
INV_A=$(node_result A federation.invite)
CODE_A=$(echo "$INV_A" | jq -r '.code // empty')
echo "A invite code: ${CODE_A:0:40}…"
if [[ -n "$CODE_A" ]]; then
node_result B federation.join "$(jq -nc --arg c "$CODE_A" '{code:$c}')" \
&& echo "B joined A" || echo "B join FAILED"
fi
bar "pair: B invites, A joins"
INV_B=$(node_result B federation.invite)
CODE_B=$(echo "$INV_B" | jq -r '.code // empty')
echo "B invite code: ${CODE_B:0:40}…"
if [[ -n "$CODE_B" ]]; then
node_result A federation.join "$(jq -nc --arg c "$CODE_B" '{code:$c}')" \
&& echo "A joined B" || echo "A join FAILED"
fi
bar "trigger sync-state on A (A dials its peers)"
node_result A federation.sync-state | jq '.'
bar "trigger sync-state on B (B dials its peers)"
node_result B federation.sync-state | jq '.'
bar "federation state AFTER (look for fresh last_seen + transport)"
echo "A knows:"; node_result A federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null
echo "B knows:"; node_result B federation.list-nodes | jq -r '.[]? | " \(.name // "?") last_seen=\(.last_seen // "never") transport=\(.last_transport // .transport // "?")"' 2>/dev/null
bar "done"
tests/multinode/smoke.sh
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env bash
# Two-node (optionally three-node) end-to-end smoke suite for the full app.
#
# Unlike repro-federation-sync.sh (a diagnostic that just prints state), this
# is an ASSERTION suite: every check is pass/fail and the script exits non-zero
# if any required check fails. It exercises the real node-to-node surface and
# specifically guards the bugs fixed in v1.7.94 / v1.7.95:
# - FIPS auto-connects to the public anchor (v1.7.94)
# - peer content browse works over the mesh, not just Tor (v1.7.95 — the
# `/content` catalog used to 404 over FIPS and never fall back to Tor)
# - a removed federation node stays removed, incl. transitive re-discovery
# (v1.7.95 tombstone) — the transitive case needs node C.
#
# Nodes (override via env):
# A_URL A_PW node A (default .116 http)
# B_URL B_PW node B (default .228 https)
# C_URL C_PW node C (OPTIONAL — enables the transitive-tombstone test)
#
# Requires both nodes on v1.7.95-alpha+ for the content-browse and tombstone
# checks; older peers SKIP those (reported, not failed).
#
# Usage:
# tests/multinode/smoke.sh
# A_URL=http://192.0.2.12 B_URL=https://192.0.2.10 tests/multinode/smoke.sh
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$HERE/lib/multinode.bash"
A_URL="${A_URL:-http://192.0.2.12}"; A_PW="${A_PW:?A_PW required — export it or set tests/multinode/.env (see .env.example)}"
B_URL="${B_URL:-https://192.0.2.10}"; B_PW="${B_PW:?B_PW required — export it or set tests/multinode/.env (see .env.example)}"
C_URL="${C_URL:-}"; C_PW="${C_PW:-}"
# ── tiny assertion framework ──────────────────────────────────────────────
PASS=0; FAIL=0; SKIP=0
declare -a FAILED_NAMES
green() { printf '\033[32m%s\033[0m' "$*"; }
red() { printf '\033[31m%s\033[0m' "$*"; }
yellow(){ printf '\033[33m%s\033[0m' "$*"; }
section() { printf '\n\033[1m── %s ──\033[0m\n' "$*"; }
ok() { printf ' %s %s\n' "$(green ✓)" "$1"; PASS=$((PASS+1)); }
no() { printf ' %s %s\n' "$(red ✗)" "$1"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); }
skip() { printf ' %s %s (%s)\n' "$(yellow —)" "$1" "$2"; SKIP=$((SKIP+1)); }
# assert_eq NAME EXPECTED ACTUAL
assert_eq() { [[ "$2" == "$3" ]] && ok "$1" || no "$1 (expected '$2', got '$3')"; }
# assert_true NAME VALUE — passes when VALUE is "true"
assert_true() { [[ "$2" == "true" ]] && ok "$1" || no "$1 (got '$2')"; }
# did_of HANDLE — this node's own DID via node.did (string or {did:...}).
did_of() {
node_result "$1" node.did 2>/dev/null \
| jq -r 'if type=="string" then . elif type=="object" then (.did // .node_did // empty) else empty end' 2>/dev/null
}
# pair HANDLE_INVITER HANDLE_JOINER — invite + join one direction. Echo "ok"/"fail".
pair() {
local inv code
inv=$(node_result "$1" federation.invite 2>/dev/null)
code=$(echo "$inv" | jq -r '.code // empty' 2>/dev/null)
[[ -z "$code" ]] && { echo "fail"; return; }
if node_result "$2" federation.join "$(jq -nc --arg c "$code" '{code:$c}')" >/dev/null 2>&1; then
echo "ok"
else echo "fail"; fi
}
node_register A "$A_URL" "$A_PW"
node_register B "$B_URL" "$B_PW"
HAVE_C=0
if [[ -n "$C_URL" && -n "$C_PW" ]]; then node_register C "$C_URL" "$C_PW"; HAVE_C=1; fi
# ── 1. reachability + auth ────────────────────────────────────────────────
section "reachability + login"
node_login A && ok "A login ($A_URL)" || { no "A login ($A_URL)"; echo "A unreachable — aborting"; exit 1; }
node_login B && ok "B login ($B_URL)" || { no "B login ($B_URL)"; echo "B unreachable — aborting"; exit 1; }
if [[ $HAVE_C == 1 ]]; then node_login C && ok "C login ($C_URL)" || { no "C login"; HAVE_C=0; }; fi
A_ONION=$(node_onion A); B_ONION=$(node_onion B)
[[ -n "$A_ONION" ]] && ok "A has onion address" || no "A has onion address"
[[ -n "$B_ONION" ]] && ok "B has onion address" || no "B has onion address"
# ── 2. FIPS mesh: daemon up + anchor connected (v1.7.94) ──────────────────
section "FIPS mesh / anchor"
for h in A B; do
s=$(node_result "$h" fips.status 2>/dev/null)
if [[ -z "$s" ]]; then skip "$h fips.status" "no FIPS RPC (old build?)"; continue; fi
assert_true "$h FIPS service active" "$(echo "$s" | jq -r '.service_active')"
ac=$(echo "$s" | jq -r '.anchor_connected')
if [[ "$ac" == "true" ]]; then ok "$h anchor connected"
else skip "$h anchor connected" "anchor_connected=$ac — node may need v1.7.94 + a moment to handshake"; fi
done
# ── 3. federation pairing (both directions) ───────────────────────────────
section "federation pairing"
assert_eq "A invites, B joins" "ok" "$(pair A B)"
assert_eq "B invites, A joins" "ok" "$(pair B A)"
# both should now list each other
node_result A federation.sync-state >/dev/null 2>&1
node_result B federation.sync-state >/dev/null 2>&1
A_SEES_B=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg o "${B_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)')
B_SEES_A=$(node_result B federation.list-nodes 2>/dev/null | jq -r --arg o "${A_ONION%.onion}" 'any((.nodes // .)[]?; (.onion // "" | gsub("\\.onion$";"")) == $o)')
assert_true "A's node list contains B" "$A_SEES_B"
assert_true "B's node list contains A" "$B_SEES_A"
# ── 4. peer content browse over the mesh (v1.7.95 fix) ────────────────────
section "peer content browse (was: 404 over mesh, no Tor fallback)"
if [[ -n "$B_ONION" ]]; then
resp=$(node_rpc A content.browse-peer "$(jq -nc --arg o "$B_ONION" '{onion:$o}')")
err=$(echo "$resp" | jq -r '.error.message // empty')
if [[ -z "$err" ]]; then
ok "A browses B's content catalog (HTTP 200)"
elif echo "$err" | grep -q '404'; then
no "A browses B's content — still 404 over mesh (is B on v1.7.95?): $err"
else
# Other errors (peer offline, no content shared) are environmental, not the bug.
skip "A browses B's content" "non-404 error: $err"
fi
else
skip "A browses B's content" "B has no onion"
fi
# ── 5. removed-node tombstone (v1.7.95) ───────────────────────────────────
section "removed-node tombstone"
B_DID=$(did_of B)
if [[ -z "$B_DID" ]]; then
skip "remove B then verify stays removed" "couldn't resolve B's DID"
else
if node_result A federation.remove-node "$(jq -nc --arg d "$B_DID" '{did:$d}')" >/dev/null 2>&1; then
still=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
assert_eq "B removed from A's list" "false" "$still"
# Transitive test needs C: A federated with B and C; C federated with B;
# A removes B; A syncs with C (who advertises B) → B must NOT reappear.
if [[ $HAVE_C == 1 ]]; then
pair A C >/dev/null; pair C A >/dev/null; pair C B >/dev/null
node_result A federation.sync-state >/dev/null 2>&1
reappeared=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
assert_eq "B does NOT reappear via transitive sync with C" "false" "$reappeared"
else
skip "transitive reappear via 3rd node" "set C_URL/C_PW to enable"
fi
# re-add restores B (explicit re-add clears the tombstone)
pair B A >/dev/null
node_result A federation.sync-state >/dev/null 2>&1
readded=$(node_result A federation.list-nodes 2>/dev/null | jq -r --arg d "$B_DID" 'any((.nodes // .)[]?; .did == $d)')
assert_true "explicit re-pair brings B back (tombstone cleared)" "$readded"
else
skip "remove B" "remove-node RPC failed (B may already be absent)"
fi
fi
# ── summary ───────────────────────────────────────────────────────────────
section "summary"
printf ' %s passed, %s failed, %s skipped\n' "$(green $PASS)" "$([[ $FAIL -gt 0 ]] && red $FAIL || echo $FAIL)" "$(yellow $SKIP)"
if [[ $FAIL -gt 0 ]]; then
printf ' failed:\n'; for n in "${FAILED_NAMES[@]}"; do printf ' - %s\n' "$n"; done
exit 1
fi
echo " all required checks passed"
tests/production-quality/aiui-proxy-closed.sh
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# aiui-proxy-closed.sh — S-15 deployed-surface check for 13-02-PLAN.md.
#
# T-13-08..T-13-12: `/aiui/api/claude/` and `/aiui/api/ollama/` used to proxy
# to an unauthenticated Python sidecar (port 3142) holding its own API key,
# and `/aiui/api/openrouter/` was a plain unauthenticated relay to a paid
# third-party API — anyone who could reach the node's web port could spend
# the owner's budget. A green `cargo test` on model_proxy.rs proves the Rust
# handler's own logic is correct in isolation; it proves NOTHING about which
# target nginx is actually pointed at on a deployed node, whether the old
# sidecar is still listening, or whether the OpenRouter relay still exists.
# This script is that proof, against a real node (13-AI-SPEC.md S-15 — "not
# a unit test and must not be treated as one").
#
# Usage: ./aiui-proxy-closed.sh <node-host> [ssh-user]
# HTTP checks always run, with NO session cookie, and must never see 200.
# SSH-based infra checks (systemd unit gone, port 3142 dark, single key
# ledger) run only when `sshpass` is installed AND $AIUI_TEST_SSH_PASS is
# set in the environment — otherwise they report SKIP, not FAIL. Never
# hardcode a password in this file (CLAUDE.md: never commit/push secrets).
#
# Exit 0 = every HTTP assertion passes AND every SSH assertion that ran passed.
set -uo pipefail
HOST="${1:?usage: aiui-proxy-closed.sh <node-host> [ssh-user]}"
SSH_USER="${2:-archipelago}"
BASE="http://${HOST}"
PASS=0; FAIL=0; SKIP=0
say() { printf '%s\n' "$*"; }
ok() { PASS=$((PASS+1)); say " PASS: $1"; }
bad() { FAIL=$((FAIL+1)); say " FAIL: $1"; }
skip() { SKIP=$((SKIP+1)); say " SKIP: $1"; }
# $1=method $2=path $3=optional body
status_of() {
if [ -n "${3:-}" ]; then
curl -s -m 10 -o /dev/null -w '%{http_code}' -X "$1" "${BASE}${2}" -d "$3" 2>/dev/null
else
curl -s -m 10 -o /dev/null -w '%{http_code}' -X "$1" "${BASE}${2}" 2>/dev/null
fi
}
# $1=label $2=observed status — closed means 401/403/404; 200 is the exposure.
assert_closed() {
case "$2" in
401|403|404) ok "$1 -> $2 (closed)" ;;
200) bad "$1 -> 200 (OPEN — an unauthenticated caller reached the backend)" ;;
*) bad "$1 -> '$2' (unexpected — want 401/403/404, and it is not 200 either)" ;;
esac
}
say "== S-15 AIUI model-proxy closure — node ${HOST} =="
# 1) /aiui/api/claude/v1/messages — no session cookie must never reach Anthropic.
S=$(status_of POST /aiui/api/claude/v1/messages '{"model":"claude-3-5-sonnet-20241022","max_tokens":1,"messages":[]}')
assert_closed "POST /aiui/api/claude/v1/messages (no session)" "$S"
# 2) /aiui/api/ollama/api/tags — no session cookie must never reach local Ollama.
S=$(status_of GET /aiui/api/ollama/api/tags)
assert_closed "GET /aiui/api/ollama/api/tags (no session)" "$S"
# 3) /aiui/api/openrouter/ — must be entirely GONE, not merely gated: 404 specifically.
S=$(status_of GET /aiui/api/openrouter/)
if [ "$S" = "404" ]; then
ok "GET /aiui/api/openrouter/ -> 404 (relay deleted)"
else
bad "GET /aiui/api/openrouter/ -> $S (want 404 — the relay must not exist at all, not just be gated)"
fi
say ""
say "-- SSH-based infra checks (${SSH_USER}@${HOST}) --"
if ! command -v sshpass >/dev/null 2>&1 || [ -z "${AIUI_TEST_SSH_PASS:-}" ]; then
skip "claude-api-proxy systemd unit (no sshpass or AIUI_TEST_SSH_PASS unset)"
skip "port 3142 listener (no sshpass or AIUI_TEST_SSH_PASS unset)"
skip "single key ledger (no sshpass or AIUI_TEST_SSH_PASS unset)"
else
ssh_run() {
sshpass -p "${AIUI_TEST_SSH_PASS}" ssh -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
"${SSH_USER}@${HOST}" "$1" 2>/dev/null
}
UNIT_STATE=$(ssh_run 'systemctl is-active claude-api-proxy 2>&1')
case "$UNIT_STATE" in
inactive|unknown) ok "claude-api-proxy unit is '$UNIT_STATE'" ;;
*"could not be found"*) ok "claude-api-proxy unit is gone (could not be found)" ;;
active) bad "claude-api-proxy unit is still ACTIVE — sidecar not torn down" ;;
*) bad "claude-api-proxy unit state unexpected: '$UNIT_STATE'" ;;
esac
PORT_COUNT=$(ssh_run "ss -ltn 2>/dev/null | grep -c ':3142 '")
[ "${PORT_COUNT:-1}" = "0" ] && ok "nothing listening on :3142" || bad "port 3142 still has a listener (count=${PORT_COUNT:-?})"
# claude-api-key's PRESENCE depends on whether an operator has configured a
# key on this node at all (via system.settings.set claude_api_key) — a
# freshly provisioned/dev node with no key set is expected to have neither
# file, and that is not a defect in this fix. The security-relevant
# invariant this plan makes is narrower and unconditional: the SECOND
# ledger (claude-api-proxy.env) must never exist, whether or not the first
# one does. Presence of claude-api-key is reported for visibility only.
LEDGER=$(ssh_run 'sudo ls /var/lib/archipelago/secrets/ 2>/dev/null')
if grep -qx 'claude-api-key' <<<"$LEDGER"; then
say " INFO: claude-api-key ledger present (a key is configured on this node)"
else
say " INFO: claude-api-key ledger absent (no key configured on this node yet — not a defect)"
fi
if grep -q 'claude-api-proxy.env' <<<"$LEDGER"; then
bad "claude-api-proxy.env still present — second key ledger not deleted"
else
ok "claude-api-proxy.env absent (single ledger enforced)"
fi
fi
say ""
say "== ${HOST}: ${PASS} passed, ${FAIL} failed, ${SKIP} skipped =="
[ "$FAIL" -eq 0 ]
tests/production-quality/deploy-guard-same-host.sh
Executable
+104
View File
@@ -0,0 +1,104 @@
#!/usr/bin/env bash
# deploy-guard-same-host.sh — regression pin for assert_safe_same_host_deploy
# (scripts/lib/common.sh), widened in 13-09 from containment-only to any
# resolved-path mismatch on the same host.
#
# The 2026-07-31 incident: a same-host `rsync --delete` deploy whose source
# was INSIDE the destination mirrored the source onto the destination and
# deleted ~1810 tracked files, a running dev server, and two sessions'
# uncommitted work. The original fix refused only containment (source-in-
# destination or destination-in-source). It missed SIBLING directories that
# share a parent but neither contains the other — e.g. this session's own
# worktree topology, archy-phase13 (source) vs archy (the main checkout,
# TARGET_DIR's resolved symlink target) — which is the identical rsync
# --delete hazard through a shape the old two-case guard let through.
#
# No SSH, no rsync, no real deploy — pure fixture strings against the
# function. Usage: ./deploy-guard-same-host.sh (takes no host argument)
# Exit 0 = all assertions pass.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")"
# shellcheck source=../../scripts/lib/common.sh
source "$PROJECT_DIR/scripts/lib/common.sh"
PASS=0; FAIL=0
say() { printf '%s\n' "$*"; }
ok() { PASS=$((PASS+1)); say " PASS: $1"; }
bad() { FAIL=$((FAIL+1)); say " FAIL: $1"; }
# Helper: run assert_safe_same_host_deploy and check its exit code against
# an expectation ("allow" or "refuse"), silencing its stderr message so the
# test output stays readable.
check() {
local desc="$1" src="$2" dst="$3" expect="$4"
local rc
assert_safe_same_host_deploy "$src" "$dst" >/dev/null 2>&1
rc=$?
if [ "$expect" = "allow" ]; then
[ "$rc" -eq 0 ] && ok "$desc" || bad "$desc (expected allow/exit 0, got exit $rc)"
else
[ "$rc" -ne 0 ] && ok "$desc" || bad "$desc (expected refuse/non-zero, got exit $rc)"
fi
}
say "== assert_safe_same_host_deploy — fixture matrix =="
# 1) Identical resolved source and destination: allowed. The normal
# in-place deploy from the main checkout onto its own symlinked
# destination.
check "identical resolved paths are allowed" \
"/home/archipelago/Projects/archy" \
"/home/archipelago/Projects/archy" \
"allow"
# 2) Source is inside (a subdirectory of) the destination: refused. The
# original 2026-07-31 containment case.
check "source-inside-destination is refused" \
"/home/archipelago/Projects/archy/.claude/worktrees/some-agent" \
"/home/archipelago/Projects/archy" \
"refuse"
# 3) Destination is inside the source: refused. The mirror-image
# containment case.
check "destination-inside-source is refused" \
"/home/archipelago/Projects/archy" \
"/home/archipelago/Projects/archy/.claude/worktrees/some-agent" \
"refuse"
# 4) Sibling-directory regression pin — the exact shape this session's own
# worktree topology exhibits, and the gap the old two-case guard let
# through: archy-phase13 (this worktree) as source, archy (the main
# checkout, TARGET_DIR's resolved symlink target) as destination. Share
# a parent (/home/archipelago/Projects); neither contains the other.
check "sibling directories (archy-phase13 vs archy) are refused [SIBLING REGRESSION PIN]" \
"/home/archipelago/Projects/archy-phase13" \
"/home/archipelago/Projects/archy" \
"refuse"
# 5) Two completely unrelated same-host paths with no shared parent at
# all: refused. Same-host plus any mismatch is refused, not just the
# two containment shapes.
check "unrelated paths with no shared parent are refused" \
"/home/archipelago/Projects/archy" \
"/opt/archipelago/web-ui" \
"refuse"
# 6) A refused case names both resolved paths and the 2026-07-31 incident
# on stderr, so a future operator understands why rather than
# reflexively retrying with a force flag.
MSG="$(assert_safe_same_host_deploy "/home/archipelago/Projects/archy-phase13" "/home/archipelago/Projects/archy" 2>&1 >/dev/null)"
if echo "$MSG" | grep -q '/home/archipelago/Projects/archy-phase13' \
&& echo "$MSG" | grep -q '/home/archipelago/Projects/archy' \
&& echo "$MSG" | grep -q '2026-07-31'; then
ok "refusal message names both resolved paths and the 2026-07-31 incident"
else
bad "refusal message missing an expected component: $MSG"
fi
say ""
say "== ${PASS} passed, ${FAIL} failed =="
[ "$FAIL" -eq 0 ]
tests/production-quality/lnd-cors-test.sh
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# lnd-cors-test.sh — assert the LND "connect your wallet" endpoints return
# correct CORS headers for the cross-origin call from the LND UI app (:18083).
#
# Bug B5: /lnd-connect-info duplicated ACAO on some nodes; /proxy/lnd/v1/* 401
# carries no ACAO fleet-wide. Browser blocks both.
#
# Usage: ./lnd-cors-test.sh <node-host> (e.g. 192.0.2.12 or 100.64.0.7)
# Exit 0 = all assertions pass.
set -uo pipefail
HOST="${1:?usage: lnd-cors-test.sh <node-host>}"
ORIGIN="http://${HOST}:18083"
BASE="http://${HOST}"
PASS=0; FAIL=0
say() { printf '%s\n' "$*"; }
ok() { PASS=$((PASS+1)); say " PASS: $1"; }
bad() { FAIL=$((FAIL+1)); say " FAIL: $1"; }
# Count ACAO header lines (case-insensitive) in a header dump.
acao_count() { grep -ci '^access-control-allow-origin:' <<<"$1"; }
acao_value() { grep -i '^access-control-allow-origin:' <<<"$1" | head -1 | sed 's/^[^:]*:[[:space:]]*//' | tr -d '\r'; }
say "== B5 LND CORS — node ${HOST} (origin ${ORIGIN}) =="
# 1) /lnd-connect-info — exactly ONE ACAO, value == origin
H=$(curl -s -m 8 -D - -o /dev/null -H "Origin: ${ORIGIN}" "${BASE}/lnd-connect-info" 2>/dev/null)
N=$(acao_count "$H"); V=$(acao_value "$H")
[ "$N" = "1" ] && ok "/lnd-connect-info has exactly 1 ACAO header" || bad "/lnd-connect-info ACAO count=$N (want 1)"
[ "$V" = "$ORIGIN" ] && ok "/lnd-connect-info ACAO value == origin" || bad "/lnd-connect-info ACAO='$V' (want '$ORIGIN')"
# 2) /proxy/lnd/v1/getinfo — ACAO present even on 401 (unauth)
H=$(curl -s -m 8 -D - -o /dev/null -H "Origin: ${ORIGIN}" "${BASE}/proxy/lnd/v1/getinfo" 2>/dev/null)
N=$(acao_count "$H")
[ "$N" -ge 1 ] && ok "/proxy/lnd/v1/getinfo has ACAO (even unauth)" || bad "/proxy/lnd/v1/getinfo missing ACAO (count=$N)"
[ "$N" -le 1 ] || bad "/proxy/lnd/v1/getinfo duplicate ACAO (count=$N)"
# 3) /proxy/lnd/v1/channels — same
H=$(curl -s -m 8 -D - -o /dev/null -H "Origin: ${ORIGIN}" "${BASE}/proxy/lnd/v1/channels" 2>/dev/null)
N=$(acao_count "$H")
[ "$N" = "1" ] && ok "/proxy/lnd/v1/channels has exactly 1 ACAO" || bad "/proxy/lnd/v1/channels ACAO count=$N (want 1)"
say ""
say "== ${HOST}: ${PASS} passed, ${FAIL} failed =="
[ "$FAIL" -eq 0 ]
tests/release/run.sh
Executable
+161
View File
@@ -0,0 +1,161 @@
#!/bin/bash
# Release gate harness — seed of the full-system test harness.
#
# Ties together the checks that already exist in this repo (catalog drift,
# release manifest, lifecycle bats, vitest, cargo tests) plus live-node
# smoke probes, so "is this release OK?" is one command instead of folklore.
#
# Usage:
# tests/release/run.sh # static + frontend + backend stages
# tests/release/run.sh --quick # static + frontend unit only
# tests/release/run.sh --with-build # also production-build the frontend
# # and verify the dist version changed
# tests/release/run.sh --manifest # also validate releases/manifest.json
# # (run AFTER create-release staged it)
# tests/release/run.sh --live [URL] # also smoke-probe a running node
# # (default http://127.0.0.1)
#
# Flags compose. Exits non-zero on the first failing stage.
#
# CAUTION (.116 and other dev nodes): full `cargo test -p archipelago` has
# hung tool PTYs here before — every cargo invocation below is wrapped in
# `timeout` and scoped to focused module filters.
set -u
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$REPO"
QUICK=0 WITH_BUILD=0 MANIFEST=0 LIVE=0 LIVE_URL="http://127.0.0.1"
while [[ $# -gt 0 ]]; do
case "$1" in
--quick) QUICK=1 ;;
--with-build) WITH_BUILD=1 ;;
--manifest) MANIFEST=1 ;;
--live) LIVE=1; [[ "${2:-}" == http* ]] && { LIVE_URL="$2"; shift; } ;;
*) echo "unknown flag: $1" >&2; exit 2 ;;
esac
shift
done
PASS=() FAIL=()
stage() { # stage <name> <cmd...>
local name="$1"; shift
echo
echo "=== [$name] $*"
if "$@"; then
echo "=== [$name] PASS"
PASS+=("$name")
else
echo "=== [$name] FAIL (exit $?)"
FAIL+=("$name")
summary 1
fi
}
summary() {
echo
echo "──────── release gate summary ────────"
printf 'PASS: %s\n' "${PASS[@]:-none}"
[[ ${#FAIL[@]} -gt 0 ]] && printf 'FAIL: %s\n' "${FAIL[@]}"
exit "${1:-0}"
}
# ── Stage 1: static ──────────────────────────────────────────────────
stage "git-diff-check" git diff --check
stage "cargo-fmt" timeout 240 cargo fmt --manifest-path core/Cargo.toml --all --check
stage "manifest-shell" python3 scripts/check-manifest-shell.py
stage "catalog-drift" python3 scripts/check-app-catalog-drift.py --release --strict
# Every release must surface its CHANGELOG entry in the Settings "What's New"
# modal. The modal hardcodes a block per version and has drifted behind before
# (sat at v1.7.84 while the fleet shipped to v1.7.92). Fail if any CHANGELOG
# version is missing a block; `python3 scripts/sync-whats-new.py` inserts them.
stage "whats-new-sync" python3 scripts/sync-whats-new.py --check
if [[ $MANIFEST -eq 1 ]]; then
stage "release-manifest" scripts/check-release-manifest.sh
fi
# ── Stage 2: frontend ────────────────────────────────────────────────
stage "ui-type-check" bash -c 'cd neode-ui && npm run --silent type-check'
stage "ui-unit-tests" bash -c 'cd neode-ui && npx vitest run --silent 2>&1 | tail -4; exit ${PIPESTATUS[0]}'
if [[ $WITH_BUILD -eq 1 ]]; then
# npm run build can fail silently (vue-tsc EACCES burned us before) —
# require the packaged output to actually contain the current version.
VERSION=$(grep -m1 '^version' core/archipelago/Cargo.toml | cut -d'"' -f2)
stage "ui-build" bash -c 'cd neode-ui && npm run build'
stage "ui-dist-version" bash -c "grep -rqo '${VERSION}' web/dist/neode-ui/assets/*.js"
fi
[[ $QUICK -eq 1 ]] && summary 0
# ── Stage 3: backend ─────────────────────────────────────────────────
stage "cargo-check" timeout 580 cargo check --manifest-path core/Cargo.toml -p archipelago
# Focused suites for the subsystems this release train touched:
# update:: — OTA download/apply/rollback/probe (v1.7.89 hardening)
# lnd — receive address + wallet readiness (v1.7.85–.89), incl. the
# structured receive-error reason-code classifier
# container::image_versions — image pinning / false-update detection
# scanner — RAII in-flight guard (v1.7.84)
# drift — published-port drift detection (the .116 self-heal)
# missing_secret — secret-resolution names the missing file (the .198 fix)
# 1500s: the non-incremental test-profile compile alone takes ~9 min on the
# .116 ThinkPad; 580s expires mid-compile (exit 124) before a single test runs.
stage "cargo-test-weekly" timeout 1500 env CARGO_INCREMENTAL=0 \
cargo test --manifest-path core/Cargo.toml -p archipelago -- \
update:: lnd container::image_versions scanner drift missing_secret
# ── Stage 4: live node smoke ─────────────────────────────────────────
if [[ $LIVE -eq 1 ]]; then
stage "live-frontend" bash -c "curl -skf -o /dev/null '$LIVE_URL/' || curl -skf -o /dev/null '${LIVE_URL/http:/https:}/'"
stage "live-aiui" curl -sf -o /dev/null "$LIVE_URL/aiui/"
stage "live-rpc" bash -c "curl -s -X POST '$LIVE_URL/rpc/v1' -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"update.status\",\"params\":{}}' | grep -qE '\"(result|error)\"'"
# Bitcoin-receive regression guard. The backend asks LND REST for a new
# on-chain address with ?type=<AddressType>. The REST gateway parses that
# as the proto enum (WITNESS_PUBKEY_HASH / 0), NOT the lncli aliases —
# sending "p2wkh" returns 400 "parsing field type ... is not a valid
# value" and bitcoin-receive silently breaks for the whole fleet (the bug
# that slipped through v1.7.88/89 because nothing exercised LND live).
# This hits LND REST directly and FAILS only on that exact parse-error
# signature; a "wallet locked" / "still syncing" reply means the type was
# accepted, which is all we're validating here.
stage "live-lnd-address-type" bash -c '
mac=$(sudo cat /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon 2>/dev/null | od -An -tx1 | tr -d " \n")
for port in 18080 8080; do
resp=$(curl -sk --max-time 8 "https://127.0.0.1:$port/v1/newaddress?type=WITNESS_PUBKEY_HASH" -H "Grpc-Metadata-macaroon: $mac" 2>/dev/null)
[ -z "$resp" ] && continue
echo "LND($port): $resp"
echo "$resp" | grep -q "is not a valid value" && { echo "FAIL: LND rejected the address type the backend sends"; exit 1; }
echo "OK: LND accepted the address type"; exit 0
done
echo "SKIP: LND REST not reachable on 18080/8080 — cannot validate address type live"; exit 0
'
# Wallet-unlock guard. After a restart/OTA, LND comes up LOCKED and the backend
# must auto-unlock it; if the unlock password is wrong (e.g. a fleet-wide
# constant vs a per-wallet password) the wallet stays LOCKED forever and ALL
# Bitcoin-receive / Lightning ops fail — fleet-wide, silently. Nothing else in
# this harness catches that: live-lnd-address-type explicitly treats "wallet
# locked" as a PASS, and os-audit treats lnd-unreachable as a WARN. This stage
# polls LND's unauthenticated /v1/state and FAILS if it is still LOCKED after a
# grace window. RPC_ACTIVE = unlocked (pass); NON_EXISTING/WAITING = no wallet
# yet (not a regression); unreachable = skip.
stage "live-lnd-unlocked" bash -c '
deadline=$(( $(date +%s) + 60 ))
while :; do
seen=""
for port in 18080 8080; do
st=$(curl -sk --max-time 6 "https://127.0.0.1:$port/v1/state" 2>/dev/null)
[ -z "$st" ] && continue
seen=1
echo "LND($port) state: $st"
echo "$st" | grep -q "RPC_ACTIVE" && { echo "OK: LND wallet is unlocked"; exit 0; }
echo "$st" | grep -qE "NON_EXISTING|WAITING_TO_START" && { echo "OK: LND wallet not initialized yet — not a lock regression"; exit 0; }
done
[ -z "$seen" ] && { echo "SKIP: LND /v1/state not reachable on 18080/8080"; exit 0; }
[ "$(date +%s)" -ge "$deadline" ] && { echo "FAIL: LND wallet still LOCKED after 60s — auto-unlock failed; Bitcoin-receive/Lightning are broken"; exit 1; }
sleep 5
done
'
fi
summary 0