Archipelago — open-source initial import
This commit is contained in:
Archipelago
Executable
+712
@@ -0,0 +1,712 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Container Doctor — diagnose and fix common container health issues
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./scripts/container-doctor.sh # Run locally on node
|
||||
# ./scripts/container-doctor.sh user@host # Run remotely via SSH
|
||||
#
|
||||
# Fixes:
|
||||
# 1. Stale podman ps/stats processes (>10 = pileup)
|
||||
# 2. Orphaned conmon/crun processes holding ports
|
||||
# 3. System tor conflicting with container tor
|
||||
# 4. Tor hidden service directory permissions (group/other must have no
|
||||
# access; Tor's own setgid 2700 is fine, restart is backed off)
|
||||
# 5. SearXNG read-only root / cap-drop ALL
|
||||
# 6. Bitcoin Knots prune+txindex conflict
|
||||
# 7. Containers stuck with exit code 127 (binary not found)
|
||||
# 8. Stopped core containers (rootless restart policy workaround)
|
||||
# 9. Missing rootless port listeners while Podman still shows published ports
|
||||
# 10. Nginx Proxy Manager public hosts not mirrored into host nginx
|
||||
# 11. BTCPay stores producing unpayable Lightning invoices (route hints off)
|
||||
# 12. Missing catatonit (Podman init binary) — init-enabled deploys fail
|
||||
#
|
||||
# Safe to run multiple times (idempotent). Never blocks deploy (exit 0 always).
|
||||
#
|
||||
|
||||
set -o pipefail
|
||||
|
||||
# Source pinned image versions (single source of truth)
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
[ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh"
|
||||
|
||||
FIXES_APPLIED=0
|
||||
CHECKS_PASSED=0
|
||||
FIX_NAMES=()
|
||||
|
||||
log() { echo "[$(date +%H:%M:%S)] DOCTOR: $*"; }
|
||||
|
||||
podman_rootless() {
|
||||
if [ "$(id -u)" = "0" ] && id archipelago >/dev/null 2>&1; then
|
||||
local archi_uid
|
||||
archi_uid=$(id -u archipelago)
|
||||
sudo -u archipelago env XDG_RUNTIME_DIR="/run/user/$archi_uid" podman "$@"
|
||||
else
|
||||
podman "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
port_is_listening() {
|
||||
local port="$1"
|
||||
ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
|
||||
}
|
||||
|
||||
run_fix() {
|
||||
local name="$1"
|
||||
shift
|
||||
if "$@"; then
|
||||
FIXES_APPLIED=$((FIXES_APPLIED + 1))
|
||||
FIX_NAMES+=("$name")
|
||||
else
|
||||
CHECKS_PASSED=$((CHECKS_PASSED + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Fix 1: Stale podman processes ────────────────────────────
|
||||
fix_stale_podman() {
|
||||
local count
|
||||
count=$(pgrep -f "podman (ps|stats)" 2>/dev/null | wc -l)
|
||||
count=${count:-0}
|
||||
if [ "$count" -gt 10 ]; then
|
||||
log "Killing $count stale podman ps/stats processes"
|
||||
pkill -f "podman (ps|stats)" 2>/dev/null || true
|
||||
sleep 2
|
||||
local after
|
||||
after=$(pgrep -f "podman (ps|stats)" 2>/dev/null | wc -l)
|
||||
after=${after:-0}
|
||||
log "Reduced from $count to $after"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 2: Orphaned conmon holding ports ─────────────────────
|
||||
fix_orphaned_conmon() {
|
||||
local fixed=false
|
||||
# Find conmon processes whose containers no longer exist
|
||||
local pids
|
||||
pids=$(pgrep -f "conmon.*--exit-command" 2>/dev/null || true)
|
||||
if [ -z "$pids" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Doctor runs as root but containers are rootless under archipelago user.
|
||||
# Must check container existence using the rootless podman database.
|
||||
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/1000 podman"
|
||||
|
||||
for pid in $pids; do
|
||||
# Extract container ID from conmon args
|
||||
local cid
|
||||
cid=$(tr '\0' ' ' < /proc/"$pid"/cmdline 2>/dev/null | grep -oP '(?<=-c )[a-f0-9]{64}' || true)
|
||||
if [ -z "$cid" ]; then
|
||||
continue
|
||||
fi
|
||||
# Check if container still exists in rootless podman
|
||||
if ! $PODMANCMD inspect "$cid" &>/dev/null; then
|
||||
local port_info
|
||||
port_info=$(ss -tlnp 2>/dev/null | grep "pid=$pid" | grep -oP ':\K\d+' | head -3 | tr '\n' ',' | sed 's/,$//')
|
||||
log "Killing orphaned conmon pid=$pid (ports: ${port_info:-none})"
|
||||
kill "$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null || true
|
||||
fixed=true
|
||||
fi
|
||||
done
|
||||
|
||||
$fixed && return 0 || return 1
|
||||
}
|
||||
|
||||
# ── Fix 3: Ensure system Tor is running (preferred over container) ──
|
||||
fix_system_tor_conflict() {
|
||||
# System Tor is preferred over container Tor.
|
||||
# If archy-tor container exists, remove it and use system Tor instead.
|
||||
if podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qE '^archy-tor$'; then
|
||||
podman stop archy-tor 2>/dev/null || true
|
||||
podman rm -f archy-tor 2>/dev/null || true
|
||||
log "Removed archy-tor container (system Tor is preferred)"
|
||||
fi
|
||||
|
||||
# Ensure system Tor is enabled and running
|
||||
if command -v tor >/dev/null 2>&1; then
|
||||
if ! systemctl is-active tor@default >/dev/null 2>&1; then
|
||||
systemctl enable tor tor@default 2>/dev/null || true
|
||||
systemctl start tor tor@default 2>/dev/null || true
|
||||
log "Started system Tor"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 4: Tor hidden service permissions ────────────────────
|
||||
# `stat -c '%a'` omits leading zeros, so a directory Tor manages itself
|
||||
# reads "2700" (setgid bit set) rather than "700" -- but the setgid bit
|
||||
# doesn't grant group/other any access. Comparing the full string against
|
||||
# the literal "700" therefore flagged Tor's own correct setting as broken
|
||||
# on every run, chmod'd it right back to what it already effectively was,
|
||||
# and restarted Tor -- forever, every ~5 minutes (the doctor timer
|
||||
# interval), so Tor never kept its consensus/HSDir cache long enough to
|
||||
# resolve any .onion peer. Only the last 3 digits of the mode are the
|
||||
# owner/group/other permission bits; anything before that is
|
||||
# setuid/setgid/sticky, which is irrelevant to "is group/other access
|
||||
# denied". Compare only those 3 digits so 700 and 2700 both pass, while a
|
||||
# genuinely permissive mode (750, 707, 2755, ...) is still corrected.
|
||||
#
|
||||
# Restart backoff: even a real, legitimate fix must not restart Tor more
|
||||
# than once per TOR_RESTART_BACKOFF_SECONDS, so no future defect in this
|
||||
# (or any other) predicate can reproduce a restart storm. The timestamp
|
||||
# file persists across doctor runs (and doctor is re-invoked from a fresh
|
||||
# shell every run, so this can't just be a shell variable).
|
||||
TOR_RESTART_BACKOFF_SECONDS=1800 # 30 minutes
|
||||
TOR_RESTART_STATE_FILE="/var/lib/archipelago/doctor-tor-last-restart"
|
||||
fix_tor_permissions() {
|
||||
local fixed=false
|
||||
local tor_dirs=("/var/lib/archipelago/tor" "/var/lib/tor")
|
||||
|
||||
for base in "${tor_dirs[@]}"; do
|
||||
if [ ! -d "$base" ]; then
|
||||
continue
|
||||
fi
|
||||
while IFS= read -r dir; do
|
||||
local perms
|
||||
perms=$(stat -c '%a' "$dir" 2>/dev/null)
|
||||
local mode_bits="${perms: -3}"
|
||||
if [ "$mode_bits" != "700" ]; then
|
||||
chmod 700 "$dir"
|
||||
log "Fixed permissions on $dir ($perms -> 700; group/other access was NOT fully denied)"
|
||||
fixed=true
|
||||
elif [ -n "$DOCTOR_DEBUG" ]; then
|
||||
log "DEBUG: $dir already denies group/other access (mode $perms) — no change"
|
||||
fi
|
||||
done < <(find "$base" -maxdepth 1 -name "hidden_service_*" -type d 2>/dev/null)
|
||||
done
|
||||
|
||||
# If we fixed a real permission defect, restart system Tor to pick up
|
||||
# the change -- but never more than once per backoff window.
|
||||
if $fixed; then
|
||||
local now last_restart elapsed
|
||||
now=$(date +%s)
|
||||
last_restart=$(cat "$TOR_RESTART_STATE_FILE" 2>/dev/null || echo 0)
|
||||
case "$last_restart" in *[!0-9]*|"") last_restart=0 ;; esac
|
||||
elapsed=$((now - last_restart))
|
||||
if [ "$elapsed" -ge "$TOR_RESTART_BACKOFF_SECONDS" ]; then
|
||||
systemctl restart tor@default 2>/dev/null || true
|
||||
mkdir -p "$(dirname "$TOR_RESTART_STATE_FILE")" 2>/dev/null
|
||||
echo "$now" > "$TOR_RESTART_STATE_FILE" 2>/dev/null
|
||||
log "Restarted Tor to apply hidden-service permission fix"
|
||||
else
|
||||
log "Skipped Tor restart: last restart was ${elapsed}s ago (backoff window ${TOR_RESTART_BACKOFF_SECONDS}s) — permissions were still corrected"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 5: SearXNG read-only / cap-drop ─────────────────────
|
||||
fix_searxng() {
|
||||
if ! podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^searxng$'; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local state
|
||||
state=$(podman inspect searxng --format '{{.State.Status}}' 2>/dev/null || true)
|
||||
local readonly_root
|
||||
readonly_root=$(podman inspect searxng --format '{{.HostConfig.ReadonlyRootfs}}' 2>/dev/null || true)
|
||||
local cap_drop
|
||||
cap_drop=$(podman inspect searxng --format '{{.HostConfig.CapDrop}}' 2>/dev/null || true)
|
||||
|
||||
# Fix if: exited, or has read-only root, or has cap-drop ALL
|
||||
local needs_fix=false
|
||||
if [ "$state" = "exited" ]; then
|
||||
needs_fix=true
|
||||
fi
|
||||
if [ "$readonly_root" = "true" ]; then
|
||||
needs_fix=true
|
||||
fi
|
||||
if [[ "$cap_drop" == *"ALL"* ]] || [[ "$cap_drop" == *"all"* ]]; then
|
||||
needs_fix=true
|
||||
fi
|
||||
|
||||
if ! $needs_fix; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Recreating SearXNG (readonly=$readonly_root, cap_drop=$cap_drop, state=$state)"
|
||||
|
||||
# Get current port mapping
|
||||
local port
|
||||
port=$(podman inspect searxng --format '{{range $k,$v := .HostConfig.PortBindings}}{{$k}}={{range $v}}{{.HostPort}}{{end}}{{println}}{{end}}' 2>/dev/null | head -1)
|
||||
local host_port="${port##*=}"
|
||||
host_port="${host_port:-8888}"
|
||||
|
||||
# Kill any stale conmon holding the port
|
||||
local conmon_pid
|
||||
conmon_pid=$(ss -tlnp 2>/dev/null | grep ":${host_port} " | grep -oP 'pid=\K\d+' | head -1)
|
||||
|
||||
podman stop searxng 2>/dev/null || true
|
||||
podman rm -f searxng 2>/dev/null || true
|
||||
|
||||
if [ -n "$conmon_pid" ]; then
|
||||
kill -9 "$conmon_pid" 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
podman run -d \
|
||||
--name searxng \
|
||||
--restart=unless-stopped \
|
||||
--security-opt=no-new-privileges:true \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=256m \
|
||||
-v searxng-config:/etc/searxng:rw \
|
||||
-v searxng-cache:/var/cache/searxng:rw \
|
||||
-p "${host_port}:8080" \
|
||||
--memory=512m \
|
||||
"${SEARXNG_IMAGE}" 2>&1 || true
|
||||
|
||||
log "SearXNG recreated (no readonly, no cap-drop ALL)"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Fix 6: Bitcoin Knots prune+txindex conflict ──────────────
|
||||
fix_bitcoin_txindex() {
|
||||
if ! podman ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^bitcoin-knots$'; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if bitcoin.conf has prune enabled
|
||||
local conf="/var/lib/archipelago/bitcoin/bitcoin.conf"
|
||||
if [ ! -f "$conf" ] || ! grep -q '^prune=' "$conf"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if container args include txindex
|
||||
local cmd
|
||||
cmd=$(podman inspect bitcoin-knots --format '{{json .Config.Cmd}}' 2>/dev/null || true)
|
||||
if ! echo "$cmd" | grep -q "txindex"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Bitcoin Knots: prune+txindex conflict detected"
|
||||
|
||||
# Get current config
|
||||
local image
|
||||
image=$(podman inspect bitcoin-knots --format '{{.ImageName}}' 2>/dev/null)
|
||||
local network
|
||||
network=$(podman inspect bitcoin-knots --format '{{.HostConfig.NetworkMode}}' 2>/dev/null)
|
||||
|
||||
# Read per-installation RPC password
|
||||
local SECRETS_DIR="/var/lib/archipelago/secrets"
|
||||
local BTC_RPC_PASS="archipelago"
|
||||
if [ -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
|
||||
BTC_RPC_PASS=$(cat "$SECRETS_DIR/bitcoin-rpc-password")
|
||||
fi
|
||||
|
||||
# Ensure bitcoin.conf has all RPC settings
|
||||
if ! grep -q 'rpcuser=' "$conf"; then
|
||||
cat > "$conf" <<BCONF
|
||||
server=1
|
||||
prune=550
|
||||
rpcuser=archipelago
|
||||
rpcpassword=$BTC_RPC_PASS
|
||||
rpcallowip=127.0.0.1/32
|
||||
rpcallowip=10.88.0.0/16
|
||||
listen=1
|
||||
printtoconsole=0
|
||||
BCONF
|
||||
log "Updated bitcoin.conf with full RPC settings"
|
||||
fi
|
||||
|
||||
# Remove stale txindex if present
|
||||
if [ -d "/var/lib/archipelago/bitcoin/indexes/txindex" ]; then
|
||||
find /var/lib/archipelago/bitcoin/indexes/txindex -type f -delete 2>/dev/null
|
||||
rmdir /var/lib/archipelago/bitcoin/indexes/txindex 2>/dev/null || true
|
||||
log "Removed stale txindex directory"
|
||||
fi
|
||||
|
||||
# Recreate without txindex
|
||||
podman stop bitcoin-knots 2>/dev/null || true
|
||||
podman rm -f bitcoin-knots 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Kill stale conmon on port 8332/8333
|
||||
for p in 8332 8333; do
|
||||
local cpid
|
||||
cpid=$(ss -tlnp 2>/dev/null | grep ":${p} " | grep -oP 'pid=\K\d+' | head -1)
|
||||
if [ -n "$cpid" ]; then
|
||||
kill -9 "$cpid" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
sleep 1
|
||||
|
||||
local net_arg=""
|
||||
if [ -n "$network" ] && [ "$network" != "bridge" ] && [ "$network" != "host" ]; then
|
||||
net_arg="--network=$network"
|
||||
elif [ "$network" = "host" ]; then
|
||||
net_arg="--network=host"
|
||||
else
|
||||
net_arg="--network=archy-net"
|
||||
fi
|
||||
|
||||
podman run -d \
|
||||
--name bitcoin-knots \
|
||||
--restart=always \
|
||||
$net_arg \
|
||||
-p 8332:8332 \
|
||||
-p 8333:8333 \
|
||||
-v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
|
||||
--memory=2g \
|
||||
--cap-drop=ALL \
|
||||
--cap-add=CHOWN \
|
||||
--cap-add=FOWNER \
|
||||
--cap-add=SETUID \
|
||||
--cap-add=SETGID \
|
||||
--cap-add=DAC_OVERRIDE \
|
||||
--security-opt=no-new-privileges:true \
|
||||
--health-cmd="bitcoin-cli -rpcuser=archipelago -rpcpassword=$BTC_RPC_PASS getblockchaininfo || exit 1" \
|
||||
--health-interval=30s \
|
||||
--health-retries=3 \
|
||||
"$image" 2>&1 || true
|
||||
|
||||
log "Bitcoin Knots recreated without txindex (prune mode)"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Fix 7: Exit code 127 containers ─────────────────────────
|
||||
fix_exit_127() {
|
||||
local containers
|
||||
containers=$(podman ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null | grep 'Exited (127)' | awk '{print $1}' || true)
|
||||
|
||||
if [ -z "$containers" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local fixed_names=()
|
||||
for name in $containers; do
|
||||
# Skip containers handled by other fixes
|
||||
if [ "$name" = "searxng" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Container $name has exit code 127 — recreating"
|
||||
# Get image and create command for recreation
|
||||
local image
|
||||
image=$(podman inspect "$name" --format '{{.ImageName}}' 2>/dev/null || true)
|
||||
local create_cmd
|
||||
create_cmd=$(podman inspect "$name" --format '{{json .Config.CreateCommand}}' 2>/dev/null || true)
|
||||
|
||||
podman rm -f "$name" 2>/dev/null || true
|
||||
|
||||
if [ -n "$create_cmd" ] && [ "$create_cmd" != "null" ]; then
|
||||
# Re-run the original create command (strip the leading "podman" and "run")
|
||||
local recreate_args
|
||||
recreate_args=$(echo "$create_cmd" | python3 -c "
|
||||
import json, sys
|
||||
args = json.load(sys.stdin)
|
||||
# Skip 'podman' and 'run', output the rest
|
||||
print(' '.join(['\"' + a + '\"' if ' ' in a else a for a in args[2:]]))
|
||||
" 2>/dev/null || true)
|
||||
|
||||
if [ -n "$recreate_args" ]; then
|
||||
eval "podman run $recreate_args" 2>&1 || true
|
||||
fixed_names+=("$name")
|
||||
log "Recreated $name from original args"
|
||||
else
|
||||
fixed_names+=("$name(removed)")
|
||||
log "Removed $name — will be recreated on next deploy"
|
||||
fi
|
||||
else
|
||||
fixed_names+=("$name(removed)")
|
||||
log "Removed $name — will be recreated on next deploy"
|
||||
fi
|
||||
done
|
||||
|
||||
[ ${#fixed_names[@]} -gt 0 ] && return 0 || return 1
|
||||
}
|
||||
|
||||
# ── Fix 8: Rootless netns egress lost ────────────────────────
|
||||
# Rootless podman uses pasta to give containers internet egress. If pasta's
|
||||
# tap vanishes (host link flap, mount churn, pasta dying during a boot-time
|
||||
# restart storm), the rootless-netns keeps inter-container traffic working
|
||||
# but silently loses outbound. Bitcoin IBD stalls at 0 peers; package pulls
|
||||
# fail. The repair must rebuild the netns from scratch: merely cycling the
|
||||
# containers reuses the existing (broken) netns because its holders
|
||||
# (aardvark-dns, podman's pause process) survive — observed on a test node
|
||||
# 2026-07-10, where the old stop/start-only cycle bounced all 35 containers
|
||||
# every timer run for ~an hour without ever restoring egress. So: stop the
|
||||
# containers, kill the netns holders, `podman system migrate`, clear the
|
||||
# stale netns state, then start everything back up.
|
||||
#
|
||||
# Destructive-action latch: cycling the whole fleet is a last resort. After
|
||||
# NETNS_CYCLE_MAX consecutive failed repairs we stop cycling (and log loudly)
|
||||
# until a run observes egress healthy again, which resets the counter.
|
||||
NETNS_CYCLE_STATE="/var/lib/archipelago/doctor-netns-cycle-failures"
|
||||
NETNS_CYCLE_MAX=3
|
||||
fix_rootless_netns_egress() {
|
||||
# Needs root for nsenter. When doctor runs as the rootless container owner,
|
||||
# a failed nsenter probe is a permissions artifact, not evidence of broken
|
||||
# egress; do not cycle the fleet from that context.
|
||||
[ "$(id -u)" = "0" ] || return 1
|
||||
|
||||
local archi_uid
|
||||
archi_uid=$(id -u archipelago 2>/dev/null) || return 1
|
||||
|
||||
# Locate the rootless-netns via aardvark-dns (it lives inside it).
|
||||
local aardvark_pid
|
||||
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
|
||||
[ -z "$aardvark_pid" ] && return 1 # no rootless network active
|
||||
|
||||
# Host precheck: if the host itself can't reach the internet, no point
|
||||
# cycling containers — this is an upstream problem.
|
||||
if ! timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Probe egress from inside the rootless-netns. One probe is noisy;
|
||||
# require two consecutive failures 10s apart to rule out transients.
|
||||
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
rm -f "$NETNS_CYCLE_STATE" # healthy again — re-arm the latch
|
||||
return 1 # first probe succeeded
|
||||
fi
|
||||
sleep 10
|
||||
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
|
||||
[ -z "$aardvark_pid" ] && return 1
|
||||
if timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
rm -f "$NETNS_CYCLE_STATE"
|
||||
return 1 # recovered on its own
|
||||
fi
|
||||
|
||||
# Latch: don't keep bouncing the fleet when the rebuild demonstrably
|
||||
# isn't fixing it.
|
||||
local failures
|
||||
failures=$(cat "$NETNS_CYCLE_STATE" 2>/dev/null || echo 0)
|
||||
case "$failures" in *[!0-9]*|"") failures=0;; esac
|
||||
if [ "$failures" -ge "$NETNS_CYCLE_MAX" ]; then
|
||||
log "Rootless-netns egress still broken but $failures rebuilds already failed — NOT cycling again (manual intervention needed; rm $NETNS_CYCLE_STATE to re-arm)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Rootless-netns egress is broken (host online, container netns unreachable) — rebuilding netns"
|
||||
|
||||
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/$archi_uid podman"
|
||||
local running
|
||||
running=$($PODMANCMD ps --format '{{.Names}}' 2>/dev/null)
|
||||
if [ -z "$running" ]; then
|
||||
log " No running containers to cycle — skipping"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local count
|
||||
count=$(echo "$running" | wc -l)
|
||||
log " Stopping $count running containers (graceful, 30s)..."
|
||||
$PODMANCMD stop --all --time 30 >/dev/null 2>&1
|
||||
sleep 5
|
||||
|
||||
# Tear the broken netns down for real: kill its holders and drop the
|
||||
# stale state so the first container start rebuilds pasta + aardvark-dns
|
||||
# from scratch. Without this, podman re-enters the old netns and the
|
||||
# missing pasta tap never comes back.
|
||||
log " Rebuilding rootless netns (killing holders, clearing state)..."
|
||||
pkill -U "$archi_uid" -x aardvark-dns 2>/dev/null
|
||||
pkill -U "$archi_uid" -x pasta 2>/dev/null
|
||||
pkill -U "$archi_uid" -x pasta.avx2 2>/dev/null
|
||||
pkill -U "$archi_uid" -x slirp4netns 2>/dev/null
|
||||
sleep 2
|
||||
$PODMANCMD system migrate >/dev/null 2>&1
|
||||
rm -rf "/run/user/$archi_uid/containers/networks"
|
||||
|
||||
log " Starting containers back up..."
|
||||
for c in $running; do
|
||||
$PODMANCMD start "$c" >/dev/null 2>&1 &
|
||||
done
|
||||
wait
|
||||
sleep 5
|
||||
|
||||
aardvark_pid=$(pgrep -U "$archi_uid" -f '^/usr/lib/podman/aardvark-dns' 2>/dev/null | head -1)
|
||||
if [ -n "$aardvark_pid" ] && timeout 3 nsenter -t "$aardvark_pid" -n bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
log " Rootless-netns egress restored ($count containers cycled)"
|
||||
rm -f "$NETNS_CYCLE_STATE"
|
||||
else
|
||||
failures=$((failures + 1))
|
||||
echo "$failures" > "$NETNS_CYCLE_STATE"
|
||||
log " WARN: egress still broken after rebuild (failure $failures/$NETNS_CYCLE_MAX) — may need manual intervention"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Fix 9: Restart stopped core containers ──────────────────
|
||||
# Rootless Podman 4.x restart policies don't auto-restart on crash.
|
||||
# This check restarts any exited core containers (tiers 0-2).
|
||||
fix_stopped_core_containers() {
|
||||
local core_containers="bitcoin-knots lnd electrumx mempool-api archy-mempool-web archy-mempool-db archy-btcpay-db archy-nbxplorer btcpay-server"
|
||||
local restarted=()
|
||||
# Doctor runs as root but containers are rootless under archipelago user
|
||||
local PODMANCMD="sudo -u archipelago XDG_RUNTIME_DIR=/run/user/1000 podman"
|
||||
|
||||
for name in $core_containers; do
|
||||
local state
|
||||
state=$($PODMANCMD inspect "$name" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
|
||||
if [ "$state" = "exited" ] || [ "$state" = "stopped" ]; then
|
||||
log "Restarting stopped container: $name"
|
||||
$PODMANCMD start "$name" 2>/dev/null && restarted+=("$name") || true
|
||||
fi
|
||||
done
|
||||
|
||||
[ ${#restarted[@]} -gt 0 ] && return 0 || return 1
|
||||
}
|
||||
|
||||
# ── Fix 10: Missing rootless port listeners ─────────────────
|
||||
# Rootless Podman can leave a container running with PortBindings still present
|
||||
# while the host-side rootlessport process has disappeared. Nginx then returns
|
||||
# 502 and direct app ports refuse connections even though `podman ps` looks OK.
|
||||
fix_missing_rootless_ports() {
|
||||
local containers
|
||||
containers=$(podman_rootless ps --format '{{.Names}}' 2>/dev/null || true)
|
||||
[ -n "$containers" ] || return 1
|
||||
|
||||
local fixed=false
|
||||
local name
|
||||
for name in $containers; do
|
||||
local ports
|
||||
ports=$(podman_rootless inspect "$name" --format '{{range $p,$bindings := .NetworkSettings.Ports}}{{if $bindings}}{{range $bindings}}{{.HostPort}}{{"\n"}}{{end}}{{end}}{{end}}' 2>/dev/null | sort -u)
|
||||
[ -n "$ports" ] || continue
|
||||
|
||||
local missing=()
|
||||
local port
|
||||
for port in $ports; do
|
||||
[ -n "$port" ] || continue
|
||||
if ! port_is_listening "$port"; then
|
||||
missing+=("$port")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
log "Restarting $name: missing rootlessport listener(s): ${missing[*]}"
|
||||
if podman_rootless restart "$name" >/dev/null 2>&1; then
|
||||
fixed=true
|
||||
else
|
||||
log "WARN: failed to restart $name for missing rootlessport listener(s)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
$fixed && return 0 || return 1
|
||||
}
|
||||
|
||||
# ── Fix 11: Nginx Proxy Manager public host bridge ───────────
|
||||
# Host nginx owns public 80/443 on Archipelago. Mirror NPM proxy hosts into
|
||||
# host nginx so issued certs and public traffic reach the intended upstreams.
|
||||
fix_npm_public_hosts() {
|
||||
local script="/opt/archipelago/scripts/sync-npm-public-hosts.sh"
|
||||
[ -x "$script" ] || script="$SCRIPT_DIR/sync-npm-public-hosts.sh"
|
||||
[ -x "$script" ] || return 1
|
||||
[ -f /var/lib/archipelago/nginx-proxy-manager/data/database.sqlite ] || return 1
|
||||
|
||||
if "$script" >/dev/null 2>&1; then
|
||||
log "Synced Nginx Proxy Manager public hosts into host nginx"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 12: BTCPay Lightning route hints ─────────────────────
|
||||
# A BTCPay store whose LND node has only private (unannounced) channels
|
||||
# produces BOLT11 invoices that external wallets cannot route to unless the
|
||||
# store's lightningPrivateRouteHints flag is on — payers see "no way to pay
|
||||
# this invoice" (observed on a test node 2026-07-10 with a Blink payer). Route
|
||||
# hints are a no-op with public channels and essential with private ones, so
|
||||
# the doctor enforces the flag on every store. BTCPay reads store blobs from
|
||||
# Postgres per request; no restart needed.
|
||||
fix_btcpay_route_hints() {
|
||||
local state
|
||||
state=$(podman_rootless inspect archy-btcpay-db --format '{{.State.Status}}' 2>/dev/null || echo "missing")
|
||||
[ "$state" = "running" ] || return 1
|
||||
|
||||
local count
|
||||
count=$(podman_rootless exec archy-btcpay-db psql -U btcpay -d btcpay -t -A -c \
|
||||
"SELECT count(*) FROM \"Stores\" WHERE (\"StoreBlob\"->>'lightningPrivateRouteHints') = 'false';" 2>/dev/null)
|
||||
[ -n "$count" ] && [ "$count" -gt 0 ] 2>/dev/null || return 1
|
||||
|
||||
if podman_rootless exec archy-btcpay-db psql -U btcpay -d btcpay -q -c \
|
||||
"UPDATE \"Stores\" SET \"StoreBlob\" = jsonb_set(\"StoreBlob\", '{lightningPrivateRouteHints}', 'true'::jsonb) WHERE (\"StoreBlob\"->>'lightningPrivateRouteHints') = 'false';" >/dev/null 2>&1; then
|
||||
log "Enabled Lightning route hints on $count BTCPay store(s) (private-channel invoices were unpayable)"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 13: Missing catatonit (container init binary) ────────
|
||||
# Podman resolves `--init` (and any Portainer/compose deploy with
|
||||
# "init: true") through catatonit; Debian's podman package only
|
||||
# Recommends it, so a node installed or upgraded without it fails those
|
||||
# deploys with a missing-init error (observed on a test node 2026-07-10
|
||||
# deploying sites via Portainer). install-podman.sh covers fresh ISO
|
||||
# installs; this heals nodes that predate it.
|
||||
fix_missing_catatonit() {
|
||||
command -v catatonit >/dev/null 2>&1 && return 1
|
||||
command -v apt-get >/dev/null 2>&1 || return 1
|
||||
|
||||
if DEBIAN_FRONTEND=noninteractive apt-get install -y catatonit >/dev/null 2>&1; then
|
||||
log "Installed catatonit (init-enabled container deploys were failing)"
|
||||
return 0
|
||||
fi
|
||||
log "WARNING: catatonit missing and apt-get install failed — init-enabled deploys will fail"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Fix 14: archipelago user missing dialout (mesh radios) ───
|
||||
# The image used to create the archipelago user with only `sudo`, so the
|
||||
# backend couldn't open /dev/ttyUSB*/ttyACM* serial LoRa radios — Mesh
|
||||
# never detected a plugged-in device (observed on the 1.7.99 RC install
|
||||
# 2026-07-13). Group change takes effect on the next service restart; we
|
||||
# restart archipelago only if a serial device is actually present.
|
||||
fix_archipelago_dialout() {
|
||||
id -nG archipelago 2>/dev/null | grep -qw dialout && return 1
|
||||
usermod -aG dialout archipelago 2>/dev/null || return 1
|
||||
log "Added archipelago to dialout (serial mesh radios were unreadable)"
|
||||
if ls /dev/ttyUSB* /dev/ttyACM* >/dev/null 2>&1; then
|
||||
systemctl try-restart archipelago 2>/dev/null || true
|
||||
log "Restarted archipelago to pick up dialout (radio present)"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────
|
||||
|
||||
# If remote host provided, run via SSH
|
||||
if [ -n "$1" ] && [ "$1" != "--local" ]; then
|
||||
REMOTE_HOST="$1"
|
||||
SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -i $SSH_KEY"
|
||||
|
||||
log "Running container doctor on $REMOTE_HOST"
|
||||
|
||||
# Copy script to remote and execute
|
||||
scp $SSH_OPTS "$0" "$REMOTE_HOST:/tmp/container-doctor.sh" 2>/dev/null
|
||||
ssh $SSH_OPTS "$REMOTE_HOST" "sudo bash /tmp/container-doctor.sh --local" 2>&1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Running locally (on the node itself)
|
||||
log "Starting container health check"
|
||||
|
||||
run_fix "stale-podman" fix_stale_podman
|
||||
run_fix "orphaned-conmon" fix_orphaned_conmon
|
||||
run_fix "system-tor" fix_system_tor_conflict
|
||||
run_fix "tor-permissions" fix_tor_permissions
|
||||
run_fix "searxng" fix_searxng
|
||||
run_fix "bitcoin-txindex" fix_bitcoin_txindex
|
||||
run_fix "exit-127" fix_exit_127
|
||||
run_fix "netns-egress" fix_rootless_netns_egress
|
||||
run_fix "stopped-core" fix_stopped_core_containers
|
||||
run_fix "rootless-ports" fix_missing_rootless_ports
|
||||
run_fix "npm-public-hosts" fix_npm_public_hosts
|
||||
run_fix "btcpay-route-hints" fix_btcpay_route_hints
|
||||
run_fix "catatonit" fix_missing_catatonit
|
||||
run_fix "dialout" fix_archipelago_dialout
|
||||
|
||||
echo ""
|
||||
if [ $FIXES_APPLIED -gt 0 ]; then
|
||||
log "Done: $FIXES_APPLIED fixes applied (${FIX_NAMES[*]}), $CHECKS_PASSED checks passed"
|
||||
else
|
||||
log "Done: all $CHECKS_PASSED checks passed — no fixes needed"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user