lfg2025/archy
1
0
Fork 0
Code Issues 24 Pull Requests 1 Actions Packages Projects Releases 64 Wiki Activity
Files
3c7a1fbbb586473135ed000b7a6f79c13c6f3951
archy/scripts/deploy-to-target.sh
T
archipelagoandClaude Opus 5 cf7a603ee5 fix(deploy): refuse deploy when rsync destination overlaps the source 
deploy-to-target.sh rsyncs with --delete to TARGET_DIR=/home/archipelago/archy,
which is a symlink to /home/archipelago/Projects/archy. archi-dev-box is this
same machine over loopback SSH, so deploying from the main checkout is a no-op
(source and destination resolve identically) — but deploying from a git worktree
nested under it mirrored that worktree onto the main checkout and deleted
everything else: ~1810 tracked files, the deploying worktree itself mid-run, a
running dev server, and two concurrent sessions' uncommitted work.

Guard compares /etc/machine-id across the SSH hop and, when source and
destination are on the same host, refuses if either path contains the other.
Identical paths still pass, so normal local deploys are unaffected.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 07:43:35 -04:00

2102 lines
104 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Deploy Archipelago code to the HP ProDesk target
#
# Usage:
# ./scripts/deploy-to-target.sh # Sync and rebuild
# ./scripts/deploy-to-target.sh --quick # Sync only, no rebuild
# ./scripts/deploy-to-target.sh --live # Deploy to live system (default: 192.168.1.228)
# ./scripts/deploy-to-target.sh --both # Deploy to 228, then copy to 198 + 253
# ./scripts/deploy-to-target.sh --frontend-only # Frontend-only deploy (skip Rust build + container rebuilds)
# ./scripts/deploy-to-target.sh --demo # Demo mode: Bitcoin pruning enabled (smaller disk)
# ./scripts/deploy-to-target.sh --dry-run --live # Show what would be deployed without executing
# ./scripts/deploy-to-target.sh --tailscale # Deploy to all 3 Tailscale alpha tester nodes
# ./scripts/deploy-to-target.sh --tailscale-node=arch2 # Deploy to a specific Tailscale node
#
set -eo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
# Load deploy config (password etc.) - deploy-config.sh is gitignored
[ -f "$SCRIPT_DIR/deploy-config.sh" ] && . "$SCRIPT_DIR/deploy-config.sh"
# Source pinned image versions (single source of truth)
[ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh"
# Source shared utility library
[ -f "$SCRIPT_DIR/lib/common.sh" ] && . "$SCRIPT_DIR/lib/common.sh"
# Configuration
TARGET_HOST="${ARCHIPELAGO_TARGET:-archipelago@192.168.1.228}"
TARGET_DIR="/home/archipelago/archy"
SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}"
SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -i $SSH_KEY"
DEPLOY_START=$(date +%s)
timestamp() { echo "[$(date +%H:%M:%S)]"; }
echo "╔════════════════════════════════════════════════════════════════╗"
echo "║ Deploying to Archipelago Target ║"
echo "╚════════════════════════════════════════════════════════════════╝"
echo ""
echo "$(timestamp) Target: $TARGET_HOST"
echo ""
# Parse arguments
QUICK=false
LIVE=false
BOTH=false
FRONTEND_ONLY=false
DEMO=false
DRY_RUN=false
CANARY=false
TAILSCALE=false
TAILSCALE_NODE=""
FLEET=false
RESET_MESH=false
for arg in "$@"; do
case $arg in
--quick) QUICK=true ;;
--live) LIVE=true ;;
--both) BOTH=true ;;
--frontend-only) FRONTEND_ONLY=true; LIVE=true ;;
--demo) DEMO=true ;;
--dry-run) DRY_RUN=true ;;
--canary) CANARY=true ;;
--tailscale) TAILSCALE=true ;;
--tailscale-node=*) TAILSCALE_NODE="${arg#*=}" ;;
--fleet) FLEET=true ;;
--all) FLEET=true ;;
--reset-mesh) RESET_MESH=true ;;
esac
done
# Fleet deploy: .228 → .198 → all 3 Tailscale nodes (all 5 servers)
if [ "$FLEET" = true ]; then
echo "╔════════════════════════════════════════════════════════════════╗"
echo "║ FLEET DEPLOY — All nodes ║"
echo "╚════════════════════════════════════════════════════════════════╝"
echo ""
echo "Phase 1: Build + deploy to .228 (primary build server)"
# Try LAN first, fall back to Tailscale IP
if ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1; then
"$0" --live || { echo "FAILED: .228 deploy"; exit 1; }
elif [ -n "${TAILSCALE_PRIMARY:-}" ] || tailscale status >/dev/null 2>&1; then
TS_PRIMARY="${TAILSCALE_PRIMARY:-$(tailscale status 2>/dev/null | grep 'archipelago-0' | awk '{print $1}')}"
if [ -n "$TS_PRIMARY" ]; then
echo " LAN unreachable — using Tailscale IP $TS_PRIMARY"
ARCHIPELAGO_TARGET="archipelago@${TS_PRIMARY}" "$0" --live || { echo "FAILED: .228 deploy via Tailscale"; exit 1; }
else
echo "FAILED: .228 unreachable on LAN or Tailscale"; exit 1
fi
else
echo "FAILED: .228 unreachable"; exit 1
fi
echo ""
echo "Phase 2: Copy to .198 + .253 (LAN secondaries — skip if unreachable)"
"$0" --both 2>/dev/null || echo " LAN secondaries unreachable, skipping"
echo ""
echo "Phase 3: Deploy to all Tailscale nodes (Arch 1/2/3)"
"$SCRIPT_DIR/deploy-tailscale.sh" --all || { echo "WARNING: Some Tailscale nodes failed"; }
echo ""
echo "════════════════════════════════════════════════════════════════"
echo "Fleet deploy complete."
exit 0
fi
# Tailscale deploy: delegate to deploy-tailscale.sh
if [ "$TAILSCALE" = true ]; then
echo "Deploying to all Tailscale nodes..."
exec "$SCRIPT_DIR/deploy-tailscale.sh" --all
fi
if [ -n "$TAILSCALE_NODE" ]; then
echo "Deploying to Tailscale node: $TAILSCALE_NODE"
exec "$SCRIPT_DIR/deploy-tailscale.sh" "$TAILSCALE_NODE"
fi
# Deploy locking — prevent concurrent deploys to the same target
TARGET_IP_FOR_LOCK="$(echo "$TARGET_HOST" | cut -d@ -f2)"
LOCK_DIR="/tmp/archipelago-deploy-${TARGET_IP_FOR_LOCK}.lock"
# Check for stale lock (older than 30 minutes)
if [ -d "$LOCK_DIR" ]; then
LOCK_STAMP="$LOCK_DIR/pid"
if [ -f "$LOCK_STAMP" ]; then
# macOS uses stat -f %m, Linux uses stat -c %Y
if stat -c %Y "$LOCK_STAMP" >/dev/null 2>&1; then
LOCK_MTIME=$(stat -c %Y "$LOCK_STAMP")
else
LOCK_MTIME=$(stat -f %m "$LOCK_STAMP")
fi
LOCK_AGE=$(( $(date +%s) - ${LOCK_MTIME:-0} ))
if [ "$LOCK_AGE" -gt 1800 ]; then
echo "$(timestamp) WARNING: Removing stale lock (${LOCK_AGE}s old)"
rm -rf "$LOCK_DIR"
fi
fi
fi
# mkdir is atomic — fails if directory already exists
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
echo "ERROR: Deploy already in progress for $TARGET_HOST (lock: $LOCK_DIR)"
exit 1
fi
echo $$ > "$LOCK_DIR"/pid
# Temp directory for intermediate files (cleaned up on exit)
TMPDIR="/tmp/archipelago-deploy-$$"
mkdir -p "$TMPDIR"
# Clean up lock and temp files on exit (normal, error, or signal)
cleanup_deploy() { rm -rf "$LOCK_DIR" "$TMPDIR"; }
trap cleanup_deploy EXIT
# Dry run mode: show what would be deployed without executing
if [[ "$DRY_RUN" == "true" ]]; then
echo "═══ DRY RUN MODE — no changes will be made ═══"
echo ""
echo "Target: $TARGET_HOST"
echo "Project: $PROJECT_DIR"
echo "Mode: $(
[[ "$BOTH" == "true" ]] && echo "both (.228 + .198)" || \
[[ "$LIVE" == "true" ]] && echo "live (.228)" || \
echo "dev (sync + build)"
)"
echo ""
echo "Files that would be synced:"
rsync -avn --exclude '.git' --exclude 'target' --exclude 'node_modules' \
--exclude 'dist' --exclude 'web/dist' --exclude '*.iso' \
--exclude 'image-recipe/_archived/build' --exclude 'image-recipe/_archived/results' \
--exclude '.codex-target-*' --exclude '.codex-tmp' --exclude 'uploads' \
"$PROJECT_DIR/" "$TARGET_HOST:$TARGET_DIR/" 2>/dev/null | \
grep -E '^[<>]|^deleting' | head -50 || echo " (rsync check failed — SSH may be unavailable)"
echo ""
echo "Frontend build: $(
[[ "$QUICK" == "true" ]] && echo "SKIP (--quick)" || echo "vue-tsc + vite build"
)"
echo "Backend build: $(
[[ "$FRONTEND_ONLY" == "true" ]] && echo "SKIP (--frontend-only)" || \
[[ "$QUICK" == "true" ]] && echo "SKIP (--quick)" || echo "cargo build --release"
)"
echo "Live deploy: $(
[[ "$LIVE" == "true" || "$BOTH" == "true" ]] && echo "YES — binary + frontend + nginx + systemd" || echo "NO"
)"
echo ""
echo "═══ DRY RUN COMPLETE — nothing was changed ═══"
exit 0
fi
# Section timing helper
section_start() { SECTION_START=$(date +%s); }
section_end() {
local elapsed=$(($(date +%s) - SECTION_START))
echo " (${elapsed}s)"
}
# ── Progress bar ──────────────────────────────────────────────
CURRENT_STEP=0
BAR_WIDTH=30
calculate_total_steps() {
local total=4 # SSH, prereqs, health, git state
if [[ "$QUICK" == "true" ]]; then
total=$((total + 1)) # sync only
echo $total; return
fi
total=$((total + 1)) # sync code
total=$((total + 1)) # frontend build
if [[ "$FRONTEND_ONLY" != "true" ]]; then
total=$((total + 1)) # backend build
fi
if [[ "$LIVE" == "true" ]]; then
total=$((total + 14)) # rollback, frontend, AIUI, nginx, systemd, claude proxy, dev mode, data dirs, nostr-provider, filebrowser, manifest, restart, HTTPS, health check
if [[ "$FRONTEND_ONLY" != "true" ]]; then
total=$((total + 1)) # deploy backend binary
total=$((total + 16)) # container rebuilds
fi
total=$((total + 3)) # UFW, IndeedHub fix, container doctor
fi
echo $total
}
TOTAL_STEPS=$(calculate_total_steps)
progress() {
CURRENT_STEP=$((CURRENT_STEP + 1))
local pct=$((CURRENT_STEP * 100 / TOTAL_STEPS))
local filled=$((pct * BAR_WIDTH / 100))
local empty=$((BAR_WIDTH - filled))
local bar
bar=$(printf '%*s' "$filled" '' | tr ' ' '█')$(printf '%*s' "$empty" '' | tr ' ' '░')
printf "\033[1;36m━━━ [%s] %3d%% (%d/%d)\033[0m %s\n" "$bar" "$pct" "$CURRENT_STEP" "$TOTAL_STEPS" "$1"
}
# ─────────────────────────────────────────────────────────────
# SSH connectivity pre-flight check
progress "Checking SSH connectivity"
if ! ssh $SSH_OPTS -o ConnectTimeout=5 "$TARGET_HOST" "echo ok" >/dev/null 2>&1; then
echo " ERROR: Cannot connect to $TARGET_HOST"
echo " Check that the server is on and reachable."
exit 1
fi
echo " Connected."
# Disk space pre-flight — abort if target is dangerously full
DISK_PCT=$(ssh $SSH_OPTS "$TARGET_HOST" "df / | tail -1 | awk '{print \$(NF-1)}' | tr -d '%'" 2>/dev/null)
if [ -n "$DISK_PCT" ] && [ "$DISK_PCT" -gt 85 ] 2>/dev/null; then
echo "ERROR: Target disk at ${DISK_PCT}% — need <85% for safe deploy. Free space and retry."
exit 1
fi
# Install prerequisites if missing (rsync for code sync, python3 for Claude API proxy)
progress "Checking prerequisites"
ssh $SSH_OPTS "$TARGET_HOST" '
NEED_INSTALL=""
command -v rsync >/dev/null 2>&1 || NEED_INSTALL="$NEED_INSTALL rsync"
command -v python3 >/dev/null 2>&1 || NEED_INSTALL="$NEED_INSTALL python3"
# python3 -m venv exists but cannot bootstrap pip without the matching
# <major>.<minor>-venv package on Debian — needed for reticulum-daemon/
# build.sh (archy-reticulum-daemon / archy-rnodeconf packaging).
if command -v python3 >/dev/null 2>&1 && ! python3 -c "import ensurepip" >/dev/null 2>&1; then
PYVER=$(python3 -c "import sys; print(f\"{sys.version_info.major}.{sys.version_info.minor}\")")
NEED_INSTALL="$NEED_INSTALL python3.${PYVER#*.}-venv"
fi
if ! command -v node >/dev/null 2>&1 || ! command -v npm >/dev/null 2>&1; then
echo " Node.js/npm not found — installing..."
curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - 2>&1 | tail -3
NEED_INSTALL="$NEED_INSTALL nodejs"
fi
if [ -n "$NEED_INSTALL" ]; then
echo " Installing:$NEED_INSTALL"
sudo apt-get update -qq && sudo apt-get install -y -qq $NEED_INSTALL 2>&1 | tail -3
else
echo " All prerequisites present"
fi
' 2>&1
# Pre-deploy health check (informational — warns but does not block)
progress "Pre-deploy health check"
TARGET_IP_ONLY="$(echo "$TARGET_HOST" | cut -d@ -f2)"
PRE_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || { echo "WARNING: Pre-deploy health check failed for $TARGET_IP_ONLY" >&2; echo "000"; })
if [ "$PRE_HEALTH" = "200" ]; then
echo " Server health: OK (200)"
else
echo " ⚠️ Server health: $PRE_HEALTH (may be down or unhealthy — deploying anyway)"
fi
echo ""
# Git state check — detect uncommitted changes and record deploy version
progress "Checking git state"
DEPLOY_COMMIT=$(git -C "$PROJECT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
DEPLOY_COMMIT_FULL=$(git -C "$PROJECT_DIR" rev-parse HEAD 2>/dev/null || echo "unknown")
DEPLOY_BRANCH=$(git -C "$PROJECT_DIR" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
DIRTY_FILES=$(git -C "$PROJECT_DIR" status --porcelain 2>/dev/null | grep -v '^??' | grep -v '\.claude/memory/' || true)
DEPLOY_DIRTY=false
echo "$(timestamp) Git state: $DEPLOY_BRANCH @ $DEPLOY_COMMIT"
if [ -n "$DIRTY_FILES" ]; then
DEPLOY_DIRTY=true
DIRTY_COUNT=$(echo "$DIRTY_FILES" | wc -l | tr -d ' ')
echo " ⚠️ WARNING: $DIRTY_COUNT uncommitted change(s) — deploying working directory, NOT last commit"
echo "$DIRTY_FILES" | head -10 | sed 's/^/ /'
[ "$DIRTY_COUNT" -gt 10 ] && echo " ... and $((DIRTY_COUNT - 10)) more"
echo ""
echo " To deploy clean: commit or stash changes first"
echo " Continuing in 3 seconds... (Ctrl+C to abort)"
sleep 3
else
echo " Working tree clean — deploying commit $DEPLOY_COMMIT"
fi
echo ""
# When --canary: deploy to 198 first, verify health, then deploy to 228
if [ "$CANARY" = true ]; then
echo "🐤 Canary deploy: .198 first, then .228 if healthy..."
echo ""
# Deploy to .228 (builds code), then copy to .198
"$0" --both
# Verify .198 is healthy before declaring success
echo ""
echo "🐤 Canary check: verifying .198 health..."
CANARY_OK=false
for i in $(seq 1 12); do
sleep 5
CANARY_HEALTH=$(curl -s --max-time 5 "http://192.168.1.198/health" 2>/dev/null || { echo "WARNING: Canary health check failed for 192.168.1.198" >&2; echo ""; })
if [ "$CANARY_HEALTH" = "OK" ]; then
echo " ✅ Canary .198 healthy after $((i * 5))s"
CANARY_OK=true
break
fi
done
if [ "$CANARY_OK" != "true" ]; then
echo " ❌ Canary .198 FAILED health check after 60s"
echo " ⚠️ .228 was also deployed. Check both servers."
exit 1
fi
echo "🐤 Canary deploy complete — both nodes healthy"
exit 0
fi
# ── deploy_secondary: copy built binary+frontend from .228 to a secondary node ──
# Usage: deploy_secondary <user@ip> <short_label> (e.g. deploy_secondary archipelago@192.168.1.198 198)
deploy_secondary() {
local SEC_TARGET="$1"
local SEC_LABEL="$2"
local SEC_IP="${SEC_TARGET#*@}"
echo ""
echo "📤 Copying to $SEC_IP (no rsync/cargo on that node)..."
scp $SSH_OPTS /tmp/archipelago-both "$SEC_TARGET:/tmp/archipelago-new"
ssh $SSH_OPTS "archipelago@192.168.1.228" "cd '$TARGET_DIR' && tar cf - web/dist/neode-ui 2>/dev/null" | ssh $SSH_OPTS "$SEC_TARGET" "mkdir -p /tmp/web-deploy && cd /tmp/web-deploy && tar xf -"
ssh $SSH_OPTS "$SEC_TARGET" '
sudo systemctl stop archipelago
sudo cp /tmp/archipelago-new /usr/local/bin/archipelago
sudo chmod +x /usr/local/bin/archipelago
rm -f /tmp/archipelago-new
sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +
sudo cp -r /tmp/web-deploy/web/dist/neode-ui/* /opt/archipelago/web-ui/ 2>/dev/null || true
sudo chown -R 1000:1000 /opt/archipelago/web-ui
'
# Deploy AIUI — prefer a sibling dist on the build host, but fall back to
# streaming from .228's /opt/archipelago/web-ui/aiui (where the ISO build
# deposited it). Without the fallback, secondaries lose AIUI whenever the
# deploy runs from a machine that doesn't have an ../AIUI checkout.
AIUI_DIST="$PROJECT_DIR/../AIUI/packages/app/dist"
if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
echo " Deploying AIUI to .$SEC_LABEL (from local sibling dist)..."
ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*"
cd "$AIUI_DIST" && tar --no-xattrs -cf - . | ssh $SSH_OPTS "$SEC_TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
cd "$PROJECT_DIR"
ssh $SSH_OPTS "$SEC_TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui"
elif ssh $SSH_OPTS archipelago@192.168.1.228 "[ -f /opt/archipelago/web-ui/aiui/index.html ]" 2>/dev/null; then
echo " Deploying AIUI to .$SEC_LABEL (streaming from .228)..."
ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*"
ssh $SSH_OPTS archipelago@192.168.1.228 "sudo tar --no-xattrs -cf - -C /opt/archipelago/web-ui/aiui ." \
| ssh $SSH_OPTS "$SEC_TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
ssh $SSH_OPTS "$SEC_TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui"
else
echo " ⚠️ AIUI not available locally or on .228 — skipping AIUI deploy to .$SEC_LABEL"
fi
# Sync nginx config + snippets
NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf"
SNIPPETS_DIR="$PROJECT_DIR/image-recipe/configs/snippets"
if [ -f "$NGINX_CFG" ]; then
echo " Syncing nginx config to .$SEC_LABEL..."
scp $SSH_OPTS "$NGINX_CFG" "$SEC_TARGET:/tmp/nginx-archipelago.conf" 2>/dev/null || true
ssh $SSH_OPTS "$SEC_TARGET" '
sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
sudo rm -f /etc/nginx/conf.d/external-app-proxies.conf
sudo sed -i "s|proxy_pass http://127.0.0.1:3141/;|proxy_pass http://127.0.0.1:3142/;|g" /etc/nginx/sites-available/archipelago
rm -f /tmp/nginx-archipelago.conf
' 2>/dev/null || true
fi
if [ -d "$SNIPPETS_DIR" ]; then
echo " Syncing nginx snippets to .$SEC_LABEL..."
ssh $SSH_OPTS "$SEC_TARGET" "sudo mkdir -p /etc/nginx/snippets" 2>/dev/null || true
for f in "$SNIPPETS_DIR"/*.conf; do
[ -f "$f" ] && scp $SSH_OPTS "$f" "$SEC_TARGET:/tmp/nginx-snippet-$(basename "$f")" 2>/dev/null || true
done
ssh $SSH_OPTS "$SEC_TARGET" '
for f in /tmp/nginx-snippet-*.conf; do
[ -f "$f" ] && sudo mv "$f" "/etc/nginx/snippets/$(basename "$f" | sed "s/^nginx-snippet-//")"
done
' 2>/dev/null || true
fi
ssh $SSH_OPTS "$SEC_TARGET" 'sudo nginx -t 2>&1 && echo " nginx config OK" || echo " nginx config test failed"' 2>/dev/null || true
# Sync systemd service file
SERVICE_FILE="$PROJECT_DIR/image-recipe/configs/archipelago.service"
if [ -f "$SERVICE_FILE" ]; then
echo " Syncing systemd service to .$SEC_LABEL..."
scp $SSH_OPTS "$SERVICE_FILE" "$SEC_TARGET:/tmp/archipelago.service" 2>/dev/null || true
ssh $SSH_OPTS "$SEC_TARGET" '
if ! diff -q /tmp/archipelago.service /etc/systemd/system/archipelago.service >/dev/null 2>&1; then
sudo cp /tmp/archipelago.service /etc/systemd/system/archipelago.service
sudo systemctl daemon-reload
echo " Service file updated"
else
echo " Service file unchanged"
fi
rm -f /tmp/archipelago.service
' 2>/dev/null || true
fi
if [ -n "${TELEMETRY_COLLECTOR_URL:-}" ]; then
echo " Syncing telemetry collector config to .$SEC_LABEL..."
TMP_TELEMETRY_ENV="$(mktemp)"
printf 'TELEMETRY_COLLECTOR_URL=%s\n' "$TELEMETRY_COLLECTOR_URL" > "$TMP_TELEMETRY_ENV"
scp $SSH_OPTS "$TMP_TELEMETRY_ENV" "$SEC_TARGET:/tmp/telemetry.env" 2>/dev/null || true
rm -f "$TMP_TELEMETRY_ENV"
ssh $SSH_OPTS "$SEC_TARGET" '
sudo mkdir -p /var/lib/archipelago
sudo cp /tmp/telemetry.env /var/lib/archipelago/telemetry.env
sudo chown archipelago:archipelago /var/lib/archipelago/telemetry.env
sudo chmod 600 /var/lib/archipelago/telemetry.env
rm -f /tmp/telemetry.env
' 2>/dev/null || true
fi
# Deploy udev rule for mesh radio
UDEV_RULE="$PROJECT_DIR/image-recipe/configs/99-mesh-radio.rules"
if [ -f "$UDEV_RULE" ]; then
echo " Syncing udev rule to .$SEC_LABEL..."
scp $SSH_OPTS "$UDEV_RULE" "$SEC_TARGET:/tmp/99-mesh-radio.rules" 2>/dev/null || true
ssh $SSH_OPTS "$SEC_TARGET" '
if ! diff -q /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules >/dev/null 2>&1; then
sudo cp /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --subsystem-match=tty
echo " Mesh radio udev rule installed"
else
echo " Mesh radio udev rule unchanged"
fi
rm -f /tmp/99-mesh-radio.rules
' 2>/dev/null || true
fi
# Deploy ALSA default-device config (routes ALSA "default" through PulseAudio/PipeWire)
ASOUND_CONF="$PROJECT_DIR/image-recipe/configs/asound.conf"
if [ -f "$ASOUND_CONF" ]; then
echo " Syncing ALSA default-device config to .$SEC_LABEL..."
scp $SSH_OPTS "$ASOUND_CONF" "$SEC_TARGET:/tmp/asound.conf" 2>/dev/null || true
ssh $SSH_OPTS "$SEC_TARGET" '
if ! diff -q /tmp/asound.conf /etc/asound.conf >/dev/null 2>&1; then
sudo cp /tmp/asound.conf /etc/asound.conf
echo " ALSA default-device config installed"
else
echo " ALSA default-device config unchanged"
fi
rm -f /tmp/asound.conf
' 2>/dev/null || true
fi
# Dev mode + FileBrowser
ssh $SSH_OPTS "$SEC_TARGET" '
# Dev mode
if ! grep -q "ARCHIPELAGO_DEV_MODE=true" /etc/systemd/system/archipelago.service.d/override.conf 2>/dev/null; then
sudo mkdir -p /etc/systemd/system/archipelago.service.d
printf "[Service]\nEnvironment=ARCHIPELAGO_DEV_MODE=true\n" | sudo tee /etc/systemd/system/archipelago.service.d/override.conf > /dev/null
sudo systemctl daemon-reload
fi
# FileBrowser fix
DOCKER=podman; command -v podman >/dev/null 2>&1 || DOCKER=docker
FB=$($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -x filebrowser || true)
if [ -n "$FB" ]; then
RO=$($DOCKER inspect filebrowser 2>/dev/null | grep -oP "\"ReadonlyRootfs\":\s*\K\w+" || echo "false")
if [ "$RO" = "true" ]; then
$DOCKER stop filebrowser 2>/dev/null; $DOCKER rm filebrowser 2>/dev/null
sudo mkdir -p /var/lib/archipelago/filebrowser
$DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>/dev/null
fi
fi
' 2>/dev/null || true
# Write deploy manifest
local DEPLOY_TS
DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
ssh $SSH_OPTS "$SEC_TARGET" "sudo tee /opt/archipelago/deploy-manifest.json > /dev/null" <<-MANIFEST_SEC_EOF
{
"commit": "$DEPLOY_COMMIT_FULL",
"commit_short": "$DEPLOY_COMMIT",
"branch": "$DEPLOY_BRANCH",
"dirty": $DEPLOY_DIRTY,
"deployed_at": "$DEPLOY_TS",
"deployed_from": "$(hostname)",
"target": "$SEC_TARGET"
}
MANIFEST_SEC_EOF
ssh $SSH_OPTS "$SEC_TARGET" "sudo systemctl start archipelago && sudo systemctl restart nginx"
# Run container doctor
echo " Running container doctor on .$SEC_LABEL..."
"$SCRIPT_DIR/container-doctor.sh" "$SEC_TARGET" 2>&1 | sed 's/^/ /' || true
# Post-deploy health check
echo " Checking .$SEC_LABEL health..."
local HEALTH="fail"
for i in $(seq 1 12); do
sleep 5
HEALTH=$(curl -s --max-time 5 "http://$SEC_IP/health" 2>/dev/null || { echo "WARNING: Health check failed for $SEC_IP" >&2; echo ""; })
if [ "$HEALTH" = "OK" ]; then
echo " ✅ $SEC_IP deployed (health OK after $((i * 5))s)"
break
fi
done
if [ "$HEALTH" != "OK" ]; then
echo " ⚠️ $SEC_IP deployed but health check failed after 60s"
fi
}
# When --both: deploy to 228 first, then copy to 198 + 253
if [ "$BOTH" = true ]; then
echo "Deploying to all LAN servers (228, then 198 + 253)..."
# Release lock so the recursive --live call can acquire it
rm -rf "$LOCK_DIR" 2>/dev/null; trap - EXIT
"$0" --live
echo ""
# Fetch built binary from .228 (shared by all secondary nodes)
if ! scp $SSH_OPTS "archipelago@192.168.1.228:$TARGET_DIR/core/target/release/archipelago" /tmp/archipelago-both 2>/dev/null; then
echo " ERROR: Failed to copy binary from .228 — is the build available?"
exit 1
fi
# Deploy to each secondary node
deploy_secondary "archipelago@192.168.1.198" "198"
deploy_secondary "archipelago@192.168.1.253" "253"
rm -f /tmp/archipelago-both
exit 0
fi
# Sync code
section_start
# GUARD (2026-07-31 incident): the rsync below is `--delete`, and TARGET_DIR
# (/home/archipelago/archy) is a SYMLINK to /home/archipelago/Projects/archy on
# archi-dev-box — which is this same machine over loopback SSH. Deploying from
# the main checkout is a harmless no-op (source and destination resolve to the
# same directory), but deploying from anywhere else on this host — a git
# worktree under .claude/worktrees/, a copy, a subdirectory — makes rsync mirror
# that source ONTO the main checkout and delete everything not present in it.
# That is exactly what happened: ~1810 tracked files deleted, the deploying
# worktree destroyed mid-run, a running dev server killed, and two concurrent
# sessions' uncommitted work lost permanently.
#
# Refuse the deploy when source and destination are on the same host AND either
# path contains the other. Identical paths are allowed (the normal local case).
_LOCAL_SRC="$(readlink -f "$PROJECT_DIR")"
_REMOTE_ID="$(ssh $SSH_OPTS "$TARGET_HOST" 'cat /etc/machine-id 2>/dev/null' 2>/dev/null || true)"
_LOCAL_ID="$(cat /etc/machine-id 2>/dev/null || true)"
if [ -n "$_REMOTE_ID" ] && [ "$_REMOTE_ID" = "$_LOCAL_ID" ]; then
_REMOTE_DST="$(ssh $SSH_OPTS "$TARGET_HOST" "readlink -f '$TARGET_DIR'" 2>/dev/null || true)"
if [ -n "$_REMOTE_DST" ] && [ "$_REMOTE_DST" != "$_LOCAL_SRC" ]; then
case "$_LOCAL_SRC/" in
"$_REMOTE_DST"/*)
echo "FATAL: refusing to deploy. Source '$_LOCAL_SRC' is INSIDE the deploy destination '$_REMOTE_DST' on this same host." >&2
echo " 'rsync --delete' would delete the destination's contents (including this source) — the 2026-07-31 data-loss incident." >&2
echo " Run this script from the main checkout ($_REMOTE_DST), not from a worktree or copy." >&2
exit 1 ;;
esac
case "$_REMOTE_DST/" in
"$_LOCAL_SRC"/*)
echo "FATAL: refusing to deploy. Destination '$_REMOTE_DST' is INSIDE the source '$_LOCAL_SRC' on this same host." >&2
exit 1 ;;
esac
fi
fi
progress "Syncing code"
rsync -avz --delete \
-e "ssh $SSH_OPTS" \
--exclude 'node_modules' \
--exclude 'target' \
--exclude 'dist' \
--exclude '.git' \
--exclude '.codex-target-*' \
--exclude '.codex-tmp' \
--exclude 'uploads' \
--exclude 'image-recipe/build' \
--exclude 'image-recipe/results' \
--exclude 'image-recipe/_archived/build' \
--exclude 'image-recipe/_archived/results' \
--exclude 'releases' \
--exclude '.venv' \
"$PROJECT_DIR/" "$TARGET_HOST:$TARGET_DIR/"
section_end
if [ "$QUICK" = true ]; then
echo ""
echo "✅ Quick sync complete!"
exit 0
fi
# Build on target
echo ""
progress "Building frontend"
section_start
ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/neode-ui && npm install --silent && npm run build" 2>&1 | sed 's/^/ /'
section_end
# Backend (if Rust is installed) — skip with --frontend-only
if [ "$FRONTEND_ONLY" = true ]; then
echo " Skipping backend build (--frontend-only)"
elif ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env 2>/dev/null && command -v cargo" >/dev/null 2>&1; then
progress "Building backend (Rust release)"
section_start
ssh $SSH_OPTS "$TARGET_HOST" "source ~/.cargo/env && cd $TARGET_DIR/core && cargo build --release 2>&1" | sed 's/^/ /'
section_end
else
echo " ⚠️ Rust not installed on target, skipping backend build"
fi
# reticulum-daemon tools (archy-reticulum-daemon, archy-rnodeconf) — skip with
# --frontend-only. Non-fatal on failure: these are supplementary mesh/radio
# tools, not required for the rest of the deploy to succeed, and build.sh
# handles its own venv/pip setup so a first run here is slower than later ones.
if [ "$FRONTEND_ONLY" = true ]; then
echo " Skipping reticulum-daemon tools build (--frontend-only)"
else
progress "Building reticulum-daemon tools (archy-reticulum-daemon, archy-rnodeconf)"
section_start
ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/reticulum-daemon && ./build.sh 2>&1" | sed 's/^/ /' \
|| echo " ⚠️ reticulum-daemon tools build failed — continuing without updating them"
section_end
fi
if [ "$LIVE" = true ]; then
# Create rollback backup before deploying
progress "Creating rollback backup"
ssh $SSH_OPTS "$TARGET_HOST" '
sudo mkdir -p /opt/archipelago/rollback
[ -f /usr/local/bin/archipelago ] && sudo cp /usr/local/bin/archipelago /opt/archipelago/rollback/archipelago.bak 2>/dev/null || true
[ -d /opt/archipelago/web-ui ] && sudo tar cf /opt/archipelago/rollback/web-ui.tar -C /opt/archipelago/web-ui . 2>/dev/null || true
' 2>/dev/null || true
# Deploy backend (check if binary exists) — skip with --frontend-only
if [ "$FRONTEND_ONLY" = true ]; then
echo " Skipping backend deploy (--frontend-only)"
elif ssh $SSH_OPTS "$TARGET_HOST" "[ -f $TARGET_DIR/core/target/release/archipelago ]" 2>/dev/null; then
progress "Deploying backend binary"
ssh $SSH_OPTS "$TARGET_HOST" 'sudo systemctl stop archipelago --no-block 2>/dev/null; sleep 2; sudo kill -9 $(pgrep -x archipelago) 2>/dev/null; sleep 1; true'
if [ "$RESET_MESH" = true ]; then
echo " Wiping mesh cache (peers/messages/sessions) per --reset-mesh"
ssh $SSH_OPTS "$TARGET_HOST" 'sudo rm -f /var/lib/archipelago/messages.json /var/lib/archipelago/sessions.json /var/lib/archipelago/mesh-outbox.json 2>/dev/null; true'
fi
ssh $SSH_OPTS "$TARGET_HOST" "sudo cp $TARGET_DIR/core/target/release/archipelago /usr/local/bin/"
fi
# Deploy reticulum-daemon tools (archy-reticulum-daemon, archy-rnodeconf) —
# skip with --frontend-only. Non-fatal: archipelago falls back to its dev
# venv path (ARCHY_RETICULUM_DAEMON_PY/_SCRIPT) if the packaged binary
# isn't present, so a missing/failed build here degrades rather than
# breaks mesh. archipelago is already stopped from the backend-binary
# step above, so this is a clean window to swap both binaries.
if [ "$FRONTEND_ONLY" = true ]; then
echo " Skipping reticulum-daemon tools deploy (--frontend-only)"
else
progress "Deploying reticulum-daemon tools"
for tool in archy-reticulum-daemon archy-rnodeconf; do
if ssh $SSH_OPTS "$TARGET_HOST" "[ -f $TARGET_DIR/reticulum-daemon/dist/$tool ]" 2>/dev/null; then
ssh $SSH_OPTS "$TARGET_HOST" "sudo cp $TARGET_DIR/reticulum-daemon/dist/$tool /usr/local/bin/ && sudo chmod +x /usr/local/bin/$tool"
echo " $tool deployed"
else
echo " ⚠️ $tool not built — leaving existing /usr/local/bin/$tool (if any) in place"
fi
done
fi
# Deploy frontend (preserve aiui/ and claude-login.html — they are NOT part of the neode-ui build)
progress "Deploying frontend"
ssh $SSH_OPTS "$TARGET_HOST" "sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name 'aiui' ! -name 'claude-login.html' -exec rm -rf {} +"
ssh $SSH_OPTS "$TARGET_HOST" "sudo cp -rf $TARGET_DIR/web/dist/neode-ui/* /opt/archipelago/web-ui/"
ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui"
# Build and deploy AIUI (non-fatal — never delete existing AIUI on failure)
progress "Building & deploying AIUI"
AIUI_DIR="$PROJECT_DIR/../AIUI"
AIUI_DIST="$AIUI_DIR/packages/app/dist"
# Auto-build AIUI if dist is missing or older than source
if [ -d "$AIUI_DIR/packages/app/src" ] && ( [ ! -f "$AIUI_DIST/index.html" ] || [ "$(find "$AIUI_DIR/packages/app/src" -newer "$AIUI_DIST/index.html" -print -quit 2>/dev/null)" != "" ] ); then
echo "$(timestamp) Building AIUI (source newer than dist or dist missing)..."
(cd "$AIUI_DIR" && VITE_BASE_PATH=/aiui/ pnpm build 2>&1 | tail -5) || echo "$(timestamp) ⚠️ AIUI build failed"
fi
# Fallback: if the AIUI sibling checkout is missing, use the pre-built
# dist shipped in this repo at demo/aiui/. That path is what we ship in
# the release tarball too, so local-and-fleet-update stay consistent.
if [ ! -f "$AIUI_DIST/index.html" ] && [ -f "$PROJECT_DIR/demo/aiui/index.html" ]; then
echo "$(timestamp) AIUI sibling dist missing — using demo/aiui/ from repo"
AIUI_DIST="$PROJECT_DIR/demo/aiui"
fi
if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
echo "$(timestamp) Deploying AIUI..."
if true; then
echo "$(timestamp) Deploying AIUI..."
ssh $SSH_OPTS "$TARGET_HOST" "sudo mkdir -p /opt/archipelago/web-ui/aiui"
ssh $SSH_OPTS "$TARGET_HOST" "sudo rm -rf /opt/archipelago/web-ui/aiui/*"
cd "$AIUI_DIST" && tar --no-xattrs -cf - . | ssh $SSH_OPTS "$TARGET_HOST" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
cd "$PROJECT_DIR"
ssh $SSH_OPTS "$TARGET_HOST" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui && sudo chmod 755 /opt/archipelago/web-ui/aiui && sudo find /opt/archipelago/web-ui/aiui -type d -exec chmod 755 {} \;"
else
echo "$(timestamp) ⚠️ AIUI build failed — keeping existing AIUI on server"
fi
else
echo "$(timestamp) ⚠️ AIUI not found at $AIUI_DIR, skipping"
fi
# Sync nginx config from image-recipe (single source of truth)
progress "Syncing nginx configuration"
NGINX_CFG="$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf"
SNIPPETS_DIR="$PROJECT_DIR/image-recipe/configs/snippets"
if [ -f "$NGINX_CFG" ]; then
scp $SSH_OPTS "$NGINX_CFG" "$TARGET_HOST:/tmp/nginx-archipelago.conf" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" '
sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
# Make sites-enabled a symlink to sites-available so future
# config updates actually take effect. Older deploys left
# sites-enabled as a regular file that fell out of sync.
if [ -f /etc/nginx/sites-enabled/archipelago ] && [ ! -L /etc/nginx/sites-enabled/archipelago ]; then
sudo rm -f /etc/nginx/sites-enabled/archipelago
sudo ln -s /etc/nginx/sites-available/archipelago /etc/nginx/sites-enabled/archipelago
elif [ ! -e /etc/nginx/sites-enabled/archipelago ]; then
sudo ln -s /etc/nginx/sites-available/archipelago /etc/nginx/sites-enabled/archipelago
fi
rm -f /tmp/nginx-archipelago.conf
' 2>/dev/null || true
fi
# Sync nginx snippet files (HTTPS app proxies, PWA headers — included by main config)
if [ -d "$SNIPPETS_DIR" ]; then
ssh $SSH_OPTS "$TARGET_HOST" "sudo mkdir -p /etc/nginx/snippets" 2>/dev/null || true
for f in "$SNIPPETS_DIR"/*.conf; do
[ -f "$f" ] && scp $SSH_OPTS "$f" "$TARGET_HOST:/tmp/nginx-snippet-$(basename "$f")" 2>/dev/null || true
done
ssh $SSH_OPTS "$TARGET_HOST" '
for f in /tmp/nginx-snippet-*.conf; do
[ -f "$f" ] && sudo mv "$f" "/etc/nginx/snippets/$(basename "$f" | sed "s/^nginx-snippet-//")"
done
' 2>/dev/null || true
fi
# Remove old port-based external app proxies config
ssh $SSH_OPTS "$TARGET_HOST" 'sudo rm -f /etc/nginx/conf.d/external-app-proxies.conf' 2>/dev/null || true
# Fix nginx Claude API proxy port (template uses 3141, proxy runs on 3142)
ssh $SSH_OPTS "$TARGET_HOST" 'sudo sed -i "s|proxy_pass http://127.0.0.1:3141/;|proxy_pass http://127.0.0.1:3142/;|g" /etc/nginx/sites-available/archipelago' 2>/dev/null || true
# Validate nginx config after all changes
ssh $SSH_OPTS "$TARGET_HOST" 'sudo nginx -t 2>&1 && echo " nginx config OK" || echo " ⚠️ nginx config test failed"' 2>/dev/null || true
# Sync systemd service file (single source of truth: image-recipe/configs/)
progress "Syncing systemd service"
SERVICE_FILE="$PROJECT_DIR/image-recipe/configs/archipelago.service"
if [ -f "$SERVICE_FILE" ]; then
scp $SSH_OPTS "$SERVICE_FILE" "$TARGET_HOST:/tmp/archipelago.service" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" '
if ! diff -q /tmp/archipelago.service /etc/systemd/system/archipelago.service >/dev/null 2>&1; then
sudo cp /tmp/archipelago.service /etc/systemd/system/archipelago.service
sudo systemctl daemon-reload
echo " Service file updated"
else
echo " Service file unchanged"
fi
rm -f /tmp/archipelago.service
' 2>/dev/null || true
fi
# Sync kiosk display helpers and units for HDMI/TV nodes. Existing nodes may
# not have a git checkout, so live deploy must carry these outside OTA too.
KIOSK_LAUNCHER="$PROJECT_DIR/image-recipe/configs/archipelago-kiosk-launcher.sh"
if [ -f "$KIOSK_LAUNCHER" ]; then
scp $SSH_OPTS "$KIOSK_LAUNCHER" "$TARGET_HOST:/tmp/archipelago-kiosk-launcher" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" '
sudo install -m 755 /tmp/archipelago-kiosk-launcher /usr/local/bin/archipelago-kiosk-launcher
rm -f /tmp/archipelago-kiosk-launcher
echo " Kiosk launcher updated"
' 2>/dev/null || true
fi
for unit in archipelago-kiosk.service archipelago-kiosk-watchdog.service; do
KIOSK_UNIT="$PROJECT_DIR/image-recipe/configs/$unit"
[ -f "$KIOSK_UNIT" ] || continue
scp $SSH_OPTS "$KIOSK_UNIT" "$TARGET_HOST:/tmp/$unit" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" "
if ! diff -q '/tmp/$unit' '/etc/systemd/system/$unit' >/dev/null 2>&1; then
sudo install -m 644 '/tmp/$unit' '/etc/systemd/system/$unit'
sudo systemctl daemon-reload
echo ' $unit updated'
else
echo ' $unit unchanged'
fi
rm -f '/tmp/$unit'
" 2>/dev/null || true
done
if [ -n "${TELEMETRY_COLLECTOR_URL:-}" ]; then
progress "Syncing telemetry collector config"
TMP_TELEMETRY_ENV="$(mktemp)"
printf 'TELEMETRY_COLLECTOR_URL=%s\n' "$TELEMETRY_COLLECTOR_URL" > "$TMP_TELEMETRY_ENV"
scp $SSH_OPTS "$TMP_TELEMETRY_ENV" "$TARGET_HOST:/tmp/telemetry.env" 2>/dev/null || true
rm -f "$TMP_TELEMETRY_ENV"
ssh $SSH_OPTS "$TARGET_HOST" '
sudo mkdir -p /var/lib/archipelago
sudo cp /tmp/telemetry.env /var/lib/archipelago/telemetry.env
sudo chown archipelago:archipelago /var/lib/archipelago/telemetry.env
sudo chmod 600 /var/lib/archipelago/telemetry.env
rm -f /tmp/telemetry.env
' 2>/dev/null || true
fi
# Deploy udev rule for mesh radio stable naming (/dev/mesh-radio)
UDEV_RULE="$PROJECT_DIR/image-recipe/configs/99-mesh-radio.rules"
if [ -f "$UDEV_RULE" ]; then
scp $SSH_OPTS "$UDEV_RULE" "$TARGET_HOST:/tmp/99-mesh-radio.rules" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" '
if ! diff -q /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules >/dev/null 2>&1; then
sudo cp /tmp/99-mesh-radio.rules /etc/udev/rules.d/99-mesh-radio.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --subsystem-match=tty
echo " Mesh radio udev rule installed"
else
echo " Mesh radio udev rule unchanged"
fi
rm -f /tmp/99-mesh-radio.rules
' 2>/dev/null || true
fi
# Deploy ALSA default-device config (routes ALSA "default" through
# PulseAudio/PipeWire — without it Chromium's raw ALSA fallback can't
# reach the HDMI sink and kiosk HDMI audio is silent).
ASOUND_CONF="$PROJECT_DIR/image-recipe/configs/asound.conf"
if [ -f "$ASOUND_CONF" ]; then
scp $SSH_OPTS "$ASOUND_CONF" "$TARGET_HOST:/tmp/asound.conf" 2>/dev/null || true
ssh $SSH_OPTS "$TARGET_HOST" '
if ! diff -q /tmp/asound.conf /etc/asound.conf >/dev/null 2>&1; then
sudo cp /tmp/asound.conf /etc/asound.conf
echo " ALSA default-device config installed"
else
echo " ALSA default-device config unchanged"
fi
rm -f /tmp/asound.conf
' 2>/dev/null || true
fi
# Deploy Claude API proxy (auto-install if missing)
progress "Setting up Claude API proxy"
ssh $SSH_OPTS "$TARGET_HOST" '
echo " Updating Claude API proxy on port 3142..."
# Check for API key in existing service or setup-aiui-server.sh
EXISTING_KEY=$(grep -oP "ANTHROPIC_API_KEY=\K.*" /etc/systemd/system/claude-api-proxy.service 2>/dev/null || true)
if [ -z "$EXISTING_KEY" ]; then
echo " ⚠️ No ANTHROPIC_API_KEY found — run setup-aiui-server.sh first to configure"
else
# Proxy script
sudo tee /opt/archipelago/claude-api-proxy.py > /dev/null << '\''PYEOF'\''
#!/usr/bin/env python3
import http.server, json, ssl, sys, os, urllib.request, urllib.error
API_KEY = os.environ.get("ANTHROPIC_API_KEY", "")
PORT = 3142
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(self):
if self.path == "/health":
self.send_response(200); self.send_header("Content-Type","application/json"); self.end_headers()
self.wfile.write(b"{\"status\":\"ok\"}"); return
cl = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(cl)
try: data = json.loads(body)
except: data = {}
if "max_tokens" not in data: data["max_tokens"] = 8096
for f in ["webSearch","web_search"]: data.pop(f, None)
# Normalize model IDs — map short/dotted names to full API model IDs
MODEL_MAP = {
"claude-haiku-4.5": "claude-haiku-4-5-20251001",
"claude-haiku-4-5": "claude-haiku-4-5-20251001",
"claude-sonnet-4": "claude-sonnet-4-20250514",
"claude-sonnet-4.5": "claude-sonnet-4-5-20250514",
"claude-sonnet-4-5": "claude-sonnet-4-5-20250514",
"claude-opus-4": "claude-opus-4-20250514",
}
m = data.get("model", "")
if m in MODEL_MAP: data["model"] = MODEL_MAP[m]
body = json.dumps(data).encode()
if not API_KEY:
err = json.dumps({"type":"error","error":{"type":"auth_error","message":"AIUI not configured. Set your Anthropic API key in Settings > AIUI to enable AI chat."}}).encode()
self.send_response(401); self.send_header("Content-Type","application/json"); self.send_header("Content-Length",str(len(err))); self.end_headers(); self.wfile.write(err); return
headers = {"Content-Type":"application/json","x-api-key":API_KEY,"anthropic-version":"2023-06-01","anthropic-dangerous-direct-browser-access":"true"}
for h in ["anthropic-version","anthropic-beta"]:
if self.headers.get(h): headers[h] = self.headers[h]
req = urllib.request.Request("https://api.anthropic.com"+self.path, data=body, headers=headers, method="POST")
try:
ctx = ssl.create_default_context()
resp = urllib.request.urlopen(req, context=ctx, timeout=300)
self.send_response(resp.status)
is_stream = "text/event-stream" in (resp.headers.get("Content-Type","") or "")
for k,v in resp.headers.items():
if k.lower() not in ("transfer-encoding","connection"): self.send_header(k,v)
if is_stream: self.send_header("Transfer-Encoding","chunked")
self.end_headers()
if is_stream:
while True:
chunk = resp.read(4096)
if not chunk: break
self.wfile.write(b"%x\r\n" % len(chunk)); self.wfile.write(chunk); self.wfile.write(b"\r\n"); self.wfile.flush()
self.wfile.write(b"0\r\n\r\n"); self.wfile.flush()
else: self.wfile.write(resp.read())
except urllib.error.HTTPError as e:
self.send_response(e.code); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(e.read())
except Exception as e:
self.send_response(502); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(json.dumps({"error":str(e)}).encode())
def do_GET(self):
if self.path == "/health":
self.send_response(200); self.send_header("Content-Type","application/json"); self.end_headers(); self.wfile.write(b"{\"status\":\"ok\"}")
else: self.send_response(404); self.end_headers()
def log_message(self, fmt, *args): pass
if not API_KEY: print("WARNING: ANTHROPIC_API_KEY not set — AIUI will return setup instructions")
server = http.server.HTTPServer(("127.0.0.1", PORT), Handler)
print(f"Claude API proxy on port {PORT}")
server.serve_forever()
PYEOF
sudo systemctl daemon-reload
sudo systemctl enable claude-api-proxy
sudo systemctl restart claude-api-proxy
sleep 1
echo " Claude API proxy: $(systemctl is-active claude-api-proxy)"
fi
' 2>/dev/null || true
# Dev mode for Tailscale HTTP access (cookies need Secure flag disabled over plain HTTP)
progress "Configuring dev mode"
ssh $SSH_OPTS "$TARGET_HOST" '
if [ -f /etc/systemd/system/archipelago.service.d/override.conf ] && grep -q "ARCHIPELAGO_DEV_MODE=true" /etc/systemd/system/archipelago.service.d/override.conf 2>/dev/null; then
echo " Dev mode already enabled"
else
echo " Enabling dev mode (for Tailscale HTTP cookie support)..."
sudo mkdir -p /etc/systemd/system/archipelago.service.d
printf "[Service]\nEnvironment=ARCHIPELAGO_DEV_MODE=true\n" | sudo tee /etc/systemd/system/archipelago.service.d/override.conf > /dev/null
sudo systemctl daemon-reload
echo " Dev mode enabled"
fi
' 2>/dev/null || true
# Create data directories for DWN, content sharing, federation, identities
progress "Creating data directories"
ssh $SSH_OPTS "$TARGET_HOST" '
# Rootless podman: allow binding to ports >= 80 (default is 1024)
if ! grep -q "unprivileged_port_start=80" /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null; then
echo "net.ipv4.ip_unprivileged_port_start=80" | sudo tee /etc/sysctl.d/99-rootless-podman.conf > /dev/null
sudo sysctl -p /etc/sysctl.d/99-rootless-podman.conf 2>/dev/null
echo " Rootless port binding enabled (>=80)"
fi
# Rootless podman: enable lingering for container persistence
if [ "$(loginctl show-user archipelago 2>/dev/null | grep Linger)" != "Linger=yes" ]; then
sudo loginctl enable-linger archipelago
echo " Linger enabled for archipelago user"
fi
# Rootless podman: enable podman socket
systemctl --user enable podman.socket 2>/dev/null || true
systemctl --user start podman.socket 2>/dev/null || true
sudo mkdir -p /var/lib/archipelago/dwn/messages
sudo mkdir -p /var/lib/archipelago/dwn/protocols
sudo mkdir -p /var/lib/archipelago/content/files
sudo mkdir -p /var/lib/archipelago/federation
sudo mkdir -p /var/lib/archipelago/identity
sudo mkdir -p /var/lib/archipelago/identities
sudo mkdir -p /var/lib/archipelago/tor-config
sudo chown -R archipelago:archipelago /var/lib/archipelago/dwn /var/lib/archipelago/content /var/lib/archipelago/federation /var/lib/archipelago/identity /var/lib/archipelago/identities /var/lib/archipelago/tor-config 2>/dev/null || true
# Fix secrets directory ownership (must be readable by archipelago user, not root)
sudo chown -R archipelago:archipelago /var/lib/archipelago/secrets 2>/dev/null || true
sudo chmod 700 /var/lib/archipelago/secrets 2>/dev/null || true
# Fix any root-owned files in data dir - dead mans switch, sessions, server-name
sudo find /var/lib/archipelago -maxdepth 1 -name "*.json" -user root -exec chown archipelago:archipelago {} \; 2>/dev/null || true
sudo chown archipelago:archipelago /var/lib/archipelago/server-name 2>/dev/null || true
echo " Data directories OK"
# Rootless podman UID mapping: fix data dir ownership so container processes
# can write. Rootless podman maps container UIDs via subuid (container UID 0 →
# host UID 1000, container UID N → host UID 100000+N).
echo " Fixing rootless podman UID mapping..."
# Containers running as root (UID 0 inside → host UID 100000 via subuid)
for dir in lnd electrumx btcpay nbxplorer immich jellyfin vaultwarden \
home-assistant fedimint fedimint-gateway photoprism ollama filebrowser; do
[ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
done
# Bitcoin Knots: container UID 101 → host UID 100101
[ -d /var/lib/archipelago/bitcoin ] && sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
# Postgres containers: container UID 70 → host UID 100070
for dir in postgres-btcpay immich-db; do
[ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null
done
# MariaDB: container UID 999 → host UID 100999
[ -d /var/lib/archipelago/mempool ] && sudo chown -R 100999:100999 /var/lib/archipelago/mempool 2>/dev/null
# Grafana: container UID 472 → host UID 100472
[ -d /var/lib/archipelago/grafana ] && sudo chown -R 100472:100472 /var/lib/archipelago/grafana 2>/dev/null
echo " UID mapping done"
' 2>/dev/null || true
# Deploy nostr-provider.js for NIP-07 iframe signing (window.nostr support)
progress "Deploying nostr-provider.js"
scp $SSH_OPTS "$PROJECT_DIR/neode-ui/public/nostr-provider.js" "$TARGET_HOST:/tmp/nostr-provider.js" 2>/dev/null && \
ssh $SSH_OPTS "$TARGET_HOST" 'sudo cp /tmp/nostr-provider.js /opt/archipelago/web-ui/nostr-provider.js && echo " nostr-provider.js deployed"' 2>/dev/null || echo " (nostr-provider.js not found, skipping)"
# Deploy tor-helper: script + systemd path unit for privileged Tor management
progress "Deploying tor-helper"
scp $SSH_OPTS \
"$PROJECT_DIR/scripts/tor-helper.sh" \
"$PROJECT_DIR/image-recipe/configs/archipelago-tor-helper.path" \
"$PROJECT_DIR/image-recipe/configs/archipelago-tor-helper.service" \
"$TARGET_HOST:/tmp/" 2>/dev/null && \
ssh $SSH_OPTS "$TARGET_HOST" '
sudo mkdir -p /opt/archipelago/scripts
sudo cp /tmp/tor-helper.sh /opt/archipelago/scripts/tor-helper.sh
sudo chmod 755 /opt/archipelago/scripts/tor-helper.sh
sudo chown root:root /opt/archipelago/scripts/tor-helper.sh
sudo cp /tmp/archipelago-tor-helper.path /etc/systemd/system/
sudo cp /tmp/archipelago-tor-helper.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable archipelago-tor-helper.path
sudo systemctl start archipelago-tor-helper.path
echo " tor-helper deployed with systemd path unit"
' 2>/dev/null || echo " (tor-helper deploy skipped)"
# Sync nginx config (second pass — includes HTTPS snippets)
scp $SSH_OPTS "$PROJECT_DIR/image-recipe/configs/nginx-archipelago.conf" "$TARGET_HOST:/tmp/nginx-archipelago.conf" 2>/dev/null && \
ssh $SSH_OPTS "$TARGET_HOST" '
sudo cp /tmp/nginx-archipelago.conf /etc/nginx/sites-available/archipelago
# Also sync HTTPS snippets if they exist
sudo mkdir -p /etc/nginx/snippets
echo " Nginx config synced"
' 2>/dev/null || echo " (nginx config sync skipped)"
# Sync HTTPS app proxies snippet if it exists
if [ -f "$PROJECT_DIR/image-recipe/configs/snippets/archipelago-https-app-proxies.conf" ]; then
scp $SSH_OPTS "$PROJECT_DIR/image-recipe/configs/snippets/archipelago-https-app-proxies.conf" "$TARGET_HOST:/tmp/https-app-proxies.conf" 2>/dev/null && \
ssh $SSH_OPTS "$TARGET_HOST" 'sudo cp /tmp/https-app-proxies.conf /etc/nginx/snippets/archipelago-https-app-proxies.conf' 2>/dev/null || true
fi
# Fix FileBrowser — recreate if read-only root, create if missing
progress "Checking FileBrowser"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
FB_EXISTS=$($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -x filebrowser || true)
if [ -n "$FB_EXISTS" ]; then
RO=$($DOCKER inspect filebrowser 2>/dev/null | grep -oP "\"ReadonlyRootfs\":\s*\K\w+" || echo "false")
if [ "$RO" = "true" ]; then
echo " FileBrowser has read-only root — recreating..."
$DOCKER stop filebrowser 2>/dev/null
$DOCKER rm filebrowser 2>/dev/null
sudo mkdir -p /var/lib/archipelago/filebrowser
$DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>&1 | tail -1
echo " FileBrowser recreated"
else
echo " FileBrowser OK"
fi
else
echo " Creating FileBrowser..."
sudo mkdir -p /var/lib/archipelago/filebrowser
$DOCKER run -d --name filebrowser --restart=unless-stopped --user 0:0 \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 8083:80 -v /var/lib/archipelago/filebrowser:/srv "$FILEBROWSER_IMAGE" 2>&1 | tail -1
echo " FileBrowser created"
fi
' 2>/dev/null || true
# Write deploy manifest — stamps the server with exactly what was deployed
progress "Writing deploy manifest"
DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
ssh $SSH_OPTS "$TARGET_HOST" "sudo tee /opt/archipelago/deploy-manifest.json > /dev/null" << MANIFEST_EOF
{
"commit": "$DEPLOY_COMMIT_FULL",
"commit_short": "$DEPLOY_COMMIT",
"branch": "$DEPLOY_BRANCH",
"dirty": $DEPLOY_DIRTY,
"deployed_at": "$DEPLOY_TS",
"deployed_from": "$(hostname)",
"target": "$TARGET_HOST"
}
MANIFEST_EOF
# Write build-info.txt — this is what the UI sidebar reads for the
# displayed version (overrides the binary's CARGO_PKG_VERSION). Keeping
# it synced with Cargo.toml on every deploy prevents the 1.3.x drift
# we saw on .198/.253 where stale build-info survived across upgrades.
DEPLOY_PKG_VERSION=$(grep '^version' "$PROJECT_DIR/core/archipelago/Cargo.toml" | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
ssh $SSH_OPTS "$TARGET_HOST" "sudo tee /opt/archipelago/build-info.txt > /dev/null" << BUILDINFO_EOF
version=$DEPLOY_PKG_VERSION
build=$DEPLOY_TS
commit=$DEPLOY_COMMIT
date=$DEPLOY_TS
type=deployed
BUILDINFO_EOF
# Ensure NTP and swap are configured (prevents OOM kills and clock drift)
progress "Ensuring NTP + swap"
ssh $SSH_OPTS "$TARGET_HOST" '
# NTP via chrony
if ! dpkg -l chrony >/dev/null 2>&1; then
sudo rm -f /usr/sbin/policy-rc.d
sudo apt-get update -qq && sudo apt-get install -y chrony 2>/dev/null
fi
sudo systemctl enable chrony 2>/dev/null
sudo systemctl start chrony 2>/dev/null
sudo timedatectl set-ntp true 2>/dev/null
# Swap
if [ ! -f /swapfile ]; then
TOTAL_KB=$(grep MemTotal /proc/meminfo | awk "{print \$2}")
SZ=$((TOTAL_KB / 1024 / 1024))
[ "$SZ" -gt 8 ] && SZ=8
[ "$SZ" -lt 2 ] && SZ=2
sudo fallocate -l ${SZ}G /swapfile && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile
grep -q "/swapfile" /etc/fstab || echo "/swapfile none swap sw 0 0" | sudo tee -a /etc/fstab
echo " Created ${SZ}G swap"
fi
sudo swapon /swapfile 2>/dev/null || true
' 2>&1 | tail -5 | sed 's/^/ /' || true
# Ensure backend binds to localhost only (security: no direct LAN access to port 5678)
progress "Securing backend bind address"
ssh $SSH_OPTS "$TARGET_HOST" '
if grep -q "ARCHIPELAGO_BIND=0.0.0.0" /etc/systemd/system/archipelago.service 2>/dev/null; then
sudo sed -i "s/ARCHIPELAGO_BIND=0.0.0.0:5678/ARCHIPELAGO_BIND=127.0.0.1:5678/" /etc/systemd/system/archipelago.service
sudo systemctl daemon-reload
echo " Fixed: backend now binds to 127.0.0.1 only"
fi
' 2>/dev/null || true
# Restart services
progress "Restarting services"
ssh $SSH_OPTS "$TARGET_HOST" "sudo systemctl start archipelago && sudo systemctl restart nginx"
# Set up HTTPS for PWA installability (browsers require secure context)
progress "Setting up HTTPS"
ssh $SSH_OPTS "$TARGET_HOST" "sudo bash $TARGET_DIR/scripts/setup-https-dev.sh" 2>&1 | sed 's/^/ /' || true
if [ "$FRONTEND_ONLY" = true ]; then
echo " Skipping container rebuilds (--frontend-only)"
fi
# App containers are now installed exclusively via the Marketplace UI.
# The deploy script only handles code sync, backend build, and frontend build.
if false; then # Legacy app installation removed — kept for reference in git history
progress "Rebuilding LND UI"
if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/lnd-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t lnd-ui:local . || docker build --no-cache -t lnd-ui:local .)" 2>&1 | tail -12 | sed 's/^/ /'; then
echo " Recreating LND UI container (port 18083)..."
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i lnd-ui); do
[ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
done
$DOCKER run -d --name archy-lnd-ui -p 18083:80 --memory=256m --restart unless-stopped lnd-ui:local
' 2>&1 | sed 's/^/ /' || true
fi
# Rebuild and recreate ElectrumX UI container (port 50002)
progress "Rebuilding ElectrumX UI"
if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/electrs-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t electrs-ui:local . || docker build --no-cache -t electrs-ui:local .)" 2>&1 | tail -12 | sed 's/^/ /'; then
echo " Recreating ElectrumX UI container (port 50002, host network)..."
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i electrs-ui); do
[ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
done
$DOCKER run -d --name archy-electrs-ui --network host --memory=256m --restart unless-stopped electrs-ui:local
' 2>&1 | sed 's/^/ /' || true
fi
# Rebuild and recreate Bitcoin UI container (host network, port 8334 in nginx.conf)
# Host network required: bitcoin-ui proxies Bitcoin RPC at 127.0.0.1:8332
progress "Rebuilding Bitcoin UI"
# Inject real RPC credentials into bitcoin-ui nginx config before building
ssh $SSH_OPTS "$TARGET_HOST" '
SECRETS_DIR="/var/lib/archipelago/secrets"
RPC_PASS=$(sudo cat "$SECRETS_DIR/bitcoin-rpc-password" 2>/dev/null)
if [ -n "$RPC_PASS" ]; then
AUTH_B64=$(echo -n "archipelago:${RPC_PASS}" | base64)
sed -i "s|__BITCOIN_RPC_AUTH__|${AUTH_B64}|g" '"$TARGET_DIR"'/docker/bitcoin-ui/nginx.conf
fi
' 2>/dev/null || true
if ssh $SSH_OPTS "$TARGET_HOST" "cd $TARGET_DIR/docker/bitcoin-ui && (command -v podman >/dev/null 2>&1 && podman build --no-cache -t bitcoin-ui:local . || docker build --no-cache -t bitcoin-ui:local .)" 2>&1 | tail -12 | sed 's/^/ /'; then
echo " Recreating Bitcoin UI container (port 8334, host network)..."
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
for c in $($DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -i bitcoin-ui); do
[ -n "$c" ] && $DOCKER stop "$c" 2>/dev/null; $DOCKER rm -f "$c" 2>/dev/null
done
$DOCKER run -d --name archy-bitcoin-ui --network host --memory=256m --restart unless-stopped bitcoin-ui:local
' 2>&1 | sed 's/^/ /' || true
fi
# Bitcoin Knots: required for Mempool, ElectrumX, BTCPay, Fedimint
TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
# Read Bitcoin RPC credentials from secrets file (rpcauth — stable across restarts)
progress "Reading Bitcoin RPC credentials"
BITCOIN_RPC_PASS=$(ssh $SSH_OPTS "$TARGET_HOST" '
SECRETS_DIR="/var/lib/archipelago/secrets"
sudo mkdir -p "$SECRETS_DIR" && sudo chmod 700 "$SECRETS_DIR"
if [ ! -f "$SECRETS_DIR/bitcoin-rpc-password" ]; then
openssl rand -hex 16 | sudo tee "$SECRETS_DIR/bitcoin-rpc-password" > /dev/null
sudo chmod 600 "$SECRETS_DIR/bitcoin-rpc-password"
fi
sudo cat "$SECRETS_DIR/bitcoin-rpc-password"
' 2>/dev/null)
BITCOIN_RPC_USER="archipelago"
if [ -z "$BITCOIN_RPC_PASS" ]; then
echo " WARNING: Could not read Bitcoin RPC password from server"
return 1
fi
# Read per-installation database passwords from server secrets
DB_PASSWORDS=$(ssh $SSH_OPTS "$TARGET_HOST" '
SECRETS_DIR="/var/lib/archipelago/secrets"
for svc in mempool btcpay immich penpot mysql-root; do
if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
openssl rand -base64 24 | sudo tee "$SECRETS_DIR/${svc}-db-password" > /dev/null
sudo chmod 600 "$SECRETS_DIR/${svc}-db-password"
fi
done
echo "MEMPOOL_DB_PASS=$(sudo cat "$SECRETS_DIR/mempool-db-password")"
echo "BTCPAY_DB_PASS=$(sudo cat "$SECRETS_DIR/btcpay-db-password")"
echo "IMMICH_DB_PASS=$(sudo cat "$SECRETS_DIR/immich-db-password")"
echo "PENPOT_DB_PASS=$(sudo cat "$SECRETS_DIR/penpot-db-password")"
echo "MYSQL_ROOT_PASS=$(sudo cat "$SECRETS_DIR/mysql-root-db-password")"
# Fedimint gateway password and hash
if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
FEDI_PASS=$(openssl rand -base64 16)
echo "$FEDI_PASS" | sudo tee "$SECRETS_DIR/fedimint-gateway-password" > /dev/null
sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-password"
if command -v htpasswd >/dev/null 2>&1; then
htpasswd -bnBC 10 "" "$FEDI_PASS" | tr -d ":\n" | sudo tee "$SECRETS_DIR/fedimint-gateway-hash" > /dev/null
sudo chmod 600 "$SECRETS_DIR/fedimint-gateway-hash"
fi
fi
if [ -f "$SECRETS_DIR/fedimint-gateway-hash" ]; then
echo "FEDI_HASH=$(sudo cat "$SECRETS_DIR/fedimint-gateway-hash")"
fi
' 2>/dev/null)
# Safe variable parsing — never eval untrusted SSH output
while IFS='=' read -r key value; do
# Skip empty lines
[ -z "$key" ] && continue
# Only allow expected variable names
case "$key" in
MEMPOOL_DB_PASS) MEMPOOL_DB_PASS="$value" ;;
BTCPAY_DB_PASS) BTCPAY_DB_PASS="$value" ;;
IMMICH_DB_PASS) IMMICH_DB_PASS="$value" ;;
PENPOT_DB_PASS) PENPOT_DB_PASS="$value" ;;
MYSQL_ROOT_PASS) MYSQL_ROOT_PASS="$value" ;;
FEDI_HASH) FEDI_HASH="$value" ;;
*) echo " WARNING: Ignoring unexpected variable from server: $key" ;;
esac
done <<< "$DB_PASSWORDS"
# Fallback if hash not available
if [ -z "${FEDI_HASH:-}" ]; then
FEDI_HASH='$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC'
fi
progress "Ensuring Bitcoin Knots"
ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
\$DOCKER network create archy-net 2>/dev/null || true
NET_OPT='--network archy-net'
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|archy-bitcoin-knots'; then
echo ' Creating Bitcoin Knots (mainnet, archipelago RPC)...'
sudo mkdir -p /var/lib/archipelago/bitcoin
# Demo mode: prune=550 saves ~194GB disk, but disables txindex (incompatible with electrumx)
if [ "$DEMO" = "true" ]; then
BTC_EXTRA_ARGS="-prune=550"
BTC_DBCACHE=512
else
BTC_EXTRA_ARGS="-txindex=1"
BTC_DBCACHE=4096
fi
\$DOCKER run -d --name bitcoin-knots --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 8332:8332 -p 8333:8333 \
-v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
${BITCOIN_KNOTS_IMAGE} \
-server=1 \$BTC_EXTRA_ARGS \
-rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
-dbcache=\$BTC_DBCACHE
echo ' Bitcoin Knots started (sync may take hours)'
else
\$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
fi
" 2>&1 | sed 's/^/ /' || true
# Fix Mempool: clean duplicates, ensure full stack - mysql, backend (8999), frontend (4080)
progress "Fixing Mempool stack"
ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
TARGET_IP='$TARGET_IP'
NET_OPT='--network archy-net'
# Clean any duplicate/old mempool containers (user may have two versions)
# EXCLUDE electrumx/mempool-electrs - indexing takes days, do not recreate on every deploy
for c in mempool mempool-api mempool-web archy-mempool-api archy-mempool-web; do
\$DOCKER stop \$c 2>/dev/null
\$DOCKER rm -f \$c 2>/dev/null
done
# Create mysql-mempool if missing
if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'mysql-mempool|archy-mempool-db'; then
echo ' Creating mysql-mempool...'
sudo mkdir -p /var/lib/archipelago/mysql-mempool
\$DOCKER run -d --name archy-mempool-db --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
-e MYSQL_DATABASE=mempool \
-e MYSQL_USER=mempool \
-e MYSQL_PASSWORD=$MEMPOOL_DB_PASS \
-e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
"$MARIADB_IMAGE"
sleep 3
fi
MYSQL_CNT=\$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
MYSQL_CNT=\${MYSQL_CNT:-archy-mempool-db}
# Ensure DB is on archy-net so mempool-api can resolve it
\$DOCKER network connect archy-net \$MYSQL_CNT 2>/dev/null || true
# Stop and remove old mempool-electrs if present (replaced by electrumx)
if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q mempool-electrs; then
echo ' Removing old mempool-electrs (replaced by ElectrumX)...'
\$DOCKER stop mempool-electrs 2>/dev/null
\$DOCKER rm -f mempool-electrs 2>/dev/null
fi
# Create electrumx ONLY if missing - do NOT recreate (indexing takes days)
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q electrumx; then
echo ' Starting existing electrumx (preserving index)...'
\$DOCKER start electrumx 2>/dev/null || true
else
echo ' Creating electrumx (indexer - may take days to sync, do not recreate)...'
sudo mkdir -p /var/lib/archipelago/electrumx
\$DOCKER run -d --name electrumx --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 50001:50001 \
-v /var/lib/archipelago/electrumx:/data \
-e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
-e COIN=Bitcoin \
-e DB_DIRECTORY=/data \
-e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
"$ELECTRUMX_IMAGE"
fi
fi
# Create/recreate mempool-api (backend on 8999) - required for mempool to work
for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mempool-api|archy-mempool-api'); do
echo ' Recreating mempool-api (backend)...'
\$DOCKER stop \"\$c\" 2>/dev/null
\$DOCKER rm -f \"\$c\" 2>/dev/null
done
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q mempool-api; then
echo ' Creating mempool-api (backend)...'
sudo mkdir -p /var/lib/archipelago/mempool
\$DOCKER run -d --name mempool-api --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 8999:8999 \
-v /var/lib/archipelago/mempool:/data \
-e MEMPOOL_BACKEND=electrum \
-e ELECTRUM_HOST=electrumx \
-e ELECTRUM_PORT=50001 \
-e ELECTRUM_TLS_ENABLED=false \
-e CORE_RPC_HOST=\$TARGET_IP \
-e CORE_RPC_PORT=8332 \
-e CORE_RPC_USERNAME=archipelago \
-e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
-e DATABASE_ENABLED=true \
-e DATABASE_HOST=\$MYSQL_CNT \
-e DATABASE_DATABASE=mempool \
-e DATABASE_USERNAME=mempool \
-e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
"$MEMPOOL_BACKEND_IMAGE"
fi
# Recreate mempool frontend - handle both 'mempool' and 'mempool-web' (frontend was on wrong port 8999)
for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^mempool\$|mempool-web|archy-mempool-web'); do
echo ' Recreating mempool frontend on 4080...'
\$DOCKER stop \"\$c\" 2>/dev/null
\$DOCKER rm -f \"\$c\" 2>/dev/null
break
done
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-mempool-web; then
echo ' Creating mempool frontend on 4080...'
\$DOCKER run -d --name archy-mempool-web --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 4080:8080 \
-e FRONTEND_HTTP_PORT=8080 \
-e BACKEND_MAINNET_HTTP_HOST=mempool-api \
"$MEMPOOL_WEB_IMAGE"
fi
" 2>&1 | sed 's/^/ /' || true
# Fix BTCPay Server: requires PostgreSQL + NBXplorer (BTCPay needs NBXplorer for block indexing)
progress "Fixing BTCPay stack"
TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
TARGET_IP='$TARGET_IP'
\$DOCKER network create archy-net 2>/dev/null || true
NET_OPT='--network archy-net'
# Ensure bitcoin-knots is on archy-net for NBXplorer/BTCPay to reach it
\$DOCKER network connect archy-net bitcoin-knots 2>/dev/null || true
# Create PostgreSQL for BTCPay if missing
if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
echo ' Creating archy-btcpay-db (PostgreSQL)...'
sudo mkdir -p /var/lib/archipelago/postgres-btcpay
\$DOCKER run -d --name archy-btcpay-db --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
-e POSTGRES_DB=btcpay \
-e POSTGRES_USER=btcpay \
-e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
"$BTCPAY_POSTGRES_IMAGE"
sleep 3
fi
# Create NBXplorer database in PostgreSQL (NBXplorer needs its own DB)
\$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \
\$DOCKER exec -e PGPASSWORD=$BTCPAY_DB_PASS archy-btcpay-db psql -U postgres -c \"CREATE DATABASE nbxplorer;\" 2>/dev/null || true
# Create NBXplorer (required by BTCPay - indexes blocks for payment tracking)
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q archy-nbxplorer; then
\$DOCKER start archy-nbxplorer 2>/dev/null || true
else
echo ' Creating archy-nbxplorer...'
sudo mkdir -p /var/lib/archipelago/nbxplorer
\$DOCKER run -d --name archy-nbxplorer --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 32838:32838 \
-v /var/lib/archipelago/nbxplorer:/data \
-e NBXPLORER_DATADIR=/data \
-e NBXPLORER_NETWORK=mainnet \
-e NBXPLORER_CHAINS=btc \
-e NBXPLORER_BIND=0.0.0.0:32838 \
-e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
-e NBXPLORER_BTCRPCUSER=$BITCOIN_RPC_USER \
-e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
"$NBXPLORER_IMAGE"
sleep 5
fi
fi
# Recreate btcpay-server with PostgreSQL, NBXplorer URL, and Bitcoin RPC
for c in btcpay-server archy-btcpay; do
if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx \"\$c\"; then
echo ' Recreating btcpay-server with NBXplorer...'
\$DOCKER stop \"\$c\" 2>/dev/null
\$DOCKER rm -f \"\$c\" 2>/dev/null
fi
done
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q btcpay-server; then
echo ' Creating btcpay-server on 23000...'
sudo mkdir -p /var/lib/archipelago/btcpay
\$DOCKER run -d --name btcpay-server --restart unless-stopped \$NET_OPT \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 23000:49392 \
-v /var/lib/archipelago/btcpay:/datadir \
-e ASPNETCORE_URLS=http://0.0.0.0:49392 \
-e BTCPAY_PROTOCOL=http \
-e BTCPAY_HOST=\$TARGET_IP:23000 \
-e BTCPAY_CHAINS=btc \
-e BTCPAY_BTCEXPLORERURL=http://archy-nbxplorer:32838 \
-e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
-e BTCPAY_BTCRPCUSER=archipelago \
-e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
-e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
"$BTCPAY_IMAGE"
fi
" 2>&1 | sed 's/^/ /' || true
# Ensure Immich stack (postgres + redis + server) - creates if missing
progress "Ensuring Immich stack"
ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
# Remove old single-container 'immich' if present (wrong port mapping, conflicts with immich_server)
if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
echo ' Removing old immich container (use immich_server)...'
\$DOCKER stop immich 2>/dev/null
\$DOCKER rm -f immich 2>/dev/null
\$DOCKER start immich_server 2>/dev/null || true
fi
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
echo ' Creating Immich stack...'
sudo mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
\$DOCKER network create immich-net 2>/dev/null || true
if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
\$DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
-v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
-e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
"$IMMICH_POSTGRES_IMAGE" 2>/dev/null || true
sleep 5
fi
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
\$DOCKER run -d --name immich_redis --restart unless-stopped --network immich-net \
"$VALKEY_IMAGE" 2>/dev/null || true
sleep 2
fi
if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
\$DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
-p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
-e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
-e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
-e UPLOAD_LOCATION=/usr/src/app/upload \
"$IMMICH_SERVER_IMAGE" 2>/dev/null || true
fi
echo ' Immich stack created (may take 1-2 min to become ready)'
else
echo ' Immich already running'
fi
" 2>&1 | sed 's/^/ /' || true
# Tor: global hidden services - each service gets its own .onion address
progress "Setting up Tor"
TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
TARGET_IP='$TARGET_IP'
sudo mkdir -p /var/lib/archipelago/tor
# Ensure services.json exists with default services
SERVICES_JSON=/var/lib/archipelago/tor/services.json
if [ ! -f "\$SERVICES_JSON" ]; then
sudo python3 -c '
import json
services = [
{"name": "archipelago", "local_port": 80, "enabled": True},
{"name": "bitcoin", "local_port": 8333, "enabled": True},
{"name": "electrumx", "local_port": 50001, "enabled": True},
{"name": "lnd", "local_port": 9735, "enabled": True},
{"name": "btcpay", "local_port": 23000, "enabled": True},
{"name": "mempool", "local_port": 4080, "enabled": True},
{"name": "fedimint", "local_port": 8175, "enabled": True}
]
with open("/var/lib/archipelago/tor/services.json", "w") as f:
json.dump({"services": services}, f, indent=2)
print("services.json created")
'
fi
# Generate torrc from services.json — use /var/lib/tor/ for hidden services
sudo python3 -c '
import json, os
# Protocol services get direct port mapping; web apps map port 80 to their local port
PROTOCOL_SERVICES = {"bitcoin", "bitcoin-knots", "electrs", "electrumx", "lnd"}
lines = ["# Auto-generated by Archipelago deploy", "SocksPort 0.0.0.0:9050", "# ControlPort disabled", ""]
# Try reading services config (check both paths for compatibility)
cfg = None
for path in ["/var/lib/archipelago/tor-config/services.json", "/var/lib/archipelago/tor/services.json"]:
try:
with open(path) as f:
cfg = json.load(f)
break
except Exception:
pass
if cfg:
for svc in cfg.get("services", []):
if not svc.get("enabled", True):
continue
n = svc["name"]
p = svc["local_port"]
lines.append("HiddenServiceDir /var/lib/tor/hidden_service_%s" % n)
if n in PROTOCOL_SERVICES:
# Protocol: direct port mapping
lines.append("HiddenServicePort %d 127.0.0.1:%d" % (p, p))
if n == "lnd":
lines.append("HiddenServicePort 9735 127.0.0.1:9735")
lines.append("HiddenServicePort 10009 127.0.0.1:10009")
else:
# Web app: map port 80 on .onion to local app port (access via app.onion without port)
lines.append("HiddenServicePort 80 127.0.0.1:%d" % p)
lines.append("")
else:
# Fallback: default services
for n, mappings in [("archipelago",[(80,80)]),("bitcoin",[(8333,8333)]),("electrs",[(50001,50001)]),("lnd",[(8080,8080),(9735,9735),(10009,10009)]),("btcpay",[(80,23000)]),("mempool",[(80,4080)]),("fedimint",[(80,8175)])]:
lines.append("HiddenServiceDir /var/lib/tor/hidden_service_%s" % n)
for remote_p, local_p in mappings:
lines.append("HiddenServicePort %d 127.0.0.1:%d" % (remote_p, local_p))
lines.append("")
with open("/etc/tor/torrc", "w") as f:
f.write("\n".join(lines) + "\n")
enabled = sum(1 for s in (cfg or {}).get("services", []) if s.get("enabled", True))
print("torrc generated with %d services" % (enabled or 7))
'
# Remove any old Tor container (system Tor is preferred)
for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'archy-tor|^tor\$'); do
\$DOCKER stop \"\$c\" 2>/dev/null
\$DOCKER rm -f \"\$c\" 2>/dev/null
done
# Use system Tor (preferred — no AppArmor issues with default paths)
if command -v tor >/dev/null 2>&1; then
sudo systemctl enable tor 2>/dev/null
sudo systemctl enable tor@default 2>/dev/null
sudo systemctl restart tor 2>/dev/null
sudo systemctl restart tor@default 2>/dev/null
echo ' Using system Tor daemon'
else
echo ' Installing system Tor...'
sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null || true
if command -v tor >/dev/null 2>&1; then
sudo systemctl enable tor 2>/dev/null
sudo systemctl enable tor@default 2>/dev/null
sudo systemctl restart tor 2>/dev/null
sudo systemctl restart tor@default 2>/dev/null
echo ' System Tor installed and started'
else
echo ' WARNING: Could not install Tor'
fi
fi
" 2>&1 | sed 's/^/ /' || true
# Tor diagnostic: check if hostname files exist (may take 30-60s after Tor starts)
echo " Checking Tor hostname files..."
ssh $SSH_OPTS "$TARGET_HOST" "
# Check all hidden_service_* dirs for hostname files (check both paths)
for dir in /var/lib/tor/hidden_service_*/ /var/lib/archipelago/tor/hidden_service_*/; do
[ -d \"\$dir\" ] || continue
svc=\$(basename \"\$dir\" | sed 's/hidden_service_//')
f=\"\${dir}hostname\"
if [ -f \"\$f\" ]; then
echo \" ✓ \$svc: \$(cat \$f)\"
else
echo \" ✗ \$svc: hostname not yet generated (Tor may need 30-60s)\"
fi
done
" 2>&1 | sed 's/^/ /' || true
# Recreate Fedimint with FM_API_URL for Guardian UI (fixes "Api URL must be configured")
section_start
progress "Fixing Fedimint"
TARGET_IP="$(echo "$TARGET_HOST" | cut -d@ -f2)"
TIMEOUT_CMD=""
command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90"
command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90"
($TIMEOUT_CMD ssh $SSH_OPTS "$TARGET_HOST" "
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
for c in \$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E '^fedimint\$'); do
echo ' Recreating fedimint with FM_API_URL...'
\$DOCKER stop \"\$c\" 2>/dev/null
\$DOCKER rm -f \"\$c\" 2>/dev/null
\$DOCKER run -d --name fedimint --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 8173:8173 -p 8174:8174 -p 8175:8175 \
-v /var/lib/archipelago/fedimint:/data \
-e FM_DATA_DIR=/data \
-e FM_BITCOIND_USERNAME=archipelago \
-e FM_BITCOIND_PASSWORD=$BITCOIN_RPC_PASS \
-e FM_BITCOIN_NETWORK=bitcoin \
-e FM_BIND_P2P=0.0.0.0:8173 \
-e FM_BIND_API=0.0.0.0:8174 \
-e FM_BIND_UI=0.0.0.0:8175 \
-e FM_P2P_URL=fedimint://$TARGET_IP:8173 \
-e FM_API_URL=ws://$TARGET_IP:8174 \
-e FM_BITCOIND_URL=http://$TARGET_IP:8332 \
"$FEDIMINT_IMAGE"
break
done
# Ensure Fedimint Gateway companion container
# Auto-detect LND: if running with credentials, use lnd mode; otherwise use ldk (built-in)
\$DOCKER rm -f fedimint-gateway 2>/dev/null || true
echo ' Creating fedimint-gateway...'
sudo mkdir -p /var/lib/archipelago/fedimint-gateway
LND_CERT=/var/lib/archipelago/lnd/tls.cert
LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
GW_COMMON=\"-p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data "$FEDIMINT_GATEWAY_IMAGE" gatewayd --data-dir /data --listen 0.0.0.0:8176 --bcrypt-password-hash '$FEDI_HASH' --network bitcoin --bitcoind-url http://$TARGET_IP:8332 --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS\"
if \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then
echo ' LND detected — using lnd mode'
\$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 8176:8176 \
-v /var/lib/archipelago/fedimint-gateway:/data \
-v /var/lib/archipelago/lnd/tls.cert:/lnd/tls.cert:ro \
-v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \
"$FEDIMINT_GATEWAY_IMAGE" \
gatewayd --data-dir /data --listen 0.0.0.0:8176 \
--bcrypt-password-hash '$FEDI_HASH' \
--network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
--bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
lnd --lnd-rpc-host $TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon
else
echo ' No LND found — using ldk (built-in Lightning)'
\$DOCKER run -d --name fedimint-gateway --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 8176:8176 -p 9737:9737 \
-v /var/lib/archipelago/fedimint-gateway:/data \
"$FEDIMINT_GATEWAY_IMAGE" \
gatewayd --data-dir /data --listen 0.0.0.0:8176 \
--bcrypt-password-hash '$FEDI_HASH' \
--network bitcoin --bitcoind-url http://$TARGET_IP:8332 \
--bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway
fi
" 2>&1 | sed 's/^/ /') || echo " (Fedimint fix timed out or skipped - run manually if needed)"
section_end
# LND: Lightning Network Daemon (requires bitcoin-knots on archy-net)
progress "Ensuring LND"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx lnd; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx lnd; then
$DOCKER start lnd 2>/dev/null || true
echo " LND started (existing)"
else
echo " Creating LND..."
sudo mkdir -p /var/lib/archipelago/lnd
if [ ! -f /var/lib/archipelago/lnd/lnd.conf ]; then
cat > /tmp/lnd.conf <<LNDCONF
[Application Options]
listen=0.0.0.0:9735
rpclisten=0.0.0.0:10009
restlisten=0.0.0.0:8080
debuglevel=info
noseedbackup=true
tor.active=false
[Bitcoin]
bitcoin.mainnet=true
bitcoin.node=bitcoind
[Bitcoind]
bitcoind.rpchost=bitcoin-knots:8332
bitcoind.rpcuser=archipelago
bitcoind.rpcpass=$BITCOIN_RPC_PASS
bitcoind.rpcpolling=true
bitcoind.estimatemode=ECONOMICAL
[autopilot]
autopilot.active=false
LNDCONF
sudo cp /tmp/lnd.conf /var/lib/archipelago/lnd/lnd.conf
else
# Always ensure LND config has correct RPC credentials from secrets
LND_CONF=/var/lib/archipelago/lnd/lnd.conf
CURRENT_PASS=$(sudo grep "bitcoind.rpcpass=" "$LND_CONF" 2>/dev/null | cut -d= -f2)
NEEDS_FIX=0
grep -q "rpccookie" "$LND_CONF" 2>/dev/null && NEEDS_FIX=1
grep -q "rpchost=127.0.0.1" "$LND_CONF" 2>/dev/null && NEEDS_FIX=1
RPC_PASS_EXPECTED=$(sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null)
[ "$CURRENT_PASS" != "$RPC_PASS_EXPECTED" ] && NEEDS_FIX=1
if [ "$NEEDS_FIX" = "1" ]; then
echo " Syncing LND config with current RPC credentials..."
sudo sed -i "/bitcoind.rpccookie/d" "$LND_CONF"
sudo sed -i "s|bitcoind.rpchost=127.0.0.1:8332|bitcoind.rpchost=bitcoin-knots:8332|" "$LND_CONF"
sudo sed -i "s|bitcoind.rpcpass=.*|bitcoind.rpcpass=$RPC_PASS_EXPECTED|" "$LND_CONF"
if ! sudo grep -q "bitcoind.rpcuser=" "$LND_CONF" 2>/dev/null; then
sudo sed -i "/bitcoind.rpchost=/a bitcoind.rpcuser=archipelago" "$LND_CONF"
fi
if ! sudo grep -q "bitcoind.rpcpass=" "$LND_CONF" 2>/dev/null; then
sudo sed -i "/bitcoind.rpcuser=/a bitcoind.rpcpass=$RPC_PASS_EXPECTED" "$LND_CONF"
fi
sudo chown 100000:100000 "$LND_CONF"
RESTART_LND=1
echo " LND config updated"
fi
fi
$DOCKER run -d --name lnd --restart unless-stopped --network archy-net \
--cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 9735:9735 -p 10009:10009 -p 18080:8080 \
-v /var/lib/archipelago/lnd:/root/.lnd \
"$LND_IMAGE"
echo " LND created"
fi
else
echo " LND already running"
fi
' 2>&1 | sed 's/^/ /' || true
# Home Assistant
progress "Ensuring Home Assistant"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx homeassistant; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx homeassistant; then
$DOCKER start homeassistant 2>/dev/null || true
else
echo " Creating Home Assistant..."
sudo mkdir -p /var/lib/archipelago/home-assistant
$DOCKER run -d --name homeassistant --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
--security-opt no-new-privileges:true \
-p 8123:8123 -v /var/lib/archipelago/home-assistant:/config \
-e TZ=UTC \
"$HOMEASSISTANT_IMAGE"
fi
else
echo " Home Assistant already running"
fi
' 2>&1 | sed 's/^/ /' || true
# Grafana
progress "Ensuring Grafana"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx grafana; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx grafana; then
$DOCKER start grafana 2>/dev/null || true
else
echo " Creating Grafana..."
sudo mkdir -p /var/lib/archipelago/grafana
sudo chown 472:472 /var/lib/archipelago/grafana 2>/dev/null || true
$DOCKER run -d --name grafana --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
--security-opt no-new-privileges:true \
-p 3000:3000 -v /var/lib/archipelago/grafana:/var/lib/grafana \
-e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \
"$GRAFANA_IMAGE"
fi
else
echo " Grafana already running"
fi
' 2>&1 | sed 's/^/ /' || true
# Jellyfin
progress "Ensuring Jellyfin"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx jellyfin; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx jellyfin; then
$DOCKER start jellyfin 2>/dev/null || true
else
echo " Creating Jellyfin..."
sudo mkdir -p /var/lib/archipelago/jellyfin/config /var/lib/archipelago/jellyfin/cache
$DOCKER run -d --name jellyfin --restart unless-stopped \
--cap-drop ALL --security-opt no-new-privileges:true \
-p 8096:8096 \
-v /var/lib/archipelago/jellyfin/config:/config \
-v /var/lib/archipelago/jellyfin/cache:/cache \
"$JELLYFIN_IMAGE"
fi
else
echo " Jellyfin already running"
fi
' 2>&1 | sed 's/^/ /' || true
# Vaultwarden
progress "Ensuring Vaultwarden"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx vaultwarden; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx vaultwarden; then
$DOCKER start vaultwarden 2>/dev/null || true
else
echo " Creating Vaultwarden..."
sudo mkdir -p /var/lib/archipelago/vaultwarden
$DOCKER run -d --name vaultwarden --restart unless-stopped \
--cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
--security-opt no-new-privileges:true \
-p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
"$VAULTWARDEN_IMAGE"
fi
else
echo " Vaultwarden already running"
fi
' 2>&1 | sed 's/^/ /' || true
# SearXNG (privacy search engine — used by AIUI web search)
progress "Ensuring SearXNG"
ssh $SSH_OPTS "$TARGET_HOST" '
DOCKER=podman
command -v podman >/dev/null 2>&1 || DOCKER=docker
if ! $DOCKER ps --format "{{.Names}}" 2>/dev/null | grep -qx searxng; then
if $DOCKER ps -a --format "{{.Names}}" 2>/dev/null | grep -qx searxng; then
$DOCKER start searxng 2>/dev/null || true
else
echo " Creating SearXNG..."
$DOCKER run -d --name searxng --restart unless-stopped \
--cap-drop ALL --security-opt no-new-privileges:true \
-p 8888:8080 \
${SEARXNG_IMAGE}
fi
else
echo " SearXNG already running"
fi
' 2>&1 | sed 's/^/ /' || true
# Ollama — optional, install from marketplace if needed
# (removed from auto-deploy: large image, not needed for core functionality)
fi # end legacy app installation (dead code, kept for git history)
# Ensure UFW allows forwarded traffic (required for podman container port access from LAN)
progress "Fixing UFW forward policy"
ssh $SSH_OPTS "$TARGET_HOST" '
if grep -q "DEFAULT_FORWARD_POLICY=\"DROP\"" /etc/default/ufw 2>/dev/null; then
sudo sed -i "s/DEFAULT_FORWARD_POLICY=\"DROP\"/DEFAULT_FORWARD_POLICY=\"ACCEPT\"/" /etc/default/ufw
sudo ufw reload 2>/dev/null
echo " Fixed UFW forward policy (was DROP, now ACCEPT)"
fi
' 2>&1 | sed 's/^/ /' || true
# Fix IndeedHub for iframe + NIP-07: remove X-Frame-Options, inject nostr-provider.js,
# resolve container IPs for nginx proxy (DNS resolver 127.0.0.11 is unreliable in podman)
progress "Fixing IndeedHub for NIP-07"
ssh $SSH_OPTS "$TARGET_HOST" '
podman_quick() { timeout 20 podman "$@"; }
podman_exec_quick() { timeout 20 podman exec "$@"; }
if podman_quick ps --format "{{.Names}}" 2>/dev/null | grep -q "^indeedhub$"; then
CHANGED=false
NETWORK=$(podman_quick inspect indeedhub --format "{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}}{{end}}" 2>/dev/null || true)
# Remove X-Frame-Options so iframe works
if podman_exec_quick indeedhub grep -q "X-Frame-Options" /etc/nginx/conf.d/default.conf 2>/dev/null; then
podman_exec_quick indeedhub sed -i "/X-Frame-Options/d" /etc/nginx/conf.d/default.conf || true
CHANGED=true
echo " Removed X-Frame-Options from IndeedHub"
fi
# Fix Host header for NIP-98 auth — $host strips port, $http_host preserves it
podman_exec_quick indeedhub sh -c "sed -i '"'"'s/proxy_set_header Host \$host;/proxy_set_header Host \$http_host;/g'"'"' /etc/nginx/conf.d/default.conf" 2>/dev/null && CHANGED=true && echo " Fixed Host header for NIP-98 auth" || true
# Inject nostr-provider.js for NIP-07 signing
if ! podman_exec_quick indeedhub test -f /usr/share/nginx/html/nostr-provider.js 2>/dev/null; then
podman_quick cp /opt/archipelago/web-ui/nostr-provider.js indeedhub:/usr/share/nginx/html/nostr-provider.js 2>/dev/null || true
echo " Copied nostr-provider.js into IndeedHub"
fi
# Add nostr-provider.js + sub_filter to nginx config
if ! podman_exec_quick indeedhub grep -q "nostr-provider" /etc/nginx/conf.d/default.conf 2>/dev/null; then
podman_exec_quick indeedhub cat /etc/nginx/conf.d/default.conf > /tmp/ih-nginx.conf 2>/dev/null || true
# Add nostr-provider location block before sw.js block
sed -i "/location = \/sw.js {/i\\ location = /nostr-provider.js {\n add_header Cache-Control \"no-cache, no-store, must-revalidate\";\n expires off;\n }\n" /tmp/ih-nginx.conf
# Add sub_filter for nostr-provider injection
sed -i "/try_files.*index.html/a\\ sub_filter_once on;\n sub_filter '"'"'</head>'"'"' '"'"'<script src=\"/nostr-provider.js\"></script></head>'"'"';" /tmp/ih-nginx.conf
podman_quick cp /tmp/ih-nginx.conf indeedhub:/etc/nginx/conf.d/default.conf 2>/dev/null || true
rm -f /tmp/ih-nginx.conf
CHANGED=true
echo " Injected nostr-provider.js into IndeedHub nginx"
fi
# Replace DNS-based upstream resolution with hardcoded container IPs
# (podman DNS resolver 127.0.0.11 is unreliable, causing 502 errors)
API_IP=$(podman_quick inspect indeedhub-build_api_1 --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)
MINIO_IP=$(podman_quick inspect indeedhub-minio --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)
RELAY_IP=$(podman_quick inspect indeedhub-relay --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" 2>/dev/null || true)
if [ -n "$API_IP" ] && [ -n "$MINIO_IP" ] && [ -n "$RELAY_IP" ]; then
podman_exec_quick indeedhub cat /etc/nginx/conf.d/default.conf > /tmp/ih-nginx.conf 2>/dev/null || true
# Remove DNS resolver lines and replace upstream variables with hardcoded IPs
sed -i "s|resolver 127.0.0.11 valid=30s ipv6=off;||g" /tmp/ih-nginx.conf
sed -i "s|set \$api_upstream http://api:4000;|set \$api_upstream http://$API_IP:4000;|g" /tmp/ih-nginx.conf
sed -i "s|set \$minio_upstream http://minio:9000;|set \$minio_upstream http://$MINIO_IP:9000;|g" /tmp/ih-nginx.conf
sed -i "s|set \$relay_upstream http://relay:8080;|set \$relay_upstream http://$RELAY_IP:8080;|g" /tmp/ih-nginx.conf
sed -i "s|proxy_set_header Host \$host;|proxy_set_header Host \$http_host;|g" /tmp/ih-nginx.conf
podman_quick cp /tmp/ih-nginx.conf indeedhub:/etc/nginx/conf.d/default.conf 2>/dev/null || true
rm -f /tmp/ih-nginx.conf
CHANGED=true
echo " Patched IndeedHub nginx with container IPs (API=$API_IP MINIO=$MINIO_IP RELAY=$RELAY_IP)"
fi
if [ "$CHANGED" = true ]; then
podman_exec_quick indeedhub nginx -s reload 2>/dev/null || true
fi
fi
' 2>&1 | sed 's/^/ /' || true
# Run container doctor — auto-fix common container health issues
progress "Running container doctor"
"$SCRIPT_DIR/container-doctor.sh" "$TARGET_HOST" 2>&1 | sed 's/^/ /' || true
# Post-deploy health check — wait up to 60s for server to come healthy
echo ""
progress "Post-deploy health check"
HEALTH_OK=false
for i in $(seq 1 12); do
POST_HEALTH=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 "http://$TARGET_IP_ONLY/health" 2>/dev/null || { echo "WARNING: Post-deploy health check failed for $TARGET_IP_ONLY" >&2; echo "000"; })
if [ "$POST_HEALTH" = "200" ]; then
echo " Health: OK (200) after $((i * 5))s"
HEALTH_OK=true
break
fi
echo " Health: $POST_HEALTH (waiting... ${i}/12)"
sleep 5
done
if [ "$HEALTH_OK" = false ]; then
echo " ⚠️ Server did not become healthy within 60s (last: $POST_HEALTH)"
echo " Attempting automatic rollback..."
ssh $SSH_OPTS "$TARGET_HOST" '
if [ -f /opt/archipelago/rollback/archipelago.bak ]; then
sudo systemctl stop archipelago 2>/dev/null
sudo cp /opt/archipelago/rollback/archipelago.bak /usr/local/bin/archipelago
if [ -f /opt/archipelago/rollback/web-ui.tar ]; then
sudo find /opt/archipelago/web-ui -mindepth 1 -maxdepth 1 ! -name "aiui" ! -name "claude-login.html" -exec rm -rf {} +
sudo tar xf /opt/archipelago/rollback/web-ui.tar -C /opt/archipelago/web-ui
fi
sudo systemctl start archipelago
echo "ROLLBACK_DONE"
else
echo "NO_ROLLBACK_AVAILABLE"
fi
' 2>/dev/null | while IFS= read -r line; do
if [ "$line" = "ROLLBACK_DONE" ]; then
echo " 🔄 Rollback complete — previous version restored"
elif [ "$line" = "NO_ROLLBACK_AVAILABLE" ]; then
echo " ⚠️ No rollback backup available"
fi
done
echo " Check: sudo journalctl -u archipelago -n 50"
fi
DEPLOY_END=$(date +%s)
DEPLOY_ELAPSED=$((DEPLOY_END - DEPLOY_START))
# Append to local deploy history log (gitignored)
DEPLOY_LOG="$PROJECT_DIR/scripts/deploy-history.log"
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) | $DEPLOY_BRANCH@$DEPLOY_COMMIT | dirty=$DEPLOY_DIRTY | target=$TARGET_HOST | ${DEPLOY_ELAPSED}s" >> "$DEPLOY_LOG"
# Auto-tag successful deploys (only on clean commits, skip if already tagged)
if [ "$DEPLOY_DIRTY" = "0" ]; then
EXISTING_TAG=$(git tag --points-at "$DEPLOY_COMMIT" 2>/dev/null | grep "^v" | head -1)
if [ -z "$EXISTING_TAG" ]; then
LAST_ALPHA=$(git tag -l 'v1.2.0-alpha.*' | sort -V | tail -1 | sed 's/.*alpha\.//')
NEXT_ALPHA=$(( ${LAST_ALPHA:-0} + 1 ))
DEPLOY_TAG="v1.2.0-alpha.${NEXT_ALPHA}"
git tag -a "$DEPLOY_TAG" "$DEPLOY_COMMIT" -m "Auto-tagged by deploy to $TARGET_IP_ONLY" 2>/dev/null && \
echo " Tagged: $DEPLOY_TAG" || true
fi
fi
echo ""
echo "$(timestamp) ✅ Deployed to live system! (${DEPLOY_ELAPSED}s total)"
echo " Commit: $DEPLOY_BRANCH @ $DEPLOY_COMMIT (dirty=$DEPLOY_DIRTY)"
echo " Backend: $(ssh $SSH_OPTS "$TARGET_HOST" 'sudo systemctl is-active archipelago')"
echo " Web UI: http://$TARGET_IP_ONLY"
echo " PWA install: https://$TARGET_IP_ONLY (use HTTPS, accept cert once, then Install app)"
else
echo ""
echo "✅ Build complete!"
echo ""
echo "To test frontend dev server:"
echo " ssh $TARGET_HOST"
echo " cd ~/archy/neode-ui && npm run dev -- --host 0.0.0.0"
echo " Then open: http://$(echo "$TARGET_HOST" | cut -d@ -f2):5173"
echo ""
echo "To deploy to live system:"
echo " ./scripts/deploy-to-target.sh --live"
fi
Reference in New Issue View Git Blame Copy Permalink