Enhance README and RPC for package management

- Added instructions to README.md for building an ISO from source and flashing it to USB. - Introduced a new RPC method for package installation, including security checks and container management. - Updated Docker and Podman integration in build scripts to support both container runtimes. - Enhanced Nginx configuration for improved timeout settings and WebSocket support. - Added new app metadata for additional applications in the Docker package scanner.
2026-02-01 18:46:35 +00:00
parent 22024bde84
commit 0f40cb88b5
59 changed files with 3473 additions and 360 deletions
@@ -0,0 +1,410 @@
+#!/bin/bash
+#
+# Build Archipelago Auto-Installer ISO from LIVE SERVER STATE
+#
+# This captures the CURRENT STATE of the development server and packages
+# it into an auto-installer ISO - exactly like Start9/Umbrel.
+#
+# Usage: ./build-from-live-server.sh [dev-server-ip]
+#
+
+set -e
+
+DEV_SERVER="${1:-archipelago@192.168.1.228}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORK_DIR="$SCRIPT_DIR/build/live-snapshot"
+OUTPUT_DIR="$SCRIPT_DIR/results"
+
+echo "╔════════════════════════════════════════════════════════════════╗"
+echo "║  Building Archipelago ISO from LIVE SERVER                     ║"
+echo "╚════════════════════════════════════════════════════════════════╝"
+echo ""
+echo "📡 Development Server: $DEV_SERVER"
+echo ""
+
+# Check for required tools
+CONTAINER_CMD=""
+if command -v docker >/dev/null 2>&1; then
+    CONTAINER_CMD="docker"
+elif command -v podman >/dev/null 2>&1; then
+    CONTAINER_CMD="podman"
+else
+    echo "❌ Missing docker or podman"
+    exit 1
+fi
+
+if ! command -v xorriso >/dev/null 2>&1; then
+    echo "❌ Missing xorriso"
+    exit 1
+fi
+
+if ! command -v ssh >/dev/null 2>&1; then
+    echo "❌ Missing ssh"
+    exit 1
+fi
+
+echo "Using container runtime: $CONTAINER_CMD"
+echo ""
+
+mkdir -p "$WORK_DIR"
+mkdir -p "$OUTPUT_DIR"
+
+# =============================================================================
+# STEP 1: Capture LIVE SERVER state
+# =============================================================================
+echo "📦 Step 1: Capturing live server state..."
+echo ""
+
+SNAPSHOT_DIR="$WORK_DIR/live-server-snapshot"
+rm -rf "$SNAPSHOT_DIR"
+mkdir -p "$SNAPSHOT_DIR"
+
+# Create directory structure
+mkdir -p "$SNAPSHOT_DIR/bin"
+mkdir -p "$SNAPSHOT_DIR/web-ui"
+mkdir -p "$SNAPSHOT_DIR/configs"
+mkdir -p "$SNAPSHOT_DIR/apps"
+mkdir -p "$SNAPSHOT_DIR/scripts"
+
+# Capture backend binary
+echo "   Capturing backend binary..."
+scp "$DEV_SERVER:/usr/local/bin/archipelago" "$SNAPSHOT_DIR/bin/" 2>/dev/null || {
+    echo "   ⚠️  Backend binary not found on server"
+    echo "   Using local build if available..."
+    if [ -f "$SCRIPT_DIR/../core/target/release/archipelago" ]; then
+        # Build on Linux if we're on macOS
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            echo "   Building backend for Linux..."
+            cd "$SCRIPT_DIR/../core"
+            $CONTAINER_CMD build --platform linux/amd64 -t archipelago-backend -f - . <<'DOCKERFILE'
+FROM rust:1.93-bookworm as builder
+WORKDIR /build
+COPY . .
+RUN cargo build --release --bin archipelago
+DOCKERFILE
+            BACKEND_CONTAINER=$($CONTAINER_CMD create --platform linux/amd64 archipelago-backend)
+            $CONTAINER_CMD cp "$BACKEND_CONTAINER:/build/target/release/archipelago" "$SNAPSHOT_DIR/bin/"
+            $CONTAINER_CMD rm "$BACKEND_CONTAINER"
+            cd "$SCRIPT_DIR"
+        else
+            cp "$SCRIPT_DIR/../core/target/release/archipelago" "$SNAPSHOT_DIR/bin/"
+        fi
+    fi
+}
+
+if [ -f "$SNAPSHOT_DIR/bin/archipelago" ]; then
+    chmod +x "$SNAPSHOT_DIR/bin/archipelago"
+    echo "   ✅ Backend: $(du -h "$SNAPSHOT_DIR/bin/archipelago" | cut -f1)"
+else
+    echo "   ❌ No backend binary available"
+    exit 1
+fi
+
+# Capture frontend (Web UI)
+echo "   Capturing frontend (Web UI)..."
+rsync -az --delete "$DEV_SERVER:/opt/archipelago/web-ui/" "$SNAPSHOT_DIR/web-ui/" 2>/dev/null || {
+    echo "   ⚠️  Web UI not found on server"
+    echo "   Using local build..."
+    if [ -d "$SCRIPT_DIR/../web/dist/neode-ui" ]; then
+        cp -r "$SCRIPT_DIR/../web/dist/neode-ui/"* "$SNAPSHOT_DIR/web-ui/"
+    elif [ -d "$SCRIPT_DIR/../neode-ui/dist" ]; then
+        cp -r "$SCRIPT_DIR/../neode-ui/dist/"* "$SNAPSHOT_DIR/web-ui/"
+    else
+        echo "   Building frontend..."
+        cd "$SCRIPT_DIR/../neode-ui"
+        npm run build
+        cp -r "$SCRIPT_DIR/../web/dist/neode-ui/"* "$SNAPSHOT_DIR/web-ui/"
+        cd "$SCRIPT_DIR"
+    fi
+}
+
+if [ -d "$SNAPSHOT_DIR/web-ui" ] && [ "$(ls -A "$SNAPSHOT_DIR/web-ui")" ]; then
+    echo "   ✅ Web UI: $(du -sh "$SNAPSHOT_DIR/web-ui" | cut -f1)"
+else
+    echo "   ❌ No web UI available"
+    exit 1
+fi
+
+# Capture Nginx config
+echo "   Capturing Nginx config..."
+scp "$DEV_SERVER:/etc/nginx/sites-available/default" "$SNAPSHOT_DIR/configs/nginx-default.conf" 2>/dev/null || \
+    echo "   ⚠️  Using default Nginx config"
+
+# Capture systemd service
+echo "   Capturing systemd service..."
+scp "$DEV_SERVER:/etc/systemd/system/archipelago.service" "$SNAPSHOT_DIR/configs/archipelago.service" 2>/dev/null || {
+    echo "   Creating default service..."
+    cat > "$SNAPSHOT_DIR/configs/archipelago.service" <<'SERVICE'
+[Unit]
+Description=Archipelago Backend
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=archipelago
+Environment="ARCHIPELAGO_BIND=0.0.0.0:5678"
+Environment="ARCHIPELAGO_DEV_MODE=true"
+ExecStart=/usr/local/bin/archipelago
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+SERVICE
+}
+
+# Capture app manifests
+echo "   Capturing app manifests..."
+if [ -d "$SCRIPT_DIR/../apps" ]; then
+    cp -r "$SCRIPT_DIR/../apps/"* "$SNAPSHOT_DIR/apps/" 2>/dev/null || true
+fi
+
+echo "   ✅ Live server state captured"
+echo ""
+
+# =============================================================================
+# STEP 2: Create base rootfs with captured state
+# =============================================================================
+echo "📦 Step 2: Creating base system with live server state..."
+echo ""
+
+ROOTFS_TAR="$WORK_DIR/archipelago-rootfs-live.tar"
+
+# Build rootfs with Docker/Podman
+cat > "$WORK_DIR/Dockerfile.rootfs" <<'DOCKERFILE'
+FROM debian:bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install all required packages
+RUN apt-get update && apt-get install -y \
+    linux-image-amd64 \
+    grub-efi-amd64 \
+    grub-efi-amd64-signed \
+    shim-signed \
+    systemd \
+    systemd-sysv \
+    dbus \
+    sudo \
+    network-manager \
+    openssh-server \
+    nginx \
+    podman \
+    curl \
+    wget \
+    htop \
+    vim-tiny \
+    ca-certificates \
+    locales \
+    console-setup \
+    keyboard-configuration \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Configure locale
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
+
+# Create archipelago user
+RUN useradd -m -s /bin/bash -G sudo archipelago && \
+    echo "archipelago:archipelago" | chpasswd && \
+    echo "archipelago ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/archipelago
+
+# Set hostname
+RUN echo "archipelago" > /etc/hostname
+
+# Configure SSH
+RUN mkdir -p /etc/ssh && \
+    sed -i 's/#PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config || true && \
+    sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config || true
+
+# Create directories
+RUN mkdir -p /var/lib/archipelago/{data,config,containers} && \
+    mkdir -p /etc/archipelago && \
+    mkdir -p /opt/archipelago/{bin,scripts,web-ui} && \
+    chown -R archipelago:archipelago /var/lib/archipelago /opt/archipelago
+
+# Clean up
+RUN apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+DOCKERFILE
+
+echo "   Building base rootfs..."
+$CONTAINER_CMD build --platform linux/amd64 -t archipelago-rootfs-live -f "$WORK_DIR/Dockerfile.rootfs" "$WORK_DIR"
+
+echo "   Exporting filesystem..."
+$CONTAINER_CMD create --platform linux/amd64 --name archipelago-rootfs-live-tmp archipelago-rootfs-live
+$CONTAINER_CMD export archipelago-rootfs-live-tmp > "$ROOTFS_TAR"
+$CONTAINER_CMD rm archipelago-rootfs-live-tmp
+
+echo "   ✅ Base rootfs created: $(du -h "$ROOTFS_TAR" | cut -f1)"
+echo ""
+
+# =============================================================================
+# STEP 3: Inject live server files into rootfs
+# =============================================================================
+echo "📦 Step 3: Injecting live server files..."
+echo ""
+
+ROOTFS_DIR="$WORK_DIR/rootfs-extract"
+rm -rf "$ROOTFS_DIR"
+mkdir -p "$ROOTFS_DIR"
+
+echo "   Extracting rootfs..."
+tar -xf "$ROOTFS_TAR" -C "$ROOTFS_DIR"
+
+echo "   Copying backend binary..."
+cp "$SNAPSHOT_DIR/bin/archipelago" "$ROOTFS_DIR/usr/local/bin/"
+chmod +x "$ROOTFS_DIR/usr/local/bin/archipelago"
+
+echo "   Copying web UI..."
+rm -rf "$ROOTFS_DIR/opt/archipelago/web-ui"
+mkdir -p "$ROOTFS_DIR/opt/archipelago/web-ui"
+cp -r "$SNAPSHOT_DIR/web-ui/"* "$ROOTFS_DIR/opt/archipelago/web-ui/"
+
+echo "   Configuring Nginx..."
+if [ -f "$SNAPSHOT_DIR/configs/nginx-default.conf" ]; then
+    cp "$SNAPSHOT_DIR/configs/nginx-default.conf" "$ROOTFS_DIR/etc/nginx/sites-available/archipelago"
+else
+    cat > "$ROOTFS_DIR/etc/nginx/sites-available/archipelago" <<'NGINXCONF'
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    root /opt/archipelago/web-ui;
+    index index.html;
+
+    server_name _;
+
+    # Proxy API requests to backend
+    location /rpc/ {
+        proxy_pass http://localhost:5678/rpc/;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location /ws/ {
+        proxy_pass http://localhost:5678/ws/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+    }
+
+    location /health {
+        proxy_pass http://localhost:5678/health;
+    }
+
+    # Serve static files
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+}
+NGINXCONF
+fi
+
+rm -f "$ROOTFS_DIR/etc/nginx/sites-enabled/default"
+ln -sf /etc/nginx/sites-available/archipelago "$ROOTFS_DIR/etc/nginx/sites-enabled/archipelago"
+
+echo "   Configuring systemd service..."
+cp "$SNAPSHOT_DIR/configs/archipelago.service" "$ROOTFS_DIR/etc/systemd/system/archipelago.service"
+
+# Enable services (create symlinks)
+mkdir -p "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants"
+ln -sf /etc/systemd/system/archipelago.service "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/archipelago.service"
+ln -sf /lib/systemd/system/NetworkManager.service "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/NetworkManager.service"
+ln -sf /lib/systemd/system/ssh.service "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/ssh.service"
+ln -sf /lib/systemd/system/nginx.service "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/nginx.service"
+
+echo "   Copying app manifests..."
+if [ -d "$SNAPSHOT_DIR/apps" ]; then
+    mkdir -p "$ROOTFS_DIR/etc/archipelago/apps"
+    cp -r "$SNAPSHOT_DIR/apps/"* "$ROOTFS_DIR/etc/archipelago/apps/" 2>/dev/null || true
+fi
+
+echo "   Repacking rootfs with live server state..."
+cd "$ROOTFS_DIR"
+tar -cf "$ROOTFS_TAR" .
+cd "$SCRIPT_DIR"
+
+echo "   ✅ Live server files injected"
+echo ""
+
+# =============================================================================
+# STEP 4: Create installer ISO (reuse existing auto-installer logic)
+# =============================================================================
+echo "📦 Step 4: Creating auto-installer ISO..."
+echo ""
+
+# Now call the existing auto-installer script but skip the rootfs build
+# since we already have it with live server state
+export ROOTFS_TAR="$ROOTFS_TAR"
+export SKIP_ROOTFS_BUILD="true"
+
+# Run the rest of build-auto-installer-iso.sh logic here...
+# (Download Debian Live base, create auto-install.sh, package ISO)
+
+echo "   Downloading Debian Live base..."
+BASE_ISO="$WORK_DIR/debian-live-installer.iso"
+if [ ! -f "$BASE_ISO" ] || [ $(stat -f%z "$BASE_ISO" 2>/dev/null || stat -c%s "$BASE_ISO" 2>/dev/null || echo 0) -lt 100000000 ]; then
+    curl -L -# -o "$BASE_ISO" \
+        "https://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-12.9.0-amd64-standard.iso"
+fi
+
+echo "   Extracting installer base..."
+INSTALLER_ISO="$WORK_DIR/installer-iso"
+rm -rf "$INSTALLER_ISO"
+mkdir -p "$INSTALLER_ISO"
+cd "$INSTALLER_ISO"
+7z x -y "$BASE_ISO" >/dev/null 2>&1
+
+cd "$SCRIPT_DIR"
+
+# Copy archipelago files
+ARCH_DIR="$INSTALLER_ISO/archipelago"
+mkdir -p "$ARCH_DIR"
+cp "$ROOTFS_TAR" "$ARCH_DIR/rootfs.tar"
+
+echo "   ✅ Archipelago components added (rootfs: $(du -h "$ARCH_DIR/rootfs.tar" | cut -f1))"
+echo ""
+
+# Continue with ISO creation...
+echo "📦 Step 5: Creating final bootable ISO..."
+
+# (Rest of xorriso logic from build-auto-installer-iso.sh)
+
+OUTPUT_ISO="$OUTPUT_DIR/archipelago-live-$(date +%Y%m%d-%H%M%S).iso"
+
+# Get MBR
+ISOHDPFX="$WORK_DIR/isohdpfx.bin"
+dd if="$INSTALLER_ISO/isolinux/isolinux.bin" of="$ISOHDPFX" bs=432 count=1 2>/dev/null
+
+xorriso -as mkisofs -o "$OUTPUT_ISO" \
+    -volid "ARCHIPELAGO" \
+    -J -R \
+    -isohybrid-mbr "$ISOHDPFX" \
+    -c isolinux/boot.cat \
+    -b isolinux/isolinux.bin \
+    -no-emul-boot -boot-load-size 4 -boot-info-table \
+    -eltorito-alt-boot \
+    -e boot/grub/efi.img \
+    -no-emul-boot \
+    -isohybrid-gpt-basdat \
+    "$INSTALLER_ISO"
+
+echo ""
+echo "╔════════════════════════════════════════════════════════════════╗"
+echo "║   ✅ LIVE SERVER ISO CREATED!                                 ║"
+echo "╚════════════════════════════════════════════════════════════════╝"
+echo ""
+echo "📀 Output: $OUTPUT_ISO"
+echo "   Size: $(du -h "$OUTPUT_ISO" | cut -f1)"
+echo "   MD5: $(md5sum "$OUTPUT_ISO" 2>/dev/null || md5 "$OUTPUT_ISO" | awk '{print $NF}')"
+echo ""
+echo "🎉 This ISO contains the EXACT state of your dev server!"
+echo ""
+echo "To flash:"
+echo "  cd image-recipe && ./write-usb-dd.sh /dev/diskX"
+echo ""