Merge remote-tracking branch 'gitea-ai/gsd/phase-13-aiui-functional-conversational-node-control-and-content-surf'

scripts/build-aiui.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+#
+# build-aiui.sh — the ONE supported way to build AIUI for a node.
+#
+# D-19 (2026-08-03): AIUI is no longer a second repository at
+# git.tx1138.com/lfg2025/AIUI. It was migrated in-repo to aiui/ via
+# `git subtree`, full history intact. There is no second checkout to clone,
+# no separate per-repo pin file to read or write, and no dirty-second-tree
+# refusal to perform — this repo's own ordinary commit discipline is what
+# keeps its history honest now, not a second-repo-specific check. D-15's
+# *delivery* half still stands and is what this script enforces:
+#
+#   - VITE_BASE_PATH must be exactly /aiui/. A wrong value produces a BLACK
+#     PAGE when embedded — the router base breaks, not the assets (this has
+#     burned this project before). Enforced here, not remembered by whoever
+#     runs the build.
+#   - A fresh checkout of this repo has no aiui/node_modules (unlike the old
+#     world, where a developer's separate AIUI clone was assumed already
+#     `pnpm install`ed) — this script installs from aiui/pnpm-lock.yaml
+#     with --frozen-lockfile before building, and treats a lockfile/
+#     package.json mismatch as a hard failure, not something to silently
+#     resolve.
+#   - The build runs AIUI's own real command (vue-tsc --noEmit && vite
+#     build via `pnpm build`), so a type error fails the build loudly
+#     instead of shipping a stale dist.
+#   - Before anything is copied anywhere, the emitted dist is verified:
+#     every local asset href carries the AIUI mount path, and this repo's
+#     own current commit (there is no second repo's SHA to pin — D-19 made
+#     them the same thing) is discoverable in the output, so a deployed
+#     node is attributable to a commit of THIS repo.
+#
+# Usage:
+#   bash scripts/build-aiui.sh
+#
+# On success, aiui/packages/app/dist/ is a fresh, verified AIUI build ready
+# to be rsynced/tar'd to a node by scripts/deploy-to-target.sh or
+# scripts/setup-aiui-server.sh.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+AIUI_ROOT="$PROJECT_DIR/aiui"
+AIUI_APP_DIR="$AIUI_ROOT/packages/app"
+AIUI_DIST="$AIUI_APP_DIR/dist"
+AIUI_MOUNT_PATH="/aiui/"
+
+timestamp() { echo "[$(date +%H:%M:%S)]"; }
+
+# ── require_base_path ──────────────────────────────────────────────────
+# Fails loudly, not silently, when VITE_BASE_PATH is unset or wrong. The
+# normal case never reaches the "unset" branch below: the caller sets a
+# default via `: "${VITE_BASE_PATH:=$AIUI_MOUNT_PATH}"` before this runs.
+# This function's job is to catch an operator override with the WRONG
+# value (T-13-57: a wrong base path ships a black page to every node).
+require_base_path() {
+    if [ -z "${VITE_BASE_PATH:-}" ]; then
+        echo "FATAL: VITE_BASE_PATH is unset." >&2
+        echo "       AIUI must be built with VITE_BASE_PATH=${AIUI_MOUNT_PATH}" >&2
+        echo "       or every asset href in the built index.html will be" >&2
+        echo "       wrong and AIUI will render as a BLACK PAGE when" >&2
+        echo "       embedded at ${AIUI_MOUNT_PATH} on a node." >&2
+        return 1
+    fi
+    if [ "$VITE_BASE_PATH" != "$AIUI_MOUNT_PATH" ]; then
+        echo "FATAL: VITE_BASE_PATH='$VITE_BASE_PATH' is wrong." >&2
+        echo "       AIUI must be built with VITE_BASE_PATH=${AIUI_MOUNT_PATH}" >&2
+        echo "       exactly. A wrong base path breaks the router base (not" >&2
+        echo "       the assets) and ships a BLACK PAGE to every node this" >&2
+        echo "       dist reaches." >&2
+        return 1
+    fi
+    return 0
+}
+
+# ── verify_dist ─────────────────────────────────────────────────────────
+# Asserts the build is safe to ship, BEFORE anything is copied anywhere.
+verify_dist() {
+    local index="$AIUI_DIST/index.html"
+    if [ ! -f "$index" ]; then
+        echo "FATAL: $index does not exist — the build did not produce a dist." >&2
+        return 1
+    fi
+
+    # Every local (leading-"/") src=/href= must carry the AIUI mount path.
+    # A hand-built bundle with the wrong base path gives a black page, and
+    # the router base is what actually breaks, not the assets.
+    local bad_refs
+    bad_refs=$(grep -oE '(src|href)="/[^"]*"' "$index" \
+        | grep -v -F "=\"${AIUI_MOUNT_PATH}" || true)
+    if [ -n "$bad_refs" ]; then
+        echo "FATAL: $index references local assets outside ${AIUI_MOUNT_PATH}:" >&2
+        echo "$bad_refs" | sed 's/^/       /' >&2
+        return 1
+    fi
+
+    # Mock quarantine (operator decision 2026-08-07, enforced 8329b826):
+    # production bundles must carry NO mock content hosts. The demo site's
+    # content pack builds with VITE_DEMO_CONTENT=true and legitimately
+    # contains them — skip this check for that build.
+    if [ "${VITE_DEMO_CONTENT:-false}" != "true" ]; then
+        local mock_hits
+        mock_hits=$(grep -rl -e 'spotify\.com/track/example' -e 'cloud\.example\.com' \
+            -e 'plex://play' -e 'image\.tmdb\.org' "$AIUI_DIST" 2>/dev/null || true)
+        if [ -n "$mock_hits" ]; then
+            echo "FATAL: production bundle contains mock content hosts" >&2
+            echo "       (mocks are demo-site-only per the 2026-08-07 operator decision):" >&2
+            echo "$mock_hits" | sed 's/^/       /' >&2
+            return 1
+        fi
+    fi
+
+    # Attribute this build to THIS repo's own current commit (D-19: no
+    # second-repo pin file — this repo's own commit IS the answer now).
+    local commit_sha
+    commit_sha="$(git -C "$PROJECT_DIR" rev-parse HEAD)"
+    {
+        echo "commit=$commit_sha"
+        echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+        echo "base_path=$VITE_BASE_PATH"
+    } > "$AIUI_DIST/BUILD-INFO"
+    if ! grep -rq "$commit_sha" "$AIUI_DIST/"; then
+        echo "FATAL: this repo's current commit ($commit_sha) is not" >&2
+        echo "       discoverable anywhere under $AIUI_DIST — a deployed" >&2
+        echo "       node would be unattributable to a source commit." >&2
+        return 1
+    fi
+
+    # Best-effort staleness check: if the source tree changed since the
+    # last recorded build but the emitted asset filenames are IDENTICAL,
+    # something didn't actually rebuild — Vite's content hashes should
+    # differ whenever the content they hash differs. State lives in a
+    # gitignored marker beside dist/ (dist/ itself gets wiped by every
+    # `vite build`, so it can't hold its own prior-run history).
+    local src_hash_file="$AIUI_APP_DIR/.build-aiui-last-src-hash"
+    local prev_asset_list="$AIUI_APP_DIR/.build-aiui-last-assets"
+    local cur_src_hash cur_assets
+    cur_src_hash=$(find "$AIUI_APP_DIR/src" -type f -print0 2>/dev/null \
+        | sort -z | xargs -0 sha256sum 2>/dev/null | sha256sum | awk '{print $1}')
+    cur_assets=$(ls "$AIUI_DIST/assets" 2>/dev/null | sort)
+    if [ -f "$src_hash_file" ] && [ -f "$prev_asset_list" ]; then
+        local prev_src_hash prev_assets
+        prev_src_hash=$(cat "$src_hash_file")
+        prev_assets=$(cat "$prev_asset_list")
+        if [ "$prev_src_hash" != "$cur_src_hash" ] && [ "$prev_assets" = "$cur_assets" ] && [ -n "$cur_assets" ]; then
+            echo "FATAL: source changed since the last build but the" >&2
+            echo "       emitted asset filenames are IDENTICAL to the" >&2
+            echo "       previous build — this looks like a stale/cached" >&2
+            echo "       dist, not a fresh build." >&2
+            return 1
+        fi
+    fi
+    echo "$cur_src_hash" > "$src_hash_file"
+    echo "$cur_assets" > "$prev_asset_list"
+
+    return 0
+}
+
+echo "$(timestamp) build-aiui.sh: building AIUI from $AIUI_ROOT (in-repo, D-19)"
+
+if [ ! -d "$AIUI_ROOT" ]; then
+    echo "FATAL: $AIUI_ROOT does not exist." >&2
+    echo "       AIUI is expected in-repo at aiui/ — it is no longer a" >&2
+    echo "       sibling checkout at ../AIUI (D-19)." >&2
+    exit 1
+fi
+
+: "${VITE_BASE_PATH:=$AIUI_MOUNT_PATH}"
+export VITE_BASE_PATH
+require_base_path
+
+echo "$(timestamp) Installing aiui/ workspace from its committed lockfile..."
+(cd "$AIUI_ROOT" && pnpm install --frozen-lockfile)
+
+echo "$(timestamp) Building AIUI (vue-tsc --noEmit && vite build)..."
+(cd "$AIUI_APP_DIR" && VITE_BASE_PATH="$VITE_BASE_PATH" pnpm build)
+
+echo "$(timestamp) Verifying dist..."
+verify_dist
+
+echo "$(timestamp) AIUI build OK — $AIUI_DIST attributable to $(git -C "$PROJECT_DIR" rev-parse --short HEAD)"

scripts/first-boot-containers.sh

@@ -1225,6 +1225,12 @@ server:
   bind_address: "0.0.0.0"
   port: 8080
   limiter: false
+# json is required: SearXNG defaults to html-only and answers 403 on the
+# JSON API, which is the only thing AIUI's web search speaks.
+search:
+  formats:
+    - html
+    - json
 ui:
   default_theme: simple
 SEARXCFG

scripts/lib/common.sh

@@ -157,6 +157,48 @@ wait_for_container() {
   return 1
 }
 
+# ── Same-host deploy safety guard (2026-07-31 incident, widened 13-09) ──
+
+# Refuse a same-host deploy whose resolved source and destination differ.
+#
+# The 2026-07-31 incident: a same-host `rsync --delete` deploy whose source
+# was INSIDE the destination (a worktree under the main checkout) mirrored
+# the worktree onto the main checkout and deleted ~1810 tracked files, a
+# running dev server, and two sessions' uncommitted work. The original fix
+# refused only that containment shape (source-in-destination or
+# destination-in-source). It missed SIBLING directories that share a parent
+# but neither contains the other — e.g. archy-phase13 (this worktree) as
+# source and archy (the main checkout, TARGET_DIR's resolved symlink
+# target) as destination — which is the identical rsync --delete hazard
+# through a shape the old two-case guard let through.
+#
+# This function takes two ALREADY-RESOLVED (`readlink -f`) absolute paths
+# and makes no SSH calls itself — same-host detection stays in the caller
+# (deploy-to-target.sh already does it via /etc/machine-id). It returns 0
+# only when the two paths are equal; every other case is refused, not just
+# the two containment shapes. Any resolved-path mismatch on the same host
+# is the same rsync --delete hazard regardless of shape, so "refuse unless
+# equal" is both the widening and a simplification.
+#
+# Usage: assert_safe_same_host_deploy <local_src> <remote_dst>
+assert_safe_same_host_deploy() {
+  local local_src="$1" remote_dst="$2"
+
+  if [ -z "$local_src" ] || [ -z "$remote_dst" ]; then
+    log_error "assert_safe_same_host_deploy: both paths are required (local_src='$local_src' remote_dst='$remote_dst')"
+    return 1
+  fi
+
+  if [ "$local_src" = "$remote_dst" ]; then
+    return 0
+  fi
+
+  echo "FATAL: refusing to deploy. Source '$local_src' and destination '$remote_dst' are on the same host but resolve to DIFFERENT paths." >&2
+  echo "       'rsync --delete' would mirror one onto the other and delete whatever the source lacks — the 2026-07-31 data-loss incident (and the sibling-directory shape its original fix missed)." >&2
+  echo "       Run this script from the deploy destination itself ('$remote_dst'), not from a worktree, sibling checkout, or copy." >&2
+  return 1
+}
+
 # ── Section timing ─────────────────────────────────────────────────────
 
 # Track elapsed time for deploy sections.

scripts/setup-node-ca.sh

@@ -154,6 +154,59 @@ install -m 644 "$CA_CRT" "$SSL_DIR/ca-download.crt"
 FP="$(openssl x509 -in "$CA_CRT" -noout -fingerprint -sha256 | cut -d= -f2)"
 log "CA fingerprint (SHA-256): $FP"
 
+# --- nginx HTTPS listener -----------------------------------------------------
+# The CA is only useful if something actually serves TLS. Bind the dashboard's
+# HTTPS on this host's LAN addresses ONLY: tailscaled already owns :443 on the
+# tailnet addresses (with its own Let's Encrypt cert), so a plain
+# `listen 443 default_server` binds 0.0.0.0 and fails with EADDRINUSE — nginx
+# then keeps running the OLD config and the reload looks like it worked.
+# Observed exactly that on archi-dev-box.
+#
+# Port 80 keeps serving: nodes are reached by IP on LANs where forcing a
+# redirect would strand anyone who has not installed the CA yet.
+ensure_nginx_https() {
+  local site="${ARCHY_NGINX_SITE:-/etc/nginx/sites-enabled/archipelago}"
+  [ -f "$site" ] || { log "No nginx site at $site — skipping HTTPS listener"; return; }
+
+  local addrs
+  addrs="$(ip -o -4 addr show scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
+    | grep -vE '^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.' | sort -u)"
+  [ -z "$addrs" ] && { log "No LAN address — skipping HTTPS listener"; return; }
+
+  if grep -q 'listen .*:443 ssl' "$site"; then
+    log "nginx HTTPS listener already present"
+    return
+  fi
+  log "Adding nginx HTTPS listener on: $(echo "$addrs" | tr '\n' ' ')"
+  python3 - "$site" "$addrs" <<'PYEOF'
+import sys, re, pathlib
+site, addrs = pathlib.Path(sys.argv[1]), sys.argv[2].split()
+s = site.read_text(); lines = s.split('\n')
+start = next(i for i,l in enumerate(lines) if l.strip() == 'server {')
+depth = 0; end = None
+for i in range(start, len(lines)):
+    depth += lines[i].count('{') - lines[i].count('}')
+    if depth == 0 and i > start:
+        end = i; break
+block = lines[start:end+1]
+https = []
+for l in block:
+    if re.match(r'\s*listen 80 default_server;', l):
+        https += [f'    listen {a}:443 ssl;' for a in addrs]
+        https += ['    ssl_certificate /etc/archipelago/ssl/archipelago.crt;',
+                  '    ssl_certificate_key /etc/archipelago/ssl/archipelago.key;',
+                  '    ssl_protocols TLSv1.2 TLSv1.3;']
+        continue
+    if re.match(r'\s*listen \[::\]:80 default_server;', l):
+        continue
+    https.append(l)
+lines = lines[:end+1] + [''] + https + lines[end+1:]
+site.write_text('\n'.join(lines))
+PYEOF
+}
+
+ensure_nginx_https
+
 if command -v systemctl >/dev/null 2>&1 && systemctl is-active --quiet nginx; then
   if nginx -t >/dev/null 2>&1; then
     systemctl reload nginx && log "nginx reloaded"

scripts/verify-aiui-deploy.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# verify-aiui-deploy.sh — post-deploy check that a marker string is actually
+# reachable by a browser loading AIUI, not merely present somewhere on disk.
+#
+# The node's /opt/archipelago/web-ui/aiui/assets/ directory is a
+# never-pruned graveyard: nginx never deletes an old build's chunk files
+# when a new build lands (only the tar+rsync step below AIUI_DIST replaces
+# what's there, and even that has left orphaned files in past incidents —
+# see feedback_node_side_frontend_verify_stale_chunks). A disk grep over
+# assets/ can therefore report "deployed" before the deploy actually
+# happened, because a DEAD chunk from an old build still contains the old
+# string. The only honest check fetches what a browser would actually load:
+# resolve the LIVE chunk set via the service worker's precache manifest
+# (sw.js), fetch each live chunk over HTTP, and grep the fetched bytes.
+#
+# This script never opens a remote shell session onto the node and never
+# greps the node's filesystem directly — every check here is an HTTP
+# fetch, exactly what a browser session would do.
+#
+# Usage:
+#   ./verify-aiui-deploy.sh <node-host> "<marker string>"
+#
+# Exit 0  = the marker was found in at least one live chunk fetched over HTTP.
+# Exit 1  = the marker was not found in any live chunk (negative control
+#           should also produce this — a check that always passes is not a
+#           check).
+
+set -uo pipefail
+
+HOST="${1:?usage: verify-aiui-deploy.sh <node-host> \"<marker string>\"}"
+MARKER="${2:?usage: verify-aiui-deploy.sh <node-host> \"<marker string>\"}"
+
+# Accept a bare host or a host:port; default to plain HTTP on :80, matching
+# how neode-ui/AIUI are actually served on a node (nginx terminates TLS
+# elsewhere; the lifecycle gate and other production-quality scripts in
+# this directory talk to nodes over plain HTTP the same way).
+BASE="http://${HOST}"
+SW_URL="${BASE}/aiui/sw.js"
+
+timestamp() { echo "[$(date +%H:%M:%S)]"; }
+
+echo "$(timestamp) Fetching service worker manifest: $SW_URL"
+SW_BODY="$(curl -sf -m 15 "$SW_URL" 2>/dev/null || true)"
+if [ -z "$SW_BODY" ]; then
+    echo "FATAL: could not fetch $SW_URL — is AIUI deployed and nginx up on $HOST?" >&2
+    exit 1
+fi
+
+# vite-plugin-pwa's generateSW mode emits sw.js with a
+# workbox.precacheAndRoute([{url:"...",revision:"..."|null}, ...]) call —
+# a JS array literal (unquoted keys), not JSON. Extract every url:"..."
+# value without a full JS parser.
+mapfile -t LIVE_PATHS < <(
+    grep -oE 'url:"[^"]*"' <<<"$SW_BODY" | sed -E 's/^url:"//; s/"$//'
+)
+
+if [ "${#LIVE_PATHS[@]}" -eq 0 ]; then
+    echo "FATAL: $SW_URL fetched but no precache entries found — cannot resolve live chunks." >&2
+    exit 1
+fi
+
+echo "$(timestamp) Resolved ${#LIVE_PATHS[@]} live chunk(s) from the precache manifest."
+
+FOUND=0
+CHECKED=0
+# Fetch to a temp file and grep that, NOT `curl | grep -q`: under this
+# script's pipefail, grep -q's early exit EPIPEs curl (exit 23) whenever
+# the marker sits before the tail of a >64KB chunk, turning a genuine
+# match into a nondeterministic FAIL.
+BODY_TMP="$(mktemp)"
+trap 'rm -f "$BODY_TMP"' EXIT
+for path in "${LIVE_PATHS[@]}"; do
+    [ -z "$path" ] && continue
+    CHECKED=$((CHECKED + 1))
+    url="${BASE}/aiui/${path}"
+    if curl -sf -m 15 -o "$BODY_TMP" "$url" 2>/dev/null && grep -q -- "$MARKER" "$BODY_TMP"; then
+        echo "$(timestamp)   MATCH: $path"
+        FOUND=1
+        break
+    fi
+done
+
+echo "$(timestamp) Checked $CHECKED live chunk(s) fetched over HTTP for marker: $MARKER"
+
+if [ "$FOUND" -eq 1 ]; then
+    echo "$(timestamp) PASS — marker found in a live, browser-fetchable chunk."
+    exit 0
+else
+    echo "$(timestamp) FAIL — marker not found in any live chunk (fetched via sw.js manifest, not a disk grep)." >&2
+    exit 1
+fi