From 3a3077529b7639d8f6efb785a22748f084396687 Mon Sep 17 00:00:00 2001 From: archipelago Date: Sun, 16 Aug 2026 05:28:53 -0400 Subject: [PATCH] feat(kiosk): companion remote drives app iframes via trusted CDP input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion tap/scroll/type now works INSIDE cross-origin app iframes and kiosk tabs. The web relay synthesizes untrusted DOM events in the top document, which can never cross an origin boundary — so apps served through the appgate were dead to the remote. The kiosk Chromium now exposes a loopback-only CDP port (default origin check intact, no --remote-allow-origins) and a backend bridge (api/handler/cdp.rs) dispatches validated companion input as Input.dispatchKeyEvent / dispatchMouseEvent / mouseWheel — trusted events that hit-test through any frame, move real focus, and insert text like a physical device. - Session keeper self-heals across kiosk Chromium restarts; inert on nodes without a kiosk unit (falls back to the existing relay path). - The kiosk relay subscriber self-tags (?kiosk=1) and the backend mutes its key/click/scroll messages while the bridge is live, so input never applies twice; cursor moves still flow for the on-screen cursor. - While companion input is active the native OS pointer is hidden (cursor:none, auto-restores 30s after the last event) so the dead physical-mouse cursor doesn't sit next to the virtual one. - docs/tv-input-iframe-apps.md scope note updated: gamepad keys stay on uinput; CDP is for companion pointer/typing only. Co-Authored-By: Claude Fable 5 --- core/archipelago/src/api/handler/cdp.rs | 421 ++++++++++++++++++ core/archipelago/src/api/handler/mod.rs | 9 + .../src/api/handler/remote_input.rs | 16 +- .../src/api/handler/remote_relay.rs | 22 + docs/tv-input-iframe-apps.md | 13 +- .../configs/archipelago-kiosk-launcher.sh | 7 + neode-ui/src/api/remote-relay.ts | 27 +- neode-ui/src/style.css | 9 + 8 files changed, 517 insertions(+), 7 deletions(-) create mode 100644 core/archipelago/src/api/handler/cdp.rs diff --git a/core/archipelago/src/api/handler/cdp.rs b/core/archipelago/src/api/handler/cdp.rs new file mode 100644 index 00000000..4114b70b --- /dev/null +++ b/core/archipelago/src/api/handler/cdp.rs @@ -0,0 +1,421 @@ +//! CDP input bridge — forwards companion remote input into the local kiosk +//! Chromium as *trusted*, browser-level input via the DevTools protocol. +//! +//! Why this exists: the web relay path (`remote-relay.ts`) synthesizes DOM +//! events in the top document, and synthetic events can never cross into a +//! cross-origin iframe — so on the kiosk, companion taps/keys/scrolls died at +//! the border of every containerized app's frame. CDP `Input.dispatch*` +//! events enter the browser's real input pipeline: they hit-test through any +//! frame, move focus, and insert text exactly like a physical device, which +//! is the only correct way to drive app iframes (tracked in the unified task +//! tracker; supersedes the earlier "no CDP" note in +//! `docs/tv-input-iframe-apps.md`, which was about gamepad *keys* only). +//! +//! The kiosk launcher opens Chromium with `--remote-debugging-port=9222` +//! bound to loopback. This keeper task discovers the page target, holds one +//! WebSocket to it, and reconnects whenever the kiosk restarts (the launcher +//! supervises Chromium in a loop, so the debugger URL changes under us). +//! When the bridge is not connected (non-kiosk installs, kiosk booting), +//! `is_active()` is false and callers fall back to the web relay unchanged. +//! +//! Security: the CDP port is loopback-only and Chromium's default origin +//! check stands (we deliberately do NOT pass `--remote-allow-origins`, so +//! browser pages can't open the debug socket; our raw client sends no +//! Origin header and is accepted). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures_util::{SinkExt, StreamExt}; +use serde_json::{json, Value}; +use tokio::sync::mpsc; +use tokio_tungstenite::tungstenite::Message; +use tracing::{debug, info, warn}; + +const CDP_HTTP: &str = "http://127.0.0.1:9222"; +/// Marker whose presence means this node drives a local kiosk display. +const KIOSK_UNIT: &str = "/etc/systemd/system/archipelago-kiosk.service"; +/// One relay scroll step ≈ this many CSS pixels (matches remote-relay.ts). +const SCROLL_STEP_PX: f64 = 100.0; + +/// Cloneable handle the WS handlers use to feed validated relay JSON into +/// the keeper task. +#[derive(Clone)] +pub struct CdpBridge { + tx: mpsc::Sender, + connected: Arc, +} + +impl CdpBridge { + /// Spawn the session keeper and return the shared handle. + pub fn spawn() -> Self { + let (tx, rx) = mpsc::channel::(256); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn(run_keeper(rx, connected.clone())); + Self { tx, connected } + } + + /// True only while a live CDP session to the kiosk Chromium exists. + pub fn is_active(&self) -> bool { + self.connected.load(Ordering::Relaxed) + } + + /// Queue a validated relay input message (the exact JSON that goes to the + /// broadcast channel) for CDP dispatch. Best-effort: if the keeper is + /// behind or gone the message is dropped — input is transient by nature. + pub fn send(&self, relay_json: &str) { + let _ = self.tx.try_send(relay_json.to_string()); + } +} + +/// Virtual cursor state, server-side. The web relay keeps this in the kiosk +/// page (`cursorX`/`cursorY`); CDP needs its own copy because trusted mouse +/// events carry absolute viewport coordinates. +struct Cursor { + x: f64, + y: f64, + w: f64, + h: f64, +} + +async fn run_keeper(mut rx: mpsc::Receiver, connected: Arc) { + loop { + // Cheap gate: no kiosk unit on this node → nothing to drive. Keep + // draining queued input so the channel never backs up. + if tokio::fs::metadata(KIOSK_UNIT).await.is_err() { + drain_for(&mut rx, Duration::from_secs(60)).await; + continue; + } + let Some(ws_url) = discover_page_target().await else { + // Kiosk configured but Chromium not up (or CDP flag not rolled + // out yet) — retry gently. + drain_for(&mut rx, Duration::from_secs(15)).await; + continue; + }; + match drive_session(&ws_url, &mut rx, &connected).await { + Ok(()) => info!("CDP kiosk input session ended cleanly"), + Err(e) => debug!(error = %e, "CDP kiosk input session dropped — will re-discover"), + } + connected.store(false, Ordering::Relaxed); + tokio::time::sleep(Duration::from_secs(3)).await; + } +} + +/// Discard queued input for `d` — used while no kiosk session exists so the +/// bounded channel can't fill with stale events. +async fn drain_for(rx: &mut mpsc::Receiver, d: Duration) { + let _ = tokio::time::timeout(d, async { + while rx.recv().await.is_some() {} + }) + .await; +} + +/// Find the kiosk page target's WebSocket debugger URL. Prefers the page on +/// localhost (the kiosk app) over e.g. devtools/extension targets. +async fn discover_page_target() -> Option { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(3)) + .build() + .ok()?; + let list: Vec = client + .get(format!("{CDP_HTTP}/json/list")) + .send() + .await + .ok()? + .json() + .await + .ok()?; + let pages: Vec<&Value> = list + .iter() + .filter(|t| t.get("type").and_then(Value::as_str) == Some("page")) + .collect(); + let preferred = pages + .iter() + .find(|t| { + t.get("url") + .and_then(Value::as_str) + .is_some_and(|u| u.contains("localhost") || u.contains("127.0.0.1")) + }) + .or_else(|| pages.first()); + preferred? + .get("webSocketDebuggerUrl") + .and_then(Value::as_str) + .map(str::to_string) +} + +async fn drive_session( + ws_url: &str, + rx: &mut mpsc::Receiver, + connected: &Arc, +) -> anyhow::Result<()> { + let (ws, _) = tokio_tungstenite::connect_async(ws_url).await?; + let (mut sink, mut stream) = ws.split(); + let mut next_id: u64 = 0; + let mut id = move || { + next_id += 1; + next_id + }; + + // Viewport size for cursor clamping. Best-effort: fall back to 1080p if + // the metrics call fails — clamping is a nicety, not a correctness need. + sink.send(Message::Text( + json!({"id": id(), "method": "Page.getLayoutMetrics"}).to_string(), + )) + .await?; + let (mut vw, mut vh) = (1920.0_f64, 1080.0_f64); + if let Ok(Some(Ok(Message::Text(txt)))) = + tokio::time::timeout(Duration::from_secs(3), stream.next()).await + { + if let Ok(v) = serde_json::from_str::(&txt) { + if let Some(vp) = v.pointer("/result/cssLayoutViewport") { + vw = vp.get("clientWidth").and_then(Value::as_f64).unwrap_or(vw); + vh = vp.get("clientHeight").and_then(Value::as_f64).unwrap_or(vh); + } + } + } + let mut cursor = Cursor { + x: vw / 2.0, + y: vh / 2.0, + w: vw, + h: vh, + }; + + connected.store(true, Ordering::Relaxed); + info!(viewport = %format!("{vw}x{vh}"), "CDP kiosk input bridge connected"); + + loop { + tokio::select! { + cmd = rx.recv() => { + let Some(cmd) = cmd else { return Ok(()) }; + for frame in translate(&cmd, &mut cursor, &mut id) { + sink.send(Message::Text(frame.to_string())).await?; + } + } + msg = stream.next() => { + match msg { + // Responses/events — nothing to correlate, but a read + // error or close means Chromium restarted. + Some(Ok(_)) => {} + Some(Err(e)) => return Err(e.into()), + None => anyhow::bail!("CDP socket closed"), + } + } + } + } +} + +/// Translate one validated relay input message into CDP command frames. +fn translate(raw: &str, cursor: &mut Cursor, id: &mut impl FnMut() -> u64) -> Vec { + let Ok(msg) = serde_json::from_str::(raw) else { + return vec![]; + }; + match msg.get("t").and_then(Value::as_str) { + Some("m") => { + let dx = msg.get("x").and_then(Value::as_i64).unwrap_or(0) as f64; + let dy = msg.get("y").and_then(Value::as_i64).unwrap_or(0) as f64; + cursor.x = (cursor.x + dx).clamp(0.0, cursor.w - 1.0); + cursor.y = (cursor.y + dy).clamp(0.0, cursor.h - 1.0); + vec![mouse_event(id(), "mouseMoved", cursor, "none", 0, 1)] + } + Some("c") => { + let b = msg.get("b").and_then(Value::as_u64).unwrap_or(1).clamp(1, 3); + let (button, buttons) = match b { + 2 => ("middle", 4), + 3 => ("right", 2), + _ => ("left", 1), + }; + vec![ + // Hover first so the press lands on current hit-test state. + mouse_event(id(), "mouseMoved", cursor, "none", 0, 1), + mouse_event(id(), "mousePressed", cursor, button, buttons, 1), + mouse_event(id(), "mouseReleased", cursor, button, 0, 1), + ] + } + Some("s") => { + let dy = msg.get("y").and_then(Value::as_i64).unwrap_or(0) as f64 * SCROLL_STEP_PX; + vec![json!({ + "id": id(), + "method": "Input.dispatchMouseEvent", + "params": { + "type": "mouseWheel", + "x": cursor.x, "y": cursor.y, + "deltaX": 0.0, "deltaY": dy, + "pointerType": "mouse", + } + })] + } + Some("k") => { + let Some(k) = msg.get("k").and_then(Value::as_str) else { + return vec![]; + }; + key_events(k, id) + } + _ => vec![], + } +} + +fn mouse_event(id: u64, kind: &str, cursor: &Cursor, button: &str, buttons: u32, clicks: u32) -> Value { + json!({ + "id": id, + "method": "Input.dispatchMouseEvent", + "params": { + "type": kind, + "x": cursor.x, "y": cursor.y, + "button": button, + "buttons": buttons, + "clickCount": if kind == "mousePressed" || kind == "mouseReleased" { clicks } else { 0 }, + "pointerType": "mouse", + } + }) +} + +/// xdotool named key → (DOM key, DOM code, Windows virtual-key code). +fn named_key(k: &str) -> Option<(&'static str, &'static str, i32)> { + Some(match k { + "Return" => ("Enter", "Enter", 13), + "BackSpace" => ("Backspace", "Backspace", 8), + "Escape" => ("Escape", "Escape", 27), + "Tab" => ("Tab", "Tab", 9), + "Delete" => ("Delete", "Delete", 46), + "Up" => ("ArrowUp", "ArrowUp", 38), + "Down" => ("ArrowDown", "ArrowDown", 40), + "Left" => ("ArrowLeft", "ArrowLeft", 37), + "Right" => ("ArrowRight", "ArrowRight", 39), + "Home" => ("Home", "Home", 36), + "End" => ("End", "End", 35), + "Prior" => ("PageUp", "PageUp", 33), + "Next" => ("PageDown", "PageDown", 34), + "F1" => ("F1", "F1", 112), + "F2" => ("F2", "F2", 113), + "F3" => ("F3", "F3", 114), + "F4" => ("F4", "F4", 115), + "F5" => ("F5", "F5", 116), + "F6" => ("F6", "F6", 117), + "F7" => ("F7", "F7", 118), + "F8" => ("F8", "F8", 119), + "F9" => ("F9", "F9", 120), + "F10" => ("F10", "F10", 121), + "F11" => ("F11", "F11", 122), + "F12" => ("F12", "F12", 123), + _ => return None, + }) +} + +/// xdotool symbol name → printable char (the relay whitelist speaks xdotool). +fn symbol_char(k: &str) -> Option { + Some(match k { + "space" => ' ', + "exclam" => '!', + "at" => '@', + "numbersign" => '#', + "dollar" => '$', + "percent" => '%', + "asciicircum" => '^', + "ampersand" => '&', + "asterisk" => '*', + "parenleft" => '(', + "parenright" => ')', + "underscore" => '_', + "plus" => '+', + "braceleft" => '{', + "braceright" => '}', + "bar" => '|', + "colon" => ':', + "quotedbl" => '"', + "less" => '<', + "greater" => '>', + "question" => '?', + "asciitilde" => '~', + "minus" => '-', + "equal" => '=', + "bracketleft" => '[', + "bracketright" => ']', + "backslash" => '\\', + "semicolon" => ';', + "apostrophe" => '\'', + "grave" => '`', + "comma" => ',', + "period" => '.', + "slash" => '/', + _ => return None, + }) +} + +/// Build the CDP frame pair (keyDown, keyUp) for one relay key name, +/// including `modifier+base` combos. A keyDown that carries `text` both +/// fires real keydown/keypress AND inserts the character — exactly how a +/// physical keystroke behaves, so games see the key and fields get the text. +fn key_events(k: &str, id: &mut impl FnMut() -> u64) -> Vec { + let (modifiers, base) = match k.split_once('+') { + Some((m, b)) => ( + match m { + "alt" => 1, + "ctrl" => 2, + "super" => 4, + "shift" => 8, + _ => 0, + }, + b, + ), + None => (0, k), + }; + + let (key, code, vk, text): (String, Option<&str>, i32, Option) = + if let Some((key, code, vk)) = named_key(base) { + // Enter carries "\r" like a real keyboard so single-line inputs + // submit and textareas newline. + let text = (key == "Enter").then(|| "\r".to_string()); + (key.to_string(), Some(code), vk, text) + } else { + let ch = if base.chars().count() == 1 { + base.chars().next() + } else { + symbol_char(base) + }; + let Some(mut ch) = ch else { + return vec![]; + }; + if modifiers == 8 && ch.is_ascii_alphabetic() { + ch = ch.to_ascii_uppercase(); + } + let vk = ch.to_ascii_uppercase() as i32; + // Ctrl/Alt/Super chords are shortcuts, not typing — no text. + let text = (modifiers & !8 == 0).then(|| ch.to_string()); + (ch.to_string(), None, vk, text) + }; + + let mut down = json!({ + "id": id(), + "method": "Input.dispatchKeyEvent", + "params": { + "type": "keyDown", + "key": key, + "modifiers": modifiers, + "windowsVirtualKeyCode": vk, + "nativeVirtualKeyCode": vk, + } + }); + if let Some(code) = code { + down["params"]["code"] = json!(code); + } + if let Some(t) = &text { + down["params"]["text"] = json!(t); + down["params"]["unmodifiedText"] = json!(t); + } + let mut up = json!({ + "id": id(), + "method": "Input.dispatchKeyEvent", + "params": { + "type": "keyUp", + "key": key, + "modifiers": modifiers, + "windowsVirtualKeyCode": vk, + "nativeVirtualKeyCode": vk, + } + }); + if let Some(code) = code { + up["params"]["code"] = json!(code); + } + vec![down, up] +} diff --git a/core/archipelago/src/api/handler/mod.rs b/core/archipelago/src/api/handler/mod.rs index 6964e98b..99a85893 100644 --- a/core/archipelago/src/api/handler/mod.rs +++ b/core/archipelago/src/api/handler/mod.rs @@ -1,4 +1,5 @@ mod blob; +mod cdp; mod content; mod dwn; mod model_proxy; @@ -51,6 +52,10 @@ pub struct ApiHandler { /// to the phone's default browser. Lets "open in external browser" apps — /// which the kiosk can't usefully open itself — launch on the controller. external_open_tx: broadcast::Sender, + /// Bridge that dispatches companion input into the local kiosk Chromium + /// as trusted CDP events (reaches inside cross-origin app iframes). + /// Inert (never connects) on nodes without a kiosk. + cdp_bridge: cdp::CdpBridge, /// Content-addressed blob store for attachments shared over mesh/federation. blob_store: Arc, /// Our own node pubkey (hex) — used to self-sign debug/test capabilities. @@ -79,6 +84,7 @@ impl ApiHandler { ); let (input_relay_tx, _) = broadcast::channel(64); let (external_open_tx, _) = broadcast::channel(16); + let cdp_bridge = cdp::CdpBridge::spawn(); // Derive a blob-store capability key from the node's Ed25519 signing // key. SHA-256 domain-separated so rotating the identity rotates @@ -109,6 +115,7 @@ impl ApiHandler { session_store, input_relay_tx, external_open_tx, + cdp_bridge, blob_store, self_pubkey_hex, }) @@ -402,6 +409,7 @@ impl ApiHandler { req, self.input_relay_tx.clone(), self.external_open_tx.subscribe(), + self.cdp_bridge.clone(), ) .await; } @@ -416,6 +424,7 @@ impl ApiHandler { req, self.input_relay_tx.subscribe(), self.external_open_tx.clone(), + self.cdp_bridge.clone(), ) .await; } diff --git a/core/archipelago/src/api/handler/remote_input.rs b/core/archipelago/src/api/handler/remote_input.rs index 04623dd9..562443ab 100644 --- a/core/archipelago/src/api/handler/remote_input.rs +++ b/core/archipelago/src/api/handler/remote_input.rs @@ -212,6 +212,7 @@ impl ApiHandler { req: Request, relay_tx: broadcast::Sender, mut external_open_rx: broadcast::Receiver, + cdp_bridge: super::cdp::CdpBridge, ) -> Result> { // Extract optional player ID from query string: /ws/remote-input?p=1 let player_id: Option = req @@ -317,9 +318,20 @@ impl ApiHandler { } else { text.clone() }; - let _ = relay_tx.send(relay_text); + let validation = handle_input(&text).await; + let _ = relay_tx.send(relay_text.clone()); + // Trusted-input path: while the kiosk CDP + // bridge is live, also dispatch validated + // input into the kiosk Chromium so it + // lands inside cross-origin app iframes + // (the web relay above can't cross that + // boundary; the kiosk subscriber mutes its + // own DOM synthesis — remote_relay.rs). + if matches!(validation, Ok(None)) && cdp_bridge.is_active() { + cdp_bridge.send(&relay_text); + } - match handle_input(&text).await { + match validation { Ok(Some(reply)) => { let _ = tx.send(Message::Text(reply)).await; } diff --git a/core/archipelago/src/api/handler/remote_relay.rs b/core/archipelago/src/api/handler/remote_relay.rs index 608b1cc1..188dfb6c 100644 --- a/core/archipelago/src/api/handler/remote_relay.rs +++ b/core/archipelago/src/api/handler/remote_relay.rs @@ -21,7 +21,16 @@ impl ApiHandler { req: Request, mut relay_rx: broadcast::Receiver, external_open_tx: broadcast::Sender, + cdp_bridge: super::cdp::CdpBridge, ) -> Result> { + // The kiosk browser self-identifies with ?kiosk=1 so we can suppress + // its DOM-synthesis path while the CDP bridge delivers trusted input + // (otherwise every key/click/scroll would apply twice). A remote + // browser claiming kiosk=1 only mutes its own input — harmless. + let is_kiosk = req + .uri() + .query() + .is_some_and(|q| q.split('&').any(|s| s == "kiosk=1")); let (response, ws_fut_opt) = hyper_ws_listener::create_ws(req) .map_err(|e| anyhow::anyhow!("WebSocket upgrade failed: {}", e))?; @@ -60,6 +69,19 @@ impl ApiHandler { msg = relay_rx.recv() => { match msg { Ok(text) => { + // Kiosk + live CDP bridge: keys/clicks/ + // scrolls arrive as trusted browser input + // via CDP; forward only cursor moves (the + // on-screen cursor is drawn by the page) + // so nothing applies twice. + if is_kiosk && cdp_bridge.is_active() { + let tag = serde_json::from_str::(&text) + .ok() + .and_then(|v| v.get("t").and_then(|t| t.as_str().map(str::to_string))); + if !matches!(tag.as_deref(), Some("m") | Some("o") | Some("p")) { + continue; + } + } if tx.send(Message::Text(text)).await.is_err() { break; } diff --git a/docs/tv-input-iframe-apps.md b/docs/tv-input-iframe-apps.md index 42bd1d3d..fea28f95 100644 --- a/docs/tv-input-iframe-apps.md +++ b/docs/tv-input-iframe-apps.md @@ -66,9 +66,16 @@ packaging docs; never required for an app to be usable. ## What NOT to do -- ❌ CDP (`--remote-debugging-port` + Input.dispatchKeyEvent): works but adds - a privileged debug port to the kiosk and a daemon↔browser coupling; the - uinput route gets the same result at kernel level with no attack surface. +- ⚠️ **Scope update 2026-08-16:** the "no CDP" rule below still holds for + *gamepad keys* (uinput remains their path). But companion **pointer input** + (tap at coordinates, scroll, focus-then-type inside cross-origin app + iframes) has no uinput equivalent that survives hit-testing across frames, + so the kiosk now runs `--remote-debugging-port=9222` (loopback-only, default + origin check intact) feeding the backend CDP bridge in + `core/archipelago/src/api/handler/cdp.rs`. +- ❌ CDP for gamepad keys (`Input.dispatchKeyEvent` for the NES pad): the + uinput route gets the same result at kernel level with no daemon↔browser + coupling; keep gamepads on uinput. - ❌ Per-app nav scripts injected into iframes: cross-origin makes this impossible for most apps, and it's exactly the per-app hack the requirement rules out. diff --git a/image-recipe/configs/archipelago-kiosk-launcher.sh b/image-recipe/configs/archipelago-kiosk-launcher.sh index ced4da15..b1126f8c 100644 --- a/image-recipe/configs/archipelago-kiosk-launcher.sh +++ b/image-recipe/configs/archipelago-kiosk-launcher.sh @@ -291,7 +291,14 @@ while true; do --disable-metrics-reporting \ --disable-domain-reliability \ --js-flags="--max-old-space-size=256" \ + --remote-debugging-port=9222 \ --user-data-dir=/var/lib/archipelago/chromium-kiosk + # --remote-debugging-port binds to 127.0.0.1 only. It feeds the + # backend's CDP input bridge (api/handler/cdp.rs), which dispatches + # companion-app remote input as TRUSTED browser events — the only way + # taps/keys/scrolls reach inside cross-origin app iframes. Chromium's + # default origin check stays on (no --remote-allow-origins), so web + # pages cannot open the debug socket. sleep 3 done diff --git a/neode-ui/src/api/remote-relay.ts b/neode-ui/src/api/remote-relay.ts index d6eee450..a7414a77 100644 --- a/neode-ui/src/api/remote-relay.ts +++ b/neode-ui/src/api/remote-relay.ts @@ -53,15 +53,30 @@ let cursorX = typeof window !== 'undefined' ? window.innerWidth / 2 : 0 let cursorY = typeof window !== 'undefined' ? window.innerHeight / 2 : 0 let cursorHideTimer: ReturnType | null = null +/** + * While the companion is actively driving input, suppress the other visible + * pointer: the native OS cursor sitting wherever the physical mouse left it + * reads as a second, dead cursor next to the companion's virtual one + * (user request 2026-08-16). Restores automatically when the companion goes + * quiet (same 30s window as `companionActive`). + */ +function setNativeCursorSuppressed(on: boolean) { + document.documentElement.classList.toggle('companion-input-active', on) +} + function markCompanionActive() { companionActive.value = true companionInputActive.value = true + setNativeCursorSuppressed(true) if (inputFlickerTimeout) clearTimeout(inputFlickerTimeout) inputFlickerTimeout = setTimeout(() => { companionInputActive.value = false }, 200) if (companionTimeout) clearTimeout(companionTimeout) - companionTimeout = setTimeout(() => { companionActive.value = false }, 30_000) + companionTimeout = setTimeout(() => { + companionActive.value = false + setNativeCursorSuppressed(false) + }, 30_000) } function createCursor(): HTMLDivElement { @@ -333,7 +348,14 @@ function doConnect() { } const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:' - const url = `${protocol}//${window.location.host}/ws/remote-relay` + // The kiosk self-identifies so the backend can mute this subscriber's + // key/click/scroll messages while the CDP bridge delivers them as trusted + // browser input (cursor moves still arrive so the on-screen cursor draws). + // Same three-way predicate as App.vue's isKiosk. + const isKiosk = localStorage.getItem('kiosk') === 'true' + || new URLSearchParams(window.location.search).has('kiosk') + || window.location.pathname === '/kiosk' + const url = `${protocol}//${window.location.host}/ws/remote-relay${isKiosk ? '?kiosk=1' : ''}` ws = new WebSocket(url) @@ -405,6 +427,7 @@ export function stopRemoteRelay() { if (cursorHideTimer) { clearTimeout(cursorHideTimer); cursorHideTimer = null } if (ws) { ws.onclose = null; ws.close(); ws = null } if (cursorEl) { cursorEl.remove(); cursorEl = null } + setNativeCursorSuppressed(false) relayConnected.value = false companionActive.value = false companionInputActive.value = false diff --git a/neode-ui/src/style.css b/neode-ui/src/style.css index aa1535ea..d3d35d27 100644 --- a/neode-ui/src/style.css +++ b/neode-ui/src/style.css @@ -1730,6 +1730,15 @@ html.kiosk-mode::before { will-change: auto !important; } +/* While the companion app is actively driving input, hide the native OS + pointer — it sits dead wherever the physical mouse left it and reads as + a second cursor next to the companion's virtual one. remote-relay.ts + toggles this class and clears it 30s after the last companion event. */ +html.companion-input-active, +html.companion-input-active * { + cursor: none !important; +} + /* Dashboard: full viewport width, no letterboxing, no body scroll */ body.dashboard-active { overflow: hidden;