feat(kiosk): companion remote drives app iframes via trusted CDP input
Demo images / Build & push demo images (push) Successful in 3m11s

Companion tap/scroll/type now works INSIDE cross-origin app iframes and
kiosk tabs. The web relay synthesizes untrusted DOM events in the top
document, which can never cross an origin boundary — so apps served
through the appgate were dead to the remote. The kiosk Chromium now
exposes a loopback-only CDP port (default origin check intact, no
--remote-allow-origins) and a backend bridge (api/handler/cdp.rs)
dispatches validated companion input as Input.dispatchKeyEvent /
dispatchMouseEvent / mouseWheel — trusted events that hit-test through
any frame, move real focus, and insert text like a physical device.

- Session keeper self-heals across kiosk Chromium restarts; inert on
  nodes without a kiosk unit (falls back to the existing relay path).
- The kiosk relay subscriber self-tags (?kiosk=1) and the backend mutes
  its key/click/scroll messages while the bridge is live, so input never
  applies twice; cursor moves still flow for the on-screen cursor.
- While companion input is active the native OS pointer is hidden
  (cursor:none, auto-restores 30s after the last event) so the dead
  physical-mouse cursor doesn't sit next to the virtual one.
- docs/tv-input-iframe-apps.md scope note updated: gamepad keys stay on
  uinput; CDP is for companion pointer/typing only.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago
2026-08-16 05:28:53 -04:00
co-authored by Claude Fable 5
parent 876ecc4bdf
commit 3a3077529b
8 changed files with 517 additions and 7 deletions
+25 -2
View File
@@ -53,15 +53,30 @@ let cursorX = typeof window !== 'undefined' ? window.innerWidth / 2 : 0
let cursorY = typeof window !== 'undefined' ? window.innerHeight / 2 : 0
let cursorHideTimer: ReturnType<typeof setTimeout> | null = null
/**
* While the companion is actively driving input, suppress the other visible
* pointer: the native OS cursor sitting wherever the physical mouse left it
* reads as a second, dead cursor next to the companion's virtual one
* (user request 2026-08-16). Restores automatically when the companion goes
* quiet (same 30s window as `companionActive`).
*/
function setNativeCursorSuppressed(on: boolean) {
document.documentElement.classList.toggle('companion-input-active', on)
}
function markCompanionActive() {
companionActive.value = true
companionInputActive.value = true
setNativeCursorSuppressed(true)
if (inputFlickerTimeout) clearTimeout(inputFlickerTimeout)
inputFlickerTimeout = setTimeout(() => { companionInputActive.value = false }, 200)
if (companionTimeout) clearTimeout(companionTimeout)
companionTimeout = setTimeout(() => { companionActive.value = false }, 30_000)
companionTimeout = setTimeout(() => {
companionActive.value = false
setNativeCursorSuppressed(false)
}, 30_000)
}
function createCursor(): HTMLDivElement {
@@ -333,7 +348,14 @@ function doConnect() {
}
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
const url = `${protocol}//${window.location.host}/ws/remote-relay`
// The kiosk self-identifies so the backend can mute this subscriber's
// key/click/scroll messages while the CDP bridge delivers them as trusted
// browser input (cursor moves still arrive so the on-screen cursor draws).
// Same three-way predicate as App.vue's isKiosk.
const isKiosk = localStorage.getItem('kiosk') === 'true'
|| new URLSearchParams(window.location.search).has('kiosk')
|| window.location.pathname === '/kiosk'
const url = `${protocol}//${window.location.host}/ws/remote-relay${isKiosk ? '?kiosk=1' : ''}`
ws = new WebSocket(url)
@@ -405,6 +427,7 @@ export function stopRemoteRelay() {
if (cursorHideTimer) { clearTimeout(cursorHideTimer); cursorHideTimer = null }
if (ws) { ws.onclose = null; ws.close(); ws = null }
if (cursorEl) { cursorEl.remove(); cursorEl = null }
setNativeCursorSuppressed(false)
relayConnected.value = false
companionActive.value = false
companionInputActive.value = false
+9
View File
@@ -1730,6 +1730,15 @@ html.kiosk-mode::before {
will-change: auto !important;
}
/* While the companion app is actively driving input, hide the native OS
pointer — it sits dead wherever the physical mouse left it and reads as
a second cursor next to the companion's virtual one. remote-relay.ts
toggles this class and clears it 30s after the last companion event. */
html.companion-input-active,
html.companion-input-active * {
cursor: none !important;
}
/* Dashboard: full viewport width, no letterboxing, no body scroll */
body.dashboard-active {
overflow: hidden;