feat(kiosk): companion remote drives app iframes via trusted CDP input
Demo images / Build & push demo images (push) Successful in 3m11s
Demo images / Build & push demo images (push) Successful in 3m11s
Companion tap/scroll/type now works INSIDE cross-origin app iframes and kiosk tabs. The web relay synthesizes untrusted DOM events in the top document, which can never cross an origin boundary — so apps served through the appgate were dead to the remote. The kiosk Chromium now exposes a loopback-only CDP port (default origin check intact, no --remote-allow-origins) and a backend bridge (api/handler/cdp.rs) dispatches validated companion input as Input.dispatchKeyEvent / dispatchMouseEvent / mouseWheel — trusted events that hit-test through any frame, move real focus, and insert text like a physical device. - Session keeper self-heals across kiosk Chromium restarts; inert on nodes without a kiosk unit (falls back to the existing relay path). - The kiosk relay subscriber self-tags (?kiosk=1) and the backend mutes its key/click/scroll messages while the bridge is live, so input never applies twice; cursor moves still flow for the on-screen cursor. - While companion input is active the native OS pointer is hidden (cursor:none, auto-restores 30s after the last event) so the dead physical-mouse cursor doesn't sit next to the virtual one. - docs/tv-input-iframe-apps.md scope note updated: gamepad keys stay on uinput; CDP is for companion pointer/typing only. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
876ecc4bdf
commit
3a3077529b
@@ -53,15 +53,30 @@ let cursorX = typeof window !== 'undefined' ? window.innerWidth / 2 : 0
|
||||
let cursorY = typeof window !== 'undefined' ? window.innerHeight / 2 : 0
|
||||
let cursorHideTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
/**
|
||||
* While the companion is actively driving input, suppress the other visible
|
||||
* pointer: the native OS cursor sitting wherever the physical mouse left it
|
||||
* reads as a second, dead cursor next to the companion's virtual one
|
||||
* (user request 2026-08-16). Restores automatically when the companion goes
|
||||
* quiet (same 30s window as `companionActive`).
|
||||
*/
|
||||
function setNativeCursorSuppressed(on: boolean) {
|
||||
document.documentElement.classList.toggle('companion-input-active', on)
|
||||
}
|
||||
|
||||
function markCompanionActive() {
|
||||
companionActive.value = true
|
||||
companionInputActive.value = true
|
||||
setNativeCursorSuppressed(true)
|
||||
|
||||
if (inputFlickerTimeout) clearTimeout(inputFlickerTimeout)
|
||||
inputFlickerTimeout = setTimeout(() => { companionInputActive.value = false }, 200)
|
||||
|
||||
if (companionTimeout) clearTimeout(companionTimeout)
|
||||
companionTimeout = setTimeout(() => { companionActive.value = false }, 30_000)
|
||||
companionTimeout = setTimeout(() => {
|
||||
companionActive.value = false
|
||||
setNativeCursorSuppressed(false)
|
||||
}, 30_000)
|
||||
}
|
||||
|
||||
function createCursor(): HTMLDivElement {
|
||||
@@ -333,7 +348,14 @@ function doConnect() {
|
||||
}
|
||||
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||
const url = `${protocol}//${window.location.host}/ws/remote-relay`
|
||||
// The kiosk self-identifies so the backend can mute this subscriber's
|
||||
// key/click/scroll messages while the CDP bridge delivers them as trusted
|
||||
// browser input (cursor moves still arrive so the on-screen cursor draws).
|
||||
// Same three-way predicate as App.vue's isKiosk.
|
||||
const isKiosk = localStorage.getItem('kiosk') === 'true'
|
||||
|| new URLSearchParams(window.location.search).has('kiosk')
|
||||
|| window.location.pathname === '/kiosk'
|
||||
const url = `${protocol}//${window.location.host}/ws/remote-relay${isKiosk ? '?kiosk=1' : ''}`
|
||||
|
||||
ws = new WebSocket(url)
|
||||
|
||||
@@ -405,6 +427,7 @@ export function stopRemoteRelay() {
|
||||
if (cursorHideTimer) { clearTimeout(cursorHideTimer); cursorHideTimer = null }
|
||||
if (ws) { ws.onclose = null; ws.close(); ws = null }
|
||||
if (cursorEl) { cursorEl.remove(); cursorEl = null }
|
||||
setNativeCursorSuppressed(false)
|
||||
relayConnected.value = false
|
||||
companionActive.value = false
|
||||
companionInputActive.value = false
|
||||
|
||||
@@ -1730,6 +1730,15 @@ html.kiosk-mode::before {
|
||||
will-change: auto !important;
|
||||
}
|
||||
|
||||
/* While the companion app is actively driving input, hide the native OS
|
||||
pointer — it sits dead wherever the physical mouse left it and reads as
|
||||
a second cursor next to the companion's virtual one. remote-relay.ts
|
||||
toggles this class and clears it 30s after the last companion event. */
|
||||
html.companion-input-active,
|
||||
html.companion-input-active * {
|
||||
cursor: none !important;
|
||||
}
|
||||
|
||||
/* Dashboard: full viewport width, no letterboxing, no body scroll */
|
||||
body.dashboard-active {
|
||||
overflow: hidden;
|
||||
|
||||
Reference in New Issue
Block a user