diff --git a/images/chromium-headful/client/src/app.vue b/images/chromium-headful/client/src/app.vue index 20f3c8e8..0481fbc5 100644 --- a/images/chromium-headful/client/src/app.vue +++ b/images/chromium-headful/client/src/app.vue @@ -13,6 +13,8 @@ ref="video" :hideControls="hideControls" :extraControls="isEmbedMode" + :showDomOverlay="showDomOverlay" + :domSyncTypes="domSyncTypes" @control-attempt="controlAttempt" /> @@ -186,6 +188,7 @@ import About from '~/components/about.vue' import Header from '~/components/header.vue' import Unsupported from '~/components/unsupported.vue' + import { DomSyncPayload, DomWebSocketMessage, DomElementType, DOM_ELEMENT_TYPES } from '~/neko/dom-types' @Component({ name: 'neko', @@ -208,6 +211,41 @@ shakeKbd = false wasConnected = false + private domWebSocket: WebSocket | null = null + private domReconnectTimeout: number | null = null + + // dom_sync: enables WebSocket connection for DOM element syncing + // Values: false (default), true (inputs only), or comma-separated types (e.g., "inputs,buttons,links") + get isDomSyncEnabled(): boolean { + const params = new URL(location.href).searchParams + const param = params.get('dom_sync') || params.get('domSync') + if (!param || param === 'false' || param === '0') return false + return true + } + + // Get enabled DOM element types from query param + // ?dom_sync=true -> ['inputs'] (default, backwards compatible) + // ?dom_sync=inputs,buttons,links -> ['inputs', 'buttons', 'links'] + get domSyncTypes(): DomElementType[] { + const params = new URL(location.href).searchParams + const param = params.get('dom_sync') || params.get('domSync') + if (!param || param === 'false' || param === '0') return [] + // If true/1, default to inputs only (backwards compatible) + if (param === 'true' || param === '1') return ['inputs'] + // Parse comma-separated list and filter to valid types + const types = param.split(',').map((t) => t.trim().toLowerCase()) as DomElementType[] + return types.filter((t) => DOM_ELEMENT_TYPES.includes(t)) + } + + // dom_overlay: shows purple overlay rectangles when dom_sync is enabled (default: true) + get showDomOverlay() { + if (!this.isDomSyncEnabled) return false + const params = new URL(location.href).searchParams + const param = params.get('dom_overlay') || params.get('domOverlay') + // Default to true if not specified, false only if explicitly set to 'false' or '0' + if (param === null) return true + return param !== 'false' && param !== '0' + } get volume() { const numberParam = parseFloat(new URL(location.href).searchParams.get('volume') || '1.0') @@ -278,6 +316,10 @@ if (value) { this.wasConnected = true this.applyQueryResolution() + // Connect to DOM sync if enabled + if (this.isDomSyncEnabled) { + this.connectDomSync() + } try { if (window.parent !== window) { window.parent.postMessage({ type: 'KERNEL_CONNECTED', connected: true }, this.parentOrigin) @@ -285,6 +327,9 @@ } catch (e) { console.error('Failed to post message to parent', e) } + } else { + // Disconnect DOM sync when main connection is lost + this.disconnectDomSync() } } @@ -331,6 +376,92 @@ // KERNEL: end custom resolution, frame rate, and readOnly control via query params + // KERNEL: DOM Sync - connects to kernel-images API WebSocket for bounding box overlay + private domRetryCount = 0 + private readonly domMaxRetries = 10 + + private connectDomSync() { + if (!this.isDomSyncEnabled) return + if (this.domWebSocket && this.domWebSocket.readyState === WebSocket.OPEN) return + + const params = new URL(location.href).searchParams + const domPort = params.get('dom_port') || '444' + const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:' + const domUrl = `${protocol}//${location.hostname}:${domPort}/dom-sync` + + console.log(`[dom-sync] Connecting to ${domUrl} (attempt ${this.domRetryCount + 1})`) + + try { + this.domWebSocket = new WebSocket(domUrl) + + this.domWebSocket.onopen = () => { + console.log('[dom-sync] Connected') + this.domRetryCount = 0 // Reset retry count on success + this.$accessor.dom.setEnabled(true) + this.$accessor.dom.setConnected(true) + } + + this.domWebSocket.onmessage = (event) => { + try { + const message: DomWebSocketMessage = JSON.parse(event.data) + if (message.event === 'dom/sync' && message.data) { + this.$accessor.dom.applySync(message.data) + } + } catch (e) { + console.error('[dom-sync] Failed to parse message:', e) + } + } + + this.domWebSocket.onclose = () => { + console.log('[dom-sync] Disconnected') + this.$accessor.dom.setConnected(false) + this.scheduleDomReconnect() + } + + this.domWebSocket.onerror = (error) => { + console.error('[dom-sync] WebSocket error:', error) + // Error will trigger onclose, which handles reconnect + } + } catch (e) { + console.error('[dom-sync] Failed to connect:', e) + this.scheduleDomReconnect() + } + } + + private scheduleDomReconnect() { + if (!this.isDomSyncEnabled || !this.connected) return + if (this.domRetryCount >= this.domMaxRetries) { + console.log('[dom-sync] Max retries reached, giving up') + return + } + // Exponential backoff: 500ms, 1s, 2s, 4s... capped at 5s + const delay = Math.min(500 * Math.pow(2, this.domRetryCount), 5000) + this.domRetryCount++ + console.log(`[dom-sync] Reconnecting in ${delay}ms`) + this.domReconnectTimeout = window.setTimeout(() => { + this.connectDomSync() + }, delay) + } + + private disconnectDomSync() { + if (this.domReconnectTimeout) { + clearTimeout(this.domReconnectTimeout) + this.domReconnectTimeout = null + } + if (this.domWebSocket) { + this.domWebSocket.close() + this.domWebSocket = null + } + this.domRetryCount = 0 + this.$accessor.dom.setEnabled(false) + this.$accessor.dom.setConnected(false) + this.$accessor.dom.reset() + } + + beforeDestroy() { + this.disconnectDomSync() + } + controlAttempt() { if (this.shakeKbd || this.$accessor.remote.hosted) return diff --git a/images/chromium-headful/client/src/components/dom-overlay.vue b/images/chromium-headful/client/src/components/dom-overlay.vue new file mode 100644 index 00000000..22f8d765 --- /dev/null +++ b/images/chromium-headful/client/src/components/dom-overlay.vue @@ -0,0 +1,176 @@ + + + + + diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue index 4c4fe0f8..e8f923ca 100644 --- a/images/chromium-headful/client/src/components/video.vue +++ b/images/chromium-headful/client/src/components/video.vue @@ -27,6 +27,7 @@ @touchstart.stop.prevent="onTouchHandler" @touchend.stop.prevent="onTouchHandler" /> +