From 5fc99e4a26a81a5e896e58d18c3a721178f0a3ea Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 20 Oct 2026 16:47:20 -0700 Subject: [PATCH] Render decoded VM desktops through the existing canvas Safari can decode a live desktop stream while leaving its native video layer black. Paint decoded frames with the existing frame callback and canvas compositor, retaining native video for browsers without frame callbacks. Clear stale pixels on disconnect and redraw on returning to the screen. Validated with real Safari 27 in m3-devbox; TypeScript check and staged dashboard build pass. Assisted-by: gpt-6-astra --- dashboard/web/components/VMConsole.tsx | 17 ++++++++++------- dashboard/web/pages/VMs.css | 3 +-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dashboard/web/components/VMConsole.tsx b/dashboard/web/components/VMConsole.tsx index 962fd36f3181a00387e44a26a49f6b89992421f9..004e921ae914ac14ddcb45565b0106670dcebfbd 100644 --- a/dashboard/web/components/VMConsole.tsx +++ b/dashboard/web/components/VMConsole.tsx @@ -100,13 +100,16 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); }; const compose = () => { const frame = windowFrame; - const composed = !!frame?.windows.length; - video.classList.toggle("vm-video-composed", composed); - canvas.hidden = !composed; - if (!frame || !composed || !width || !height || video.readyState < 2) return; + canvas.hidden = !video.requestVideoFrameCallback || !video.srcObject || video.readyState < 2; + if (canvas.hidden || !props.active || !width || !height) return; if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; } const context = canvas.getContext("2d", { alpha: false }); if (!context) return; + canvas.dataset.windows = String(frame?.windows.length ?? 0); + if (!frame?.windows.length) { + context.drawImage(video, 0, 0, width, height); + return; + } context.fillStyle = "#101014"; context.fillRect(0, 0, width, height); for (const window of [...frame.windows].reverse()) { @@ -118,7 +121,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e } context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height); } - canvas.dataset.windows = String(frame.windows.length); }; const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); }; const connect = async () => { @@ -227,9 +229,9 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined; movement = undefined; pointerSequence = 0; setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined); - compose(); video.onplaying = null; video.srcObject = null; + compose(); clearTimeout(cursorTimer); cursorFrames = []; cursorFrame = 0; @@ -385,6 +387,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e const current = attempt; playbackFrame = requestAnimationFrame(() => { if (disposed || current !== attempt || !props.active || !enabled() || document.hidden || !video.srcObject) return; + compose(); void video.play().catch(() => { if (!disposed && current === attempt && props.active && enabled() && !document.hidden && video.srcObject) { setProblem("The browser couldn't play the screen. Reconnect to try again."); @@ -488,7 +491,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e -