authoravatarclover caruso<git@paperclover.net>2026-10-20 16:47:20-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log5fc99e4a26a81a5e896e58d18c3a721178f0a3ea
treed4474fc40afc2ce44d3b594b5283e798514f9e4e
parent58b46f47f01152bdbe98197d20a6a06064728aec
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Render decoded VM desktops through the existing canvas

Safari can decode a live desktop stream while leaving its native video layer black. Paint decoded frames with the existing frame callback and canvas compositor, retaining native video for browsers without frame callbacks. Clear stale pixels on disconnect and redraw on returning to the screen. Validated with real Safari 27 in m3-devbox; TypeScript check and staged dashboard build pass. Assisted-by: gpt-6-astra

2 files changed, 11 insertions(+), 9 deletions(-)

dashboard/web/components/VMConsole.tsx+10-7
......@@ -100,13 +100,16 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
100100 const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); };
101101 const compose = () => {
102102 const frame = windowFrame;
103 const composed = !!frame?.windows.length;
104 video.classList.toggle("vm-video-composed", composed);
105 canvas.hidden = !composed;
106 if (!frame || !composed || !width || !height || video.readyState < 2) return;
103 canvas.hidden = !video.requestVideoFrameCallback || !video.srcObject || video.readyState < 2;
104 if (canvas.hidden || !props.active || !width || !height) return;
107105 if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; }
108106 const context = canvas.getContext("2d", { alpha: false });
109107 if (!context) return;
108 canvas.dataset.windows = String(frame?.windows.length ?? 0);
109 if (!frame?.windows.length) {
110 context.drawImage(video, 0, 0, width, height);
111 return;
112 }
110113 context.fillStyle = "#101014";
111114 context.fillRect(0, 0, width, height);
112115 for (const window of [...frame.windows].reverse()) {
......@@ -118,7 +121,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
118121 }
119122 context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height);
120123 }
121 canvas.dataset.windows = String(frame.windows.length);
122124 };
123125 const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); };
124126 const connect = async () => {
......@@ -227,9 +229,9 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
227229 windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined;
228230 movement = undefined; pointerSequence = 0;
229231 setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined);
230 compose();
231232 video.onplaying = null;
232233 video.srcObject = null;
234 compose();
233235 clearTimeout(cursorTimer);
234236 cursorFrames = [];
235237 cursorFrame = 0;
......@@ -385,6 +387,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
385387 const current = attempt;
386388 playbackFrame = requestAnimationFrame(() => {
387389 if (disposed || current !== attempt || !props.active || !enabled() || document.hidden || !video.srcObject) return;
390 compose();
388391 void video.play().catch(() => {
389392 if (!disposed && current === attempt && props.active && enabled() && !document.hidden && video.srcObject) {
390393 setProblem("The browser couldn't play the screen. Reconnect to try again.");
......@@ -488,7 +491,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
488491 <button class="button small" onClick={() => setBootScreen(true)}>View boot screen</button>
489492 </div>
490493 </Show>
491 <canvas ref={canvas} class="vm-window-canvas" hidden aria-hidden="true" />
492494 <video ref={video} autoplay muted playsinline tabindex={preparing() ? -1 : 0} aria-hidden={preparing()} aria-label="VM screen" aria-describedby={`keyboard-release-${props.name}`}
493495 onContextMenu={(event) => event.preventDefault()}
494496 onPointerMove={pointer}
......@@ -513,6 +515,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
513515 const mask = event.deltaY ? (event.deltaY < 0 ? 8 : 16) : event.deltaX < 0 ? 32 : 64;
514516 pointer(event, buttons | mask); pointer(event);
515517 }} />
518 <canvas ref={canvas} class="vm-screen-canvas" hidden aria-hidden="true" />
516519 <img ref={cursorImage} class="vm-cursor" alt="" aria-hidden="true" />
517520 <img ref={cursorInvert} class="vm-cursor vm-cursor-invert" alt="" aria-hidden="true" />
518521 </div>
dashboard/web/pages/VMs.css+1-2
......@@ -83,8 +83,7 @@
8383.vm-password input { width: 26ch; min-width: 0; padding: 5px 6px; color: var(--text); font: 12px monospace; }
8484.vm-console-screen { flex: 1; background: #101014; min-height: 0; position: relative; }
8585.vm-console-screen video { display: block; width: 100%; height: 100%; object-fit: contain; outline: none; touch-action: none; }
86.vm-console-screen video.vm-video-composed { opacity: 0; }
87.vm-window-canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; pointer-events: none; }
86.vm-screen-canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; pointer-events: none; background: #101014; }
8887.vm-stream-stats { display: flex; flex-wrap: wrap; gap: 4px 16px; padding: 6px 12px; font-size: 12px; font-variant-numeric: tabular-nums; }
8988.vm-cursor { display: none; position: fixed; left: 0; top: 0; pointer-events: none; }
9089.vm-cursor-invert { mix-blend-mode: difference; }