authoravatarclover caruso<git@paperclover.net>2026-10-20 16:47:20-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log5fc99e4a26a81a5e896e58d18c3a721178f0a3ea
treed4474fc40afc2ce44d3b594b5283e798514f9e4e
parent58b46f47f01152bdbe98197d20a6a06064728aec
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Render decoded VM desktops through the existing canvas

Safari can decode a live desktop stream while leaving its native video layer black. Paint decoded frames with the existing frame callback and canvas compositor, retaining native video for browsers without frame callbacks. Clear stale pixels on disconnect and redraw on returning to the screen. Validated with real Safari 27 in m3-devbox; TypeScript check and staged dashboard build pass. Assisted-by: gpt-6-astra

2 files changed, 11 insertions(+), 9 deletions(-)

dashboard/web/components/VMConsole.tsx+10-7
...@@ -100,13 +100,16 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -100,13 +100,16 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
100 const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); };100 const send = (message: object) => { if (input?.readyState === "open") input.send(JSON.stringify(message)); };
101 const compose = () => {101 const compose = () => {
102 const frame = windowFrame;102 const frame = windowFrame;
103 const composed = !!frame?.windows.length;103 canvas.hidden = !video.requestVideoFrameCallback || !video.srcObject || video.readyState < 2;
104 video.classList.toggle("vm-video-composed", composed);104 if (canvas.hidden || !props.active || !width || !height) return;
105 canvas.hidden = !composed;
106 if (!frame || !composed || !width || !height || video.readyState < 2) return;
107 if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; }105 if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; }
108 const context = canvas.getContext("2d", { alpha: false });106 const context = canvas.getContext("2d", { alpha: false });
109 if (!context) return;107 if (!context) return;
108 canvas.dataset.windows = String(frame?.windows.length ?? 0);
109 if (!frame?.windows.length) {
110 context.drawImage(video, 0, 0, width, height);
111 return;
112 }
110 context.fillStyle = "#101014";113 context.fillStyle = "#101014";
111 context.fillRect(0, 0, width, height);114 context.fillRect(0, 0, width, height);
112 for (const window of [...frame.windows].reverse()) {115 for (const window of [...frame.windows].reverse()) {
...@@ -118,7 +121,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -118,7 +121,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
118 }121 }
119 context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height);122 context.drawImage(video, window.atlas_x, window.atlas_y, window.width, window.height, x, y, window.width, window.height);
120 }123 }
121 canvas.dataset.windows = String(frame.windows.length);
122 };124 };
123 const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); };125 const release = () => { buttons = 0; drag = undefined; compose(); send({ type: "release", sequence: ++pointerSequence }); };
124 const connect = async () => {126 const connect = async () => {
...@@ -227,9 +229,9 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -227,9 +229,9 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
227 windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined;229 windowFrame = undefined; pendingFrames = []; frameTimestamp = undefined; drag = undefined;
228 movement = undefined; pointerSequence = 0;230 movement = undefined; pointerSequence = 0;
229 setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined);231 setFrameSync(false); setWindowsAvailable(false); setWindowCapture(false); setStats(undefined); setCapture(undefined);
230 compose();
231 video.onplaying = null;232 video.onplaying = null;
232 video.srcObject = null;233 video.srcObject = null;
234 compose();
233 clearTimeout(cursorTimer);235 clearTimeout(cursorTimer);
234 cursorFrames = [];236 cursorFrames = [];
235 cursorFrame = 0;237 cursorFrame = 0;
...@@ -385,6 +387,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -385,6 +387,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
385 const current = attempt;387 const current = attempt;
386 playbackFrame = requestAnimationFrame(() => {388 playbackFrame = requestAnimationFrame(() => {
387 if (disposed || current !== attempt || !props.active || !enabled() || document.hidden || !video.srcObject) return;389 if (disposed || current !== attempt || !props.active || !enabled() || document.hidden || !video.srcObject) return;
390 compose();
388 void video.play().catch(() => {391 void video.play().catch(() => {
389 if (!disposed && current === attempt && props.active && enabled() && !document.hidden && video.srcObject) {392 if (!disposed && current === attempt && props.active && enabled() && !document.hidden && video.srcObject) {
390 setProblem("The browser couldn't play the screen. Reconnect to try again.");393 setProblem("The browser couldn't play the screen. Reconnect to try again.");
...@@ -488,7 +491,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -488,7 +491,6 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
488 <button class="button small" onClick={() => setBootScreen(true)}>View boot screen</button>491 <button class="button small" onClick={() => setBootScreen(true)}>View boot screen</button>
489 </div>492 </div>
490 </Show>493 </Show>
491 <canvas ref={canvas} class="vm-window-canvas" hidden aria-hidden="true" />
492 <video ref={video} autoplay muted playsinline tabindex={preparing() ? -1 : 0} aria-hidden={preparing()} aria-label="VM screen" aria-describedby={`keyboard-release-${props.name}`}494 <video ref={video} autoplay muted playsinline tabindex={preparing() ? -1 : 0} aria-hidden={preparing()} aria-label="VM screen" aria-describedby={`keyboard-release-${props.name}`}
493 onContextMenu={(event) => event.preventDefault()}495 onContextMenu={(event) => event.preventDefault()}
494 onPointerMove={pointer}496 onPointerMove={pointer}
...@@ -513,6 +515,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e...@@ -513,6 +515,7 @@ export function VMConsole(props: { name: string; platform: Domain["platform"]; e
513 const mask = event.deltaY ? (event.deltaY < 0 ? 8 : 16) : event.deltaX < 0 ? 32 : 64;515 const mask = event.deltaY ? (event.deltaY < 0 ? 8 : 16) : event.deltaX < 0 ? 32 : 64;
514 pointer(event, buttons | mask); pointer(event);516 pointer(event, buttons | mask); pointer(event);
515 }} />517 }} />
518 <canvas ref={canvas} class="vm-screen-canvas" hidden aria-hidden="true" />
516 <img ref={cursorImage} class="vm-cursor" alt="" aria-hidden="true" />519 <img ref={cursorImage} class="vm-cursor" alt="" aria-hidden="true" />
517 <img ref={cursorInvert} class="vm-cursor vm-cursor-invert" alt="" aria-hidden="true" />520 <img ref={cursorInvert} class="vm-cursor vm-cursor-invert" alt="" aria-hidden="true" />
518 </div>521 </div>
dashboard/web/pages/VMs.css+1-2
...@@ -83,8 +83,7 @@...@@ -83,8 +83,7 @@
83.vm-password input { width: 26ch; min-width: 0; padding: 5px 6px; color: var(--text); font: 12px monospace; }83.vm-password input { width: 26ch; min-width: 0; padding: 5px 6px; color: var(--text); font: 12px monospace; }
84.vm-console-screen { flex: 1; background: #101014; min-height: 0; position: relative; }84.vm-console-screen { flex: 1; background: #101014; min-height: 0; position: relative; }
85.vm-console-screen video { display: block; width: 100%; height: 100%; object-fit: contain; outline: none; touch-action: none; }85.vm-console-screen video { display: block; width: 100%; height: 100%; object-fit: contain; outline: none; touch-action: none; }
86.vm-console-screen video.vm-video-composed { opacity: 0; }86.vm-screen-canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; pointer-events: none; background: #101014; }
87.vm-window-canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; pointer-events: none; }
88.vm-stream-stats { display: flex; flex-wrap: wrap; gap: 4px 16px; padding: 6px 12px; font-size: 12px; font-variant-numeric: tabular-nums; }87.vm-stream-stats { display: flex; flex-wrap: wrap; gap: 4px 16px; padding: 6px 12px; font-size: 12px; font-variant-numeric: tabular-nums; }
89.vm-cursor { display: none; position: fixed; left: 0; top: 0; pointer-events: none; }88.vm-cursor { display: none; position: fixed; left: 0; top: 0; pointer-events: none; }
90.vm-cursor-invert { mix-blend-mode: difference; }89.vm-cursor-invert { mix-blend-mode: difference; }