1#!/usr/bin/env python3
2import asyncio
3import base64
4import contextlib
5from collections import deque
6import json
7import logging
8import os
9from pathlib import Path
10import signal
11import socket
12import struct
13import threading
14import time
15import zlib
16
17import gi
18
19gi.require_version("Gst", "1.0")
20gi.require_version("GstSdp", "1.0")
21gi.require_version("GstWebRTC", "1.0")
22gi.require_version("GstRtp", "1.0")
23from gi.repository import GLib, Gst, GstRtp, GstSdp, GstWebRTC
24
25
26from importlib import import_module
27
28rfb = import_module("vm-rfb")
29Desktop, png = rfb.Desktop, rfb.png
30STUN = os.environ.get("SNOWGLOBE_VM_STUN_SERVER", "stun://stun.cloudflare.com:3478")
31SLOTS = asyncio.Semaphore(4)
32GUEST_PROTOCOL = json.loads(Path(os.environ.get("SNOWGLOBE_VM_GUEST_PROTOCOL", Path(__file__).resolve().parent.parent / "guest/protocol.json")).read_text())
33GUESTS = {}
34TWCC = "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"
35
36
37class WindowDesktop:
38 def __init__(self, width, height, windows, previous=None):
39 limits = GUEST_PROTOCOL["limits"]
40 if not 0 < width <= limits["dimension"] or not 0 < height <= limits["dimension"] or width * height > limits["pixels"] or not 0 < len(windows) <= limits["windows"]:
41 raise ValueError("Invalid guest window desktop")
42 self.windows = windows
43 self.surfaces = {}
44 self.width = max(width, *(window["width"] for window in windows))
45 x = y = row_height = 0
46 ids = set()
47 for window in sorted(windows, key=lambda window: window["id"]):
48 w, h = window["width"], window["height"]
49 if not window["id"] or window["id"] in ids or not 0 < w <= limits["dimension"] or not 0 < h <= limits["dimension"] or abs(window["x"]) > limits["dimension"] or abs(window["y"]) > limits["dimension"]:
50 raise ValueError("Invalid guest window bounds")
51 ids.add(window["id"])
52 if x + w > self.width:
53 x, y, row_height = 0, y + row_height, 0
54 window["atlas_x"], window["atlas_y"] = x, y
55 x, row_height = x + w, max(row_height, h)
56 surface = previous.surfaces.get(window["id"]) if previous else None
57 self.surfaces[window["id"]] = surface if surface and (surface[0], surface[1]) == (w, h) else (w, h, None)
58 self.height = y + row_height
59 if self.height > limits["dimension"] or self.width * self.height > limits["pixels"]:
60 raise OverflowError("Guest windows exceed the video atlas")
61 self.repacked = not previous or (self.width, self.height, [(w["id"], w["width"], w["height"]) for w in sorted(windows, key=lambda w: w["id"])]) != (
62 previous.width, previous.height, [(w["id"], w["width"], w["height"]) for w in sorted(previous.windows, key=lambda w: w["id"])])
63 self.pixels = bytearray(self.width * self.height * 4) if self.repacked else previous.pixels
64 for window in windows:
65 surface = self.surfaces[window["id"]]
66 if self.repacked and surface[2] is not None:
67 self.copy(window, 0, 0, window["width"], window["height"], surface[2])
68
69 @property
70 def complete(self):
71 return all(surface[2] is not None for surface in self.surfaces.values())
72
73 def copy(self, window, x, y, width, height, pixels):
74 for row in range(height):
75 offset = ((window["atlas_y"] + y + row) * self.width + window["atlas_x"] + x) * 4
76 self.pixels[offset:offset + width * 4] = pixels[row * width * 4:(row + 1) * width * 4]
77
78 def update(self, values, data):
79 window = next((window for window in self.windows if window["id"] == values["id"]), None)
80 if not window:
81 raise ValueError("Unknown guest window")
82 width, height = window["width"], window["height"]
83 x, y, w, h = values["x"], values["y"], values["rect_width"], values["rect_height"]
84 if (values["width"], values["height"]) != (width, height) or not w or not h or x + w > width or y + h > height or len(data) != w * h * 4:
85 raise ValueError("Invalid guest window rectangle")
86 pixels = self.surfaces[window["id"]][2]
87 if pixels is None:
88 if (x, y, w, h) != (0, 0, width, height):
89 raise ValueError("Guest window needs a complete first frame")
90 pixels = bytearray(data)
91 self.surfaces[window["id"]] = (width, height, pixels)
92 else:
93 for row in range(h):
94 offset = ((y + row) * width + x) * 4
95 pixels[offset:offset + w * 4] = data[row * w * 4:(row + 1) * w * 4]
96 self.copy(window, x, y, w, h, data)
97
98
99class GuestDesktop:
100 def __init__(self, name):
101 self.name = name
102 self.screens = set()
103 self.pixels = bytearray()
104 self.cursor = []
105 self.pointer = None
106 self.layers = None
107 self.writer = None
108 self.windows_available = None
109 self.stop_supported = None
110 self.stopping = False
111
112 def request(self, refresh=False):
113 if not self.stopping and self.writer and not self.writer.is_closing():
114 if refresh:
115 self.writer.write(struct.pack("!4sII", GUEST_PROTOCOL["magic"].encode(), GUEST_PROTOCOL["packets"]["refresh"]["id"], 0))
116 if self.windows_available is not None:
117 self.writer.write(struct.pack("!4sIII", GUEST_PROTOCOL["magic"].encode(), GUEST_PROTOCOL["packets"]["stream"]["id"], 4, bool(self.screens)))
118 if self.windows_available:
119 enabled = any(screen.experiment for screen in self.screens)
120 self.writer.write(struct.pack("!4sIII", GUEST_PROTOCOL["magic"].encode(), GUEST_PROTOCOL["packets"]["experiment"]["id"], 4, enabled))
121
122 def stop(self):
123 self.stopping = True
124 if self.writer and not self.writer.is_closing():
125 if self.stop_supported:
126 self.writer.write(struct.pack("!4sII", GUEST_PROTOCOL["magic"].encode(), GUEST_PROTOCOL["packets"]["stop"]["id"], 0))
127 elif self.stop_supported is False:
128 self.writer.close()
129
130 def sync(self, screen):
131 screen.send({"type": "capabilities", "windows": bool(self.windows_available)})
132 if self.pixels:
133 if self.pointer:
134 screen.send(self.pointer)
135 for cursor in self.cursor:
136 screen.send(cursor)
137
138 def suspend(self):
139 active = bool(self.pixels)
140 self.pixels = bytearray()
141 self.cursor.clear()
142 self.pointer = None
143 self.layers = None
144 for screen in tuple(self.screens):
145 screen.send({"type": "capabilities", "windows": False})
146 if active:
147 for screen in tuple(self.screens):
148 screen.resize()
149 screen.send(screen.desktop.cursor or {"type": "cursor", "source": "qemu", "frame": 0, "fallback": True, "x": 0, "y": 0})
150 screen.present()
151
152 async def read(self, reader):
153 limits = GUEST_PROTOCOL["limits"]
154 packets = {packet["id"]: (name, packet["fields"]) for name, packet in GUEST_PROTOCOL["packets"].items()}
155 while True:
156 async with asyncio.timeout(4):
157 magic, kind, length = struct.unpack("!4sII", await reader.readexactly(12))
158 if magic != GUEST_PROTOCOL["magic"].encode() or kind not in packets or length > limits["packet_bytes"]:
159 raise ValueError("Invalid guest display packet")
160 name, fields = packets[kind]
161 if name == "capabilities" and length == 4:
162 fields = fields[:1]
163 if length < len(fields) * 4:
164 raise ValueError("Incomplete guest display packet")
165 payload = await reader.readexactly(length)
166 values = dict(zip(fields, struct.unpack("!" + "I" * len(fields), payload[:len(fields) * 4])))
167 data = payload[len(fields) * 4:]
168 if self.stop_supported is None and name != "capabilities":
169 self.stop_supported = False
170 if self.stopping:
171 return
172 if name == "screen":
173 width, height = values["width"], values["height"]
174 x, y, w, h = values["x"], values["y"], values["rect_width"], values["rect_height"]
175 if not 0 < width <= limits["dimension"] or not 0 < height <= limits["dimension"] or width * height > limits["pixels"] or not w or not h or x + w > width or y + h > height or len(data) != w * h * 4:
176 raise ValueError("Invalid guest display rectangle")
177 resized = not self.pixels or (width, height) != (self.width, self.height)
178 if resized:
179 if (x, y, w, h) != (0, 0, width, height):
180 raise ValueError("Guest display needs a complete first frame")
181 self.width, self.height = width, height
182 self.pixels = bytearray(width * height * 4)
183 for row in range(h):
184 offset = ((y + row) * width + x) * 4
185 self.pixels[offset:offset + w * 4] = data[row * w * 4:(row + 1) * w * 4]
186 for screen in tuple(self.screens):
187 if resized:
188 screen.resize()
189 self.sync(screen)
190 if screen.source is self:
191 screen.present()
192 await asyncio.sleep(0)
193 elif name == "cursor":
194 width, height, count = values["width"], values["height"], values["frames"]
195 if not width and not height and not count and not data and not values["hot_x"] and not values["hot_y"]:
196 self.cursor = [{"type": "cursor", "source": "guest", "id": values["id"], "width": 0, "height": 0, "frame": 0, "frames": 0, "fallback": True, "x": 0, "y": 0}]
197 if self.pixels:
198 for screen in tuple(self.screens):
199 screen.send(self.cursor[0])
200 continue
201 pixels = width * height
202 frame_bytes = 4 + pixels * 5
203 if not 0 < width <= limits["cursor_size"] or not 0 < height <= limits["cursor_size"] or not 0 < count <= limits["cursor_frames"] or values["hot_x"] >= width or values["hot_y"] >= height or len(data) != frame_bytes * count:
204 raise ValueError("Invalid guest cursor")
205 self.cursor.clear()
206 for index in range(count):
207 frame = data[index * frame_bytes:(index + 1) * frame_bytes]
208 duration = struct.unpack("!I", frame[:4])[0]
209 if not 1 <= duration <= 60000:
210 raise ValueError("Invalid guest cursor duration")
211 mask = frame[4 + pixels * 4:]
212 if any(value & ~sum(GUEST_PROTOCOL["xor_bits"].values()) for value in mask):
213 raise ValueError("Invalid guest cursor inversion")
214 message = {"type": "cursor", "source": "guest", "id": values["id"], "width": width, "height": height,
215 "x": values["hot_x"], "y": values["hot_y"], "frame": index, "frames": count,
216 "duration": duration, "image": png(width, height, frame[4:4 + pixels * 4])}
217 if any(mask):
218 bits = GUEST_PROTOCOL["xor_bits"]
219 message["invert"] = png(width, height, b"".join(bytes([255 if value & bits["red"] else 0,
220 255 if value & bits["green"] else 0,
221 255 if value & bits["blue"] else 0,
222 255 if value else 0]) for value in mask))
223 self.cursor.append(message)
224 if self.pixels:
225 for screen in tuple(self.screens):
226 screen.send(message)
227 await asyncio.sleep(0)
228 elif name == "pointer":
229 if data or values["visible"] > 1:
230 raise ValueError("Invalid guest pointer")
231 self.pointer = {"type": "cursor-position", "id": values["id"], "visible": bool(values["visible"]),
232 "x": (values["x"] ^ 0x80000000) - 0x80000000, "y": (values["y"] ^ 0x80000000) - 0x80000000}
233 if self.pixels:
234 for screen in tuple(self.screens):
235 screen.send(self.pointer)
236 elif name == "suspend":
237 if data:
238 raise ValueError("Invalid guest display suspension")
239 self.suspend()
240 elif name == "heartbeat":
241 if data:
242 raise ValueError("Invalid guest display heartbeat")
243 elif name == "stop":
244 if data or not self.stopping:
245 raise ValueError("Unexpected guest display stop")
246 return
247 elif name == "capabilities":
248 if data or values["windows"] > 1 or values.get("stop", 0) > 1:
249 raise ValueError("Invalid guest display capabilities")
250 self.windows_available = bool(values["windows"])
251 self.stop_supported = bool(values.get("stop", 0))
252 for screen in tuple(self.screens):
253 screen.send({"type": "capabilities", "windows": bool(self.windows_available)})
254 if self.stopping:
255 self.stop()
256 else:
257 self.request()
258 elif name == "capture_stats":
259 if data or not 0 < values["elapsed_ms"] <= 60000 or values["frames"] > 10000 or any(values[key] > 60000000 for key in ("capture_us", "scan_us", "write_us")):
260 raise ValueError("Invalid guest capture statistics")
261 for screen in tuple(self.screens):
262 screen.send({"type": "capture-stats", **values})
263 elif name == "windows":
264 count = values["count"]
265 fields = GUEST_PROTOCOL["window_fields"]
266 if count > limits["windows"] or len(data) != count * len(fields) * 4:
267 raise ValueError("Invalid guest window list")
268 previous = self.layers
269 if count:
270 records = []
271 for record in struct.iter_unpack("!" + "I" * len(fields), data):
272 window = dict(zip(fields, record))
273 for key in ("x", "y", "caption_left", "caption_top", "caption_right", "caption_bottom"):
274 window[key] = (window[key] ^ 0x80000000) - 0x80000000
275 records.append(window)
276 try:
277 self.layers = WindowDesktop(values["width"], values["height"], records, previous)
278 except OverflowError:
279 self.layers = None
280 else:
281 self.layers = None
282 for screen in tuple(self.screens):
283 if screen.experiment:
284 if self.layers and not self.layers.repacked and self.layers.complete and screen.source is self.layers:
285 screen.send({"type": "window-positions", "windows": self.layers.windows})
286 else:
287 screen.resize()
288 screen.present()
289 screen.send({"type": "window-capture", "available": bool(self.layers)})
290 elif name == "surface":
291 if self.layers:
292 complete = self.layers.complete
293 self.layers.update(values, data)
294 for screen in tuple(self.screens):
295 if screen.experiment and self.layers.complete:
296 if not complete:
297 screen.resize()
298 screen.present()
299 else:
300 raise ValueError("Unexpected guest display request")
301
302 async def run(self):
303 while True:
304 if self.stopping:
305 GUESTS.pop(self.name, None)
306 return
307 writer = None
308 try:
309 reader, writer = await asyncio.wait_for(asyncio.open_unix_connection(
310 os.environ.get("SNOWGLOBE_HOST_SOCKET", "/run/snowglobe-host/host.sock")), 10)
311 peer = writer.get_extra_info("socket")
312 if struct.unpack("3i", peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))[1] != 0:
313 raise ValueError("Untrusted guest display broker")
314 writer.write(json.dumps({"operation": "vm.guest", "payload": {"name": self.name}}).encode() + b"\n")
315 size = struct.unpack("!I", await asyncio.wait_for(reader.readexactly(4), 70))[0]
316 if size > 65536:
317 raise ValueError("Invalid guest display broker response")
318 response = json.loads(await reader.readexactly(size))
319 if "error" not in response:
320 self.writer = writer
321 writer.write(struct.pack("!4sII", GUEST_PROTOCOL["magic"].encode(), GUEST_PROTOCOL["packets"]["refresh"]["id"], 0))
322 await self.read(reader)
323 except (OSError, ValueError, EOFError, asyncio.IncompleteReadError, TimeoutError) as error:
324 logging.info("Guest display %s: %s", self.name, error)
325 finally:
326 self.writer = None
327 self.windows_available = None
328 self.stop_supported = None
329 self.suspend()
330 if writer:
331 writer.close()
332 with contextlib.suppress(OSError, TimeoutError):
333 await asyncio.wait_for(writer.wait_closed(), 2)
334 if self.stopping or not self.screens:
335 GUESTS.pop(self.name, None)
336 return
337 await asyncio.sleep(2)
338
339
340class Screen:
341 def __init__(self, writer, desktop):
342 self.loop = asyncio.get_running_loop()
343 self.writer, self.desktop = writer, desktop
344 self.guest = None
345 self.channel = None
346 self.pointer_channel = None
347 self.pointer_sequence = -1
348 self.experiment = False
349 self.frame_timer = None
350 self.last_frame = 0
351 self.layouts = {}
352 self.encoding_layouts = deque(maxlen=64)
353 self.last_rtp_pts = None
354 self.sent_layout = None
355 self.bwe = None
356 width, height = self.desktop.width, self.desktop.height
357 ssrc = struct.unpack("!I", os.urandom(4))[0] | 1
358 # TWCC and RTX headers are added after payload packetization.
359 self.pipeline = Gst.parse_launch(
360 f"appsrc name=frames is-live=true format=time max-buffers=1 leaky-type=downstream "
361 f"caps=video/x-raw,format=BGRx,width={width},height={height},framerate=60/1 "
362 f"! videoconvert ! videoscale ! capsfilter name=size caps=video/x-raw,format=NV12,width={width + width % 2},height={height + height % 2} "
363 "! x264enc name=encoder tune=zerolatency speed-preset=ultrafast bitrate=4000 key-int-max=60 bframes=0 "
364 "! video/x-h264,profile=constrained-baseline ! h264parse "
365 f"! rtph264pay name=pay pt=96 ssrc={ssrc} mtu=1190 config-interval=-1 aggregate-mode=zero-latency "
366 f"! capsfilter name=codec caps=application/x-rtp,media=video,encoding-name=H264,clock-rate=90000,ssrc=(uint){ssrc} "
367 "! webrtcbin name=rtc bundle-policy=max-bundle")
368 self.frames = self.pipeline.get_by_name("frames")
369 self.rtc = self.pipeline.get_by_name("rtc")
370 self.ice = self.rtc.get_property("ice-agent")
371 self.ice._ref_sink() # PyGObject sinks the default agent's floating reference, which webrtcbin still owns.
372 self.ice.set_property("min-rtp-port", 50000)
373 self.ice.set_property("max-rtp-port", 50031)
374 if STUN:
375 self.rtc.set_property("stun-server", STUN)
376 self.handlers = [
377 (self.rtc, self.rtc.connect("on-ice-candidate", lambda _, line, candidate: self.loop.call_soon_threadsafe(
378 self.send, {"type": "candidate", "sdpMLineIndex": line, "candidate": candidate}))),
379 (self.rtc, self.rtc.connect("on-data-channel", self.data_channel)),
380 (self.rtc, self.rtc.connect("notify::connection-state", self.connection_state)),
381 (self.rtc, self.rtc.connect("request-aux-sender", lambda *_: self.bwe)),
382 ]
383 self.encoder = self.pipeline.get_by_name("encoder")
384 self.encoder_probe = self.encoder.get_static_pad("sink").add_probe(Gst.PadProbeType.BUFFER, self.encoding)
385 self.rtp_probe = self.pipeline.get_by_name("pay").get_static_pad("src").add_probe(Gst.PadProbeType.BUFFER, self.rtp_layout)
386 self.probe = self.frames.get_static_pad("src").add_probe(Gst.PadProbeType.EVENT_UPSTREAM, self.refresh)
387 self.bus = self.pipeline.get_bus()
388 self.bus.add_signal_watch()
389 self.handlers.append((self.bus, self.bus.connect("message::error", self.pipeline_error)))
390 self.pipeline.set_state(Gst.State.PLAYING)
391 self.send({"type": "display", "width": width, "height": height, "encoder": "x264"})
392
393 def close(self):
394 self.writer.close()
395 if self.frame_timer:
396 self.frame_timer.cancel()
397 self.desktop.notify = None
398 if self.guest:
399 self.guest.screens.discard(self)
400 if self.guest.screens:
401 self.guest.request()
402 else:
403 self.guest.stop()
404 for element, handler in self.handlers:
405 element.disconnect(handler)
406 self.handlers.clear()
407 self.frames.get_static_pad("src").remove_probe(self.probe)
408 self.encoder.get_static_pad("sink").remove_probe(self.encoder_probe)
409 self.pipeline.get_by_name("pay").get_static_pad("src").remove_probe(self.rtp_probe)
410 self.bus.remove_signal_watch()
411 self.pipeline.set_state(Gst.State.NULL)
412 self.pipeline = self.encoder = None
413
414 def send(self, message):
415 if not self.writer.is_closing():
416 self.writer.write(json.dumps(message, separators=(",", ":")).encode() + b"\n")
417 if self.writer.transport.get_write_buffer_size() > 262144:
418 self.writer.close()
419
420 def fail(self, message):
421 self.send({"type": "error", "message": message, "fatal": True})
422 self.writer.close()
423
424 def pipeline_error(self, _, message):
425 error, debug = message.parse_error()
426 logging.error("VM screen stream: %s (%s)", error, debug)
427 self.loop.call_soon_threadsafe(self.fail, "The screen stream stopped. Reconnect to try again.")
428
429 def resize(self):
430 if self.writer.is_closing():
431 return
432 width, height = self.source.width, self.source.height
433 self.frames.set_property("caps", Gst.Caps.from_string(
434 f"video/x-raw,format=BGRx,width={width},height={height},framerate=60/1"))
435 self.pipeline.get_by_name("size").set_property("caps", Gst.Caps.from_string(
436 f"video/x-raw,format=NV12,width={width + width % 2},height={height + height % 2}"))
437 self.send({"type": "display", "width": self.guest.width if self.guest and self.guest.pixels else width,
438 "height": self.guest.height if self.guest and self.guest.pixels else height,
439 "source": "guest" if self.guest and self.guest.pixels else "qemu",
440 "encoder": "x264"})
441
442 def refresh(self, _, info):
443 event = info.get_event().get_structure()
444 if event and event.get_name() == "GstForceKeyUnit":
445 self.loop.call_soon_threadsafe(self.present)
446 return Gst.PadProbeReturn.OK
447
448 def connection_state(self, rtc, _):
449 state = rtc.get_property("connection-state")
450 if state == GstWebRTC.WebRTCPeerConnectionState.CONNECTED:
451 self.loop.call_soon_threadsafe(self.present)
452 self.loop.call_soon_threadsafe(self.desktop.request, False)
453 if self.guest:
454 self.loop.call_soon_threadsafe(self.guest.sync, self)
455 self.loop.call_soon_threadsafe(self.guest.request, True)
456 elif state in (GstWebRTC.WebRTCPeerConnectionState.FAILED, GstWebRTC.WebRTCPeerConnectionState.CLOSED):
457 self.loop.call_soon_threadsafe(self.fail, "The screen disconnected. Reconnect to try again.")
458
459 def data_channel(self, _, channel):
460 label = channel.get_property("label")
461 if label not in ("input", "pointer") or (self.channel if label == "input" else self.pointer_channel):
462 channel.emit("close")
463 return
464 if label == "input":
465 self.channel = channel
466 else:
467 self.pointer_channel = channel
468 self.handlers.append((channel, channel.connect("on-message-string", lambda _, message: self.loop.call_soon_threadsafe(self.input, message, label))))
469
470 def input(self, raw, channel="input"):
471 if self.writer.is_closing():
472 return
473 try:
474 if len(raw) > 131072:
475 raise ValueError("Shorten the clipboard text, then send it again.")
476 message = json.loads(raw)
477 if not isinstance(message, dict):
478 raise ValueError("Unsupported input event.")
479 if channel == "pointer" and message.get("type") != "pointer":
480 raise ValueError("Unsupported pointer event.")
481 if message.get("type") == "experiment":
482 if type(message.get("enabled")) is not bool or not self.guest:
483 raise ValueError("Unsupported window experiment.")
484 self.experiment = message["enabled"] and bool(self.guest.windows_available)
485 self.guest.request()
486 self.resize()
487 self.present()
488 return
489 if message.get("type") in ("pointer", "release") and "sequence" in message:
490 sequence = message["sequence"]
491 if type(sequence) is not int or not 0 <= sequence <= 0x1fffffffffffff:
492 raise ValueError("Unsupported pointer event.")
493 if channel == "pointer" and sequence <= self.pointer_sequence:
494 return
495 self.pointer_sequence = max(sequence, self.pointer_sequence)
496 if channel == "pointer":
497 message["buttons"] = self.desktop.buttons
498 self.desktop.input(message)
499 if self.desktop.writer.transport.get_write_buffer_size() > 262144:
500 self.fail("The VM stopped accepting input. Reconnect to try again.")
501 except (ValueError, TypeError) as error:
502 self.send({"type": "error", "message": str(error)})
503
504 @property
505 def source(self):
506 if self.experiment and self.guest and self.guest.pixels and self.guest.layers and self.guest.layers.complete:
507 return self.guest.layers
508 return self.guest if self.guest and self.guest.pixels else self.desktop
509
510 def present(self):
511 if self.writer.is_closing():
512 return
513 if self.frame_timer is None and self.rtc.get_property("connection-state") == GstWebRTC.WebRTCPeerConnectionState.CONNECTED:
514 self.frame_timer = self.loop.call_later(max(0, self.last_frame + 1 / 60 - time.monotonic()), self.push_frame)
515
516 def push_frame(self):
517 self.frame_timer = None
518 if self.writer.is_closing():
519 return
520 self.last_frame = time.monotonic()
521 source = self.source
522 buffer = Gst.Buffer.new_wrapped(bytes(source.pixels))
523 buffer.pts = self.pipeline.get_clock().get_time() - self.pipeline.get_base_time()
524 buffer.duration = Gst.SECOND // 60
525 layout = {"type": "windows", "windows": source.windows if isinstance(source, WindowDesktop) else [], "width": source.width, "height": source.height}
526 self.layouts[buffer.pts] = layout
527 while len(self.layouts) > 64:
528 del self.layouts[next(iter(self.layouts))]
529 self.frames.emit("push-buffer", buffer)
530
531 def encoding(self, _, info):
532 layout = self.layouts.pop(info.get_buffer().pts, None)
533 if layout is not None:
534 self.encoding_layouts.append(layout)
535 return Gst.PadProbeReturn.OK
536
537 def rtp_layout(self, _, info):
538 buffer = info.get_buffer()
539 if buffer.pts != self.last_rtp_pts and self.encoding_layouts:
540 self.last_rtp_pts = buffer.pts
541 layout = self.encoding_layouts.popleft()
542 if layout != self.sent_layout:
543 self.sent_layout = layout
544 timestamp = struct.unpack("!I", buffer.extract_dup(4, 4))[0]
545 self.loop.call_soon_threadsafe(self.send, {**layout, "rtpTimestamp": timestamp})
546 return Gst.PadProbeReturn.OK
547
548 def signal(self, message):
549 if message.get("type") == "offer" and isinstance(message.get("sdp"), str):
550 if self.rtc.get_property("remote-description"):
551 raise ValueError("Reconnect to start a new screen connection.")
552 result, sdp = GstSdp.SDPMessage.new_from_text(message["sdp"])
553 if result != GstSdp.SDPResult.OK:
554 raise ValueError("Unable to connect the screen. Reconnect to try again.")
555 caps = None
556 for index in range(sdp.medias_len()):
557 media = sdp.get_media(index)
558 if media.get_media() != "video":
559 continue
560 for format_index in range(media.formats_len()):
561 payload = int(media.get_format(format_index))
562 offered = media.get_caps_from_media(payload)
563 if not offered:
564 continue
565 codec = offered.get_structure(0)
566 profile = codec.get_string("profile-level-id")
567 if codec.get_string("encoding-name") == "H264" and codec.get_string("packetization-mode") == "1" and (
568 not profile or profile.startswith("42")):
569 caps = offered
570 break
571 if caps:
572 break
573 if not caps:
574 raise ValueError("This browser doesn't support the VM's H.264 stream. Open it in a current browser.")
575 payload = caps.get_structure(0).get_value("payload")
576 extension = None
577 for index in range(media.attributes_len()):
578 attribute = media.get_attribute(index)
579 if attribute.key == "extmap" and attribute.value.split()[1:2] == [TWCC]:
580 extension = int(attribute.value.split()[0].split("/")[0])
581 if not 1 <= extension <= 255:
582 raise ValueError("Unsupported screen connection extension.")
583 if extension and Gst.ElementFactory.find("rtpgccbwe"):
584 self.bwe = Gst.ElementFactory.make("rtpgccbwe")
585 self.bwe.set_property("min-bitrate", 500000)
586 self.bwe.set_property("max-bitrate", 12000000)
587 self.bwe.set_property("estimated-bitrate", 4000000)
588 self.handlers.append((self.bwe, self.bwe.connect("notify::estimated-bitrate", lambda bwe, _: self.encoder.set_property("bitrate", bwe.get_property("estimated-bitrate") // 1000))))
589 header = GstRtp.RTPHeaderExtension.create_from_uri(TWCC)
590 header.set_id(extension)
591 self.pipeline.get_by_name("pay").emit("add-extension", header)
592 caps = Gst.Caps.from_string(
593 f"application/x-rtp,media=video,encoding-name=H264,clock-rate=90000,payload={payload},"
594 f"ssrc=(uint){self.pipeline.get_by_name('pay').get_property('ssrc')},"
595 "packetization-mode=(string)1,rtcp-fb-nack=(boolean)true,rtcp-fb-nack-pli=(boolean)true")
596 if self.bwe:
597 caps.set_value("rtcp-fb-transport-cc", True)
598 caps.set_value(f"extmap-{extension}", TWCC)
599 self.pipeline.get_by_name("pay").set_property("pt", payload)
600 self.pipeline.get_by_name("codec").set_property("caps", caps)
601 transceiver = self.rtc.get_static_pad("sink_0").get_property("transceiver")
602 preferences = caps.copy()
603 for index in range(media.formats_len()):
604 offered = media.get_caps_from_media(int(media.get_format(index)))
605 if offered:
606 codec = offered.get_structure(0)
607 if codec.get_string("encoding-name") == "RTX" and codec.get_string("apt") == str(payload):
608 preferences.append(offered)
609 transceiver.set_property("codec-preferences", preferences)
610 transceiver.set_property("direction", GstWebRTC.WebRTCRTPTransceiverDirection.SENDONLY)
611 transceiver.set_property("do-nack", True)
612 description = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.OFFER, sdp)
613 promise = Gst.Promise.new_with_change_func(self.remote_description, None, None)
614 self.rtc.emit("set-remote-description", description, promise)
615 elif message.get("type") == "candidate":
616 line, candidate = message.get("sdpMLineIndex"), message.get("candidate")
617 if type(line) is not int or not 0 <= line <= 16 or not isinstance(candidate, str):
618 raise ValueError("Unsupported screen connection candidate.")
619 self.rtc.emit("add-ice-candidate", line, candidate)
620 else:
621 raise ValueError("Unsupported screen connection message.")
622
623 def remote_description(self, promise, *_):
624 if self.writer.is_closing():
625 return
626 reply = promise.get_reply()
627 if reply and reply.has_field("error"):
628 logging.error("VM screen offer: %s", reply.to_string())
629 self.loop.call_soon_threadsafe(self.fail, "Unable to connect the screen. Reconnect to try again.")
630 return
631 self.rtc.emit("create-answer", None, Gst.Promise.new_with_change_func(self.answer, None, None))
632
633 def answer(self, promise, *_):
634 if self.writer.is_closing():
635 return
636 reply = promise.get_reply()
637 if not reply or not reply.has_field("answer"):
638 logging.error("VM screen answer: %s", reply.to_string() if reply else "missing reply")
639 self.loop.call_soon_threadsafe(self.fail, "Unable to connect the screen. Reconnect to try again.")
640 return
641 answer = reply.get_value("answer")
642 self.rtc.emit("set-local-description", answer, Gst.Promise.new())
643 self.loop.call_soon_threadsafe(self.send, {"type": "answer", "sdp": answer.sdp.as_text()})
644
645
646async def serve(reader, writer):
647 desktop = screen = task = None
648 if SLOTS.locked():
649 writer.write(b'{"type":"error","fatal":true,"message":"Four VM screens are already open. Close one, then reconnect."}\n')
650 writer.close()
651 return
652 async with SLOTS:
653 try:
654 peer = writer.get_extra_info("socket")
655 if struct.unpack("3i", peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))[1] not in (0, os.getuid()):
656 return
657 name = json.loads(await asyncio.wait_for(reader.readline(), 10))["name"]
658 if not isinstance(name, str) or not name or len(name) > 63:
659 raise ValueError("Choose a VM from the list.")
660 host_reader, host_writer = await asyncio.wait_for(asyncio.open_unix_connection(
661 os.environ.get("SNOWGLOBE_HOST_SOCKET", "/run/snowglobe-host/host.sock")), 10)
662 desktop = Desktop(host_reader, host_writer, lambda message: screen.send(message) if message["type"] != "cursor" or screen.source is desktop else None)
663 host_peer = host_writer.get_extra_info("socket")
664 if struct.unpack("3i", host_peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))[1] != 0:
665 raise ValueError("The host identity couldn't be verified. Check its configuration.")
666 host_writer.write(json.dumps({"operation": "vm.console", "payload": {"name": name}}).encode() + b"\n")
667 size = struct.unpack("!I", await asyncio.wait_for(host_reader.readexactly(4), 70))[0]
668 if size > 65536:
669 raise ValueError("The host screen response was too large.")
670 result = json.loads(await host_reader.readexactly(size))
671 if "error" in result:
672 raise ValueError(result["error"])
673 await asyncio.wait_for(desktop.start(), 10)
674 screen = Screen(writer, desktop)
675 if (result.get("value") or {}).get("guestDisplay", True):
676 guest = GUESTS.get(name)
677 if guest and guest.stopping:
678 await asyncio.shield(guest.task)
679 if name not in GUESTS:
680 GUESTS[name] = GuestDesktop(name)
681 GUESTS[name].task = asyncio.create_task(GUESTS[name].run())
682 screen.guest = GUESTS[name]
683 screen.guest.screens.add(screen)
684 if screen.guest.pixels:
685 screen.resize()
686 if screen.guest.pointer:
687 screen.send(screen.guest.pointer)
688 for cursor in screen.guest.cursor:
689 screen.send(cursor)
690 screen.send({"type": "ready", "iceServers": [{"urls": STUN.replace("stun://", "stun:")}] if STUN else []})
691 task = asyncio.create_task(desktop.read(lambda: screen.present() if screen.source is desktop else None,
692 lambda: screen.resize() if screen.source is desktop else None))
693 def captured(done):
694 if done.cancelled():
695 return
696 error = done.exception()
697 if error:
698 logging.error("VM screen capture: %s", error)
699 screen.fail(str(error) if isinstance(error, ValueError) else
700 "The VM screen stopped. Check that the VM is running, then reconnect.")
701 task.add_done_callback(captured)
702 async with asyncio.timeout(8 * 3600):
703 while raw := await reader.readline():
704 message = json.loads(raw)
705 if not isinstance(message, dict):
706 raise ValueError("Unsupported screen connection message.")
707 screen.signal(message)
708 if task.done():
709 task.result()
710 except ValueError as error:
711 if screen:
712 screen.send({"type": "error", "message": str(error), "fatal": True})
713 else:
714 writer.write(json.dumps({"type": "error", "message": str(error), "fatal": True}).encode() + b"\n")
715 except (OSError, EOFError, TimeoutError, GLib.Error, asyncio.IncompleteReadError):
716 logging.exception("VM screen connection stopped")
717 if screen:
718 screen.send({"type": "error", "fatal": True, "message": "Unable to connect the VM screen. Reconnect to try again."})
719 finally:
720 if task:
721 task.cancel()
722 with contextlib.suppress(asyncio.CancelledError, Exception):
723 await task
724 if screen:
725 screen.close()
726 if desktop:
727 with contextlib.suppress(OSError):
728 desktop.input({"type": "release"})
729 desktop.writer.close()
730 try:
731 await asyncio.wait_for(desktop.writer.wait_closed(), 2)
732 except (OSError, TimeoutError):
733 desktop.writer.transport.abort()
734 writer.close()
735 try:
736 await asyncio.wait_for(writer.wait_closed(), 2)
737 except (OSError, TimeoutError):
738 writer.transport.abort()
739
740
741async def main():
742 Gst.init(None)
743 glib = GLib.MainLoop()
744 threading.Thread(target=glib.run, daemon=True).start()
745 stopped = asyncio.Event()
746 loop = asyncio.get_running_loop()
747 for signum in (signal.SIGTERM, signal.SIGINT):
748 loop.add_signal_handler(signum, stopped.set)
749 clients = set()
750 def connected(reader, writer):
751 if stopped.is_set():
752 writer.close()
753 return
754 task = asyncio.create_task(serve(reader, writer))
755 clients.add(task)
756 task.add_done_callback(clients.discard)
757 path = Path(os.environ.get("SNOWGLOBE_VM_SCREEN_SOCKET", "/run/snowglobe-vm-screen/screen.sock"))
758 path.unlink(missing_ok=True)
759 server = await asyncio.start_unix_server(connected, path=str(path), limit=131072)
760 path.chmod(0o600)
761 await stopped.wait()
762 server.close()
763 guests = tuple(GUESTS.values())
764 for guest in guests:
765 guest.stop()
766 tasks = tuple(clients)
767 for task in tasks:
768 task.cancel()
769 await asyncio.gather(*tasks, return_exceptions=True)
770 await server.wait_closed()
771 await asyncio.gather(*(guest.task for guest in guests), return_exceptions=True)
772 glib.quit()
773
774
775if __name__ == "__main__":
776 asyncio.run(main())