1#!/usr/bin/env python3
2import concurrent.futures
3import contextlib
4import fcntl
5import importlib.util
6import json
7import math
8import os
9from pathlib import Path
10import pwd
11import re
12import selectors
13import select
14import stat
15import signal
16import socket
17import struct
18import subprocess
19import sys
20import threading
21import time
22import vms
23import tty
24
25spec = importlib.util.spec_from_file_location("copyparty", Path(__file__).resolve().parent.parent / "service/copyparty/manage.py")
26copyparty = importlib.util.module_from_spec(spec)
27spec.loader.exec_module(copyparty)
28
29
30dashboard_runs = importlib.import_module("dashboard-run")
31shale_labels = importlib.import_module("shale-labels")
32
33
34FIELDS = (
35 "name", "used", "usedbydataset", "usedbysnapshots", "referenced", "available",
36 "compressratio", "logicalused", "compression", "recordsize", "mountpoint",
37 "quota", "origin", "mounted",
38)
39TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"}
40PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text())
41MAX_REQUEST = PROTOCOL['request']
42MAX_RESPONSE = PROTOCOL['response']
43STREAM_SLOTS = threading.BoundedSemaphore(4)
44GUEST_SLOTS = threading.BoundedSemaphore(4)
45CLONE_SLOTS = threading.BoundedSemaphore(2)
46VM_PREPARATION_SLOT = threading.BoundedSemaphore(1)
47INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?"
48
49
50class Rejected(Exception):
51 pass
52
53
54class Host:
55 def __init__(self, pool):
56 if not re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_.:-]*", pool):
57 raise ValueError("incorrect storage pool")
58 self.pool = pool
59 self.containers = None
60 self.inventory_lock = threading.Lock()
61
62 def dataset(self, value):
63 if (not isinstance(value, str) or len(value) > 255
64 or not re.fullmatch(r"[\w.: ][\w.: -]*(/[\w.: -]+)*", value)
65 or any(part in {".", ".."} for part in value.split("/"))
66 or not (value == self.pool or value.startswith(self.pool + "/"))):
67 raise Rejected("Choose a dataset from this storage pool.")
68 return value
69
70 def snapshot(self, value):
71 if (not isinstance(value, str) or len(value) > 255
72 or not re.fullmatch(r"[\w.: ][\w.: -]*", value)
73 or value in {".", ".."}):
74 raise Rejected("Choose a snapshot from the list.")
75 return value
76
77 def handle(self, request):
78 if not isinstance(request, dict):
79 raise Rejected("Choose a supported host operation.")
80 operation = request.get("operation")
81 parameters = {
82 "storage.datasets": set(),
83 "storage.pool": set(),
84 "storage.mounts": set(),
85 "storage.snapshots": {"dataset"},
86 "storage.reclaim": {"dataset", "from", "to"},
87 "storage.destroy": {"dataset", "from", "to"},
88 "storage.removed": {"dataset", "snapshot"},
89 "files.snapshot": {"datasets"},
90 "files.snapshots": {"datasets"},
91 "files.discard": {"dataset", "snapshot"},
92 "index.snapshot": {"dataset"},
93 "index.snapshots": {"dataset"},
94 "index.diff": {"dataset", "from", "to"},
95 "index.discard": {"dataset", "snapshot"},
96 "shale.metadata": {"repository", "accounts"},
97 "shale.access": {"service", "uri", "account", "authorization"},
98 "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"},
99 "shale.write": {"grant", "repository", "action", "arguments"},
100 "host.sample": set(),
101 "host.usage": {"refresh"},
102 "copyparty.manage": {"action", "payload", "users"},
103 "vm.ai": {"payload"},
104 **dashboard_runs.FIELDS,
105 **{"vm." + action: set() if fields is None else {"payload"} for action, fields in vms.ACTION_FIELDS.items()},
106 }
107 if not isinstance(operation, str) or operation not in parameters:
108 raise Rejected("Choose a supported host operation.")
109 if set(request) != parameters[operation] | {"operation"}:
110 raise Rejected("Use only the fields required by this host operation.")
111 if operation == "copyparty.manage":
112 return copyparty.handle(request)
113 if operation == "shale.access":
114 return importlib.import_module("shale-access").access(request)
115 if operation == "shale.clone":
116 return importlib.import_module("shale-access").clone(request)
117 if operation == "shale.metadata":
118 return shale_labels.metadata(request)
119 if operation == "shale.write":
120 return shale_labels.handle(request)
121 if operation in dashboard_runs.FIELDS:
122 return dashboard_runs.handle(request)
123 if operation == "host.sample":
124 stat = Path("/proc/stat").read_text().splitlines()
125 # Guest columns are already included in user and nice time.
126 cpu = list(map(int, stat[0].split()[1:9]))
127 total = sum(cpu)
128 memory = {line.split()[0].rstrip(":"): int(line.split()[1]) * 1024
129 for line in Path("/proc/meminfo").read_text().splitlines()}
130 arc = None
131 try:
132 arc = next(int(line.split()[-1]) for line in Path("/proc/spl/kstat/zfs/arcstats").read_text().splitlines() if line.split()[0] == "size")
133 except (OSError, StopIteration):
134 pass
135 temperature = None
136 for root in Path("/sys/class/hwmon").glob("hwmon*"):
137 try:
138 if root.joinpath("name").read_text().strip() in {"k10temp", "coretemp", "zenpower"}:
139 value = float(root.joinpath("temp1_input").read_text()) / 1000
140 if math.isfinite(value) and value > 0:
141 temperature = value
142 break
143 except (OSError, ValueError):
144 pass
145 rx, tx = 0, 0
146 for line in Path("/proc/net/dev").read_text().splitlines():
147 name, separator, values = line.partition(":")
148 if separator and name.strip().startswith(("en", "eth", "wl")):
149 values = values.split()
150 rx += int(values[0])
151 tx += int(values[8])
152 gpu = None
153 try:
154 values = [float(line) for line in command("nvidia-smi", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits", timeout=3).splitlines()]
155 values = [value for value in values if math.isfinite(value)]
156 if values:
157 gpu = sum(values) / len(values)
158 except (OSError, ValueError, subprocess.SubprocessError, TimeoutError):
159 pass
160 return {"at": time.monotonic(), "cpu": {"busy": total - cpu[3] - cpu[4], "total": total},
161 "cores": sum(bool(re.match(r"cpu[0-9]+\b", line)) for line in stat),
162 "memory": {"used": memory["MemTotal"] - memory["MemAvailable"], "total": memory["MemTotal"]},
163 "bootedAt": time.time() - float(Path("/proc/uptime").read_text().split()[0]),
164 "load": os.getloadavg()[1], "arc": arc, "temperature": temperature, "gpu": gpu,
165 "network": {"rx": rx, "tx": tx}}
166 if operation == "host.usage":
167 if type(request["refresh"]) is not bool:
168 raise Rejected("Choose whether to refresh the container list.")
169 with self.inventory_lock:
170 if request["refresh"] or not self.containers:
171 self.containers = json.loads(command("podman", "--remote", "--url", "unix:///run/podman/podman.sock", "ps", "--format", "json", timeout=10))
172 rows = []
173 for container in self.containers:
174 pid, identity = container["Pid"], container["Id"]
175 if not isinstance(pid, int) or pid <= 0 or not re.fullmatch(r"[0-9a-f]{64}", identity):
176 continue
177 try:
178 cgroup = next(line[3:] for line in Path(f"/proc/{pid}/cgroup").read_text().splitlines() if line.startswith("0::"))
179 if f"libpod-{identity}.scope" not in Path(cgroup).parts:
180 raise ValueError("container cgroup changed")
181 root = Path("/sys/fs/cgroup") / cgroup.lstrip("/")
182 cpu = next(int(line.split()[1]) for line in root.joinpath("cpu.stat").read_text().splitlines() if line.startswith("usage_usec "))
183 memory = int(root.joinpath("memory.current").read_text())
184 rows.append({"id": identity, "name": container["Names"][0], "cpu": cpu, "memory": memory})
185 except (OSError, ValueError, StopIteration):
186 self.containers = None
187 return {"at": time.monotonic(), "containers": rows}
188 if operation == "vm.ai":
189 if not VM_PREPARATION_SLOT.acquire(blocking=False):
190 raise Rejected("A test VM operation is running. Wait for it to finish.")
191 try:
192 process = subprocess.run(["python3", str(Path(__file__).with_name("vm-ai.py"))],
193 input=json.dumps(request["payload"]), capture_output=True, text=True, timeout=900)
194 if process.returncode == 2:
195 raise Rejected(process.stderr.strip())
196 process.check_returncode()
197 return json.loads(process.stdout)
198 finally:
199 VM_PREPARATION_SLOT.release()
200 if operation.startswith("vm."):
201 action = operation[3:]
202 payload = request.get("payload")
203 try:
204 vms.validate(action, payload)
205 except ValueError as error:
206 raise Rejected(str(error)) from error
207 if action == "computer":
208 try:
209 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"])
210 except BlockingIOError:
211 raise dashboard_runs.Error(409, "Another computer operation is running. Wait for it to finish.") from None
212 except ValueError as error:
213 raise Rejected(str(error)) from error
214 if action == "clipboard":
215 if not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"]:
216 raise Rejected("This VM does not support this guest operation.")
217 try:
218 return vms.mac_vms.handle(action, payload)
219 except ValueError as error:
220 raise Rejected(str(error)) from error
221 args = ["python3", str(Path(__file__).with_name("vms.py")), action]
222 if payload is not None:
223 args.append(json.dumps(payload))
224 try:
225 if action in {"create", "media", "preset"}:
226 if not VM_PREPARATION_SLOT.acquire(blocking=False):
227 raise Rejected("A VM image is being prepared. Wait for it to finish, then retry.")
228 try:
229 return json.loads(command(*args, timeout=900))
230 finally:
231 VM_PREPARATION_SLOT.release()
232 return json.loads(command(*args, timeout=230 if action == "act" else 60))
233 except subprocess.CalledProcessError as error:
234 if error.returncode == 2:
235 raise Rejected(error.stderr) from error
236 raise
237 if operation in {"files.snapshot", "files.snapshots"}:
238 values = request["datasets"]
239 if not isinstance(values, list) or not values:
240 raise Rejected("Choose at least one dataset.")
241 datasets = sorted({self.dataset(value) for value in values})
242 if operation == "files.snapshot":
243 name = f"dash-{time.time_ns()}"
244 command("zfs", "snapshot", *(f"{dataset}@{name}" for dataset in datasets))
245 return name
246 text = command("zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-d", "1", *datasets)
247 return [name for name in text.splitlines() if re.fullmatch(r"dash-[0-9]+", name.partition("@")[2])]
248 dataset = self.dataset(request["dataset"]) if "dataset" in request else None
249 if operation.startswith("index."):
250 if operation == "index.snapshot":
251 snapshot = f"{dataset}@index-{time.time_ns()}"
252 command("zfs", "snapshot", snapshot)
253 return snapshot
254 if operation == "index.snapshots":
255 text = command("zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-d", "1", dataset)
256 return [name for name in text.splitlines() if re.fullmatch(INDEX_SNAPSHOT, name.partition("@")[2])]
257 names = [request["from"], request["to"]] if operation == "index.diff" else [request["snapshot"]]
258 if any(not re.fullmatch(INDEX_SNAPSHOT, self.snapshot(name)) for name in names):
259 raise Rejected("Choose an index snapshot.")
260 snapshots = [f"{dataset}@{name}" for name in names]
261 if operation == "index.diff":
262 return command("zfs", "diff", "-FH", *snapshots)
263 command("zfs", "destroy", "-d", *snapshots)
264 return None
265 if operation == "storage.datasets":
266 text = command("zfs", "list", "-Hp", "-r", "-t", "filesystem",
267 "-o", ",".join(FIELDS), self.pool)
268 rows = []
269 for line in text.splitlines():
270 values = line.split("\t")
271 if len(values) != len(FIELDS):
272 raise ValueError("unexpected ZFS dataset response")
273 row = dict(zip(FIELDS, values))
274 for field in FIELDS:
275 if field not in TEXT_FIELDS:
276 row[field] = numeric(row[field])
277 if row.pop("mounted") != "yes" or row["mountpoint"] == "none":
278 row["mountpoint"] = None
279 if row["origin"] == "-":
280 row["origin"] = None
281 rows.append(row)
282 return rows
283 if operation == "storage.pool":
284 fields = ("size", "allocated", "free", "fragmentation")
285 summary = command("zpool", "list", "-Hp", "-o", ",".join(fields), self.pool)
286 status = json.loads(command("zpool", "status", "-jp", "--json-int", self.pool))
287 return {"summary": dict(zip(fields, map(numeric, summary.strip().split("\t")))),
288 "status": status, "name": self.pool}
289 if operation == "storage.mounts":
290 mounts = json.loads(command("findmnt", "-J", "-l", "-t", "zfs", "-o", "SOURCE,TARGET"))
291 return {"filesystems": [item for item in mounts.get("filesystems", [])
292 if "@" not in item["source"]
293 and (item["source"] == self.pool or item["source"].startswith(self.pool + "/"))]}
294 if operation == "storage.snapshots":
295 text = command("zfs", "list", "-Hp", "-t", "snapshot", "-d", "1", "-s", "creation",
296 "-o", "name,creation,used,referenced,clones", dataset)
297 rows = []
298 for line in text.splitlines():
299 name, creation, used, referenced, clones = line.split("\t")
300 rows.append({"name": name.split("@", 1)[1], "creation": numeric(creation),
301 "used": numeric(used), "referenced": numeric(referenced),
302 "clones": [] if clones == "-" else clones.split(",")})
303 return rows
304 if operation in {"storage.reclaim", "storage.destroy"}:
305 start, end = self.snapshot(request["from"]), self.snapshot(request["to"])
306 flags = ["-nvp"] if operation == "storage.reclaim" else []
307 return command("zfs", "destroy", *flags, f"{dataset}@{start}%{end}")
308 snapshot = self.snapshot(request["snapshot"])
309 if operation == "files.discard":
310 if not re.fullmatch(r"dash-[0-9]+", snapshot):
311 raise Rejected("Choose a file undo snapshot.")
312 command("zfs", "destroy", f"{dataset}@{snapshot}")
313 return None
314 return command("zfs", "diff", "-H", f"{dataset}@{snapshot}", dataset)
315
316
317def numeric(value):
318 try:
319 return float(value.rstrip("x"))
320 except ValueError:
321 return 0
322
323
324def command(*args, timeout=60):
325 with subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
326 stderr=subprocess.PIPE, start_new_session=True) as process:
327 output, errors = bytearray(), bytearray()
328 deadline = time.monotonic() + timeout
329 try:
330 with selectors.DefaultSelector() as selector:
331 selector.register(process.stdout, selectors.EVENT_READ, output)
332 selector.register(process.stderr, selectors.EVENT_READ, errors)
333 while selector.get_map():
334 remaining = deadline - time.monotonic()
335 if remaining <= 0:
336 raise TimeoutError("host command timed out")
337 for key, _ in selector.select(remaining):
338 chunk = os.read(key.fd, 65536)
339 if not chunk:
340 selector.unregister(key.fileobj)
341 continue
342 key.data.extend(chunk)
343 if len(output) + len(errors) > MAX_RESPONSE // 2:
344 raise RuntimeError("The host response is too large. Narrow the selection.")
345 process.wait(timeout=max(0.001, deadline - time.monotonic()))
346 except BaseException:
347 try:
348 os.killpg(process.pid, signal.SIGKILL)
349 except ProcessLookupError:
350 pass
351 raise
352 if process.returncode:
353 raise subprocess.CalledProcessError(process.returncode, args, stderr=errors.decode(errors="replace").strip())
354 return output.decode()
355
356
357def send_response(connection, response):
358 payload = json.dumps(response, allow_nan=False).encode()
359 if len(payload) > MAX_RESPONSE:
360 payload = b'{"error":"The host response is too large. Narrow the selection.","status":502}'
361 connection.sendall(struct.pack("!I", len(payload)) + payload)
362
363
364def stream_console(connection, host, request):
365 details = host.handle(request)
366 with contextlib.ExitStack() as resources:
367 if request["operation"] == "vm.serial":
368 descriptor = os.open(details["path"], os.O_RDWR | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK)
369 screen = resources.enter_context(os.fdopen(descriptor, "r+b", buffering=0))
370 if not stat.S_ISCHR(os.fstat(descriptor).st_mode):
371 raise Rejected("This VM's serial console is unavailable. Restart the VM and reconnect.")
372 try:
373 fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
374 except BlockingIOError:
375 raise Rejected("This console is already open. Close its other console tab, then reconnect.") from None
376 tty.setraw(descriptor)
377 elif request["operation"] == "vm.guest":
378 screen = resources.enter_context(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM))
379 screen.settimeout(10)
380 screen.connect(details["path"])
381 screen.setblocking(False)
382 elif details.get("remote"):
383 _, screen = resources.enter_context(vms.mac_vms.connect({"action": "console"}, timeout=20))
384 screen.setblocking(False)
385 else:
386 identity = details["uuid"]
387 deadline = time.monotonic() + 5
388 while True:
389 try:
390 screen = resources.enter_context(socket.create_connection(("127.0.0.1", details["port"]), timeout=10))
391 break
392 except ConnectionRefusedError:
393 if time.monotonic() >= deadline:
394 raise Rejected("This VM's screen isn't ready. Wait a moment, then reconnect.") from None
395 time.sleep(0.1)
396 details = host.handle(request)
397 if details["uuid"] != identity:
398 raise Rejected("The VM changed while its screen was opening. Open it again.")
399 screen.setblocking(False)
400 send_response(connection, {"value": {"guestDisplay": False} if details.get("remote") else None})
401 try:
402 connection.setblocking(False)
403 with selectors.DefaultSelector() as selector:
404 selector.register(connection, selectors.EVENT_READ, screen)
405 selector.register(screen, selectors.EVENT_READ, connection)
406 deadline = time.monotonic() + 8 * 3600
407 while time.monotonic() < deadline:
408 ready = selector.select(300)
409 if not ready:
410 continue
411 for key, _ in ready:
412 try:
413 data = os.read(key.fd, 65536)
414 except BlockingIOError:
415 continue
416 if not data:
417 return
418 remaining = memoryview(data)
419 stalled = time.monotonic() + 30
420 while remaining:
421 try:
422 written = os.write(key.data.fileno(), remaining)
423 if written == 0:
424 return
425 remaining = remaining[written:]
426 except BlockingIOError:
427 if not select.select([], [key.data], [], max(0, stalled - time.monotonic()))[1]:
428 return
429 except OSError:
430 return
431
432
433def stream_upload(connection, request):
434 if set(request) != {"operation", "payload"}:
435 raise Rejected("Use only the fields required to upload an ISO.")
436 spec = request["payload"]
437 try:
438 vms.validate("upload", spec)
439 except ValueError as error:
440 raise Rejected(str(error)) from error
441 directory = vms.UPLOADS
442 vms.managed_directory(directory)
443 target = directory / spec["volume"]
444 if target.exists() or target.is_symlink():
445 raise Rejected("An ISO already has this filename. Rename the file before uploading.")
446 import tempfile
447 fd, temporary = tempfile.mkstemp(prefix=".upload-", dir=directory)
448 try:
449 send_response(connection, {"value": None})
450 connection.settimeout(120)
451 deadline = time.monotonic() + 2 * 3600
452 with os.fdopen(fd, "wb") as outgoing:
453 remaining = spec["size"]
454 while remaining:
455 if time.monotonic() > deadline:
456 raise TimeoutError("ISO upload timed out. Try uploading again.")
457 data = connection.recv(min(1024 * 1024, remaining))
458 if not data:
459 raise Rejected("The ISO upload was interrupted. Upload the file again.")
460 outgoing.write(data)
461 remaining -= len(data)
462 outgoing.flush()
463 os.fsync(outgoing.fileno())
464 # Never overwrite a file that another upload published in the meantime.
465 os.link(temporary, target)
466 send_response(connection, {"value": {"volume": vms.image_id(target)}})
467 finally:
468 Path(temporary).unlink(missing_ok=True)
469
470
471def stream_clone(connection, plan):
472 owner = os.stat(plan["objects"])
473 # Service identities have numeric owners without passwd entries, which systemd's User= rejects.
474 command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec",
475 *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes",
476 "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")),
477 *(f"--setenv={name}={value}" for name, value in plan["environment"].items()),
478 "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--",
479 sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]]
480 send_response(connection, {"value": None})
481 connection.setblocking(True)
482 subprocess.run(command, stdin=connection, stdout=connection)
483
484
485def serve_connection(connection, host, allowed_uid, allowed_operations=None):
486 with connection:
487 connection.settimeout(65)
488 _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
489 if uid != allowed_uid:
490 return
491 try:
492 # Leave stream payload bytes on the socket after the JSON header.
493 line = bytearray()
494 while len(line) <= MAX_REQUEST:
495 chunk = connection.recv(min(65536, MAX_REQUEST + 1 - len(line)), socket.MSG_PEEK)
496 if not chunk:
497 break
498 end = chunk.find(b"\n")
499 chunk = connection.recv(len(chunk) if end < 0 else end + 1)
500 line.extend(chunk)
501 if line.endswith(b"\n"):
502 break
503 if len(line) > MAX_REQUEST or not line.endswith(b"\n"):
504 raise Rejected("The host request is too large or incomplete.")
505 request = json.loads(line)
506 if allowed_operations is not None and (
507 not isinstance(request, dict) or not isinstance(request.get("operation"), str)
508 or request["operation"] not in allowed_operations
509 ):
510 raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.")
511 if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}:
512 streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS
513 if not streams.acquire(blocking=False):
514 raise Rejected("Four VM connections or uploads are open. Close one and try again.")
515 try:
516 if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}:
517 stream_console(connection, host, request)
518 else:
519 stream_upload(connection, request)
520 return
521 finally:
522 streams.release()
523 if isinstance(request, dict) and request.get("operation") == "shale.clone":
524 if not CLONE_SLOTS.acquire(blocking=False):
525 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
526 try:
527 stream_clone(connection, host.handle(request))
528 return
529 finally:
530 CLONE_SLOTS.release()
531 response = {"value": host.handle(request)}
532 except Rejected as error:
533 response = {"error": str(error), "status": 400}
534 except dashboard_runs.Error as error:
535 response = {"error": str(error), "status": error.status}
536 except (json.JSONDecodeError, UnicodeDecodeError):
537 response = {"error": "Send the host request as UTF-8 JSON.", "status": 400}
538 except Exception as error:
539 message = error.stderr if isinstance(error, subprocess.CalledProcessError) else str(error)
540 print(f"host operation: {message}", flush=True)
541 if isinstance(error, subprocess.CalledProcessError) and any(line.endswith(": snapshot has dependent clones") for line in message.splitlines()):
542 response = {"error": "A snapshot has clones. Delete its clones before deleting the snapshot.", "status": 409}
543 else:
544 response = {"error": "The host operation couldn't finish. Check its logs, then retry.", "status": 502}
545 try:
546 send_response(connection, response)
547 except (BrokenPipeError, ConnectionResetError):
548 pass
549
550
551def main():
552 host = Host(os.environ["SNOWGLOBE_POOL"])
553 allowed_uid = pwd.getpwnam(os.environ["SNOWGLOBE_DASHBOARD_USER"]).pw_uid
554 socket_path = Path(os.environ.get("SNOWGLOBE_HOST_SOCKET", "/run/snowglobe-host/host.sock"))
555 socket_path.unlink(missing_ok=True)
556 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
557 listener.bind(str(socket_path))
558 socket_path.chmod(0o660)
559 listener.listen(8)
560 if address := os.environ.get("NOTIFY_SOCKET"):
561 with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification:
562 notification.connect("\0" + address[1:] if address.startswith("@") else address)
563 notification.sendall(b"READY=1")
564 slots = threading.BoundedSemaphore(16)
565 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:
566 while True:
567 connection, _ = listener.accept()
568 if not slots.acquire(blocking=False):
569 connection.close()
570 continue
571 work = workers.submit(serve_connection, connection, host, allowed_uid)
572 work.add_done_callback(lambda _: slots.release())
573
574
575if __name__ == "__main__":
576 main()