| 1 | #!/usr/bin/env python3 |
| 2 | import concurrent.futures |
| 3 | import contextlib |
| 4 | import fcntl |
| 5 | import importlib.util |
| 6 | import json |
| 7 | import math |
| 8 | import os |
| 9 | from pathlib import Path |
| 10 | import pwd |
| 11 | import re |
| 12 | import selectors |
| 13 | import select |
| 14 | import stat |
| 15 | import signal |
| 16 | import socket |
| 17 | import struct |
| 18 | import subprocess |
| 19 | import sys |
| 20 | import threading |
| 21 | import time |
| 22 | import vms |
| 23 | import tty |
| 24 | |
| 25 | spec = importlib.util.spec_from_file_location("copyparty", Path(__file__).resolve().parent.parent / "service/copyparty/manage.py") |
| 26 | copyparty = importlib.util.module_from_spec(spec) |
| 27 | spec.loader.exec_module(copyparty) |
| 28 | |
| 29 | |
| 30 | dashboard_runs = importlib.import_module("dashboard-run") |
| 31 | shale_labels = importlib.import_module("shale-labels") |
| 32 | |
| 33 | |
| 34 | FIELDS = ( |
| 35 | "name", "used", "usedbydataset", "usedbysnapshots", "referenced", "available", |
| 36 | "compressratio", "logicalused", "compression", "recordsize", "mountpoint", |
| 37 | "quota", "origin", "mounted", |
| 38 | ) |
| 39 | TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"} |
| 40 | PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text()) |
| 41 | MAX_REQUEST = PROTOCOL['request'] |
| 42 | MAX_RESPONSE = PROTOCOL['response'] |
| 43 | STREAM_SLOTS = threading.BoundedSemaphore(4) |
| 44 | GUEST_SLOTS = threading.BoundedSemaphore(4) |
| 45 | CLONE_SLOTS = threading.BoundedSemaphore(2) |
| 46 | VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) |
| 47 | INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" |
| 48 | |
| 49 | |
| 50 | class Rejected(Exception): |
| 51 | pass |
| 52 | |
| 53 | |
| 54 | class Host: |
| 55 | def __init__(self, pool): |
| 56 | if not re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_.:-]*", pool): |
| 57 | raise ValueError("incorrect storage pool") |
| 58 | self.pool = pool |
| 59 | self.containers = None |
| 60 | self.inventory_lock = threading.Lock() |
| 61 | |
| 62 | def dataset(self, value): |
| 63 | if (not isinstance(value, str) or len(value) > 255 |
| 64 | or not re.fullmatch(r"[\w.: ][\w.: -]*(/[\w.: -]+)*", value) |
| 65 | or any(part in {".", ".."} for part in value.split("/")) |
| 66 | or not (value == self.pool or value.startswith(self.pool + "/"))): |
| 67 | raise Rejected("Choose a dataset from this storage pool.") |
| 68 | return value |
| 69 | |
| 70 | def snapshot(self, value): |
| 71 | if (not isinstance(value, str) or len(value) > 255 |
| 72 | or not re.fullmatch(r"[\w.: ][\w.: -]*", value) |
| 73 | or value in {".", ".."}): |
| 74 | raise Rejected("Choose a snapshot from the list.") |
| 75 | return value |
| 76 | |
| 77 | def handle(self, request): |
| 78 | if not isinstance(request, dict): |
| 79 | raise Rejected("Choose a supported host operation.") |
| 80 | operation = request.get("operation") |
| 81 | parameters = { |
| 82 | "storage.datasets": set(), |
| 83 | "storage.pool": set(), |
| 84 | "storage.mounts": set(), |
| 85 | "storage.snapshots": {"dataset"}, |
| 86 | "storage.reclaim": {"dataset", "from", "to"}, |
| 87 | "storage.destroy": {"dataset", "from", "to"}, |
| 88 | "storage.removed": {"dataset", "snapshot"}, |
| 89 | "files.snapshot": {"datasets"}, |
| 90 | "files.snapshots": {"datasets"}, |
| 91 | "files.discard": {"dataset", "snapshot"}, |
| 92 | "index.snapshot": {"dataset"}, |
| 93 | "index.snapshots": {"dataset"}, |
| 94 | "index.diff": {"dataset", "from", "to"}, |
| 95 | "index.discard": {"dataset", "snapshot"}, |
| 96 | "shale.metadata": {"repository", "accounts"}, |
| 97 | "shale.access": {"service", "uri", "account", "authorization"}, |
| 98 | "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, |
| 99 | "shale.write": {"grant", "repository", "action", "arguments"}, |
| 100 | "host.sample": set(), |
| 101 | "host.usage": {"refresh"}, |
| 102 | "copyparty.manage": {"action", "payload", "users"}, |
| 103 | "vm.ai": {"payload"}, |
| 104 | **dashboard_runs.FIELDS, |
| 105 | **{"vm." + action: set() if fields is None else {"payload"} for action, fields in vms.ACTION_FIELDS.items()}, |
| 106 | } |
| 107 | if not isinstance(operation, str) or operation not in parameters: |
| 108 | raise Rejected("Choose a supported host operation.") |
| 109 | if set(request) != parameters[operation] | {"operation"}: |
| 110 | raise Rejected("Use only the fields required by this host operation.") |
| 111 | if operation == "copyparty.manage": |
| 112 | return copyparty.handle(request) |
| 113 | if operation == "shale.access": |
| 114 | return importlib.import_module("shale-access").access(request) |
| 115 | if operation == "shale.clone": |
| 116 | return importlib.import_module("shale-access").clone(request) |
| 117 | if operation == "shale.metadata": |
| 118 | return shale_labels.metadata(request) |
| 119 | if operation == "shale.write": |
| 120 | return shale_labels.handle(request) |
| 121 | if operation in dashboard_runs.FIELDS: |
| 122 | return dashboard_runs.handle(request) |
| 123 | if operation == "host.sample": |
| 124 | stat = Path("/proc/stat").read_text().splitlines() |
| 125 | # Guest columns are already included in user and nice time. |
| 126 | cpu = list(map(int, stat[0].split()[1:9])) |
| 127 | total = sum(cpu) |
| 128 | memory = {line.split()[0].rstrip(":"): int(line.split()[1]) * 1024 |
| 129 | for line in Path("/proc/meminfo").read_text().splitlines()} |
| 130 | arc = None |
| 131 | try: |
| 132 | arc = next(int(line.split()[-1]) for line in Path("/proc/spl/kstat/zfs/arcstats").read_text().splitlines() if line.split()[0] == "size") |
| 133 | except (OSError, StopIteration): |
| 134 | pass |
| 135 | temperature = None |
| 136 | for root in Path("/sys/class/hwmon").glob("hwmon*"): |
| 137 | try: |
| 138 | if root.joinpath("name").read_text().strip() in {"k10temp", "coretemp", "zenpower"}: |
| 139 | value = float(root.joinpath("temp1_input").read_text()) / 1000 |
| 140 | if math.isfinite(value) and value > 0: |
| 141 | temperature = value |
| 142 | break |
| 143 | except (OSError, ValueError): |
| 144 | pass |
| 145 | rx, tx = 0, 0 |
| 146 | for line in Path("/proc/net/dev").read_text().splitlines(): |
| 147 | name, separator, values = line.partition(":") |
| 148 | if separator and name.strip().startswith(("en", "eth", "wl")): |
| 149 | values = values.split() |
| 150 | rx += int(values[0]) |
| 151 | tx += int(values[8]) |
| 152 | gpu = None |
| 153 | try: |
| 154 | values = [float(line) for line in command("nvidia-smi", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits", timeout=3).splitlines()] |
| 155 | values = [value for value in values if math.isfinite(value)] |
| 156 | if values: |
| 157 | gpu = sum(values) / len(values) |
| 158 | except (OSError, ValueError, subprocess.SubprocessError, TimeoutError): |
| 159 | pass |
| 160 | return {"at": time.monotonic(), "cpu": {"busy": total - cpu[3] - cpu[4], "total": total}, |
| 161 | "cores": sum(bool(re.match(r"cpu[0-9]+\b", line)) for line in stat), |
| 162 | "memory": {"used": memory["MemTotal"] - memory["MemAvailable"], "total": memory["MemTotal"]}, |
| 163 | "bootedAt": time.time() - float(Path("/proc/uptime").read_text().split()[0]), |
| 164 | "load": os.getloadavg()[1], "arc": arc, "temperature": temperature, "gpu": gpu, |
| 165 | "network": {"rx": rx, "tx": tx}} |
| 166 | if operation == "host.usage": |
| 167 | if type(request["refresh"]) is not bool: |
| 168 | raise Rejected("Choose whether to refresh the container list.") |
| 169 | with self.inventory_lock: |
| 170 | if request["refresh"] or not self.containers: |
| 171 | self.containers = json.loads(command("podman", "--remote", "--url", "unix:///run/podman/podman.sock", "ps", "--format", "json", timeout=10)) |
| 172 | rows = [] |
| 173 | for container in self.containers: |
| 174 | pid, identity = container["Pid"], container["Id"] |
| 175 | if not isinstance(pid, int) or pid <= 0 or not re.fullmatch(r"[0-9a-f]{64}", identity): |
| 176 | continue |
| 177 | try: |
| 178 | cgroup = next(line[3:] for line in Path(f"/proc/{pid}/cgroup").read_text().splitlines() if line.startswith("0::")) |
| 179 | if f"libpod-{identity}.scope" not in Path(cgroup).parts: |
| 180 | raise ValueError("container cgroup changed") |
| 181 | root = Path("/sys/fs/cgroup") / cgroup.lstrip("/") |
| 182 | cpu = next(int(line.split()[1]) for line in root.joinpath("cpu.stat").read_text().splitlines() if line.startswith("usage_usec ")) |
| 183 | memory = int(root.joinpath("memory.current").read_text()) |
| 184 | rows.append({"id": identity, "name": container["Names"][0], "cpu": cpu, "memory": memory}) |
| 185 | except (OSError, ValueError, StopIteration): |
| 186 | self.containers = None |
| 187 | return {"at": time.monotonic(), "containers": rows} |
| 188 | if operation == "vm.ai": |
| 189 | if not VM_PREPARATION_SLOT.acquire(blocking=False): |
| 190 | raise Rejected("A test VM operation is running. Wait for it to finish.") |
| 191 | try: |
| 192 | process = subprocess.run(["python3", str(Path(__file__).with_name("vm-ai.py"))], |
| 193 | input=json.dumps(request["payload"]), capture_output=True, text=True, timeout=900) |
| 194 | if process.returncode == 2: |
| 195 | raise Rejected(process.stderr.strip()) |
| 196 | process.check_returncode() |
| 197 | return json.loads(process.stdout) |
| 198 | finally: |
| 199 | VM_PREPARATION_SLOT.release() |
| 200 | if operation.startswith("vm."): |
| 201 | action = operation[3:] |
| 202 | payload = request.get("payload") |
| 203 | try: |
| 204 | vms.validate(action, payload) |
| 205 | except ValueError as error: |
| 206 | raise Rejected(str(error)) from error |
| 207 | if action == "computer": |
| 208 | try: |
| 209 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) |
| 210 | except BlockingIOError: |
| 211 | raise dashboard_runs.Error(409, "Another computer operation is running. Wait for it to finish.") from None |
| 212 | except ValueError as error: |
| 213 | raise Rejected(str(error)) from error |
| 214 | if action == "clipboard": |
| 215 | if not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"]: |
| 216 | raise Rejected("This VM does not support this guest operation.") |
| 217 | try: |
| 218 | return vms.mac_vms.handle(action, payload) |
| 219 | except ValueError as error: |
| 220 | raise Rejected(str(error)) from error |
| 221 | args = ["python3", str(Path(__file__).with_name("vms.py")), action] |
| 222 | if payload is not None: |
| 223 | args.append(json.dumps(payload)) |
| 224 | try: |
| 225 | if action in {"create", "media", "preset"}: |
| 226 | if not VM_PREPARATION_SLOT.acquire(blocking=False): |
| 227 | raise Rejected("A VM image is being prepared. Wait for it to finish, then retry.") |
| 228 | try: |
| 229 | return json.loads(command(*args, timeout=900)) |
| 230 | finally: |
| 231 | VM_PREPARATION_SLOT.release() |
| 232 | return json.loads(command(*args, timeout=230 if action == "act" else 60)) |
| 233 | except subprocess.CalledProcessError as error: |
| 234 | if error.returncode == 2: |
| 235 | raise Rejected(error.stderr) from error |
| 236 | raise |
| 237 | if operation in {"files.snapshot", "files.snapshots"}: |
| 238 | values = request["datasets"] |
| 239 | if not isinstance(values, list) or not values: |
| 240 | raise Rejected("Choose at least one dataset.") |
| 241 | datasets = sorted({self.dataset(value) for value in values}) |
| 242 | if operation == "files.snapshot": |
| 243 | name = f"dash-{time.time_ns()}" |
| 244 | command("zfs", "snapshot", *(f"{dataset}@{name}" for dataset in datasets)) |
| 245 | return name |
| 246 | text = command("zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-d", "1", *datasets) |
| 247 | return [name for name in text.splitlines() if re.fullmatch(r"dash-[0-9]+", name.partition("@")[2])] |
| 248 | dataset = self.dataset(request["dataset"]) if "dataset" in request else None |
| 249 | if operation.startswith("index."): |
| 250 | if operation == "index.snapshot": |
| 251 | snapshot = f"{dataset}@index-{time.time_ns()}" |
| 252 | command("zfs", "snapshot", snapshot) |
| 253 | return snapshot |
| 254 | if operation == "index.snapshots": |
| 255 | text = command("zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-d", "1", dataset) |
| 256 | return [name for name in text.splitlines() if re.fullmatch(INDEX_SNAPSHOT, name.partition("@")[2])] |
| 257 | names = [request["from"], request["to"]] if operation == "index.diff" else [request["snapshot"]] |
| 258 | if any(not re.fullmatch(INDEX_SNAPSHOT, self.snapshot(name)) for name in names): |
| 259 | raise Rejected("Choose an index snapshot.") |
| 260 | snapshots = [f"{dataset}@{name}" for name in names] |
| 261 | if operation == "index.diff": |
| 262 | return command("zfs", "diff", "-FH", *snapshots) |
| 263 | command("zfs", "destroy", "-d", *snapshots) |
| 264 | return None |
| 265 | if operation == "storage.datasets": |
| 266 | text = command("zfs", "list", "-Hp", "-r", "-t", "filesystem", |
| 267 | "-o", ",".join(FIELDS), self.pool) |
| 268 | rows = [] |
| 269 | for line in text.splitlines(): |
| 270 | values = line.split("\t") |
| 271 | if len(values) != len(FIELDS): |
| 272 | raise ValueError("unexpected ZFS dataset response") |
| 273 | row = dict(zip(FIELDS, values)) |
| 274 | for field in FIELDS: |
| 275 | if field not in TEXT_FIELDS: |
| 276 | row[field] = numeric(row[field]) |
| 277 | if row.pop("mounted") != "yes" or row["mountpoint"] == "none": |
| 278 | row["mountpoint"] = None |
| 279 | if row["origin"] == "-": |
| 280 | row["origin"] = None |
| 281 | rows.append(row) |
| 282 | return rows |
| 283 | if operation == "storage.pool": |
| 284 | fields = ("size", "allocated", "free", "fragmentation") |
| 285 | summary = command("zpool", "list", "-Hp", "-o", ",".join(fields), self.pool) |
| 286 | status = json.loads(command("zpool", "status", "-jp", "--json-int", self.pool)) |
| 287 | return {"summary": dict(zip(fields, map(numeric, summary.strip().split("\t")))), |
| 288 | "status": status, "name": self.pool} |
| 289 | if operation == "storage.mounts": |
| 290 | mounts = json.loads(command("findmnt", "-J", "-l", "-t", "zfs", "-o", "SOURCE,TARGET")) |
| 291 | return {"filesystems": [item for item in mounts.get("filesystems", []) |
| 292 | if "@" not in item["source"] |
| 293 | and (item["source"] == self.pool or item["source"].startswith(self.pool + "/"))]} |
| 294 | if operation == "storage.snapshots": |
| 295 | text = command("zfs", "list", "-Hp", "-t", "snapshot", "-d", "1", "-s", "creation", |
| 296 | "-o", "name,creation,used,referenced,clones", dataset) |
| 297 | rows = [] |
| 298 | for line in text.splitlines(): |
| 299 | name, creation, used, referenced, clones = line.split("\t") |
| 300 | rows.append({"name": name.split("@", 1)[1], "creation": numeric(creation), |
| 301 | "used": numeric(used), "referenced": numeric(referenced), |
| 302 | "clones": [] if clones == "-" else clones.split(",")}) |
| 303 | return rows |
| 304 | if operation in {"storage.reclaim", "storage.destroy"}: |
| 305 | start, end = self.snapshot(request["from"]), self.snapshot(request["to"]) |
| 306 | flags = ["-nvp"] if operation == "storage.reclaim" else [] |
| 307 | return command("zfs", "destroy", *flags, f"{dataset}@{start}%{end}") |
| 308 | snapshot = self.snapshot(request["snapshot"]) |
| 309 | if operation == "files.discard": |
| 310 | if not re.fullmatch(r"dash-[0-9]+", snapshot): |
| 311 | raise Rejected("Choose a file undo snapshot.") |
| 312 | command("zfs", "destroy", f"{dataset}@{snapshot}") |
| 313 | return None |
| 314 | return command("zfs", "diff", "-H", f"{dataset}@{snapshot}", dataset) |
| 315 | |
| 316 | |
| 317 | def numeric(value): |
| 318 | try: |
| 319 | return float(value.rstrip("x")) |
| 320 | except ValueError: |
| 321 | return 0 |
| 322 | |
| 323 | |
| 324 | def command(*args, timeout=60): |
| 325 | with subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, |
| 326 | stderr=subprocess.PIPE, start_new_session=True) as process: |
| 327 | output, errors = bytearray(), bytearray() |
| 328 | deadline = time.monotonic() + timeout |
| 329 | try: |
| 330 | with selectors.DefaultSelector() as selector: |
| 331 | selector.register(process.stdout, selectors.EVENT_READ, output) |
| 332 | selector.register(process.stderr, selectors.EVENT_READ, errors) |
| 333 | while selector.get_map(): |
| 334 | remaining = deadline - time.monotonic() |
| 335 | if remaining <= 0: |
| 336 | raise TimeoutError("host command timed out") |
| 337 | for key, _ in selector.select(remaining): |
| 338 | chunk = os.read(key.fd, 65536) |
| 339 | if not chunk: |
| 340 | selector.unregister(key.fileobj) |
| 341 | continue |
| 342 | key.data.extend(chunk) |
| 343 | if len(output) + len(errors) > MAX_RESPONSE // 2: |
| 344 | raise RuntimeError("The host response is too large. Narrow the selection.") |
| 345 | process.wait(timeout=max(0.001, deadline - time.monotonic())) |
| 346 | except BaseException: |
| 347 | try: |
| 348 | os.killpg(process.pid, signal.SIGKILL) |
| 349 | except ProcessLookupError: |
| 350 | pass |
| 351 | raise |
| 352 | if process.returncode: |
| 353 | raise subprocess.CalledProcessError(process.returncode, args, stderr=errors.decode(errors="replace").strip()) |
| 354 | return output.decode() |
| 355 | |
| 356 | |
| 357 | def send_response(connection, response): |
| 358 | payload = json.dumps(response, allow_nan=False).encode() |
| 359 | if len(payload) > MAX_RESPONSE: |
| 360 | payload = b'{"error":"The host response is too large. Narrow the selection.","status":502}' |
| 361 | connection.sendall(struct.pack("!I", len(payload)) + payload) |
| 362 | |
| 363 | |
| 364 | def stream_console(connection, host, request): |
| 365 | details = host.handle(request) |
| 366 | with contextlib.ExitStack() as resources: |
| 367 | if request["operation"] == "vm.serial": |
| 368 | descriptor = os.open(details["path"], os.O_RDWR | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK) |
| 369 | screen = resources.enter_context(os.fdopen(descriptor, "r+b", buffering=0)) |
| 370 | if not stat.S_ISCHR(os.fstat(descriptor).st_mode): |
| 371 | raise Rejected("This VM's serial console is unavailable. Restart the VM and reconnect.") |
| 372 | try: |
| 373 | fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) |
| 374 | except BlockingIOError: |
| 375 | raise Rejected("This console is already open. Close its other console tab, then reconnect.") from None |
| 376 | tty.setraw(descriptor) |
| 377 | elif request["operation"] == "vm.guest": |
| 378 | screen = resources.enter_context(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) |
| 379 | screen.settimeout(10) |
| 380 | screen.connect(details["path"]) |
| 381 | screen.setblocking(False) |
| 382 | elif details.get("remote"): |
| 383 | _, screen = resources.enter_context(vms.mac_vms.connect({"action": "console"}, timeout=20)) |
| 384 | screen.setblocking(False) |
| 385 | else: |
| 386 | identity = details["uuid"] |
| 387 | deadline = time.monotonic() + 5 |
| 388 | while True: |
| 389 | try: |
| 390 | screen = resources.enter_context(socket.create_connection(("127.0.0.1", details["port"]), timeout=10)) |
| 391 | break |
| 392 | except ConnectionRefusedError: |
| 393 | if time.monotonic() >= deadline: |
| 394 | raise Rejected("This VM's screen isn't ready. Wait a moment, then reconnect.") from None |
| 395 | time.sleep(0.1) |
| 396 | details = host.handle(request) |
| 397 | if details["uuid"] != identity: |
| 398 | raise Rejected("The VM changed while its screen was opening. Open it again.") |
| 399 | screen.setblocking(False) |
| 400 | send_response(connection, {"value": {"guestDisplay": False} if details.get("remote") else None}) |
| 401 | try: |
| 402 | connection.setblocking(False) |
| 403 | with selectors.DefaultSelector() as selector: |
| 404 | selector.register(connection, selectors.EVENT_READ, screen) |
| 405 | selector.register(screen, selectors.EVENT_READ, connection) |
| 406 | deadline = time.monotonic() + 8 * 3600 |
| 407 | while time.monotonic() < deadline: |
| 408 | ready = selector.select(300) |
| 409 | if not ready: |
| 410 | continue |
| 411 | for key, _ in ready: |
| 412 | try: |
| 413 | data = os.read(key.fd, 65536) |
| 414 | except BlockingIOError: |
| 415 | continue |
| 416 | if not data: |
| 417 | return |
| 418 | remaining = memoryview(data) |
| 419 | stalled = time.monotonic() + 30 |
| 420 | while remaining: |
| 421 | try: |
| 422 | written = os.write(key.data.fileno(), remaining) |
| 423 | if written == 0: |
| 424 | return |
| 425 | remaining = remaining[written:] |
| 426 | except BlockingIOError: |
| 427 | if not select.select([], [key.data], [], max(0, stalled - time.monotonic()))[1]: |
| 428 | return |
| 429 | except OSError: |
| 430 | return |
| 431 | |
| 432 | |
| 433 | def stream_upload(connection, request): |
| 434 | if set(request) != {"operation", "payload"}: |
| 435 | raise Rejected("Use only the fields required to upload an ISO.") |
| 436 | spec = request["payload"] |
| 437 | try: |
| 438 | vms.validate("upload", spec) |
| 439 | except ValueError as error: |
| 440 | raise Rejected(str(error)) from error |
| 441 | directory = vms.UPLOADS |
| 442 | vms.managed_directory(directory) |
| 443 | target = directory / spec["volume"] |
| 444 | if target.exists() or target.is_symlink(): |
| 445 | raise Rejected("An ISO already has this filename. Rename the file before uploading.") |
| 446 | import tempfile |
| 447 | fd, temporary = tempfile.mkstemp(prefix=".upload-", dir=directory) |
| 448 | try: |
| 449 | send_response(connection, {"value": None}) |
| 450 | connection.settimeout(120) |
| 451 | deadline = time.monotonic() + 2 * 3600 |
| 452 | with os.fdopen(fd, "wb") as outgoing: |
| 453 | remaining = spec["size"] |
| 454 | while remaining: |
| 455 | if time.monotonic() > deadline: |
| 456 | raise TimeoutError("ISO upload timed out. Try uploading again.") |
| 457 | data = connection.recv(min(1024 * 1024, remaining)) |
| 458 | if not data: |
| 459 | raise Rejected("The ISO upload was interrupted. Upload the file again.") |
| 460 | outgoing.write(data) |
| 461 | remaining -= len(data) |
| 462 | outgoing.flush() |
| 463 | os.fsync(outgoing.fileno()) |
| 464 | # Never overwrite a file that another upload published in the meantime. |
| 465 | os.link(temporary, target) |
| 466 | send_response(connection, {"value": {"volume": vms.image_id(target)}}) |
| 467 | finally: |
| 468 | Path(temporary).unlink(missing_ok=True) |
| 469 | |
| 470 | |
| 471 | def stream_clone(connection, plan): |
| 472 | owner = os.stat(plan["objects"]) |
| 473 | # Service identities have numeric owners without passwd entries, which systemd's User= rejects. |
| 474 | command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", |
| 475 | *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", |
| 476 | "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), |
| 477 | *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), |
| 478 | "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", |
| 479 | sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] |
| 480 | send_response(connection, {"value": None}) |
| 481 | connection.setblocking(True) |
| 482 | subprocess.run(command, stdin=connection, stdout=connection) |
| 483 | |
| 484 | |
| 485 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 486 | with connection: |
| 487 | connection.settimeout(65) |
| 488 | _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) |
| 489 | if uid != allowed_uid: |
| 490 | return |
| 491 | try: |
| 492 | # Leave stream payload bytes on the socket after the JSON header. |
| 493 | line = bytearray() |
| 494 | while len(line) <= MAX_REQUEST: |
| 495 | chunk = connection.recv(min(65536, MAX_REQUEST + 1 - len(line)), socket.MSG_PEEK) |
| 496 | if not chunk: |
| 497 | break |
| 498 | end = chunk.find(b"\n") |
| 499 | chunk = connection.recv(len(chunk) if end < 0 else end + 1) |
| 500 | line.extend(chunk) |
| 501 | if line.endswith(b"\n"): |
| 502 | break |
| 503 | if len(line) > MAX_REQUEST or not line.endswith(b"\n"): |
| 504 | raise Rejected("The host request is too large or incomplete.") |
| 505 | request = json.loads(line) |
| 506 | if allowed_operations is not None and ( |
| 507 | not isinstance(request, dict) or not isinstance(request.get("operation"), str) |
| 508 | or request["operation"] not in allowed_operations |
| 509 | ): |
| 510 | raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.") |
| 511 | if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: |
| 512 | streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS |
| 513 | if not streams.acquire(blocking=False): |
| 514 | raise Rejected("Four VM connections or uploads are open. Close one and try again.") |
| 515 | try: |
| 516 | if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}: |
| 517 | stream_console(connection, host, request) |
| 518 | else: |
| 519 | stream_upload(connection, request) |
| 520 | return |
| 521 | finally: |
| 522 | streams.release() |
| 523 | if isinstance(request, dict) and request.get("operation") == "shale.clone": |
| 524 | if not CLONE_SLOTS.acquire(blocking=False): |
| 525 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") |
| 526 | try: |
| 527 | stream_clone(connection, host.handle(request)) |
| 528 | return |
| 529 | finally: |
| 530 | CLONE_SLOTS.release() |
| 531 | response = {"value": host.handle(request)} |
| 532 | except Rejected as error: |
| 533 | response = {"error": str(error), "status": 400} |
| 534 | except dashboard_runs.Error as error: |
| 535 | response = {"error": str(error), "status": error.status} |
| 536 | except (json.JSONDecodeError, UnicodeDecodeError): |
| 537 | response = {"error": "Send the host request as UTF-8 JSON.", "status": 400} |
| 538 | except Exception as error: |
| 539 | message = error.stderr if isinstance(error, subprocess.CalledProcessError) else str(error) |
| 540 | print(f"host operation: {message}", flush=True) |
| 541 | if isinstance(error, subprocess.CalledProcessError) and any(line.endswith(": snapshot has dependent clones") for line in message.splitlines()): |
| 542 | response = {"error": "A snapshot has clones. Delete its clones before deleting the snapshot.", "status": 409} |
| 543 | else: |
| 544 | response = {"error": "The host operation couldn't finish. Check its logs, then retry.", "status": 502} |
| 545 | try: |
| 546 | send_response(connection, response) |
| 547 | except (BrokenPipeError, ConnectionResetError): |
| 548 | pass |
| 549 | |
| 550 | |
| 551 | def main(): |
| 552 | host = Host(os.environ["SNOWGLOBE_POOL"]) |
| 553 | allowed_uid = pwd.getpwnam(os.environ["SNOWGLOBE_DASHBOARD_USER"]).pw_uid |
| 554 | socket_path = Path(os.environ.get("SNOWGLOBE_HOST_SOCKET", "/run/snowglobe-host/host.sock")) |
| 555 | socket_path.unlink(missing_ok=True) |
| 556 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: |
| 557 | listener.bind(str(socket_path)) |
| 558 | socket_path.chmod(0o660) |
| 559 | listener.listen(8) |
| 560 | if address := os.environ.get("NOTIFY_SOCKET"): |
| 561 | with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification: |
| 562 | notification.connect("\0" + address[1:] if address.startswith("@") else address) |
| 563 | notification.sendall(b"READY=1") |
| 564 | slots = threading.BoundedSemaphore(16) |
| 565 | with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: |
| 566 | while True: |
| 567 | connection, _ = listener.accept() |
| 568 | if not slots.acquire(blocking=False): |
| 569 | connection.close() |
| 570 | continue |
| 571 | work = workers.submit(serve_connection, connection, host, allowed_uid) |
| 572 | work.add_done_callback(lambda _: slots.release()) |
| 573 | |
| 574 | |
| 575 | if __name__ == "__main__": |
| 576 | main() |