From df4161ba6e1bae0f7c7596554bde1ccfe49ea94d Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sun, 1 Nov 2026 22:19:03 -0800 Subject: [PATCH] Enable the Snow Globe GPU tensor unlock Build pinned gpu-unlock firmware and an initrd module for the RTX 3090 before the NVIDIA driver loads. Add host boot preparation that preserves the existing default until promotion, plus repeatable inference and tensor benchmarks. Verified two isolated cold boots, a real generation-198 boot with the MAC-bound network intact, all service routes, authenticated inference, and 45 before/after benchmark samples per phase. Keep the installed kernel and NVIDIA driver versions unchanged. Assisted-by: gpt-6.1-sol --- nixos/configuration.nix | 2 +- nixos/gpu-unlock.nix | 52 +++++++ nixos/snowglobe.nix | 2 +- nixos/tests/snowglobe-network.nix | 41 ++++++ tools/benchmark-gpu-unlock.py | 222 ++++++++++++++++++++++++++++++ tools/deploy-test.py | 10 ++ tools/deploy.py | 7 +- tools/gpu-unlock.md | 35 +++++ tools/mac-agent-vm.md | 5 +- tools/mac-agent-vm.py | 22 ++- tools/release-host-test.py | 39 ++++++ tools/release.py | 26 +++- 12 files changed, 445 insertions(+), 18 deletions(-) create mode 100644 nixos/gpu-unlock.nix create mode 100644 nixos/tests/snowglobe-network.nix create mode 100644 tools/benchmark-gpu-unlock.py create mode 100644 tools/gpu-unlock.md diff --git a/nixos/configuration.nix b/nixos/configuration.nix index d3c524e1659df9588b41f93561eeca536e440328..d67af41031c65d2ef4bdd1d15b9cb34f7563d1cb 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -47,7 +47,7 @@ let }); in { - nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ]; + nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" "gpu-unlock-firmware" ]; networking.firewall = { enable = true; diff --git a/nixos/gpu-unlock.nix b/nixos/gpu-unlock.nix new file mode 100644 index 0000000000000000000000000000000000000000..5b67d801698edc41be611a2b3463550a4f18bb9b --- /dev/null +++ b/nixos/gpu-unlock.nix @@ -0,0 +1,52 @@ +{ config, lib, pkgs, ... }: +let + source = pkgs.fetchzip { + url = "https://github.com/Zagitta/gpu-unlock/archive/7eb51bccae0a0b7e599e084d971170e9a1e82d0c.tar.gz"; + hash = "sha256-1KD/J1S1uyI5GGoTwQ1cU/sXeEKlG54exSvi6U+j73M="; + }; + nvidiaVersion = "610.43.03"; + nvidiaSource = pkgs.fetchurl { + url = "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-${nvidiaVersion}.tar.xz"; + hash = "sha256-fhGJI8eiPtw2EU1jJzpG4+BOmvmGlaQiA+esLf6fwdw="; + }; + nvidiaInstaller = pkgs.fetchurl { + url = "https://download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/NVIDIA-Linux-x86_64-${nvidiaVersion}-no-compat32.run"; + hash = "sha256-4IRFHbcwpt8p09dPZVdJxNI248lF99lLUgXoQxj+L5A="; + }; + firmware = pkgs.runCommand "gpu-unlock-firmware" { + nativeBuildInputs = [ pkgs.python3 pkgs.zstd ]; + meta.license = lib.licenses.unfreeRedistributableFirmware; + } '' + tar -xf ${nvidiaSource} + python3 ${source}/firmware/extract_gsp.py ${nvidiaInstaller} assets + python3 ${source}/firmware/extract_boot_assets.py NVIDIA-kernel-module-source-${nvidiaVersion} assets + mkdir -p $out/lib/firmware/gpu-unlock/${nvidiaVersion} $out/share/licenses/gpu-unlock + install -m444 assets/*.bin $out/lib/firmware/gpu-unlock/${nvidiaVersion}/ + install -m444 assets/NVIDIA-LICENSE ${source}/firmware/NOTICE.md $out/share/licenses/gpu-unlock/ + ''; + kernel = config.boot.kernelPackages.kernel; + module = kernel.stdenv.mkDerivation { + pname = "gpu-unlock"; + version = "2.0.0"; + src = source; + nativeBuildInputs = kernel.moduleBuildDependencies; + hardeningDisable = [ "pic" ]; + buildPhase = '' + runHook preBuild + make -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M="$PWD/kernel/gpu_unlock" modules + runHook postBuild + ''; + installPhase = '' + runHook preInstall + install -Dm444 kernel/gpu_unlock/gpu_unlock.ko $out/lib/modules/${kernel.modDirVersion}/extra/gpu_unlock.ko + runHook postInstall + ''; + meta.license = lib.licenses.gpl2Only; + }; +in +{ + boot.extraModulePackages = [ module ]; + hardware.firmware = [ firmware ]; + boot.initrd.kernelModules = [ "gpu_unlock" ]; + boot.kernelParams = [ "gpu_unlock.targets=0000:09:00.0" ]; +} diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 868bdbf89e809465f53ca2b553dd45bc9ec2da76..9eea6412ae8b0ca0c4d0e643d3b903c471dc36ea 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -15,7 +15,7 @@ let }; in { - imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ]; + imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; networking.hostName = "snowglobe"; environment.etc."machine-info".text = '' diff --git a/nixos/tests/snowglobe-network.nix b/nixos/tests/snowglobe-network.nix new file mode 100644 index 0000000000000000000000000000000000000000..d8fdad9cd11d3093d5f57e300d3e0b6c0c63607a --- /dev/null +++ b/nixos/tests/snowglobe-network.nix @@ -0,0 +1,41 @@ +{ pkgs }: +pkgs.testers.runNixOSTest { + name = "snowglobe-network"; + globalTimeout = 300; + nodes.machine = { config, lib, ... }: { + imports = [ ../snowglobe-network.nix ../gpu-unlock.nix ]; + boot.initrd.systemd.enable = true; + boot.initrd.kernelModules = [ "virtio_net" ]; + networking.usePredictableInterfaceNames = lib.mkForce true; + services.openssh.enable = true; + virtualisation = { + memorySize = 1024; + cores = 2; + vlans = []; + qemu.networkingOptions = lib.mkForce [ + "-netdev user,id=ethernet,net=10.0.0.0/24,host=10.0.0.2" + "-device virtio-net-pci,netdev=ethernet,mac=${config.systemd.network.links."10-snowglobe-ethernet".matchConfig.MACAddress},addr=0x5" + ]; + }; + }; + testScript = '' + import json + start_all() + for boot in range(2): + machine.wait_for_unit("network-addresses-enp4s0.service") + machine.wait_for_unit("sshd.service") + address = json.loads(machine.succeed("ip -j -4 address show dev enp4s0"))[0] + assert {item["local"] for item in address["addr_info"]} == {"10.0.0.1", "192.168.0.1"}, address + route = json.loads(machine.succeed("ip -j route show default")) + assert len(route) == 1 and route[0]["gateway"] == "10.0.0.2" and route[0]["dev"] == "enp4s0", route + properties = machine.succeed("udevadm info -q property /sys/class/net/enp4s0") + assert "ID_NET_LINK_FILE=/etc/systemd/network/10-snowglobe-ethernet.link" in properties, properties + machine.succeed("ping -c 1 -W 2 10.0.0.2") + machine.succeed("test -d /sys/module/gpu_unlock") + machine.succeed("test ! -d /sys/module/nvidia") + print(f"Boot {boot}: MAC naming, both addresses, default gateway, SSH, initrd GPU module passed") + if boot == 0: + machine.shutdown() + machine.start() + ''; +} diff --git a/tools/benchmark-gpu-unlock.py b/tools/benchmark-gpu-unlock.py new file mode 100644 index 0000000000000000000000000000000000000000..d7aeac5ea9ce4f3342329e01802ae27060a988ba --- /dev/null +++ b/tools/benchmark-gpu-unlock.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Emit repeatable GPU-unlock measurements using synthetic inputs only.""" +import argparse +import ctypes as ct +from datetime import datetime, timezone +import hashlib +import http.client +import importlib.metadata +import json +import os +from pathlib import Path +import random +import time + + +def emit(**record): + print(json.dumps(record), flush=True) + + +def request(path, data=None): + connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600) + connection.request("GET" if data is None else "POST", path, + None if data is None else json.dumps(data), { + "Authorization": "Bearer " + os.environ["VLLM_API_KEY"], + "Content-Type": "application/json", + }) + response = connection.getresponse() + if response.status != 200: + response.read() + connection.close() + raise RuntimeError(f"HTTP {response.status} at {path}") + return connection, response + + +def metrics(): + connection, response = request("/metrics") + try: + values = {} + for line in response.read().decode().splitlines(): + if line.startswith("vllm:") and "_bucket{" not in line: + name = line.split("{", 1)[0].split(" ", 1)[0] + values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1]) + return values + finally: + connection.close() + + +def idle(): + values = metrics() + if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]: + raise RuntimeError("Inference endpoint is busy") + return values + + +def inference(args): + connection, response = request("/tokenize", { + "model": "qwen3.8-27b", + "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n", + }) + vocabulary = json.loads(response.read())["tokens"] + connection.close() + for length in args.lengths: + for trial in range(args.repeats): + prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length) + before = idle() + started = time.perf_counter() + connection, response = request("/v1/completions", { + "model": "qwen3.8-27b", "prompt": prompt, + "max_tokens": args.output_tokens, "ignore_eos": True, + "temperature": 0, "seed": 15027, + "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}", + "stream": True, "stream_options": {"include_usage": True}, + }) + first = None + usage = None + digest = hashlib.sha256() + chunks = 0 + try: + for line in response: + if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]": + continue + event = json.loads(line[6:]) + for choice in event.get("choices", []): + content = choice.get("text", "") + if content: + if first is None: + first = time.perf_counter() + chunks += 1 + digest.update(content.encode()) + if event.get("usage"): + usage = event["usage"] + finally: + connection.close() + elapsed = time.perf_counter() - started + after = metrics() + delta = {name: value - before.get(name, 0) for name, value in after.items()} + valid = (usage is not None and usage["prompt_tokens"] == length + and usage["completion_tokens"] == args.output_tokens + and delta.get("vllm:request_success_total") == 1 + and delta.get("vllm:prefix_cache_hits_total", 0) == 0) + emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial, + output_tokens=None if usage is None else usage["completion_tokens"], + wall_seconds=elapsed, ttft_seconds=None if first is None else first - started, + prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"), + decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"), + queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"), + output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid, + counters={name: value for name, value in delta.items() + if name.endswith(("_sum", "_total")) and value}) + if not valid: + raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output") + + +def tensor(args): + # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model. + cuda = ct.CDLL("libcudart.so.13") + libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia")) + blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13")))) + cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t] + cuda.cudaFree.argtypes = [ct.c_void_p] + cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t] + cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)] + cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)] + cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p] + cuda.cudaEventSynchronize.argtypes = [ct.c_void_p] + cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p] + cuda.cudaEventDestroy.argtypes = [ct.c_void_p] + blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)] + blas.cublasDestroy_v2.argtypes = [ct.c_void_p] + blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int] + blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int, + ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int, + ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p, + ct.c_int, ct.c_int, ct.c_int, ct.c_int] + + def checked(function, *values): + status = function(*values) + if status: + raise RuntimeError(f"{function.__name__}: status {status}") + + idle() + handle = ct.c_void_p() + checked(blas.cublasCreate_v2, ct.byref(handle)) + start, end = ct.c_void_p(), ct.c_void_p() + try: + checked(cuda.cudaEventCreate, ct.byref(start)) + checked(cuda.cudaEventCreate, ct.byref(end)) + checked(blas.cublasSetMathMode, handle, 16) + for dtype, input_type, output_type, element_bytes, compute in ( + ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68), + ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68), + ("int8_int32", 3, 10, 1, 72), + ): + for size in args.sizes: + idle() + free, total = ct.c_size_t(), ct.c_size_t() + checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total)) + needed = size * size * (element_bytes * 2 + 4) + if free.value < needed + 64 * 1024**2: + emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, + skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed) + continue + buffers = [] + try: + for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,): + pointer = ct.c_void_p() + checked(cuda.cudaMalloc, ct.byref(pointer), byte_count) + buffers.append(pointer) + checked(cuda.cudaMemset, pointer, 60, byte_count) + scalar = ct.c_int if output_type == 10 else ct.c_float + alpha, beta = scalar(1), scalar(0) + + def multiply(): + checked(blas.cublasGemmEx, handle, 0, 0, size, size, size, + ct.byref(alpha), buffers[0], input_type, size, + buffers[1], input_type, size, ct.byref(beta), + buffers[2], output_type, size, compute, 99) + + for _ in range(10): + multiply() + checked(cuda.cudaDeviceSynchronize) + for trial in range(args.repeats): + checked(cuda.cudaEventRecord, start, None) + for _ in range(100): + multiply() + checked(cuda.cudaEventRecord, end, None) + checked(cuda.cudaEventSynchronize, end) + milliseconds = ct.c_float() + checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end) + seconds = milliseconds.value / 1000 / 100 + emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, + trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12) + finally: + for pointer in buffers: + checked(cuda.cudaFree, pointer) + finally: + if start: + checked(cuda.cudaEventDestroy, start) + if end: + checked(cuda.cudaEventDestroy, end) + checked(blas.cublasDestroy_v2, handle) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("kind", choices=("inference", "tensor")) + parser.add_argument("--phase", required=True) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000]) + parser.add_argument("--output-tokens", type=int, default=256) + parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048]) + args = parser.parse_args() + emit(kind="metadata", phase=args.phase, benchmark=args.kind, + utc=datetime.now(timezone.utc).isoformat(), + script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(), + versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")}, + arguments=vars(args)) + if args.kind == "inference": + inference(args) + else: + tensor(args) diff --git a/tools/deploy-test.py b/tools/deploy-test.py index b569b48499e2b599badf06ac9e6be3ab1c5430be..f9292ea9090e6bbccdb7fbcb897b8e11e29f1898 100644 --- a/tools/deploy-test.py +++ b/tools/deploy-test.py @@ -13,6 +13,16 @@ import deploy import release +class HostStage(unittest.TestCase): + def test_prepares_working_host_without_publishing_or_deploying_services(self): + with patch("sys.argv", ["deploy.py", "stage", "host"]), patch.object( + deploy, "upload", return_value="0123456789abcdef", + ) as upload, patch.object(deploy, "run_logged") as run: + deploy.main() + upload.assert_called_once_with() + run.assert_called_once_with("stage", "host", "python3 /opt/snowglobe/releases/0123456789abcdef/tools/release.py prepare-boot 0123456789abcdef") + + class MainExport(unittest.TestCase): def test_production_exports_main_while_staging_exports_working_files(self): with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary: diff --git a/tools/deploy.py b/tools/deploy.py index c7d5c92b49daf0f40c3334770aa8069cc032f43e..2aac68b721adc550115878fbce497cfd51802a81 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -187,8 +187,8 @@ def main(): parser.error("rollback target must be a release ID") if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target: parser.error(f"{args.mode} takes no target") - if args.env and args.mode != "stage": - parser.error("--env is available only for stage") + if args.env and (args.mode != "stage" or args.target == "host"): + parser.error("--env requires a service or dashboard stage") if bool(args.file) != (args.mode == "secrets"): parser.error("--file is required only for secrets") if args.key and args.mode != "secrets": @@ -212,6 +212,9 @@ def main(): manager = f"python3 {REMOTE}/release.py" if args.mode == "stage": release = upload() + if args.target == "host": + run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}") + return overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env) if args.computer: overrides += " --computer " + shlex.quote(args.computer) diff --git a/tools/gpu-unlock.md b/tools/gpu-unlock.md new file mode 100644 index 0000000000000000000000000000000000000000..bc3edd8865ffa1fc82b9a57fab1c65d1ce779482 --- /dev/null +++ b/tools/gpu-unlock.md @@ -0,0 +1,35 @@ +# GPU tensor unlock + +[The host module](../nixos/gpu-unlock.nix) builds the pinned upstream kernel +module and extracts its NVIDIA firmware without executing the driver installer. +It runs in the initrd before the normal NVIDIA driver. The installed driver +and kernel stay on the repository's existing versions. The target is the +current GPU's PCI address; a different address is skipped by upstream. + +`python3 tools/deploy.py stage host` builds and installs a candidate boot +generation from an immutable uploaded release. It preserves the previous +default boot entry and leaves the running host, services, and published release +unchanged. It prints the candidate entry and fallback. Only after reboot +approval, select the printed entry with `bootctl set-oneshot ` and +reboot. A subsequent boot uses the previous default again. Promote the reviewed +source through the normal main deployment workflow when retaining the change. +After promotion, clear the trial's EFI default with `bootctl set-default ''` +so NixOS's `loader.conf` selects the production generation on subsequent boots. + +Before rebooting, inspect the candidate and fallback initrds for the MAC-bound +Ethernet link rule, and verify both static addresses and the gateway in the +built system. The [isolated boot test](../nixos/tests/snowglobe-network.nix) +exercises the same network module and GPU module in a disposable VM. + +Run [the benchmark](benchmark-gpu-unlock.py) with the existing inference image's +Python and environment, using `inference --phase before-` and +`tensor --phase before-`. Repeat with `after-` after the approved boot. +Keep the script, image, model, serving settings, lengths, and seeds identical. +Each context length has three uncached requests with a fixed output budget; +measurements include server prefill/decode time, streamed first-token latency, +speculation counters, and output hashes. Tensor tests use CUDA events and +FP32 accumulation, plus FP32 and integer controls, while retaining GPU memory +for the resident model. The script rejects busy inference and records skipped +tensor sizes if memory is insufficient. Save stdout with host/GPU metadata +and clock, power, and temperature telemetry. These are synthetic performance +probes, not model-quality evaluations. diff --git a/tools/mac-agent-vm.md b/tools/mac-agent-vm.md index 66f81a2acef9727b324e12444630ed3d1df02098..38cf7e87c111fea2bafb7c03bb0548bb8d5fa1ce 100644 --- a/tools/mac-agent-vm.md +++ b/tools/mac-agent-vm.md @@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da `start` resumes a suspended VM or boots a stopped VM, then waits for commands to work through the logged-in guest agent. `stop` requests a graceful -guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM +guest shutdown and waits for it to finish. `start` launches Tart through +`ssh localhost` (needs Remote Login and this user's key) because closing an +agent session kills every process it spawned, even in a new session; launchd +jobs cannot read the VM volume (TCC). `suspend` waits for Tart's RAM snapshot; it writes memory to disk and is not an instant CPU pause. A running VM holds a process-scoped idle-sleep assertion, so locking the host does not cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. diff --git a/tools/mac-agent-vm.py b/tools/mac-agent-vm.py index c92206e4a00d19cf3cc3ea58a688795458c0cbaf..327c3f81946b8f4f27712acada7fe525393057a9 100644 --- a/tools/mac-agent-vm.py +++ b/tools/mac-agent-vm.py @@ -10,6 +10,7 @@ import os from pathlib import Path import select import secrets +import shlex import signal import socket import socketserver @@ -247,20 +248,17 @@ def main(): if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]: print("GUI-agent VM is already running.") return - with (PRIVATE / "run.log").open("w") as log: - os.chmod(PRIVATE / "run.log", 0o600) - process = subprocess.Popen([str(TART), "run", NAME, "--no-graphics", "--no-audio", - "--no-usb-accessories", "--no-clipboard", "--suspendable", - "--root-disk-opts=sync=full"], - env={**os.environ, "TART_HOME": str(ROOT / "state")}, - stdin=subprocess.DEVNULL, stdout=log, stderr=log, - start_new_session=True) - subprocess.Popen(["/usr/bin/caffeinate", "-i", "-w", str(process.pid)], - stdin=subprocess.DEVNULL, stdout=log, stderr=log, - start_new_session=True) + # Closing an agent session kills every process it ever spawned, so sshd must own the VM. + run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", + str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", + "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"]) + (PRIVATE / "run.log").write_text("") + os.chmod(PRIVATE / "run.log", 0o600) + subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", + f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1