diff --git a/nixos/configuration.nix b/nixos/configuration.nix index d3c524e1659df9588b41f93561eeca536e440328..d67af41031c65d2ef4bdd1d15b9cb34f7563d1cb 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -47,7 +47,7 @@ let }); in { - nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ]; + nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" "gpu-unlock-firmware" ]; networking.firewall = { enable = true; diff --git a/nixos/gpu-unlock.nix b/nixos/gpu-unlock.nix new file mode 100644 index 0000000000000000000000000000000000000000..5b67d801698edc41be611a2b3463550a4f18bb9b --- /dev/null +++ b/nixos/gpu-unlock.nix @@ -0,0 +1,52 @@ +{ config, lib, pkgs, ... }: +let + source = pkgs.fetchzip { + url = "https://github.com/Zagitta/gpu-unlock/archive/7eb51bccae0a0b7e599e084d971170e9a1e82d0c.tar.gz"; + hash = "sha256-1KD/J1S1uyI5GGoTwQ1cU/sXeEKlG54exSvi6U+j73M="; + }; + nvidiaVersion = "610.43.03"; + nvidiaSource = pkgs.fetchurl { + url = "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-${nvidiaVersion}.tar.xz"; + hash = "sha256-fhGJI8eiPtw2EU1jJzpG4+BOmvmGlaQiA+esLf6fwdw="; + }; + nvidiaInstaller = pkgs.fetchurl { + url = "https://download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/NVIDIA-Linux-x86_64-${nvidiaVersion}-no-compat32.run"; + hash = "sha256-4IRFHbcwpt8p09dPZVdJxNI248lF99lLUgXoQxj+L5A="; + }; + firmware = pkgs.runCommand "gpu-unlock-firmware" { + nativeBuildInputs = [ pkgs.python3 pkgs.zstd ]; + meta.license = lib.licenses.unfreeRedistributableFirmware; + } '' + tar -xf ${nvidiaSource} + python3 ${source}/firmware/extract_gsp.py ${nvidiaInstaller} assets + python3 ${source}/firmware/extract_boot_assets.py NVIDIA-kernel-module-source-${nvidiaVersion} assets + mkdir -p $out/lib/firmware/gpu-unlock/${nvidiaVersion} $out/share/licenses/gpu-unlock + install -m444 assets/*.bin $out/lib/firmware/gpu-unlock/${nvidiaVersion}/ + install -m444 assets/NVIDIA-LICENSE ${source}/firmware/NOTICE.md $out/share/licenses/gpu-unlock/ + ''; + kernel = config.boot.kernelPackages.kernel; + module = kernel.stdenv.mkDerivation { + pname = "gpu-unlock"; + version = "2.0.0"; + src = source; + nativeBuildInputs = kernel.moduleBuildDependencies; + hardeningDisable = [ "pic" ]; + buildPhase = '' + runHook preBuild + make -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M="$PWD/kernel/gpu_unlock" modules + runHook postBuild + ''; + installPhase = '' + runHook preInstall + install -Dm444 kernel/gpu_unlock/gpu_unlock.ko $out/lib/modules/${kernel.modDirVersion}/extra/gpu_unlock.ko + runHook postInstall + ''; + meta.license = lib.licenses.gpl2Only; + }; +in +{ + boot.extraModulePackages = [ module ]; + hardware.firmware = [ firmware ]; + boot.initrd.kernelModules = [ "gpu_unlock" ]; + boot.kernelParams = [ "gpu_unlock.targets=0000:09:00.0" ]; +} diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 868bdbf89e809465f53ca2b553dd45bc9ec2da76..9eea6412ae8b0ca0c4d0e643d3b903c471dc36ea 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -15,7 +15,7 @@ let }; in { - imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ]; + imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; networking.hostName = "snowglobe"; environment.etc."machine-info".text = '' diff --git a/nixos/tests/snowglobe-network.nix b/nixos/tests/snowglobe-network.nix new file mode 100644 index 0000000000000000000000000000000000000000..d8fdad9cd11d3093d5f57e300d3e0b6c0c63607a --- /dev/null +++ b/nixos/tests/snowglobe-network.nix @@ -0,0 +1,41 @@ +{ pkgs }: +pkgs.testers.runNixOSTest { + name = "snowglobe-network"; + globalTimeout = 300; + nodes.machine = { config, lib, ... }: { + imports = [ ../snowglobe-network.nix ../gpu-unlock.nix ]; + boot.initrd.systemd.enable = true; + boot.initrd.kernelModules = [ "virtio_net" ]; + networking.usePredictableInterfaceNames = lib.mkForce true; + services.openssh.enable = true; + virtualisation = { + memorySize = 1024; + cores = 2; + vlans = []; + qemu.networkingOptions = lib.mkForce [ + "-netdev user,id=ethernet,net=10.0.0.0/24,host=10.0.0.2" + "-device virtio-net-pci,netdev=ethernet,mac=${config.systemd.network.links."10-snowglobe-ethernet".matchConfig.MACAddress},addr=0x5" + ]; + }; + }; + testScript = '' + import json + start_all() + for boot in range(2): + machine.wait_for_unit("network-addresses-enp4s0.service") + machine.wait_for_unit("sshd.service") + address = json.loads(machine.succeed("ip -j -4 address show dev enp4s0"))[0] + assert {item["local"] for item in address["addr_info"]} == {"10.0.0.1", "192.168.0.1"}, address + route = json.loads(machine.succeed("ip -j route show default")) + assert len(route) == 1 and route[0]["gateway"] == "10.0.0.2" and route[0]["dev"] == "enp4s0", route + properties = machine.succeed("udevadm info -q property /sys/class/net/enp4s0") + assert "ID_NET_LINK_FILE=/etc/systemd/network/10-snowglobe-ethernet.link" in properties, properties + machine.succeed("ping -c 1 -W 2 10.0.0.2") + machine.succeed("test -d /sys/module/gpu_unlock") + machine.succeed("test ! -d /sys/module/nvidia") + print(f"Boot {boot}: MAC naming, both addresses, default gateway, SSH, initrd GPU module passed") + if boot == 0: + machine.shutdown() + machine.start() + ''; +} diff --git a/tools/benchmark-gpu-unlock.py b/tools/benchmark-gpu-unlock.py new file mode 100644 index 0000000000000000000000000000000000000000..d7aeac5ea9ce4f3342329e01802ae27060a988ba --- /dev/null +++ b/tools/benchmark-gpu-unlock.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Emit repeatable GPU-unlock measurements using synthetic inputs only.""" +import argparse +import ctypes as ct +from datetime import datetime, timezone +import hashlib +import http.client +import importlib.metadata +import json +import os +from pathlib import Path +import random +import time + + +def emit(**record): + print(json.dumps(record), flush=True) + + +def request(path, data=None): + connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600) + connection.request("GET" if data is None else "POST", path, + None if data is None else json.dumps(data), { + "Authorization": "Bearer " + os.environ["VLLM_API_KEY"], + "Content-Type": "application/json", + }) + response = connection.getresponse() + if response.status != 200: + response.read() + connection.close() + raise RuntimeError(f"HTTP {response.status} at {path}") + return connection, response + + +def metrics(): + connection, response = request("/metrics") + try: + values = {} + for line in response.read().decode().splitlines(): + if line.startswith("vllm:") and "_bucket{" not in line: + name = line.split("{", 1)[0].split(" ", 1)[0] + values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1]) + return values + finally: + connection.close() + + +def idle(): + values = metrics() + if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]: + raise RuntimeError("Inference endpoint is busy") + return values + + +def inference(args): + connection, response = request("/tokenize", { + "model": "qwen3.8-27b", + "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n", + }) + vocabulary = json.loads(response.read())["tokens"] + connection.close() + for length in args.lengths: + for trial in range(args.repeats): + prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length) + before = idle() + started = time.perf_counter() + connection, response = request("/v1/completions", { + "model": "qwen3.8-27b", "prompt": prompt, + "max_tokens": args.output_tokens, "ignore_eos": True, + "temperature": 0, "seed": 15027, + "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}", + "stream": True, "stream_options": {"include_usage": True}, + }) + first = None + usage = None + digest = hashlib.sha256() + chunks = 0 + try: + for line in response: + if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]": + continue + event = json.loads(line[6:]) + for choice in event.get("choices", []): + content = choice.get("text", "") + if content: + if first is None: + first = time.perf_counter() + chunks += 1 + digest.update(content.encode()) + if event.get("usage"): + usage = event["usage"] + finally: + connection.close() + elapsed = time.perf_counter() - started + after = metrics() + delta = {name: value - before.get(name, 0) for name, value in after.items()} + valid = (usage is not None and usage["prompt_tokens"] == length + and usage["completion_tokens"] == args.output_tokens + and delta.get("vllm:request_success_total") == 1 + and delta.get("vllm:prefix_cache_hits_total", 0) == 0) + emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial, + output_tokens=None if usage is None else usage["completion_tokens"], + wall_seconds=elapsed, ttft_seconds=None if first is None else first - started, + prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"), + decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"), + queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"), + output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid, + counters={name: value for name, value in delta.items() + if name.endswith(("_sum", "_total")) and value}) + if not valid: + raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output") + + +def tensor(args): + # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model. + cuda = ct.CDLL("libcudart.so.13") + libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia")) + blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13")))) + cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t] + cuda.cudaFree.argtypes = [ct.c_void_p] + cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t] + cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)] + cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)] + cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p] + cuda.cudaEventSynchronize.argtypes = [ct.c_void_p] + cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p] + cuda.cudaEventDestroy.argtypes = [ct.c_void_p] + blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)] + blas.cublasDestroy_v2.argtypes = [ct.c_void_p] + blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int] + blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int, + ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int, + ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p, + ct.c_int, ct.c_int, ct.c_int, ct.c_int] + + def checked(function, *values): + status = function(*values) + if status: + raise RuntimeError(f"{function.__name__}: status {status}") + + idle() + handle = ct.c_void_p() + checked(blas.cublasCreate_v2, ct.byref(handle)) + start, end = ct.c_void_p(), ct.c_void_p() + try: + checked(cuda.cudaEventCreate, ct.byref(start)) + checked(cuda.cudaEventCreate, ct.byref(end)) + checked(blas.cublasSetMathMode, handle, 16) + for dtype, input_type, output_type, element_bytes, compute in ( + ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68), + ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68), + ("int8_int32", 3, 10, 1, 72), + ): + for size in args.sizes: + idle() + free, total = ct.c_size_t(), ct.c_size_t() + checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total)) + needed = size * size * (element_bytes * 2 + 4) + if free.value < needed + 64 * 1024**2: + emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, + skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed) + continue + buffers = [] + try: + for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,): + pointer = ct.c_void_p() + checked(cuda.cudaMalloc, ct.byref(pointer), byte_count) + buffers.append(pointer) + checked(cuda.cudaMemset, pointer, 60, byte_count) + scalar = ct.c_int if output_type == 10 else ct.c_float + alpha, beta = scalar(1), scalar(0) + + def multiply(): + checked(blas.cublasGemmEx, handle, 0, 0, size, size, size, + ct.byref(alpha), buffers[0], input_type, size, + buffers[1], input_type, size, ct.byref(beta), + buffers[2], output_type, size, compute, 99) + + for _ in range(10): + multiply() + checked(cuda.cudaDeviceSynchronize) + for trial in range(args.repeats): + checked(cuda.cudaEventRecord, start, None) + for _ in range(100): + multiply() + checked(cuda.cudaEventRecord, end, None) + checked(cuda.cudaEventSynchronize, end) + milliseconds = ct.c_float() + checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end) + seconds = milliseconds.value / 1000 / 100 + emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, + trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12) + finally: + for pointer in buffers: + checked(cuda.cudaFree, pointer) + finally: + if start: + checked(cuda.cudaEventDestroy, start) + if end: + checked(cuda.cudaEventDestroy, end) + checked(blas.cublasDestroy_v2, handle) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("kind", choices=("inference", "tensor")) + parser.add_argument("--phase", required=True) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000]) + parser.add_argument("--output-tokens", type=int, default=256) + parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048]) + args = parser.parse_args() + emit(kind="metadata", phase=args.phase, benchmark=args.kind, + utc=datetime.now(timezone.utc).isoformat(), + script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(), + versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")}, + arguments=vars(args)) + if args.kind == "inference": + inference(args) + else: + tensor(args) diff --git a/tools/deploy-test.py b/tools/deploy-test.py index b569b48499e2b599badf06ac9e6be3ab1c5430be..f9292ea9090e6bbccdb7fbcb897b8e11e29f1898 100644 --- a/tools/deploy-test.py +++ b/tools/deploy-test.py @@ -13,6 +13,16 @@ import deploy import release +class HostStage(unittest.TestCase): + def test_prepares_working_host_without_publishing_or_deploying_services(self): + with patch("sys.argv", ["deploy.py", "stage", "host"]), patch.object( + deploy, "upload", return_value="0123456789abcdef", + ) as upload, patch.object(deploy, "run_logged") as run: + deploy.main() + upload.assert_called_once_with() + run.assert_called_once_with("stage", "host", "python3 /opt/snowglobe/releases/0123456789abcdef/tools/release.py prepare-boot 0123456789abcdef") + + class MainExport(unittest.TestCase): def test_production_exports_main_while_staging_exports_working_files(self): with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary: diff --git a/tools/deploy.py b/tools/deploy.py index c7d5c92b49daf0f40c3334770aa8069cc032f43e..2aac68b721adc550115878fbce497cfd51802a81 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -187,8 +187,8 @@ def main(): parser.error("rollback target must be a release ID") if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target: parser.error(f"{args.mode} takes no target") - if args.env and args.mode != "stage": - parser.error("--env is available only for stage") + if args.env and (args.mode != "stage" or args.target == "host"): + parser.error("--env requires a service or dashboard stage") if bool(args.file) != (args.mode == "secrets"): parser.error("--file is required only for secrets") if args.key and args.mode != "secrets": @@ -212,6 +212,9 @@ def main(): manager = f"python3 {REMOTE}/release.py" if args.mode == "stage": release = upload() + if args.target == "host": + run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}") + return overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env) if args.computer: overrides += " --computer " + shlex.quote(args.computer) diff --git a/tools/gpu-unlock.md b/tools/gpu-unlock.md new file mode 100644 index 0000000000000000000000000000000000000000..bc3edd8865ffa1fc82b9a57fab1c65d1ce779482 --- /dev/null +++ b/tools/gpu-unlock.md @@ -0,0 +1,35 @@ +# GPU tensor unlock + +[The host module](../nixos/gpu-unlock.nix) builds the pinned upstream kernel +module and extracts its NVIDIA firmware without executing the driver installer. +It runs in the initrd before the normal NVIDIA driver. The installed driver +and kernel stay on the repository's existing versions. The target is the +current GPU's PCI address; a different address is skipped by upstream. + +`python3 tools/deploy.py stage host` builds and installs a candidate boot +generation from an immutable uploaded release. It preserves the previous +default boot entry and leaves the running host, services, and published release +unchanged. It prints the candidate entry and fallback. Only after reboot +approval, select the printed entry with `bootctl set-oneshot ` and +reboot. A subsequent boot uses the previous default again. Promote the reviewed +source through the normal main deployment workflow when retaining the change. +After promotion, clear the trial's EFI default with `bootctl set-default ''` +so NixOS's `loader.conf` selects the production generation on subsequent boots. + +Before rebooting, inspect the candidate and fallback initrds for the MAC-bound +Ethernet link rule, and verify both static addresses and the gateway in the +built system. The [isolated boot test](../nixos/tests/snowglobe-network.nix) +exercises the same network module and GPU module in a disposable VM. + +Run [the benchmark](benchmark-gpu-unlock.py) with the existing inference image's +Python and environment, using `inference --phase before-` and +`tensor --phase before-`. Repeat with `after-` after the approved boot. +Keep the script, image, model, serving settings, lengths, and seeds identical. +Each context length has three uncached requests with a fixed output budget; +measurements include server prefill/decode time, streamed first-token latency, +speculation counters, and output hashes. Tensor tests use CUDA events and +FP32 accumulation, plus FP32 and integer controls, while retaining GPU memory +for the resident model. The script rejects busy inference and records skipped +tensor sizes if memory is insufficient. Save stdout with host/GPU metadata +and clock, power, and temperature telemetry. These are synthetic performance +probes, not model-quality evaluations. diff --git a/tools/mac-agent-vm.md b/tools/mac-agent-vm.md index 66f81a2acef9727b324e12444630ed3d1df02098..38cf7e87c111fea2bafb7c03bb0548bb8d5fa1ce 100644 --- a/tools/mac-agent-vm.md +++ b/tools/mac-agent-vm.md @@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da `start` resumes a suspended VM or boots a stopped VM, then waits for commands to work through the logged-in guest agent. `stop` requests a graceful -guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM +guest shutdown and waits for it to finish. `start` launches Tart through +`ssh localhost` (needs Remote Login and this user's key) because closing an +agent session kills every process it spawned, even in a new session; launchd +jobs cannot read the VM volume (TCC). `suspend` waits for Tart's RAM snapshot; it writes memory to disk and is not an instant CPU pause. A running VM holds a process-scoped idle-sleep assertion, so locking the host does not cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. diff --git a/tools/mac-agent-vm.py b/tools/mac-agent-vm.py index c92206e4a00d19cf3cc3ea58a688795458c0cbaf..327c3f81946b8f4f27712acada7fe525393057a9 100644 --- a/tools/mac-agent-vm.py +++ b/tools/mac-agent-vm.py @@ -10,6 +10,7 @@ import os from pathlib import Path import select import secrets +import shlex import signal import socket import socketserver @@ -247,20 +248,17 @@ def main(): if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]: print("GUI-agent VM is already running.") return - with (PRIVATE / "run.log").open("w") as log: - os.chmod(PRIVATE / "run.log", 0o600) - process = subprocess.Popen([str(TART), "run", NAME, "--no-graphics", "--no-audio", - "--no-usb-accessories", "--no-clipboard", "--suspendable", - "--root-disk-opts=sync=full"], - env={**os.environ, "TART_HOME": str(ROOT / "state")}, - stdin=subprocess.DEVNULL, stdout=log, stderr=log, - start_new_session=True) - subprocess.Popen(["/usr/bin/caffeinate", "-i", "-w", str(process.pid)], - stdin=subprocess.DEVNULL, stdout=log, stderr=log, - start_new_session=True) + # Closing an agent session kills every process it ever spawned, so sshd must own the VM. + run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", + str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", + "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"]) + (PRIVATE / "run.log").write_text("") + os.chmod(PRIVATE / "run.log", 0o600) + subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", + f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1