| author | |
| committer | |
| log | df4161ba6e1bae0f7c7596554bde1ccfe49ea94d |
| tree | 60ab485b2cfae8a17c940be301cd4cffb338bdca |
| parent | ffc3896c9d5ee7c48a19bfa615c002d2c9de7077 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Build pinned gpu-unlock firmware and an initrd module for the RTX 3090 before the NVIDIA driver loads. Add host boot preparation that preserves the existing default until promotion, plus repeatable inference and tensor benchmarks.
Verified two isolated cold boots, a real generation-198 boot with the MAC-bound network intact, all service routes, authenticated inference, and 45 before/after benchmark samples per phase. Keep the installed kernel and NVIDIA driver versions unchanged.
Assisted-by: gpt-6.1-sol12 files changed, 445 insertions(+), 18 deletions(-)
nixos/configuration.nix+1-1| ... | ... | @@ -47,7 +47,7 @@ let |
| 47 | 47 | }); |
| 48 | 48 | in |
| 49 | 49 | { |
| 50 | nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ]; | |
| 50 | nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" "gpu-unlock-firmware" ]; | |
| 51 | 51 | |
| 52 | 52 | networking.firewall = { |
| 53 | 53 | enable = true; |
nixos/gpu-unlock.nix created+52| ... | ... | @@ -0,0 +1,52 @@ |
| 1 | { config, lib, pkgs, ... }: | |
| 2 | let | |
| 3 | source = pkgs.fetchzip { | |
| 4 | url = "https://github.com/Zagitta/gpu-unlock/archive/7eb51bccae0a0b7e599e084d971170e9a1e82d0c.tar.gz"; | |
| 5 | hash = "sha256-1KD/J1S1uyI5GGoTwQ1cU/sXeEKlG54exSvi6U+j73M="; | |
| 6 | }; | |
| 7 | nvidiaVersion = "610.43.03"; | |
| 8 | nvidiaSource = pkgs.fetchurl { | |
| 9 | url = "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-${nvidiaVersion}.tar.xz"; | |
| 10 | hash = "sha256-fhGJI8eiPtw2EU1jJzpG4+BOmvmGlaQiA+esLf6fwdw="; | |
| 11 | }; | |
| 12 | nvidiaInstaller = pkgs.fetchurl { | |
| 13 | url = "https://download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/NVIDIA-Linux-x86_64-${nvidiaVersion}-no-compat32.run"; | |
| 14 | hash = "sha256-4IRFHbcwpt8p09dPZVdJxNI248lF99lLUgXoQxj+L5A="; | |
| 15 | }; | |
| 16 | firmware = pkgs.runCommand "gpu-unlock-firmware" { | |
| 17 | nativeBuildInputs = [ pkgs.python3 pkgs.zstd ]; | |
| 18 | meta.license = lib.licenses.unfreeRedistributableFirmware; | |
| 19 | } '' | |
| 20 | tar -xf ${nvidiaSource} | |
| 21 | python3 ${source}/firmware/extract_gsp.py ${nvidiaInstaller} assets | |
| 22 | python3 ${source}/firmware/extract_boot_assets.py NVIDIA-kernel-module-source-${nvidiaVersion} assets | |
| 23 | mkdir -p $out/lib/firmware/gpu-unlock/${nvidiaVersion} $out/share/licenses/gpu-unlock | |
| 24 | install -m444 assets/*.bin $out/lib/firmware/gpu-unlock/${nvidiaVersion}/ | |
| 25 | install -m444 assets/NVIDIA-LICENSE ${source}/firmware/NOTICE.md $out/share/licenses/gpu-unlock/ | |
| 26 | ''; | |
| 27 | kernel = config.boot.kernelPackages.kernel; | |
| 28 | module = kernel.stdenv.mkDerivation { | |
| 29 | pname = "gpu-unlock"; | |
| 30 | version = "2.0.0"; | |
| 31 | src = source; | |
| 32 | nativeBuildInputs = kernel.moduleBuildDependencies; | |
| 33 | hardeningDisable = [ "pic" ]; | |
| 34 | buildPhase = '' | |
| 35 | runHook preBuild | |
| 36 | make -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M="$PWD/kernel/gpu_unlock" modules | |
| 37 | runHook postBuild | |
| 38 | ''; | |
| 39 | installPhase = '' | |
| 40 | runHook preInstall | |
| 41 | install -Dm444 kernel/gpu_unlock/gpu_unlock.ko $out/lib/modules/${kernel.modDirVersion}/extra/gpu_unlock.ko | |
| 42 | runHook postInstall | |
| 43 | ''; | |
| 44 | meta.license = lib.licenses.gpl2Only; | |
| 45 | }; | |
| 46 | in | |
| 47 | { | |
| 48 | boot.extraModulePackages = [ module ]; | |
| 49 | hardware.firmware = [ firmware ]; | |
| 50 | boot.initrd.kernelModules = [ "gpu_unlock" ]; | |
| 51 | boot.kernelParams = [ "gpu_unlock.targets=0000:09:00.0" ]; | |
| 52 | } |
nixos/snowglobe.nix+1-1| ... | ... | @@ -15,7 +15,7 @@ let |
| 15 | 15 | }; |
| 16 | 16 | in |
| 17 | 17 | { |
| 18 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ]; | |
| 18 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ]; | |
| 19 | 19 | |
| 20 | 20 | networking.hostName = "snowglobe"; |
| 21 | 21 | environment.etc."machine-info".text = '' |
nixos/tests/snowglobe-network.nix created+41| ... | ... | @@ -0,0 +1,41 @@ |
| 1 | { pkgs }: | |
| 2 | pkgs.testers.runNixOSTest { | |
| 3 | name = "snowglobe-network"; | |
| 4 | globalTimeout = 300; | |
| 5 | nodes.machine = { config, lib, ... }: { | |
| 6 | imports = [ ../snowglobe-network.nix ../gpu-unlock.nix ]; | |
| 7 | boot.initrd.systemd.enable = true; | |
| 8 | boot.initrd.kernelModules = [ "virtio_net" ]; | |
| 9 | networking.usePredictableInterfaceNames = lib.mkForce true; | |
| 10 | services.openssh.enable = true; | |
| 11 | virtualisation = { | |
| 12 | memorySize = 1024; | |
| 13 | cores = 2; | |
| 14 | vlans = []; | |
| 15 | qemu.networkingOptions = lib.mkForce [ | |
| 16 | "-netdev user,id=ethernet,net=10.0.0.0/24,host=10.0.0.2" | |
| 17 | "-device virtio-net-pci,netdev=ethernet,mac=${config.systemd.network.links."10-snowglobe-ethernet".matchConfig.MACAddress},addr=0x5" | |
| 18 | ]; | |
| 19 | }; | |
| 20 | }; | |
| 21 | testScript = '' | |
| 22 | import json | |
| 23 | start_all() | |
| 24 | for boot in range(2): | |
| 25 | machine.wait_for_unit("network-addresses-enp4s0.service") | |
| 26 | machine.wait_for_unit("sshd.service") | |
| 27 | address = json.loads(machine.succeed("ip -j -4 address show dev enp4s0"))[0] | |
| 28 | assert {item["local"] for item in address["addr_info"]} == {"10.0.0.1", "192.168.0.1"}, address | |
| 29 | route = json.loads(machine.succeed("ip -j route show default")) | |
| 30 | assert len(route) == 1 and route[0]["gateway"] == "10.0.0.2" and route[0]["dev"] == "enp4s0", route | |
| 31 | properties = machine.succeed("udevadm info -q property /sys/class/net/enp4s0") | |
| 32 | assert "ID_NET_LINK_FILE=/etc/systemd/network/10-snowglobe-ethernet.link" in properties, properties | |
| 33 | machine.succeed("ping -c 1 -W 2 10.0.0.2") | |
| 34 | machine.succeed("test -d /sys/module/gpu_unlock") | |
| 35 | machine.succeed("test ! -d /sys/module/nvidia") | |
| 36 | print(f"Boot {boot}: MAC naming, both addresses, default gateway, SSH, initrd GPU module passed") | |
| 37 | if boot == 0: | |
| 38 | machine.shutdown() | |
| 39 | machine.start() | |
| 40 | ''; | |
| 41 | } |
tools/benchmark-gpu-unlock.py created+222| ... | ... | @@ -0,0 +1,222 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | """Emit repeatable GPU-unlock measurements using synthetic inputs only.""" | |
| 3 | import argparse | |
| 4 | import ctypes as ct | |
| 5 | from datetime import datetime, timezone | |
| 6 | import hashlib | |
| 7 | import http.client | |
| 8 | import importlib.metadata | |
| 9 | import json | |
| 10 | import os | |
| 11 | from pathlib import Path | |
| 12 | import random | |
| 13 | import time | |
| 14 | ||
| 15 | ||
| 16 | def emit(**record): | |
| 17 | print(json.dumps(record), flush=True) | |
| 18 | ||
| 19 | ||
| 20 | def request(path, data=None): | |
| 21 | connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600) | |
| 22 | connection.request("GET" if data is None else "POST", path, | |
| 23 | None if data is None else json.dumps(data), { | |
| 24 | "Authorization": "Bearer " + os.environ["VLLM_API_KEY"], | |
| 25 | "Content-Type": "application/json", | |
| 26 | }) | |
| 27 | response = connection.getresponse() | |
| 28 | if response.status != 200: | |
| 29 | response.read() | |
| 30 | connection.close() | |
| 31 | raise RuntimeError(f"HTTP {response.status} at {path}") | |
| 32 | return connection, response | |
| 33 | ||
| 34 | ||
| 35 | def metrics(): | |
| 36 | connection, response = request("/metrics") | |
| 37 | try: | |
| 38 | values = {} | |
| 39 | for line in response.read().decode().splitlines(): | |
| 40 | if line.startswith("vllm:") and "_bucket{" not in line: | |
| 41 | name = line.split("{", 1)[0].split(" ", 1)[0] | |
| 42 | values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1]) | |
| 43 | return values | |
| 44 | finally: | |
| 45 | connection.close() | |
| 46 | ||
| 47 | ||
| 48 | def idle(): | |
| 49 | values = metrics() | |
| 50 | if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]: | |
| 51 | raise RuntimeError("Inference endpoint is busy") | |
| 52 | return values | |
| 53 | ||
| 54 | ||
| 55 | def inference(args): | |
| 56 | connection, response = request("/tokenize", { | |
| 57 | "model": "qwen3.8-27b", | |
| 58 | "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n", | |
| 59 | }) | |
| 60 | vocabulary = json.loads(response.read())["tokens"] | |
| 61 | connection.close() | |
| 62 | for length in args.lengths: | |
| 63 | for trial in range(args.repeats): | |
| 64 | prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length) | |
| 65 | before = idle() | |
| 66 | started = time.perf_counter() | |
| 67 | connection, response = request("/v1/completions", { | |
| 68 | "model": "qwen3.8-27b", "prompt": prompt, | |
| 69 | "max_tokens": args.output_tokens, "ignore_eos": True, | |
| 70 | "temperature": 0, "seed": 15027, | |
| 71 | "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}", | |
| 72 | "stream": True, "stream_options": {"include_usage": True}, | |
| 73 | }) | |
| 74 | first = None | |
| 75 | usage = None | |
| 76 | digest = hashlib.sha256() | |
| 77 | chunks = 0 | |
| 78 | try: | |
| 79 | for line in response: | |
| 80 | if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]": | |
| 81 | continue | |
| 82 | event = json.loads(line[6:]) | |
| 83 | for choice in event.get("choices", []): | |
| 84 | content = choice.get("text", "") | |
| 85 | if content: | |
| 86 | if first is None: | |
| 87 | first = time.perf_counter() | |
| 88 | chunks += 1 | |
| 89 | digest.update(content.encode()) | |
| 90 | if event.get("usage"): | |
| 91 | usage = event["usage"] | |
| 92 | finally: | |
| 93 | connection.close() | |
| 94 | elapsed = time.perf_counter() - started | |
| 95 | after = metrics() | |
| 96 | delta = {name: value - before.get(name, 0) for name, value in after.items()} | |
| 97 | valid = (usage is not None and usage["prompt_tokens"] == length | |
| 98 | and usage["completion_tokens"] == args.output_tokens | |
| 99 | and delta.get("vllm:request_success_total") == 1 | |
| 100 | and delta.get("vllm:prefix_cache_hits_total", 0) == 0) | |
| 101 | emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial, | |
| 102 | output_tokens=None if usage is None else usage["completion_tokens"], | |
| 103 | wall_seconds=elapsed, ttft_seconds=None if first is None else first - started, | |
| 104 | prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"), | |
| 105 | decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"), | |
| 106 | queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"), | |
| 107 | output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid, | |
| 108 | counters={name: value for name, value in delta.items() | |
| 109 | if name.endswith(("_sum", "_total")) and value}) | |
| 110 | if not valid: | |
| 111 | raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output") | |
| 112 | ||
| 113 | ||
| 114 | def tensor(args): | |
| 115 | # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model. | |
| 116 | cuda = ct.CDLL("libcudart.so.13") | |
| 117 | libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia")) | |
| 118 | blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13")))) | |
| 119 | cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t] | |
| 120 | cuda.cudaFree.argtypes = [ct.c_void_p] | |
| 121 | cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t] | |
| 122 | cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)] | |
| 123 | cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)] | |
| 124 | cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p] | |
| 125 | cuda.cudaEventSynchronize.argtypes = [ct.c_void_p] | |
| 126 | cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p] | |
| 127 | cuda.cudaEventDestroy.argtypes = [ct.c_void_p] | |
| 128 | blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)] | |
| 129 | blas.cublasDestroy_v2.argtypes = [ct.c_void_p] | |
| 130 | blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int] | |
| 131 | blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int, | |
| 132 | ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int, | |
| 133 | ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p, | |
| 134 | ct.c_int, ct.c_int, ct.c_int, ct.c_int] | |
| 135 | ||
| 136 | def checked(function, *values): | |
| 137 | status = function(*values) | |
| 138 | if status: | |
| 139 | raise RuntimeError(f"{function.__name__}: status {status}") | |
| 140 | ||
| 141 | idle() | |
| 142 | handle = ct.c_void_p() | |
| 143 | checked(blas.cublasCreate_v2, ct.byref(handle)) | |
| 144 | start, end = ct.c_void_p(), ct.c_void_p() | |
| 145 | try: | |
| 146 | checked(cuda.cudaEventCreate, ct.byref(start)) | |
| 147 | checked(cuda.cudaEventCreate, ct.byref(end)) | |
| 148 | checked(blas.cublasSetMathMode, handle, 16) | |
| 149 | for dtype, input_type, output_type, element_bytes, compute in ( | |
| 150 | ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68), | |
| 151 | ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68), | |
| 152 | ("int8_int32", 3, 10, 1, 72), | |
| 153 | ): | |
| 154 | for size in args.sizes: | |
| 155 | idle() | |
| 156 | free, total = ct.c_size_t(), ct.c_size_t() | |
| 157 | checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total)) | |
| 158 | needed = size * size * (element_bytes * 2 + 4) | |
| 159 | if free.value < needed + 64 * 1024**2: | |
| 160 | emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, | |
| 161 | skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed) | |
| 162 | continue | |
| 163 | buffers = [] | |
| 164 | try: | |
| 165 | for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,): | |
| 166 | pointer = ct.c_void_p() | |
| 167 | checked(cuda.cudaMalloc, ct.byref(pointer), byte_count) | |
| 168 | buffers.append(pointer) | |
| 169 | checked(cuda.cudaMemset, pointer, 60, byte_count) | |
| 170 | scalar = ct.c_int if output_type == 10 else ct.c_float | |
| 171 | alpha, beta = scalar(1), scalar(0) | |
| 172 | ||
| 173 | def multiply(): | |
| 174 | checked(blas.cublasGemmEx, handle, 0, 0, size, size, size, | |
| 175 | ct.byref(alpha), buffers[0], input_type, size, | |
| 176 | buffers[1], input_type, size, ct.byref(beta), | |
| 177 | buffers[2], output_type, size, compute, 99) | |
| 178 | ||
| 179 | for _ in range(10): | |
| 180 | multiply() | |
| 181 | checked(cuda.cudaDeviceSynchronize) | |
| 182 | for trial in range(args.repeats): | |
| 183 | checked(cuda.cudaEventRecord, start, None) | |
| 184 | for _ in range(100): | |
| 185 | multiply() | |
| 186 | checked(cuda.cudaEventRecord, end, None) | |
| 187 | checked(cuda.cudaEventSynchronize, end) | |
| 188 | milliseconds = ct.c_float() | |
| 189 | checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end) | |
| 190 | seconds = milliseconds.value / 1000 / 100 | |
| 191 | emit(kind="tensor", phase=args.phase, dtype=dtype, size=size, | |
| 192 | trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12) | |
| 193 | finally: | |
| 194 | for pointer in buffers: | |
| 195 | checked(cuda.cudaFree, pointer) | |
| 196 | finally: | |
| 197 | if start: | |
| 198 | checked(cuda.cudaEventDestroy, start) | |
| 199 | if end: | |
| 200 | checked(cuda.cudaEventDestroy, end) | |
| 201 | checked(blas.cublasDestroy_v2, handle) | |
| 202 | ||
| 203 | ||
| 204 | if __name__ == "__main__": | |
| 205 | parser = argparse.ArgumentParser(description=__doc__) | |
| 206 | parser.add_argument("kind", choices=("inference", "tensor")) | |
| 207 | parser.add_argument("--phase", required=True) | |
| 208 | parser.add_argument("--repeats", type=int, default=3) | |
| 209 | parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000]) | |
| 210 | parser.add_argument("--output-tokens", type=int, default=256) | |
| 211 | parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048]) | |
| 212 | args = parser.parse_args() | |
| 213 | emit(kind="metadata", phase=args.phase, benchmark=args.kind, | |
| 214 | utc=datetime.now(timezone.utc).isoformat(), | |
| 215 | script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| 216 | model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(), | |
| 217 | versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")}, | |
| 218 | arguments=vars(args)) | |
| 219 | if args.kind == "inference": | |
| 220 | inference(args) | |
| 221 | else: | |
| 222 | tensor(args) |
tools/deploy-test.py+10| ... | ... | @@ -13,6 +13,16 @@ import deploy |
| 13 | 13 | import release |
| 14 | 14 | |
| 15 | 15 | |
| 16 | class HostStage(unittest.TestCase): | |
| 17 | def test_prepares_working_host_without_publishing_or_deploying_services(self): | |
| 18 | with patch("sys.argv", ["deploy.py", "stage", "host"]), patch.object( | |
| 19 | deploy, "upload", return_value="0123456789abcdef", | |
| 20 | ) as upload, patch.object(deploy, "run_logged") as run: | |
| 21 | deploy.main() | |
| 22 | upload.assert_called_once_with() | |
| 23 | run.assert_called_once_with("stage", "host", "python3 /opt/snowglobe/releases/0123456789abcdef/tools/release.py prepare-boot 0123456789abcdef") | |
| 24 | ||
| 25 | ||
| 16 | 26 | class MainExport(unittest.TestCase): |
| 17 | 27 | def test_production_exports_main_while_staging_exports_working_files(self): |
| 18 | 28 | with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary: |
tools/deploy.py+5-2| ... | ... | @@ -187,8 +187,8 @@ def main(): |
| 187 | 187 | parser.error("rollback target must be a release ID") |
| 188 | 188 | if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target: |
| 189 | 189 | parser.error(f"{args.mode} takes no target") |
| 190 | if args.env and args.mode != "stage": | |
| 191 | parser.error("--env is available only for stage") | |
| 190 | if args.env and (args.mode != "stage" or args.target == "host"): | |
| 191 | parser.error("--env requires a service or dashboard stage") | |
| 192 | 192 | if bool(args.file) != (args.mode == "secrets"): |
| 193 | 193 | parser.error("--file is required only for secrets") |
| 194 | 194 | if args.key and args.mode != "secrets": |
| ... | ... | @@ -212,6 +212,9 @@ def main(): |
| 212 | 212 | manager = f"python3 {REMOTE}/release.py" |
| 213 | 213 | if args.mode == "stage": |
| 214 | 214 | release = upload() |
| 215 | if args.target == "host": | |
| 216 | run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}") | |
| 217 | return | |
| 215 | 218 | overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env) |
| 216 | 219 | if args.computer: |
| 217 | 220 | overrides += " --computer " + shlex.quote(args.computer) |
tools/gpu-unlock.md created+35| ... | ... | @@ -0,0 +1,35 @@ |
| 1 | # GPU tensor unlock | |
| 2 | ||
| 3 | [The host module](../nixos/gpu-unlock.nix) builds the pinned upstream kernel | |
| 4 | module and extracts its NVIDIA firmware without executing the driver installer. | |
| 5 | It runs in the initrd before the normal NVIDIA driver. The installed driver | |
| 6 | and kernel stay on the repository's existing versions. The target is the | |
| 7 | current GPU's PCI address; a different address is skipped by upstream. | |
| 8 | ||
| 9 | `python3 tools/deploy.py stage host` builds and installs a candidate boot | |
| 10 | generation from an immutable uploaded release. It preserves the previous | |
| 11 | default boot entry and leaves the running host, services, and published release | |
| 12 | unchanged. It prints the candidate entry and fallback. Only after reboot | |
| 13 | approval, select the printed entry with `bootctl set-oneshot <entry>` and | |
| 14 | reboot. A subsequent boot uses the previous default again. Promote the reviewed | |
| 15 | source through the normal main deployment workflow when retaining the change. | |
| 16 | After promotion, clear the trial's EFI default with `bootctl set-default ''` | |
| 17 | so NixOS's `loader.conf` selects the production generation on subsequent boots. | |
| 18 | ||
| 19 | Before rebooting, inspect the candidate and fallback initrds for the MAC-bound | |
| 20 | Ethernet link rule, and verify both static addresses and the gateway in the | |
| 21 | built system. The [isolated boot test](../nixos/tests/snowglobe-network.nix) | |
| 22 | exercises the same network module and GPU module in a disposable VM. | |
| 23 | ||
| 24 | Run [the benchmark](benchmark-gpu-unlock.py) with the existing inference image's | |
| 25 | Python and environment, using `inference --phase before-<run>` and | |
| 26 | `tensor --phase before-<run>`. Repeat with `after-<run>` after the approved boot. | |
| 27 | Keep the script, image, model, serving settings, lengths, and seeds identical. | |
| 28 | Each context length has three uncached requests with a fixed output budget; | |
| 29 | measurements include server prefill/decode time, streamed first-token latency, | |
| 30 | speculation counters, and output hashes. Tensor tests use CUDA events and | |
| 31 | FP32 accumulation, plus FP32 and integer controls, while retaining GPU memory | |
| 32 | for the resident model. The script rejects busy inference and records skipped | |
| 33 | tensor sizes if memory is insufficient. Save stdout with host/GPU metadata | |
| 34 | and clock, power, and temperature telemetry. These are synthetic performance | |
| 35 | probes, not model-quality evaluations. |
tools/mac-agent-vm.md+4-1| ... | ... | @@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da |
| 42 | 42 | |
| 43 | 43 | `start` resumes a suspended VM or boots a stopped VM, then waits for commands |
| 44 | 44 | to work through the logged-in guest agent. `stop` requests a graceful |
| 45 | guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM | |
| 45 | guest shutdown and waits for it to finish. `start` launches Tart through | |
| 46 | `ssh localhost` (needs Remote Login and this user's key) because closing an | |
| 47 | agent session kills every process it spawned, even in a new session; launchd | |
| 48 | jobs cannot read the VM volume (TCC). `suspend` waits for Tart's RAM | |
| 46 | 49 | snapshot; it writes memory to disk and is not an instant CPU pause. A running |
| 47 | 50 | VM holds a process-scoped idle-sleep assertion, so locking the host does not |
| 48 | 51 | cause idle sleep. Host login, FileVault, and explicit sleep remain host policy. |
tools/mac-agent-vm.py+10-12| ... | ... | @@ -10,6 +10,7 @@ import os |
| 10 | 10 | from pathlib import Path |
| 11 | 11 | import select |
| 12 | 12 | import secrets |
| 13 | import shlex | |
| 13 | 14 | import signal |
| 14 | 15 | import socket |
| 15 | 16 | import socketserver |
| ... | ... | @@ -247,20 +248,17 @@ def main(): |
| 247 | 248 | if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]: |
| 248 | 249 | print("GUI-agent VM is already running.") |
| 249 | 250 | return |
| 250 | with (PRIVATE / "run.log").open("w") as log: | |
| 251 | os.chmod(PRIVATE / "run.log", 0o600) | |
| 252 | process = subprocess.Popen([str(TART), "run", NAME, "--no-graphics", "--no-audio", | |
| 253 | "--no-usb-accessories", "--no-clipboard", "--suspendable", | |
| 254 | "--root-disk-opts=sync=full"], | |
| 255 | env={**os.environ, "TART_HOME": str(ROOT / "state")}, | |
| 256 | stdin=subprocess.DEVNULL, stdout=log, stderr=log, | |
| 257 | start_new_session=True) | |
| 258 | subprocess.Popen(["/usr/bin/caffeinate", "-i", "-w", str(process.pid)], | |
| 259 | stdin=subprocess.DEVNULL, stdout=log, stderr=log, | |
| 260 | start_new_session=True) | |
| 251 | # Closing an agent session kills every process it ever spawned, so sshd must own the VM. | |
| 252 | run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i", | |
| 253 | str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories", | |
| 254 | "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"]) | |
| 255 | (PRIVATE / "run.log").write_text("") | |
| 256 | os.chmod(PRIVATE / "run.log", 0o600) | |
| 257 | subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost", | |
| 258 | f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True) | |
| 261 | 259 | deadline = time.monotonic() + 90 |
| 262 | 260 | while time.monotonic() < deadline: |
| 263 | if process.poll() is not None: | |
| 261 | if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode: | |
| 264 | 262 | raise ValueError("VM startup failed; inspect the private run log locally.") |
| 265 | 263 | try: |
| 266 | 264 | tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5) |
tools/release-host-test.py+39| ... | ... | @@ -97,5 +97,44 @@ class ActivationEnvironment(unittest.TestCase): |
| 97 | 97 | self.assertLess(rename, deploy) |
| 98 | 98 | |
| 99 | 99 | |
| 100 | class BootPreparation(unittest.TestCase): | |
| 101 | def test_installs_a_boot_entry_and_preserves_the_default_without_activation(self): | |
| 102 | entries = [ | |
| 103 | [{"id": "previous.conf", "isDefault": True}], | |
| 104 | [{"id": "candidate.conf", "options": "init=/candidate/init loglevel=4"}], | |
| 105 | ] | |
| 106 | calls = [] | |
| 107 | ||
| 108 | def run(argv, **kwargs): | |
| 109 | calls.append(argv) | |
| 110 | return subprocess.CompletedProcess(argv, 0, json.dumps(entries.pop(0)) if argv[:2] == ["bootctl", "list"] else "") | |
| 111 | ||
| 112 | with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object( | |
| 113 | release, "host_configuration", return_value="snowglobe", | |
| 114 | ), patch.object(release.subprocess, "run", side_effect=run), patch.object( | |
| 115 | Path, "resolve", return_value=Path("/candidate"), | |
| 116 | ): | |
| 117 | release.prepare_boot("0123456789abcdef") | |
| 118 | self.assertEqual(calls, [ | |
| 119 | ["bootctl", "list", "--json=short"], | |
| 120 | ["nixos-rebuild", "boot", "--flake", "path:/candidate#snowglobe"], | |
| 121 | ["bootctl", "set-default", "previous.conf"], | |
| 122 | ["bootctl", "list", "--json=short"], | |
| 123 | ]) | |
| 124 | ||
| 125 | def test_failed_build_restores_the_default_without_selecting_a_candidate(self): | |
| 126 | with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object( | |
| 127 | release, "host_configuration", return_value="snowglobe", | |
| 128 | ), patch.object(release.subprocess, "run", side_effect=[ | |
| 129 | subprocess.CompletedProcess([], 0, '[{"id":"previous.conf","isDefault":true}]'), | |
| 130 | subprocess.CalledProcessError(1, "nixos-rebuild"), | |
| 131 | subprocess.CompletedProcess([], 0), | |
| 132 | ]) as command: | |
| 133 | with self.assertRaises(subprocess.CalledProcessError): | |
| 134 | release.prepare_boot("0123456789abcdef") | |
| 135 | self.assertEqual(command.call_args.args[0], ["bootctl", "set-default", "previous.conf"]) | |
| 136 | self.assertEqual(command.call_count, 3) | |
| 137 | ||
| 138 | ||
| 100 | 139 | if __name__ == "__main__": |
| 101 | 140 | unittest.main() |
tools/release.py+25-1| ... | ... | @@ -140,6 +140,25 @@ def host_configuration(path): |
| 140 | 140 | return name |
| 141 | 141 | |
| 142 | 142 | |
| 143 | def prepare_boot(release): | |
| 144 | path = check_release(release) | |
| 145 | configuration = host_configuration(path) | |
| 146 | entries = json.loads(subprocess.run( | |
| 147 | ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True, | |
| 148 | ).stdout) | |
| 149 | fallback = next(entry["id"] for entry in entries if entry["isDefault"]) | |
| 150 | try: | |
| 151 | subprocess.run(["nixos-rebuild", "boot", "--flake", f"path:{path.resolve()}#{configuration}"], check=True) | |
| 152 | finally: | |
| 153 | subprocess.run(["bootctl", "set-default", fallback], check=True) | |
| 154 | system = Path("/nix/var/nix/profiles/system").resolve() | |
| 155 | entries = json.loads(subprocess.run( | |
| 156 | ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True, | |
| 157 | ).stdout) | |
| 158 | candidate = next(entry["id"] for entry in entries if f"init={system}/init" in entry.get("options", "").split()) | |
| 159 | print(json.dumps({"boot_prepared": release, "entry": candidate, "fallback": fallback, "system": str(system)})) | |
| 160 | ||
| 161 | ||
| 143 | 162 | def activate(release, initial=False): |
| 144 | 163 | path = check_release(release) |
| 145 | 164 | script = path / "tools/snowglobe.py" |
| ... | ... | @@ -219,7 +238,7 @@ def activate(release, initial=False): |
| 219 | 238 | |
| 220 | 239 | def main(): |
| 221 | 240 | parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release") |
| 222 | parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy"]) | |
| 241 | parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy", "prepare-boot"]) | |
| 223 | 242 | parser.add_argument("target", nargs="?") |
| 224 | 243 | args = parser.parse_args() |
| 225 | 244 | STATE.mkdir(parents=True, exist_ok=True) |
| ... | ... | @@ -249,6 +268,11 @@ def main(): |
| 249 | 268 | return |
| 250 | 269 | with (STATE / "release.lock").open("w") as lock: |
| 251 | 270 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 271 | if args.mode == "prepare-boot": | |
| 272 | if not args.target: | |
| 273 | parser.error("prepare-boot requires a release ID") | |
| 274 | prepare_boot(args.target) | |
| 275 | return | |
| 252 | 276 | if args.mode == "publish": |
| 253 | 277 | if not args.target: |
| 254 | 278 | parser.error("publish requires a main release ID") |