authoravatarclover caruso<git@paperclover.net>2026-11-01 22:19:03-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
logdf4161ba6e1bae0f7c7596554bde1ccfe49ea94d
tree60ab485b2cfae8a17c940be301cd4cffb338bdca
parentffc3896c9d5ee7c48a19bfa615c002d2c9de7077
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Enable the Snow Globe GPU tensor unlock

Build pinned gpu-unlock firmware and an initrd module for the RTX 3090 before the NVIDIA driver loads. Add host boot preparation that preserves the existing default until promotion, plus repeatable inference and tensor benchmarks. Verified two isolated cold boots, a real generation-198 boot with the MAC-bound network intact, all service routes, authenticated inference, and 45 before/after benchmark samples per phase. Keep the installed kernel and NVIDIA driver versions unchanged. Assisted-by: gpt-6.1-sol

12 files changed, 445 insertions(+), 18 deletions(-)

nixos/configuration.nix+1-1
......@@ -47,7 +47,7 @@ let
4747 });
4848in
4949{
50 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ];
50 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" "gpu-unlock-firmware" ];
5151
5252 networking.firewall = {
5353 enable = true;
nixos/gpu-unlock.nix created+52
......@@ -0,0 +1,52 @@
1{ config, lib, pkgs, ... }:
2let
3 source = pkgs.fetchzip {
4 url = "https://github.com/Zagitta/gpu-unlock/archive/7eb51bccae0a0b7e599e084d971170e9a1e82d0c.tar.gz";
5 hash = "sha256-1KD/J1S1uyI5GGoTwQ1cU/sXeEKlG54exSvi6U+j73M=";
6 };
7 nvidiaVersion = "610.43.03";
8 nvidiaSource = pkgs.fetchurl {
9 url = "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-${nvidiaVersion}.tar.xz";
10 hash = "sha256-fhGJI8eiPtw2EU1jJzpG4+BOmvmGlaQiA+esLf6fwdw=";
11 };
12 nvidiaInstaller = pkgs.fetchurl {
13 url = "https://download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/NVIDIA-Linux-x86_64-${nvidiaVersion}-no-compat32.run";
14 hash = "sha256-4IRFHbcwpt8p09dPZVdJxNI248lF99lLUgXoQxj+L5A=";
15 };
16 firmware = pkgs.runCommand "gpu-unlock-firmware" {
17 nativeBuildInputs = [ pkgs.python3 pkgs.zstd ];
18 meta.license = lib.licenses.unfreeRedistributableFirmware;
19 } ''
20 tar -xf ${nvidiaSource}
21 python3 ${source}/firmware/extract_gsp.py ${nvidiaInstaller} assets
22 python3 ${source}/firmware/extract_boot_assets.py NVIDIA-kernel-module-source-${nvidiaVersion} assets
23 mkdir -p $out/lib/firmware/gpu-unlock/${nvidiaVersion} $out/share/licenses/gpu-unlock
24 install -m444 assets/*.bin $out/lib/firmware/gpu-unlock/${nvidiaVersion}/
25 install -m444 assets/NVIDIA-LICENSE ${source}/firmware/NOTICE.md $out/share/licenses/gpu-unlock/
26 '';
27 kernel = config.boot.kernelPackages.kernel;
28 module = kernel.stdenv.mkDerivation {
29 pname = "gpu-unlock";
30 version = "2.0.0";
31 src = source;
32 nativeBuildInputs = kernel.moduleBuildDependencies;
33 hardeningDisable = [ "pic" ];
34 buildPhase = ''
35 runHook preBuild
36 make -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M="$PWD/kernel/gpu_unlock" modules
37 runHook postBuild
38 '';
39 installPhase = ''
40 runHook preInstall
41 install -Dm444 kernel/gpu_unlock/gpu_unlock.ko $out/lib/modules/${kernel.modDirVersion}/extra/gpu_unlock.ko
42 runHook postInstall
43 '';
44 meta.license = lib.licenses.gpl2Only;
45 };
46in
47{
48 boot.extraModulePackages = [ module ];
49 hardware.firmware = [ firmware ];
50 boot.initrd.kernelModules = [ "gpu_unlock" ];
51 boot.kernelParams = [ "gpu_unlock.targets=0000:09:00.0" ];
52}
nixos/snowglobe.nix+1-1
......@@ -15,7 +15,7 @@ let
1515 };
1616in
1717{
18 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ];
18 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ];
1919
2020 networking.hostName = "snowglobe";
2121 environment.etc."machine-info".text = ''
nixos/tests/snowglobe-network.nix created+41
......@@ -0,0 +1,41 @@
1{ pkgs }:
2pkgs.testers.runNixOSTest {
3 name = "snowglobe-network";
4 globalTimeout = 300;
5 nodes.machine = { config, lib, ... }: {
6 imports = [ ../snowglobe-network.nix ../gpu-unlock.nix ];
7 boot.initrd.systemd.enable = true;
8 boot.initrd.kernelModules = [ "virtio_net" ];
9 networking.usePredictableInterfaceNames = lib.mkForce true;
10 services.openssh.enable = true;
11 virtualisation = {
12 memorySize = 1024;
13 cores = 2;
14 vlans = [];
15 qemu.networkingOptions = lib.mkForce [
16 "-netdev user,id=ethernet,net=10.0.0.0/24,host=10.0.0.2"
17 "-device virtio-net-pci,netdev=ethernet,mac=${config.systemd.network.links."10-snowglobe-ethernet".matchConfig.MACAddress},addr=0x5"
18 ];
19 };
20 };
21 testScript = ''
22 import json
23 start_all()
24 for boot in range(2):
25 machine.wait_for_unit("network-addresses-enp4s0.service")
26 machine.wait_for_unit("sshd.service")
27 address = json.loads(machine.succeed("ip -j -4 address show dev enp4s0"))[0]
28 assert {item["local"] for item in address["addr_info"]} == {"10.0.0.1", "192.168.0.1"}, address
29 route = json.loads(machine.succeed("ip -j route show default"))
30 assert len(route) == 1 and route[0]["gateway"] == "10.0.0.2" and route[0]["dev"] == "enp4s0", route
31 properties = machine.succeed("udevadm info -q property /sys/class/net/enp4s0")
32 assert "ID_NET_LINK_FILE=/etc/systemd/network/10-snowglobe-ethernet.link" in properties, properties
33 machine.succeed("ping -c 1 -W 2 10.0.0.2")
34 machine.succeed("test -d /sys/module/gpu_unlock")
35 machine.succeed("test ! -d /sys/module/nvidia")
36 print(f"Boot {boot}: MAC naming, both addresses, default gateway, SSH, initrd GPU module passed")
37 if boot == 0:
38 machine.shutdown()
39 machine.start()
40 '';
41}
tools/benchmark-gpu-unlock.py created+222
......@@ -0,0 +1,222 @@
1#!/usr/bin/env python3
2"""Emit repeatable GPU-unlock measurements using synthetic inputs only."""
3import argparse
4import ctypes as ct
5from datetime import datetime, timezone
6import hashlib
7import http.client
8import importlib.metadata
9import json
10import os
11from pathlib import Path
12import random
13import time
14
15
16def emit(**record):
17 print(json.dumps(record), flush=True)
18
19
20def request(path, data=None):
21 connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600)
22 connection.request("GET" if data is None else "POST", path,
23 None if data is None else json.dumps(data), {
24 "Authorization": "Bearer " + os.environ["VLLM_API_KEY"],
25 "Content-Type": "application/json",
26 })
27 response = connection.getresponse()
28 if response.status != 200:
29 response.read()
30 connection.close()
31 raise RuntimeError(f"HTTP {response.status} at {path}")
32 return connection, response
33
34
35def metrics():
36 connection, response = request("/metrics")
37 try:
38 values = {}
39 for line in response.read().decode().splitlines():
40 if line.startswith("vllm:") and "_bucket{" not in line:
41 name = line.split("{", 1)[0].split(" ", 1)[0]
42 values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1])
43 return values
44 finally:
45 connection.close()
46
47
48def idle():
49 values = metrics()
50 if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]:
51 raise RuntimeError("Inference endpoint is busy")
52 return values
53
54
55def inference(args):
56 connection, response = request("/tokenize", {
57 "model": "qwen3.8-27b",
58 "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n",
59 })
60 vocabulary = json.loads(response.read())["tokens"]
61 connection.close()
62 for length in args.lengths:
63 for trial in range(args.repeats):
64 prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length)
65 before = idle()
66 started = time.perf_counter()
67 connection, response = request("/v1/completions", {
68 "model": "qwen3.8-27b", "prompt": prompt,
69 "max_tokens": args.output_tokens, "ignore_eos": True,
70 "temperature": 0, "seed": 15027,
71 "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}",
72 "stream": True, "stream_options": {"include_usage": True},
73 })
74 first = None
75 usage = None
76 digest = hashlib.sha256()
77 chunks = 0
78 try:
79 for line in response:
80 if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]":
81 continue
82 event = json.loads(line[6:])
83 for choice in event.get("choices", []):
84 content = choice.get("text", "")
85 if content:
86 if first is None:
87 first = time.perf_counter()
88 chunks += 1
89 digest.update(content.encode())
90 if event.get("usage"):
91 usage = event["usage"]
92 finally:
93 connection.close()
94 elapsed = time.perf_counter() - started
95 after = metrics()
96 delta = {name: value - before.get(name, 0) for name, value in after.items()}
97 valid = (usage is not None and usage["prompt_tokens"] == length
98 and usage["completion_tokens"] == args.output_tokens
99 and delta.get("vllm:request_success_total") == 1
100 and delta.get("vllm:prefix_cache_hits_total", 0) == 0)
101 emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial,
102 output_tokens=None if usage is None else usage["completion_tokens"],
103 wall_seconds=elapsed, ttft_seconds=None if first is None else first - started,
104 prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"),
105 decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"),
106 queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"),
107 output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid,
108 counters={name: value for name, value in delta.items()
109 if name.endswith(("_sum", "_total")) and value})
110 if not valid:
111 raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output")
112
113
114def tensor(args):
115 # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model.
116 cuda = ct.CDLL("libcudart.so.13")
117 libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia"))
118 blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13"))))
119 cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t]
120 cuda.cudaFree.argtypes = [ct.c_void_p]
121 cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t]
122 cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)]
123 cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)]
124 cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p]
125 cuda.cudaEventSynchronize.argtypes = [ct.c_void_p]
126 cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p]
127 cuda.cudaEventDestroy.argtypes = [ct.c_void_p]
128 blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)]
129 blas.cublasDestroy_v2.argtypes = [ct.c_void_p]
130 blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int]
131 blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int,
132 ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int,
133 ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p,
134 ct.c_int, ct.c_int, ct.c_int, ct.c_int]
135
136 def checked(function, *values):
137 status = function(*values)
138 if status:
139 raise RuntimeError(f"{function.__name__}: status {status}")
140
141 idle()
142 handle = ct.c_void_p()
143 checked(blas.cublasCreate_v2, ct.byref(handle))
144 start, end = ct.c_void_p(), ct.c_void_p()
145 try:
146 checked(cuda.cudaEventCreate, ct.byref(start))
147 checked(cuda.cudaEventCreate, ct.byref(end))
148 checked(blas.cublasSetMathMode, handle, 16)
149 for dtype, input_type, output_type, element_bytes, compute in (
150 ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68),
151 ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68),
152 ("int8_int32", 3, 10, 1, 72),
153 ):
154 for size in args.sizes:
155 idle()
156 free, total = ct.c_size_t(), ct.c_size_t()
157 checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total))
158 needed = size * size * (element_bytes * 2 + 4)
159 if free.value < needed + 64 * 1024**2:
160 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
161 skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed)
162 continue
163 buffers = []
164 try:
165 for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,):
166 pointer = ct.c_void_p()
167 checked(cuda.cudaMalloc, ct.byref(pointer), byte_count)
168 buffers.append(pointer)
169 checked(cuda.cudaMemset, pointer, 60, byte_count)
170 scalar = ct.c_int if output_type == 10 else ct.c_float
171 alpha, beta = scalar(1), scalar(0)
172
173 def multiply():
174 checked(blas.cublasGemmEx, handle, 0, 0, size, size, size,
175 ct.byref(alpha), buffers[0], input_type, size,
176 buffers[1], input_type, size, ct.byref(beta),
177 buffers[2], output_type, size, compute, 99)
178
179 for _ in range(10):
180 multiply()
181 checked(cuda.cudaDeviceSynchronize)
182 for trial in range(args.repeats):
183 checked(cuda.cudaEventRecord, start, None)
184 for _ in range(100):
185 multiply()
186 checked(cuda.cudaEventRecord, end, None)
187 checked(cuda.cudaEventSynchronize, end)
188 milliseconds = ct.c_float()
189 checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end)
190 seconds = milliseconds.value / 1000 / 100
191 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
192 trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12)
193 finally:
194 for pointer in buffers:
195 checked(cuda.cudaFree, pointer)
196 finally:
197 if start:
198 checked(cuda.cudaEventDestroy, start)
199 if end:
200 checked(cuda.cudaEventDestroy, end)
201 checked(blas.cublasDestroy_v2, handle)
202
203
204if __name__ == "__main__":
205 parser = argparse.ArgumentParser(description=__doc__)
206 parser.add_argument("kind", choices=("inference", "tensor"))
207 parser.add_argument("--phase", required=True)
208 parser.add_argument("--repeats", type=int, default=3)
209 parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000])
210 parser.add_argument("--output-tokens", type=int, default=256)
211 parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048])
212 args = parser.parse_args()
213 emit(kind="metadata", phase=args.phase, benchmark=args.kind,
214 utc=datetime.now(timezone.utc).isoformat(),
215 script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
216 model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(),
217 versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")},
218 arguments=vars(args))
219 if args.kind == "inference":
220 inference(args)
221 else:
222 tensor(args)
tools/deploy-test.py+10
......@@ -13,6 +13,16 @@ import deploy
1313import release
1414
1515
16class HostStage(unittest.TestCase):
17 def test_prepares_working_host_without_publishing_or_deploying_services(self):
18 with patch("sys.argv", ["deploy.py", "stage", "host"]), patch.object(
19 deploy, "upload", return_value="0123456789abcdef",
20 ) as upload, patch.object(deploy, "run_logged") as run:
21 deploy.main()
22 upload.assert_called_once_with()
23 run.assert_called_once_with("stage", "host", "python3 /opt/snowglobe/releases/0123456789abcdef/tools/release.py prepare-boot 0123456789abcdef")
24
25
1626class MainExport(unittest.TestCase):
1727 def test_production_exports_main_while_staging_exports_working_files(self):
1828 with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary:
tools/deploy.py+5-2
......@@ -187,8 +187,8 @@ def main():
187187 parser.error("rollback target must be a release ID")
188188 if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target:
189189 parser.error(f"{args.mode} takes no target")
190 if args.env and args.mode != "stage":
191 parser.error("--env is available only for stage")
190 if args.env and (args.mode != "stage" or args.target == "host"):
191 parser.error("--env requires a service or dashboard stage")
192192 if bool(args.file) != (args.mode == "secrets"):
193193 parser.error("--file is required only for secrets")
194194 if args.key and args.mode != "secrets":
......@@ -212,6 +212,9 @@ def main():
212212 manager = f"python3 {REMOTE}/release.py"
213213 if args.mode == "stage":
214214 release = upload()
215 if args.target == "host":
216 run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}")
217 return
215218 overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env)
216219 if args.computer:
217220 overrides += " --computer " + shlex.quote(args.computer)
tools/gpu-unlock.md created+35
......@@ -0,0 +1,35 @@
1# GPU tensor unlock
2
3[The host module](../nixos/gpu-unlock.nix) builds the pinned upstream kernel
4module and extracts its NVIDIA firmware without executing the driver installer.
5It runs in the initrd before the normal NVIDIA driver. The installed driver
6and kernel stay on the repository's existing versions. The target is the
7current GPU's PCI address; a different address is skipped by upstream.
8
9`python3 tools/deploy.py stage host` builds and installs a candidate boot
10generation from an immutable uploaded release. It preserves the previous
11default boot entry and leaves the running host, services, and published release
12unchanged. It prints the candidate entry and fallback. Only after reboot
13approval, select the printed entry with `bootctl set-oneshot <entry>` and
14reboot. A subsequent boot uses the previous default again. Promote the reviewed
15source through the normal main deployment workflow when retaining the change.
16After promotion, clear the trial's EFI default with `bootctl set-default ''`
17so NixOS's `loader.conf` selects the production generation on subsequent boots.
18
19Before rebooting, inspect the candidate and fallback initrds for the MAC-bound
20Ethernet link rule, and verify both static addresses and the gateway in the
21built system. The [isolated boot test](../nixos/tests/snowglobe-network.nix)
22exercises the same network module and GPU module in a disposable VM.
23
24Run [the benchmark](benchmark-gpu-unlock.py) with the existing inference image's
25Python and environment, using `inference --phase before-<run>` and
26`tensor --phase before-<run>`. Repeat with `after-<run>` after the approved boot.
27Keep the script, image, model, serving settings, lengths, and seeds identical.
28Each context length has three uncached requests with a fixed output budget;
29measurements include server prefill/decode time, streamed first-token latency,
30speculation counters, and output hashes. Tensor tests use CUDA events and
31FP32 accumulation, plus FP32 and integer controls, while retaining GPU memory
32for the resident model. The script rejects busy inference and records skipped
33tensor sizes if memory is insufficient. Save stdout with host/GPU metadata
34and clock, power, and temperature telemetry. These are synthetic performance
35probes, not model-quality evaluations.
tools/mac-agent-vm.md+4-1
......@@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da
4242
4343`start` resumes a suspended VM or boots a stopped VM, then waits for commands
4444to work through the logged-in guest agent. `stop` requests a graceful
45guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM
45guest shutdown and waits for it to finish. `start` launches Tart through
46`ssh localhost` (needs Remote Login and this user's key) because closing an
47agent session kills every process it spawned, even in a new session; launchd
48jobs cannot read the VM volume (TCC). `suspend` waits for Tart's RAM
4649snapshot; it writes memory to disk and is not an instant CPU pause. A running
4750VM holds a process-scoped idle-sleep assertion, so locking the host does not
4851cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.
tools/mac-agent-vm.py+10-12
......@@ -10,6 +10,7 @@ import os
1010from pathlib import Path
1111import select
1212import secrets
13import shlex
1314import signal
1415import socket
1516import socketserver
......@@ -247,20 +248,17 @@ def main():
247248 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]:
248249 print("GUI-agent VM is already running.")
249250 return
250 with (PRIVATE / "run.log").open("w") as log:
251 os.chmod(PRIVATE / "run.log", 0o600)
252 process = subprocess.Popen([str(TART), "run", NAME, "--no-graphics", "--no-audio",
253 "--no-usb-accessories", "--no-clipboard", "--suspendable",
254 "--root-disk-opts=sync=full"],
255 env={**os.environ, "TART_HOME": str(ROOT / "state")},
256 stdin=subprocess.DEVNULL, stdout=log, stderr=log,
257 start_new_session=True)
258 subprocess.Popen(["/usr/bin/caffeinate", "-i", "-w", str(process.pid)],
259 stdin=subprocess.DEVNULL, stdout=log, stderr=log,
260 start_new_session=True)
251 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.
252 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",
253 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",
254 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"])
255 (PRIVATE / "run.log").write_text("")
256 os.chmod(PRIVATE / "run.log", 0o600)
257 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",
258 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)
261259 deadline = time.monotonic() + 90
262260 while time.monotonic() < deadline:
263 if process.poll() is not None:
261 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:
264262 raise ValueError("VM startup failed; inspect the private run log locally.")
265263 try:
266264 tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5)
tools/release-host-test.py+39
......@@ -97,5 +97,44 @@ class ActivationEnvironment(unittest.TestCase):
9797 self.assertLess(rename, deploy)
9898
9999
100class BootPreparation(unittest.TestCase):
101 def test_installs_a_boot_entry_and_preserves_the_default_without_activation(self):
102 entries = [
103 [{"id": "previous.conf", "isDefault": True}],
104 [{"id": "candidate.conf", "options": "init=/candidate/init loglevel=4"}],
105 ]
106 calls = []
107
108 def run(argv, **kwargs):
109 calls.append(argv)
110 return subprocess.CompletedProcess(argv, 0, json.dumps(entries.pop(0)) if argv[:2] == ["bootctl", "list"] else "")
111
112 with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object(
113 release, "host_configuration", return_value="snowglobe",
114 ), patch.object(release.subprocess, "run", side_effect=run), patch.object(
115 Path, "resolve", return_value=Path("/candidate"),
116 ):
117 release.prepare_boot("0123456789abcdef")
118 self.assertEqual(calls, [
119 ["bootctl", "list", "--json=short"],
120 ["nixos-rebuild", "boot", "--flake", "path:/candidate#snowglobe"],
121 ["bootctl", "set-default", "previous.conf"],
122 ["bootctl", "list", "--json=short"],
123 ])
124
125 def test_failed_build_restores_the_default_without_selecting_a_candidate(self):
126 with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object(
127 release, "host_configuration", return_value="snowglobe",
128 ), patch.object(release.subprocess, "run", side_effect=[
129 subprocess.CompletedProcess([], 0, '[{"id":"previous.conf","isDefault":true}]'),
130 subprocess.CalledProcessError(1, "nixos-rebuild"),
131 subprocess.CompletedProcess([], 0),
132 ]) as command:
133 with self.assertRaises(subprocess.CalledProcessError):
134 release.prepare_boot("0123456789abcdef")
135 self.assertEqual(command.call_args.args[0], ["bootctl", "set-default", "previous.conf"])
136 self.assertEqual(command.call_count, 3)
137
138
100139if __name__ == "__main__":
101140 unittest.main()
tools/release.py+25-1
......@@ -140,6 +140,25 @@ def host_configuration(path):
140140 return name
141141
142142
143def prepare_boot(release):
144 path = check_release(release)
145 configuration = host_configuration(path)
146 entries = json.loads(subprocess.run(
147 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
148 ).stdout)
149 fallback = next(entry["id"] for entry in entries if entry["isDefault"])
150 try:
151 subprocess.run(["nixos-rebuild", "boot", "--flake", f"path:{path.resolve()}#{configuration}"], check=True)
152 finally:
153 subprocess.run(["bootctl", "set-default", fallback], check=True)
154 system = Path("/nix/var/nix/profiles/system").resolve()
155 entries = json.loads(subprocess.run(
156 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
157 ).stdout)
158 candidate = next(entry["id"] for entry in entries if f"init={system}/init" in entry.get("options", "").split())
159 print(json.dumps({"boot_prepared": release, "entry": candidate, "fallback": fallback, "system": str(system)}))
160
161
143162def activate(release, initial=False):
144163 path = check_release(release)
145164 script = path / "tools/snowglobe.py"
......@@ -219,7 +238,7 @@ def activate(release, initial=False):
219238
220239def main():
221240 parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release")
222 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy"])
241 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy", "prepare-boot"])
223242 parser.add_argument("target", nargs="?")
224243 args = parser.parse_args()
225244 STATE.mkdir(parents=True, exist_ok=True)
......@@ -249,6 +268,11 @@ def main():
249268 return
250269 with (STATE / "release.lock").open("w") as lock:
251270 fcntl.flock(lock, fcntl.LOCK_EX)
271 if args.mode == "prepare-boot":
272 if not args.target:
273 parser.error("prepare-boot requires a release ID")
274 prepare_boot(args.target)
275 return
252276 if args.mode == "publish":
253277 if not args.target:
254278 parser.error("publish requires a main release ID")