1#!/usr/bin/env python3
2import argparse
3from datetime import datetime, timezone
4import fcntl
5import hashlib
6import json
7import os
8from pathlib import Path
9import re
10import subprocess
11import time
12
13
14ROOT = Path("/opt/snowglobe")
15RELEASES = ROOT / "releases"
16STATE = Path("/var/lib/snowglobe")
17HISTORY = STATE / "deployments.json"
18SOURCES = ("config", "service", "tools", "nixos", "dashboard", "guest", "flake.nix", "flake.lock", "readme.md", "AGENTS.md")
19BUILD_DIRS = ("dashboard/node_modules", "dashboard/dist", "dashboard/.cache", "dashboard/data", "dashboard/target", "guest/build")
20RELEASE_ID = re.compile(r"[0-9a-f]{16}\Z")
21STAGE_ID = re.compile(r"[a-z][a-z0-9-]*\Z")
22
23
24def files(root):
25 for name in SOURCES:
26 source = root / name
27 if not source.exists() and not source.is_symlink():
28 continue
29 paths = source.rglob("*") if source.is_dir() else [source]
30 for path in sorted(paths):
31 relative = path.relative_to(root)
32 if any(relative.is_relative_to(directory) for directory in BUILD_DIRS):
33 continue
34 if any(part in {".DS_Store", "__pycache__", ".identities.lock", "identities.pending"} or part.startswith("._") or part.endswith(".pyc") for part in relative.parts):
35 continue
36 if path.is_symlink():
37 raise ValueError(f"unsupported release file: {relative}")
38 if path.is_dir():
39 continue
40 if not path.is_file():
41 raise ValueError(f"unsupported release file: {relative}")
42 yield path, relative
43
44
45def _hash_contents(digest, path):
46 with path.open("rb") as source:
47 while chunk := source.read(65536):
48 digest.update(chunk)
49
50
51def tree_digest(root):
52 digest = hashlib.sha256()
53 for path, relative in files(root):
54 digest.update(str(relative).encode() + b"\0")
55 _hash_contents(digest, path)
56 return digest.hexdigest()
57
58
59def host_digest(root):
60 digest = hashlib.sha256()
61 paths = sorted(path for path, relative in files(root)
62 if relative.parts[0] in {"nixos", "dashboard", "config", "service"}
63 or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"})
64 for path in paths:
65 digest.update(str(path.relative_to(root)).encode())
66 _hash_contents(digest, path)
67 return digest.hexdigest()
68
69
70def check_release(release):
71 if not RELEASE_ID.fullmatch(release):
72 raise ValueError("invalid release ID")
73 path = RELEASES / release
74 if not path.is_dir():
75 raise ValueError(f"release is missing: {release}")
76 manifest = path / ".snowglobe-release.json"
77 if not manifest.exists():
78 raise ValueError(f"release has no manifest: {release}")
79 details = json.loads(manifest.read_text())
80 if details.get("id") != release or details.get("digest") != tree_digest(path) or details.get("version", 1) not in (1, 2):
81 raise ValueError(f"release contents changed: {release}")
82 return path
83
84
85def current_release(link="current"):
86 current = ROOT / link
87 if not current.is_symlink():
88 return None
89 target = current.resolve()
90 if target.parent != RELEASES.resolve() or not RELEASE_ID.fullmatch(target.name):
91 raise ValueError(f"{link} points outside releases: {target}")
92 return target.name
93
94
95def main_release(version=None):
96 version = version or current_release("main")
97 if version is None:
98 return None
99 if not RELEASE_ID.fullmatch(version):
100 raise ValueError("invalid release ID")
101 details = json.loads((RELEASES / version / ".snowglobe-release.json").read_text())
102 revision = details.get("main")
103 if (not isinstance(revision, dict) or not isinstance(revision.get("commit"), str)
104 or not re.fullmatch(r"[0-9a-f]{40}", revision["commit"])
105 or not isinstance(revision.get("description"), str) or not revision["description"].strip()):
106 raise ValueError("This release did not come from a described main commit. Publish main first.")
107 return {"release": version, "commit": revision["commit"], "description": revision["description"]}
108
109
110def history():
111 return json.loads(HISTORY.read_text()) if HISTORY.exists() else []
112
113
114def save_history(entries):
115 pending = HISTORY.with_suffix(".pending")
116 pending.write_text(json.dumps(entries, indent=2) + "\n")
117 pending.replace(HISTORY)
118
119
120def jobs(path):
121 rendered = subprocess.run(
122 ["python3", str(path / "tools/snowglobe.py"), "render"],
123 check=True, capture_output=True, text=True,
124 ).stdout
125 result = set(re.findall(r'^job "([a-z][a-z0-9-]*)" \{$', rendered, re.MULTILINE))
126 if not result:
127 raise ValueError(f"no Nomad jobs rendered by {path}")
128 return result
129
130
131def host_configuration(path):
132 hostname = os.uname().nodename.split(".", 1)[0]
133 available = json.loads(subprocess.run(
134 ["nix", "--extra-experimental-features", "nix-command flakes", "eval", "--json", f"path:{path.resolve()}#nixosConfigurations", "--apply", "builtins.attrNames"],
135 check=True, capture_output=True, text=True,
136 ).stdout)
137 name = "vm" if hostname == "snowglobe-rehearsal" else hostname
138 if name not in available:
139 raise ValueError(f"release has no NixOS configuration for {hostname}")
140 return name
141
142
143def prepare_boot(release):
144 path = check_release(release)
145 configuration = host_configuration(path)
146 entries = json.loads(subprocess.run(
147 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
148 ).stdout)
149 fallback = next(entry["id"] for entry in entries if entry["isDefault"])
150 try:
151 subprocess.run(["nixos-rebuild", "boot", "--flake", f"path:{path.resolve()}#{configuration}"], check=True)
152 finally:
153 subprocess.run(["bootctl", "set-default", fallback], check=True)
154 system = Path("/nix/var/nix/profiles/system").resolve()
155 entries = json.loads(subprocess.run(
156 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
157 ).stdout)
158 candidate = next(entry["id"] for entry in entries if f"init={system}/init" in entry.get("options", "").split())
159 print(json.dumps({"boot_prepared": release, "entry": candidate, "fallback": fallback, "system": str(system)}))
160
161
162def activate(release, initial=False):
163 path = check_release(release)
164 script = path / "tools/snowglobe.py"
165 environment = dict(os.environ)
166 managed = STATE / "managed-jobs.json"
167 current = current_release()
168 previous = set(json.loads(managed.read_text())) if managed.exists() else (
169 jobs(RELEASES / current) if current and not initial else set()
170 )
171 digest = host_digest(path)
172 recorded = ROOT / "host.digest"
173 old_digest = recorded.read_text().strip() if recorded.exists() else None
174 if old_digest != digest:
175 configuration = host_configuration(path)
176 subprocess.run(["nixos-rebuild", "dry-build", "--flake", f"path:{path.resolve()}#{configuration}"], check=True)
177 subprocess.run(["python3", str(script), "preflight"], check=True, env=environment)
178 backup = None
179 if current and not initial:
180 backup_script = ROOT / "data.py"
181 if not backup_script.is_file():
182 backup_script = path / "tools/data.py"
183 result = subprocess.run(
184 ["python3", str(backup_script), "backup", current, release],
185 check=True, capture_output=True, text=True,
186 )
187 print(result.stdout.strip(), flush=True)
188 backup = result.stdout.split("backup=", 1)[1].split()[0]
189 if old_digest != digest:
190 if current is None:
191 (ROOT / "current").symlink_to(path)
192 hostname = os.uname().nodename.split(".", 1)[0]
193 subprocess.run(["nixos-rebuild", "switch", "--flake", f"path:{path.resolve()}#{configuration}"], check=True)
194 configured_hostname = Path("/etc/hostname").read_text().strip()
195 if os.uname().nodename != configured_hostname:
196 subprocess.run(["hostnamectl", "set-hostname", "--transient", configured_hostname], check=True)
197 if os.environ.get("SNOWGLOBE_NODE_NAME", hostname) == hostname:
198 os.environ["SNOWGLOBE_NODE_NAME"] = os.uname().nodename.split(".", 1)[0]
199 environment = dict(os.environ)
200 next_link = ROOT / ("next-" + release)
201 next_link.unlink(missing_ok=True)
202 next_link.symlink_to(path)
203 next_link.replace(ROOT / "current")
204 if old_digest != digest:
205 recorded.write_text(digest + "\n")
206 for _ in range(60):
207 response = subprocess.run(
208 ["curl", "--fail", "--silent", "--max-time", "2", "http://127.0.0.1:4646/v1/status/leader"],
209 capture_output=True, text=True,
210 )
211 if response.returncode == 0 and json.loads(response.stdout):
212 break
213 time.sleep(2)
214 else:
215 raise RuntimeError("Nomad API did not become ready")
216 subprocess.run(["systemctl", "restart", "snowglobe-router.service"], check=True)
217 if (path / "nixos/dashboard.nix").exists():
218 subprocess.run(["systemctl", "start", "snowglobe-dashboard.service"], check=True)
219 subprocess.run(["systemctl", "is-active", "--quiet", "snowglobe-dashboard.service"], check=True)
220 subprocess.run(["python3", str(script), "pool"], check=True, env=environment)
221 subprocess.run(["python3", str(script), "deploy"], check=True, env=environment)
222 desired = jobs(path)
223 for name in sorted(previous - desired):
224 subprocess.run(
225 ["nomad", "job", "stop", "-purge", "-yes", name], check=True,
226 env={**os.environ, "NOMAD_TOKEN": (STATE / "nomad.token").read_text().strip()},
227 )
228 pending = managed.with_suffix(".pending")
229 pending.write_text(json.dumps(sorted(desired)) + "\n")
230 pending.replace(managed)
231 if (path / "tools/log-shipper.py").exists():
232 subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True)
233 if (path / "tools/substation-watch.py").exists():
234 subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True)
235 manifest = path / ".snowglobe-release.json"
236 if json.loads(manifest.read_text()).get("version", 1) >= 2:
237 subprocess.run(["python3", str(script), "check"], check=True, env=environment)
238 return backup
239
240
241def main():
242 parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release")
243 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy", "prepare-boot"])
244 parser.add_argument("target", nargs="?")
245 args = parser.parse_args()
246 STATE.mkdir(parents=True, exist_ok=True)
247 if args.mode == "stage-destroy":
248 if not args.target or not STAGE_ID.fullmatch(args.target):
249 parser.error("stage-destroy requires a stage ID")
250 saved = json.loads((STATE / "stages" / (args.target + ".json")).read_text())
251 path = check_release(saved["release"])
252 subprocess.run(["python3", str(path / "tools/snowglobe.py"), "destroy", args.target], check=True)
253 return
254 if args.mode == "verify":
255 if not args.target:
256 parser.error("verify requires a release ID")
257 check_release(args.target)
258 return
259 if args.mode == "history":
260 if args.target:
261 parser.error("history takes no target")
262 current = current_release()
263 entries = history()
264 for index, entry in enumerate(entries[-12:], start=max(0, len(entries) - 12)):
265 marker = "*" if index == len(entries) - 1 and entry["release"] == current else " "
266 when = datetime.fromtimestamp(entry["time"], timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
267 print(f"{marker} {entry['release']} {when} {entry['source']}")
268 if current and (not entries or entries[-1]["release"] != current):
269 print(f"* {current} deployment incomplete")
270 return
271 with (STATE / "release.lock").open("w") as lock:
272 fcntl.flock(lock, fcntl.LOCK_EX)
273 if args.mode == "prepare-boot":
274 if not args.target:
275 parser.error("prepare-boot requires a release ID")
276 prepare_boot(args.target)
277 return
278 if args.mode == "publish":
279 if not args.target:
280 parser.error("publish requires a main release ID")
281 path = check_release(args.target)
282 main_release(args.target)
283 pending = ROOT / "main.pending"
284 pending.unlink(missing_ok=True)
285 pending.symlink_to(path)
286 pending.replace(ROOT / "main")
287 print(f"main={args.target}")
288 return
289 entries = history()
290 current = current_release()
291 if current and not entries and args.mode != "bootstrap":
292 entries.append({"release": current, "source": "previous", "time": int(time.time())})
293 save_history(entries)
294 if args.mode == "deploy":
295 candidate = main_release()
296 if not candidate or args.target != candidate["release"]:
297 raise ValueError("main changed or is unavailable. Publish main and retry deployment.")
298 release = candidate["release"]
299 source = "main"
300 elif args.mode == "bootstrap":
301 if not args.target or not RELEASE_ID.fullmatch(args.target) or entries:
302 parser.error("bootstrap requires a release ID and no successful deployment")
303 release, source = args.target, "bootstrap"
304 main_release(release)
305 else:
306 if args.target:
307 match = next((item for item in reversed(entries) if item["release"] == args.target), None)
308 if not match:
309 parser.error("rollback target is not in production history")
310 else:
311 match = next((item for item in reversed(entries) if item["release"] != current), None)
312 if not match:
313 parser.error("no earlier production release")
314 release, source = match["release"], "rollback"
315 if release == current and entries and entries[-1]["release"] == current:
316 print(f"already running {release}")
317 return
318 backup = activate(release, args.mode == "bootstrap")
319 entries.append({"release": release, "source": source, "time": int(time.time()), "backup": backup})
320 save_history(entries)
321 print(f"production={release} previous={current or ''}")
322
323
324if __name__ == "__main__":
325 main()