| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | from datetime import datetime, timezone |
| 4 | import fcntl |
| 5 | import hashlib |
| 6 | import json |
| 7 | import os |
| 8 | from pathlib import Path |
| 9 | import re |
| 10 | import subprocess |
| 11 | import time |
| 12 | |
| 13 | |
| 14 | ROOT = Path("/opt/snowglobe") |
| 15 | RELEASES = ROOT / "releases" |
| 16 | STATE = Path("/var/lib/snowglobe") |
| 17 | HISTORY = STATE / "deployments.json" |
| 18 | SOURCES = ("config", "service", "tools", "nixos", "dashboard", "guest", "flake.nix", "flake.lock", "readme.md", "AGENTS.md") |
| 19 | BUILD_DIRS = ("dashboard/node_modules", "dashboard/dist", "dashboard/.cache", "dashboard/data", "dashboard/target", "guest/build") |
| 20 | RELEASE_ID = re.compile(r"[0-9a-f]{16}\Z") |
| 21 | STAGE_ID = re.compile(r"[a-z][a-z0-9-]*\Z") |
| 22 | |
| 23 | |
| 24 | def files(root): |
| 25 | for name in SOURCES: |
| 26 | source = root / name |
| 27 | if not source.exists() and not source.is_symlink(): |
| 28 | continue |
| 29 | paths = source.rglob("*") if source.is_dir() else [source] |
| 30 | for path in sorted(paths): |
| 31 | relative = path.relative_to(root) |
| 32 | if any(relative.is_relative_to(directory) for directory in BUILD_DIRS): |
| 33 | continue |
| 34 | if any(part in {".DS_Store", "__pycache__", ".identities.lock", "identities.pending"} or part.startswith("._") or part.endswith(".pyc") for part in relative.parts): |
| 35 | continue |
| 36 | if path.is_symlink(): |
| 37 | raise ValueError(f"unsupported release file: {relative}") |
| 38 | if path.is_dir(): |
| 39 | continue |
| 40 | if not path.is_file(): |
| 41 | raise ValueError(f"unsupported release file: {relative}") |
| 42 | yield path, relative |
| 43 | |
| 44 | |
| 45 | def _hash_contents(digest, path): |
| 46 | with path.open("rb") as source: |
| 47 | while chunk := source.read(65536): |
| 48 | digest.update(chunk) |
| 49 | |
| 50 | |
| 51 | def tree_digest(root): |
| 52 | digest = hashlib.sha256() |
| 53 | for path, relative in files(root): |
| 54 | digest.update(str(relative).encode() + b"\0") |
| 55 | _hash_contents(digest, path) |
| 56 | return digest.hexdigest() |
| 57 | |
| 58 | |
| 59 | def host_digest(root): |
| 60 | digest = hashlib.sha256() |
| 61 | paths = sorted(path for path, relative in files(root) |
| 62 | if relative.parts[0] in {"nixos", "dashboard", "config", "service"} |
| 63 | or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "tools/vm-rfb.py", "tools/vm-computer.py", "tools/vm-ai.py", "tools/mac-computer.py", "guest/computer-windows.cs", "guest/computer-legacy-mac.py", "guest/computer-linux.py", "guest/computer-linux-launch", "guest/computer-linux.nix", "guest/protocol.json"}) |
| 64 | for path in paths: |
| 65 | digest.update(str(path.relative_to(root)).encode()) |
| 66 | _hash_contents(digest, path) |
| 67 | return digest.hexdigest() |
| 68 | |
| 69 | |
| 70 | def check_release(release): |
| 71 | if not RELEASE_ID.fullmatch(release): |
| 72 | raise ValueError("invalid release ID") |
| 73 | path = RELEASES / release |
| 74 | if not path.is_dir(): |
| 75 | raise ValueError(f"release is missing: {release}") |
| 76 | manifest = path / ".snowglobe-release.json" |
| 77 | if not manifest.exists(): |
| 78 | raise ValueError(f"release has no manifest: {release}") |
| 79 | details = json.loads(manifest.read_text()) |
| 80 | if details.get("id") != release or details.get("digest") != tree_digest(path) or details.get("version", 1) not in (1, 2): |
| 81 | raise ValueError(f"release contents changed: {release}") |
| 82 | return path |
| 83 | |
| 84 | |
| 85 | def current_release(link="current"): |
| 86 | current = ROOT / link |
| 87 | if not current.is_symlink(): |
| 88 | return None |
| 89 | target = current.resolve() |
| 90 | if target.parent != RELEASES.resolve() or not RELEASE_ID.fullmatch(target.name): |
| 91 | raise ValueError(f"{link} points outside releases: {target}") |
| 92 | return target.name |
| 93 | |
| 94 | |
| 95 | def main_release(version=None): |
| 96 | version = version or current_release("main") |
| 97 | if version is None: |
| 98 | return None |
| 99 | if not RELEASE_ID.fullmatch(version): |
| 100 | raise ValueError("invalid release ID") |
| 101 | details = json.loads((RELEASES / version / ".snowglobe-release.json").read_text()) |
| 102 | revision = details.get("main") |
| 103 | if (not isinstance(revision, dict) or not isinstance(revision.get("commit"), str) |
| 104 | or not re.fullmatch(r"[0-9a-f]{40}", revision["commit"]) |
| 105 | or not isinstance(revision.get("description"), str) or not revision["description"].strip()): |
| 106 | raise ValueError("This release did not come from a described main commit. Publish main first.") |
| 107 | return {"release": version, "commit": revision["commit"], "description": revision["description"]} |
| 108 | |
| 109 | |
| 110 | def history(): |
| 111 | return json.loads(HISTORY.read_text()) if HISTORY.exists() else [] |
| 112 | |
| 113 | |
| 114 | def save_history(entries): |
| 115 | pending = HISTORY.with_suffix(".pending") |
| 116 | pending.write_text(json.dumps(entries, indent=2) + "\n") |
| 117 | pending.replace(HISTORY) |
| 118 | |
| 119 | |
| 120 | def jobs(path): |
| 121 | rendered = subprocess.run( |
| 122 | ["python3", str(path / "tools/snowglobe.py"), "render"], |
| 123 | check=True, capture_output=True, text=True, |
| 124 | ).stdout |
| 125 | result = set(re.findall(r'^job "([a-z][a-z0-9-]*)" \{$', rendered, re.MULTILINE)) |
| 126 | if not result: |
| 127 | raise ValueError(f"no Nomad jobs rendered by {path}") |
| 128 | return result |
| 129 | |
| 130 | |
| 131 | def host_configuration(path): |
| 132 | hostname = os.uname().nodename.split(".", 1)[0] |
| 133 | available = json.loads(subprocess.run( |
| 134 | ["nix", "--extra-experimental-features", "nix-command flakes", "eval", "--json", f"path:{path.resolve()}#nixosConfigurations", "--apply", "builtins.attrNames"], |
| 135 | check=True, capture_output=True, text=True, |
| 136 | ).stdout) |
| 137 | name = "vm" if hostname == "snowglobe-rehearsal" else hostname |
| 138 | if name not in available: |
| 139 | raise ValueError(f"release has no NixOS configuration for {hostname}") |
| 140 | return name |
| 141 | |
| 142 | |
| 143 | def prepare_boot(release): |
| 144 | path = check_release(release) |
| 145 | configuration = host_configuration(path) |
| 146 | entries = json.loads(subprocess.run( |
| 147 | ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True, |
| 148 | ).stdout) |
| 149 | fallback = next(entry["id"] for entry in entries if entry["isDefault"]) |
| 150 | try: |
| 151 | subprocess.run(["nixos-rebuild", "boot", "--flake", f"path:{path.resolve()}#{configuration}"], check=True) |
| 152 | finally: |
| 153 | subprocess.run(["bootctl", "set-default", fallback], check=True) |
| 154 | system = Path("/nix/var/nix/profiles/system").resolve() |
| 155 | entries = json.loads(subprocess.run( |
| 156 | ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True, |
| 157 | ).stdout) |
| 158 | candidate = next(entry["id"] for entry in entries if f"init={system}/init" in entry.get("options", "").split()) |
| 159 | print(json.dumps({"boot_prepared": release, "entry": candidate, "fallback": fallback, "system": str(system)})) |
| 160 | |
| 161 | |
| 162 | def activate(release, initial=False): |
| 163 | path = check_release(release) |
| 164 | script = path / "tools/snowglobe.py" |
| 165 | environment = dict(os.environ) |
| 166 | managed = STATE / "managed-jobs.json" |
| 167 | current = current_release() |
| 168 | previous = set(json.loads(managed.read_text())) if managed.exists() else ( |
| 169 | jobs(RELEASES / current) if current and not initial else set() |
| 170 | ) |
| 171 | digest = host_digest(path) |
| 172 | recorded = ROOT / "host.digest" |
| 173 | old_digest = recorded.read_text().strip() if recorded.exists() else None |
| 174 | if old_digest != digest: |
| 175 | configuration = host_configuration(path) |
| 176 | subprocess.run(["nixos-rebuild", "dry-build", "--flake", f"path:{path.resolve()}#{configuration}"], check=True) |
| 177 | subprocess.run(["python3", str(script), "preflight"], check=True, env=environment) |
| 178 | backup = None |
| 179 | if current and not initial: |
| 180 | backup_script = ROOT / "data.py" |
| 181 | if not backup_script.is_file(): |
| 182 | backup_script = path / "tools/data.py" |
| 183 | result = subprocess.run( |
| 184 | ["python3", str(backup_script), "backup", current, release], |
| 185 | check=True, capture_output=True, text=True, |
| 186 | ) |
| 187 | print(result.stdout.strip(), flush=True) |
| 188 | backup = result.stdout.split("backup=", 1)[1].split()[0] |
| 189 | if old_digest != digest: |
| 190 | if current is None: |
| 191 | (ROOT / "current").symlink_to(path) |
| 192 | hostname = os.uname().nodename.split(".", 1)[0] |
| 193 | subprocess.run(["nixos-rebuild", "switch", "--flake", f"path:{path.resolve()}#{configuration}"], check=True) |
| 194 | configured_hostname = Path("/etc/hostname").read_text().strip() |
| 195 | if os.uname().nodename != configured_hostname: |
| 196 | subprocess.run(["hostnamectl", "set-hostname", "--transient", configured_hostname], check=True) |
| 197 | if os.environ.get("SNOWGLOBE_NODE_NAME", hostname) == hostname: |
| 198 | os.environ["SNOWGLOBE_NODE_NAME"] = os.uname().nodename.split(".", 1)[0] |
| 199 | environment = dict(os.environ) |
| 200 | next_link = ROOT / ("next-" + release) |
| 201 | next_link.unlink(missing_ok=True) |
| 202 | next_link.symlink_to(path) |
| 203 | next_link.replace(ROOT / "current") |
| 204 | if old_digest != digest: |
| 205 | recorded.write_text(digest + "\n") |
| 206 | for _ in range(60): |
| 207 | response = subprocess.run( |
| 208 | ["curl", "--fail", "--silent", "--max-time", "2", "http://127.0.0.1:4646/v1/status/leader"], |
| 209 | capture_output=True, text=True, |
| 210 | ) |
| 211 | if response.returncode == 0 and json.loads(response.stdout): |
| 212 | break |
| 213 | time.sleep(2) |
| 214 | else: |
| 215 | raise RuntimeError("Nomad API did not become ready") |
| 216 | subprocess.run(["systemctl", "restart", "snowglobe-router.service"], check=True) |
| 217 | if (path / "nixos/dashboard.nix").exists(): |
| 218 | subprocess.run(["systemctl", "start", "snowglobe-dashboard.service"], check=True) |
| 219 | subprocess.run(["systemctl", "is-active", "--quiet", "snowglobe-dashboard.service"], check=True) |
| 220 | subprocess.run(["python3", str(script), "pool"], check=True, env=environment) |
| 221 | subprocess.run(["python3", str(script), "deploy"], check=True, env=environment) |
| 222 | desired = jobs(path) |
| 223 | for name in sorted(previous - desired): |
| 224 | subprocess.run( |
| 225 | ["nomad", "job", "stop", "-purge", "-yes", name], check=True, |
| 226 | env={**os.environ, "NOMAD_TOKEN": (STATE / "nomad.token").read_text().strip()}, |
| 227 | ) |
| 228 | pending = managed.with_suffix(".pending") |
| 229 | pending.write_text(json.dumps(sorted(desired)) + "\n") |
| 230 | pending.replace(managed) |
| 231 | if (path / "tools/log-shipper.py").exists(): |
| 232 | subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True) |
| 233 | if (path / "tools/substation-watch.py").exists(): |
| 234 | subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True) |
| 235 | manifest = path / ".snowglobe-release.json" |
| 236 | if json.loads(manifest.read_text()).get("version", 1) >= 2: |
| 237 | subprocess.run(["python3", str(script), "check"], check=True, env=environment) |
| 238 | return backup |
| 239 | |
| 240 | |
| 241 | def main(): |
| 242 | parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release") |
| 243 | parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy", "prepare-boot"]) |
| 244 | parser.add_argument("target", nargs="?") |
| 245 | args = parser.parse_args() |
| 246 | STATE.mkdir(parents=True, exist_ok=True) |
| 247 | if args.mode == "stage-destroy": |
| 248 | if not args.target or not STAGE_ID.fullmatch(args.target): |
| 249 | parser.error("stage-destroy requires a stage ID") |
| 250 | saved = json.loads((STATE / "stages" / (args.target + ".json")).read_text()) |
| 251 | path = check_release(saved["release"]) |
| 252 | subprocess.run(["python3", str(path / "tools/snowglobe.py"), "destroy", args.target], check=True) |
| 253 | return |
| 254 | if args.mode == "verify": |
| 255 | if not args.target: |
| 256 | parser.error("verify requires a release ID") |
| 257 | check_release(args.target) |
| 258 | return |
| 259 | if args.mode == "history": |
| 260 | if args.target: |
| 261 | parser.error("history takes no target") |
| 262 | current = current_release() |
| 263 | entries = history() |
| 264 | for index, entry in enumerate(entries[-12:], start=max(0, len(entries) - 12)): |
| 265 | marker = "*" if index == len(entries) - 1 and entry["release"] == current else " " |
| 266 | when = datetime.fromtimestamp(entry["time"], timezone.utc).strftime("%Y-%m-%d %H:%M UTC") |
| 267 | print(f"{marker} {entry['release']} {when} {entry['source']}") |
| 268 | if current and (not entries or entries[-1]["release"] != current): |
| 269 | print(f"* {current} deployment incomplete") |
| 270 | return |
| 271 | with (STATE / "release.lock").open("w") as lock: |
| 272 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 273 | if args.mode == "prepare-boot": |
| 274 | if not args.target: |
| 275 | parser.error("prepare-boot requires a release ID") |
| 276 | prepare_boot(args.target) |
| 277 | return |
| 278 | if args.mode == "publish": |
| 279 | if not args.target: |
| 280 | parser.error("publish requires a main release ID") |
| 281 | path = check_release(args.target) |
| 282 | main_release(args.target) |
| 283 | pending = ROOT / "main.pending" |
| 284 | pending.unlink(missing_ok=True) |
| 285 | pending.symlink_to(path) |
| 286 | pending.replace(ROOT / "main") |
| 287 | print(f"main={args.target}") |
| 288 | return |
| 289 | entries = history() |
| 290 | current = current_release() |
| 291 | if current and not entries and args.mode != "bootstrap": |
| 292 | entries.append({"release": current, "source": "previous", "time": int(time.time())}) |
| 293 | save_history(entries) |
| 294 | if args.mode == "deploy": |
| 295 | candidate = main_release() |
| 296 | if not candidate or args.target != candidate["release"]: |
| 297 | raise ValueError("main changed or is unavailable. Publish main and retry deployment.") |
| 298 | release = candidate["release"] |
| 299 | source = "main" |
| 300 | elif args.mode == "bootstrap": |
| 301 | if not args.target or not RELEASE_ID.fullmatch(args.target) or entries: |
| 302 | parser.error("bootstrap requires a release ID and no successful deployment") |
| 303 | release, source = args.target, "bootstrap" |
| 304 | main_release(release) |
| 305 | else: |
| 306 | if args.target: |
| 307 | match = next((item for item in reversed(entries) if item["release"] == args.target), None) |
| 308 | if not match: |
| 309 | parser.error("rollback target is not in production history") |
| 310 | else: |
| 311 | match = next((item for item in reversed(entries) if item["release"] != current), None) |
| 312 | if not match: |
| 313 | parser.error("no earlier production release") |
| 314 | release, source = match["release"], "rollback" |
| 315 | if release == current and entries and entries[-1]["release"] == current: |
| 316 | print(f"already running {release}") |
| 317 | return |
| 318 | backup = activate(release, args.mode == "bootstrap") |
| 319 | entries.append({"release": release, "source": source, "time": int(time.time()), "backup": backup}) |
| 320 | save_history(entries) |
| 321 | print(f"production={release} previous={current or ''}") |
| 322 | |
| 323 | |
| 324 | if __name__ == "__main__": |
| 325 | main() |