1#!/usr/bin/env python3
2import argparse
3from contextlib import nullcontext
4import fcntl
5import hashlib
6import json
7import os
8from pathlib import Path
9import re
10import secrets
11import shutil
12import subprocess
13import sys
14import tempfile
15import time
16
17from data import cloned_postgres, dataset_for
18
19
20REPO = Path(__file__).resolve().parent.parent
21SERVICES = REPO / "service"
22STATE = Path("/var/lib/snowglobe")
23IDENTITIES = STATE / "identities.json"
24ASSETS = Path("/var/lib/caddy/snowglobe")
25NAME = re.compile(r"[a-z][a-z0-9-]*\Z")
26
27
28def command(*args, capture=False, **kwargs):
29 return subprocess.run(args, check=True, text=True, capture_output=capture, **kwargs)
30
31
32def service_files():
33 files = {}
34 for directory in SERVICES.iterdir():
35 if not directory.is_dir():
36 continue
37 for path in directory.glob("*.pkl"):
38 name = directory.name if path.name == "service.pkl" else path.stem
39 if name in files:
40 raise ValueError(f"duplicate service definition: {name}")
41 files[name] = path
42 return files
43
44
45def services():
46 return sorted(service_files())
47
48
49def service_dir(name):
50 path = service_files()[name]
51 return path.parent if path.name == "service.pkl" else path.parent / name
52
53
54def dependencies(data):
55 return set(data["dependsOn"]) | {item["provider"] for item in data["inputs"].values() if item["provider"] != "snowglobe"}
56
57
58def ordered(data):
59 pending = {item["id"]: item for item in data}
60 result = []
61 while pending:
62 ready = [
63 item
64 for item in pending.values()
65 if not dependencies(item) & pending.keys()
66 ]
67 if not ready:
68 raise ValueError("cycle in service dependencies")
69 for item in ready:
70 result.append(item)
71 del pending[item["id"]]
72 return result
73
74
75def identity(name):
76 STATE.mkdir(parents=True, exist_ok=True)
77 if not IDENTITIES.exists():
78 write_private(IDENTITIES, (REPO / "config/identities.json").read_text())
79 with (IDENTITIES.parent / ".identities.lock").open("w") as lock:
80 fcntl.flock(lock, fcntl.LOCK_EX)
81 with IDENTITIES.open() as file:
82 ids = json.load(file)
83 if len(set(ids.values())) != len(ids):
84 raise ValueError("duplicate service UIDs")
85 if name not in ids:
86 used = set(ids.values())
87 ids[name] = next(uid for uid in range(3100, 60000) if uid not in used)
88 pending = IDENTITIES.with_suffix(".pending")
89 pending.write_text(json.dumps(ids, indent=2) + "\n")
90 pending.replace(IDENTITIES)
91 return ids[name]
92
93
94def load(name, properties, instance_id=None):
95 files = service_files()
96 if not NAME.fullmatch(name) or name not in files:
97 raise ValueError(f"unknown service: {name}")
98 instance_id = instance_id or name
99 if not NAME.fullmatch(instance_id):
100 raise ValueError(f"invalid instance ID: {instance_id}")
101 uid = identity(name)
102 result = command(
103 "pkl",
104 "eval",
105 *(
106 part
107 for key, value in {**properties, "serviceId": instance_id, "uid": uid, "preview": str(instance_id != name).lower()}.items()
108 for part in ("-p", f"{key}={value}")
109 ),
110 str(files[name]),
111 capture=True,
112 )
113 data = json.loads(result.stdout)
114 if data["id"] != instance_id:
115 raise ValueError(f"service ID must match deployment: {instance_id}")
116 inputs = {}
117 for requirement in data.pop("requirements"):
118 alias = requirement["alias"]
119 if not NAME.fullmatch(alias) or alias == "own" or alias in inputs:
120 raise ValueError(f"invalid or duplicate requirement alias: {alias}")
121 inputs[alias] = requirement
122 issuer_hosts = [request.pop("issuerHostname") for request in inputs.values()
123 if request["provider"] == "snowglobe" and request["kind"] == "client"]
124 data["inputs"] = inputs
125 data["uid"] = uid
126 data["sourceId"] = name
127 for task_name, task in data["containers"].items():
128 task["extraHosts"] = list(dict.fromkeys(task["extraHosts"]))
129 for hostname in issuer_hosts:
130 if all(host.split(":", 1)[0].lower() != hostname.lower() for host in task["extraHosts"]):
131 task["extraHosts"].append(f"{hostname}:host-gateway")
132 if bool(task.get("image")) == bool(task.get("build")):
133 raise ValueError(f"{name}.{task_name} needs one image or build directory")
134 if task.get("build"):
135 if REPO.parent == Path("/opt/snowglobe/releases").resolve():
136 context = build_context(name, task["build"])
137 digest = hashlib.sha256()
138 for path in sorted(context.rglob("*")):
139 if path.is_file():
140 digest.update(str(path.relative_to(context)).encode() + b"\0")
141 digest.update(path.read_bytes())
142 tag = digest.hexdigest()[:16]
143 elif not (service_dir(name) / task["build"] / "Dockerfile").is_file() and not (service_dir(name) / "build-source.json").is_file():
144 raise ValueError(f"missing build source: {name}.{task_name}")
145 else:
146 tag = REPO.name
147 task["image"] = f"localhost/snowglobe/{name}-{task_name}:{tag}"
148 return data
149
150
151def q(value):
152 return json.dumps(value, ensure_ascii=False)
153
154
155def write_private(path, content):
156 pending = None
157 try:
158 with tempfile.NamedTemporaryFile("w", dir=path.parent, delete=False) as file:
159 pending = Path(file.name)
160 file.write(content)
161 pending.replace(path)
162 finally:
163 if pending:
164 pending.unlink(missing_ok=True)
165
166
167def host_path(root, relative):
168 if relative == ".":
169 return root
170 part = Path(relative)
171 if part.is_absolute() or ".." in part.parts or not part.parts:
172 raise ValueError(f"invalid relative path: {relative}")
173 return str(Path(root) / part)
174
175
176def config_path(service, relative):
177 root = service_dir(service).resolve()
178 source = (root / relative).resolve()
179 if not source.is_relative_to(root) or not source.exists():
180 raise ValueError(f"invalid config path: {relative}")
181 return source
182
183
184def build_context(service, relative):
185 source = (service_dir(service) / relative).resolve()
186 if not source.is_relative_to((REPO / "service").resolve()) or not (source / "Dockerfile").is_file():
187 raise ValueError(f"invalid build context: {relative}")
188 return source
189
190
191def config_assets(data):
192 paths = set()
193 for task in data["containers"].values():
194 paths.update(volume["config"] for volume in task["volumes"].values() if volume.get("config"))
195 if task.get("http"):
196 paths.update(task["http"]["overrideFiles"].values())
197 sources = {rel: config_path(data.get("sourceId", data["id"]), rel) for rel in paths}
198 digest = hashlib.sha256()
199 for rel, source in sorted(sources.items()):
200 digest.update(rel.encode())
201 files = sorted(source.rglob("*")) if source.is_dir() else [source]
202 for file in files:
203 if file.is_symlink():
204 raise ValueError(f"config asset cannot be a symlink: {file}")
205 if file.is_file():
206 digest.update(str(Path(rel) / file.relative_to(source)).encode() if source.is_dir() else rel.encode())
207 digest.update(file.read_bytes())
208 return ASSETS / data["id"] / digest.hexdigest()[:16], sources
209
210
211def render(data, definitions):
212 if not data["containers"]:
213 return ""
214 name = data["id"]
215 asset_dir, _ = config_assets(data)
216 prepare_digest = None
217 if data.get("prepare"):
218 source = service_dir(data.get("sourceId", name))
219 digest = hashlib.sha256()
220 for path in sorted(source.rglob("*")):
221 if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc":
222 continue
223 if path.name == "service.pkl" or path.name.startswith("icon"):
224 continue
225 digest.update(str(path.relative_to(source)).encode() + b"\0")
226 digest.update(path.read_bytes())
227 prepare_digest = digest.hexdigest()[:16]
228 routed = [task["http"] for task in data["containers"].values()
229 if task.get("http") and task["http"].get("hostname")]
230 primary = routed[0] if routed else {}
231 requirements = {provider: "startup" for provider in data["dependsOn"]}
232 requirements.update({item["provider"]: item["kind"] for item in data["inputs"].values()})
233 secrets_meta = [
234 {"name": name, "generated": True, "bytes": spec["bytes"]}
235 for name, spec in data["secrets"].items()
236 ] + [{"name": name, "generated": False} for name in data["requiredSecrets"]]
237 lines = [
238 f"job {q(name)} {{",
239 ' datacenters = ["clover"]',
240 ' type = "service"',
241 ' meta {',
242 f" snowglobe_service = {q(data['sourceId'])}",
243 f" snowglobe_name = {q(data['name'])}",
244 f" snowglobe_tagline = {q(data['tagline'])}",
245 f" snowglobe_hostname = {q(primary.get('hostname') or '')}",
246 f" snowglobe_launcher = {q(str(data['launcher']).lower())}",
247 f" snowglobe_auth_role = {q(data.get('access') or primary.get('authRole') or '')}",
248 f" snowglobe_trace_service = {q(data.get('traceServiceName') or '')}",
249 f" snowglobe_metrics_pushed = {q(str(data['metricsPushed']).lower())}",
250 f" snowglobe_requires = {q(json.dumps(requirements))}",
251 f" snowglobe_secrets = {q(json.dumps(secrets_meta))}",
252 ' }',
253 ' group "app" {',
254 ]
255 if dependencies(data):
256 lines += [
257 " restart {",
258 " attempts = 3",
259 ' delay = "1m"',
260 ' interval = "4m"',
261 ' mode = "delay"',
262 " }",
263 ]
264 if data["rollout"] == "overlapped":
265 for task in data["containers"].values():
266 if task["hostNetwork"] or any(
267 endpoint and endpoint.get("hostPort") is not None
268 for endpoint in (task.get("http"), *task["tcp"])
269 ) or any(not volume["readOnly"] for volume in task["volumes"].values()):
270 raise ValueError(f"overlapped rollout requires dynamic ports and read-only mounts: {name}")
271 elif data["rollout"] != "simple":
272 raise ValueError(f"unknown rollout: {data['rollout']}")
273 if data["rollout"] == "overlapped" or data.get("healthyDeadline"):
274 lines.append(" update {")
275 if data["rollout"] == "overlapped":
276 lines += [" canary = 1", " auto_promote = true", " auto_revert = true"]
277 if data.get("healthyDeadline"):
278 lines += [f" healthy_deadline = {q(data['healthyDeadline'])}", ' progress_deadline = "0"']
279 lines.append(" }")
280 if data.get("healthyDeadline"):
281 lines += [" migrate {", f" healthy_deadline = {q(data['healthyDeadline'])}", " }"]
282 ports = {}
283 network = []
284 host_network = any(task["hostNetwork"] for task in data["containers"].values())
285 if host_network and not all(task["hostNetwork"] for task in data["containers"].values()):
286 raise ValueError("mixed host and bridge networking in one group")
287 for task_name, task in data["containers"].items():
288 for kind, endpoint in [("http", task.get("http")), *((port["name"], port) for port in task["tcp"])]:
289 if endpoint:
290 label = kind if len(data["containers"]) == 1 else f"{task_name}-{kind}"
291 if label in ports.values():
292 raise ValueError(f"duplicate port label: {label}")
293 ports[(task_name, kind)] = label
294 network.append(f" port {q(label)} {{")
295 if endpoint.get("hostPort") is not None:
296 network.append(f" static = {endpoint['hostPort']}")
297 if not task["hostNetwork"]:
298 network.append(f" to = {endpoint['containerPort']}")
299 if endpoint.get("loopback") or (not task["hostNetwork"] and kind == "http" and endpoint.get("hostPort") is None):
300 network.append(' host_network = "loopback"')
301 network.append(" }")
302 if network:
303 lines.append(" network {")
304 if host_network:
305 lines.append(' mode = "host"')
306 lines += network + [" }"]
307 for provider_id in sorted(dependencies(data)):
308 provider = definitions[provider_id]
309 for provider_task, container in provider["containers"].items():
310 endpoint = container.get("http")
311 if not endpoint or not endpoint.get("hostname") or endpoint.get("authRole"):
312 continue
313 host = endpoint["hostname"]
314 task_name = f"wait-{provider_id}-{provider_task}"
315 lines += [
316 f" task {q(task_name)} {{",
317 ' driver = "podman"',
318 " lifecycle {",
319 ' hook = "prestart"',
320 " sidecar = false",
321 " }",
322 " config {",
323 ' image = "docker.io/curlimages/curl@sha256:43ebaa53d3806db6b1ce4353b6b26ae638ec1c167ee351524b05690f988bb20d"',
324 f" extra_hosts = {q([host + ':host-gateway'])}",
325 f" args = {q(['--insecure', '--fail', '--silent', '--show-error', '--retry', '999999', '--retry-all-errors', '--retry-delay', '5', '--connect-timeout', '3', '--max-time', '5', 'https://' + host + endpoint['checkPath']])}",
326 " }",
327 " resources {",
328 " cpu = 50",
329 " memory = 64",
330 " }",
331 " }",
332 ]
333 for task_name, task in data["containers"].items():
334 if task.get("http") or task.get("tcp"):
335 kind = "http" if task.get("http") else "tcp"
336 endpoint = task["http"] if kind == "http" else task["tcp"][0]
337 label = ports[(task_name, "http" if kind == "http" else endpoint["name"])]
338 lines += [" service {", f" name = {q(name if task_name == 'app' else name + '-' + task_name)}", ' provider = "nomad"', f" port = {q(label)}"]
339 if len(data["containers"]) == 1:
340 lines.append(f" task = {q(task_name)}")
341 tags = []
342 if kind == "http" and endpoint.get("metricsPath"):
343 tags.append("snowglobe-metrics-path=" + endpoint["metricsPath"])
344 if kind == "http" and endpoint.get("hostname"):
345 tags.append("caddy-host=" + endpoint["hostname"])
346 tags += ["caddy-plain-host=" + host for host in endpoint["plainHostnames"]]
347 if endpoint.get("authRole"):
348 tags.append("caddy-auth-role=" + endpoint["authRole"])
349 if endpoint.get("userHeader"):
350 tags.append("caddy-user-header=" + endpoint["userHeader"])
351 if endpoint["forwardRealIp"]:
352 tags.append("caddy-real-ip=true")
353 if endpoint.get("secureCookies"):
354 tags.append("caddy-secure-cookies=true")
355 if endpoint["tlsInternal"]:
356 tags.append("caddy-internal=true")
357 if tags:
358 lines.append(f" tags = {q(tags)}")
359 lines += [" check {", f" type = {q(kind)}"]
360 if kind == "http":
361 lines.append(f" path = {q(endpoint['checkPath'])}")
362 if endpoint["checkHeaders"]:
363 lines.append(" header {")
364 for key, value in endpoint["checkHeaders"].items():
365 if not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", key):
366 raise ValueError(f"invalid health check header: {key}")
367 lines.append(f" {key} = [{q(value)}]")
368 lines.append(" }")
369 lines += [
370 ' interval = "10s"',
371 ' timeout = "15s"',
372 ' check_restart {',
373 ' limit = 12',
374 f" grace = {q(data.get('healthRestartGrace') or data.get('healthyDeadline') or '5m')}",
375 ' }',
376 ' }',
377 ' }',
378 ]
379 for task_name, task in data["containers"].items():
380 lines += [f" task {q(task_name)} {{", ' driver = "podman"']
381 if task.get("killTimeout"):
382 lines.append(f" kill_timeout = {q(task['killTimeout'])}")
383 if task["lifecycle"] != "main":
384 lines += [' lifecycle {', ' hook = "prestart"',
385 f" sidecar = {str(task['lifecycle'] == 'prestartSidecar').lower()}", ' }']
386 if not task["imageUser"]:
387 uses_clover = any(
388 volume.get("clover") or (
389 volume.get("src")
390 and Path(volume["src"]).resolve().is_relative_to(Path(data["cloverRoot"]).resolve())
391 )
392 for volume in task["volumes"].values()
393 )
394 gid = data["cloverGid"] if uses_clover else 0 if task["rootGroup"] else data["uid"]
395 lines.append(f" user = {q(str(data['uid']) + ':' + str(gid))}")
396 lines += [" config {", f" image = {q(task['image'])}"]
397 if task.get("entrypoint"):
398 lines.append(f" entrypoint = {q(task['entrypoint'])}")
399 if task["args"]:
400 lines.append(f" args = {q(task['args'])}")
401 if task["hostNetwork"]:
402 lines.append(' network_mode = "host"')
403 if task.get("readOnlyRoot"):
404 lines.append(' readonly_rootfs = true')
405 task_ports = [label for (owner, _), label in ports.items() if owner == task_name]
406 if task_ports and not task["hostNetwork"]:
407 lines.append(f" ports = {q(task_ports)}")
408 for field, values in (("cap_add", task["capAdd"]), ("devices", task["devices"]), ("extra_hosts", task["extraHosts"])):
409 if values:
410 lines.append(f" {field} = {q(values)}")
411 volumes = []
412 secret_files = task.get("secretFiles", {})
413 filenames = set()
414 for target, template in secret_files.items():
415 filename = Path(target).name
416 if (not re.fullmatch(r"/[A-Za-z0-9_./-]+", target) or target.endswith("/")
417 or any(part in {".", "..", ""} for part in target[1:].split("/"))
418 or filename in filenames
419 or filename in {task_name + ".env", task_name + "-secret.env", "nomad_token"}
420 or target in task["volumes"] or target == "/secrets" or target.startswith("/secrets/")):
421 raise ValueError(f"invalid secret file target: {target}")
422 if task["imageUser"]:
423 raise ValueError("secret files require a service-owned container user")
424 if re.search(r"(?m)^EOF\r?$", template):
425 raise ValueError("invalid secret template delimiter")
426 filenames.add(filename)
427 volumes.append(f"secrets/{filename}:{target}:ro")
428 for target, volume in task["volumes"].items():
429 if not target.startswith("/") or ":" in target:
430 raise ValueError(f"invalid volume target: {target}")
431 if volume.get("config") is not None:
432 if volume.get("src") is not None or not volume["readOnly"]:
433 raise ValueError("config mounts must be read only and have no host source")
434 config_path(data.get("sourceId", name), volume["config"])
435 source = host_path(str(asset_dir), volume["config"])
436 elif volume.get("src") is not None:
437 source = volume["src"]
438 if not Path(source).is_absolute():
439 raise ValueError(f"volume source must be absolute: {source}")
440 else:
441 source = host_path(data["hostRoot"], target.lstrip("/"))
442 if ":" in source:
443 raise ValueError(f"invalid volume source: {source}")
444 volumes.append(f"{source}:{target}" + (":ro" if volume["readOnly"] else ""))
445 if volumes:
446 lines.append(f" volumes = {q(volumes)}")
447 if task["tmpfs"]:
448 lines.append(f" tmpfs = {q(task['tmpfs'])}")
449 lines.append(" }")
450 secret_env = []
451 plain_env = {}
452 for key, value in task["env"].items():
453 if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
454 raise ValueError(f"invalid env key: {key}")
455 parts = re.split(r"(\$\{secret\.[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*\})", value)
456 if len(parts) == 1:
457 if "${secret." in value:
458 raise ValueError(f"invalid secret reference: {value}")
459 plain_env[key] = value
460 continue
461 args = []
462 paths = {}
463 for part in parts:
464 match = re.fullmatch(r"\$\{secret\.([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\}", part)
465 if match:
466 alias, field = match.groups()
467 if alias != "own" and alias not in data["inputs"]:
468 raise ValueError(f"unknown secret alias: {alias}")
469 path = f"nomad/jobs/{name}" + (f"/inputs/{alias}" if alias != "own" else "")
470 variable = paths.setdefault(path, f"$s{len(paths)}")
471 args.append(f"(index {variable} {q(field)}).Value")
472 elif part:
473 args.append(q(part))
474 start = "".join(f'{{{{ with {variable} := nomadVar {q(path)} }}}}' for path, variable in paths.items())
475 end = "{{ end }}" * len(paths)
476 secret_env.append(f'{start}{key}={{{{ print {" ".join(args)} | toJSON }}}}{end}')
477 if "SNOWGLOBE_PREPARE_DIGEST" in task["env"]:
478 raise ValueError("SNOWGLOBE_PREPARE_DIGEST is reserved")
479 lines.append(" env {")
480 for key, value in plain_env.items():
481 lines.append(f" {key} = {q(value)}")
482 if prepare_digest:
483 lines.append(f" SNOWGLOBE_PREPARE_DIGEST = {q(prepare_digest)}")
484 lines.append(" }")
485 if secret_env:
486 lines += [" template {", " data = <<EOF", *secret_env, "EOF", f' destination = {q("secrets/" + task_name + "-secret.env")}', " env = true", " error_on_missing_key = true", " }"]
487 if task.get("envTemplate"):
488 if "\nEOF\n" in task["envTemplate"]:
489 raise ValueError("invalid env template delimiter")
490 lines += [" template {", " data = <<EOF", task["envTemplate"].rstrip(), "EOF", f' destination = {q("secrets/" + task_name + ".env")}', " env = true", " }"]
491 for target, template in secret_files.items():
492 lines += [" template {", " data = <<EOF", template.rstrip(), "EOF",
493 f' destination = {q("secrets/" + Path(target).name)}', ' perms = "0400"',
494 f' uid = {data["uid"]}', f' gid = {data["uid"]}',
495 " error_on_missing_key = true", " }"]
496 lines += [" resources {", f" cpu = {task['cpu']}", f" memory = {task['memory']}", " }", " }"]
497 lines += [" }", "}"]
498 return "\n".join(lines) + "\n"
499
500
501def token():
502 if "NOMAD_TOKEN" not in os.environ:
503 os.environ["NOMAD_TOKEN"] = (STATE / "nomad.token").read_text().strip()
504
505
506def get_variable(path):
507 result = subprocess.run(["nomad", "var", "get", "-out=json", path], capture_output=True, text=True)
508 if result.returncode == 0:
509 return json.loads(result.stdout)
510 if "Variable not found" in result.stderr + result.stdout:
511 return None
512 raise RuntimeError(f"Nomad variable read failed for {path}: {result.stderr.strip()}")
513
514
515def provision(data):
516 if not data["containers"]:
517 return
518 if os.geteuid() != 0:
519 raise PermissionError("provisioning requires root")
520 root = Path(data["hostRoot"])
521 if data["hasManagedVolumes"]:
522 root.mkdir(parents=True, exist_ok=True)
523 if command(
524 "findmnt", "-n", "-o", "FSTYPE", "--mountpoint", str(root), capture=True
525 ).stdout.strip() != "zfs":
526 raise ValueError(f"expected ZFS dataset at {root}")
527 for task in data["containers"].values():
528 gid = 0 if task["rootGroup"] else data["uid"]
529 for target, volume in task["volumes"].items():
530 if volume.get("config") is not None or volume.get("src") is not None:
531 continue
532 path = Path(host_path(str(root), target.lstrip("/")))
533 path.mkdir(parents=True, exist_ok=True)
534 os.chown(path, data["uid"], gid)
535 path.chmod(0o750)
536 if data["sourceId"] == "ytdl" and target == "/data":
537 command("systemd-tmpfiles", "--create", "--prefix=" + str(path))
538 bundle = STATE / "ca-bundle.crt"
539 if any(
540 volume.get("src") == str(bundle)
541 for task in data["containers"].values()
542 for volume in task["volumes"].values()
543 ):
544 contents = Path("/etc/ssl/certs/ca-certificates.crt").read_bytes()
545 local_ca = Path("/var/lib/caddy/.local/share/caddy/pki/authorities/local/root.crt")
546 if local_ca.exists():
547 contents += b"\n" + local_ca.read_bytes()
548 if not bundle.exists() or bundle.read_bytes() != contents:
549 pending = bundle.with_suffix(".pending")
550 pending.write_bytes(contents)
551 pending.chmod(0o644)
552 pending.replace(bundle)
553 route_dir = STATE / "routes"
554 route_dir.mkdir(parents=True, exist_ok=True)
555 asset_dir, assets = config_assets(data)
556 static = {}
557 directories = {}
558 identity = {}
559 head_html = {}
560 metrics_paths = {}
561 api_paths = {}
562 for task in data["containers"].values():
563 if task.get("http"):
564 http = task["http"]
565 if http["identityHeaders"]:
566 identity[http["hostname"]] = http["identityHeaders"]
567 if http["headHtml"]:
568 head_html[http["hostname"]] = http["headHtml"]
569 if http.get("metricsPath") and http["hostname"]:
570 metrics_paths[http["hostname"]] = http["metricsPath"]
571 if http.get("apiPaths"):
572 targets = {}
573 for path, task_name in http["apiPaths"].items():
574 target = data["containers"].get(task_name)
575 if (not http["hostname"] or not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path)
576 or not target or not (target.get("http") or target.get("tcp"))):
577 raise ValueError(f"invalid API route for {data['id']}: {path}")
578 targets[path] = data["id"] if task_name == "app" else data["id"] + "-" + task_name
579 api_paths[http["hostname"]] = targets
580 for request, rel in http["overrideFiles"].items():
581 origin = assets[rel]
582 target = directories if origin.is_dir() else static
583 target[request] = host_path(str(asset_dir), rel)
584 if assets:
585 asset_dir.parent.mkdir(parents=True, exist_ok=True)
586 asset_dir.parent.chmod(0o755)
587 if assets and not asset_dir.exists():
588 temporary = tempfile.mkdtemp(dir=asset_dir.parent)
589 try:
590 for rel, origin in assets.items():
591 destination = Path(temporary) / rel
592 destination.parent.mkdir(parents=True, exist_ok=True)
593 if origin.is_dir():
594 shutil.copytree(origin, destination)
595 for child in destination.rglob("*"):
596 child.chmod(0o755 if child.is_dir() else 0o644)
597 else:
598 shutil.copy2(origin, destination)
599 destination.chmod(0o644)
600 Path(temporary).chmod(0o755)
601 os.replace(temporary, asset_dir)
602 finally:
603 shutil.rmtree(temporary, ignore_errors=True)
604 secret_dir = STATE / "secrets"
605 secret_dir.mkdir(parents=True, exist_ok=True)
606 secret_file = secret_dir / f"{data['id']}.nv.hcl"
607 generated = data["secrets"]
608 required = data["requiredSecrets"]
609 secret_path = "nomad/jobs/" + data["id"]
610 existing = get_variable(secret_path) if generated or required else None
611 if generated and not existing and secret_file.exists():
612 command("nomad", "var", "put", "-in=hcl", "-out=none", secret_path, "@" + str(secret_file))
613 existing = get_variable(secret_path)
614 values = dict(existing["Items"]) if existing else {}
615 if generated:
616 for item, spec in generated.items():
617 if item not in values:
618 values[item] = secrets.token_hex(spec["bytes"])
619 if not existing or values != existing["Items"]:
620 put_variable(secret_path, values, existing["ModifyIndex"] if existing else 0)
621 backup = "items {\n" + "".join(f" {item} = {q(value)}\n" for item, value in values.items()) + "}\n"
622 if not secret_file.exists() or secret_file.read_text() != backup:
623 write_private(secret_file, backup)
624 if required:
625 missing = [name for name in required if not values.get(name)]
626 if missing:
627 raise ValueError(f"missing required secrets for {data['id']}: {', '.join(missing)}")
628 proof = {}
629 for task in data["containers"].values():
630 http = task.get("http")
631 if http and http.get("identityProofSecret"):
632 name = http["identityProofSecret"]
633 if not http["identityHeaders"] or name not in values:
634 raise ValueError(f"missing identity proof secret for {data['id']}")
635 proof[http["hostname"]] = "X-Snowglobe-Idp-" + values[name]
636 write_private(route_dir / f"{data['id']}.json", json.dumps({
637 "files": static, "dirs": directories, "identity": identity,
638 "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, "apiPaths": api_paths,
639 }))
640
641
642def submit(data, mode, definitions):
643 if not data["containers"]:
644 return
645 with tempfile.NamedTemporaryFile("w", suffix=".nomad.hcl", delete=False) as file:
646 file.write(render(data, definitions))
647 path = file.name
648 try:
649 command("nomad", "job", mode, path)
650 finally:
651 Path(path).unlink()
652
653
654def build_images(data):
655 for task in data["containers"].values():
656 if task.get("build"):
657 image = task["image"]
658 if subprocess.run(["podman", "image", "exists", image]).returncode:
659 command("podman", "build", "-t", image, str(build_context(data["sourceId"], task["build"])))
660
661
662def clone_dataset(source, snapshot, dataset, mount):
663 acltype = command("zfs", "get", "-H", "-o", "value", "acltype", source, capture=True).stdout.strip()
664 command("zfs", "clone", "-o", "mountpoint=" + mount, "-o", "acltype=" + acltype, snapshot, dataset)
665
666
667def check_http(host, path, tls_internal=False, attempts=120):
668 for attempt in range(attempts):
669 try:
670 if subprocess.run(
671 ["curl", "--noproxy", "*", "-sf", *(["--cacert", "/var/lib/caddy/.local/share/caddy/pki/authorities/local/root.crt"] if tls_internal else []), "--resolve", f"{host}:443:127.0.0.1", f"https://{host}{path}"],
672 capture_output=True, timeout=3,
673 ).returncode == 0:
674 print(f"Healthy {host}")
675 return
676 except subprocess.TimeoutExpired:
677 pass
678 time.sleep(2)
679 raise RuntimeError(f"HTTPS route is unhealthy: {host}")
680
681
682def put_variable(path, items, index=None):
683 with tempfile.NamedTemporaryFile("w", suffix=".nv.hcl", delete=False) as file:
684 file.write("items {\n" + "".join(f" {key} = {q(value)}\n" for key, value in items.items()) + "}\n")
685 filename = file.name
686 try:
687 command("nomad", "var", "put", "-in=hcl", "-out=none", *([f"-check-index={index}"] if index is not None else []), path, "@" + filename)
688 finally:
689 Path(filename).unlink()
690
691
692def configure(data):
693 if not data.get("setup"):
694 return
695 if data["setup"].startswith("tools/"):
696 script = (REPO / data["setup"]).resolve()
697 if not script.is_relative_to(REPO / "tools") or not script.is_file():
698 raise ValueError(f"invalid shared setup script: {data['setup']}")
699 else:
700 script = config_path(data.get("sourceId", data["id"]), data["setup"])
701 routes = [task["http"] for task in data["containers"].values()
702 if task.get("http") and task["http"].get("hostname")]
703 if len(routes) != 1:
704 raise ValueError("service setup requires one HTTP hostname")
705 route = routes[0]
706 check_http(route["hostname"], route["checkPath"], route["tlsInternal"], attempts=600)
707 variable = get_variable("nomad/jobs/" + data["id"])
708 own = variable["Items"] if variable else {}
709 command("python3", str(script), input=json.dumps({
710 "serviceId": data["id"], "host": route["hostname"], "hostRoot": data["hostRoot"],
711 "preview": data["id"] != data.get("sourceId", data["id"]),
712 "ownerEmail": data["ownerEmail"], **own,
713 }))
714 print(f"Configured {data['id']}")
715
716
717def prepare(data):
718 if data.get("prepare"):
719 script = config_path(data.get("sourceId", data["id"]), data["prepare"])
720 command("python3", str(script), input=json.dumps({
721 "hostRoot": data["hostRoot"], "uid": data["uid"],
722 }))
723
724
725def provision_inputs(data, definitions, stage_id=None, postgres_source=None):
726 for alias, request in data["inputs"].items():
727 if not NAME.fullmatch(alias):
728 raise ValueError(f"invalid input alias: {alias}")
729 native = request["provider"] == "snowglobe"
730 provider = {"id": "snowglobe", "provide": True, "containers": {}} if native else definitions[request["provider"]]
731 if not provider.get("provide"):
732 raise ValueError(f"{provider['id']} does not provide inputs")
733 path = f"nomad/jobs/{data['id']}/inputs/{alias}"
734 variable = get_variable(path)
735 own = (get_variable("nomad/jobs/" + provider["id"]) or {}).get("Items", {})
736 http = next((task["http"] for task in provider["containers"].values()
737 if task.get("http") and task["http"].get("hostname")), None)
738 if http:
739 check_http(http["hostname"], http["checkPath"], http["tlsInternal"])
740 payload = {
741 "request": request,
742 "existing": variable["Items"] if variable else None,
743 "providerSecrets": own,
744 "host": http["hostname"] if http else None,
745 }
746 if stage_id:
747 payload["stageId"] = stage_id
748 if postgres_source and provider["id"] == "postgres":
749 payload["sourceContainer"] = postgres_source
750 script = REPO / "tools/oidc-provider.py" if native else config_path(provider["id"], provider["provide"])
751 result = command("python3", str(script), input=json.dumps(payload), capture=True)
752 values = json.loads(result.stdout)
753 if not isinstance(values, dict) or not values or any(
754 not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) or not isinstance(value, str) or not value
755 for key, value in values.items()
756 ):
757 raise ValueError(f"invalid outputs from {provider['id']}")
758 if not variable or variable["Items"] != values:
759 put_variable(path, values, variable["ModifyIndex"] if variable else None)
760 print(f"Allocated {data['id']}.{alias} from {provider['id']}")
761
762
763def bootstrap(all_data):
764 if os.geteuid() != 0:
765 raise PermissionError("bootstrap requires root")
766 STATE.mkdir(parents=True, exist_ok=True)
767 if not (STATE / "nomad.token").exists():
768 result = command("nomad", "acl", "bootstrap", "-json", capture=True)
769 write_private(STATE / "nomad.token", json.loads(result.stdout)["SecretID"] + "\n")
770 token()
771 command(
772 "nomad",
773 "acl",
774 "policy",
775 "apply",
776 "snowglobe-router",
777 str(REPO / "config/policies/router.hcl"),
778 )
779 command("nomad", "acl", "policy", "apply", "snowglobe-dashboard",
780 str(REPO / "config/policies/dashboard.hcl"))
781 router_token = STATE / "router.token"
782 if not router_token.exists():
783 result = command(
784 "nomad",
785 "acl",
786 "token",
787 "create",
788 "-policy=snowglobe-router",
789 "-name=snowglobe-router",
790 "-json",
791 capture=True,
792 )
793 write_private(router_token, json.loads(result.stdout)["SecretID"] + "\n")
794 dashboard_token = STATE / "dashboard.token"
795 if not dashboard_token.exists():
796 result = command("nomad", "acl", "token", "create", "-policy=snowglobe-dashboard",
797 "-name=snowglobe-dashboard", "-json", capture=True)
798 write_private(dashboard_token, json.loads(result.stdout)["SecretID"] + "\n")
799 for data in all_data:
800 paths = {
801 f"nomad/jobs/{data['id']}/inputs/{alias}" for alias in data["inputs"]
802 }
803 if paths and data["containers"]:
804 policy = (
805 'namespace "default" {\n variables {\n'
806 + "".join(
807 f' path {q(path)} {{ capabilities = ["read"] }}\n'
808 for path in sorted(paths)
809 )
810 + " }\n}\n"
811 )
812 with tempfile.NamedTemporaryFile("w", delete=False) as file:
813 file.write(policy)
814 path = file.name
815 try:
816 for task_name in data["containers"]:
817 command(
818 "nomad", "acl", "policy", "apply", "-namespace", "default",
819 "-job", data["id"], "-group", "app", "-task", task_name,
820 data["id"] + "-" + task_name + "-imports", path,
821 )
822 finally:
823 Path(path).unlink()
824 current = Path("/opt/snowglobe/current")
825 if current.is_symlink():
826 command("systemctl", "start", "snowglobe-router.service")
827
828
829def destroy_stage(stage, metadata, definitions):
830 subprocess.run(["nomad", "job", "stop", "-purge", "-yes", stage], check=False)
831 for alias, request in metadata["inputs"].items():
832 native = request["provider"] == "snowglobe"
833 provider = {"id": "snowglobe", "provide": True, "containers": {}} if native else definitions[request["provider"]]
834 own = (get_variable("nomad/jobs/" + provider["id"]) or {}).get("Items", {})
835 variable = get_variable(f"nomad/jobs/{stage}/inputs/{alias}")
836 existing = variable["Items"] if variable else None
837 hosts = [task["http"]["hostname"] for task in provider["containers"].values()
838 if task.get("http") and task["http"].get("hostname")]
839 script = REPO / "tools/oidc-provider.py" if native else config_path(provider["id"], provider["provide"])
840 command("python3", str(script), input=json.dumps({
841 "operation": "delete", "request": request, "stageId": stage,
842 "providerSecrets": own, "host": hosts[0] if hosts else None,
843 "existing": existing,
844 }))
845 datasets = list(metadata.get("external", {}).values())
846 if metadata.get("clone"):
847 datasets.append({"clone": metadata["clone"], "snapshot": metadata.get("snapshot")})
848 for dataset in datasets:
849 for attempt in range(30):
850 if subprocess.run(["zfs", "destroy", dataset["clone"]], capture_output=True).returncode == 0:
851 break
852 time.sleep(1)
853 else:
854 raise RuntimeError("preview dataset is still in use")
855 if dataset.get("snapshot"):
856 command("zfs", "destroy", dataset["snapshot"])
857 if not metadata.get("clone"):
858 shutil.rmtree(metadata["mount"], ignore_errors=True)
859 for alias in metadata["inputs"]:
860 subprocess.run(["nomad", "var", "purge", f"nomad/jobs/{stage}/inputs/{alias}"], check=False)
861 subprocess.run(["nomad", "var", "purge", "nomad/jobs/" + stage], check=False)
862 for task in metadata["tasks"]:
863 subprocess.run(["nomad", "acl", "policy", "delete", stage + "-" + task + "-imports"], check=False)
864 for path in (
865 STATE / "stages" / f"{stage}.json",
866 STATE / "routes" / f"{stage}.json",
867 STATE / "secrets" / f"{stage}.nv.hcl",
868 ):
869 path.unlink(missing_ok=True)
870 shutil.rmtree(ASSETS / stage, ignore_errors=True)
871
872
873def check_pool(data):
874 if os.geteuid() != 0:
875 raise PermissionError("pool setup requires root")
876 clover = subprocess.run(
877 ["findmnt", "-n", "-o", "SOURCE,FSTYPE", "--mountpoint", data[0]["cloverRoot"]],
878 capture_output=True, text=True,
879 )
880 clover_info = clover.stdout.split()
881 if clover.returncode or len(clover_info) != 2 or clover_info[1] != "zfs":
882 raise ValueError(f"clover must be a mounted ZFS dataset: {data[0]['cloverRoot']}")
883 media = data[0]["mediaRoot"]
884 mounted = subprocess.run(
885 ["findmnt", "-n", "-o", "SOURCE,FSTYPE", "--mountpoint", media],
886 capture_output=True, text=True,
887 )
888 media_info = mounted.stdout.split()
889 expected = {"fuse"} if os.environ.get("SNOWGLOBE_MEDIA_READ_ONLY") == "true" else {"zfs"}
890 if mounted.returncode or len(media_info) != 2 or media_info[1] not in expected or media_info[0] == clover_info[0]:
891 raise ValueError(f"media must be a separate mounted dataset: {media}")
892 for path, info in ((data[0]["cloverRoot"], clover_info), (media, media_info)):
893 if info[1] == "zfs":
894 acltype = command("zfs", "get", "-H", "-o", "value", "acltype", info[0], capture=True).stdout.strip()
895 if acltype == "nfsv4":
896 raise ValueError(f"{path} uses NFSv4 ACLs, which NixOS cannot enforce. Rehearse a POSIX permission conversion on a ZFS clone before migration.")
897 pool = data[0]["pool"]
898 if subprocess.run(["zpool", "list", pool], capture_output=True).returncode:
899 raise ValueError(f"ZFS pool is unavailable: {pool}")
900 prod = pool + "/prod"
901 acl_source = prod if subprocess.run(["zfs", "list", prod], capture_output=True).returncode == 0 else pool
902 if command("zfs", "get", "-H", "-o", "value", "acltype", acl_source, capture=True).stdout.strip() == "nfsv4":
903 raise ValueError(f"{acl_source} uses NFSv4 ACLs; service datasets need POSIX ACLs on NixOS.")
904 encrypted = any(
905 command("zfs", "get", "-H", "-o", "value", "encryption", info[0], capture=True).stdout.strip() != "off"
906 for info in (clover_info, media_info) if info[1] == "zfs"
907 )
908 if encrypted:
909 root = str(Path(data[0]["hostRoot"]).parent)
910 mounted = subprocess.run(["findmnt", "-n", "-o", "SOURCE", "--mountpoint", root], capture_output=True, text=True)
911 encryption = subprocess.run(["zfs", "get", "-H", "-o", "value", "encryption", prod], capture_output=True, text=True)
912 if encryption.returncode or encryption.stdout.strip() == "off" or mounted.stdout.strip() != prod:
913 raise ValueError(f"Create and mount encrypted {prod} at {root} before deployment; the existing storage is encrypted.")
914
915
916def check_staging_pool(data):
917 check_pool([data])
918 prod = data["pool"] + "/prod"
919 staging = data["pool"] + "/staging"
920 acl_source = staging if subprocess.run(["zfs", "list", staging], capture_output=True).returncode == 0 else data["pool"]
921 if command("zfs", "get", "-H", "-o", "value", "acltype", acl_source, capture=True).stdout.strip() == "nfsv4":
922 raise ValueError(f"{acl_source} uses NFSv4 ACLs; staged datasets need POSIX ACLs on NixOS.")
923 encryption = subprocess.run(["zfs", "get", "-H", "-o", "value", "encryption", prod], capture_output=True, text=True)
924 if encryption.returncode or encryption.stdout.strip() == "off":
925 return
926 mounted = subprocess.run(["findmnt", "-n", "-o", "SOURCE", "--mountpoint", data["stagingRoot"]], capture_output=True, text=True)
927 staging_encryption = subprocess.run(["zfs", "get", "-H", "-o", "value", "encryption", staging], capture_output=True, text=True)
928 if staging_encryption.returncode or staging_encryption.stdout.strip() == "off" or mounted.stdout.strip() != staging:
929 raise ValueError(f"Create and mount encrypted {staging} at {data['stagingRoot']} before staging.")
930
931
932def ensure_pool(data):
933 check_pool(data)
934 STATE.mkdir(parents=True, exist_ok=True)
935 pool = data[0]["pool"]
936 for item in data:
937 if item["hasManagedVolumes"]:
938 dataset = pool + "/prod/" + item["id"]
939 if subprocess.run(["zfs", "list", dataset], capture_output=True).returncode:
940 root = Path(item["hostRoot"])
941 if root.exists() and any(root.iterdir()):
942 raise ValueError(f"refusing to mount over {root}")
943 if subprocess.run(["zfs", "list", pool + "/prod"], capture_output=True).returncode:
944 parent = str(root.parent)
945 if Path(parent).exists() and any(Path(parent).iterdir()):
946 raise ValueError(f"refusing to mount over {parent}")
947 command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod")
948 command("zfs", "create", "-o", "mountpoint=" + str(root), dataset)
949 command("zfs", "set", "snowglobe:snapshots=" + str(item["snapshots"]).lower(), dataset)
950
951
952def main():
953 parser = argparse.ArgumentParser()
954 parser.add_argument(
955 "mode",
956 choices=[
957 "render",
958 "secrets",
959 "validate",
960 "check",
961 "bootstrap",
962 "deploy",
963 "preflight",
964 "pool",
965 "allocate",
966 "stage",
967 "destroy",
968 ],
969 )
970 parser.add_argument("name", nargs="?")
971 parser.add_argument("--computer")
972 parser.add_argument("--base-domain")
973 parser.add_argument("--root")
974 parser.add_argument("--pool")
975 parser.add_argument("--media-root")
976 parser.add_argument("--env", action="append", default=[])
977 parser.add_argument("--key", action="append", default=[])
978 args = parser.parse_args()
979 if args.computer and (args.mode != "stage" or args.name != "dashboard"):
980 parser.error("--computer requires stage dashboard")
981 if args.mode == "stage" and args.name == "dashboard":
982 if args.env or args.base_domain or args.root or args.pool or args.media_root:
983 parser.error("dashboard staging uses the host configuration")
984 __import__("dashboard-stage").stage(args.computer)
985 return
986 if args.mode == "destroy" and args.name and NAME.fullmatch(args.name):
987 saved = STATE / "stages" / f"{args.name}.json"
988 if saved.exists() and json.loads(saved.read_text()).get("sourceId") == "dashboard":
989 __import__("dashboard-stage").destroy(args.name)
990 return
991 if args.mode == "allocate" and not args.name:
992 parser.error("allocate requires a service name")
993 if args.key and args.mode != "secrets":
994 parser.error("--key is available only for secrets")
995 if args.name and args.mode != "destroy" and args.name not in service_files():
996 raise ValueError(f"unknown service: {args.name}")
997 properties = {
998 key: value
999 for key, value in (("domain", args.base_domain), ("root", args.root), ("pool", args.pool), ("mediaRoot", args.media_root))
1000 if value
1001 }
1002 names = [args.name] if args.name else services()
1003 if args.mode == "secrets":
1004 if not args.name or not NAME.fullmatch(args.name):
1005 parser.error("secrets requires a service name")
1006 data = load(args.name, properties)
1007 declared = [*data["requiredSecrets"], *data["secrets"]]
1008 fields = args.key or declared
1009 if not declared or len(set(declared)) != len(declared) or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", field) for field in declared):
1010 raise ValueError(f"invalid required secrets for {args.name}")
1011 if len(set(fields)) != len(fields) or any(field not in declared for field in fields):
1012 raise ValueError(f"unknown or duplicate secret key for {args.name}")
1013 values = sys.stdin.read().splitlines()
1014 if len(values) != len(fields) or any(not value for value in values):
1015 raise ValueError(f"expected {len(fields)} nonempty secret lines for {args.name}")
1016 token()
1017 path = "nomad/jobs/" + args.name
1018 existing = get_variable(path)
1019 items = dict(existing["Items"]) if existing else {}
1020 items.update(zip(fields, values))
1021 put_variable(path, items, existing["ModifyIndex"] if existing else 0)
1022 print(f"Imported {len(fields)} secrets for {args.name}")
1023 return
1024 if args.mode == "destroy":
1025 if not args.name or not NAME.fullmatch(args.name):
1026 parser.error("destroy requires a stage ID")
1027 token()
1028 metadata = json.loads((STATE / "stages" / f"{args.name}.json").read_text())
1029 names = {metadata["sourceId"]} | {request["provider"] for request in metadata["inputs"].values() if request["provider"] != "snowglobe"}
1030 definitions = {name: load(name, properties) for name in names}
1031 destroy_stage(args.name, metadata, definitions)
1032 return
1033 if args.mode == "stage":
1034 if not args.name:
1035 parser.error("stage requires a service name")
1036 token()
1037 data = load(args.name, properties)
1038 check_staging_pool(data)
1039 definitions = {args.name: data}
1040 definitions.update({provider: load(provider, properties) for provider in dependencies(data)})
1041 if not data["containers"]:
1042 raise ValueError(f"service has no container to stage: {args.name}")
1043 stage_dir = STATE / "stages"
1044 previous = []
1045 for path in stage_dir.glob("*.json"):
1046 saved = json.loads(path.read_text())
1047 if saved.get("sourceId") == args.name:
1048 previous.append((path, saved))
1049 if len(previous) > 1:
1050 raise ValueError(f"multiple stages exist for {args.name}; destroy one first")
1051 created = not previous
1052 stage = f"{args.name}-preview-{secrets.token_hex(4)}" if created else previous[0][0].stem
1053 http = [task["http"] for task in data["containers"].values() if task.get("http") and task["http"].get("hostname")]
1054 if len(http) > 1:
1055 raise ValueError("preview requires at most one HTTP route")
1056 original_host = http[0].get("hostname") if http else None
1057 hostname = stage + "." + original_host.split(".", 1)[1] if original_host else None
1058 data = load(args.name, properties, stage)
1059 data["rollout"] = "simple"
1060 data["hostRoot"] = str(Path(data["stagingRoot"]) / stage)
1061 if data["stageIsolation"] == "fresh" and any(
1062 volume.get("src") and not volume["readOnly"]
1063 for task in data["containers"].values()
1064 for volume in task["volumes"].values()
1065 ):
1066 raise ValueError("fresh stages cannot use writable external mounts")
1067 for task in data["containers"].values():
1068 if task.get("tcp") and not task["hostNetwork"]:
1069 for port in task["tcp"]:
1070 port["hostPort"] = None
1071 if hostname:
1072 hosts = {original_host: hostname, **{host: stage + "-" + host for host in http[0]["plainHostnames"]}}
1073 for task in data["containers"].values():
1074 if task.get("http") and task["http"].get("hostname") == original_host:
1075 task["http"]["hostname"] = hostname
1076 task["http"]["plainHostnames"] = [hosts[host] for host in task["http"]["plainHostnames"]]
1077 for source, target in hosts.items():
1078 task["env"] = {key: value.replace(source, target) for key, value in task["env"].items()}
1079 if task.get("secretFiles"):
1080 task["secretFiles"] = {key: value.replace(source, target) for key, value in task["secretFiles"].items()}
1081 for request in data["inputs"].values():
1082 if request["provider"] == "snowglobe":
1083 request["redirectUris"] = [uri.replace(original_host, hostname) for uri in request["redirectUris"]]
1084 for assignment in args.env:
1085 key, separator, value = assignment.partition("=")
1086 if not separator or len(data["containers"]) != 1:
1087 parser.error(f"unknown environment override: {key}")
1088 env = next(iter(data["containers"].values()))["env"]
1089 if key not in env:
1090 parser.error(f"unknown environment override: {key}")
1091 env[key] = value
1092 metadata = (
1093 {"mount": data["hostRoot"], "inputs": data["inputs"],
1094 "tasks": list(data["containers"]), "sourceId": args.name,
1095 "stageIsolation": data["stageIsolation"]}
1096 if created else previous[0][1]
1097 )
1098 if not created:
1099 if metadata.get("stageIsolation", "clone") != data["stageIsolation"]:
1100 raise ValueError("stage isolation changed; destroy the stage before recreating it")
1101 if bool(metadata.get("clone")) != data["hasManagedVolumes"]:
1102 raise ValueError("storage layout changed; destroy the stage before recreating it")
1103 for alias, request in data["inputs"].items():
1104 if alias in metadata["inputs"] and metadata["inputs"][alias] != request:
1105 raise ValueError(f"requirement {alias} changed; destroy the stage before recreating it")
1106 if alias not in metadata["inputs"] and request["provider"] == "postgres":
1107 raise ValueError("new database requirement needs a new stage")
1108 metadata["inputs"].update(data["inputs"])
1109 metadata["tasks"] = sorted(set(metadata["tasks"]) | set(data["containers"]))
1110 stage_dir.mkdir(parents=True, exist_ok=True)
1111 postgres_snapshot = None
1112 pending_snapshots = set()
1113 try:
1114 postgres_inputs = created and data["stageIsolation"] == "clone" and any(
1115 request["provider"] == "postgres" and request["kind"] == "database"
1116 for request in data["inputs"].values()
1117 )
1118 snapshots = []
1119 source = None
1120 if created and data["hasManagedVolumes"]:
1121 pool = data["pool"]
1122 source = subprocess.run(
1123 ["findmnt", "-n", "-o", "SOURCE", "--mountpoint", definitions[args.name]["hostRoot"]],
1124 capture_output=True, text=True,
1125 ).stdout.strip()
1126 dataset = pool + "/staging/" + stage
1127 if subprocess.run(["zfs", "list", pool + "/staging"], capture_output=True).returncode:
1128 command("zfs", "create", "-o", "mountpoint=none", pool + "/staging")
1129 if source and data["stageIsolation"] == "clone":
1130 if source.split("/", 1)[0] != pool:
1131 raise ValueError(f"production dataset belongs to another pool: {source}")
1132 snapshot = source + "@" + stage
1133 snapshots.append(snapshot)
1134 if postgres_inputs:
1135 postgres_dataset = dataset_for("postgres")
1136 if not postgres_dataset:
1137 raise ValueError("Postgres needs a ZFS dataset for consistent stages")
1138 postgres_snapshot = postgres_dataset + "@" + stage
1139 snapshots.append(postgres_snapshot)
1140 external = {}
1141 for task in data["containers"].values():
1142 for volume in task["volumes"].values():
1143 if volume.get("src") and not volume["readOnly"]:
1144 external.setdefault(volume["src"], []).append(volume)
1145 external_mounts = {}
1146 for external_source in external:
1147 path = Path(external_source).resolve(strict=True)
1148 mount = json.loads(command(
1149 "findmnt", "--json", "-o", "SOURCE,TARGET,FSTYPE,OPTIONS", "--target", str(path), capture=True,
1150 ).stdout)["filesystems"][0]
1151 if "ro" not in mount["options"].split(",") and mount["fstype"] != "zfs":
1152 raise ValueError(f"writable external mount is not ZFS: {external_source}")
1153 external_mounts[external_source] = (path, mount)
1154 if created and mount["fstype"] == "zfs" and "ro" not in mount["options"].split(","):
1155 target = mount["source"] + "@" + stage
1156 if target not in snapshots:
1157 snapshots.append(target)
1158 if snapshots:
1159 if len({target.split("/", 1)[0] for target in snapshots}) != 1:
1160 raise ValueError("staged datasets must belong to one ZFS pool")
1161 command("zfs", "snapshot", *snapshots)
1162 pending_snapshots.update(snapshots)
1163 if created and data["hasManagedVolumes"]:
1164 if source and data["stageIsolation"] == "clone":
1165 try:
1166 clone_dataset(source, snapshot, dataset, data["hostRoot"])
1167 except Exception:
1168 command("zfs", "destroy", snapshot)
1169 pending_snapshots.remove(snapshot)
1170 raise
1171 metadata["snapshot"] = snapshot
1172 pending_snapshots.remove(snapshot)
1173 else:
1174 command("zfs", "create", "-o", "mountpoint=" + data["hostRoot"], dataset)
1175 metadata["clone"] = dataset
1176 bindings = {}
1177 for source, volumes in sorted(external.items()):
1178 path, mount = external_mounts[source]
1179 if "ro" in mount["options"].split(","):
1180 binding = {"src": source, "readOnly": True}
1181 elif mount["fstype"] == "zfs":
1182 dataset = mount["source"]
1183 if metadata.get("snapshot") == dataset + "@" + stage:
1184 clone_mount = data["hostRoot"]
1185 else:
1186 staged = metadata.setdefault("external", {})
1187 if dataset not in staged:
1188 if not created:
1189 raise ValueError("external storage changed; destroy the stage before recreating it")
1190 suffix = hashlib.sha256(dataset.encode()).hexdigest()[:8]
1191 pool = dataset.split("/", 1)[0]
1192 clone = pool + "/staging/" + stage + "-external-" + suffix
1193 clone_mount = str(Path(data["stagingRoot"]) / (stage + "-external-" + suffix))
1194 snapshot = dataset + "@" + stage
1195 if subprocess.run(["zfs", "list", pool + "/staging"], capture_output=True).returncode:
1196 command("zfs", "create", "-o", "mountpoint=none", pool + "/staging")
1197 try:
1198 clone_dataset(dataset, snapshot, clone, clone_mount)
1199 except Exception:
1200 command("zfs", "destroy", snapshot)
1201 pending_snapshots.remove(snapshot)
1202 raise
1203 staged[dataset] = {"clone": clone, "snapshot": snapshot, "mount": clone_mount}
1204 pending_snapshots.remove(snapshot)
1205 clone_mount = staged[dataset]["mount"]
1206 relative = path.relative_to(Path(mount["target"]).resolve())
1207 binding = {"src": str(Path(clone_mount) / relative), "readOnly": False}
1208 else:
1209 raise ValueError(f"writable external mount is not ZFS: {source}")
1210 bindings[source] = binding
1211 for volume in volumes:
1212 if path.is_relative_to(Path(data["cloverRoot"]).resolve()):
1213 volume["clover"] = True
1214 volume.update(binding)
1215 if not created and bindings != metadata.get("externalSources", {}):
1216 raise ValueError("external storage changed; destroy the stage before recreating it")
1217 metadata["externalSources"] = bindings
1218 if REPO.parent == Path("/opt/snowglobe/releases").resolve():
1219 metadata["release"] = REPO.name
1220 metadata["ready"] = False
1221 write_private(stage_dir / f"{stage}.json", json.dumps(metadata))
1222 bootstrap([data])
1223 if created and data["stageIsolation"] == "clone" and (data["secrets"] or data["requiredSecrets"]):
1224 source = get_variable("nomad/jobs/" + args.name)
1225 if source:
1226 put_variable("nomad/jobs/" + stage, source["Items"], 0)
1227 if postgres_snapshot:
1228 postgres_clone = postgres_dataset.rsplit("/prod/", 1)[0] + "/staging/" + stage + "-postgres"
1229 postgres_mount = Path(data["stagingRoot"]) / (stage + "-postgres")
1230 source_context = cloned_postgres(postgres_snapshot, postgres_clone, postgres_mount)
1231 else:
1232 source_context = nullcontext(None)
1233 with source_context as postgres_source:
1234 provision_inputs(data, definitions, stage, postgres_source)
1235 provision(data)
1236 prepare(data)
1237 build_images(data)
1238 submit(data, "run", definitions)
1239 except Exception:
1240 if created:
1241 destroy_stage(stage, metadata, definitions)
1242 raise
1243 finally:
1244 for snapshot in sorted(pending_snapshots):
1245 command("zfs", "destroy", snapshot)
1246 if hostname:
1247 check_http(hostname, http[0]["checkPath"], http[0]["tlsInternal"])
1248 configure(data)
1249 if REPO.parent == Path("/opt/snowglobe/releases").resolve():
1250 metadata = {**json.loads((stage_dir / f"{stage}.json").read_text()), **metadata}
1251 metadata["ready"] = True
1252 metadata["overrides"] = [assignment.partition("=")[0] for assignment in args.env]
1253 write_private(stage_dir / f"{stage}.json", json.dumps(metadata))
1254 print(f"stage={stage}\nrelease={metadata.get('release', '')}\nhost={hostname}\ndataset={metadata.get('clone', '')}")
1255 return
1256 definitions = {name: load(name, properties) for name in services()}
1257 if args.name and args.name not in definitions:
1258 raise ValueError(f"unknown service: {args.name}")
1259 if not args.name and args.mode == "deploy":
1260 names = [name for name in names if definitions[name]["containers"]]
1261 selected = set(names)
1262 while True:
1263 required = selected | set().union(*(dependencies(definitions[name]) for name in selected))
1264 missing = required - definitions.keys()
1265 if missing:
1266 raise ValueError(f"unknown dependency: {sorted(missing)}")
1267 if required == selected:
1268 break
1269 selected = required
1270 data = ordered([definitions[name] for name in sorted(selected)])
1271 if args.mode == "render":
1272 for item in data:
1273 if item["containers"]:
1274 print(render(item, definitions))
1275 elif args.mode == "validate":
1276 token()
1277 for item in data:
1278 submit(item, "validate", definitions)
1279 elif args.mode == "check":
1280 for item in data:
1281 for task in item["containers"].values():
1282 if task.get("http") and task["http"].get("hostname"):
1283 check_http(task["http"]["hostname"], task["http"]["checkPath"], task["http"]["tlsInternal"])
1284 elif args.mode == "preflight":
1285 check_pool(data)
1286 token()
1287 missing_secrets = {}
1288 for item in data:
1289 if item["containers"] and item["requiredSecrets"]:
1290 variable = get_variable("nomad/jobs/" + item["id"])
1291 values = variable["Items"] if variable else {}
1292 missing = [name for name in item["requiredSecrets"] if not values.get(name)]
1293 if missing:
1294 missing_secrets[item["id"]] = missing
1295 if missing_secrets:
1296 raise ValueError("missing required secrets: " + "; ".join(
1297 f"{name}: {', '.join(keys)}" for name, keys in missing_secrets.items()))
1298 elif args.mode == "pool":
1299 ensure_pool(data)
1300 elif args.mode == "allocate":
1301 ensure_pool(data)
1302 bootstrap(data)
1303 for item in data:
1304 provision_inputs(item, definitions)
1305 provision(item)
1306 prepare(item)
1307 build_images(item)
1308 elif args.mode == "bootstrap":
1309 bootstrap(data)
1310 elif args.mode == "deploy":
1311 bootstrap(data)
1312 for item in data:
1313 provision_inputs(item, definitions)
1314 provision(item)
1315 prepare(item)
1316 build_images(item)
1317 submit(item, "run", definitions)
1318 configure(item)
1319
1320
1321if __name__ == "__main__":
1322 try:
1323 main()
1324 except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
1325 print(error, file=sys.stderr)
1326 sys.exit(1)