1#!/usr/bin/env python3
2import concurrent.futures
3from importlib import import_module
4import json
5import os
6from pathlib import Path
7import re
8import secrets
9import shlex
10import shutil
11import socket
12import sqlite3
13import subprocess
14import sys
15import tempfile
16import threading
17import time
18
19import snowglobe
20
21
22OPERATIONS = frozenset({
23 "shale.access", "shale.clone",
24 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",
25 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",
26 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",
27 "files.snapshot", "files.snapshots", "files.discard",
28})
29STAGE_ID = re.compile(r"dashboard-preview-[0-9a-f]{8}\Z")
30boundary = import_module("dashboard-host")
31
32
33class PreviewHost(boundary.Host):
34 def __init__(self, saved):
35 super().__init__(saved["clone"].split("/", 1)[0])
36 self.stage = saved
37
38 def handle(self, request):
39 operation = request.get("operation") if isinstance(request, dict) else None
40 owner = None
41 if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}:
42 with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts:
43 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
44 if not owner:
45 raise ValueError("The preview account is unavailable.")
46 if operation == "vm.ai":
47 payload = request.get("payload")
48 if not owner or not isinstance(payload, dict) or payload.get("owner") != owner[0]:
49 raise boundary.dashboard_runs.Error(403, "Test VMs must belong to this preview account.")
50 if operation in {"vm.computer", "vm.owner", "vm.access", "vm.act", "vm.console", "vm.guest"}:
51 payload = request.get("payload")
52 if not self.stage.get("computer") or not isinstance(payload, dict):
53 raise boundary.dashboard_runs.Error(403, "Choose the VM explicitly attached to this preview.")
54 if payload.get("name") not in self.stage["computer"].split(","):
55 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}
56 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:
57 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")
58 if operation in {"shale.access", "shale.clone"}:
59 service = request.get("service", "")
60 if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
61 raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.")
62 linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text())
63 if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name:
64 raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.")
65 if operation == "shale.clone":
66 return import_module("shale-access").clone(request)
67 return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data")
68 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:
69 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]
70 if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]:
71 raise boundary.dashboard_runs.Error(403, "File changes are limited to this preview's files.")
72 if operation == "storage.mounts" and set(request) == {"operation"}:
73 mounted = json.loads(snowglobe.command(
74 "findmnt", "--json", "-o", "SOURCE,TARGET,FSTYPE,OPTIONS", "--mountpoint", self.stage["mount"] + "/files", capture=True,
75 ).stdout)
76 mounted["filesystems"][0]["target"] = "/srv"
77 return mounted
78 result = super().handle(request)
79 if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}:
80 if operation == "vm.owner":
81 result = {**result, "owner": owner[0], "ownerOnly": True}
82 else:
83 result = [{**vm, "owner": owner[0], "ownerOnly": True} if vm["name"] in self.stage["computer"].split(",") else vm for vm in result]
84 return result
85
86
87def metadata(stage):
88 if not STAGE_ID.fullmatch(stage):
89 raise ValueError("invalid dashboard stage ID")
90 saved = json.loads((snowglobe.STATE / "stages" / f"{stage}.json").read_text())
91 if saved.get("sourceId") != "dashboard":
92 raise ValueError("not a dashboard stage")
93 return saved
94
95
96def serve(stage):
97 saved = metadata(stage)
98 host = PreviewHost(saved)
99 runtime = Path("/run/snowglobe-preview") / stage
100 runtime.mkdir(parents=True, exist_ok=True)
101 os.chown(runtime, 0, saved["uid"])
102 runtime.chmod(0o750)
103 path = runtime / "host.sock"
104 path.unlink(missing_ok=True)
105 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
106 listener.bind(str(path))
107 os.chown(path, 0, saved["uid"])
108 path.chmod(0o660)
109 listener.listen(8)
110 slots = threading.BoundedSemaphore(16)
111 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:
112 while True:
113 connection, _ = listener.accept()
114 if not slots.acquire(blocking=False):
115 connection.close()
116 continue
117 operations = OPERATIONS | {"vm.computer", "vm.owner", "vm.access", "vm.ai", "vm.act", "vm.console", "vm.guest"} if saved.get("computer") else OPERATIONS
118 work = workers.submit(boundary.serve_connection, connection, host, saved["uid"], operations)
119 work.add_done_callback(lambda _: slots.release())
120
121
122def stage(computer=None):
123 snowglobe.token()
124 site = json.loads(snowglobe.command("pkl", "eval", "-f", "json", str(snowglobe.REPO / "config/site.pkl"), capture=True).stdout)
125 site["hostRoot"] = str(Path(site["root"]) / "prod/dashboard")
126 site["stagingRoot"] = str(Path(site["root"]) / "staging")
127 snowglobe.check_staging_pool(site)
128 directory = snowglobe.STATE / "stages"
129 previous = [path for path in directory.glob("*.json") if json.loads(path.read_text()).get("sourceId") == "dashboard"]
130 if len(previous) > 1:
131 raise ValueError("multiple dashboard stages exist; destroy one first")
132 created = not previous
133 identity = "dashboard-preview-" + secrets.token_hex(4) if created else previous[0].stem
134 hostname = identity + "." + site["domain"]
135 root = Path(site["stagingRoot"]) / identity
136 saved = {"sourceId": "dashboard", "tasks": ["app"], "inputs": {}, "mount": str(root),
137 "clone": site["pool"] + "/staging/" + identity, "uid": snowglobe.identity("dashboard"),
138 "stageIsolation": "fresh"} if created else metadata(identity)
139 if computer:
140 saved["computer"] = computer
141 host_environment = shlex.split(snowglobe.command("systemctl", "show", "snowglobe-host", "--property=Environment", "--value", capture=True).stdout)
142 host_path = next(value for value in host_environment if value.startswith("PATH="))
143 mac = None
144 if saved.get("computer"):
145 configured = next((value.partition("=")[2] for value in host_environment if value.startswith("SNOWGLOBE_MAC_VM=")), "null")
146 mac = json.loads(configured)
147 selected = saved["computer"].split(",")
148 for name in selected:
149 boundary.vms.ensure_name(name)
150 if not mac or name != mac["name"]:
151 snowglobe.command("virsh", "domuuid", name, capture=True)
152 if mac and mac["name"] not in selected:
153 mac = None
154 nix = ["nix", "--extra-experimental-features", "nix-command flakes"]
155 package = "path:" + str(snowglobe.REPO) + "#dashboard-image"
156 image = subprocess.run([*nix, "eval", "--raw", package, "--apply", 'image: "${image.imageName}:${image.imageTag}"'],
157 stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
158 if subprocess.run(["podman", "image", "exists", image], capture_output=True).returncode:
159 with tempfile.TemporaryDirectory(prefix="dashboard-image-", dir=site["stagingRoot"]) as temporary:
160 built = subprocess.run([*nix, "build", package + ".stream", "--option", "build-dir", temporary,
161 "--out-link", str(Path(temporary) / "stream"), "--print-out-paths"],
162 stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
163 environment = {**os.environ, "TMPDIR": temporary}
164 with subprocess.Popen([built], stdout=subprocess.PIPE, env=environment) as stream:
165 try:
166 subprocess.run(["podman", "load", "--quiet"], stdin=stream.stdout, env=environment, check=True)
167 finally:
168 stream.stdout.close()
169 if stream.wait():
170 raise RuntimeError("dashboard image stream failed")
171 env = {
172 "SNOWGLOBE_DOMAIN": site["domain"], "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + hostname,
173 "SNOWGLOBE_FILE_ORIGIN": "https://files-" + hostname, "SNOWGLOBE_AUTH_RP_ID": hostname,
174 "SNOWGLOBE_AUTH_REQUIRED": "1", "SNOWGLOBE_PROXY_TOKEN_FILE": "/run/secrets/proxy.token", "SNOWGLOBE_DATA_DIR": "/data",
175 "SNOWGLOBE_PREVIEW": "1", "SNOWGLOBE_METRICS": "follow",
176 "SNOWGLOBE_LISTEN_ADDRESS": "127.0.0.1", "PORT": "${NOMAD_PORT_http}",
177 "SNOWGLOBE_NOMAD_TOKEN_FILE": "/run/secrets/nomad.token", "NOMAD_ADDR": "http://127.0.0.1:4646",
178 "SNOWGLOBE_HOST_SOCKET": "/run/snowglobe-host/host.sock", "SNOWGLOBE_NODE_NAME": site["nodeName"],
179 "SNOWGLOBE_STORE_ROOT": "/srv", "SNOWGLOBE_MEDIA_ROOT": "/srv/Media",
180 "SNOWGLOBE_FILES_WRITABLE": "1", "SNOWGLOBE_CLOVER_READ_ONLY": "false", "SNOWGLOBE_MEDIA_READ_ONLY": "false",
181 "HOME": "/data", "XDG_CACHE_HOME": "/data/cache",
182 }
183 try:
184 if created:
185 snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root), saved["clone"])
186 saved["external"] = {"files": {"clone": saved["clone"] + "/files"}}
187 directory.mkdir(parents=True, exist_ok=True)
188 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
189 snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root / "files"), saved["clone"] + "/files")
190 for path in (root / "data", root / "files", root / "files/Media"):
191 path.mkdir(parents=True, exist_ok=True)
192 os.chown(path, saved["uid"], saved["uid"])
193 path.chmod(0o750)
194 fixture = root / "files/preview.txt"
195 fixture.write_text("Disposable Snow Globe dashboard preview.\n")
196 fixture.chmod(0o644)
197 proof = root / "proxy.token"
198 if not proof.exists():
199 snowglobe.write_private(proof, secrets.token_hex(32) + "\n")
200 os.chown(proof, saved["uid"], saved["uid"])
201 proof.chmod(0o400)
202 directory.mkdir(parents=True, exist_ok=True)
203 saved.update(release=snowglobe.REPO.name, ready=False)
204 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
205 if created:
206 policy = 'namespace "default" { capabilities = ["list-jobs", "read-job"] }\nnode { policy = "read" }\n'
207 snowglobe.command("nomad", "acl", "policy", "apply", identity, "-", input=policy)
208 token = json.loads(snowglobe.command("nomad", "acl", "token", "create", "-policy=" + identity,
209 "-name=" + identity, "-json", capture=True).stdout)
210 saved["tokenAccessor"] = token["AccessorID"]
211 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
212 snowglobe.write_private(root / "nomad.token", token["SecretID"] + "\n")
213 os.chown(root / "nomad.token", saved["uid"], saved["uid"])
214 imported = snowglobe.command("podman", "run", "--rm", "--network=none", "--read-only",
215 "--user=" + str(saved["uid"]), "--cap-drop=ALL", "--security-opt=no-new-privileges",
216 "--entrypoint=/bin/home-dashboard",
217 "--volume=" + str(root / "data") + ":/data:rw",
218 "--volume=" + str(proof) + ":/run/secrets/proxy.token:ro",
219 *["--env=" + key + "=" + value for key, value in env.items()], image, "--preview-account", capture=True)
220 snowglobe.write_private(root / "access.json", imported.stdout)
221 runtime = "/run/snowglobe-preview/" + identity
222 computer_environment = []
223 if saved.get("computer"):
224 ahk = snowglobe.command(*nix, "build", "--out-link", str(root / "computer-ahk"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#computer-ahk", capture=True).stdout.strip()
225 computer_environment.append("--setenv=SNOWGLOBE_COMPUTER_AHK=" + ahk)
226 if mac:
227 with sqlite3.connect(f"file:{root / 'data/accounts.sqlite'}?mode=ro", uri=True) as accounts:
228 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
229 if not owner:
230 raise ValueError("The preview account is unavailable.")
231 mac["owner"] = owner[0]
232 computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac))
233 host_python = snowglobe.command(*nix, "build", "--out-link", str(root / "host-python"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip()
234 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
235 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
236 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity,
237 "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32",
238 "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"],
239 *computer_environment,
240 host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity)
241 for _ in range(100):
242 if Path(runtime, "host.sock").exists():
243 break
244 time.sleep(.1)
245 else:
246 raise RuntimeError("dashboard preview host socket did not start")
247 screen_volumes = []
248 if saved.get("computer"):
249 screen = snowglobe.command(*nix, "build", "--out-link", str(root / "vm-screen"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#vm-screen", capture=True).stdout.strip()
250 screen_runtime = "/run/snowglobe-preview-screen-" + identity
251 launcher = f"import os; os.chown({screen_runtime!r}, {saved['uid']}, {saved['uid']}); os.setgroups([]); os.setgid({saved['uid']}); os.setuid({saved['uid']}); os.execv({screen + '/bin/snowglobe-vm-screen'!r}, [{screen + '/bin/snowglobe-vm-screen'!r}])"
252 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-screen-" + identity,
253 "--property=RuntimeDirectory=snowglobe-preview-screen-" + identity, "--property=RuntimeDirectoryMode=0700",
254 "--property=RuntimeDirectoryPreserve=yes",
255 "--property=Restart=on-failure", "--property=MemoryMax=2G", "--property=TasksMax=128",
256 "--property=NoNewPrivileges=yes", "--property=ProtectSystem=strict", "--property=ProtectHome=yes", "--property=PrivateTmp=yes",
257 "--property=DevicePolicy=closed",
258 "--setenv=SNOWGLOBE_HOST_SOCKET=" + runtime + "/host.sock",
259 "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock",
260 "--setenv=XDG_RUNTIME_DIR=" + screen_runtime,
261 "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache",
262 host_python + "/bin/python3", "-c", launcher)
263 for _ in range(100):
264 if Path(screen_runtime, "screen.sock").exists():
265 break
266 time.sleep(.1)
267 else:
268 raise RuntimeError("dashboard preview screen socket did not start")
269 screen_volumes.append(screen_runtime + ":/run/snowglobe-vm-screen:ro")
270 job = {"ID": identity, "Name": identity, "Type": "service", "Datacenters": ["clover"],
271 "Meta": {"snowglobe_service": "dashboard", "snowglobe_name": "Snow Globe preview", "snowglobe_hostname": hostname},
272 "TaskGroups": [{"Name": "app", "Count": 1, "Networks": [{"Mode": "host", "DynamicPorts": [{"Label": "http", "HostNetwork": "loopback"}]}],
273 "Services": [{"Name": identity, "Provider": "nomad", "PortLabel": "http", "TaskName": "app", "Tags": ["caddy-host=" + hostname],
274 "Checks": [{"Name": "Dashboard listener", "Type": "tcp", "Interval": 10000000000, "Timeout": 5000000000}]}],
275 "Tasks": [{"Name": "app", "Driver": "podman", "User": str(saved["uid"]) + ":" + str(saved["uid"]), "Env": env,
276 "Config": {"image": image, "network_mode": "host", "cap_drop": ["ALL"], "security_opt": ["no-new-privileges"],
277 "volumes": [str(root / "data") + ":/data:rw", str(root / "files") + ":/srv:rw,rslave", runtime + ":/run/snowglobe-host:ro",
278 str(root / "nomad.token") + ":/run/secrets/nomad.token:ro",
279 str(proof) + ":/run/secrets/proxy.token:ro", *screen_volumes]},
280 "Resources": {"CPU": 1000, "MemoryMB": 2048}}]}]}
281 snowglobe.command("nomad", "job", "run", "-json", "-", input=json.dumps(job))
282 snowglobe.check_http(hostname, "/sign-in", site["tlsInternal"])
283 saved["ready"] = True
284 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
285 except BaseException:
286 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
287 if created and (directory / f"{identity}.json").exists():
288 destroy(identity)
289 raise
290 print(f"stage={identity}\nrelease={saved['release']}\nhost={hostname}\ndataset={saved['clone']}\naccess={root / 'access.json'}")
291
292
293def destroy(identity):
294 snowglobe.token()
295 saved = metadata(identity)
296 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
297 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
298 if saved.get("tokenAccessor"):
299 revoked = subprocess.run(["nomad", "acl", "token", "delete", saved["tokenAccessor"]], capture_output=True, text=True)
300 if revoked.returncode and "400 (Cannot delete nonexistent tokens:" not in revoked.stderr:
301 raise RuntimeError(revoked.stderr.strip())
302 snowglobe.command("nomad", "acl", "policy", "delete", identity, capture=True)
303 datasets = set(snowglobe.command("zfs", "list", "-H", "-o", "name", capture=True).stdout.splitlines())
304 saved["external"] = {name: item for name, item in saved.get("external", {}).items() if item["clone"] in datasets}
305 if saved.get("external", {}).get("files", {}).get("clone") == saved["clone"] + "/files":
306 snapshots = snowglobe.command("zfs", "list", "-H", "-t", "snapshot", "-r", "-o", "name", saved["clone"] + "/files", capture=True).stdout.splitlines()
307 for snapshot in snapshots:
308 if snapshot.startswith(saved["clone"] + "/files@"):
309 snowglobe.command("zfs", "destroy", snapshot)
310 if saved["clone"] not in datasets:
311 del saved["clone"]
312 snowglobe.destroy_stage(identity, saved, {})
313 shutil.rmtree(Path("/run/snowglobe-preview") / identity, ignore_errors=True)
314 shutil.rmtree(Path("/run") / ("snowglobe-preview-screen-" + identity), ignore_errors=True)
315
316
317if __name__ == "__main__":
318 if len(sys.argv) == 3 and sys.argv[1] == "--serve":
319 serve(sys.argv[2])
320 else:
321 raise SystemExit("Expected --serve STAGE")