1#!/usr/bin/env python3
2import argparse
3import json
4import os
5from pathlib import Path
6import re
7import shlex
8import shutil
9import subprocess
10import sys
11import tempfile
12import time
13
14from release import BUILD_DIRS, SOURCES, tree_digest
15
16
17REPO = Path(__file__).resolve().parent.parent
18HOST = os.environ.get("SNOWGLOBE_DEPLOY_HOST", "root@10.0.0.1")
19PORT = os.environ.get("SNOWGLOBE_DEPLOY_PORT", "22")
20REMOTE = Path("/opt/snowglobe")
21NAME = re.compile(r"[a-z][a-z0-9-]*\Z")
22RELEASE_ID = re.compile(r"[0-9a-f]{16}\Z")
23
24
25def ssh(command, capture=False):
26 return subprocess.run(
27 ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, command],
28 check=True, text=True, capture_output=capture,
29 )
30
31
32def run_logged(mode, target, command):
33 run = f"{int(time.time() * 1000)}-{mode}-{target or 'current'}"
34 path = f"/var/lib/snowglobe/runs/{run}.log"
35 script = f"umask 077; mkdir -p /var/lib/snowglobe/runs; set -o pipefail; {command} 2>&1 | tee {shlex.quote(path)}"
36 ssh("bash -c " + shlex.quote(script))
37 print(f"run={run}")
38
39
40def sync_manager(release):
41 source = REMOTE / "releases" / release / "tools"
42 ssh(f"cp {source}/release.py {source}/data.py {REMOTE}/")
43
44
45def export_revision(source, commit, paths, destination):
46 entries = subprocess.run(
47 ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T",
48 '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"',
49 *("cwd:" + json.dumps(path) for path in paths)],
50 cwd=source, check=True, capture_output=True, text=True,
51 ).stdout.splitlines()
52 for entry in entries:
53 name, kind, executable = json.loads(entry)
54 relative = Path(name)
55 if kind != "file" or relative.is_absolute() or ".." in relative.parts:
56 raise ValueError(f"unsupported revision file: {name}")
57 target = destination / relative
58 target.parent.mkdir(parents=True, exist_ok=True)
59 target.write_bytes(subprocess.run(
60 ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name],
61 cwd=source, check=True, capture_output=True,
62 ).stdout)
63 target.chmod(0o755 if executable else 0o644)
64
65
66def upload(main=False):
67 with tempfile.TemporaryDirectory() as temporary:
68 snapshot = Path(temporary).resolve()
69 revision = None
70 if main:
71 output = subprocess.run(
72 ["jj", "--ignore-working-copy", "log", "-r", "main", "--no-graph", "-T",
73 'json(commit_id) ++ "\\n" ++ json(conflict) ++ "\\n" ++ json(description)'],
74 cwd=REPO, check=True, capture_output=True, text=True,
75 ).stdout.splitlines()
76 commit, conflicted, description = map(json.loads, output)
77 if conflicted or not description.strip():
78 raise ValueError("main needs a commit description and no conflicts before deployment")
79 revision = {"commit": commit, "description": description}
80 export_revision(REPO, commit, SOURCES, snapshot)
81 else:
82 subprocess.run(
83 ["rsync", "-a", "--exclude=.DS_Store", "--exclude=__pycache__", "--exclude=*.pyc",
84 "--exclude=.identities.lock", "--exclude=identities.pending", "--exclude=._*",
85 *(f"--exclude={directory}" for directory in BUILD_DIRS),
86 *(str(REPO / path) for path in SOURCES),
87 str(snapshot) + "/"], check=True,
88 )
89 for manifest in snapshot.glob("service/*/build-source.json"):
90 spec = json.loads(manifest.read_text())
91 context = manifest.parent / "build"
92 if context.exists():
93 raise ValueError(f"build context already exists: {manifest.parent.name}")
94 context.mkdir()
95 for index, source_spec in enumerate(spec.get("sources", [spec])):
96 source = (REPO / source_spec["source"]).resolve(strict=True)
97 if not source.is_dir() or not source.is_relative_to(REPO.parent):
98 raise ValueError(f"invalid build source: {manifest.parent.name}")
99 target = Path(source_spec.get("destination", "."))
100 if target.is_absolute() or ".." in target.parts or (index and not target.parts):
101 raise ValueError(f"invalid build source destination: {target}")
102 source_context = context / target
103 if source_context != context:
104 source_context.mkdir(parents=True, exist_ok=False)
105 dockerfile_name = source_spec.get("dockerfile")
106 paths = [*source_spec["include"], *([dockerfile_name] if dockerfile_name else [])]
107 for name in paths:
108 relative = Path(name)
109 if relative.is_absolute() or ".." in relative.parts or not relative.parts:
110 raise ValueError(f"invalid build source path: {name}")
111 if "revision" in source_spec:
112 commit = source_spec["revision"]
113 if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{40}", commit):
114 raise ValueError(f"invalid build source revision: {manifest.parent.name}")
115 export_revision(source, commit, paths, source_context)
116 for name in paths:
117 if not (source_context / name).exists():
118 raise ValueError(f"missing build source path at revision: {name}")
119 if dockerfile_name:
120 dockerfile = source_context / dockerfile_name
121 if not dockerfile.is_file():
122 raise ValueError(f"invalid build source Dockerfile: {dockerfile_name}")
123 if dockerfile != source_context / "Dockerfile":
124 if (source_context / "Dockerfile").exists():
125 raise ValueError("build source already includes Dockerfile")
126 if any(dockerfile.is_relative_to(source_context / name) for name in source_spec["include"]):
127 shutil.copy2(dockerfile, source_context / "Dockerfile")
128 else:
129 dockerfile.rename(source_context / "Dockerfile")
130 continue
131 for name in paths:
132 relative = Path(name)
133 origin = source / relative
134 if not origin.resolve(strict=True).is_relative_to(source):
135 raise ValueError(f"build source escapes context: {name}")
136 destination = source_context / ("Dockerfile" if name == dockerfile_name else name)
137 destination.parent.mkdir(parents=True, exist_ok=True)
138 if origin.is_dir():
139 shutil.copytree(origin, destination, symlinks=True)
140 else:
141 shutil.copy2(origin, destination)
142 if "sources" in spec and spec.get("dockerfile"):
143 relative = Path(spec["dockerfile"])
144 dockerfile = manifest.parent / relative
145 if (relative.is_absolute() or ".." in relative.parts or not relative.parts
146 or not dockerfile.resolve(strict=True).is_relative_to(manifest.parent)
147 or not dockerfile.is_file() or (context / "Dockerfile").exists()):
148 raise ValueError(f"invalid build context Dockerfile: {relative}")
149 shutil.copy2(dockerfile, context / "Dockerfile")
150 if not (context / "Dockerfile").is_file():
151 raise ValueError(f"missing build source Dockerfile: {manifest.parent.name}")
152 if spec.get("includeManifest"):
153 shutil.copy2(manifest, context / ".snowglobe-source.json")
154 digest = tree_digest(snapshot)
155 release = digest[:16]
156 remote_release = REMOTE / "releases" / release
157 details = {"id": release, "digest": digest, "version": 2}
158 if revision:
159 details["main"] = revision
160 (snapshot / ".snowglobe-release.json").write_text(json.dumps(details) + "\n")
161 if subprocess.run(
162 ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, f"test -d {remote_release}"],
163 check=False,
164 ).returncode == 0:
165 ssh(f"python3 {remote_release}/tools/release.py verify {release}")
166 if revision:
167 script = "import pathlib,sys; p=pathlib.Path(" + repr(str(remote_release / ".snowglobe-release.json")) + "); t=p.with_suffix('.pending'); t.write_text(sys.stdin.read()); t.chmod(0o444); t.replace(p)"
168 subprocess.run(["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST,
169 "python3 -c " + shlex.quote(script)], input=json.dumps(details) + "\n", text=True, check=True)
170 return release
171 ssh(f"mkdir -p {remote_release}")
172 subprocess.run(
173 ["rsync", "-a", "-e", f"ssh -p {PORT} -o BatchMode=yes",
174 *(str(snapshot / path) for path in SOURCES), str(snapshot / ".snowglobe-release.json"),
175 f"{HOST}:{remote_release}/"], check=True,
176 )
177 ssh(f"chmod -R a-w {remote_release}")
178 return release
179
180
181def main():
182 parser = argparse.ArgumentParser(description="Preview working changes and deploy main")
183 parser.add_argument("mode", choices=["stage", "publish", "prod", "rollback", "history", "backups", "data-restore", "bootstrap", "allocate", "destroy", "secrets"])
184 parser.add_argument("--computer", help="Give a dashboard preview access to these comma-separated VM names")
185 parser.add_argument("target", nargs="?")
186 parser.add_argument("--env", action="append", default=[])
187 parser.add_argument("--file", type=Path)
188 parser.add_argument("--key", action="append", default=[])
189 parser.add_argument("--backup")
190 parser.add_argument("--discard-writes", action="store_true")
191 args = parser.parse_args()
192 if args.computer and (args.mode != "stage" or args.target != "dashboard" or not all(NAME.fullmatch(name) for name in args.computer.split(","))):
193 parser.error("--computer requires stage dashboard and VM names")
194 if args.mode in {"stage", "allocate", "destroy", "secrets", "data-restore"} and (not args.target or not NAME.fullmatch(args.target)):
195 parser.error(f"{args.mode} requires a service or stage ID")
196 if args.mode == "rollback" and args.target and not RELEASE_ID.fullmatch(args.target):
197 parser.error("rollback target must be a release ID")
198 if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target:
199 parser.error(f"{args.mode} takes no target")
200 if args.env and (args.mode != "stage" or args.target == "host"):
201 parser.error("--env requires a service or dashboard stage")
202 if bool(args.file) != (args.mode == "secrets"):
203 parser.error("--file is required only for secrets")
204 if args.key and args.mode != "secrets":
205 parser.error("--key is available only for secrets")
206 if len(set(args.key)) != len(args.key) or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) for key in args.key):
207 parser.error("--key values must be unique secret names")
208 if args.mode == "data-restore":
209 if not args.backup or not re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{6}", args.backup) or not args.discard_writes:
210 parser.error("data-restore requires --backup ID and --discard-writes")
211 elif args.backup or args.discard_writes:
212 parser.error("--backup and --discard-writes are available only for data-restore")
213 if args.mode == "secrets":
214 if str(args.file) == "-":
215 values = sys.stdin.read()
216 else:
217 if args.file.stat().st_mode & 0o077:
218 raise ValueError("secret file must be readable only by its owner")
219 values = args.file.read_text()
220 if not values.strip():
221 raise ValueError("secret input is empty")
222 manager = f"python3 {REMOTE}/release.py"
223 if args.mode == "stage":
224 release = upload()
225 if args.target == "host":
226 run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}")
227 return
228 overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env)
229 if args.computer:
230 overrides += " --computer " + shlex.quote(args.computer)
231 run_logged("stage", args.target, f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py stage {args.target}{overrides}")
232 elif args.mode == "secrets":
233 release = upload()
234 keys = "".join(f" --key {shlex.quote(key)}" for key in args.key)
235 subprocess.run(
236 ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST,
237 f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py secrets {args.target}{keys}"],
238 input=values, text=True, check=True,
239 )
240 elif args.mode == "bootstrap":
241 release = upload(main=True)
242 sync_manager(release)
243 ssh(f"{manager} publish {release}")
244 ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py pool")
245 ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py bootstrap")
246 ssh(f"python3 {REMOTE}/releases/{release}/tools/release.py bootstrap {release}")
247 elif args.mode == "allocate":
248 release = upload()
249 ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py allocate {args.target}")
250 elif args.mode in {"publish", "prod"}:
251 release = upload(main=True)
252 sync_manager(release)
253 ssh(f"{manager} publish {release}")
254 if args.mode == "prod":
255 run_logged("prod", release, f"{manager} deploy {release}")
256 elif args.mode == "rollback":
257 run_logged("rollback", args.target, f"{manager} rollback{(' ' + args.target) if args.target else ''}")
258 elif args.mode == "history":
259 ssh(f"{manager} history")
260 elif args.mode == "backups":
261 release = upload()
262 ssh(f"python3 {REMOTE}/releases/{release}/tools/data.py list")
263 elif args.mode == "data-restore":
264 release = upload()
265 run_logged("data-restore", args.target, f"python3 {REMOTE}/releases/{release}/tools/data.py restore {args.backup} {args.target} --discard-writes")
266 else:
267 release = upload()
268 run_logged("destroy", args.target, f"python3 {REMOTE}/releases/{release}/tools/release.py stage-destroy {args.target}")
269
270
271if __name__ == "__main__":
272 try:
273 main()
274 except (OSError, ValueError, subprocess.CalledProcessError) as error:
275 print(f"Home server deployment stopped: {error}", file=sys.stderr)
276 sys.exit(error.returncode if isinstance(error, subprocess.CalledProcessError) else 1)