| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | import json |
| 4 | import os |
| 5 | from pathlib import Path |
| 6 | import re |
| 7 | import shlex |
| 8 | import shutil |
| 9 | import subprocess |
| 10 | import sys |
| 11 | import tempfile |
| 12 | import time |
| 13 | |
| 14 | from release import BUILD_DIRS, SOURCES, tree_digest |
| 15 | |
| 16 | |
| 17 | REPO = Path(__file__).resolve().parent.parent |
| 18 | HOST = os.environ.get("SNOWGLOBE_DEPLOY_HOST", "root@10.0.0.1") |
| 19 | PORT = os.environ.get("SNOWGLOBE_DEPLOY_PORT", "22") |
| 20 | REMOTE = Path("/opt/snowglobe") |
| 21 | NAME = re.compile(r"[a-z][a-z0-9-]*\Z") |
| 22 | RELEASE_ID = re.compile(r"[0-9a-f]{16}\Z") |
| 23 | |
| 24 | |
| 25 | def ssh(command, capture=False): |
| 26 | return subprocess.run( |
| 27 | ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, command], |
| 28 | check=True, text=True, capture_output=capture, |
| 29 | ) |
| 30 | |
| 31 | |
| 32 | def run_logged(mode, target, command): |
| 33 | run = f"{int(time.time() * 1000)}-{mode}-{target or 'current'}" |
| 34 | path = f"/var/lib/snowglobe/runs/{run}.log" |
| 35 | script = f"umask 077; mkdir -p /var/lib/snowglobe/runs; set -o pipefail; {command} 2>&1 | tee {shlex.quote(path)}" |
| 36 | ssh("bash -c " + shlex.quote(script)) |
| 37 | print(f"run={run}") |
| 38 | |
| 39 | |
| 40 | def sync_manager(release): |
| 41 | source = REMOTE / "releases" / release / "tools" |
| 42 | ssh(f"cp {source}/release.py {source}/data.py {REMOTE}/") |
| 43 | |
| 44 | |
| 45 | def export_revision(source, commit, paths, destination): |
| 46 | entries = subprocess.run( |
| 47 | ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T", |
| 48 | '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"', |
| 49 | *("cwd:" + json.dumps(path) for path in paths)], |
| 50 | cwd=source, check=True, capture_output=True, text=True, |
| 51 | ).stdout.splitlines() |
| 52 | for entry in entries: |
| 53 | name, kind, executable = json.loads(entry) |
| 54 | relative = Path(name) |
| 55 | if kind != "file" or relative.is_absolute() or ".." in relative.parts: |
| 56 | raise ValueError(f"unsupported revision file: {name}") |
| 57 | target = destination / relative |
| 58 | target.parent.mkdir(parents=True, exist_ok=True) |
| 59 | target.write_bytes(subprocess.run( |
| 60 | ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name], |
| 61 | cwd=source, check=True, capture_output=True, |
| 62 | ).stdout) |
| 63 | target.chmod(0o755 if executable else 0o644) |
| 64 | |
| 65 | |
| 66 | def upload(main=False): |
| 67 | with tempfile.TemporaryDirectory() as temporary: |
| 68 | snapshot = Path(temporary).resolve() |
| 69 | revision = None |
| 70 | if main: |
| 71 | output = subprocess.run( |
| 72 | ["jj", "--ignore-working-copy", "log", "-r", "main", "--no-graph", "-T", |
| 73 | 'json(commit_id) ++ "\\n" ++ json(conflict) ++ "\\n" ++ json(description)'], |
| 74 | cwd=REPO, check=True, capture_output=True, text=True, |
| 75 | ).stdout.splitlines() |
| 76 | commit, conflicted, description = map(json.loads, output) |
| 77 | if conflicted or not description.strip(): |
| 78 | raise ValueError("main needs a commit description and no conflicts before deployment") |
| 79 | revision = {"commit": commit, "description": description} |
| 80 | export_revision(REPO, commit, SOURCES, snapshot) |
| 81 | else: |
| 82 | subprocess.run( |
| 83 | ["rsync", "-a", "--exclude=.DS_Store", "--exclude=__pycache__", "--exclude=*.pyc", |
| 84 | "--exclude=.identities.lock", "--exclude=identities.pending", "--exclude=._*", |
| 85 | *(f"--exclude={directory}" for directory in BUILD_DIRS), |
| 86 | *(str(REPO / path) for path in SOURCES), |
| 87 | str(snapshot) + "/"], check=True, |
| 88 | ) |
| 89 | for manifest in snapshot.glob("service/*/build-source.json"): |
| 90 | spec = json.loads(manifest.read_text()) |
| 91 | context = manifest.parent / "build" |
| 92 | if context.exists(): |
| 93 | raise ValueError(f"build context already exists: {manifest.parent.name}") |
| 94 | context.mkdir() |
| 95 | for index, source_spec in enumerate(spec.get("sources", [spec])): |
| 96 | source = (REPO / source_spec["source"]).resolve(strict=True) |
| 97 | if not source.is_dir() or not source.is_relative_to(REPO.parent): |
| 98 | raise ValueError(f"invalid build source: {manifest.parent.name}") |
| 99 | target = Path(source_spec.get("destination", ".")) |
| 100 | if target.is_absolute() or ".." in target.parts or (index and not target.parts): |
| 101 | raise ValueError(f"invalid build source destination: {target}") |
| 102 | source_context = context / target |
| 103 | if source_context != context: |
| 104 | source_context.mkdir(parents=True, exist_ok=False) |
| 105 | dockerfile_name = source_spec.get("dockerfile") |
| 106 | paths = [*source_spec["include"], *([dockerfile_name] if dockerfile_name else [])] |
| 107 | for name in paths: |
| 108 | relative = Path(name) |
| 109 | if relative.is_absolute() or ".." in relative.parts or not relative.parts: |
| 110 | raise ValueError(f"invalid build source path: {name}") |
| 111 | if "revision" in source_spec: |
| 112 | commit = source_spec["revision"] |
| 113 | if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{40}", commit): |
| 114 | raise ValueError(f"invalid build source revision: {manifest.parent.name}") |
| 115 | export_revision(source, commit, paths, source_context) |
| 116 | for name in paths: |
| 117 | if not (source_context / name).exists(): |
| 118 | raise ValueError(f"missing build source path at revision: {name}") |
| 119 | if dockerfile_name: |
| 120 | dockerfile = source_context / dockerfile_name |
| 121 | if not dockerfile.is_file(): |
| 122 | raise ValueError(f"invalid build source Dockerfile: {dockerfile_name}") |
| 123 | if dockerfile != source_context / "Dockerfile": |
| 124 | if (source_context / "Dockerfile").exists(): |
| 125 | raise ValueError("build source already includes Dockerfile") |
| 126 | if any(dockerfile.is_relative_to(source_context / name) for name in source_spec["include"]): |
| 127 | shutil.copy2(dockerfile, source_context / "Dockerfile") |
| 128 | else: |
| 129 | dockerfile.rename(source_context / "Dockerfile") |
| 130 | continue |
| 131 | for name in paths: |
| 132 | relative = Path(name) |
| 133 | origin = source / relative |
| 134 | if not origin.resolve(strict=True).is_relative_to(source): |
| 135 | raise ValueError(f"build source escapes context: {name}") |
| 136 | destination = source_context / ("Dockerfile" if name == dockerfile_name else name) |
| 137 | destination.parent.mkdir(parents=True, exist_ok=True) |
| 138 | if origin.is_dir(): |
| 139 | shutil.copytree(origin, destination, symlinks=True) |
| 140 | else: |
| 141 | shutil.copy2(origin, destination) |
| 142 | if "sources" in spec and spec.get("dockerfile"): |
| 143 | relative = Path(spec["dockerfile"]) |
| 144 | dockerfile = manifest.parent / relative |
| 145 | if (relative.is_absolute() or ".." in relative.parts or not relative.parts |
| 146 | or not dockerfile.resolve(strict=True).is_relative_to(manifest.parent) |
| 147 | or not dockerfile.is_file() or (context / "Dockerfile").exists()): |
| 148 | raise ValueError(f"invalid build context Dockerfile: {relative}") |
| 149 | shutil.copy2(dockerfile, context / "Dockerfile") |
| 150 | if not (context / "Dockerfile").is_file(): |
| 151 | raise ValueError(f"missing build source Dockerfile: {manifest.parent.name}") |
| 152 | if spec.get("includeManifest"): |
| 153 | shutil.copy2(manifest, context / ".snowglobe-source.json") |
| 154 | digest = tree_digest(snapshot) |
| 155 | release = digest[:16] |
| 156 | remote_release = REMOTE / "releases" / release |
| 157 | details = {"id": release, "digest": digest, "version": 2} |
| 158 | if revision: |
| 159 | details["main"] = revision |
| 160 | (snapshot / ".snowglobe-release.json").write_text(json.dumps(details) + "\n") |
| 161 | if subprocess.run( |
| 162 | ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, f"test -d {remote_release}"], |
| 163 | check=False, |
| 164 | ).returncode == 0: |
| 165 | ssh(f"python3 {remote_release}/tools/release.py verify {release}") |
| 166 | if revision: |
| 167 | script = "import pathlib,sys; p=pathlib.Path(" + repr(str(remote_release / ".snowglobe-release.json")) + "); t=p.with_suffix('.pending'); t.write_text(sys.stdin.read()); t.chmod(0o444); t.replace(p)" |
| 168 | subprocess.run(["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, |
| 169 | "python3 -c " + shlex.quote(script)], input=json.dumps(details) + "\n", text=True, check=True) |
| 170 | return release |
| 171 | ssh(f"mkdir -p {remote_release}") |
| 172 | subprocess.run( |
| 173 | ["rsync", "-a", "-e", f"ssh -p {PORT} -o BatchMode=yes", |
| 174 | *(str(snapshot / path) for path in SOURCES), str(snapshot / ".snowglobe-release.json"), |
| 175 | f"{HOST}:{remote_release}/"], check=True, |
| 176 | ) |
| 177 | ssh(f"chmod -R a-w {remote_release}") |
| 178 | return release |
| 179 | |
| 180 | |
| 181 | def main(): |
| 182 | parser = argparse.ArgumentParser(description="Preview working changes and deploy main") |
| 183 | parser.add_argument("mode", choices=["stage", "publish", "prod", "rollback", "history", "backups", "data-restore", "bootstrap", "allocate", "destroy", "secrets"]) |
| 184 | parser.add_argument("--computer", help="Give a dashboard preview access to these comma-separated VM names") |
| 185 | parser.add_argument("target", nargs="?") |
| 186 | parser.add_argument("--env", action="append", default=[]) |
| 187 | parser.add_argument("--file", type=Path) |
| 188 | parser.add_argument("--key", action="append", default=[]) |
| 189 | parser.add_argument("--backup") |
| 190 | parser.add_argument("--discard-writes", action="store_true") |
| 191 | args = parser.parse_args() |
| 192 | if args.computer and (args.mode != "stage" or args.target != "dashboard" or not all(NAME.fullmatch(name) for name in args.computer.split(","))): |
| 193 | parser.error("--computer requires stage dashboard and VM names") |
| 194 | if args.mode in {"stage", "allocate", "destroy", "secrets", "data-restore"} and (not args.target or not NAME.fullmatch(args.target)): |
| 195 | parser.error(f"{args.mode} requires a service or stage ID") |
| 196 | if args.mode == "rollback" and args.target and not RELEASE_ID.fullmatch(args.target): |
| 197 | parser.error("rollback target must be a release ID") |
| 198 | if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target: |
| 199 | parser.error(f"{args.mode} takes no target") |
| 200 | if args.env and (args.mode != "stage" or args.target == "host"): |
| 201 | parser.error("--env requires a service or dashboard stage") |
| 202 | if bool(args.file) != (args.mode == "secrets"): |
| 203 | parser.error("--file is required only for secrets") |
| 204 | if args.key and args.mode != "secrets": |
| 205 | parser.error("--key is available only for secrets") |
| 206 | if len(set(args.key)) != len(args.key) or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) for key in args.key): |
| 207 | parser.error("--key values must be unique secret names") |
| 208 | if args.mode == "data-restore": |
| 209 | if not args.backup or not re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{6}", args.backup) or not args.discard_writes: |
| 210 | parser.error("data-restore requires --backup ID and --discard-writes") |
| 211 | elif args.backup or args.discard_writes: |
| 212 | parser.error("--backup and --discard-writes are available only for data-restore") |
| 213 | if args.mode == "secrets": |
| 214 | if str(args.file) == "-": |
| 215 | values = sys.stdin.read() |
| 216 | else: |
| 217 | if args.file.stat().st_mode & 0o077: |
| 218 | raise ValueError("secret file must be readable only by its owner") |
| 219 | values = args.file.read_text() |
| 220 | if not values.strip(): |
| 221 | raise ValueError("secret input is empty") |
| 222 | manager = f"python3 {REMOTE}/release.py" |
| 223 | if args.mode == "stage": |
| 224 | release = upload() |
| 225 | if args.target == "host": |
| 226 | run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}") |
| 227 | return |
| 228 | overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env) |
| 229 | if args.computer: |
| 230 | overrides += " --computer " + shlex.quote(args.computer) |
| 231 | run_logged("stage", args.target, f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py stage {args.target}{overrides}") |
| 232 | elif args.mode == "secrets": |
| 233 | release = upload() |
| 234 | keys = "".join(f" --key {shlex.quote(key)}" for key in args.key) |
| 235 | subprocess.run( |
| 236 | ["ssh", "-p", PORT, "-o", "BatchMode=yes", HOST, |
| 237 | f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py secrets {args.target}{keys}"], |
| 238 | input=values, text=True, check=True, |
| 239 | ) |
| 240 | elif args.mode == "bootstrap": |
| 241 | release = upload(main=True) |
| 242 | sync_manager(release) |
| 243 | ssh(f"{manager} publish {release}") |
| 244 | ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py pool") |
| 245 | ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py bootstrap") |
| 246 | ssh(f"python3 {REMOTE}/releases/{release}/tools/release.py bootstrap {release}") |
| 247 | elif args.mode == "allocate": |
| 248 | release = upload() |
| 249 | ssh(f"python3 {REMOTE}/releases/{release}/tools/snowglobe.py allocate {args.target}") |
| 250 | elif args.mode in {"publish", "prod"}: |
| 251 | release = upload(main=True) |
| 252 | sync_manager(release) |
| 253 | ssh(f"{manager} publish {release}") |
| 254 | if args.mode == "prod": |
| 255 | run_logged("prod", release, f"{manager} deploy {release}") |
| 256 | elif args.mode == "rollback": |
| 257 | run_logged("rollback", args.target, f"{manager} rollback{(' ' + args.target) if args.target else ''}") |
| 258 | elif args.mode == "history": |
| 259 | ssh(f"{manager} history") |
| 260 | elif args.mode == "backups": |
| 261 | release = upload() |
| 262 | ssh(f"python3 {REMOTE}/releases/{release}/tools/data.py list") |
| 263 | elif args.mode == "data-restore": |
| 264 | release = upload() |
| 265 | run_logged("data-restore", args.target, f"python3 {REMOTE}/releases/{release}/tools/data.py restore {args.backup} {args.target} --discard-writes") |
| 266 | else: |
| 267 | release = upload() |
| 268 | run_logged("destroy", args.target, f"python3 {REMOTE}/releases/{release}/tools/release.py stage-destroy {args.target}") |
| 269 | |
| 270 | |
| 271 | if __name__ == "__main__": |
| 272 | try: |
| 273 | main() |
| 274 | except (OSError, ValueError, subprocess.CalledProcessError) as error: |
| 275 | print(f"Home server deployment stopped: {error}", file=sys.stderr) |
| 276 | sys.exit(error.returncode if isinstance(error, subprocess.CalledProcessError) else 1) |