| author | |
| committer | |
| log | d250b064f061df930c6febd22702fcb9250c9a2b |
| tree | 6fa0ec8e6e10d27788a7944a6178c25af982cacc |
| parent | 69ba225c6c6854475eff976b11d59f97cea15983 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Services set snapshots = false for disposable managed data; provisioning
records it as the snowglobe:snapshots dataset property and backups skip those
datasets. Local AI opts out for its context cache.
Assisted-by: claude-opus-5.56 files changed, 115 insertions(+), 1 deletions(-)
config/Service.pkl+3| ... | @@ -111,6 +111,8 @@ enabled: Boolean = true | ... | @@ -111,6 +111,8 @@ enabled: Boolean = true |
| 111 | rollout: String = "simple" | 111 | rollout: String = "simple" |
| 112 | /// Fresh previews create empty storage and new secrets instead of forking production. | 112 | /// Fresh previews create empty storage and new secrets instead of forking production. |
| 113 | stageIsolation: "clone"|"fresh" = "clone" | 113 | stageIsolation: "clone"|"fresh" = "clone" |
| 114 | /// Include the managed dataset in deployment backup snapshots. | ||
| 115 | snapshots: Boolean = true | ||
| 114 | /// Time allowed for a new allocation to pass its service checks. | 116 | /// Time allowed for a new allocation to pass its service checks. |
| 115 | healthyDeadline: String? | 117 | healthyDeadline: String? |
| 116 | /// Delay before persistent health-check failures may restart a running allocation. | 118 | /// Delay before persistent health-check failures may restart a running allocation. |
| ... | @@ -165,6 +167,7 @@ output { | ... | @@ -165,6 +167,7 @@ output { |
| 165 | access = module.access | 167 | access = module.access |
| 166 | rollout = module.rollout | 168 | rollout = module.rollout |
| 167 | stageIsolation = module.stageIsolation | 169 | stageIsolation = module.stageIsolation |
| 170 | snapshots = module.snapshots | ||
| 168 | healthyDeadline = vmStartupGrace ?? module.healthyDeadline | 171 | healthyDeadline = vmStartupGrace ?? module.healthyDeadline |
| 169 | healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace | 172 | healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace |
| 170 | hostRoot = "\(site.root)/prod/\(module.id)" | 173 | hostRoot = "\(site.root)/prod/\(module.id)" |
service/local-ai/service.pkl+1| ... | @@ -12,6 +12,7 @@ launcher = false | ... | @@ -12,6 +12,7 @@ launcher = false |
| 12 | access = "admin" | 12 | access = "admin" |
| 13 | enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true" | 13 | enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true" |
| 14 | stageIsolation = "fresh" | 14 | stageIsolation = "fresh" |
| 15 | snapshots = false | ||
| 15 | prepare = "prepare.py" | 16 | prepare = "prepare.py" |
| 16 | healthyDeadline = "30m" | 17 | healthyDeadline = "30m" |
| 17 | healthRestartGrace = "30m" | 18 | healthRestartGrace = "30m" |
tools/data-restore.md+5| ... | @@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the | ... | @@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the |
| 7 | backup is published. A failed deployment can leave a completed backup without | 7 | backup is published. A failed deployment can leave a completed backup without |
| 8 | a successful release-history entry. | 8 | a successful release-history entry. |
| 9 | 9 | ||
| 10 | Services can set `snapshots = false` in Pkl for disposable managed data, as Local | ||
| 11 | AI does for its context cache. Pool provisioning writes the policy to the | ||
| 12 | dataset's `snowglobe:snapshots` property. Database dumps and their shared | ||
| 13 | PostgreSQL snapshot remain included. | ||
| 14 | |||
| 10 | `python3 tools/deploy.py backups` lists backup IDs and services. Each backup's | 15 | `python3 tools/deploy.py backups` lists backup IDs and services. Each backup's |
| 11 | private manifest records the service snapshots, database dumps and checksums, | 16 | private manifest records the service snapshots, database dumps and checksums, |
| 12 | and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret | 17 | and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret |
tools/data-test.py created+103| ... | @@ -0,0 +1,103 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | from contextlib import ExitStack, nullcontext | ||
| 3 | import json | ||
| 4 | import os | ||
| 5 | from pathlib import Path | ||
| 6 | import subprocess | ||
| 7 | import tempfile | ||
| 8 | import unittest | ||
| 9 | from unittest.mock import patch | ||
| 10 | |||
| 11 | import data | ||
| 12 | import snowglobe | ||
| 13 | |||
| 14 | |||
| 15 | class SnapshotPolicy(unittest.TestCase): | ||
| 16 | def test_pkl_policy_reaches_existing_and_new_datasets(self): | ||
| 17 | with tempfile.TemporaryDirectory(prefix="snowglobe-snapshot-policy-") as temporary: | ||
| 18 | root = Path(temporary) | ||
| 19 | fixture = root / "service.pkl" | ||
| 20 | for declaration, expected in (("", True), ("snapshots = false", False)): | ||
| 21 | fixture.write_text(f'''amends {json.dumps((snowglobe.REPO / "config/Service.pkl").as_uri())} | ||
| 22 | name = "Fixture" | ||
| 23 | container {{ image = "fixture"; volumes {{ ["/cache"] {{}} }} }} | ||
| 24 | {declaration} | ||
| 25 | ''') | ||
| 26 | rendered = json.loads(subprocess.check_output([ | ||
| 27 | os.environ.get("PKL", "pkl"), "eval", "-p", "serviceId=fixture", "-p", "uid=3132", | ||
| 28 | "-p", f"root={root}", "-p", "pool=fixture", str(fixture), | ||
| 29 | ], text=True)) | ||
| 30 | self.assertIs(rendered["snapshots"], expected) | ||
| 31 | for exists in (True, False): | ||
| 32 | with self.subTest(declaration=declaration, exists=exists), patch.object( | ||
| 33 | snowglobe, "check_pool", | ||
| 34 | ), patch.object(snowglobe, "STATE", root), patch.object( | ||
| 35 | snowglobe.subprocess, "run", return_value=subprocess.CompletedProcess([], 0 if exists else 1), | ||
| 36 | ), patch.object(snowglobe, "command") as command: | ||
| 37 | snowglobe.ensure_pool([rendered]) | ||
| 38 | self.assertEqual(command.call_args.args, ( | ||
| 39 | "zfs", "set", f"snowglobe:snapshots={str(expected).lower()}", "fixture/prod/fixture", | ||
| 40 | )) | ||
| 41 | self.assertEqual(command.call_count, 1 if exists else 3) | ||
| 42 | rendered["hasManagedVolumes"] = False | ||
| 43 | with patch.object(snowglobe, "check_pool"), patch.object( | ||
| 44 | snowglobe, "STATE", root, | ||
| 45 | ), patch.object(snowglobe, "command") as command: | ||
| 46 | snowglobe.ensure_pool([rendered]) | ||
| 47 | command.assert_not_called() | ||
| 48 | |||
| 49 | def test_backup_excludes_cache_and_preserves_database_recovery(self): | ||
| 50 | for policy, database in (("-", False), ("true", False), ("false", False), ("false", True)): | ||
| 51 | with self.subTest(policy=policy, database=database), tempfile.TemporaryDirectory( | ||
| 52 | prefix="snowglobe-backup-policy-", | ||
| 53 | ) as temporary, ExitStack() as stack: | ||
| 54 | root = Path(temporary) | ||
| 55 | (root / "managed-jobs.json").write_text('["cache", "files", "postgres"]') | ||
| 56 | snapshots = [] | ||
| 57 | |||
| 58 | def run(*args, **kwargs): | ||
| 59 | if args[:2] == ("zfs", "get"): | ||
| 60 | value = "true" if args[-1].endswith("/files") else policy | ||
| 61 | return subprocess.CompletedProcess(args, 0, value + "\n") | ||
| 62 | if args[:2] == ("zfs", "snapshot"): | ||
| 63 | snapshots.extend(args[2:]) | ||
| 64 | elif args[:4] == ("nomad", "operator", "snapshot", "save"): | ||
| 65 | Path(args[4]).write_bytes(b"synthetic Nomad state") | ||
| 66 | elif args[-2:] == ("-c", "SELECT extname FROM pg_extension WHERE extname <> 'plpgsql' ORDER BY extname"): | ||
| 67 | return subprocess.CompletedProcess(args, 0, "") | ||
| 68 | else: | ||
| 69 | self.fail(f"Unexpected command: {args}") | ||
| 70 | return subprocess.CompletedProcess(args, 0) | ||
| 71 | |||
| 72 | for name, value in { | ||
| 73 | "STATE": root, "BACKUPS": root / "backups", "run": run, | ||
| 74 | "current_release": lambda: root / ("a" * 16), | ||
| 75 | "dataset_for": lambda name: "fixture/prod/" + name, | ||
| 76 | "database_inputs": lambda name: {"db": ("fixture", "svc_fixture")} if database and name == "cache" else {}, | ||
| 77 | "cloned_postgres": lambda *args: nullcontext("fixture-postgres"), | ||
| 78 | "database_dump": lambda container, name, destination, extensions: destination.write_bytes(b"synthetic database"), | ||
| 79 | }.items(): | ||
| 80 | stack.enter_context(patch.object(data, name, value)) | ||
| 81 | backup_id = data.backup("a" * 16, "b" * 16) | ||
| 82 | manifest = json.loads((data.BACKUPS / backup_id / "manifest.json").read_text()) | ||
| 83 | expected = {"fixture/prod/files"} | ||
| 84 | if policy != "false": | ||
| 85 | expected |= {"fixture/prod/cache", "fixture/prod/postgres"} | ||
| 86 | elif database: | ||
| 87 | expected.add("fixture/prod/postgres") | ||
| 88 | self.assertEqual({item.split("@", 1)[0] for item in snapshots}, expected) | ||
| 89 | services = {"files"} | ||
| 90 | if policy != "false": | ||
| 91 | services |= {"cache", "postgres"} | ||
| 92 | elif database: | ||
| 93 | services.add("cache") | ||
| 94 | self.assertEqual(set(manifest["services"]), services) | ||
| 95 | if database: | ||
| 96 | entry = manifest["services"]["cache"] | ||
| 97 | self.assertNotIn("dataset", entry) | ||
| 98 | dump = entry["databases"]["db"] | ||
| 99 | self.assertEqual(dump["sha256"], data.checksum(data.BACKUPS / backup_id / dump["dump"])) | ||
| 100 | |||
| 101 | |||
| 102 | if __name__ == "__main__": | ||
| 103 | unittest.main() | ||
tools/data.py+2-1| ... | @@ -181,7 +181,8 @@ def backup(from_release, to_release): | ... | @@ -181,7 +181,8 @@ def backup(from_release, to_release): |
| 181 | for service in sorted(names): | 181 | for service in sorted(names): |
| 182 | entry = {} | 182 | entry = {} |
| 183 | dataset = dataset_for(service) | 183 | dataset = dataset_for(service) |
| 184 | if dataset: | 184 | if dataset and run("zfs", "get", "-H", "-o", "value", "snowglobe:snapshots", dataset, |
| 185 | capture=True, text=True).stdout.strip() != "false": | ||
| 185 | datasets.add(dataset) | 186 | datasets.add(dataset) |
| 186 | entry["dataset"] = dataset | 187 | entry["dataset"] = dataset |
| 187 | entry["snapshot"] = snapshot | 188 | entry["snapshot"] = snapshot |
tools/snowglobe.py+1| ... | @@ -929,6 +929,7 @@ def ensure_pool(data): | ... | @@ -929,6 +929,7 @@ def ensure_pool(data): |
| 929 | raise ValueError(f"refusing to mount over {parent}") | 929 | raise ValueError(f"refusing to mount over {parent}") |
| 930 | command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod") | 930 | command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod") |
| 931 | command("zfs", "create", "-o", "mountpoint=" + str(root), dataset) | 931 | command("zfs", "create", "-o", "mountpoint=" + str(root), dataset) |
| 932 | command("zfs", "set", "snowglobe:snapshots=" + str(item["snapshots"]).lower(), dataset) | ||
| 932 | 933 | ||
| 933 | 934 | ||
| 934 | def main(): | 935 | def main(): |