authoravatarclover caruso<git@paperclover.net>2026-10-23 18:18:08-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logd250b064f061df930c6febd22702fcb9250c9a2b
tree6fa0ec8e6e10d27788a7944a6178c25af982cacc
parent69ba225c6c6854475eff976b11d59f97cea15983
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Let services opt their datasets out of backup snapshots

Services set snapshots = false for disposable managed data; provisioning records it as the snowglobe:snapshots dataset property and backups skip those datasets. Local AI opts out for its context cache. Assisted-by: claude-opus-5.5

6 files changed, 115 insertions(+), 1 deletions(-)

config/Service.pkl+3
......@@ -111,6 +111,8 @@ enabled: Boolean = true
111111rollout: String = "simple"
112112/// Fresh previews create empty storage and new secrets instead of forking production.
113113stageIsolation: "clone"|"fresh" = "clone"
114/// Include the managed dataset in deployment backup snapshots.
115snapshots: Boolean = true
114116/// Time allowed for a new allocation to pass its service checks.
115117healthyDeadline: String?
116118/// Delay before persistent health-check failures may restart a running allocation.
......@@ -165,6 +167,7 @@ output {
165167 access = module.access
166168 rollout = module.rollout
167169 stageIsolation = module.stageIsolation
170 snapshots = module.snapshots
168171 healthyDeadline = vmStartupGrace ?? module.healthyDeadline
169172 healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace
170173 hostRoot = "\(site.root)/prod/\(module.id)"
service/local-ai/service.pkl+1
......@@ -12,6 +12,7 @@ launcher = false
1212access = "admin"
1313enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true"
1414stageIsolation = "fresh"
15snapshots = false
1516prepare = "prepare.py"
1617healthyDeadline = "30m"
1718healthRestartGrace = "30m"
tools/data-restore.md+5
......@@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the
77backup is published. A failed deployment can leave a completed backup without
88a successful release-history entry.
99
10Services can set `snapshots = false` in Pkl for disposable managed data, as Local
11AI does for its context cache. Pool provisioning writes the policy to the
12dataset's `snowglobe:snapshots` property. Database dumps and their shared
13PostgreSQL snapshot remain included.
14
1015`python3 tools/deploy.py backups` lists backup IDs and services. Each backup's
1116private manifest records the service snapshots, database dumps and checksums,
1217and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret
tools/data-test.py created+103
......@@ -0,0 +1,103 @@
1#!/usr/bin/env python3
2from contextlib import ExitStack, nullcontext
3import json
4import os
5from pathlib import Path
6import subprocess
7import tempfile
8import unittest
9from unittest.mock import patch
10
11import data
12import snowglobe
13
14
15class SnapshotPolicy(unittest.TestCase):
16 def test_pkl_policy_reaches_existing_and_new_datasets(self):
17 with tempfile.TemporaryDirectory(prefix="snowglobe-snapshot-policy-") as temporary:
18 root = Path(temporary)
19 fixture = root / "service.pkl"
20 for declaration, expected in (("", True), ("snapshots = false", False)):
21 fixture.write_text(f'''amends {json.dumps((snowglobe.REPO / "config/Service.pkl").as_uri())}
22name = "Fixture"
23container {{ image = "fixture"; volumes {{ ["/cache"] {{}} }} }}
24{declaration}
25''')
26 rendered = json.loads(subprocess.check_output([
27 os.environ.get("PKL", "pkl"), "eval", "-p", "serviceId=fixture", "-p", "uid=3132",
28 "-p", f"root={root}", "-p", "pool=fixture", str(fixture),
29 ], text=True))
30 self.assertIs(rendered["snapshots"], expected)
31 for exists in (True, False):
32 with self.subTest(declaration=declaration, exists=exists), patch.object(
33 snowglobe, "check_pool",
34 ), patch.object(snowglobe, "STATE", root), patch.object(
35 snowglobe.subprocess, "run", return_value=subprocess.CompletedProcess([], 0 if exists else 1),
36 ), patch.object(snowglobe, "command") as command:
37 snowglobe.ensure_pool([rendered])
38 self.assertEqual(command.call_args.args, (
39 "zfs", "set", f"snowglobe:snapshots={str(expected).lower()}", "fixture/prod/fixture",
40 ))
41 self.assertEqual(command.call_count, 1 if exists else 3)
42 rendered["hasManagedVolumes"] = False
43 with patch.object(snowglobe, "check_pool"), patch.object(
44 snowglobe, "STATE", root,
45 ), patch.object(snowglobe, "command") as command:
46 snowglobe.ensure_pool([rendered])
47 command.assert_not_called()
48
49 def test_backup_excludes_cache_and_preserves_database_recovery(self):
50 for policy, database in (("-", False), ("true", False), ("false", False), ("false", True)):
51 with self.subTest(policy=policy, database=database), tempfile.TemporaryDirectory(
52 prefix="snowglobe-backup-policy-",
53 ) as temporary, ExitStack() as stack:
54 root = Path(temporary)
55 (root / "managed-jobs.json").write_text('["cache", "files", "postgres"]')
56 snapshots = []
57
58 def run(*args, **kwargs):
59 if args[:2] == ("zfs", "get"):
60 value = "true" if args[-1].endswith("/files") else policy
61 return subprocess.CompletedProcess(args, 0, value + "\n")
62 if args[:2] == ("zfs", "snapshot"):
63 snapshots.extend(args[2:])
64 elif args[:4] == ("nomad", "operator", "snapshot", "save"):
65 Path(args[4]).write_bytes(b"synthetic Nomad state")
66 elif args[-2:] == ("-c", "SELECT extname FROM pg_extension WHERE extname <> 'plpgsql' ORDER BY extname"):
67 return subprocess.CompletedProcess(args, 0, "")
68 else:
69 self.fail(f"Unexpected command: {args}")
70 return subprocess.CompletedProcess(args, 0)
71
72 for name, value in {
73 "STATE": root, "BACKUPS": root / "backups", "run": run,
74 "current_release": lambda: root / ("a" * 16),
75 "dataset_for": lambda name: "fixture/prod/" + name,
76 "database_inputs": lambda name: {"db": ("fixture", "svc_fixture")} if database and name == "cache" else {},
77 "cloned_postgres": lambda *args: nullcontext("fixture-postgres"),
78 "database_dump": lambda container, name, destination, extensions: destination.write_bytes(b"synthetic database"),
79 }.items():
80 stack.enter_context(patch.object(data, name, value))
81 backup_id = data.backup("a" * 16, "b" * 16)
82 manifest = json.loads((data.BACKUPS / backup_id / "manifest.json").read_text())
83 expected = {"fixture/prod/files"}
84 if policy != "false":
85 expected |= {"fixture/prod/cache", "fixture/prod/postgres"}
86 elif database:
87 expected.add("fixture/prod/postgres")
88 self.assertEqual({item.split("@", 1)[0] for item in snapshots}, expected)
89 services = {"files"}
90 if policy != "false":
91 services |= {"cache", "postgres"}
92 elif database:
93 services.add("cache")
94 self.assertEqual(set(manifest["services"]), services)
95 if database:
96 entry = manifest["services"]["cache"]
97 self.assertNotIn("dataset", entry)
98 dump = entry["databases"]["db"]
99 self.assertEqual(dump["sha256"], data.checksum(data.BACKUPS / backup_id / dump["dump"]))
100
101
102if __name__ == "__main__":
103 unittest.main()
tools/data.py+2-1
......@@ -181,7 +181,8 @@ def backup(from_release, to_release):
181181 for service in sorted(names):
182182 entry = {}
183183 dataset = dataset_for(service)
184 if dataset:
184 if dataset and run("zfs", "get", "-H", "-o", "value", "snowglobe:snapshots", dataset,
185 capture=True, text=True).stdout.strip() != "false":
185186 datasets.add(dataset)
186187 entry["dataset"] = dataset
187188 entry["snapshot"] = snapshot
tools/snowglobe.py+1
......@@ -929,6 +929,7 @@ def ensure_pool(data):
929929 raise ValueError(f"refusing to mount over {parent}")
930930 command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod")
931931 command("zfs", "create", "-o", "mountpoint=" + str(root), dataset)
932 command("zfs", "set", "snowglobe:snapshots=" + str(item["snapshots"]).lower(), dataset)
932933
933934
934935def main():