authoravatarclover caruso<git@paperclover.net>2026-10-16 02:35:44-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log83bb016ae253ba9ec424a98efd19351f9531e6b4
tree6acc3b942bb5142caea771895a0ecd463b9dac40
parent553ce6ae636c10536639d8c15893e182c9f0c6ee
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Pin external service builds to reviewed source revisions

Export pinned build contexts from immutable Jujutsu revisions and retain working-source exports for unpinned manifests. Pin Clover DB to the exact healthy deployed source so a dashboard rollout cannot import unrelated database migrations. Assisted-by: gpt-6.1-sol

4 files changed, 190 insertions(+), 23 deletions(-)

service/clover-source-of-truth/build-source.json+1
......@@ -1,5 +1,6 @@
11{
22 "source": "../sitegen",
3 "revision": "13f64d71038f5e72ab34dc1f55805e2fee05f368",
34 "dockerfile": "src/source-of-truth.dockerfile",
45 "include": [
56 "package.json",
tools/copyparty.md+12
......@@ -86,3 +86,15 @@ outside its scroller. Anchoring that hidden text and bounding the list during
8686refresh errors kept the document at 720 × 320 in a 720 × 320 viewport, with
8787only the table scrolling. Its headers and last-row keyboard actions stayed
8888reachable; the hidden live status remained in the accessibility tree.
89
90Production account creation completed in 0.60 seconds, rotation in 0.47
91seconds, and deletion in 0.15 seconds under the effective host restrictions.
92The disposable account was removed; shares and grants remained unchanged.
93
94The first rollout, `a8982234e99ab23c`, also exported newer sibling Clover DB
95sources. That image failed before database startup because its nonroot user
96could not create its launcher cache. Clover DB was restored through the
97previous immutable release's service deployment. Its newer source also carries
98database migrations outside this change's scope. The build manifest now pins
99sitegen revision `13f64d71038f5e72ab34dc1f55805e2fee05f368`; all 183 exported
100files, file types, and executable bits match the previous healthy context.
tools/deploy-test.py+127
......@@ -1,4 +1,5 @@
11#!/usr/bin/env python3
2from contextlib import ExitStack
23import json
34import os
45from pathlib import Path
......@@ -75,5 +76,131 @@ class MainExport(unittest.TestCase):
7576 deploy.upload(main=True)
7677
7778
79class BuildSourceExport(unittest.TestCase):
80 def setUp(self):
81 stack = ExitStack()
82 self.addCleanup(stack.close)
83 root = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="studio-build-source-"))).resolve()
84 self.repo = root / "repo"
85 self.source = root / "source"
86 self.remote = root / "remote"
87 for repo in (self.repo, self.source):
88 subprocess.run(["jj", "git", "init", str(repo)], check=True, capture_output=True)
89 for name in release.SOURCES:
90 path = self.repo / name
91 if "." in name:
92 path.write_text("fixture")
93 else:
94 path.mkdir()
95 (path / "fixture").write_text("fixture")
96 (self.repo / "config/excluded-services.json").write_text("[]\n")
97 (self.source / "lib").mkdir()
98 (self.source / "lib/committed").write_text("committed library")
99 (self.source / "src").mkdir()
100 (self.source / "src/app.dockerfile").write_text("FROM committed\n")
101 (self.source / "run.sh").write_text("#!/bin/sh\necho committed\n")
102 (self.source / "run.sh").chmod(0o755)
103 subprocess.run(["jj", "describe", "-m", "Committed build source"], cwd=self.source, check=True, capture_output=True)
104 self.commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout
105 subprocess.run(["jj", "new"], cwd=self.source, check=True, capture_output=True)
106 (self.source / "run.sh").write_text("unfinished runner")
107 (self.source / "run.sh").chmod(0o644)
108 (self.source / "src/app.dockerfile").write_text("FROM unfinished\n")
109 (self.source / "lib/committed").unlink()
110 (self.source / "lib/new").write_text("uncommitted library")
111 self.manifest = self.repo / "service/app/build-source.json"
112 self.manifest.parent.mkdir()
113 self.spec = {"source": "../source", "revision": self.commit,
114 "dockerfile": "src/app.dockerfile", "include": ["run.sh", "lib"]}
115 self.manifest.write_text(json.dumps(self.spec))
116 for args in [["describe", "-m", "Pinned build source"], ["bookmark", "set", "main"]]:
117 subprocess.run(["jj", *args], cwd=self.repo, check=True, capture_output=True)
118 original = subprocess.run
119
120 def transport(argv, **kwargs):
121 if argv[0] == "ssh":
122 exists = argv[-1].startswith("test -d ") and Path(argv[-1][8:]).is_dir()
123 return subprocess.CompletedProcess(argv, 0 if exists else 1)
124 if argv[0] == "rsync" and "-e" in argv:
125 destination = Path(argv[-1].partition(":")[2])
126 destination.mkdir(parents=True)
127 for name in argv[4:-1]:
128 source = Path(name)
129 if source.is_dir():
130 shutil.copytree(source, destination / source.name)
131 else:
132 shutil.copy2(source, destination / source.name)
133 return subprocess.CompletedProcess(argv, 0)
134 return original(argv, **kwargs)
135
136 stack.enter_context(patch.object(deploy, "REPO", self.repo))
137 stack.enter_context(patch.object(deploy, "REMOTE", self.remote))
138 stack.enter_context(patch.object(deploy, "ssh"))
139 stack.enter_context(patch.object(deploy.subprocess, "run", side_effect=transport))
140
141 def test_pinned_source_ignores_working_files_for_main_and_stage(self):
142 for main in (True, False):
143 with self.subTest(main=main):
144 context = self.remote / "releases" / deploy.upload(main=main) / "service/app/build"
145 self.assertEqual((context / "run.sh").read_text(), "#!/bin/sh\necho committed\n")
146 self.assertTrue(os.access(context / "run.sh", os.X_OK))
147 self.assertEqual((context / "lib/committed").read_text(), "committed library")
148 self.assertFalse((context / "lib/new").exists())
149 self.assertEqual((context / "Dockerfile").read_text(), "FROM committed\n")
150 self.assertFalse((context / "src/app.dockerfile").exists())
151
152 def test_unpinned_source_uses_working_files(self):
153 del self.spec["revision"]
154 self.manifest.write_text(json.dumps(self.spec))
155 context = self.remote / "releases" / deploy.upload() / "service/app/build"
156 self.assertEqual((context / "run.sh").read_text(), "unfinished runner")
157 self.assertFalse(os.access(context / "run.sh", os.X_OK))
158 self.assertFalse((context / "lib/committed").exists())
159 self.assertEqual((context / "lib/new").read_text(), "uncommitted library")
160 self.assertEqual((context / "Dockerfile").read_text(), "FROM unfinished\n")
161
162 def test_dockerfile_also_selected_by_include_is_retained(self):
163 self.manifest.write_text(json.dumps({**self.spec, "include": ["src"]}))
164 context = self.remote / "releases" / deploy.upload() / "service/app/build"
165 self.assertEqual((context / "src/app.dockerfile").read_text(), "FROM committed\n")
166 self.assertEqual((context / "Dockerfile").read_text(), "FROM committed\n")
167
168 def test_invalid_revision_and_paths_are_rejected(self):
169 for revision in ("main", "f" * 39, "g" * 40, None):
170 with self.subTest(revision=revision):
171 self.manifest.write_text(json.dumps({**self.spec, "revision": revision}))
172 with self.assertRaisesRegex(ValueError, "invalid build source revision"):
173 deploy.upload()
174 for name in ("../outside", "/outside", "", "missing", "glob:*"):
175 with self.subTest(path=name):
176 self.manifest.write_text(json.dumps({**self.spec, "include": [name]}))
177 with self.assertRaisesRegex(ValueError, "build source path"):
178 deploy.upload()
179 self.manifest.write_text(json.dumps({**self.spec, "source": "../.."}))
180 with self.assertRaisesRegex(ValueError, "invalid build source"):
181 deploy.upload()
182
183 def test_selected_symlink_is_rejected(self):
184 (self.source / "link").symlink_to("run.sh")
185 subprocess.run(["jj", "describe", "-m", "Source with symlink"], cwd=self.source, check=True, capture_output=True)
186 commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout
187 self.manifest.write_text(json.dumps({**self.spec, "revision": commit, "include": ["link"]}))
188 with self.assertRaisesRegex(ValueError, "unsupported revision file: link"):
189 deploy.upload()
190
191 def test_selected_conflict_is_rejected(self):
192 commits = []
193 for content in ("left", "right"):
194 subprocess.run(["jj", "new", self.commit], cwd=self.source, check=True, capture_output=True)
195 (self.source / "run.sh").write_text(content)
196 subprocess.run(["jj", "describe", "-m", content], cwd=self.source, check=True, capture_output=True)
197 commits.append(subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout)
198 subprocess.run(["jj", "new", *commits], cwd=self.source, check=True, capture_output=True)
199 commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout
200 self.manifest.write_text(json.dumps({**self.spec, "revision": commit}))
201 with self.assertRaisesRegex(ValueError, "unsupported revision file: run.sh"):
202 deploy.upload()
203
204
78205if __name__ == "__main__":
79206 unittest.main()
tools/deploy.py+50-23
......@@ -43,6 +43,32 @@ def sync_manager(release):
4343 ssh(f"cp {source}/release.py {source}/data.py {REMOTE}/")
4444
4545
46def export_revision(source, commit, paths, destination, excluded=frozenset()):
47 entries = subprocess.run(
48 ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T",
49 '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"',
50 *("cwd:" + json.dumps(path) for path in paths)],
51 cwd=source, check=True, capture_output=True, text=True,
52 ).stdout.splitlines()
53 for entry in entries:
54 name, kind, executable = json.loads(entry)
55 relative = Path(name)
56 if relative.parts[:1] == ("service",) and (
57 set(relative.parts[1:]) & excluded or
58 relative.suffix == ".pkl" and relative.stem in excluded
59 ):
60 continue
61 if kind != "file" or relative.is_absolute() or ".." in relative.parts:
62 raise ValueError(f"unsupported revision file: {name}")
63 target = destination / relative
64 target.parent.mkdir(parents=True, exist_ok=True)
65 target.write_bytes(subprocess.run(
66 ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name],
67 cwd=source, check=True, capture_output=True,
68 ).stdout)
69 target.chmod(0o755 if executable else 0o644)
70
71
4672def upload(main=False):
4773 excluded = set() if main else excluded_services(REPO)
4874 with tempfile.TemporaryDirectory() as temporary:
......@@ -65,28 +91,7 @@ def upload(main=False):
6591 "config/excluded-services.json"],
6692 cwd=REPO, check=True, capture_output=True, text=True,
6793 ).stdout))
68 entries = subprocess.run(
69 ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T",
70 '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"',
71 *SOURCES], cwd=REPO, check=True, capture_output=True, text=True,
72 ).stdout.splitlines()
73 for entry in entries:
74 name, kind, executable = json.loads(entry)
75 relative = Path(name)
76 if relative.parts[:1] == ("service",) and (
77 set(relative.parts[1:]) & excluded or
78 relative.suffix == ".pkl" and relative.stem in excluded
79 ):
80 continue
81 if kind != "file" or relative.is_absolute() or ".." in relative.parts:
82 raise ValueError(f"unsupported main release file: {name}")
83 destination = snapshot / relative
84 destination.parent.mkdir(parents=True, exist_ok=True)
85 destination.write_bytes(subprocess.run(
86 ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name],
87 cwd=REPO, check=True, capture_output=True,
88 ).stdout)
89 destination.chmod(0o755 if executable else 0o644)
94 export_revision(REPO, commit, SOURCES, snapshot, excluded)
9095 else:
9196 subprocess.run(
9297 ["rsync", "-a", "--exclude=.DS_Store", "--exclude=__pycache__", "--exclude=*.pyc",
......@@ -108,10 +113,32 @@ def upload(main=False):
108113 if context.exists():
109114 raise ValueError(f"build context already exists: {manifest.parent.name}")
110115 context.mkdir()
111 for name in [*spec["include"], spec["dockerfile"]]:
116 paths = [*spec["include"], spec["dockerfile"]]
117 for name in paths:
112118 relative = Path(name)
113119 if relative.is_absolute() or ".." in relative.parts or not relative.parts:
114120 raise ValueError(f"invalid build source path: {name}")
121 if "revision" in spec:
122 commit = spec["revision"]
123 if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{40}", commit):
124 raise ValueError(f"invalid build source revision: {manifest.parent.name}")
125 export_revision(source, commit, paths, context)
126 for name in paths:
127 if not (context / name).exists():
128 raise ValueError(f"missing build source path at revision: {name}")
129 dockerfile = context / spec["dockerfile"]
130 if not dockerfile.is_file():
131 raise ValueError(f"invalid build source Dockerfile: {spec['dockerfile']}")
132 if dockerfile != context / "Dockerfile":
133 if (context / "Dockerfile").exists():
134 raise ValueError("build source already includes Dockerfile")
135 if any(dockerfile.is_relative_to(context / name) for name in spec["include"]):
136 shutil.copy2(dockerfile, context / "Dockerfile")
137 else:
138 dockerfile.rename(context / "Dockerfile")
139 continue
140 for name in paths:
141 relative = Path(name)
115142 origin = source / relative
116143 if not origin.resolve(strict=True).is_relative_to(source):
117144 raise ValueError(f"build source escapes context: {name}")