authoravatarclover caruso<git@paperclover.net>2026-10-17 12:43:21-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
loga4c36c6d8f5893c91e75def1bc06dad8917475bc
tree6ad7df4cddc8c9bb09a4fdff1a5ec7b27625c737
parenta061380e5790ff217adfe1d0ca9dd0426d126739
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Show Zig release navigation and preserve archive dates

Generate published documentation links and version navigation, authenticate development build dates, and preserve root and ancestor timestamps on publication. Simplify anonymous mirror browsing in Copyparty. Run reconciliation explicitly with a private recovery journal. Assisted-by: gpt-6.1-sol

13 files changed, 815 insertions(+), 31 deletions(-)

config/Service.pkl+1
...@@ -85,6 +85,7 @@ class Container {...@@ -85,6 +85,7 @@ class Container {
85 secretFiles: Mapping<String, String> = new {}85 secretFiles: Mapping<String, String> = new {}
86 args: Listing<String> = new {}86 args: Listing<String> = new {}
87 capAdd: Listing<String> = new {}87 capAdd: Listing<String> = new {}
88 readOnlyRoot: Boolean = false
88 devices: Listing<String> = new {}89 devices: Listing<String> = new {}
89 extraHosts: Listing<String> = new {}90 extraHosts: Listing<String> = new {}
90 hostNetwork: Boolean = false91 hostNetwork: Boolean = false
service/copyparty/copyparty.conf+1
...@@ -24,6 +24,7 @@...@@ -24,6 +24,7 @@
24 idp-logout: /auth/file/sign-out24 idp-logout: /auth/file/sign-out
25 idp-login-t: with snow globe25 idp-login-t: with snow globe
26 html-head: <link rel="stylesheet" href="/.static/copyparty.css">26 html-head: <link rel="stylesheet" href="/.static/copyparty.css">
27 js-browser: /.static/mirrors.js
2728
28[/]29[/]
29 /w30 /w
service/copyparty/static/copyparty.css+16
...@@ -7,3 +7,19 @@...@@ -7,3 +7,19 @@
7 --font-serif: "Name Sans", serif;7 --font-serif: "Name Sans", serif;
8 --font-mono: "Name Mono", monospace;8 --font-mono: "Name Mono", monospace;
9}9}
10.mirror-anonymous #files tr > :first-child,
11.mirror-anonymous #tree,
12.mirror-anonymous #entree,
13.mirror-anonymous #opa_auc,
14.mirror-anonymous #op_player,
15.mirror-anonymous #widget {
16 display: none !important;
17}
18.mirror-anonymous #path {
19 display: inline-block !important;
20}
21.mirror-anonymous #wrap {
22 margin-left: 1.5em !important;
23 padding-bottom: 1.5em;
24 overflow-x: auto;
25}
service/copyparty/static/mirrors.js created+12
...@@ -0,0 +1,12 @@
1(function () {
2 var render = treectl.gentab;
3 treectl.gentab = function (top, listing) {
4 var anonymous = (listing.acct === undefined ? acct : listing.acct) === "*" && /^\/mirrors(?:\/|$)/.test(top);
5 document.body.classList.toggle("mirror-anonymous", anonymous);
6 if (anonymous)
7 read_dsort("-ts");
8 return render.call(this, top, listing);
9 };
10 if (ls0 !== null)
11 treectl.gentab(get_evpath(), ls0);
12})();
service/zig-mirror/build/mirror.py+446-15
...@@ -1,8 +1,15 @@...@@ -1,8 +1,15 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import argparse2import argparse
3import base64
4import ctypes
5import ctypes.util
6import contextlib
7import datetime
3import email.utils8import email.utils
4import fcntl9import fcntl
10import functools
5import hashlib11import hashlib
12import html
6from html.parser import HTMLParser13from html.parser import HTMLParser
7import http.server14import http.server
8import json15import json
...@@ -10,6 +17,7 @@ import os...@@ -10,6 +17,7 @@ import os
10from pathlib import Path17from pathlib import Path
11import re18import re
12import shutil19import shutil
20import stat
13import subprocess21import subprocess
14import tempfile22import tempfile
15import threading23import threading
...@@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U'...@@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U'
24OFFICIAL = 'https://ziglang.org'32OFFICIAL = 'https://ziglang.org'
25NEKTRO = 'http://mirrors.nektro.net/zig/'33NEKTRO = 'http://mirrors.nektro.net/zig/'
26MAX_ARTIFACT = 2 * 1024**334MAX_ARTIFACT = 2 * 1024**3
35_index_cache = (float('-inf'), {})
36_index_lock = threading.Lock()
2737
2838
29class Unavailable(Exception):39class Unavailable(Exception):
30 pass40 pass
3141
3242
43@functools.lru_cache(maxsize=1)
44def comment_verifier():
45 name = ctypes.util.find_library('sodium')
46 if not name:
47 raise Unavailable('Minisign comment verification unavailable')
48 library = ctypes.CDLL(name)
49 if library.sodium_init() < 0:
50 raise Unavailable('Minisign comment verification unavailable')
51 verify = library.crypto_sign_verify_detached
52 verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p]
53 verify.restype = ctypes.c_int
54 return verify
55
56
57def metadata_open(path, final_flags=os.O_RDONLY):
58 path = path.absolute()
59 descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0))
60 try:
61 for index, part in enumerate(path.parts[1:]):
62 flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0)
63 if index < len(path.parts) - 2:
64 flags |= os.O_DIRECTORY
65 next_descriptor = os.open(part, flags, dir_fd=descriptor)
66 os.close(descriptor)
67 descriptor = next_descriptor
68 return descriptor
69 except BaseException:
70 os.close(descriptor)
71 raise
72
73
74def signed_time(signature):
75 with os.fdopen(metadata_open(signature), 'rb') as stream:
76 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
77 raise Unavailable('signature is not a regular file')
78 body = stream.read(4097)
79 lines = body.splitlines()
80 try:
81 if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '):
82 raise ValueError()
83 key = base64.b64decode(PUBLIC_KEY, validate=True)
84 sig = base64.b64decode(lines[1], validate=True)
85 global_sig = base64.b64decode(lines[3], validate=True)
86 comment = lines[2][len(b'trusted comment: '):]
87 if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]:
88 raise ValueError()
89 # Minisign authenticates the artifact signature followed by the exact trusted comment.
90 message = sig[10:] + comment
91 if comment_verifier()(global_sig, message, len(message), key[10:]) != 0:
92 raise ValueError()
93 match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment)
94 if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'):
95 raise ValueError()
96 return int(match[1]) * 1_000_000_000
97 except (ValueError, IndexError) as error:
98 raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error
99
100
101@functools.lru_cache(maxsize=128)
102def stable_time(version):
103 try:
104 entries = index()
105 if version not in entries:
106 entries = index(refresh=True)
107 date = datetime.date.fromisoformat(entries[version]['date'])
108 return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000
109 except (KeyError, ValueError) as error:
110 raise Unavailable('official release date unavailable: ' + version) from error
111
112
113def artifact_time(artifact):
114 version = filename(artifact.name)
115 if '-dev.' in version or version == '0.15.0':
116 return signed_time(Path(str(artifact) + '.minisig'))
117 return stable_time(version)
118
119
120def children(directory):
121 descriptor = metadata_open(directory)
122 try:
123 with os.scandir(descriptor) as entries:
124 return [(directory / entry.name, entry.stat(follow_symlinks=False))
125 for entry in entries if not entry.name.startswith('.')]
126 finally:
127 os.close(descriptor)
128
129
130def set_mtime(path, timestamp):
131 descriptor = metadata_open(path)
132 try:
133 before = os.fstat(descriptor)
134 if before.st_mtime_ns != timestamp:
135 os.utime(descriptor, ns=(before.st_atime_ns, timestamp))
136 finally:
137 os.close(descriptor)
138
139
140def file_state(value):
141 return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')}
142
143
144def version_readme(version, artifacts, development=False):
145 systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows',
146 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'}
147 source = []
148 targets = {}
149
150 def link(artifact):
151 name = artifact.name
152 url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/')
153 label = 'tar.xz' if name.endswith('.tar.xz') else 'zip'
154 result = f'[{label}]({url})'
155 signature = Path(str(artifact) + '.minisig')
156 if signature.is_file() and not signature.is_symlink():
157 result += f' <a href="{url}.minisig" style="background:none!important;border:none!important;font-size:13px;margin-left:-5px;margin-right:-3px" aria-label="Signature for {html.escape(name, quote=True)}">&#128272;</a>'
158 return result
159
160 for artifact in sorted(artifacts):
161 name = artifact.name
162 extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip'
163 target = name[len('zig'): -len('-' + version + extension)]
164 if target in ('', '-bootstrap'):
165 source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact))
166 continue
167 if target == '-win64':
168 architecture, system = 'x86_64', 'windows'
169 else:
170 first, second = target.removeprefix('-').split('-')
171 system, architecture = (first, second) if first in systems else (second, first)
172 targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact))
173 columns = list(systems)
174 columns += sorted({system for row in targets.values() for system in row} - set(columns))
175 heading = '<br>'.join(source) or 'Architecture'
176 lines = [f'### Zig {version}', '']
177 if not artifacts and development:
178 return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode()
179 if artifacts and '-dev.' not in version:
180 release = index().get(version, {})
181 for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]:
182 url = release.get(key)
183 if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url):
184 lines.append(f'- [{label}]({url})')
185 if lines[-1]:
186 lines.append('')
187 if development:
188 lines += [f'[Development builds](/mirrors/zig/{version}/dev/)', '']
189 elif '-dev.' in version:
190 lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', '']
191 if not artifacts:
192 return '\n'.join(lines).encode()
193 lines += ['<div role="region" aria-label="Zig downloads" tabindex="0" style="max-width:100%;overflow-x:auto;white-space:nowrap;word-break:normal;overflow-wrap:normal">', '',
194 '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |',
195 '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |']
196 priority = ('x86_64', 'aarch64', 'x86', 'arm')
197 for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)):
198 lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |')
199 lines += ['', '</div>']
200 if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts):
201 lines += ['', '<span style="font-style:italic;color:orange">Verify your build signatures.</span> <span style="opacity:0.7">The &#128272; links open the Minisign signatures.</span>', '']
202 else:
203 lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', '']
204 return '\n'.join(lines).encode()
205
206
33def filename(name):207def filename(name):
34 match = PATTERN.fullmatch(name)208 match = PATTERN.fullmatch(name)
35 if len(name) > 240 or not match:209 if len(name) > 240 or not match:
...@@ -72,9 +246,14 @@ def fetch(url, target, limit):...@@ -72,9 +246,14 @@ def fetch(url, target, limit):
72 raise Unavailable('upstream download failed') from error246 raise Unavailable('upstream download failed') from error
73247
74248
75def index():249def index(refresh=False):
76 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:250 global _index_cache
77 return json.load(response)251 with _index_lock:
252 if refresh or time.monotonic() - _index_cache[0] >= 3600:
253 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:
254 entries = json.load(response)
255 _index_cache = (time.monotonic(), entries)
256 return _index_cache[1]
78257
79258
80def digest(path):259def digest(path):
...@@ -87,7 +266,10 @@ def digest(path):...@@ -87,7 +266,10 @@ def digest(path):
87266
88class Mirror:267class Mirror:
89 def __init__(self, root):268 def __init__(self, root):
90 self.root = Path(root)269 self.root = Path(root).absolute()
270 if self.root.is_symlink():
271 raise ValueError('mirror root is a symlink')
272 self.root = self.root.resolve()
91 self.root.mkdir(parents=True, exist_ok=True)273 self.root.mkdir(parents=True, exist_ok=True)
92 self.slots = threading.BoundedSemaphore(4)274 self.slots = threading.BoundedSemaphore(4)
93 self.refresh_lock = threading.Lock()275 self.refresh_lock = threading.Lock()
...@@ -98,6 +280,50 @@ class Mirror:...@@ -98,6 +280,50 @@ class Mirror:
98 raise ValueError('artifact escapes mirror')280 raise ValueError('artifact escapes mirror')
99 return path281 return path
100282
283 def lock_directory(self):
284 locks = self.root / '.mirror-locks'
285 descriptor = metadata_open(self.root)
286 try:
287 fcntl.flock(descriptor, fcntl.LOCK_EX)
288 timestamp = os.fstat(descriptor).st_mtime_ns
289 try:
290 locks.mkdir()
291 except FileExistsError:
292 pass
293 else:
294 set_mtime(self.root, timestamp)
295 finally:
296 os.close(descriptor)
297 descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY)
298 os.close(descriptor)
299 return locks
300
301 @contextlib.contextmanager
302 def publishing(self):
303 locks = self.lock_directory()
304 with (locks / '.timestamps').open('a') as lock:
305 fcntl.flock(lock, fcntl.LOCK_EX)
306 yield
307
308 def parent_times(self, artifact):
309 def latest(directory):
310 values = []
311 for path, value in children(directory):
312 if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path:
313 values.append(value.st_mtime_ns)
314 elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'):
315 values.append(latest(path))
316 return max(values, default=0)
317
318 directory = artifact.parent
319 while True:
320 timestamp = latest(directory)
321 if timestamp:
322 set_mtime(directory, timestamp)
323 if directory == self.root:
324 break
325 directory = directory.parent
326
101 def ensure(self, name, source=None, checksum=None):327 def ensure(self, name, source=None, checksum=None):
102 target = self.path(name)328 target = self.path(name)
103 artifact_name = name.removesuffix('.minisig')329 artifact_name = name.removesuffix('.minisig')
...@@ -109,8 +335,7 @@ class Mirror:...@@ -109,8 +335,7 @@ class Mirror:
109 try:335 try:
110 if shutil.disk_usage(self.root).free < 16 * 1024**3:336 if shutil.disk_usage(self.root).free < 16 * 1024**3:
111 raise Unavailable('mirror free-space reserve reached')337 raise Unavailable('mirror free-space reserve reached')
112 locks = self.root / '.mirror-locks'338 locks = self.lock_directory()
113 locks.mkdir(exist_ok=True)
114 with (locks / artifact_name).open('a') as lock:339 with (locks / artifact_name).open('a') as lock:
115 fcntl.flock(lock, fcntl.LOCK_EX)340 fcntl.flock(lock, fcntl.LOCK_EX)
116 if target.is_file():341 if target.is_file():
...@@ -123,8 +348,7 @@ class Mirror:...@@ -123,8 +348,7 @@ class Mirror:
123 and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None)348 and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None)
124 if not checksum:349 if not checksum:
125 raise Unavailable('unsigned artifact has no official SHA-256')350 raise Unavailable('unsigned artifact has no official SHA-256')
126 artifact.parent.mkdir(parents=True, exist_ok=True)351 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
127 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:
128 pending = Path(temporary) / artifact_name352 pending = Path(temporary) / artifact_name
129 signature = Path(str(pending) + '.minisig')353 signature = Path(str(pending) + '.minisig')
130 url = source or upstream(artifact_name)354 url = source or upstream(artifact_name)
...@@ -137,21 +361,216 @@ class Mirror:...@@ -137,21 +361,216 @@ class Mirror:
137 capture_output=True)361 capture_output=True)
138 if result.returncode:362 if result.returncode:
139 raise Unavailable('artifact signature verification failed')363 raise Unavailable('artifact signature verification failed')
364 timestamp = artifact_time(pending)
140 os.chmod(pending, 0o664)365 os.chmod(pending, 0o664)
141 os.replace(pending, artifact)366 set_mtime(pending, timestamp)
142 if signature.exists():367 if signature.exists():
143 os.chmod(signature, 0o664)368 os.chmod(signature, 0o664)
144 os.replace(signature, Path(str(artifact) + '.minisig'))369 set_mtime(signature, timestamp)
370 with self.publishing():
371 self.path(name)
372 artifact.parent.mkdir(parents=True, exist_ok=True)
373 os.replace(pending, artifact)
374 if signature.exists():
375 os.replace(signature, Path(str(artifact) + '.minisig'))
376 try:
377 self.write_readmes(artifact.parent)
378 finally:
379 self.parent_times(artifact)
145 if name.endswith('.minisig') and not target.exists():380 if name.endswith('.minisig') and not target.exists():
146 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:381 with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary:
147 pending = Path(temporary) / name382 pending = Path(temporary) / name
148 fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096)383 fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096)
149 os.chmod(pending, 0o664)384 os.chmod(pending, 0o664)
150 os.replace(pending, target)385 set_mtime(pending, artifact.stat().st_mtime_ns)
386 with self.publishing():
387 os.replace(pending, target)
388 try:
389 self.write_readmes(artifact.parent)
390 finally:
391 self.parent_times(artifact)
151 return target392 return target
152 finally:393 finally:
153 self.slots.release()394 self.slots.release()
154395
396 def readme_change(self, directory, artifacts, content=None):
397 path = directory / 'preadme.md'
398 if content is None:
399 content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir())
400 before = None
401 previous = b''
402 if path.exists() or path.is_symlink():
403 descriptor = metadata_open(path)
404 with os.fdopen(descriptor, 'rb') as stream:
405 value = os.fstat(stream.fileno())
406 if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024:
407 raise Unavailable('version readme is not a small regular file')
408 before = file_state(value)
409 previous = stream.read(512 * 1024 + 1)
410 required = {artifact.name for artifact in artifacts}
411 required |= {artifact.name + '.minisig' for artifact in artifacts
412 if Path(str(artifact) + '.minisig').is_file()}
413 present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1])
414 for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())}
415 linked = {name for name in present if PATTERN.fullmatch(name)}
416 introduction = content.split(b'<div ', 1)[0]
417 if artifacts and linked == required and previous.startswith(introduction) and b'https://file.paperclover.net' not in previous:
418 return None
419 if previous == content:
420 return None
421 return {'path': str(path.relative_to(self.root)), 'before': before,
422 'previous': base64.b64encode(previous).decode(), 'content': content.decode()}
423
424 def artifacts(self, directory):
425 return [path for path, value in children(directory)
426 if stat.S_ISREG(value.st_mode) and PATTERN.fullmatch(path.name)
427 and not path.name.endswith('.minisig') and self.path(path.name) == path]
428
429 def directory_readme(self, directory):
430 if directory.name == 'dev':
431 content = f'### Zig {directory.parent.name} development builds\n'.encode()
432 return self.readme_change(directory, [], content)
433 artifacts = self.artifacts(directory)
434 if artifacts or (directory / 'dev').is_dir():
435 return self.readme_change(directory, artifacts)
436
437 def write_readmes(self, directory):
438 directories = [directory]
439 if directory.parent.name == 'dev':
440 directories += [directory.parent, directory.parent.parent]
441 for current in directories:
442 change = self.directory_readme(current)
443 if change:
444 self.apply_readme(change)
445
446 def apply_readme(self, change):
447 path = self.root / change['path']
448 if change['before'] is None:
449 parent = metadata_open(path.parent)
450 try:
451 descriptor = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o664, dir_fd=parent)
452 os.fchown(descriptor, -1, os.fstat(parent).st_gid)
453 finally:
454 os.close(parent)
455 before = os.fstat(descriptor)
456 else:
457 descriptor = metadata_open(path, os.O_WRONLY)
458 before = os.fstat(descriptor)
459 if any(file_state(before)[key] != value for key, value in change['before'].items() if key != 'atime_ns'):
460 os.close(descriptor)
461 raise Unavailable('version readme changed since plan')
462 try:
463 os.ftruncate(descriptor, 0)
464 with os.fdopen(os.dup(descriptor), 'wb') as output:
465 output.write(change['content'].encode())
466 output.flush()
467 os.fsync(output.fileno())
468 os.utime(descriptor, ns=(before.st_atime_ns, before.st_mtime_ns))
469 finally:
470 os.close(descriptor)
471
472 def metadata_plan(self):
473 files = {}
474 folders = {}
475 errors = []
476 pending = [self.root]
477 while pending:
478 for artifact, value in children(pending.pop()):
479 if stat.S_ISDIR(value.st_mode):
480 pending.append(artifact)
481 continue
482 if not stat.S_ISREG(value.st_mode) or not PATTERN.fullmatch(artifact.name) or artifact.name.endswith('.minisig'):
483 continue
484 try:
485 if artifact.relative_to(self.root) != relative(artifact.name):
486 raise Unavailable('misplaced Zig artifact')
487 timestamp = artifact_time(artifact)
488 files[artifact] = timestamp
489 signature = Path(str(artifact) + '.minisig')
490 if signature.exists():
491 with os.fdopen(metadata_open(signature), 'rb') as stream:
492 if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
493 raise Unavailable('signature is not a regular file')
494 files[signature] = timestamp
495 directory = artifact.parent
496 while True:
497 folders[directory] = max(folders.get(directory, 0), timestamp)
498 if directory == self.root:
499 break
500 directory = directory.parent
501 except (Unavailable, OSError, ValueError) as error:
502 errors.append({'path': str(artifact.relative_to(self.root)), 'error': str(error)})
503 readmes = []
504 for directory in sorted(set(folders) - {self.root}):
505 try:
506 change = self.directory_readme(directory)
507 if change:
508 readmes.append(change)
509 except (Unavailable, OSError, ValueError) as error:
510 errors.append({'path': str(directory.relative_to(self.root)), 'error': str(error)})
511 entries = []
512 for path, timestamp in sorted({**files, **folders}.items()):
513 descriptor = metadata_open(path)
514 try:
515 entries.append({'path': str(path.relative_to(self.root)), 'before': file_state(os.fstat(descriptor)), 'mtime_ns': timestamp})
516 finally:
517 os.close(descriptor)
518 return {'root': str(self.root), 'files': len(files), 'folders': len(folders),
519 'changed': sum(entry['before']['mtime_ns'] != entry['mtime_ns'] for entry in entries),
520 'provenance': {'stable': OFFICIAL + '/download/index.json UTC date', 'development': 'pinned Minisign authenticated trusted-comment timestamp', 'withdrawn0.15.0': 'authenticated signing time; release retracted before 0.15.1', 'folders': 'newest contained artifact timestamp'},
521 'errors': errors, 'entries': entries, 'readmes': readmes}
522
523 def reconcile(self, apply=False, journal=None):
524 with self.publishing() if apply else contextlib.nullcontext():
525 plan = self.metadata_plan()
526 if apply:
527 if plan['errors']:
528 raise Unavailable('timestamp plan has unresolved dates')
529 if journal is None or journal.resolve().is_relative_to(self.root):
530 raise ValueError('private journal outside the mirror is required')
531 journal = journal.resolve()
532 descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
533 with os.fdopen(descriptor, 'w') as output:
534 json.dump(plan, output)
535 output.write('\n')
536 output.flush()
537 os.fsync(output.fileno())
538 descriptor = metadata_open(journal.parent)
539 try:
540 os.fsync(descriptor)
541 finally:
542 os.close(descriptor)
543 for change in plan['readmes']:
544 self.apply_readme(change)
545 for entry in plan['entries']:
546 if entry['before']['mtime_ns'] == entry['mtime_ns'] and not stat.S_ISDIR(entry['before']['mode']):
547 continue
548 descriptor = metadata_open(self.root / entry['path'])
549 try:
550 current = file_state(os.fstat(descriptor))
551 expected = entry['before']
552 fields = ('dev', 'ino', 'mode', 'uid', 'gid') if stat.S_ISDIR(current['mode']) else tuple(key for key in expected if key != 'atime_ns')
553 if any(current[key] != expected[key] for key in fields):
554 raise Unavailable('file changed since timestamp plan: ' + entry['path'])
555 if current['mtime_ns'] != entry['mtime_ns']:
556 os.utime(descriptor, ns=(current['atime_ns'], entry['mtime_ns']))
557 finally:
558 os.close(descriptor)
559 applied = {}
560 paths = {entry['path'] for entry in plan['entries']} | {change['path'] for change in plan['readmes']}
561 for path in sorted(paths):
562 descriptor = metadata_open(self.root / path)
563 try:
564 applied[path] = file_state(os.fstat(descriptor))
565 finally:
566 os.close(descriptor)
567 with os.fdopen(metadata_open(journal, os.O_WRONLY | os.O_APPEND), 'w') as output:
568 json.dump({'applied': applied}, output)
569 output.write('\n')
570 output.flush()
571 os.fsync(output.fileno())
572 return plan
573
155 def verify(self):574 def verify(self):
156 entries = index()575 entries = index()
157 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']576 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
...@@ -181,13 +600,20 @@ class Mirror:...@@ -181,13 +600,20 @@ class Mirror:
181 if not self.refresh_lock.acquire(blocking=False):600 if not self.refresh_lock.acquire(blocking=False):
182 return601 return
183 try:602 try:
184 entries = index()603 entries = index(refresh=True)
185 latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))]604 latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))]
186 for release in (latest, entries['master']):605 for release in (latest, entries['master']):
606 artifact = None
187 for item in release.values():607 for item in release.values():
188 if isinstance(item, dict) and 'tarball' in item:608 if isinstance(item, dict) and 'tarball' in item:
189 name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1])609 name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1])
190 self.ensure(name, checksum=item['shasum'])610 artifact = self.ensure(name, checksum=item['shasum'])
611 if artifact is not None:
612 with self.publishing():
613 try:
614 self.write_readmes(artifact.parent)
615 finally:
616 self.parent_times(artifact)
191 finally:617 finally:
192 self.refresh_lock.release()618 self.refresh_lock.release()
193619
...@@ -306,15 +732,20 @@ def backfill(mirror, budget, apply):...@@ -306,15 +732,20 @@ def backfill(mirror, budget, apply):
306732
307def main():733def main():
308 parser = argparse.ArgumentParser()734 parser = argparse.ArgumentParser()
309 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify'])735 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify', 'reconcile'])
310 parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror'))736 parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror'))
311 parser.add_argument('--budget-gib', type=int, default=32)737 parser.add_argument('--budget-gib', type=int, default=32)
312 parser.add_argument('--apply', action='store_true')738 parser.add_argument('--apply', action='store_true')
739 parser.add_argument('--journal', type=Path)
313 args = parser.parse_args()740 args = parser.parse_args()
314 os.umask(0o002)741 os.umask(0o002)
742 if args.command == 'reconcile' and not Path(args.root).is_dir():
743 raise ValueError('existing mirror root is required')
315 mirror = Mirror(args.root)744 mirror = Mirror(args.root)
316 if args.command == 'verify':745 if args.command == 'verify':
317 mirror.verify()746 mirror.verify()
747 elif args.command == 'reconcile':
748 print(json.dumps(mirror.reconcile(args.apply, args.journal)))
318 elif args.command == 'refresh':749 elif args.command == 'refresh':
319 mirror.refresh()750 mirror.refresh()
320 elif args.command == 'backfill':751 elif args.command == 'backfill':
service/zig-mirror/fixture/zig-x86_64-linux-0.15.0.tar.xz.minisig created+4
...@@ -0,0 +1,4 @@
1untrusted comment: signature from minisign secret key
2RUSGOq2NVecA2RV/TllU7u4sMW2JwbgaUIX6upgmh5dqHIle3ERNdeh58+iO14pz84aSsJzW8yWISzgmbf/AKPvj09piCUZmGws=
3trusted comment: timestamp:1755639580 file:zig-x86_64-linux-0.15.0.tar.xz hashed
4RO3woTlcv5vkx53pBuILPCuk3u74YyiPQqWvbLJzcOacBRhZ6nQcBM7RvBCyNts7v2Jt3CmbEULL0yfppsrpAA==
service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig created+4
...@@ -0,0 +1,4 @@
1untrusted comment: signature from minisign secret key
2RUSGOq2NVecA2RwFmP6WKntiiWhQqtOhDbzNAVDP/x3uN6v1EcAMLi5uCCw1kisYZ1Dl6SOJTU6/6FT4pJDgRK093OhyF8QHfQg=
3trusted comment: timestamp:1791194091 file:zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip hashed
4m6Yh4Pw9wtV1WB26pz7pMTQ6gdjUABVuOTN66QXuwvuoKn0ixv6DO01+QtJQnaZfcQg1EnJGp854sGjb627pDA==
service/zig-mirror/prepare.py deleted-14
...@@ -1,14 +0,0 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import sys
6
7data = json.load(sys.stdin)
8if Path(data['hostRoot']).name != 'zig-mirror':
9 root = Path(data['hostRoot']) / 'mirror'
10 artifact = root / '0.0.0/zig-0.0.0.tar.xz'
11 artifact.parent.mkdir(parents=True, exist_ok=True)
12 artifact.write_bytes(b'Synthetic Zig mirror preview.\n')
13 for path in (artifact.parent, artifact):
14 os.chown(path, data['uid'], data['uid'])
service/zig-mirror/service.pkl+2-1
...@@ -5,10 +5,11 @@ import "../../config/site.pkl" as site...@@ -5,10 +5,11 @@ import "../../config/site.pkl" as site
5name = "Zig mirror"5name = "Zig mirror"
6launcher = false6launcher = false
7stageIsolation = "fresh"7stageIsolation = "fresh"
8prepare = "prepare.py"
98
10container {9container {
11 build = "build"10 build = "build"
11 readOnlyRoot = true
12 capAdd { "FOWNER" }
12 http {13 http {
13 containerPort = 808014 containerPort = 8080
14 subdomain = "zig"15 subdomain = "zig"
tools/snowglobe-secret-files-test.py+12
...@@ -43,6 +43,18 @@ class SecretFiles(unittest.TestCase):...@@ -43,6 +43,18 @@ class SecretFiles(unittest.TestCase):
43 self.assertNotIn('template {', rendered)43 self.assertNotIn('template {', rendered)
44 self.assertNotIn('secrets/', rendered)44 self.assertNotIn('secrets/', rendered)
4545
46 def test_read_only_root_is_opt_in_and_preserves_writable_data_mounts(self):
47 self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data}))
48 self.task["readOnlyRoot"] = True
49 self.task["volumes"] = {"/mirror": {"src": "/fixture/mirror", "readOnly": False}}
50 self.task["capAdd"] = ["FOWNER"]
51 rendered = snowglobe.render(self.data, {"fixture": self.data})
52 self.assertIn('readonly_rootfs = true', rendered)
53 self.assertIn('cap_add = ["FOWNER"]', rendered)
54 self.assertIn('/fixture/mirror:/mirror"', rendered)
55 del self.task["readOnlyRoot"]
56 self.assertNotIn('readonly_rootfs', snowglobe.render(self.data, {"fixture": self.data}))
57
46 def test_unsafe_and_colliding_targets_are_rejected(self):58 def test_unsafe_and_colliding_targets_are_rejected(self):
47 for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"):59 for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"):
48 with self.subTest(target=target):60 with self.subTest(target=target):
tools/snowglobe.py+2
...@@ -398,6 +398,8 @@ def render(data, definitions):...@@ -398,6 +398,8 @@ def render(data, definitions):
398 lines.append(f" args = {q(task['args'])}")398 lines.append(f" args = {q(task['args'])}")
399 if task["hostNetwork"]:399 if task["hostNetwork"]:
400 lines.append(' network_mode = "host"')400 lines.append(' network_mode = "host"')
401 if task.get("readOnlyRoot"):
402 lines.append(' readonly_rootfs = true')
401 task_ports = [label for (owner, _), label in ports.items() if owner == task_name]403 task_ports = [label for (owner, _), label in ports.items() if owner == task_name]
402 if task_ports and not task["hostNetwork"]:404 if task_ports and not task["hostNetwork"]:
403 lines.append(f" ports = {q(task_ports)}")405 lines.append(f" ports = {q(task_ports)}")
tools/zig-mirror-test.py+257-1
...@@ -3,7 +3,10 @@ import concurrent.futures...@@ -3,7 +3,10 @@ import concurrent.futures
3import importlib.util3import importlib.util
4import io4import io
5import contextlib5import contextlib
6import ctypes.util
7import datetime
6import json8import json
9import os
7from pathlib import Path10from pathlib import Path
8from types import SimpleNamespace11from types import SimpleNamespace
9import tempfile12import tempfile
...@@ -19,6 +22,50 @@ mirror = importlib.util.module_from_spec(spec)...@@ -19,6 +22,50 @@ mirror = importlib.util.module_from_spec(spec)
19spec.loader.exec_module(mirror)22spec.loader.exec_module(mirror)
2023
2124
25class IndexTests(unittest.TestCase):
26 def setUp(self):
27 cache = patch.object(mirror, '_index_cache', (float('-inf'), {}))
28 cache.start()
29 self.addCleanup(cache.stop)
30 mirror.stable_time.cache_clear()
31
32 def test_index_coalesces_concurrent_reads_expires_and_forces_refresh(self):
33 snapshot = {'0.17.0': {'docs': 'https://ziglang.org/documentation/0.17.0/'}}
34 def response(*args, **kwargs):
35 time.sleep(.01)
36 return io.BytesIO(json.dumps(snapshot).encode())
37 with patch.object(mirror.urllib.request, 'urlopen', side_effect=response) as request, patch.object(mirror.time, 'monotonic', return_value=0) as clock:
38 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers:
39 entries = list(workers.map(lambda _: mirror.index(), range(8)))
40 self.assertTrue(all(entry == snapshot for entry in entries))
41 self.assertEqual(request.call_count, 1)
42 clock.return_value = 3599
43 mirror.index()
44 self.assertEqual(request.call_count, 1)
45 clock.return_value = 3600
46 mirror.index()
47 self.assertEqual(request.call_count, 2)
48 mirror.index(refresh=True)
49 self.assertEqual(request.call_count, 3)
50 self.assertTrue(all(call.args[0] == 'https://ziglang.org/download/index.json' for call in request.call_args_list))
51
52 def test_new_release_miss_refreshes_stale_index_before_publication(self):
53 release = {'date': '2027-04-01', 'docs': 'https://ziglang.org/documentation/0.19.0/'}
54 responses = [{}, {'0.19.0': release}]
55 with tempfile.TemporaryDirectory() as root, patch.object(mirror.urllib.request, 'urlopen', side_effect=lambda *args, **kwargs: io.BytesIO(json.dumps(responses.pop(0)).encode())) as request:
56 mirror.index()
57 cache = mirror.Mirror(root)
58 with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3)), patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')), patch.object(mirror.subprocess, 'run', return_value=SimpleNamespace(returncode=0)):
59 artifact = cache.ensure('zig-x86_64-linux-0.19.0.tar.xz')
60 self.assertEqual(request.call_count, 2)
61 self.assertIn('[Language Reference](' + release['docs'] + ')', (artifact.parent / 'preadme.md').read_text())
62 expected = int(datetime.datetime(2027, 4, 1, tzinfo=datetime.timezone.utc).timestamp()) * 10**9
63 self.assertEqual(artifact.stat().st_mtime_ns, expected)
64 self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, expected)
65 self.assertEqual(artifact.parent.stat().st_mtime_ns, expected)
66 self.assertEqual(cache.root.stat().st_mtime_ns, expected)
67
68
22class MirrorTests(unittest.TestCase):69class MirrorTests(unittest.TestCase):
23 def setUp(self):70 def setUp(self):
24 self.temporary = tempfile.TemporaryDirectory()71 self.temporary = tempfile.TemporaryDirectory()
...@@ -27,6 +74,16 @@ class MirrorTests(unittest.TestCase):...@@ -27,6 +74,16 @@ class MirrorTests(unittest.TestCase):
27 disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3))74 disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3))
28 disk.start()75 disk.start()
29 self.addCleanup(disk.stop)76 self.addCleanup(disk.stop)
77 mirror.stable_time.cache_clear()
78 dates = {version: {'date': date} for version, date in [('0.1.1', '2017-10-17'), ('0.2.0', '2018-03-15'),
79 ('0.3.0', '2018-09-28'), ('0.4.0', '2019-04-08'), ('0.5.0', '2019-09-30'), ('0.17.0', '2026-10-01')]}
80 metadata = patch.object(mirror, 'index', return_value=dates)
81 metadata.start()
82 self.addCleanup(metadata.stop)
83
84 def dev_fixture(self):
85 fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig'
86 return fixture.name.removesuffix('.minisig'), fixture.read_bytes()
3087
31 def test_spec_filenames_and_existing_layout(self):88 def test_spec_filenames_and_existing_layout(self):
32 cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'),89 cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'),
...@@ -150,7 +207,7 @@ class MirrorTests(unittest.TestCase):...@@ -150,7 +207,7 @@ class MirrorTests(unittest.TestCase):
150 def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self):207 def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self):
151 names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz']208 names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz']
152 body = b'official historical archive'209 body = b'official historical archive'
153 entries = {mirror.filename(name): {'platform': {'tarball': mirror.upstream(name),210 entries = {mirror.filename(name): {'date': '2017-10-17', 'platform': {'tarball': mirror.upstream(name),
154 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}}211 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}}
155 for i, name in enumerate(names)}212 for i, name in enumerate(names)}
156 def fetch(url, target, limit):213 def fetch(url, target, limit):
...@@ -179,6 +236,205 @@ class MirrorTests(unittest.TestCase):...@@ -179,6 +236,205 @@ class MirrorTests(unittest.TestCase):
179 self.assertEqual(response.read(), body)236 self.assertEqual(response.read(), body)
180 self.assertFalse(list(self.cache.root.rglob('.download-*')))237 self.assertFalse(list(self.cache.root.rglob('.download-*')))
181238
239 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
240 def test_real_minisign_comment_and_tampering(self):
241 name, body = self.dev_fixture()
242 signature = self.cache.root / (name + '.minisig')
243 signature.write_bytes(body)
244 self.assertEqual(mirror.signed_time(signature), 1791194091 * 10**9)
245 for changed in [body.replace(b'1791194091', b'1791194092'), body.replace(b'RUSGO', b'RUSGP'),
246 body.replace(b'trusted comment: timestamp:', b'trusted comment: forged:')]:
247 signature.write_bytes(changed)
248 with self.assertRaises(mirror.Unavailable):
249 mirror.signed_time(signature)
250 signature.unlink()
251 os.mkfifo(signature)
252 with self.assertRaises(mirror.Unavailable):
253 mirror.signed_time(signature)
254
255 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
256 def test_publisher_dates_pairs_and_folders_after_cleanup(self):
257 name, signature = self.dev_fixture()
258 def fetch(url, target, limit):
259 target.write_bytes(signature if url.endswith('.minisig') and '-dev.' in url else b'fixture')
260 with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify:
261 verify.return_value.returncode = 0
262 artifact = self.cache.ensure(name)
263 stamp = 1791194091 * 10**9
264 self.assertEqual(artifact.stat().st_mtime_ns, stamp)
265 self.assertEqual(Path(str(artifact) + '.minisig').stat().st_mtime_ns, stamp)
266 for directory in [artifact.parent, artifact.parent.parent, artifact.parent.parent.parent, self.cache.root]:
267 self.assertEqual(directory.stat().st_mtime_ns, stamp)
268 (self.cache.root / '0.99.0').mkdir()
269 older = self.cache.ensure('zig-0.17.0.tar.xz')
270 self.assertEqual(older.stat().st_mtime_ns, 1790812800 * 10**9)
271 self.assertEqual(self.cache.root.stat().st_mtime_ns, stamp)
272 self.assertFalse(list(self.cache.root.rglob('.download-*')))
273 self.assertIn('Signature for '+name, (artifact.parent/'preadme.md').read_text())
274 self.assertIn('this release only has [development builds]', (artifact.parent.parent.parent/'preadme.md').read_text())
275
276 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
277 def test_reconcile_is_metadata_only_idempotent_and_journalled(self):
278 name, signature = self.dev_fixture()
279 artifact = self.cache.path(name);artifact.parent.mkdir(parents=True)
280 artifact.write_bytes(b'public compiler fixture')
281 Path(str(artifact)+'.minisig').write_bytes(signature)
282 stable = self.cache.path('zig-win64-0.1.1.zip');stable.parent.mkdir();stable.write_bytes(b'legacy')
283 root_readme = self.cache.root/'preadme.md';root_readme.write_text('owner root notes')
284 version_readme = artifact.parent/'preadme.md';version_readme.write_text('old public version notes')
285 hidden = self.cache.root/'.hist';hidden.mkdir();(hidden/'zig-0.17.0.tar.xz').write_bytes(b'private history')
286 original = {path: (path.stat().st_ino,path.stat().st_uid,path.stat().st_gid,path.stat().st_mode,path.stat().st_size)
287 for path in [artifact,Path(str(artifact)+'.minisig'),stable,root_readme,hidden/'zig-0.17.0.tar.xz']}
288 plan = self.cache.reconcile()
289 self.assertFalse(plan['errors'])
290 self.assertFalse((self.cache.root/'.mirror-locks').exists())
291 with tempfile.TemporaryDirectory() as private:
292 journal=Path(private)/'undo.json'
293 self.cache.reconcile(True,journal)
294 self.assertEqual(journal.stat().st_mode&0o777,0o600)
295 records = [json.loads(line) for line in journal.read_text().splitlines()]
296 self.assertTrue(records[0]['readmes'])
297 self.assertEqual(set(records[1]['applied']),
298 {entry['path'] for entry in records[0]['entries']} |
299 {change['path'] for change in records[0]['readmes']})
300 with self.assertRaises(FileExistsError):
301 self.cache.reconcile(True, journal)
302 with self.assertRaises(ValueError):
303 self.cache.reconcile(True, self.cache.root / 'public-undo.json')
304 for path,state in original.items():
305 actual=path.stat();self.assertEqual((actual.st_ino,actual.st_uid,actual.st_gid,actual.st_mode,actual.st_size),state)
306 self.assertEqual(root_readme.read_text(),'owner root notes')
307 self.assertEqual(artifact.read_bytes(),b'public compiler fixture')
308 self.assertEqual(self.cache.reconcile()['changed'],0)
309 self.assertEqual(self.cache.reconcile()['readmes'],[])
310
311 def test_hidden_lock_creation_preserves_visible_root_date(self):
312 os.utime(self.cache.root, ns=(1234, 5678))
313 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers:
314 list(workers.map(lambda _: self.cache.lock_directory(), range(12)))
315 self.assertEqual(self.cache.root.stat().st_mtime_ns, 5678)
316
317 @unittest.skipUnless(ctypes.util.find_library('sodium'), 'libsodium required')
318 def test_withdrawn_0150_uses_authenticated_signing_time(self):
319 name = 'zig-x86_64-linux-0.15.0.tar.xz'
320 fixture = Path(__file__).resolve().parents[1] / 'service/zig-mirror/fixture' / (name + '.minisig')
321 artifact = self.cache.path(name)
322 artifact.parent.mkdir()
323 artifact.write_bytes(b'already verified archive')
324 signature = Path(str(artifact) + '.minisig')
325 signature.write_bytes(fixture.read_bytes())
326 with patch.object(mirror, 'index', side_effect=AssertionError('withdrawn release must not use another release date')):
327 self.assertEqual(mirror.artifact_time(artifact), mirror.signed_time(signature))
328 signature.write_bytes(fixture.read_bytes().replace(b'timestamp:', b'tampered:'))
329 with self.assertRaises(mirror.Unavailable):
330 mirror.artifact_time(artifact)
331
332 def test_missing_release_date_never_publishes(self):
333 with patch.object(mirror, 'fetch', side_effect=lambda url, target, limit: target.write_bytes(b'fixture')):
334 with self.assertRaises(mirror.Unavailable):
335 self.cache.ensure('zig-0.4.1.tar.xz', checksum=mirror.hashlib.sha256(b'fixture').hexdigest())
336 self.assertFalse(self.cache.path('zig-0.4.1.tar.xz').exists())
337
338 def test_metadata_traversal_rejects_intermediate_symlinks(self):
339 with tempfile.TemporaryDirectory() as outside:
340 target=Path(outside)/'file';target.write_bytes(b'untouched')
341 (self.cache.root/'alias').symlink_to(outside)
342 with self.assertRaises(OSError):
343 mirror.set_mtime(self.cache.root/'alias/file',1)
344
345 def test_dev_index_replaces_duplicate_list_with_heading(self):
346 artifact = self.cache.path('zig-x86_64-linux-0.18.0-dev.4+5a23bf4b2.tar.xz')
347 artifact.parent.mkdir(parents=True)
348 artifact.write_bytes(b'public compiler fixture')
349 directory = artifact.parent.parent
350 readme = directory / 'preadme.md'
351 readme.write_text('### Zig 0.18.0 development builds\n\n- [0.18.0-dev.4+5a23bf4b2](old-link/)\n')
352 previous = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, directory, directory.parent, self.cache.root]}
353 change = self.cache.directory_readme(directory)
354 self.assertEqual(change['content'], '### Zig 0.18.0 development builds\n')
355 self.cache.apply_readme(change)
356 self.assertIsNone(self.cache.directory_readme(directory))
357 self.assertEqual(readme.read_text(), change['content'])
358 for path, timestamp in previous.items():
359 self.assertEqual(path.stat().st_mtime_ns, timestamp)
360
361 def test_development_only_base_has_exact_sentence_without_release_docs(self):
362 directory = self.cache.root / '0.18.0'
363 (directory / 'dev').mkdir(parents=True)
364 with patch.object(mirror, 'index', side_effect=AssertionError('development navigation needs no release metadata')):
365 change = self.cache.directory_readme(directory)
366 self.assertEqual(change['content'], '### Zig 0.18.0\n\nthis release only has [development builds](/mirrors/zig/0.18.0/dev/) out.\n')
367 self.cache.apply_readme(change)
368 self.assertIsNone(self.cache.directory_readme(directory))
369
370 def test_release_links_follow_index_presence_and_precede_artifact_table(self):
371 artifact = self.cache.path('zig-x86_64-linux-0.19.0.tar.xz')
372 release = {'notes': 'https://ziglang.org/download/0.19.0/release-notes.html',
373 'docs': 'https://ziglang.org/documentation/0.19.0/',
374 'stdDocs': 'https://ziglang.org/documentation/0.19.0/std/'}
375 labels = {'notes': 'Release Notes', 'docs': 'Language Reference', 'stdDocs': 'Standard Library Documentation'}
376 for absent in [None, *release]:
377 with self.subTest(absent=absent), patch.object(mirror, 'index', return_value={'0.19.0': {key: url for key, url in release.items() if key != absent}}):
378 text = mirror.version_readme('0.19.0', [artifact], True).decode()
379 for key, url in release.items():
380 if key == absent:
381 self.assertNotIn(f'[{labels[key]}]({url})', text)
382 else:
383 self.assertIn(f'- [{labels[key]}]({url})', text)
384 self.assertLess(text.index(url), text.index('[Development builds]'))
385 self.assertLess(text.index('[Development builds]'), text.index('<div '))
386 with patch.object(mirror, 'index', return_value={'0.19.0': {'notes': 'https://example.org/notes', 'docs': None, 'stdDocs': 'https://ziglang.org/bad)link'}}):
387 text = mirror.version_readme('0.19.0', [artifact]).decode()
388 for label in labels.values():
389 self.assertNotIn(label, text)
390 self.assertIn(artifact.name, text)
391
392 def test_refresh_updates_cached_release_links_without_changing_dates(self):
393 artifact = self.cache.path('zig-x86_64-linux-0.17.0.tar.xz')
394 artifact.parent.mkdir()
395 artifact.write_bytes(b'already verified archive')
396 stamp = mirror.stable_time('0.17.0')
397 for path in [artifact, artifact.parent, self.cache.root]:
398 mirror.set_mtime(path, stamp)
399 self.cache.write_readmes(artifact.parent)
400 mirror.set_mtime(artifact.parent, stamp)
401 readme = artifact.parent / 'preadme.md'
402 original = {path: path.stat().st_mtime_ns for path in [artifact, artifact.parent, self.cache.root, readme]}
403 release = {'date': '2026-10-01', 'docs': 'https://ziglang.org/documentation/0.17.0/',
404 'src': {'tarball': mirror.upstream(artifact.name), 'shasum': 'unused-cached-hash'}}
405 with patch.object(mirror, 'index', return_value={'0.17.0': release, 'master': {}}) as metadata, patch.object(mirror, 'fetch', side_effect=AssertionError('cached refresh must not redownload')):
406 self.cache.refresh()
407 self.assertIn('[Language Reference](' + release['docs'] + ')', readme.read_text())
408 self.assertIsNone(self.cache.directory_readme(artifact.parent))
409 metadata.assert_any_call(refresh=True)
410 release['notes'] = 'https://ziglang.org/download/0.17.0/release-notes.html'
411 self.cache.refresh()
412 self.assertIn('[Release Notes](' + release['notes'] + ')', readme.read_text())
413 self.assertIsNone(self.cache.directory_readme(artifact.parent))
414 del release['docs']
415 self.cache.refresh()
416 self.assertNotIn('Language Reference', readme.read_text())
417 readme.write_text(readme.read_text() + '\nCustom table notes.\n')
418 mirror.set_mtime(readme, original[readme])
419 self.assertIsNone(self.cache.directory_readme(artifact.parent))
420 for path, timestamp in original.items():
421 self.assertEqual(path.stat().st_mtime_ns, timestamp)
422
423 def test_readme_covers_legacy_bootstrap_both_target_orders_and_dynamic_os(self):
424 names=['zig-0.1.1.tar.xz','zig-bootstrap-0.1.1.zip','zig-win64-0.1.1.zip',
425 'zig-linux-aarch64-0.1.1.tar.xz','zig-x86_64-macos-0.1.1.tar.xz','zig-newarch-newos-0.1.1.zip']
426 artifacts=[self.cache.path(name) for name in names]
427 for artifact in artifacts:artifact.parent.mkdir(parents=True,exist_ok=True);artifact.write_bytes(b'fixture')
428 text=mirror.version_readme('0.1.1',artifacts).decode()
429 for name in names:self.assertIn(name,text)
430 self.assertIn('Bootstrap',text);self.assertIn('newos',text);self.assertNotIn('&#128272;',text)
431 self.assertIn('role="region" aria-label="Zig downloads" tabindex="0"',text)
432 self.assertIn('>\n\n| Source ',text)
433 self.assertLess(text.index('</div>'),text.index('This historical release'))
434 self.assertNotIn('https://file.paperclover.net',text)
435 empty=mirror.version_readme('0.17.0',[],True).decode()
436 self.assertIn('[development builds]',empty);self.assertNotIn('| Linux',empty);self.assertNotIn('role="region"',empty)
437
182438
183if __name__ == '__main__':439if __name__ == '__main__':
184 unittest.main()440 unittest.main()
tools/zig-mirror.md+58
...@@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after...@@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after
23SHA-256 and/or Zig Minisign verification. Existing archive files need the23SHA-256 and/or Zig Minisign verification. Existing archive files need the
24one-time `verify` operation before first public exposure.24one-time `verify` operation before first public exposure.
2525
26## Release dates and navigation
27
28Indexed stable releases use the official download index's date at UTC midnight.
29The retracted 0.15.0 artifacts use authenticated signing time instead: the
30[official retraction](https://github.com/ziglang/www.ziglang.org/commit/1be0f80b15d256d42411a21f10e208f9b269a5e2)
31removed that release from the index. Other unknown stable releases fail closed.
32Development archives use the build timestamp from their Minisign trusted comment;
33its global Ed25519 signature is checked against Zig's pinned key before the date
34is accepted. The signature file receives the archive's date. Each visible folder,
35including the mirror root, uses its newest contained archive date. Empty folders,
36README files and hidden operational/history files do not contribute to that date.
37Every cache miss, periodic refresh and backfill uses this same publication path.
38
39Version folders contain an architecture/OS download table built only from present
40canonical artifacts and signatures. Released versions list Release Notes,
41Language Reference and Standard Library Documentation below their heading only
42when the official index publishes the corresponding `notes`, `docs` or `stdDocs`
43URL. No documentation URL is synthesized or probed per artifact. The index has
44one process-wide, lock-protected snapshot, expiring after an hour; periodic/manual
45refresh bypasses it, as does a cache miss for an unindexed stable version's date.
46Refresh also updates the current release/build navigation when archives are
47already cached, preserving artifact and folder dates.
48
49Base versions link to their development index; a base folder with only development
50builds says “this release only has development builds out.” Each development
51version links back to its index, whose heading leaves the native directory listing
52to show the builds. Complete existing relative-link tables are retained when
53their introduction matches the published links and navigation; incomplete or
54production-only links are replaced. The owner's root `preadme.md` and hidden
55history remain untouched.
56
57Existing verified archives need a one-time metadata reconciliation. It reads
58signature comments and file metadata, not archive bodies, and preserves atime,
59ownership and modes. Preview it first:
60
61```sh
62python3 /app/mirror.py reconcile > /alloc/metadata-proposal.json
63python3 /app/mirror.py reconcile --apply --journal /alloc/private-metadata-undo.jsonl
64python3 /app/mirror.py reconcile > /alloc/metadata-after.json
65```
66
67Retain proposal and undo files privately outside `/mirror`; never publish them
68through Copyparty. Apply fails closed on missing dates, unsafe paths or changed
69files. The journal is created exclusively with mode0600 and fsynced before any
70change. Its first record retains old metadata and replaced public README bytes;
71a second record captures completed inode/metadata states. A journal without the
72completion record requires recovery review. A guarded one-time undo must verify
73those completed identities and generated README bytes before restoring old
74metadata/docs; later writes must cause it to stop. Keep that recovery helper with
75the private deployment evidence, rather than adding an automatic rollback hook.
76
77The worker has only `FOWNER` beyond its normal capability set, allowing explicit
78historic dates on the existing UID3000-owned NAS folders without changing their
79ownership. Its root filesystem is read-only and its only writable external mount
80is the canonical Zig subtree. Download temporaries and locks stay beneath hidden
81`.mirror-locks`; they cannot change visible folder dates. Fresh worker previews
82start empty, disable polling and fetch public artifacts only when requested.
83
26## Production cutover84## Production cutover
2785
28The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files86The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files