From a4c36c6d8f5893c91e75def1bc06dad8917475bc Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 17 Oct 2026 12:43:21 -0700 Subject: [PATCH] Show Zig release navigation and preserve archive dates Generate published documentation links and version navigation, authenticate development build dates, and preserve root and ancestor timestamps on publication. Simplify anonymous mirror browsing in Copyparty. Run reconciliation explicitly with a private recovery journal. Assisted-by: gpt-6.1-sol --- config/Service.pkl | 1 + service/copyparty/copyparty.conf | 1 + service/copyparty/static/copyparty.css | 16 + service/copyparty/static/mirrors.js | 12 + service/zig-mirror/build/mirror.py | 461 +++++++++++++++++- .../zig-x86_64-linux-0.15.0.tar.xz.minisig | 4 + ...windows-0.18.0-dev.4+5a23bf4b2.zip.minisig | 4 + service/zig-mirror/prepare.py | 14 - service/zig-mirror/service.pkl | 3 +- tools/snowglobe-secret-files-test.py | 12 + tools/snowglobe.py | 2 + tools/zig-mirror-test.py | 258 +++++++++- tools/zig-mirror.md | 58 +++ 13 files changed, 815 insertions(+), 31 deletions(-) create mode 100644 service/copyparty/static/mirrors.js create mode 100644 service/zig-mirror/fixture/zig-x86_64-linux-0.15.0.tar.xz.minisig create mode 100644 service/zig-mirror/fixture/zig-x86_64-windows-0.18.0-dev.4+5a23bf4b2.zip.minisig delete mode 100644 service/zig-mirror/prepare.py diff --git a/config/Service.pkl b/config/Service.pkl index d0473741351172c0d1fb9d096b6cd67834cfb62a..2cb1ccfc7d8fc89e72b824b28181c86f58e0e984 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -85,6 +85,7 @@ class Container { secretFiles: Mapping = new {} args: Listing = new {} capAdd: Listing = new {} + readOnlyRoot: Boolean = false devices: Listing = new {} extraHosts: Listing = new {} hostNetwork: Boolean = false diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index 7e663a4d279a966eb37b17066dbb0519677c7c98..2466309442cbea7b5b22d32101276f07e00e6fc8 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -24,6 +24,7 @@ idp-logout: /auth/file/sign-out idp-login-t: with snow globe html-head: + js-browser: /.static/mirrors.js [/] /w diff --git a/service/copyparty/static/copyparty.css b/service/copyparty/static/copyparty.css index 7d88b7f6d3b5b732d5d6efea3470be73b1b2cd5e..857e385e68f10128d449281e7e100cc715e534fc 100644 --- a/service/copyparty/static/copyparty.css +++ b/service/copyparty/static/copyparty.css @@ -7,3 +7,19 @@ --font-serif: "Name Sans", serif; --font-mono: "Name Mono", monospace; } +.mirror-anonymous #files tr > :first-child, +.mirror-anonymous #tree, +.mirror-anonymous #entree, +.mirror-anonymous #opa_auc, +.mirror-anonymous #op_player, +.mirror-anonymous #widget { + display: none !important; +} +.mirror-anonymous #path { + display: inline-block !important; +} +.mirror-anonymous #wrap { + margin-left: 1.5em !important; + padding-bottom: 1.5em; + overflow-x: auto; +} diff --git a/service/copyparty/static/mirrors.js b/service/copyparty/static/mirrors.js new file mode 100644 index 0000000000000000000000000000000000000000..37938be439d80f0c2b23f2879879920cce831808 --- /dev/null +++ b/service/copyparty/static/mirrors.js @@ -0,0 +1,12 @@ +(function () { + var render = treectl.gentab; + treectl.gentab = function (top, listing) { + var anonymous = (listing.acct === undefined ? acct : listing.acct) === "*" && /^\/mirrors(?:\/|$)/.test(top); + document.body.classList.toggle("mirror-anonymous", anonymous); + if (anonymous) + read_dsort("-ts"); + return render.call(this, top, listing); + }; + if (ls0 !== null) + treectl.gentab(get_evpath(), ls0); +})(); diff --git a/service/zig-mirror/build/mirror.py b/service/zig-mirror/build/mirror.py index ff9c7dba03e9025d2b3ae9bc9b577d9bae5eaccb..c2eb485e097ced645166d09d62d29c8c491f9155 100644 --- a/service/zig-mirror/build/mirror.py +++ b/service/zig-mirror/build/mirror.py @@ -1,8 +1,15 @@ #!/usr/bin/env python3 import argparse +import base64 +import ctypes +import ctypes.util +import contextlib +import datetime import email.utils import fcntl +import functools import hashlib +import html from html.parser import HTMLParser import http.server import json @@ -10,6 +17,7 @@ import os from pathlib import Path import re import shutil +import stat import subprocess import tempfile import threading @@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' OFFICIAL = 'https://ziglang.org' NEKTRO = 'http://mirrors.nektro.net/zig/' MAX_ARTIFACT = 2 * 1024**3 +_index_cache = (float('-inf'), {}) +_index_lock = threading.Lock() class Unavailable(Exception): pass +@functools.lru_cache(maxsize=1) +def comment_verifier(): + name = ctypes.util.find_library('sodium') + if not name: + raise Unavailable('Minisign comment verification unavailable') + library = ctypes.CDLL(name) + if library.sodium_init() < 0: + raise Unavailable('Minisign comment verification unavailable') + verify = library.crypto_sign_verify_detached + verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p] + verify.restype = ctypes.c_int + return verify + + +def metadata_open(path, final_flags=os.O_RDONLY): + path = path.absolute() + descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0)) + try: + for index, part in enumerate(path.parts[1:]): + flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0) + if index < len(path.parts) - 2: + flags |= os.O_DIRECTORY + next_descriptor = os.open(part, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = next_descriptor + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def signed_time(signature): + with os.fdopen(metadata_open(signature), 'rb') as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise Unavailable('signature is not a regular file') + body = stream.read(4097) + lines = body.splitlines() + try: + if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '): + raise ValueError() + key = base64.b64decode(PUBLIC_KEY, validate=True) + sig = base64.b64decode(lines[1], validate=True) + global_sig = base64.b64decode(lines[3], validate=True) + comment = lines[2][len(b'trusted comment: '):] + if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]: + raise ValueError() + # Minisign authenticates the artifact signature followed by the exact trusted comment. + message = sig[10:] + comment + if comment_verifier()(global_sig, message, len(message), key[10:]) != 0: + raise ValueError() + match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment) + if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'): + raise ValueError() + return int(match[1]) * 1_000_000_000 + except (ValueError, IndexError) as error: + raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error + + +@functools.lru_cache(maxsize=128) +def stable_time(version): + try: + entries = index() + if version not in entries: + entries = index(refresh=True) + date = datetime.date.fromisoformat(entries[version]['date']) + return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000 + except (KeyError, ValueError) as error: + raise Unavailable('official release date unavailable: ' + version) from error + + +def artifact_time(artifact): + version = filename(artifact.name) + if '-dev.' in version or version == '0.15.0': + return signed_time(Path(str(artifact) + '.minisig')) + return stable_time(version) + + +def children(directory): + descriptor = metadata_open(directory) + try: + with os.scandir(descriptor) as entries: + return [(directory / entry.name, entry.stat(follow_symlinks=False)) + for entry in entries if not entry.name.startswith('.')] + finally: + os.close(descriptor) + + +def set_mtime(path, timestamp): + descriptor = metadata_open(path) + try: + before = os.fstat(descriptor) + if before.st_mtime_ns != timestamp: + os.utime(descriptor, ns=(before.st_atime_ns, timestamp)) + finally: + os.close(descriptor) + + +def file_state(value): + return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')} + + +def version_readme(version, artifacts, development=False): + systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows', + 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'} + source = [] + targets = {} + + def link(artifact): + name = artifact.name + url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/') + label = 'tar.xz' if name.endswith('.tar.xz') else 'zip' + result = f'[{label}]({url})' + signature = Path(str(artifact) + '.minisig') + if signature.is_file() and not signature.is_symlink(): + result += f' 🔐' + return result + + for artifact in sorted(artifacts): + name = artifact.name + extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip' + target = name[len('zig'): -len('-' + version + extension)] + if target in ('', '-bootstrap'): + source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact)) + continue + if target == '-win64': + architecture, system = 'x86_64', 'windows' + else: + first, second = target.removeprefix('-').split('-') + system, architecture = (first, second) if first in systems else (second, first) + targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact)) + columns = list(systems) + columns += sorted({system for row in targets.values() for system in row} - set(columns)) + heading = '
'.join(source) or 'Architecture' + lines = [f'### Zig {version}', ''] + if not artifacts and development: + return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode() + if artifacts and '-dev.' not in version: + release = index().get(version, {}) + for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]: + url = release.get(key) + if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url): + lines.append(f'- [{label}]({url})') + if lines[-1]: + lines.append('') + if development: + lines += [f'[Development builds](/mirrors/zig/{version}/dev/)', ''] + elif '-dev.' in version: + lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', ''] + if not artifacts: + return '\n'.join(lines).encode() + lines += ['
', '', + '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |', + '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |'] + priority = ('x86_64', 'aarch64', 'x86', 'arm') + for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)): + lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |') + lines += ['', '
'] + if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts): + lines += ['', 'Verify your build signatures. The 🔐 links open the Minisign signatures.', ''] + else: + lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', ''] + return '\n'.join(lines).encode() + + def filename(name): match = PATTERN.fullmatch(name) if len(name) > 240 or not match: @@ -72,9 +246,14 @@ def fetch(url, target, limit): raise Unavailable('upstream download failed') from error -def index(): - with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: - return json.load(response) +def index(refresh=False): + global _index_cache + with _index_lock: + if refresh or time.monotonic() - _index_cache[0] >= 3600: + with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: + entries = json.load(response) + _index_cache = (time.monotonic(), entries) + return _index_cache[1] def digest(path): @@ -87,7 +266,10 @@ def digest(path): class Mirror: def __init__(self, root): - self.root = Path(root) + self.root = Path(root).absolute() + if self.root.is_symlink(): + raise ValueError('mirror root is a symlink') + self.root = self.root.resolve() self.root.mkdir(parents=True, exist_ok=True) self.slots = threading.BoundedSemaphore(4) self.refresh_lock = threading.Lock() @@ -98,6 +280,50 @@ class Mirror: raise ValueError('artifact escapes mirror') return path + def lock_directory(self): + locks = self.root / '.mirror-locks' + descriptor = metadata_open(self.root) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + timestamp = os.fstat(descriptor).st_mtime_ns + try: + locks.mkdir() + except FileExistsError: + pass + else: + set_mtime(self.root, timestamp) + finally: + os.close(descriptor) + descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY) + os.close(descriptor) + return locks + + @contextlib.contextmanager + def publishing(self): + locks = self.lock_directory() + with (locks / '.timestamps').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + def parent_times(self, artifact): + def latest(directory): + values = [] + for path, value in children(directory): + if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path: + values.append(value.st_mtime_ns) + elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'): + values.append(latest(path)) + return max(values, default=0) + + directory = artifact.parent + while True: + timestamp = latest(directory) + if timestamp: + set_mtime(directory, timestamp) + if directory == self.root: + break + directory = directory.parent + def ensure(self, name, source=None, checksum=None): target = self.path(name) artifact_name = name.removesuffix('.minisig') @@ -109,8 +335,7 @@ class Mirror: try: if shutil.disk_usage(self.root).free < 16 * 1024**3: raise Unavailable('mirror free-space reserve reached') - locks = self.root / '.mirror-locks' - locks.mkdir(exist_ok=True) + locks = self.lock_directory() with (locks / artifact_name).open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) if target.is_file(): @@ -123,8 +348,7 @@ class Mirror: and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) if not checksum: raise Unavailable('unsigned artifact has no official SHA-256') - artifact.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: pending = Path(temporary) / artifact_name signature = Path(str(pending) + '.minisig') url = source or upstream(artifact_name) @@ -137,21 +361,216 @@ class Mirror: capture_output=True) if result.returncode: raise Unavailable('artifact signature verification failed') + timestamp = artifact_time(pending) os.chmod(pending, 0o664) - os.replace(pending, artifact) + set_mtime(pending, timestamp) if signature.exists(): os.chmod(signature, 0o664) - os.replace(signature, Path(str(artifact) + '.minisig')) + set_mtime(signature, timestamp) + with self.publishing(): + self.path(name) + artifact.parent.mkdir(parents=True, exist_ok=True) + os.replace(pending, artifact) + if signature.exists(): + os.replace(signature, Path(str(artifact) + '.minisig')) + try: + self.write_readmes(artifact.parent) + finally: + self.parent_times(artifact) if name.endswith('.minisig') and not target.exists(): - with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: pending = Path(temporary) / name fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) os.chmod(pending, 0o664) - os.replace(pending, target) + set_mtime(pending, artifact.stat().st_mtime_ns) + with self.publishing(): + os.replace(pending, target) + try: + self.write_readmes(artifact.parent) + finally: + self.parent_times(artifact) return target finally: self.slots.release() + def readme_change(self, directory, artifacts, content=None): + path = directory / 'preadme.md' + if content is None: + content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir()) + before = None + previous = b'' + if path.exists() or path.is_symlink(): + descriptor = metadata_open(path) + with os.fdopen(descriptor, 'rb') as stream: + value = os.fstat(stream.fileno()) + if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024: + raise Unavailable('version readme is not a small regular file') + before = file_state(value) + previous = stream.read(512 * 1024 + 1) + required = {artifact.name for artifact in artifacts} + required |= {artifact.name + '.minisig' for artifact in artifacts + if Path(str(artifact) + '.minisig').is_file()} + present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1]) + for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())} + linked = {name for name in present if PATTERN.fullmatch(name)} + introduction = content.split(b''),text.index('This historical release')) + self.assertNotIn('https://file.paperclover.net',text) + empty=mirror.version_readme('0.17.0',[],True).decode() + self.assertIn('[development builds]',empty);self.assertNotIn('| Linux',empty);self.assertNotIn('role="region"',empty) + if __name__ == '__main__': unittest.main() diff --git a/tools/zig-mirror.md b/tools/zig-mirror.md index a461acab5d09d88d1e9faf0a15d96ecdd2ac1dd9..463d69cbb86c82de8c932766066673a773f6750c 100644 --- a/tools/zig-mirror.md +++ b/tools/zig-mirror.md @@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after SHA-256 and/or Zig Minisign verification. Existing archive files need the one-time `verify` operation before first public exposure. +## Release dates and navigation + +Indexed stable releases use the official download index's date at UTC midnight. +The retracted 0.15.0 artifacts use authenticated signing time instead: the +[official retraction](https://github.com/ziglang/www.ziglang.org/commit/1be0f80b15d256d42411a21f10e208f9b269a5e2) +removed that release from the index. Other unknown stable releases fail closed. +Development archives use the build timestamp from their Minisign trusted comment; +its global Ed25519 signature is checked against Zig's pinned key before the date +is accepted. The signature file receives the archive's date. Each visible folder, +including the mirror root, uses its newest contained archive date. Empty folders, +README files and hidden operational/history files do not contribute to that date. +Every cache miss, periodic refresh and backfill uses this same publication path. + +Version folders contain an architecture/OS download table built only from present +canonical artifacts and signatures. Released versions list Release Notes, +Language Reference and Standard Library Documentation below their heading only +when the official index publishes the corresponding `notes`, `docs` or `stdDocs` +URL. No documentation URL is synthesized or probed per artifact. The index has +one process-wide, lock-protected snapshot, expiring after an hour; periodic/manual +refresh bypasses it, as does a cache miss for an unindexed stable version's date. +Refresh also updates the current release/build navigation when archives are +already cached, preserving artifact and folder dates. + +Base versions link to their development index; a base folder with only development +builds says “this release only has development builds out.” Each development +version links back to its index, whose heading leaves the native directory listing +to show the builds. Complete existing relative-link tables are retained when +their introduction matches the published links and navigation; incomplete or +production-only links are replaced. The owner's root `preadme.md` and hidden +history remain untouched. + +Existing verified archives need a one-time metadata reconciliation. It reads +signature comments and file metadata, not archive bodies, and preserves atime, +ownership and modes. Preview it first: + +```sh +python3 /app/mirror.py reconcile > /alloc/metadata-proposal.json +python3 /app/mirror.py reconcile --apply --journal /alloc/private-metadata-undo.jsonl +python3 /app/mirror.py reconcile > /alloc/metadata-after.json +``` + +Retain proposal and undo files privately outside `/mirror`; never publish them +through Copyparty. Apply fails closed on missing dates, unsafe paths or changed +files. The journal is created exclusively with mode0600 and fsynced before any +change. Its first record retains old metadata and replaced public README bytes; +a second record captures completed inode/metadata states. A journal without the +completion record requires recovery review. A guarded one-time undo must verify +those completed identities and generated README bytes before restoring old +metadata/docs; later writes must cause it to stop. Keep that recovery helper with +the private deployment evidence, rather than adding an automatic rollback hook. + +The worker has only `FOWNER` beyond its normal capability set, allowing explicit +historic dates on the existing UID3000-owned NAS folders without changing their +ownership. Its root filesystem is read-only and its only writable external mount +is the canonical Zig subtree. Download temporaries and locks stay beneath hidden +`.mirror-locks`; they cannot change visible folder dates. Fresh worker previews +start empty, disable polling and fetch public artifacts only when requested. + ## Production cutover The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files -- 2.54.1