diff --git a/config/Service.pkl b/config/Service.pkl index d0473741351172c0d1fb9d096b6cd67834cfb62a..2cb1ccfc7d8fc89e72b824b28181c86f58e0e984 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -85,6 +85,7 @@ class Container { secretFiles: Mapping = new {} args: Listing = new {} capAdd: Listing = new {} + readOnlyRoot: Boolean = false devices: Listing = new {} extraHosts: Listing = new {} hostNetwork: Boolean = false diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index 7e663a4d279a966eb37b17066dbb0519677c7c98..2466309442cbea7b5b22d32101276f07e00e6fc8 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -24,6 +24,7 @@ idp-logout: /auth/file/sign-out idp-login-t: with snow globe html-head: + js-browser: /.static/mirrors.js [/] /w diff --git a/service/copyparty/static/copyparty.css b/service/copyparty/static/copyparty.css index 7d88b7f6d3b5b732d5d6efea3470be73b1b2cd5e..857e385e68f10128d449281e7e100cc715e534fc 100644 --- a/service/copyparty/static/copyparty.css +++ b/service/copyparty/static/copyparty.css @@ -7,3 +7,19 @@ --font-serif: "Name Sans", serif; --font-mono: "Name Mono", monospace; } +.mirror-anonymous #files tr > :first-child, +.mirror-anonymous #tree, +.mirror-anonymous #entree, +.mirror-anonymous #opa_auc, +.mirror-anonymous #op_player, +.mirror-anonymous #widget { + display: none !important; +} +.mirror-anonymous #path { + display: inline-block !important; +} +.mirror-anonymous #wrap { + margin-left: 1.5em !important; + padding-bottom: 1.5em; + overflow-x: auto; +} diff --git a/service/copyparty/static/mirrors.js b/service/copyparty/static/mirrors.js new file mode 100644 index 0000000000000000000000000000000000000000..37938be439d80f0c2b23f2879879920cce831808 --- /dev/null +++ b/service/copyparty/static/mirrors.js @@ -0,0 +1,12 @@ +(function () { + var render = treectl.gentab; + treectl.gentab = function (top, listing) { + var anonymous = (listing.acct === undefined ? acct : listing.acct) === "*" && /^\/mirrors(?:\/|$)/.test(top); + document.body.classList.toggle("mirror-anonymous", anonymous); + if (anonymous) + read_dsort("-ts"); + return render.call(this, top, listing); + }; + if (ls0 !== null) + treectl.gentab(get_evpath(), ls0); +})(); diff --git a/service/zig-mirror/build/mirror.py b/service/zig-mirror/build/mirror.py index ff9c7dba03e9025d2b3ae9bc9b577d9bae5eaccb..c2eb485e097ced645166d09d62d29c8c491f9155 100644 --- a/service/zig-mirror/build/mirror.py +++ b/service/zig-mirror/build/mirror.py @@ -1,8 +1,15 @@ #!/usr/bin/env python3 import argparse +import base64 +import ctypes +import ctypes.util +import contextlib +import datetime import email.utils import fcntl +import functools import hashlib +import html from html.parser import HTMLParser import http.server import json @@ -10,6 +17,7 @@ import os from pathlib import Path import re import shutil +import stat import subprocess import tempfile import threading @@ -24,12 +32,178 @@ PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' OFFICIAL = 'https://ziglang.org' NEKTRO = 'http://mirrors.nektro.net/zig/' MAX_ARTIFACT = 2 * 1024**3 +_index_cache = (float('-inf'), {}) +_index_lock = threading.Lock() class Unavailable(Exception): pass +@functools.lru_cache(maxsize=1) +def comment_verifier(): + name = ctypes.util.find_library('sodium') + if not name: + raise Unavailable('Minisign comment verification unavailable') + library = ctypes.CDLL(name) + if library.sodium_init() < 0: + raise Unavailable('Minisign comment verification unavailable') + verify = library.crypto_sign_verify_detached + verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p] + verify.restype = ctypes.c_int + return verify + + +def metadata_open(path, final_flags=os.O_RDONLY): + path = path.absolute() + descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0)) + try: + for index, part in enumerate(path.parts[1:]): + flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0) + if index < len(path.parts) - 2: + flags |= os.O_DIRECTORY + next_descriptor = os.open(part, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = next_descriptor + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def signed_time(signature): + with os.fdopen(metadata_open(signature), 'rb') as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise Unavailable('signature is not a regular file') + body = stream.read(4097) + lines = body.splitlines() + try: + if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '): + raise ValueError() + key = base64.b64decode(PUBLIC_KEY, validate=True) + sig = base64.b64decode(lines[1], validate=True) + global_sig = base64.b64decode(lines[3], validate=True) + comment = lines[2][len(b'trusted comment: '):] + if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]: + raise ValueError() + # Minisign authenticates the artifact signature followed by the exact trusted comment. + message = sig[10:] + comment + if comment_verifier()(global_sig, message, len(message), key[10:]) != 0: + raise ValueError() + match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment) + if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'): + raise ValueError() + return int(match[1]) * 1_000_000_000 + except (ValueError, IndexError) as error: + raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error + + +@functools.lru_cache(maxsize=128) +def stable_time(version): + try: + entries = index() + if version not in entries: + entries = index(refresh=True) + date = datetime.date.fromisoformat(entries[version]['date']) + return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000 + except (KeyError, ValueError) as error: + raise Unavailable('official release date unavailable: ' + version) from error + + +def artifact_time(artifact): + version = filename(artifact.name) + if '-dev.' in version or version == '0.15.0': + return signed_time(Path(str(artifact) + '.minisig')) + return stable_time(version) + + +def children(directory): + descriptor = metadata_open(directory) + try: + with os.scandir(descriptor) as entries: + return [(directory / entry.name, entry.stat(follow_symlinks=False)) + for entry in entries if not entry.name.startswith('.')] + finally: + os.close(descriptor) + + +def set_mtime(path, timestamp): + descriptor = metadata_open(path) + try: + before = os.fstat(descriptor) + if before.st_mtime_ns != timestamp: + os.utime(descriptor, ns=(before.st_atime_ns, timestamp)) + finally: + os.close(descriptor) + + +def file_state(value): + return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')} + + +def version_readme(version, artifacts, development=False): + systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows', + 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'} + source = [] + targets = {} + + def link(artifact): + name = artifact.name + url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/') + label = 'tar.xz' if name.endswith('.tar.xz') else 'zip' + result = f'[{label}]({url})' + signature = Path(str(artifact) + '.minisig') + if signature.is_file() and not signature.is_symlink(): + result += f' 🔐' + return result + + for artifact in sorted(artifacts): + name = artifact.name + extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip' + target = name[len('zig'): -len('-' + version + extension)] + if target in ('', '-bootstrap'): + source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact)) + continue + if target == '-win64': + architecture, system = 'x86_64', 'windows' + else: + first, second = target.removeprefix('-').split('-') + system, architecture = (first, second) if first in systems else (second, first) + targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact)) + columns = list(systems) + columns += sorted({system for row in targets.values() for system in row} - set(columns)) + heading = '
'.join(source) or 'Architecture' + lines = [f'### Zig {version}', ''] + if not artifacts and development: + return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode() + if artifacts and '-dev.' not in version: + release = index().get(version, {}) + for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]: + url = release.get(key) + if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url): + lines.append(f'- [{label}]({url})') + if lines[-1]: + lines.append('') + if development: + lines += [f'[Development builds](/mirrors/zig/{version}/dev/)', ''] + elif '-dev.' in version: + lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', ''] + if not artifacts: + return '\n'.join(lines).encode() + lines += ['
', '', + '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |', + '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |'] + priority = ('x86_64', 'aarch64', 'x86', 'arm') + for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)): + lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |') + lines += ['', '
'] + if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts): + lines += ['', 'Verify your build signatures. The 🔐 links open the Minisign signatures.', ''] + else: + lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', ''] + return '\n'.join(lines).encode() + + def filename(name): match = PATTERN.fullmatch(name) if len(name) > 240 or not match: @@ -72,9 +246,14 @@ def fetch(url, target, limit): raise Unavailable('upstream download failed') from error -def index(): - with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: - return json.load(response) +def index(refresh=False): + global _index_cache + with _index_lock: + if refresh or time.monotonic() - _index_cache[0] >= 3600: + with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: + entries = json.load(response) + _index_cache = (time.monotonic(), entries) + return _index_cache[1] def digest(path): @@ -87,7 +266,10 @@ def digest(path): class Mirror: def __init__(self, root): - self.root = Path(root) + self.root = Path(root).absolute() + if self.root.is_symlink(): + raise ValueError('mirror root is a symlink') + self.root = self.root.resolve() self.root.mkdir(parents=True, exist_ok=True) self.slots = threading.BoundedSemaphore(4) self.refresh_lock = threading.Lock() @@ -98,6 +280,50 @@ class Mirror: raise ValueError('artifact escapes mirror') return path + def lock_directory(self): + locks = self.root / '.mirror-locks' + descriptor = metadata_open(self.root) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + timestamp = os.fstat(descriptor).st_mtime_ns + try: + locks.mkdir() + except FileExistsError: + pass + else: + set_mtime(self.root, timestamp) + finally: + os.close(descriptor) + descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY) + os.close(descriptor) + return locks + + @contextlib.contextmanager + def publishing(self): + locks = self.lock_directory() + with (locks / '.timestamps').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + def parent_times(self, artifact): + def latest(directory): + values = [] + for path, value in children(directory): + if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path: + values.append(value.st_mtime_ns) + elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'): + values.append(latest(path)) + return max(values, default=0) + + directory = artifact.parent + while True: + timestamp = latest(directory) + if timestamp: + set_mtime(directory, timestamp) + if directory == self.root: + break + directory = directory.parent + def ensure(self, name, source=None, checksum=None): target = self.path(name) artifact_name = name.removesuffix('.minisig') @@ -109,8 +335,7 @@ class Mirror: try: if shutil.disk_usage(self.root).free < 16 * 1024**3: raise Unavailable('mirror free-space reserve reached') - locks = self.root / '.mirror-locks' - locks.mkdir(exist_ok=True) + locks = self.lock_directory() with (locks / artifact_name).open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) if target.is_file(): @@ -123,8 +348,7 @@ class Mirror: and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) if not checksum: raise Unavailable('unsigned artifact has no official SHA-256') - artifact.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: pending = Path(temporary) / artifact_name signature = Path(str(pending) + '.minisig') url = source or upstream(artifact_name) @@ -137,21 +361,216 @@ class Mirror: capture_output=True) if result.returncode: raise Unavailable('artifact signature verification failed') + timestamp = artifact_time(pending) os.chmod(pending, 0o664) - os.replace(pending, artifact) + set_mtime(pending, timestamp) if signature.exists(): os.chmod(signature, 0o664) - os.replace(signature, Path(str(artifact) + '.minisig')) + set_mtime(signature, timestamp) + with self.publishing(): + self.path(name) + artifact.parent.mkdir(parents=True, exist_ok=True) + os.replace(pending, artifact) + if signature.exists(): + os.replace(signature, Path(str(artifact) + '.minisig')) + try: + self.write_readmes(artifact.parent) + finally: + self.parent_times(artifact) if name.endswith('.minisig') and not target.exists(): - with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: + with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: pending = Path(temporary) / name fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) os.chmod(pending, 0o664) - os.replace(pending, target) + set_mtime(pending, artifact.stat().st_mtime_ns) + with self.publishing(): + os.replace(pending, target) + try: + self.write_readmes(artifact.parent) + finally: + self.parent_times(artifact) return target finally: self.slots.release() + def readme_change(self, directory, artifacts, content=None): + path = directory / 'preadme.md' + if content is None: + content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir()) + before = None + previous = b'' + if path.exists() or path.is_symlink(): + descriptor = metadata_open(path) + with os.fdopen(descriptor, 'rb') as stream: + value = os.fstat(stream.fileno()) + if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024: + raise Unavailable('version readme is not a small regular file') + before = file_state(value) + previous = stream.read(512 * 1024 + 1) + required = {artifact.name for artifact in artifacts} + required |= {artifact.name + '.minisig' for artifact in artifacts + if Path(str(artifact) + '.minisig').is_file()} + present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1]) + for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())} + linked = {name for name in present if PATTERN.fullmatch(name)} + introduction = content.split(b''),text.index('This historical release')) + self.assertNotIn('https://file.paperclover.net',text) + empty=mirror.version_readme('0.17.0',[],True).decode() + self.assertIn('[development builds]',empty);self.assertNotIn('| Linux',empty);self.assertNotIn('role="region"',empty) + if __name__ == '__main__': unittest.main() diff --git a/tools/zig-mirror.md b/tools/zig-mirror.md index a461acab5d09d88d1e9faf0a15d96ecdd2ac1dd9..463d69cbb86c82de8c932766066673a773f6750c 100644 --- a/tools/zig-mirror.md +++ b/tools/zig-mirror.md @@ -23,6 +23,64 @@ free-space reserve. Completed archives and signatures publish atomically after SHA-256 and/or Zig Minisign verification. Existing archive files need the one-time `verify` operation before first public exposure. +## Release dates and navigation + +Indexed stable releases use the official download index's date at UTC midnight. +The retracted 0.15.0 artifacts use authenticated signing time instead: the +[official retraction](https://github.com/ziglang/www.ziglang.org/commit/1be0f80b15d256d42411a21f10e208f9b269a5e2) +removed that release from the index. Other unknown stable releases fail closed. +Development archives use the build timestamp from their Minisign trusted comment; +its global Ed25519 signature is checked against Zig's pinned key before the date +is accepted. The signature file receives the archive's date. Each visible folder, +including the mirror root, uses its newest contained archive date. Empty folders, +README files and hidden operational/history files do not contribute to that date. +Every cache miss, periodic refresh and backfill uses this same publication path. + +Version folders contain an architecture/OS download table built only from present +canonical artifacts and signatures. Released versions list Release Notes, +Language Reference and Standard Library Documentation below their heading only +when the official index publishes the corresponding `notes`, `docs` or `stdDocs` +URL. No documentation URL is synthesized or probed per artifact. The index has +one process-wide, lock-protected snapshot, expiring after an hour; periodic/manual +refresh bypasses it, as does a cache miss for an unindexed stable version's date. +Refresh also updates the current release/build navigation when archives are +already cached, preserving artifact and folder dates. + +Base versions link to their development index; a base folder with only development +builds says “this release only has development builds out.” Each development +version links back to its index, whose heading leaves the native directory listing +to show the builds. Complete existing relative-link tables are retained when +their introduction matches the published links and navigation; incomplete or +production-only links are replaced. The owner's root `preadme.md` and hidden +history remain untouched. + +Existing verified archives need a one-time metadata reconciliation. It reads +signature comments and file metadata, not archive bodies, and preserves atime, +ownership and modes. Preview it first: + +```sh +python3 /app/mirror.py reconcile > /alloc/metadata-proposal.json +python3 /app/mirror.py reconcile --apply --journal /alloc/private-metadata-undo.jsonl +python3 /app/mirror.py reconcile > /alloc/metadata-after.json +``` + +Retain proposal and undo files privately outside `/mirror`; never publish them +through Copyparty. Apply fails closed on missing dates, unsafe paths or changed +files. The journal is created exclusively with mode0600 and fsynced before any +change. Its first record retains old metadata and replaced public README bytes; +a second record captures completed inode/metadata states. A journal without the +completion record requires recovery review. A guarded one-time undo must verify +those completed identities and generated README bytes before restoring old +metadata/docs; later writes must cause it to stop. Keep that recovery helper with +the private deployment evidence, rather than adding an automatic rollback hook. + +The worker has only `FOWNER` beyond its normal capability set, allowing explicit +historic dates on the existing UID3000-owned NAS folders without changing their +ownership. Its root filesystem is read-only and its only writable external mount +is the canonical Zig subtree. Download temporaries and locks stay beneath hidden +`.mirror-locks`; they cannot change visible folder dates. Fresh worker previews +start empty, disable polling and fetch public artifacts only when requested. + ## Production cutover The 2026-10-17 production inventory found 289,931,243,359 bytes / 10,063 files