| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | import base64 |
| 4 | import ctypes |
| 5 | import ctypes.util |
| 6 | import contextlib |
| 7 | import datetime |
| 8 | import email.utils |
| 9 | import fcntl |
| 10 | import functools |
| 11 | import hashlib |
| 12 | import html |
| 13 | from html.parser import HTMLParser |
| 14 | import http.server |
| 15 | import json |
| 16 | import os |
| 17 | from pathlib import Path |
| 18 | import re |
| 19 | import shutil |
| 20 | import stat |
| 21 | import subprocess |
| 22 | import tempfile |
| 23 | import threading |
| 24 | import time |
| 25 | import urllib.error |
| 26 | import urllib.parse |
| 27 | import urllib.request |
| 28 | |
| 29 | PATTERN = re.compile(r'^zig(?:|-bootstrap|-win64|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') |
| 30 | VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') |
| 31 | PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' |
| 32 | OFFICIAL = 'https://ziglang.org' |
| 33 | NEKTRO = 'http://mirrors.nektro.net/zig/' |
| 34 | SIGNATURE_NOTE = '<span style="font-style:italic;color:orange">Don\'t forget to verify your build signatures!</span> <span style="opacity:0.7">&nbsp;(the 🔐 is a link to the minisig)' |
| 35 | MAX_ARTIFACT = 2 * 1024**3 |
| 36 | _index_cache = (float('-inf'), {}) |
| 37 | _index_lock = threading.Lock() |
| 38 | |
| 39 | |
| 40 | class Unavailable(Exception): |
| 41 | pass |
| 42 | |
| 43 | |
| 44 | @functools.lru_cache(maxsize=1) |
| 45 | def comment_verifier(): |
| 46 | name = ctypes.util.find_library('sodium') |
| 47 | if not name: |
| 48 | raise Unavailable('Minisign comment verification unavailable') |
| 49 | library = ctypes.CDLL(name) |
| 50 | if library.sodium_init() < 0: |
| 51 | raise Unavailable('Minisign comment verification unavailable') |
| 52 | verify = library.crypto_sign_verify_detached |
| 53 | verify.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulonglong, ctypes.c_char_p] |
| 54 | verify.restype = ctypes.c_int |
| 55 | return verify |
| 56 | |
| 57 | |
| 58 | def metadata_open(path, final_flags=os.O_RDONLY): |
| 59 | path = path.absolute() |
| 60 | descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOATIME', 0)) |
| 61 | try: |
| 62 | for index, part in enumerate(path.parts[1:]): |
| 63 | flags = (final_flags if index == len(path.parts) - 2 else os.O_RDONLY) | os.O_NOFOLLOW | os.O_NONBLOCK | getattr(os, 'O_NOATIME', 0) |
| 64 | if index < len(path.parts) - 2: |
| 65 | flags |= os.O_DIRECTORY |
| 66 | next_descriptor = os.open(part, flags, dir_fd=descriptor) |
| 67 | os.close(descriptor) |
| 68 | descriptor = next_descriptor |
| 69 | return descriptor |
| 70 | except BaseException: |
| 71 | os.close(descriptor) |
| 72 | raise |
| 73 | |
| 74 | |
| 75 | def signed_time(signature): |
| 76 | with os.fdopen(metadata_open(signature), 'rb') as stream: |
| 77 | if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): |
| 78 | raise Unavailable('signature is not a regular file') |
| 79 | body = stream.read(4097) |
| 80 | lines = body.splitlines() |
| 81 | try: |
| 82 | if len(body) > 4096 or len(lines) != 4 or not lines[2].startswith(b'trusted comment: '): |
| 83 | raise ValueError() |
| 84 | key = base64.b64decode(PUBLIC_KEY, validate=True) |
| 85 | sig = base64.b64decode(lines[1], validate=True) |
| 86 | global_sig = base64.b64decode(lines[3], validate=True) |
| 87 | comment = lines[2][len(b'trusted comment: '):] |
| 88 | if len(key) != 42 or len(sig) != 74 or len(global_sig) != 64 or sig[:2] not in (b'Ed', b'ED') or sig[2:10] != key[2:10]: |
| 89 | raise ValueError() |
| 90 | # Minisign authenticates the artifact signature followed by the exact trusted comment. |
| 91 | message = sig[10:] + comment |
| 92 | if comment_verifier()(global_sig, message, len(message), key[10:]) != 0: |
| 93 | raise ValueError() |
| 94 | match = re.fullmatch(rb'timestamp:(\d+)\tfile:([^\t]+)(?:\thashed)?', comment) |
| 95 | if not match or os.fsdecode(match[2]) != signature.name.removesuffix('.minisig'): |
| 96 | raise ValueError() |
| 97 | return int(match[1]) * 1_000_000_000 |
| 98 | except (ValueError, IndexError) as error: |
| 99 | raise Unavailable('authenticated build timestamp unavailable: ' + signature.name) from error |
| 100 | |
| 101 | |
| 102 | @functools.lru_cache(maxsize=128) |
| 103 | def stable_time(version): |
| 104 | try: |
| 105 | entries = index() |
| 106 | if version not in entries: |
| 107 | entries = index(refresh=True) |
| 108 | date = datetime.date.fromisoformat(entries[version]['date']) |
| 109 | return int(datetime.datetime.combine(date, datetime.time(), datetime.timezone.utc).timestamp()) * 1_000_000_000 |
| 110 | except (KeyError, ValueError) as error: |
| 111 | raise Unavailable('official release date unavailable: ' + version) from error |
| 112 | |
| 113 | |
| 114 | def artifact_time(artifact): |
| 115 | version = filename(artifact.name) |
| 116 | if '-dev.' in version or version == '0.15.0': |
| 117 | return signed_time(Path(str(artifact) + '.minisig')) |
| 118 | return stable_time(version) |
| 119 | |
| 120 | |
| 121 | def children(directory): |
| 122 | descriptor = metadata_open(directory) |
| 123 | try: |
| 124 | with os.scandir(descriptor) as entries: |
| 125 | return [(directory / entry.name, entry.stat(follow_symlinks=False)) |
| 126 | for entry in entries if not entry.name.startswith('.')] |
| 127 | finally: |
| 128 | os.close(descriptor) |
| 129 | |
| 130 | |
| 131 | def set_mtime(path, timestamp): |
| 132 | descriptor = metadata_open(path) |
| 133 | try: |
| 134 | before = os.fstat(descriptor) |
| 135 | if before.st_mtime_ns != timestamp: |
| 136 | os.utime(descriptor, ns=(before.st_atime_ns, timestamp)) |
| 137 | finally: |
| 138 | os.close(descriptor) |
| 139 | |
| 140 | |
| 141 | def file_state(value): |
| 142 | return {key: getattr(value, 'st_' + key) for key in ('dev', 'ino', 'mode', 'uid', 'gid', 'size', 'atime_ns', 'mtime_ns')} |
| 143 | |
| 144 | |
| 145 | def version_readme(version, artifacts, development=False): |
| 146 | systems = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows', |
| 147 | 'freebsd': 'FreeBSD', 'netbsd': 'NetBSD', 'openbsd': 'OpenBSD'} |
| 148 | source = [] |
| 149 | targets = {} |
| 150 | |
| 151 | def link(artifact): |
| 152 | name = artifact.name |
| 153 | url = '/mirrors/zig/' + urllib.parse.quote(str(relative(name)), safe='/') |
| 154 | label = 'tar.xz' if name.endswith('.tar.xz') else 'zip' |
| 155 | result = f'[{label}]({url})' |
| 156 | signature = Path(str(artifact) + '.minisig') |
| 157 | if signature.is_file() and not signature.is_symlink(): |
| 158 | result += f' <a href="{url}.minisig" style="background:none!important;border:none!important;font-size:13px;margin-left:-5px;margin-right:-3px" aria-label="Signature for {html.escape(name, quote=True)}">&#128272;</a>' |
| 159 | return result |
| 160 | |
| 161 | for artifact in sorted(artifacts): |
| 162 | name = artifact.name |
| 163 | extension = '.tar.xz' if name.endswith('.tar.xz') else '.zip' |
| 164 | target = name[len('zig'): -len('-' + version + extension)] |
| 165 | if target in ('', '-bootstrap'): |
| 166 | source.append(('Source' if not target else 'Bootstrap') + ' ' + link(artifact)) |
| 167 | continue |
| 168 | if target == '-win64': |
| 169 | architecture, system = 'x86_64', 'windows' |
| 170 | else: |
| 171 | first, second = target.removeprefix('-').split('-') |
| 172 | system, architecture = (first, second) if first in systems else (second, first) |
| 173 | targets.setdefault(architecture, {}).setdefault(system, []).append(link(artifact)) |
| 174 | columns = list(systems) |
| 175 | columns += sorted({system for row in targets.values() for system in row} - set(columns)) |
| 176 | heading = '<br>'.join(source) or 'Architecture' |
| 177 | lines = [f'### Zig {version}', ''] |
| 178 | if not artifacts and development: |
| 179 | return '\n'.join(lines + [f'this release only has [development builds](/mirrors/zig/{version}/dev/) out.', '']).encode() |
| 180 | if artifacts and '-dev.' not in version: |
| 181 | release = index().get(version, {}) |
| 182 | for key, label in [('notes', 'Release Notes'), ('docs', 'Language Reference'), ('stdDocs', 'Standard Library Documentation')]: |
| 183 | url = release.get(key) |
| 184 | if isinstance(url, str) and re.fullmatch(re.escape(OFFICIAL) + r'/[^\s()<>\[\]]+', url): |
| 185 | lines.append(f'- [{label}]({url})') |
| 186 | if lines[-1]: |
| 187 | lines.append('') |
| 188 | if '-dev.' in version: |
| 189 | lines += [f'[All development builds](/mirrors/zig/{version.split("-")[0]}/dev/)', ''] |
| 190 | if not artifacts: |
| 191 | return '\n'.join(lines).encode() |
| 192 | lines += ['<div role="region" aria-label="Zig downloads" tabindex="0" style="max-width:100%;overflow-x:auto;white-space:nowrap;word-break:normal;overflow-wrap:normal">', '', |
| 193 | '| ' + heading + ' | ' + ' | '.join(systems.get(system, system) for system in columns) + ' |', |
| 194 | '| ' + ' | '.join(['---'] * (len(columns) + 1)) + ' |'] |
| 195 | priority = ('x86_64', 'aarch64', 'x86', 'arm') |
| 196 | for architecture in sorted(targets, key=lambda value: (priority.index(value) if value in priority else len(priority), value)): |
| 197 | lines.append('| ' + architecture + ' | ' + ' | '.join(' '.join(targets[architecture].get(system, [])) or '—' for system in columns) + ' |') |
| 198 | lines += ['', '</div>'] |
| 199 | if any(Path(str(artifact) + '.minisig').is_file() for artifact in artifacts): |
| 200 | lines += ['', SIGNATURE_NOTE, ''] |
| 201 | else: |
| 202 | lines += ['', 'This historical release has no Minisign signatures. Verify its [official SHA-256](https://ziglang.org/download/index.json).', ''] |
| 203 | return '\n'.join(lines).encode() |
| 204 | |
| 205 | |
| 206 | def filename(name): |
| 207 | match = PATTERN.fullmatch(name) |
| 208 | if len(name) > 240 or not match: |
| 209 | raise ValueError('invalid Zig artifact name') |
| 210 | if name.startswith('zig-win64-') and name not in {'zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip'}: |
| 211 | raise ValueError('invalid legacy Zig artifact name') |
| 212 | version = match[1] |
| 213 | major, minor, patch = version.split('-')[0].split('.') |
| 214 | if any(part != str(int(part)) for part in (major, minor, patch)): |
| 215 | raise ValueError('invalid Zig version') |
| 216 | return version |
| 217 | |
| 218 | |
| 219 | def relative(name): |
| 220 | version = filename(name) |
| 221 | if '-dev.' in version: |
| 222 | return Path(version.split('-')[0]) / 'dev' / version / name |
| 223 | return Path(version) / name |
| 224 | |
| 225 | |
| 226 | def upstream(name): |
| 227 | version = filename(name) |
| 228 | prefix = '/builds/' if '-dev.' in version else '/download/' + version + '/' |
| 229 | return OFFICIAL + prefix + urllib.parse.quote(name) |
| 230 | |
| 231 | |
| 232 | def fetch(url, target, limit): |
| 233 | request = urllib.request.Request(url, headers={'User-Agent': 'snowglobe-zig-mirror/1'}) |
| 234 | try: |
| 235 | with urllib.request.urlopen(request, timeout=180) as response, target.open('wb') as output: |
| 236 | count = 0 |
| 237 | while chunk := response.read(1024 * 1024): |
| 238 | count += len(chunk) |
| 239 | if count > limit: |
| 240 | raise Unavailable('upstream artifact exceeds size limit') |
| 241 | output.write(chunk) |
| 242 | output.flush() |
| 243 | os.fsync(output.fileno()) |
| 244 | except (OSError, urllib.error.URLError) as error: |
| 245 | raise Unavailable('upstream download failed') from error |
| 246 | |
| 247 | |
| 248 | def index(refresh=False): |
| 249 | global _index_cache |
| 250 | with _index_lock: |
| 251 | if refresh or time.monotonic() - _index_cache[0] >= 3600: |
| 252 | with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response: |
| 253 | entries = json.load(response) |
| 254 | _index_cache = (time.monotonic(), entries) |
| 255 | return _index_cache[1] |
| 256 | |
| 257 | |
| 258 | def digest(path): |
| 259 | checksum = hashlib.sha256() |
| 260 | with path.open('rb') as stream: |
| 261 | while chunk := stream.read(1024 * 1024): |
| 262 | checksum.update(chunk) |
| 263 | return checksum.hexdigest() |
| 264 | |
| 265 | |
| 266 | class Mirror: |
| 267 | def __init__(self, root): |
| 268 | self.root = Path(root).absolute() |
| 269 | if self.root.is_symlink(): |
| 270 | raise ValueError('mirror root is a symlink') |
| 271 | self.root = self.root.resolve() |
| 272 | self.root.mkdir(parents=True, exist_ok=True) |
| 273 | self.slots = threading.BoundedSemaphore(4) |
| 274 | self.refresh_lock = threading.Lock() |
| 275 | |
| 276 | def path(self, name): |
| 277 | path = self.root / relative(name) |
| 278 | if not path.resolve().is_relative_to(self.root.resolve()): |
| 279 | raise ValueError('artifact escapes mirror') |
| 280 | return path |
| 281 | |
| 282 | def lock_directory(self): |
| 283 | locks = self.root / '.mirror-locks' |
| 284 | descriptor = metadata_open(self.root) |
| 285 | try: |
| 286 | fcntl.flock(descriptor, fcntl.LOCK_EX) |
| 287 | timestamp = os.fstat(descriptor).st_mtime_ns |
| 288 | try: |
| 289 | locks.mkdir() |
| 290 | except FileExistsError: |
| 291 | pass |
| 292 | else: |
| 293 | set_mtime(self.root, timestamp) |
| 294 | finally: |
| 295 | os.close(descriptor) |
| 296 | descriptor = metadata_open(locks, os.O_RDONLY | os.O_DIRECTORY) |
| 297 | os.close(descriptor) |
| 298 | return locks |
| 299 | |
| 300 | @contextlib.contextmanager |
| 301 | def publishing(self): |
| 302 | locks = self.lock_directory() |
| 303 | with (locks / '.timestamps').open('a') as lock: |
| 304 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 305 | yield |
| 306 | |
| 307 | def parent_times(self, artifact): |
| 308 | def latest(directory): |
| 309 | values = [] |
| 310 | for path, value in children(directory): |
| 311 | if stat.S_ISREG(value.st_mode) and not path.name.endswith('.minisig') and PATTERN.fullmatch(path.name) and self.path(path.name) == path: |
| 312 | values.append(value.st_mtime_ns) |
| 313 | elif stat.S_ISDIR(value.st_mode) and (VERSION.fullmatch(path.name) or path.name == 'dev'): |
| 314 | values.append(latest(path)) |
| 315 | return max(values, default=0) |
| 316 | |
| 317 | directory = artifact.parent |
| 318 | while True: |
| 319 | timestamp = latest(directory) |
| 320 | if timestamp: |
| 321 | set_mtime(directory, timestamp) |
| 322 | if directory == self.root: |
| 323 | break |
| 324 | directory = directory.parent |
| 325 | |
| 326 | def ensure(self, name, source=None, checksum=None): |
| 327 | target = self.path(name) |
| 328 | artifact_name = name.removesuffix('.minisig') |
| 329 | artifact = self.path(artifact_name) |
| 330 | if target.is_file(): |
| 331 | return target |
| 332 | if not self.slots.acquire(timeout=1): |
| 333 | raise BlockingIOError('mirror download capacity reached') |
| 334 | try: |
| 335 | if shutil.disk_usage(self.root).free < 16 * 1024**3: |
| 336 | raise Unavailable('mirror free-space reserve reached') |
| 337 | locks = self.lock_directory() |
| 338 | with (locks / artifact_name).open('a') as lock: |
| 339 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 340 | if target.is_file(): |
| 341 | return target |
| 342 | version = filename(artifact_name) |
| 343 | signed = '-dev.' in version or tuple(map(int, version.split('.'))) >= (0, 6, 0) |
| 344 | if not signed and not checksum: |
| 345 | checksum = next((item['shasum'] for item in index().get(version, {}).values() |
| 346 | if isinstance(item, dict) and 'tarball' in item |
| 347 | and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) |
| 348 | if not checksum: |
| 349 | raise Unavailable('unsigned artifact has no official SHA-256') |
| 350 | with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: |
| 351 | pending = Path(temporary) / artifact_name |
| 352 | signature = Path(str(pending) + '.minisig') |
| 353 | url = source or upstream(artifact_name) |
| 354 | fetch(url, pending, MAX_ARTIFACT) |
| 355 | if checksum and digest(pending) != checksum: |
| 356 | raise Unavailable('artifact SHA-256 mismatch') |
| 357 | if signed: |
| 358 | fetch(url + '.minisig', signature, 4096) |
| 359 | result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], |
| 360 | capture_output=True) |
| 361 | if result.returncode: |
| 362 | raise Unavailable('artifact signature verification failed') |
| 363 | timestamp = artifact_time(pending) |
| 364 | os.chmod(pending, 0o664) |
| 365 | set_mtime(pending, timestamp) |
| 366 | if signature.exists(): |
| 367 | os.chmod(signature, 0o664) |
| 368 | set_mtime(signature, timestamp) |
| 369 | with self.publishing(): |
| 370 | self.path(name) |
| 371 | artifact.parent.mkdir(parents=True, exist_ok=True) |
| 372 | os.replace(pending, artifact) |
| 373 | if signature.exists(): |
| 374 | os.replace(signature, Path(str(artifact) + '.minisig')) |
| 375 | try: |
| 376 | self.write_readmes(artifact.parent) |
| 377 | finally: |
| 378 | self.parent_times(artifact) |
| 379 | if name.endswith('.minisig') and not target.exists(): |
| 380 | with tempfile.TemporaryDirectory(prefix='.download-', dir=locks) as temporary: |
| 381 | pending = Path(temporary) / name |
| 382 | fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096) |
| 383 | os.chmod(pending, 0o664) |
| 384 | set_mtime(pending, artifact.stat().st_mtime_ns) |
| 385 | with self.publishing(): |
| 386 | os.replace(pending, target) |
| 387 | try: |
| 388 | self.write_readmes(artifact.parent) |
| 389 | finally: |
| 390 | self.parent_times(artifact) |
| 391 | return target |
| 392 | finally: |
| 393 | self.slots.release() |
| 394 | |
| 395 | def readme_change(self, directory, artifacts, content=None): |
| 396 | path = directory / 'preadme.md' |
| 397 | if content is None: |
| 398 | content = version_readme(directory.name, artifacts, (directory / 'dev').is_dir()) |
| 399 | before = None |
| 400 | previous = b'' |
| 401 | if path.exists() or path.is_symlink(): |
| 402 | descriptor = metadata_open(path) |
| 403 | with os.fdopen(descriptor, 'rb') as stream: |
| 404 | value = os.fstat(stream.fileno()) |
| 405 | if not stat.S_ISREG(value.st_mode) or value.st_size > 512 * 1024: |
| 406 | raise Unavailable('version readme is not a small regular file') |
| 407 | before = file_state(value) |
| 408 | previous = stream.read(512 * 1024 + 1) |
| 409 | required = {artifact.name for artifact in artifacts} |
| 410 | required |= {artifact.name + '.minisig' for artifact in artifacts |
| 411 | if Path(str(artifact) + '.minisig').is_file()} |
| 412 | present = {urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/', 1)[-1]) |
| 413 | for url in re.findall(r'''(?:href=["']|\]\()([^"')\s]+)''', previous.decode())} |
| 414 | linked = {name for name in present if PATTERN.fullmatch(name)} |
| 415 | introduction = content.split(b'<div ', 1)[0] |
| 416 | preserved = previous |
| 417 | if '-dev.' not in directory.name: |
| 418 | preserved = preserved.replace(f'\n[Development builds](/mirrors/zig/{directory.name}/dev/)\n'.encode(), b'') |
| 419 | if artifacts and linked == required and preserved.startswith(introduction) and b'https://file.paperclover.net' not in preserved: |
| 420 | content = preserved.replace( |
| 421 | b'\n<span style="font-style:italic;color:orange">Verify your build signatures.</span> <span style="opacity:0.7">The &#128272; links open the Minisign signatures.</span>\n', |
| 422 | b'\n' + SIGNATURE_NOTE.encode() + b'\n', 1) |
| 423 | if previous == content: |
| 424 | return None |
| 425 | return {'path': str(path.relative_to(self.root)), 'before': before, |
| 426 | 'previous': base64.b64encode(previous).decode(), 'content': content.decode()} |
| 427 | |
| 428 | def artifacts(self, directory): |
| 429 | return [path for path, value in children(directory) |
| 430 | if stat.S_ISREG(value.st_mode) and PATTERN.fullmatch(path.name) |
| 431 | and not path.name.endswith('.minisig') and self.path(path.name) == path] |
| 432 | |
| 433 | def directory_readme(self, directory): |
| 434 | if directory.name == 'dev': |
| 435 | content = f'### Zig {directory.parent.name} development builds\n'.encode() |
| 436 | return self.readme_change(directory, [], content) |
| 437 | artifacts = self.artifacts(directory) |
| 438 | if artifacts or (directory / 'dev').is_dir(): |
| 439 | return self.readme_change(directory, artifacts) |
| 440 | |
| 441 | def write_readmes(self, directory): |
| 442 | directories = [directory] |
| 443 | if directory.parent.name == 'dev': |
| 444 | directories += [directory.parent, directory.parent.parent] |
| 445 | for current in directories: |
| 446 | change = self.directory_readme(current) |
| 447 | if change: |
| 448 | self.apply_readme(change) |
| 449 | |
| 450 | def apply_readme(self, change): |
| 451 | path = self.root / change['path'] |
| 452 | if change['before'] is None: |
| 453 | parent = metadata_open(path.parent) |
| 454 | try: |
| 455 | descriptor = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o664, dir_fd=parent) |
| 456 | os.fchown(descriptor, -1, os.fstat(parent).st_gid) |
| 457 | finally: |
| 458 | os.close(parent) |
| 459 | before = os.fstat(descriptor) |
| 460 | else: |
| 461 | descriptor = metadata_open(path, os.O_WRONLY) |
| 462 | before = os.fstat(descriptor) |
| 463 | if any(file_state(before)[key] != value for key, value in change['before'].items() if key != 'atime_ns'): |
| 464 | os.close(descriptor) |
| 465 | raise Unavailable('version readme changed since plan') |
| 466 | try: |
| 467 | os.ftruncate(descriptor, 0) |
| 468 | with os.fdopen(os.dup(descriptor), 'wb') as output: |
| 469 | output.write(change['content'].encode()) |
| 470 | output.flush() |
| 471 | os.fsync(output.fileno()) |
| 472 | os.utime(descriptor, ns=(before.st_atime_ns, before.st_mtime_ns)) |
| 473 | finally: |
| 474 | os.close(descriptor) |
| 475 | |
| 476 | def metadata_plan(self): |
| 477 | files = {} |
| 478 | folders = {} |
| 479 | errors = [] |
| 480 | pending = [self.root] |
| 481 | while pending: |
| 482 | for artifact, value in children(pending.pop()): |
| 483 | if stat.S_ISDIR(value.st_mode): |
| 484 | pending.append(artifact) |
| 485 | continue |
| 486 | if not stat.S_ISREG(value.st_mode) or not PATTERN.fullmatch(artifact.name) or artifact.name.endswith('.minisig'): |
| 487 | continue |
| 488 | try: |
| 489 | if artifact.relative_to(self.root) != relative(artifact.name): |
| 490 | raise Unavailable('misplaced Zig artifact') |
| 491 | timestamp = artifact_time(artifact) |
| 492 | files[artifact] = timestamp |
| 493 | signature = Path(str(artifact) + '.minisig') |
| 494 | if signature.exists(): |
| 495 | with os.fdopen(metadata_open(signature), 'rb') as stream: |
| 496 | if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): |
| 497 | raise Unavailable('signature is not a regular file') |
| 498 | files[signature] = timestamp |
| 499 | directory = artifact.parent |
| 500 | while True: |
| 501 | folders[directory] = max(folders.get(directory, 0), timestamp) |
| 502 | if directory == self.root: |
| 503 | break |
| 504 | directory = directory.parent |
| 505 | except (Unavailable, OSError, ValueError) as error: |
| 506 | errors.append({'path': str(artifact.relative_to(self.root)), 'error': str(error)}) |
| 507 | readmes = [] |
| 508 | for directory in sorted(set(folders) - {self.root}): |
| 509 | try: |
| 510 | change = self.directory_readme(directory) |
| 511 | if change: |
| 512 | readmes.append(change) |
| 513 | except (Unavailable, OSError, ValueError) as error: |
| 514 | errors.append({'path': str(directory.relative_to(self.root)), 'error': str(error)}) |
| 515 | entries = [] |
| 516 | for path, timestamp in sorted({**files, **folders}.items()): |
| 517 | descriptor = metadata_open(path) |
| 518 | try: |
| 519 | entries.append({'path': str(path.relative_to(self.root)), 'before': file_state(os.fstat(descriptor)), 'mtime_ns': timestamp}) |
| 520 | finally: |
| 521 | os.close(descriptor) |
| 522 | return {'root': str(self.root), 'files': len(files), 'folders': len(folders), |
| 523 | 'changed': sum(entry['before']['mtime_ns'] != entry['mtime_ns'] for entry in entries), |
| 524 | 'provenance': {'stable': OFFICIAL + '/download/index.json UTC date', 'development': 'pinned Minisign authenticated trusted-comment timestamp', 'withdrawn0.15.0': 'authenticated signing time; release retracted before 0.15.1', 'folders': 'newest contained artifact timestamp'}, |
| 525 | 'errors': errors, 'entries': entries, 'readmes': readmes} |
| 526 | |
| 527 | def reconcile(self, apply=False, journal=None): |
| 528 | with self.publishing() if apply else contextlib.nullcontext(): |
| 529 | plan = self.metadata_plan() |
| 530 | if apply: |
| 531 | if plan['errors']: |
| 532 | raise Unavailable('timestamp plan has unresolved dates') |
| 533 | if journal is None or journal.resolve().is_relative_to(self.root): |
| 534 | raise ValueError('private journal outside the mirror is required') |
| 535 | journal = journal.resolve() |
| 536 | descriptor = os.open(journal, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) |
| 537 | with os.fdopen(descriptor, 'w') as output: |
| 538 | json.dump(plan, output) |
| 539 | output.write('\n') |
| 540 | output.flush() |
| 541 | os.fsync(output.fileno()) |
| 542 | descriptor = metadata_open(journal.parent) |
| 543 | try: |
| 544 | os.fsync(descriptor) |
| 545 | finally: |
| 546 | os.close(descriptor) |
| 547 | for change in plan['readmes']: |
| 548 | self.apply_readme(change) |
| 549 | for entry in plan['entries']: |
| 550 | if entry['before']['mtime_ns'] == entry['mtime_ns'] and not stat.S_ISDIR(entry['before']['mode']): |
| 551 | continue |
| 552 | descriptor = metadata_open(self.root / entry['path']) |
| 553 | try: |
| 554 | current = file_state(os.fstat(descriptor)) |
| 555 | expected = entry['before'] |
| 556 | fields = ('dev', 'ino', 'mode', 'uid', 'gid') if stat.S_ISDIR(current['mode']) else tuple(key for key in expected if key != 'atime_ns') |
| 557 | if any(current[key] != expected[key] for key in fields): |
| 558 | raise Unavailable('file changed since timestamp plan: ' + entry['path']) |
| 559 | if current['mtime_ns'] != entry['mtime_ns']: |
| 560 | os.utime(descriptor, ns=(current['atime_ns'], entry['mtime_ns'])) |
| 561 | finally: |
| 562 | os.close(descriptor) |
| 563 | applied = {} |
| 564 | paths = {entry['path'] for entry in plan['entries']} | {change['path'] for change in plan['readmes']} |
| 565 | for path in sorted(paths): |
| 566 | descriptor = metadata_open(self.root / path) |
| 567 | try: |
| 568 | applied[path] = file_state(os.fstat(descriptor)) |
| 569 | finally: |
| 570 | os.close(descriptor) |
| 571 | with os.fdopen(metadata_open(journal, os.O_WRONLY | os.O_APPEND), 'w') as output: |
| 572 | json.dump({'applied': applied}, output) |
| 573 | output.write('\n') |
| 574 | output.flush() |
| 575 | os.fsync(output.fileno()) |
| 576 | return plan |
| 577 | |
| 578 | def verify(self): |
| 579 | entries = index() |
| 580 | hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] |
| 581 | for release in entries.values() for item in release.values() |
| 582 | if isinstance(item, dict) and 'tarball' in item} |
| 583 | checked = 0 |
| 584 | for artifact in sorted(self.root.rglob('zig-*')): |
| 585 | if any(part.startswith('.') for part in artifact.relative_to(self.root).parts): |
| 586 | continue |
| 587 | if not artifact.is_file() or artifact.name.endswith('.minisig'): |
| 588 | continue |
| 589 | filename(artifact.name) |
| 590 | if not artifact.resolve().is_relative_to(self.root.resolve()): |
| 591 | raise Unavailable('artifact escapes mirror') |
| 592 | checksum = hashes.get(artifact.name) |
| 593 | if checksum: |
| 594 | if digest(artifact) != checksum: |
| 595 | raise Unavailable('artifact SHA-256 mismatch: ' + str(artifact)) |
| 596 | else: |
| 597 | result = subprocess.run(['minisign', '-Vm', str(artifact), '-x', str(artifact) + '.minisig', '-P', PUBLIC_KEY], capture_output=True) |
| 598 | if result.returncode: |
| 599 | raise Unavailable('artifact signature verification failed: ' + str(artifact)) |
| 600 | checked += 1 |
| 601 | print(json.dumps({'verifiedArtifacts': checked})) |
| 602 | |
| 603 | def refresh(self): |
| 604 | if not self.refresh_lock.acquire(blocking=False): |
| 605 | return |
| 606 | try: |
| 607 | entries = index(refresh=True) |
| 608 | latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))] |
| 609 | for release in (latest, entries['master']): |
| 610 | artifact = None |
| 611 | for item in release.values(): |
| 612 | if isinstance(item, dict) and 'tarball' in item: |
| 613 | name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]) |
| 614 | artifact = self.ensure(name, checksum=item['shasum']) |
| 615 | if artifact is not None: |
| 616 | with self.publishing(): |
| 617 | try: |
| 618 | self.write_readmes(artifact.parent) |
| 619 | finally: |
| 620 | self.parent_times(artifact) |
| 621 | finally: |
| 622 | self.refresh_lock.release() |
| 623 | |
| 624 | |
| 625 | class Handler(http.server.BaseHTTPRequestHandler): |
| 626 | protocol_version = 'HTTP/1.1' |
| 627 | timeout = 30 |
| 628 | |
| 629 | def do_HEAD(self): |
| 630 | self.do_GET() |
| 631 | |
| 632 | def do_GET(self): |
| 633 | path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) |
| 634 | if path == '/health': |
| 635 | self.send_response(200) |
| 636 | self.send_header('Content-Length', '0') |
| 637 | self.end_headers() |
| 638 | return |
| 639 | name = path.removeprefix('/mirrors/zig/').removeprefix('/') |
| 640 | try: |
| 641 | filename(name) |
| 642 | target = self.server.mirror.ensure(name) |
| 643 | size = target.stat().st_size |
| 644 | first, last, status = 0, size - 1, 200 |
| 645 | if self.headers.get('Range'): |
| 646 | match = re.fullmatch(r'bytes=(\d*)-(\d*)', self.headers['Range']) |
| 647 | if not match or not any(match.groups()): |
| 648 | raise IndexError() |
| 649 | if not match[1]: |
| 650 | first = max(0, size - int(match[2])) |
| 651 | else: |
| 652 | first = int(match[1]) |
| 653 | if match[2]: |
| 654 | last = min(last, int(match[2])) |
| 655 | if first > last or first >= size: |
| 656 | raise IndexError() |
| 657 | status = 206 |
| 658 | self.send_response(status) |
| 659 | self.send_header('Content-Type', 'application/octet-stream') |
| 660 | self.send_header('Content-Length', str(last - first + 1)) |
| 661 | self.send_header('Accept-Ranges', 'bytes') |
| 662 | self.send_header('Cache-Control', 'public, max-age=31536000, immutable') |
| 663 | self.send_header('Last-Modified', email.utils.formatdate(target.stat().st_mtime, usegmt=True)) |
| 664 | if status == 206: |
| 665 | self.send_header('Content-Range', f'bytes {first}-{last}/{size}') |
| 666 | self.end_headers() |
| 667 | if self.command != 'HEAD': |
| 668 | with target.open('rb') as stream: |
| 669 | stream.seek(first) |
| 670 | remaining = last - first + 1 |
| 671 | while remaining and (chunk := stream.read(min(1024 * 1024, remaining))): |
| 672 | self.wfile.write(chunk) |
| 673 | remaining -= len(chunk) |
| 674 | except ValueError: |
| 675 | self.send_error(404) |
| 676 | except IndexError: |
| 677 | self.send_response(416) |
| 678 | self.send_header('Content-Range', f'bytes */{size}') |
| 679 | self.send_header('Content-Length', '0') |
| 680 | self.end_headers() |
| 681 | except BlockingIOError: |
| 682 | self.send_error(429) |
| 683 | except (Unavailable, OSError): |
| 684 | self.send_error(504) |
| 685 | |
| 686 | def do_POST(self): |
| 687 | self.send_error(405) |
| 688 | |
| 689 | |
| 690 | class Links(HTMLParser): |
| 691 | def __init__(self): |
| 692 | super().__init__() |
| 693 | self.names = [] |
| 694 | |
| 695 | def handle_starttag(self, tag, attrs): |
| 696 | if tag == 'a': |
| 697 | href = dict(attrs).get('href', '') |
| 698 | if '/' not in href.rstrip('/') and not href.startswith('.'): |
| 699 | self.names.append(urllib.parse.unquote(href)) |
| 700 | |
| 701 | |
| 702 | def listing(url): |
| 703 | with urllib.request.urlopen(url, timeout=30) as response: |
| 704 | links = Links() |
| 705 | links.feed(response.read(4 * 1024**2).decode()) |
| 706 | return links.names |
| 707 | |
| 708 | |
| 709 | def backfill(mirror, budget, apply): |
| 710 | entries = index() |
| 711 | hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum'] |
| 712 | for release in entries.values() for item in release.values() |
| 713 | if isinstance(item, dict) and 'tarball' in item} |
| 714 | copied = 0 |
| 715 | for version in listing(NEKTRO): |
| 716 | version = version.rstrip('/') |
| 717 | if not VERSION.fullmatch(version): |
| 718 | continue |
| 719 | url = NEKTRO + urllib.parse.quote(version) + '/' |
| 720 | for name in listing(url): |
| 721 | try: |
| 722 | filename(name) |
| 723 | except ValueError: |
| 724 | continue |
| 725 | if name.endswith('.minisig') or mirror.path(name).exists(): |
| 726 | continue |
| 727 | if not apply: |
| 728 | print(url + urllib.parse.quote(name)) |
| 729 | continue |
| 730 | if copied + MAX_ARTIFACT > budget: |
| 731 | raise SystemExit('backfill budget reached; rerun to resume') |
| 732 | path = mirror.ensure(name, url + urllib.parse.quote(name), hashes.get(name)) |
| 733 | copied += path.stat().st_size |
| 734 | print(json.dumps({'downloadedBytes': copied, 'apply': apply})) |
| 735 | |
| 736 | |
| 737 | def main(): |
| 738 | parser = argparse.ArgumentParser() |
| 739 | parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify', 'reconcile']) |
| 740 | parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror')) |
| 741 | parser.add_argument('--budget-gib', type=int, default=32) |
| 742 | parser.add_argument('--apply', action='store_true') |
| 743 | parser.add_argument('--journal', type=Path) |
| 744 | args = parser.parse_args() |
| 745 | os.umask(0o002) |
| 746 | if args.command == 'reconcile' and not Path(args.root).is_dir(): |
| 747 | raise ValueError('existing mirror root is required') |
| 748 | mirror = Mirror(args.root) |
| 749 | if args.command == 'verify': |
| 750 | mirror.verify() |
| 751 | elif args.command == 'reconcile': |
| 752 | print(json.dumps(mirror.reconcile(args.apply, args.journal))) |
| 753 | elif args.command == 'refresh': |
| 754 | mirror.refresh() |
| 755 | elif args.command == 'backfill': |
| 756 | backfill(mirror, args.budget_gib * 1024**3, args.apply) |
| 757 | else: |
| 758 | interval = int(os.environ.get('ZIG_MIRROR_REFRESH', '21600')) |
| 759 | if interval: |
| 760 | def refresh(): |
| 761 | while True: |
| 762 | try: |
| 763 | mirror.refresh() |
| 764 | except Exception as error: |
| 765 | print(type(error).__name__ + ': refresh failed', flush=True) |
| 766 | time.sleep(interval) |
| 767 | threading.Thread(target=refresh, daemon=True).start() |
| 768 | server = http.server.ThreadingHTTPServer(('0.0.0.0', 8080), Handler) |
| 769 | server.mirror = mirror |
| 770 | server.serve_forever() |
| 771 | |
| 772 | |
| 773 | if __name__ == '__main__': |
| 774 | main() |