diff --git a/service/zig-mirror/build/mirror.py b/service/zig-mirror/build/mirror.py index 407b1bfe7b6e98344ffa10ac86261b8a46ee3069..f819eb0cff0994081f7382e6266c88ce0af03bac 100644 --- a/service/zig-mirror/build/mirror.py +++ b/service/zig-mirror/build/mirror.py @@ -18,7 +18,7 @@ import urllib.error import urllib.parse import urllib.request -PATTERN = re.compile(r'^zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') +PATTERN = re.compile(r'^zig(?:|-bootstrap|-win64|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$') VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z') PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U' OFFICIAL = 'https://ziglang.org' @@ -34,6 +34,8 @@ def filename(name): match = PATTERN.fullmatch(name) if len(name) > 240 or not match: raise ValueError('invalid Zig artifact name') + if name.startswith('zig-win64-') and name not in {'zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip'}: + raise ValueError('invalid legacy Zig artifact name') version = match[1] major, minor, patch = version.split('-')[0].split('.') if any(part != str(int(part)) for part in (major, minor, patch)): @@ -113,6 +115,14 @@ class Mirror: fcntl.flock(lock, fcntl.LOCK_EX) if target.is_file(): return target + version = filename(artifact_name) + signed = '-dev.' in version or tuple(map(int, version.split('.'))) >= (0, 6, 0) + if not signed and not checksum: + checksum = next((item['shasum'] for item in index().get(version, {}).values() + if isinstance(item, dict) and 'tarball' in item + and urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1] == artifact_name), None) + if not checksum: + raise Unavailable('unsigned artifact has no official SHA-256') artifact.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary: pending = Path(temporary) / artifact_name @@ -121,7 +131,7 @@ class Mirror: fetch(url, pending, MAX_ARTIFACT) if checksum and digest(pending) != checksum: raise Unavailable('artifact SHA-256 mismatch') - if not checksum or tuple(map(int, filename(artifact_name).split('-')[0].split('.'))) >= (0, 6, 0): + if signed: fetch(url + '.minisig', signature, 4096) result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY], capture_output=True) diff --git a/tools/router-zig-test.py b/tools/router-zig-test.py index 4f0cdbaac15d8dc6830613d3b82f6dc8c11fd12a..da9796a8b6bccce0545074d4fc71855d9e6dc23a 100644 --- a/tools/router-zig-test.py +++ b/tools/router-zig-test.py @@ -83,7 +83,11 @@ def main(): assert path == archive + '?download=1' assert 'Cookie' not in headers and 'Authorization' not in headers assert request(host, archive, method='HEAD') == (200, b'') - for path in ['/mirrors/zig/', '/mirrors/zig/releases/', '/mirrors/zig/dev/', '/mirrors/zig/releases/zig-0.0.0.tar.xz']: + for name in ['zig-win64-0.1.1.zip', 'zig-win64-0.2.0.zip', 'zig-future-0.99.0.zip']: + assert request(host, '/mirrors/zig/' + name) == (200, b'hello') + assert request(host, '/mirrors/zig/' + name, method='HEAD') == (200, b'') + for path in ['/mirrors/zig/', '/mirrors/zig/releases/', '/mirrors/zig/dev/', '/mirrors/zig/releases/zig-0.0.0.tar.xz', + '/mirrors/zig/preadme.md', '/mirrors/zig/not-zig.zip']: assert request(host, path) == (204, b'') assert request(host, archive, method='POST') == (204, b'') count = len(Mirror.requests) diff --git a/tools/router.py b/tools/router.py index d99d06f2279c8c1eda5afde82d7a15e2a709b319..b339ec8ecbe49ce405c06c1f295f0d3722ac6cc8 100644 --- a/tools/router.py +++ b/tools/router.py @@ -118,7 +118,7 @@ def zig_mirror_routes(service, routes): except FileNotFoundError: mirror = None upstreams = set().union(*(route["upstreams"] for route in routes.values() if mirror in route["services"])) - pattern = r"^/mirrors/zig/zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-([0-9]+\.[0-9]+\.[0-9]+(?:-dev\.[0-9]+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$" + pattern = r"^/mirrors/zig/zig[^/]*\.(?:tar\.xz|zip)(?:\.minisig)?$" lines = [" @zig_mirror {", " method GET HEAD", f" path_regexp zig_mirror {pattern}", " }", " handle @zig_mirror {"] if upstreams: diff --git a/tools/zig-mirror-test.py b/tools/zig-mirror-test.py index eb084cae5f4faf6c1acb44cd57e8c8dcfa54a05d..f64d374451022e2f5ceb3d6961fdad8c39bf49b7 100644 --- a/tools/zig-mirror-test.py +++ b/tools/zig-mirror-test.py @@ -30,12 +30,14 @@ class MirrorTests(unittest.TestCase): def test_spec_filenames_and_existing_layout(self): cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'), + ('zig-win64-0.1.1.zip', '0.1.1/'), ('zig-win64-0.2.0.zip', '0.2.0/'), ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'), ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')] for name, prefix in cases: self.assertEqual(str(mirror.relative(name)), prefix + name) self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name)) - for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz']: + for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz', + 'zig-win64-0.3.0.zip', 'zig-win64-0.1.1.tar.xz', 'zig-win64-0.1.1.zip.minisig', 'zig-linux-0.1.1.zip']: with self.assertRaises(ValueError): mirror.filename(name) @@ -130,6 +132,38 @@ class MirrorTests(unittest.TestCase): urllib.request.urlopen(base + '/zig-0.17.0.tar.xz') self.assertEqual(error.exception.code, 504) + def test_unsigned_stable_http_miss_uses_official_hash_and_fails_closed(self): + names = ['zig-win64-0.1.1.zip', 'zig-linux-x86_64-0.5.0.tar.xz', 'zig-linux-x86_64-0.4.0.tar.xz'] + body = b'official historical archive' + entries = {mirror.filename(name): {'platform': {'tarball': mirror.upstream(name), + 'shasum': mirror.hashlib.sha256(body).hexdigest() if i < 2 else '0' * 64}} + for i, name in enumerate(names)} + def fetch(url, target, limit): + self.assertNotIn('.minisig', url) + self.assertEqual(url, mirror.upstream(target.name)) + target.write_bytes(body) + server = mirror.http.server.ThreadingHTTPServer(('127.0.0.1', 0), mirror.Handler) + server.mirror = self.cache + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base = 'http://127.0.0.1:' + str(server.server_port) + '/mirrors/zig/' + with patch.object(mirror, 'index', return_value=entries) as index, patch.object(mirror, 'fetch', side_effect=fetch) as download, patch.object(mirror.subprocess, 'run') as signature: + for name in names[:2]: + with urllib.request.urlopen(base + name) as response: + self.assertEqual(response.read(), body) + for name in [names[2], 'zig-linux-x86_64-0.3.0.tar.xz']: + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(base + name) + self.assertEqual(error.exception.code, 504) + self.assertFalse(self.cache.path(name).exists()) + self.assertEqual(download.call_count, 3) + signature.assert_not_called() + index.side_effect = mirror.Unavailable('offline') + with urllib.request.urlopen(base + names[0]) as response: + self.assertEqual(response.read(), body) + self.assertFalse(list(self.cache.root.rglob('.download-*'))) + if __name__ == '__main__': unittest.main()