authoravatarclover caruso<git@paperclover.net>2026-10-16 23:59:48-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log248649cf074333f5aec822a0616f916ccab97dac
treec726b640165c2d707154dc7455971f1a4bec2485
parent96c5ccc7c2fdad289ee3de6c44e2ba49ca9568c2
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Serve verified Zig artifacts through the public mirrors share

Add a bounded cache with on-demand upstream fetches, periodic current-build refresh and resumable Nektro backfill. Keep preview data isolated and retain the existing tree until the documented copy and checksum cutover is approved. Verified the public preview download, ranges, anonymous write denial and private-share boundary. IPv6 routing and production cutover remain separately controlled. Assisted-by: gpt-6.1-sol

9 files changed, 618 insertions(+), 4 deletions(-)

service/copyparty/copyparty.conf+8
...@@ -49,3 +49,11 @@...@@ -49,3 +49,11 @@
49 accs:49 accs:
50 rwmda.: snow50 rwmda.: snow
51 g: *51 g: *
52[/mirrors]
53 /w/media/Software/Public
54 accs:
55 r: *
56 A: snow
57 flags:
58 chmod_f: 664
59 chmod_d: 775
service/copyparty/prepare.py+11
...@@ -44,3 +44,14 @@ if not target.exists() or target.read_bytes() != rendered:...@@ -44,3 +44,14 @@ if not target.exists() or target.read_bytes() != rendered:
44 copyparty.atomic(target, rendered, (data['uid'], data['uid']))44 copyparty.atomic(target, rendered, (data['uid'], data['uid']))
45os.chmod(target, 0o600)45os.chmod(target, 0o600)
46os.chown(target, data['uid'], data['uid'])46os.chown(target, data['uid'], data['uid'])
47
48if service != 'copyparty':
49 public = Path(data['hostRoot']) / 'w/media/Software/Public/zig/0.0.0'
50 public.mkdir(parents=True, exist_ok=True)
51 fixture = public / 'zig-0.0.0.tar.xz'
52 fixture.write_bytes(b'Synthetic Zig mirror preview.\n')
53 for directory in [public, *public.parents]:
54 if directory == Path(data['hostRoot']):
55 break
56 os.chown(directory, data['uid'], data['uid'])
57 os.chown(fixture, data['uid'], data['uid'])
service/copyparty/service.pkl+5-4
...@@ -4,6 +4,7 @@ import "../../config/site.pkl" as site...@@ -4,6 +4,7 @@ import "../../config/site.pkl" as site
44
5name = "Copyparty"5name = "Copyparty"
6prepare = "prepare.py"6prepare = "prepare.py"
7stageIsolation = "fresh"
78
8secrets {9secrets {
9 ["idp_header"] { bytes = 16 }10 ["idp_header"] { bytes = 16 }
...@@ -31,9 +32,9 @@ container {...@@ -31,9 +32,9 @@ container {
31 ["/cfg"] {}32 ["/cfg"] {}
32 ["/w"] {}33 ["/w"] {}
33 ["/hooks/reject-apple-cruft.py"] { config = "reject-apple-cruft.py" }34 ["/hooks/reject-apple-cruft.py"] { config = "reject-apple-cruft.py" }
34 ["/w/clover"] { src = site.cloverRoot; readOnly = site.cloverReadOnly }35 ["/w/clover"] { src = if (site.preview) null else site.cloverRoot; readOnly = site.cloverReadOnly }
35 ["/w/clover/Media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly }36 ["/w/clover/Media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly }
36 ["/w/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly }37 ["/w/media"] { src = if (site.preview) null else site.mediaRoot; readOnly = site.mediaReadOnly }
37 ["/w/logs"] { src = "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly }38 ["/w/logs"] { src = if (site.preview) null else "\(site.root)/logs"; readOnly = site.preview || site.cloverReadOnly }
38 }39 }
39}40}
service/zig-mirror/build/Dockerfile created+4
...@@ -0,0 +1,4 @@
1FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332
2RUN apt-get update && apt-get install -y --no-install-recommends minisign ca-certificates && rm -rf /var/lib/apt/lists/*
3COPY mirror.py /app/mirror.py
4CMD ["python3", "-u", "/app/mirror.py", "serve"]
service/zig-mirror/build/mirror.py created+326
...@@ -0,0 +1,326 @@
1#!/usr/bin/env python3
2import argparse
3import email.utils
4import fcntl
5import hashlib
6from html.parser import HTMLParser
7import http.server
8import json
9import os
10from pathlib import Path
11import re
12import shutil
13import subprocess
14import tempfile
15import threading
16import time
17import urllib.error
18import urllib.parse
19import urllib.request
20
21PATTERN = re.compile(r'^zig(?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-(\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?)\.(?:tar\.xz|zip)(?:\.minisig)?$')
22VERSION = re.compile(r'\d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)?\Z')
23PUBLIC_KEY = 'RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U'
24OFFICIAL = 'https://ziglang.org'
25NEKTRO = 'http://mirrors.nektro.net/zig/'
26MAX_ARTIFACT = 2 * 1024**3
27
28
29class Unavailable(Exception):
30 pass
31
32
33def filename(name):
34 match = PATTERN.fullmatch(name)
35 if len(name) > 240 or not match:
36 raise ValueError('invalid Zig artifact name')
37 version = match[1]
38 major, minor, patch = version.split('-')[0].split('.')
39 if any(part != str(int(part)) for part in (major, minor, patch)):
40 raise ValueError('invalid Zig version')
41 return version
42
43
44def relative(name):
45 version = filename(name)
46 if '-dev.' in version:
47 return Path(version.split('-')[0]) / 'dev' / version / name
48 return Path(version) / name
49
50
51def upstream(name):
52 version = filename(name)
53 prefix = '/builds/' if '-dev.' in version else '/download/' + version + '/'
54 return OFFICIAL + prefix + urllib.parse.quote(name)
55
56
57def fetch(url, target, limit):
58 request = urllib.request.Request(url, headers={'User-Agent': 'snowglobe-zig-mirror/1'})
59 try:
60 with urllib.request.urlopen(request, timeout=180) as response, target.open('wb') as output:
61 count = 0
62 while chunk := response.read(1024 * 1024):
63 count += len(chunk)
64 if count > limit:
65 raise Unavailable('upstream artifact exceeds size limit')
66 output.write(chunk)
67 output.flush()
68 os.fsync(output.fileno())
69 except (OSError, urllib.error.URLError) as error:
70 raise Unavailable('upstream download failed') from error
71
72
73def index():
74 with urllib.request.urlopen(OFFICIAL + '/download/index.json', timeout=30) as response:
75 return json.load(response)
76
77
78def digest(path):
79 checksum = hashlib.sha256()
80 with path.open('rb') as stream:
81 while chunk := stream.read(1024 * 1024):
82 checksum.update(chunk)
83 return checksum.hexdigest()
84
85
86class Mirror:
87 def __init__(self, root):
88 self.root = Path(root)
89 self.root.mkdir(parents=True, exist_ok=True)
90 self.locks = self.root / '.mirror-locks'
91 self.slots = threading.BoundedSemaphore(4)
92 self.refresh_lock = threading.Lock()
93
94 def path(self, name):
95 path = self.root / relative(name)
96 if not path.resolve().is_relative_to(self.root.resolve()):
97 raise ValueError('artifact escapes mirror')
98 return path
99
100 def ensure(self, name, source=None, checksum=None):
101 target = self.path(name)
102 artifact_name = name.removesuffix('.minisig')
103 artifact = self.path(artifact_name)
104 if target.is_file():
105 return target
106 if not self.slots.acquire(timeout=1):
107 raise BlockingIOError('mirror download capacity reached')
108 try:
109 if shutil.disk_usage(self.root).free < 16 * 1024**3:
110 raise Unavailable('mirror free-space reserve reached')
111 self.locks.mkdir(exist_ok=True)
112 with (self.locks / artifact_name).open('a') as lock:
113 fcntl.flock(lock, fcntl.LOCK_EX)
114 if target.is_file():
115 return target
116 artifact.parent.mkdir(parents=True, exist_ok=True)
117 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:
118 pending = Path(temporary) / artifact_name
119 signature = Path(str(pending) + '.minisig')
120 url = source or upstream(artifact_name)
121 fetch(url, pending, MAX_ARTIFACT)
122 if checksum and digest(pending) != checksum:
123 raise Unavailable('artifact SHA-256 mismatch')
124 if not checksum or tuple(map(int, filename(artifact_name).split('-')[0].split('.'))) >= (0, 6, 0):
125 fetch(url + '.minisig', signature, 4096)
126 result = subprocess.run(['minisign', '-Vm', str(pending), '-x', str(signature), '-P', PUBLIC_KEY],
127 capture_output=True)
128 if result.returncode:
129 raise Unavailable('artifact signature verification failed')
130 os.chmod(pending, 0o664)
131 os.replace(pending, artifact)
132 if signature.exists():
133 os.chmod(signature, 0o664)
134 os.replace(signature, Path(str(artifact) + '.minisig'))
135 if name.endswith('.minisig') and not target.exists():
136 with tempfile.TemporaryDirectory(prefix='.download-', dir=artifact.parent) as temporary:
137 pending = Path(temporary) / name
138 fetch((source or upstream(artifact_name)) + '.minisig', pending, 4096)
139 os.chmod(pending, 0o664)
140 os.replace(pending, target)
141 return target
142 finally:
143 self.slots.release()
144
145 def verify(self):
146 entries = index()
147 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
148 for release in entries.values() for item in release.values()
149 if isinstance(item, dict) and 'tarball' in item}
150 checked = 0
151 for artifact in self.root.rglob('zig-*'):
152 if not artifact.is_file() or artifact.name.endswith('.minisig'):
153 continue
154 filename(artifact.name)
155 if not artifact.resolve().is_relative_to(self.root.resolve()):
156 raise Unavailable('artifact escapes mirror')
157 checksum = hashes.get(artifact.name)
158 if checksum:
159 if digest(artifact) != checksum:
160 raise Unavailable('artifact SHA-256 mismatch: ' + str(artifact))
161 else:
162 result = subprocess.run(['minisign', '-Vm', str(artifact), '-x', str(artifact) + '.minisig', '-P', PUBLIC_KEY], capture_output=True)
163 if result.returncode:
164 raise Unavailable('artifact signature verification failed: ' + str(artifact))
165 checked += 1
166 print(json.dumps({'verifiedArtifacts': checked}))
167
168 def refresh(self):
169 if not self.refresh_lock.acquire(blocking=False):
170 return
171 try:
172 entries = index()
173 latest = entries[max((key for key in entries if key != 'master'), key=lambda version: tuple(map(int, version.split('.'))))]
174 for release in (latest, entries['master']):
175 for item in release.values():
176 if isinstance(item, dict) and 'tarball' in item:
177 name = urllib.parse.unquote(urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1])
178 self.ensure(name, checksum=item['shasum'])
179 finally:
180 self.refresh_lock.release()
181
182
183class Handler(http.server.BaseHTTPRequestHandler):
184 protocol_version = 'HTTP/1.1'
185 timeout = 30
186
187 def do_HEAD(self):
188 self.do_GET()
189
190 def do_GET(self):
191 path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path)
192 if path == '/health':
193 self.send_response(200)
194 self.send_header('Content-Length', '0')
195 self.end_headers()
196 return
197 name = path.removeprefix('/mirrors/zig/').removeprefix('/')
198 try:
199 filename(name)
200 target = self.server.mirror.ensure(name)
201 size = target.stat().st_size
202 first, last, status = 0, size - 1, 200
203 if self.headers.get('Range'):
204 match = re.fullmatch(r'bytes=(\d*)-(\d*)', self.headers['Range'])
205 if not match or not any(match.groups()):
206 raise IndexError()
207 if not match[1]:
208 first = max(0, size - int(match[2]))
209 else:
210 first = int(match[1])
211 if match[2]:
212 last = min(last, int(match[2]))
213 if first > last or first >= size:
214 raise IndexError()
215 status = 206
216 self.send_response(status)
217 self.send_header('Content-Type', 'application/octet-stream')
218 self.send_header('Content-Length', str(last - first + 1))
219 self.send_header('Accept-Ranges', 'bytes')
220 self.send_header('Cache-Control', 'public, max-age=31536000, immutable')
221 self.send_header('Last-Modified', email.utils.formatdate(target.stat().st_mtime, usegmt=True))
222 if status == 206:
223 self.send_header('Content-Range', f'bytes {first}-{last}/{size}')
224 self.end_headers()
225 if self.command != 'HEAD':
226 with target.open('rb') as stream:
227 stream.seek(first)
228 remaining = last - first + 1
229 while remaining and (chunk := stream.read(min(1024 * 1024, remaining))):
230 self.wfile.write(chunk)
231 remaining -= len(chunk)
232 except ValueError:
233 self.send_error(404)
234 except IndexError:
235 self.send_response(416)
236 self.send_header('Content-Range', f'bytes */{size}')
237 self.send_header('Content-Length', '0')
238 self.end_headers()
239 except BlockingIOError:
240 self.send_error(429)
241 except (Unavailable, OSError):
242 self.send_error(504)
243
244 def do_POST(self):
245 self.send_error(405)
246
247
248class Links(HTMLParser):
249 def __init__(self):
250 super().__init__()
251 self.names = []
252
253 def handle_starttag(self, tag, attrs):
254 if tag == 'a':
255 href = dict(attrs).get('href', '')
256 if '/' not in href.rstrip('/') and not href.startswith('.'):
257 self.names.append(urllib.parse.unquote(href))
258
259
260def listing(url):
261 with urllib.request.urlopen(url, timeout=30) as response:
262 links = Links()
263 links.feed(response.read(4 * 1024**2).decode())
264 return links.names
265
266
267def backfill(mirror, budget, apply):
268 entries = index()
269 hashes = {urllib.parse.urlsplit(item['tarball']).path.rsplit('/', 1)[1]: item['shasum']
270 for release in entries.values() for item in release.values()
271 if isinstance(item, dict) and 'tarball' in item}
272 copied = 0
273 for version in listing(NEKTRO):
274 version = version.rstrip('/')
275 if not VERSION.fullmatch(version):
276 continue
277 url = NEKTRO + urllib.parse.quote(version) + '/'
278 for name in listing(url):
279 try:
280 filename(name)
281 except ValueError:
282 continue
283 if name.endswith('.minisig') or mirror.path(name).exists():
284 continue
285 if not apply:
286 print(url + urllib.parse.quote(name))
287 continue
288 if copied + MAX_ARTIFACT > budget:
289 raise SystemExit('backfill budget reached; rerun to resume')
290 path = mirror.ensure(name, url + urllib.parse.quote(name), hashes.get(name))
291 copied += path.stat().st_size
292 print(json.dumps({'downloadedBytes': copied, 'apply': apply}))
293
294
295def main():
296 parser = argparse.ArgumentParser()
297 parser.add_argument('command', choices=['serve', 'refresh', 'backfill', 'verify'])
298 parser.add_argument('--root', default=os.environ.get('ZIG_MIRROR_ROOT', '/mirror'))
299 parser.add_argument('--budget-gib', type=int, default=32)
300 parser.add_argument('--apply', action='store_true')
301 args = parser.parse_args()
302 mirror = Mirror(args.root)
303 if args.command == 'verify':
304 mirror.verify()
305 elif args.command == 'refresh':
306 mirror.refresh()
307 elif args.command == 'backfill':
308 backfill(mirror, args.budget_gib * 1024**3, args.apply)
309 else:
310 interval = int(os.environ.get('ZIG_MIRROR_REFRESH', '21600'))
311 if interval:
312 def refresh():
313 while True:
314 try:
315 mirror.refresh()
316 except Exception as error:
317 print(type(error).__name__ + ': refresh failed', flush=True)
318 time.sleep(interval)
319 threading.Thread(target=refresh, daemon=True).start()
320 server = http.server.ThreadingHTTPServer(('0.0.0.0', 8080), Handler)
321 server.mirror = mirror
322 server.serve_forever()
323
324
325if __name__ == '__main__':
326 main()
service/zig-mirror/prepare.py created+14
...@@ -0,0 +1,14 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import sys
6
7data = json.load(sys.stdin)
8if Path(data['hostRoot']).name != 'zig-mirror':
9 root = Path(data['hostRoot']) / 'mirror'
10 artifact = root / '0.0.0/zig-0.0.0.tar.xz'
11 artifact.parent.mkdir(parents=True, exist_ok=True)
12 artifact.write_bytes(b'Synthetic Zig mirror preview.\n')
13 for path in (artifact.parent, artifact):
14 os.chown(path, data['uid'], data['uid'])
service/zig-mirror/service.pkl created+24
...@@ -0,0 +1,24 @@
1amends "../../config/Service.pkl"
2
3import "../../config/site.pkl" as site
4
5name = "Zig mirror"
6launcher = false
7stageIsolation = "fresh"
8prepare = "prepare.py"
9
10container {
11 build = "build"
12 http {
13 containerPort = 8080
14 subdomain = "zig"
15 checkPath = "/health"
16 }
17 volumes {
18 ["/mirror"] { src = if (site.preview) null else "\(site.mediaRoot)/Software/Public/zig" }
19 }
20 env {
21 ["ZIG_MIRROR_ROOT"] = "/mirror"
22 ["ZIG_MIRROR_REFRESH"] = if (site.preview) "0" else "21600"
23 }
24}
tools/zig-mirror-test.py created+135
...@@ -0,0 +1,135 @@
1#!/usr/bin/env python3
2import concurrent.futures
3import importlib.util
4import io
5import contextlib
6import json
7from pathlib import Path
8from types import SimpleNamespace
9import tempfile
10import threading
11import time
12import unittest
13import urllib.error
14import urllib.request
15from unittest.mock import patch
16
17spec = importlib.util.spec_from_file_location('mirror', Path(__file__).resolve().parents[1] / 'service/zig-mirror/build/mirror.py')
18mirror = importlib.util.module_from_spec(spec)
19spec.loader.exec_module(mirror)
20
21
22class MirrorTests(unittest.TestCase):
23 def setUp(self):
24 self.temporary = tempfile.TemporaryDirectory()
25 self.addCleanup(self.temporary.cleanup)
26 self.cache = mirror.Mirror(self.temporary.name)
27 disk = patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=64 * 1024**3))
28 disk.start()
29 self.addCleanup(disk.stop)
30
31 def test_spec_filenames_and_existing_layout(self):
32 cases = [('zig-0.14.1.tar.xz', '0.14.1/'), ('zig-bootstrap-0.17.0.zip.minisig', '0.17.0/'),
33 ('zig-x86_64-linux-0.14.1.tar.xz', '0.14.1/'), ('zig-linux-x86_64-0.13.0.tar.xz', '0.13.0/'),
34 ('zig-newarch-newos-0.18.0-dev.1+abc123.tar.xz', '0.18.0/dev/0.18.0-dev.1+abc123/')]
35 for name, prefix in cases:
36 self.assertEqual(str(mirror.relative(name)), prefix + name)
37 self.assertEqual(mirror.upstream(name), 'https://ziglang.org/' + ('builds/' if '-dev.' in name else 'download/' + mirror.filename(name) + '/') + urllib.parse.quote(name))
38 for name in ['../zig-0.17.0.tar.xz', 'zig-00.17.0.tar.xz', 'zig-0.17.0.tar.xz/anything', 'foo.zip', 'zig-a-b-0.18.0-dev.1+ab?x.tar.xz']:
39 with self.assertRaises(ValueError):
40 mirror.filename(name)
41
42 def test_concurrent_miss_downloads_once_and_atomic_signature_pair(self):
43 calls = []
44 def fetch(url, target, limit):
45 calls.append(url)
46 time.sleep(.03)
47 target.write_bytes(b'signed data' if not url.endswith('.minisig') else b'signature')
48 with patch.object(mirror, 'fetch', side_effect=fetch), patch.object(mirror.subprocess, 'run') as verify:
49 verify.return_value.returncode = 0
50 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers:
51 paths = list(workers.map(self.cache.ensure, ['zig-0.17.0.tar.xz'] * 4))
52 self.assertEqual(len(calls), 2)
53 self.assertTrue(all(path.read_bytes() == b'signed data' for path in paths))
54 self.assertEqual(Path(str(paths[0]) + '.minisig').read_bytes(), b'signature')
55 self.assertFalse(list(self.cache.root.rglob('.download-*')))
56
57 def test_bad_hash_and_signature_never_publish(self):
58 def fetch(url, target, limit):
59 target.write_bytes(b'bad data')
60 with patch.object(mirror, 'fetch', side_effect=fetch):
61 with self.assertRaises(mirror.Unavailable):
62 self.cache.ensure('zig-0.17.0.tar.xz', checksum='0' * 64)
63 with patch.object(mirror.subprocess, 'run') as verify:
64 verify.return_value.returncode = 1
65 with self.assertRaises(mirror.Unavailable):
66 self.cache.ensure('zig-0.17.0.tar.xz')
67 self.assertFalse(self.cache.path('zig-0.17.0.tar.xz').exists())
68 self.assertFalse(list(self.cache.root.rglob('.download-*')))
69
70 def test_symlink_cannot_escape_cache(self):
71 (self.cache.root / '0.17.0').symlink_to(self.cache.root.parent)
72 with self.assertRaises(ValueError):
73 self.cache.ensure('zig-0.17.0.tar.xz')
74
75 def test_reserve_rejects_new_download_but_retains_cached_reads(self):
76 with patch.object(mirror.shutil, 'disk_usage', return_value=SimpleNamespace(free=0)), patch.object(mirror, 'fetch') as fetch:
77 with self.assertRaises(mirror.Unavailable):
78 self.cache.ensure('zig-0.17.0.tar.xz')
79 fetch.assert_not_called()
80 target = self.cache.path('zig-0.17.0.tar.xz')
81 target.parent.mkdir()
82 target.write_bytes(b'cached')
83 self.assertEqual(self.cache.ensure(target.name), target)
84
85 def test_backfill_dry_run_and_budget_do_not_download(self):
86 versions = ['0.17.0/', '../', 'not-a-version/']
87 artifacts = ['zig-0.17.0.tar.xz', 'zig-0.17.0.tar.xz.minisig', 'private.txt']
88 with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure, contextlib.redirect_stdout(io.StringIO()) as output:
89 mirror.backfill(self.cache, 0, False)
90 ensure.assert_not_called()
91 self.assertIn('http://mirrors.nektro.net/zig/0.17.0/zig-0.17.0.tar.xz', output.getvalue())
92 self.assertNotIn('private.txt', output.getvalue())
93 with patch.object(mirror, 'listing', side_effect=[versions, artifacts]), patch.object(mirror, 'index', return_value={}), patch.object(self.cache, 'ensure') as ensure:
94 with self.assertRaises(SystemExit):
95 mirror.backfill(self.cache, 0, True)
96 ensure.assert_not_called()
97
98 def test_verify_rejects_corrupted_existing_archive(self):
99 target = self.cache.path('zig-0.17.0.tar.xz')
100 target.parent.mkdir()
101 target.write_bytes(b'corrupt')
102 entries = {'0.17.0': {'src': {'tarball': 'https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz', 'shasum': '0' * 64}}}
103 with patch.object(mirror, 'index', return_value=entries):
104 with self.assertRaises(mirror.Unavailable):
105 self.cache.verify()
106
107 def test_http_query_range_head_denials_and_timeout(self):
108 target = self.cache.path('zig-0.17.0.tar.xz')
109 target.parent.mkdir()
110 target.write_bytes(b'0123456789')
111 server = mirror.http.server.ThreadingHTTPServer(('127.0.0.1', 0), mirror.Handler)
112 server.mirror = self.cache
113 threading.Thread(target=server.serve_forever, daemon=True).start()
114 self.addCleanup(server.server_close)
115 self.addCleanup(server.shutdown)
116 base = 'http://127.0.0.1:' + str(server.server_port)
117 with urllib.request.urlopen(base + '/mirrors/zig/zig-0.17.0.tar.xz?source=spec-test') as response:
118 self.assertEqual(response.status, 200)
119 self.assertEqual(response.read(), b'0123456789')
120 for method, value, expected in [('GET', 'bytes=2-5', b'2345'), ('GET', 'bytes=-3', b'789'), ('HEAD', 'bytes=2-5', b'')]:
121 with urllib.request.urlopen(urllib.request.Request(base + '/zig-0.17.0.tar.xz', method=method, headers={'Range': value})) as response:
122 self.assertEqual(response.status, 206)
123 self.assertEqual(response.read(), expected)
124 for path, headers, status in [('/zig-0.17.0.tar.xz', {'Range': 'bytes=100-'}, 416), ('/../secret', {}, 404), ('/0.17.0/zig-0.17.0.tar.xz', {}, 404)]:
125 with self.assertRaises(urllib.error.HTTPError) as error:
126 urllib.request.urlopen(urllib.request.Request(base + path, headers=headers))
127 self.assertEqual(error.exception.code, status)
128 with patch.object(self.cache, 'ensure', side_effect=mirror.Unavailable()):
129 with self.assertRaises(urllib.error.HTTPError) as error:
130 urllib.request.urlopen(base + '/zig-0.17.0.tar.xz')
131 self.assertEqual(error.exception.code, 504)
132
133
134if __name__ == '__main__':
135 unittest.main()
tools/zig-mirror.md created+91
...@@ -0,0 +1,91 @@
1# Zig mirror
2
3The canonical artifact tree is `/srv/clover/Media/Software/Public/zig`.
4Copyparty publishes only `Software/Public` as anonymous read-only `/mirrors`;
5its existing authenticated `/media` and `/clover` mounts keep their permissions.
6The mirror container shares only the Zig subtree. Flat protocol URLs fetch
7missing files synchronously, then serve them locally; nested release/dev paths
8remain browsable through Copyparty.
9
10[Zig's mirror contract](https://codeberg.org/ziglang/ziglang.org/src/branch/main/MIRRORS.md)
11requires HTTPS on **both IPv4 and IPv6**. Do not submit this mirror to Zig's list
12until both address families work from outside the LAN. On 2026-10-17 the host
13had a public IPv6 RA address/default route, but outbound IPv6 to Zig and Google
14failed and `file.paperclover.net` had no AAAA record. DDNS currently updates A
15records only. A working delegated prefix/router firewall and matching AAAA
16publication must be verified before enabling dual-stack discovery.
17
18The worker polls the official index every six hours and caches the latest stable
19release and current development build. A request for any uncached supported name
20checks its exact official URL immediately; a newly published release needs no
21manual trigger or wait for the poll. `refresh` runs the periodic operation on
22demand. Missing upstream files/timeouts return 504; saturated downloads return
23429. Downloads have four slots, a 2 GiB per-artifact ceiling, and a 16 GiB
24free-space reserve. Completed archives and signatures publish atomically after
25SHA-256 and/or Zig Minisign verification. Existing archive files need the
26one-time `verify` operation before first public exposure.
27
28## Production cutover
29
30This is a reviewable runbook, **not an executed migration**. The 2026-10-17
31inventory found 267 GiB / 10,063 files in `/srv/mirrors/zig`, on `globe/mirrors`.
32The destination Zig directory existed empty on `globe/clover/Media`, with about
339.13 TiB available. These are different filesystems: an atomic rename cannot
34move the tree. Preserve the old tree until copy verification and public checks
35finish. Both roots use group `chloe` (GID 3000).
36
37After production approval, confirm the host machine ID
38`691734bfb3c24b19ae66e5cacd17f5e2`, mounts, free space, and that no writer is
39changing the old tree. Then copy without deleting the source:
40
41```sh
42findmnt -T /srv/mirrors/zig
43findmnt -T /srv/clover/Media/Software/Public/zig
44rsync -aH --info=progress2 /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/
45rsync -anrc --delete /srv/mirrors/zig/ /srv/clover/Media/Software/Public/zig/
46```
47
48The checksum dry run must report no differences. Preserve existing numeric
49ownership and ACLs; do not recursively chown the Media dataset. The destination
50root must retain group traversal/write access for the mirror's declared Clover
51group. Run the mirror image's `verify` command against the copied destination
52before publishing it. A disposable container can bind that exact subtree
53read-only. Retain
54any old `.hist` metadata privately; anonymous users lack dot-file permission.
55
56Promote the reviewed configuration through `tools/deploy.py prod`. Do not replace
57containers by hand. Confirm anonymous GET/HEAD/Range on flat and nested URLs,
58anonymous upload/delete denial, authenticated existing shares, and that parent
59folders remain inaccessible. The source can be retired separately after the
60verified copy and service behavior are accepted; it is not an active second
61mirror and no alias is required.
62
63## Backfill and manual refresh
64
65Inside the running mirror task, the same script exposes:
66
67```sh
68python3 /app/mirror.py verify
69python3 /app/mirror.py refresh
70python3 /app/mirror.py backfill
71python3 /app/mirror.py backfill --apply --budget-gib 32
72```
73
74Use the normal authenticated Nomad allocation exec path to run these. The
75no-`--apply` backfill lists missing artifact URLs without downloading them.
76The source is [Meghan's mirror](http://mirrors.nektro.net/zig/), which listed
77893 version directories on 2026-10-17. Stable files also check the official
78index hash; development files require Zig's signature. Invalid bytes never
79become public. The 32 GiB budget reserves the full per-artifact ceiling before
80each download and stops conservatively; rerun to resume, preserving cached
81files. Missing historical builds cannot be recovered from Zig once removed,
82so failed verification/downloads stop the batch for review.
83
84## Staging
85
86Both services use fresh previews. Copyparty mounts disposable directories instead
87of the live NAS and generates one clearly synthetic public archive. Mirror
88previews disable periodic downloads. A flat cache miss may fetch a public Zig
89artifact on demand for verification. The preview association lives in the
90Copyparty stage's `mirrorStage` metadata; an unpaired preview returns 503 for
91flat protocol URLs and never falls back to production.