1#!/usr/bin/env python3
2import argparse
3from contextlib import ExitStack
4from html.parser import HTMLParser
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import os
8from pathlib import Path
9import re
10import socket
11import socketserver
12import sqlite3
13import subprocess
14import tempfile
15import threading
16import time
17import urllib.error
18import urllib.request
19
20import router
21
22
23class Links(HTMLParser):
24 def __init__(self, html):
25 super().__init__()
26 self.links = []
27 self.feed(html)
28
29 def handle_starttag(self, tag, attrs):
30 if tag == 'a':
31 self.links.append(dict(attrs))
32
33
34def port():
35 with socket.socket() as sock:
36 sock.bind(('127.0.0.1', 0))
37 return sock.getsockname()[1]
38
39
40def main():
41 parser = argparse.ArgumentParser()
42 parser.add_argument('--caddy', required=True, type=Path)
43 parser.add_argument('--binary', type=Path, default=Path(__file__).resolve().parent.parent / 'dashboard/target/debug/home-dashboard')
44 parser.add_argument('--page', type=Path)
45 parser.add_argument('--browser-ready-file', type=Path)
46 args = parser.parse_args()
47 repo = Path(__file__).resolve().parent.parent
48 observations = []
49 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'
50 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'
51 blob = bytes(range(256)) * 65536
52 csrf = 'initial'
53
54 class Fixture(BaseHTTPRequestHandler):
55 protocol_version = 'HTTP/1.1'
56
57 def do_GET(self):
58 nonlocal csrf
59 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
60 if self.path == '/':
61 account = b'<a class="usa-nav-link" href="/~fixture-member">~fixture-member</a>' if self.headers.get('Cookie') == 'SessionID=preserved' else b''
62 return self.respond(b'<html><body><header class="usa-header--basic"><nav>' + account + b'</nav></header></body></html>', 'text/html')
63 if self.path == '/-/settings' or self.path.endswith('/issues/new'):
64 csrf += 'x'
65 account = '<body id="page-user-settings"><kbd>fixture-member</kbd>' if self.path == '/-/settings' else '<body>'
66 return self.respond(('<html>' + account + '<form method="post"><input type="hidden" name="csrf_token" value="' + csrf + '"></form></body></html>').encode(), 'text/html')
67 if self.path == '/binary':
68 return self.respond(blob, 'application/octet-stream')
69 if self.path == '/json':
70 return self.respond(b'{"issue":{"title":"<angle> & Unicode"}}', 'application/json; charset=utf-8')
71 if self.path == '/large':
72 return self.respond(b' ' * (9 * 1024 * 1024), 'text/html')
73 if self.path == '/-/login':
74 callback = 'https://' + self.headers['Host'] + '/-/callback'
75 return self.respond(b'', 'text/plain', 302, [('Location', 'https://snowglobe.snowglobe.test/auth/oidc/authorize?redirect_uri=' + urllib.parse.quote(callback, safe=''))])
76 if self.path == '/redirect':
77 return self.respond(b'', 'text/plain', 302, [('Location', '/snowbound/issues/26'), ('Set-Cookie', 'SessionID=new; Path=/; HttpOnly'), ('Set-Cookie', 'other=kept; Path=/')])
78 if self.path.startswith('/-/') and args.page:
79 with urllib.request.urlopen('https://shale.paperclover.net' + self.path, timeout=15) as response:
80 return self.respond(response.read(), response.headers['Content-Type'])
81 return self.respond(issue + sample, 'text/html; charset=utf-8', headers=[('ETag', '"stale"'), ('Last-Modified', 'Mon, 05 Oct 2026 00:00:00 GMT')])
82
83 def do_HEAD(self):
84 self.respond(b'', 'text/html')
85
86 def do_POST(self):
87 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
88 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
89 if self.path.endswith('/issues/new') and urllib.parse.parse_qs(body.decode()).get('csrf_token') != [csrf]:
90 return self.respond(b'E: unexpected csrf token', 'text/plain', 403)
91 self.respond(body, 'application/octet-stream', 201)
92
93 def respond(self, body, content_type, status=200, headers=()):
94 self.send_response(status)
95 self.send_header('Content-Type', content_type)
96 self.send_header('Content-Length', str(len(body)))
97 for key, value in headers:
98 self.send_header(key, value)
99 self.end_headers()
100 self.wfile.write(body)
101
102 def log_message(self, *args):
103 pass
104
105 class NoRedirect(urllib.request.HTTPRedirectHandler):
106 def redirect_request(self, *args):
107 return None
108
109 opener = urllib.request.build_opener(NoRedirect)
110 with tempfile.TemporaryDirectory(prefix='shale-page-') as temporary, ExitStack() as stack:
111 root = Path(temporary).resolve()
112 fixture = ThreadingHTTPServer(('127.0.0.1', 0), Fixture)
113 stack.callback(fixture.server_close)
114 stack.callback(fixture.shutdown)
115 threading.Thread(target=fixture.serve_forever, daemon=True).start()
116 dashboard_port, gateway_port, internal_port = port(), port(), port()
117 proof = 'a' * 64
118 token = root / 'proxy.token'
119 token.write_text(proof)
120 certificate, gateway_key = root / 'gateway.pem', root / 'gateway.key'
121 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
122 data = root / 'data'
123 data.mkdir()
124 class Host(socketserver.StreamRequestHandler):
125 def handle(self):
126 request = json.loads(self.rfile.readline())
127 value = {}
128 if request['operation'] == 'shale.access':
129 allowed = request['account'] == 'fixture-member'
130 value = {'allowed': allowed, 'deny': not allowed}
131 response = json.dumps({'value': value}).encode()
132 self.wfile.write(len(response).to_bytes(4, 'big') + response)
133
134 host = socketserver.UnixStreamServer(str(root / 'host.sock'), Host)
135 stack.callback(host.server_close)
136 stack.callback(host.shutdown)
137 threading.Thread(target=host.serve_forever, daemon=True).start()
138 environment = {**os.environ, 'PORT': str(dashboard_port), 'SNOWGLOBE_DOMAIN': 'snowglobe.test', 'SNOWGLOBE_DATA_DIR': str(data), 'SNOWGLOBE_PROXY_TOKEN_FILE': str(token), 'SNOWGLOBE_HOST_SOCKET': str(root / 'host.sock'), 'SNOWGLOBE_INTERNAL_URL': f'https://localhost:{internal_port}', 'SNOWGLOBE_CA_BUNDLE': str(certificate), 'SNOWGLOBE_REPO': str(repo), 'SNOWGLOBE_WEB_DIR': str(repo / 'dashboard/dist')}
139 for key in ['SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_YT_STATE']:
140 environment.pop(key, None)
141 log = stack.enter_context((root / 'dashboard.log').open('w'))
142 dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log)
143 stack.callback(lambda: dashboard.wait(timeout=10))
144 stack.callback(dashboard.terminate)
145 deadline = time.monotonic() + 15
146 while True:
147 assert dashboard.poll() is None, (root / 'dashboard.log').read_text()
148 try:
149 with socket.create_connection(('127.0.0.1', dashboard_port), timeout=.1):
150 break
151 except OSError:
152 assert time.monotonic() < deadline
153 time.sleep(.05)
154 with sqlite3.connect(data / 'accounts.sqlite') as db:
155 for provider, subject, suffix, name in [('github', '24465214', '24465214', 'paperclover'), ('astheno', 'pairwise-astheno-subject', 'abc', 'Astheno user'), ('github', '777', '777', None), ('astheno', 'unmapped-pairwise-subject', 'unknown', 'Unmapped guest')]:
156 username = f'guest-{provider}-{suffix}'
157 profile = {'kind': 'guest', 'enabled': True, 'username': username, 'guestProvider': provider, 'firstName': name}
158 db.execute('INSERT INTO users(id,profile) VALUES (?,?)', [username, json.dumps(profile)])
159 db.execute('INSERT INTO external_identities VALUES (?,?,?)', [provider, subject, username])
160 (root / 'stages').mkdir()
161 (root / 'stages/shale-preview-12345678.json').write_text(json.dumps({'sourceId': 'shale'}))
162 os.environ.update(SNOWGLOBE_DOMAIN='snowglobe.test', SNOWGLOBE_DASHBOARD_PORT=str(dashboard_port), SNOWGLOBE_PROXY_TOKEN_FILE=str(token), SNOWGLOBE_INTERNAL_PORT=str(internal_port), SNOWGLOBE_STATE=str(root))
163 router.ROUTE_DIR = str(root / 'routes')
164 Path(router.ROUTE_DIR).mkdir()
165 (Path(router.ROUTE_DIR) / 'shale.json').write_text(json.dumps({'headHtml': {'shale.snowglobe.test': {'/snowbound/*': '<meta name="rewrite-test" content="kept">'}}}))
166
167 def nomad(path, token):
168 if path == '/v1/services':
169 return [{'Namespace': 'default', 'Services': [{'ServiceName': 'shale'}, {'ServiceName': 'shale-preview-12345678'}]}]
170 if path.startswith('/v1/service/'):
171 service = path.rsplit('/', 1)[1]
172 host = 'shale.snowglobe.test' if service == 'shale' else service + '.snowglobe.test'
173 return [{'Address': '127.0.0.1', 'Port': fixture.server_port, 'ServiceName': service, 'AllocID': service, 'JobID': service, 'Tags': ['caddy-host=' + host]}]
174 return {'ready': {'Status': 'success'}}
175
176 router.nomad = nomad
177 rendered = router.render('fixture')
178 full_config = root / 'routes.caddy'
179 full_config.write_text('{\n admin off\n auto_https off\n}\n' + rendered)
180 caddy_env = {**os.environ, 'XDG_DATA_HOME': str(root / 'caddy-data'), 'XDG_CONFIG_HOME': str(root / 'caddy-config')}
181 subprocess.run([str(args.caddy), 'validate', '--config', str(full_config), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
182 def local_site(host, listener):
183 start = rendered.index(host + ' {')
184 end = rendered.index('\n}', start) + 2
185 return rendered[start:end].replace(host + ' {', f'http://127.0.0.1:{listener} {{', 1).replace(' tls internal\n', '').replace(' route {\n', f' route {{\n request_header Host {host}\n', 1)
186
187 preview_port = port()
188 config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.snowglobe.test', gateway_port) + '\n' + local_site('shale-preview-12345678.snowglobe.test', preview_port)
189 internal_host = f'dashboard.internal.snowglobe.test:{internal_port}'
190 start = rendered.index(internal_host + ' {')
191 config += '\n' + rendered[start:].replace(internal_host + ' {', f'https://localhost:{internal_port} {{', 1).replace(' tls internal\n', f' tls {certificate} {gateway_key}\n', 1)
192 config_path = root / 'Caddyfile'
193 config_path.write_text(config)
194 subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
195 caddy_log = stack.enter_context((root / 'caddy.log').open('w'))
196 caddy = subprocess.Popen([str(args.caddy), 'run', '--config', str(config_path), '--adapter', 'caddyfile'], stdout=caddy_log, stderr=caddy_log, env=caddy_env)
197 stack.callback(lambda: caddy.wait(timeout=10))
198 stack.callback(caddy.terminate)
199 origin = f'http://127.0.0.1:{gateway_port}'
200
201 def request(path, *, headers=None, body=None, direct=False):
202 supplied = {'Cookie': 'SessionID=preserved', 'Origin': 'https://shale.snowglobe.test', **(headers or {})}
203 url = (f'http://127.0.0.1:{dashboard_port}' if direct else origin) + path
204 try:
205 response = opener.open(urllib.request.Request(url, headers=supplied, data=body), timeout=15)
206 except urllib.error.HTTPError as error:
207 response = error
208 with response:
209 return response.status, response.headers, response.read()
210
211 deadline = time.monotonic() + 15
212 while True:
213 try:
214 status, headers, body = request('/snowbound/issues/26?query=kept')
215 break
216 except urllib.error.URLError:
217 assert time.monotonic() < deadline and caddy.poll() is None
218 time.sleep(.05)
219 assert status == 200, (status, body, (root / 'dashboard.log').read_text(), (root / 'caddy.log').read_text())
220 links = Links(body.decode()).links
221 link = next(link for link in links if link.get('href') == 'https://github.com/paperclover')
222 assert link['target'] == '_blank' and link['rel'] == 'noreferrer', link
223 assert any(link.get('href') == '/~guest-github-unknown' for link in links)
224 assert any(link.get('href') == '/~guest-github-24465214/repo' for link in links)
225 assert b'Astheno user' in body and not any(link.get('href') == '/~guest-astheno-abc' for link in links)
226 assert b'identity.astheno.software/user/' not in body and b'pairwise-astheno-subject' not in body
227 assert b'https://avatars.githubusercontent.com/u/24465214?s=64' in body
228 assert b'rewrite-test' in body
229 assert headers['Cache-Control'] == 'no-store' and headers.get('ETag') is None and headers.get('Last-Modified') is None
230 assert observations[-1][1] == '/snowbound/issues/26?query=kept', observations[-1]
231 assert observations[-1][2]['host'] == 'shale.snowglobe.test', observations[-1]
232 assert observations[-1][2]['cookie'] == 'SessionID=preserved', observations[-1]
233 status, headers, body = request('/json', headers={'Accept': 'application/json'})
234 assert status == 200 and headers['Content-Type'].startswith('application/json')
235 assert body == b'{"issue":{"title":"<angle> & Unicode"}}'
236 assert observations[-1][2]['accept'] == 'application/json'
237 assert not any(key.lower().startswith('snowglobe-') for key in observations[-1][2]), observations[-1]
238 status, headers, body = request('/~guest-github-24465214')
239 assert status == 302 and headers['Location'] == 'https://github.com/paperclover' and headers['Referrer-Policy'] == 'no-referrer'
240 assert request('/~guest-astheno-abc/')[0] == 200
241 assert observations[-1][1] == '/~guest-astheno-abc/'
242 assert request('/~guest-astheno-unknown')[0] == 200
243 status, headers, body = request('/-/login')
244 assert status == 302 and urllib.parse.parse_qs(urllib.parse.urlparse(headers['Location']).query)['redirect_uri'] == ['https://shale.snowglobe.test/-/callback']
245 status, headers, body = request('/redirect')
246 assert status == 302 and headers['Location'] == '/snowbound/issues/26' and len(headers.get_all('Set-Cookie')) == 2
247 with opener.open(f'http://127.0.0.1:{preview_port}/snowbound/issues/26') as response:
248 assert response.status == 200
249 assert observations[-1][2]['host'] == 'shale-preview-12345678.snowglobe.test'
250 assert any(link.get('href') == 'https://github.com/paperclover' for link in Links(response.read().decode()).links)
251 assert b'/~guest-github-24465214' in request('/repo/info/refs')[2]
252 assert request('/binary')[2] == blob
253 assert len(request('/large')[2]) == 9 * 1024 * 1024
254 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')
255 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'
256 for path in ['/the-knife/issues/new', '/another-repo/issues/new']:
257 status, _, form = request(path)
258 assert status == 200
259 csrf_field = re.search(rb'name="csrf_token" value="([^"]+)"', form)[1]
260 assert request('/the-knife/')[0] == 200
261 submitted = b'csrf_token=' + csrf_field + b'&title=preserved'
262 status, _, body = request(path, body=submitted)
263 assert status == 201 and body == submitted, (path, status, body)
264 for invalid in [b'title=missing', b'csrf_token=invalid']:
265 assert request(path, body=invalid)[0] == 403
266 assert not any(item[1] == '/-/settings' for item in observations)
267 assert request('/the-knife/issues/new', headers={'Cookie': ''})[0] == 403
268 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403
269 assert request('/auth/shale/page', direct=True)[0] == 403
270 assert request('/auth/shale/page', direct=True, headers={'Snowglobe-Proxy-Token': proof, 'Snowglobe-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Snowglobe-Shale-Uri': '/', 'Host': 'snowglobe.snowglobe.test'})[0] == 403
271 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'csrf_preservation_and_rejection': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
272 if args.browser_ready_file:
273 args.browser_ready_file.write_text(json.dumps({'origin': origin}))
274 stop = args.browser_ready_file.with_suffix('.stop')
275 deadline = time.monotonic() + 600
276 while not stop.exists() and time.monotonic() < deadline:
277 time.sleep(.2)
278 args.browser_ready_file.unlink(missing_ok=True)
279 stop.unlink(missing_ok=True)
280
281
282if __name__ == '__main__':
283 main()