1#!/usr/bin/env python3
2import argparse
3from http.client import HTTPResponse, HTTPSConnection
4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json
6import os
7from pathlib import Path
8import re
9import socket
10import ssl
11import subprocess
12import tempfile
13import threading
14import time
15from unittest.mock import patch
16
17import router
18import snowglobe
19
20
21class Hub(BaseHTTPRequestHandler):
22 protocol_version = "HTTP/1.1"
23 started = threading.Event()
24
25 def do_GET(self):
26 if self.path == "/v1/upload-canceled":
27 type(self).started.set()
28 self.rfile.read(int(self.headers.get("Content-Length", 0)))
29 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
30 if self.headers.get("Authorization") != "Bearer " + key:
31 status, body = 401, b'{"error":"unauthorized"}'
32 elif self.path.startswith("/v1/transport-canceled"):
33 status, body = 499, b""
34 elif self.path.startswith("/v1/upstream-timeout"):
35 status, body = 504, b""
36 elif self.path.startswith("/v1/slow"):
37 type(self).started.set()
38 time.sleep(12)
39 status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n"
40 elif self.path.startswith("/v1/cancel"):
41 type(self).started.set()
42 time.sleep(1)
43 status, body = 200, b"finished"
44 elif self.path.startswith("/v1/broken"):
45 self.close_connection = True
46 return
47 else:
48 status, body = 200, b'{"object":"list"}'
49 try:
50 self.send_response(status)
51 self.send_header("Content-Length", str(len(body)))
52 self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json")
53 self.send_header("Set-Cookie", "response-cookie-private-fixture")
54 self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", ""))
55 self.send_header("X-Request-ID", "upstream-private-fixture")
56 if self.path in {"/v1/transport-canceled-no-retry", "/v1/upstream-timeout"}:
57 self.send_header("X-Should-Retry", "false")
58 self.end_headers()
59 self.wfile.write(body)
60 except (BrokenPipeError, ConnectionResetError):
61 pass
62
63 do_POST = do_GET
64
65 def log_message(self, *_):
66 pass
67
68
69class Login(BaseHTTPRequestHandler):
70 def do_GET(self):
71 self.send_response(302)
72 self.send_header("Location", "/snow.oauth2/start")
73 self.end_headers()
74
75 def log_message(self, *_):
76 pass
77
78
79def main():
80 parser = argparse.ArgumentParser()
81 parser.add_argument("--caddy", required=True)
82 args = parser.parse_args()
83 with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login:
84 root = Path(temporary)
85 routes = root / "routes"
86 routes.mkdir()
87 proof = root / "proof"
88 proof.write_text("a" * 64)
89 service = "substation-preview-1234abcd"
90 host = service + ".snowglobe.test"
91 paths = {"/v1/*": "hub", "/v0/management/*": "hub"}
92 data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False,
93 "uid": os.getuid(), "secrets": {}, "requiredSecrets": [],
94 "containers": {"app": {"rootGroup": False, "volumes": {}, "http": {
95 "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}},
96 "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}}
97 with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0):
98 snowglobe.provision(data)
99 manifest = json.loads((routes / (service + ".json")).read_text())
100 assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths}
101 instances = {
102 service: {"ServiceName": service, "JobID": service, "AllocID": service,
103 "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]},
104 service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service,
105 "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []},
106 "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria",
107 "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]},
108 }
109
110 def nomad(path, _):
111 if path == "/v1/services":
112 return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}]
113 if path.startswith("/v1/service/"):
114 item = instances.get(path.rsplit("/", 1)[1])
115 return [item] if item else []
116 return {"http": {"Status": "success"}}
117
118 environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port),
119 "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary,
120 "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")}
121 environment.pop("SNOWGLOBE_INTERNAL_PORT", None)
122 with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)):
123 rendered = router.render("fixture")
124 assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered
125 instances[service + "-hub"]["JobID"] = "substation"
126 try:
127 router.render("fixture")
128 except ValueError:
129 pass
130 else:
131 raise AssertionError("preview routed to production")
132 instances[service + "-hub"]["JobID"] = service
133 missing = instances.pop(service + "-hub")
134 assert "respond 503" in router.render("fixture")
135 instances[service + "-hub"] = missing
136 ports = []
137 for _ in range(2):
138 with socket.socket() as listener:
139 listener.bind(("127.0.0.1", 0))
140 ports.append(listener.getsockname()[1])
141 port, admin = ports
142 site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0]
143 site = site.replace(" tls internal\n", "")
144 config = root / "Caddyfile"
145 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n"
146 config.write_text(content)
147 for backend in (hub, login):
148 threading.Thread(target=backend.serve_forever, daemon=True).start()
149 log_path = root / "caddy.log"
150 tls = ssl._create_unverified_context()
151 with log_path.open("w") as log:
152 process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment)
153 try:
154 def request(path, key=None, method="GET", body=None, **headers):
155 if key:
156 headers["Authorization"] = "Bearer " + key
157 connection = HTTPSConnection("localhost", port, timeout=20, context=tls)
158 connection.request(method, path, body=body, headers=headers)
159 response = connection.getresponse()
160 result = response.status, dict(response.getheaders()), response.read()
161 connection.close()
162 return result
163
164 deadline = time.monotonic() + 10
165 while True:
166 try:
167 assert request("/")[0] == 302
168 break
169 except OSError:
170 assert process.poll() is None and time.monotonic() < deadline, log_path.read_text()
171 time.sleep(.05)
172 for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401),
173 ("/v0/management/version", "client-fixture", 401),
174 ("/v1/models", "client-fixture", 200),
175 ("/v0/management/version", "management-fixture", 200)]:
176 result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"})
177 assert result == status, (path, result)
178 assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"]
179 assert headers["X-Request-Id"] != "spoof-private-fixture"
180 assert "X-Should-Retry" not in headers
181 status, headers, _ = request("/v1/transport-canceled", "client-fixture")
182 assert status == 499 and headers["X-Should-Retry"] == "true"
183 status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture")
184 assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers)
185 assert "Connection" not in headers
186 status, headers, _ = request("/v1/upstream-timeout", "client-fixture")
187 assert status == 504 and headers["X-Should-Retry"] == "false" and "Connection" not in headers
188 for size in (100000, 2000000):
189 wire = tls.wrap_socket(socket.create_connection(("127.0.0.1", port), timeout=5), server_hostname="localhost")
190 wire.sendall(("POST /v1/upload-idle HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer client-fixture\r\n"
191 f"Content-Length: {size}\r\n\r\n").encode() + b"x" * 40000)
192 response = HTTPResponse(wire)
193 response.begin()
194 assert response.status in (499, 504), response.status
195 response.read()
196 assert response.headers["Connection"] == "close", dict(response.getheaders())
197 assert response.headers["X-Should-Retry"] == "true"
198 assert re.fullmatch(r"[0-9a-f-]{36}", response.headers["X-Request-ID"])
199 try:
200 wire.sendall(b"x" * (size - 40000))
201 assert wire.recv(4096) == b"", "late request body became another response"
202 except (BrokenPipeError, ConnectionResetError, ssl.SSLError):
203 pass
204 finally:
205 response.close()
206 wire.close()
207 client = HTTPSConnection("localhost", port, timeout=5, context=tls)
208 for _ in range(2):
209 client.request("GET", "/v1/models", headers={"Authorization": "Bearer client-fixture"})
210 response = client.getresponse()
211 assert response.status == 200 and response.headers.get("Connection") != "close"
212 response.read()
213 client.close()
214 assert request("/api/summary", "client-fixture")[0] == 302
215 assert request("/v1", "client-fixture")[0] == 302
216 Hub.started.clear()
217 outcome = []
218 worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture")))
219 worker.start()
220 assert Hub.started.wait(5)
221 config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0"))
222 subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment)
223 worker.join(timeout=20)
224 assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome
225 assert request("/v1/broken", "client-fixture")[0] == 502
226 Hub.started.clear()
227 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
228 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
229 assert Hub.started.wait(5)
230 client.close()
231 Hub.started.clear()
232 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
233 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
234 assert Hub.started.wait(5)
235 client.close()
236 deadline = time.monotonic() + 5
237 while True:
238 entries = []
239 for line in log_path.read_text().splitlines():
240 try:
241 entry = json.loads(line)
242 except ValueError:
243 continue
244 if entry.get("service") == service:
245 entries.append(entry)
246 if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries):
247 break
248 assert time.monotonic() < deadline, entries
249 time.sleep(.05)
250 assert all("request" not in entry and "resp_headers" not in entry for entry in entries)
251 assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries)
252 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)
253 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
254 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
255 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel")
256 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")
257 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000
258 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")
259 assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture")
260 assert "private-fixture" not in log_path.read_text()
261 assert "client-fixture" not in log_path.read_text()
262 assert "management-fixture" not in log_path.read_text()
263 finally:
264 process.terminate()
265 process.wait(timeout=10)
266 for backend in (hub, login):
267 backend.shutdown()
268 print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed")
269
270
271if __name__ == "__main__":
272 main()