| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | from http.client import HTTPResponse, HTTPSConnection |
| 4 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 5 | import json |
| 6 | import os |
| 7 | from pathlib import Path |
| 8 | import re |
| 9 | import socket |
| 10 | import ssl |
| 11 | import subprocess |
| 12 | import tempfile |
| 13 | import threading |
| 14 | import time |
| 15 | from unittest.mock import patch |
| 16 | |
| 17 | import router |
| 18 | import snowglobe |
| 19 | |
| 20 | |
| 21 | class Hub(BaseHTTPRequestHandler): |
| 22 | protocol_version = "HTTP/1.1" |
| 23 | started = threading.Event() |
| 24 | |
| 25 | def do_GET(self): |
| 26 | if self.path == "/v1/upload-canceled": |
| 27 | type(self).started.set() |
| 28 | self.rfile.read(int(self.headers.get("Content-Length", 0))) |
| 29 | key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" |
| 30 | if self.headers.get("Authorization") != "Bearer " + key: |
| 31 | status, body = 401, b'{"error":"unauthorized"}' |
| 32 | elif self.path.startswith("/v1/transport-canceled"): |
| 33 | status, body = 499, b"" |
| 34 | elif self.path.startswith("/v1/upstream-timeout"): |
| 35 | status, body = 504, b"" |
| 36 | elif self.path.startswith("/v1/slow"): |
| 37 | type(self).started.set() |
| 38 | time.sleep(12) |
| 39 | status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n" |
| 40 | elif self.path.startswith("/v1/cancel"): |
| 41 | type(self).started.set() |
| 42 | time.sleep(1) |
| 43 | status, body = 200, b"finished" |
| 44 | elif self.path.startswith("/v1/broken"): |
| 45 | self.close_connection = True |
| 46 | return |
| 47 | else: |
| 48 | status, body = 200, b'{"object":"list"}' |
| 49 | try: |
| 50 | self.send_response(status) |
| 51 | self.send_header("Content-Length", str(len(body))) |
| 52 | self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json") |
| 53 | self.send_header("Set-Cookie", "response-cookie-private-fixture") |
| 54 | self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) |
| 55 | self.send_header("X-Request-ID", "upstream-private-fixture") |
| 56 | if self.path in {"/v1/transport-canceled-no-retry", "/v1/upstream-timeout"}: |
| 57 | self.send_header("X-Should-Retry", "false") |
| 58 | self.end_headers() |
| 59 | self.wfile.write(body) |
| 60 | except (BrokenPipeError, ConnectionResetError): |
| 61 | pass |
| 62 | |
| 63 | do_POST = do_GET |
| 64 | |
| 65 | def log_message(self, *_): |
| 66 | pass |
| 67 | |
| 68 | |
| 69 | class Login(BaseHTTPRequestHandler): |
| 70 | def do_GET(self): |
| 71 | self.send_response(302) |
| 72 | self.send_header("Location", "/snow.oauth2/start") |
| 73 | self.end_headers() |
| 74 | |
| 75 | def log_message(self, *_): |
| 76 | pass |
| 77 | |
| 78 | |
| 79 | def main(): |
| 80 | parser = argparse.ArgumentParser() |
| 81 | parser.add_argument("--caddy", required=True) |
| 82 | args = parser.parse_args() |
| 83 | with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login: |
| 84 | root = Path(temporary) |
| 85 | routes = root / "routes" |
| 86 | routes.mkdir() |
| 87 | proof = root / "proof" |
| 88 | proof.write_text("a" * 64) |
| 89 | service = "substation-preview-1234abcd" |
| 90 | host = service + ".snowglobe.test" |
| 91 | paths = {"/v1/*": "hub", "/v0/management/*": "hub"} |
| 92 | data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False, |
| 93 | "uid": os.getuid(), "secrets": {}, "requiredSecrets": [], |
| 94 | "containers": {"app": {"rootGroup": False, "volumes": {}, "http": { |
| 95 | "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}}, |
| 96 | "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}} |
| 97 | with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0): |
| 98 | snowglobe.provision(data) |
| 99 | manifest = json.loads((routes / (service + ".json")).read_text()) |
| 100 | assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths} |
| 101 | instances = { |
| 102 | service: {"ServiceName": service, "JobID": service, "AllocID": service, |
| 103 | "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]}, |
| 104 | service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service, |
| 105 | "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []}, |
| 106 | "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria", |
| 107 | "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]}, |
| 108 | } |
| 109 | |
| 110 | def nomad(path, _): |
| 111 | if path == "/v1/services": |
| 112 | return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}] |
| 113 | if path.startswith("/v1/service/"): |
| 114 | item = instances.get(path.rsplit("/", 1)[1]) |
| 115 | return [item] if item else [] |
| 116 | return {"http": {"Status": "success"}} |
| 117 | |
| 118 | environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port), |
| 119 | "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary, |
| 120 | "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")} |
| 121 | environment.pop("SNOWGLOBE_INTERNAL_PORT", None) |
| 122 | with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)): |
| 123 | rendered = router.render("fixture") |
| 124 | assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered |
| 125 | instances[service + "-hub"]["JobID"] = "substation" |
| 126 | try: |
| 127 | router.render("fixture") |
| 128 | except ValueError: |
| 129 | pass |
| 130 | else: |
| 131 | raise AssertionError("preview routed to production") |
| 132 | instances[service + "-hub"]["JobID"] = service |
| 133 | missing = instances.pop(service + "-hub") |
| 134 | assert "respond 503" in router.render("fixture") |
| 135 | instances[service + "-hub"] = missing |
| 136 | ports = [] |
| 137 | for _ in range(2): |
| 138 | with socket.socket() as listener: |
| 139 | listener.bind(("127.0.0.1", 0)) |
| 140 | ports.append(listener.getsockname()[1]) |
| 141 | port, admin = ports |
| 142 | site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] |
| 143 | site = site.replace(" tls internal\n", "") |
| 144 | config = root / "Caddyfile" |
| 145 | content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n" |
| 146 | config.write_text(content) |
| 147 | for backend in (hub, login): |
| 148 | threading.Thread(target=backend.serve_forever, daemon=True).start() |
| 149 | log_path = root / "caddy.log" |
| 150 | tls = ssl._create_unverified_context() |
| 151 | with log_path.open("w") as log: |
| 152 | process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) |
| 153 | try: |
| 154 | def request(path, key=None, method="GET", body=None, **headers): |
| 155 | if key: |
| 156 | headers["Authorization"] = "Bearer " + key |
| 157 | connection = HTTPSConnection("localhost", port, timeout=20, context=tls) |
| 158 | connection.request(method, path, body=body, headers=headers) |
| 159 | response = connection.getresponse() |
| 160 | result = response.status, dict(response.getheaders()), response.read() |
| 161 | connection.close() |
| 162 | return result |
| 163 | |
| 164 | deadline = time.monotonic() + 10 |
| 165 | while True: |
| 166 | try: |
| 167 | assert request("/")[0] == 302 |
| 168 | break |
| 169 | except OSError: |
| 170 | assert process.poll() is None and time.monotonic() < deadline, log_path.read_text() |
| 171 | time.sleep(.05) |
| 172 | for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401), |
| 173 | ("/v0/management/version", "client-fixture", 401), |
| 174 | ("/v1/models", "client-fixture", 200), |
| 175 | ("/v0/management/version", "management-fixture", 200)]: |
| 176 | result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"}) |
| 177 | assert result == status, (path, result) |
| 178 | assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] |
| 179 | assert headers["X-Request-Id"] != "spoof-private-fixture" |
| 180 | assert "X-Should-Retry" not in headers |
| 181 | status, headers, _ = request("/v1/transport-canceled", "client-fixture") |
| 182 | assert status == 499 and headers["X-Should-Retry"] == "true" |
| 183 | status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture") |
| 184 | assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers) |
| 185 | assert "Connection" not in headers |
| 186 | status, headers, _ = request("/v1/upstream-timeout", "client-fixture") |
| 187 | assert status == 504 and headers["X-Should-Retry"] == "false" and "Connection" not in headers |
| 188 | for size in (100000, 2000000): |
| 189 | wire = tls.wrap_socket(socket.create_connection(("127.0.0.1", port), timeout=5), server_hostname="localhost") |
| 190 | wire.sendall(("POST /v1/upload-idle HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer client-fixture\r\n" |
| 191 | f"Content-Length: {size}\r\n\r\n").encode() + b"x" * 40000) |
| 192 | response = HTTPResponse(wire) |
| 193 | response.begin() |
| 194 | assert response.status in (499, 504), response.status |
| 195 | response.read() |
| 196 | assert response.headers["Connection"] == "close", dict(response.getheaders()) |
| 197 | assert response.headers["X-Should-Retry"] == "true" |
| 198 | assert re.fullmatch(r"[0-9a-f-]{36}", response.headers["X-Request-ID"]) |
| 199 | try: |
| 200 | wire.sendall(b"x" * (size - 40000)) |
| 201 | assert wire.recv(4096) == b"", "late request body became another response" |
| 202 | except (BrokenPipeError, ConnectionResetError, ssl.SSLError): |
| 203 | pass |
| 204 | finally: |
| 205 | response.close() |
| 206 | wire.close() |
| 207 | client = HTTPSConnection("localhost", port, timeout=5, context=tls) |
| 208 | for _ in range(2): |
| 209 | client.request("GET", "/v1/models", headers={"Authorization": "Bearer client-fixture"}) |
| 210 | response = client.getresponse() |
| 211 | assert response.status == 200 and response.headers.get("Connection") != "close" |
| 212 | response.read() |
| 213 | client.close() |
| 214 | assert request("/api/summary", "client-fixture")[0] == 302 |
| 215 | assert request("/v1", "client-fixture")[0] == 302 |
| 216 | Hub.started.clear() |
| 217 | outcome = [] |
| 218 | worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture"))) |
| 219 | worker.start() |
| 220 | assert Hub.started.wait(5) |
| 221 | config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0")) |
| 222 | subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment) |
| 223 | worker.join(timeout=20) |
| 224 | assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome |
| 225 | assert request("/v1/broken", "client-fixture")[0] == 502 |
| 226 | Hub.started.clear() |
| 227 | client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") |
| 228 | client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) |
| 229 | assert Hub.started.wait(5) |
| 230 | client.close() |
| 231 | Hub.started.clear() |
| 232 | client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost") |
| 233 | client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) |
| 234 | assert Hub.started.wait(5) |
| 235 | client.close() |
| 236 | deadline = time.monotonic() + 5 |
| 237 | while True: |
| 238 | entries = [] |
| 239 | for line in log_path.read_text().splitlines(): |
| 240 | try: |
| 241 | entry = json.loads(line) |
| 242 | except ValueError: |
| 243 | continue |
| 244 | if entry.get("service") == service: |
| 245 | entries.append(entry) |
| 246 | if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries): |
| 247 | break |
| 248 | assert time.monotonic() < deadline, entries |
| 249 | time.sleep(.05) |
| 250 | assert all("request" not in entry and "resp_headers" not in entry for entry in entries) |
| 251 | assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) |
| 252 | assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) |
| 253 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) |
| 254 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) |
| 255 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel") |
| 256 | upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") |
| 257 | assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 |
| 258 | complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") |
| 259 | assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture") |
| 260 | assert "private-fixture" not in log_path.read_text() |
| 261 | assert "client-fixture" not in log_path.read_text() |
| 262 | assert "management-fixture" not in log_path.read_text() |
| 263 | finally: |
| 264 | process.terminate() |
| 265 | process.wait(timeout=10) |
| 266 | for backend in (hub, login): |
| 267 | backend.shutdown() |
| 268 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed") |
| 269 | |
| 270 | |
| 271 | if __name__ == "__main__": |
| 272 | main() |