| author | |
| committer | |
| log | 665ddec25df72359c856c0603d356d5ce7b2831f |
| tree | c0fadfaa96f627e0c26dd89cf927a080af5850ad |
| parent | 97b5aceb6cd7fb036fe8b18c8cee7b24065b0b82 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Route declared API paths to sibling tasks over loopback. Substation keeps
client and management key checks in the hub; OAuth protects its dashboard.
Generate edge request IDs and record metadata without headers, queries or
bodies, including upstream status and header latency for cancellation diagnosis.
Probe anonymous rejection and both key roles through HTTPS every minute.
Restore the trace ingestion route by selecting the registered task name.
Verified actual staging hub, OAuth and monitor containers behind isolated
Caddy; key separation and preview isolation; slow headers across reload;
client cancellation 499 and upstream failure 502; private logs; existing
WebSocket, cookie, Shale and Zig routes; ten deployment export tests.
Assisted-by: gpt-6.1-sol8 files changed, 383 insertions(+), 6 deletions(-)
config/Service.pkl+2| ... | ... | @@ -29,6 +29,8 @@ class Http { |
| 29 | 29 | /// Internal Prometheus endpoint collected by the home server dashboard. |
| 30 | 30 | metricsPath: String? |
| 31 | 31 | checkHeaders: Mapping<String, String> = new {} |
| 32 | /// API paths authenticated by a sibling task, bypassing the browser login proxy. | |
| 33 | apiPaths: Mapping<String, String> = new {} | |
| 32 | 34 | /// Request path to a file or directory in this service's folder. |
| 33 | 35 | overrideFiles: Mapping<String, String> = new {} |
| 34 | 36 | /// HTML inserted before </head> for the listed page paths. |
service/substation/monitor.js created+47| ... | ... | @@ -0,0 +1,47 @@ |
| 1 | export async function probe(url, clientKey, managementKey) { | |
| 2 | const checks = []; | |
| 3 | for (const [path, key, expected] of [ | |
| 4 | ["/v1/models", null, 401], | |
| 5 | ["/v1/models", clientKey, 200], | |
| 6 | ["/v0/management/version", managementKey, 200], | |
| 7 | ]) { | |
| 8 | const started = performance.now(); | |
| 9 | let status = null; | |
| 10 | let requestId = null; | |
| 11 | let ok = false; | |
| 12 | try { | |
| 13 | const response = await fetch(new URL(path, url), { | |
| 14 | headers: key ? { Authorization: `Bearer ${key}` } : {}, | |
| 15 | redirect: "manual", | |
| 16 | signal: AbortSignal.timeout(10_000), | |
| 17 | }); | |
| 18 | status = response.status; | |
| 19 | const id = response.headers.get("x-request-id"); | |
| 20 | if (/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(id ?? "")) requestId = id; | |
| 21 | const body = await response.json(); | |
| 22 | ok = status === expected && requestId !== null && (expected === 401 | |
| 23 | ? typeof body.error === "string" | |
| 24 | : path === "/v1/models" | |
| 25 | ? body.object === "list" && Array.isArray(body.data) | |
| 26 | : typeof body.version === "string"); | |
| 27 | } catch {} | |
| 28 | checks.push({ path, authenticated: key !== null, status, requestId, ok, | |
| 29 | elapsedMs: Math.round(performance.now() - started) }); | |
| 30 | } | |
| 31 | return checks; | |
| 32 | } | |
| 33 | ||
| 34 | if (import.meta.main) { | |
| 35 | const { SUBSTATION_URL: url, SUBSTATION_CLIENT_KEY: clientKey, SUBSTATION_MANAGEMENT_KEY: managementKey } = process.env; | |
| 36 | if (!url || !clientKey || !managementKey) throw new Error("Substation monitor configuration is incomplete"); | |
| 37 | for (;;) { | |
| 38 | const checks = await probe(url, clientKey, managementKey); | |
| 39 | const ok = checks.every(check => check.ok); | |
| 40 | console.log(JSON.stringify({ event: "substation-probe", level: ok ? "info" : "error", checks })); | |
| 41 | if (process.argv.includes("--once")) { | |
| 42 | process.exitCode = ok ? 0 : 1; | |
| 43 | break; | |
| 44 | } | |
| 45 | await Bun.sleep(60_000); | |
| 46 | } | |
| 47 | } |
service/substation/service.pkl+21-2| ... | ... | @@ -88,10 +88,14 @@ containers { |
| 88 | 88 | containerPort = 4180 |
| 89 | 89 | loopback = true |
| 90 | 90 | subdomain = "substation.ai" |
| 91 | apiPaths { | |
| 92 | ["/v1/*"] = "hub" | |
| 93 | ["/v0/management/*"] = "hub" | |
| 94 | } | |
| 91 | 95 | } |
| 92 | 96 | envTemplate = """ |
| 93 | 97 | OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }} |
| 94 | OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ | |
| 98 | OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/ | |
| 95 | 99 | """ |
| 96 | 100 | env { |
| 97 | 101 | ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)" |
| ... | ... | @@ -105,7 +109,22 @@ containers { |
| 105 | 109 | ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false" |
| 106 | 110 | ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false" |
| 107 | 111 | ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s" |
| 108 | ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/" | |
| 112 | } | |
| 113 | } | |
| 114 | ["monitor"] = (runtime) { | |
| 115 | entrypoint = "/usr/local/bin/bun" | |
| 116 | args { "/monitor.js" } | |
| 117 | cpu = 25 | |
| 118 | memory = 64 | |
| 119 | volumes = new { | |
| 120 | ["/monitor.js"] { config = "monitor.js" } | |
| 121 | ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true } | |
| 122 | } | |
| 123 | extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" } | |
| 124 | env = new { | |
| 125 | ["SUBSTATION_URL"] = "https://\(module.containers["app"].http.hostname)" | |
| 126 | ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}" | |
| 127 | ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}" | |
| 109 | 128 | } |
| 110 | 129 | } |
| 111 | 130 | } |
tools/router-api-test.py created+217| ... | ... | @@ -0,0 +1,217 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import argparse | |
| 3 | from http.client import HTTPConnection | |
| 4 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| 5 | import json | |
| 6 | import os | |
| 7 | from pathlib import Path | |
| 8 | import socket | |
| 9 | import subprocess | |
| 10 | import tempfile | |
| 11 | import threading | |
| 12 | import time | |
| 13 | from unittest.mock import patch | |
| 14 | ||
| 15 | import router | |
| 16 | import snowglobe | |
| 17 | ||
| 18 | ||
| 19 | class Hub(BaseHTTPRequestHandler): | |
| 20 | protocol_version = "HTTP/1.1" | |
| 21 | started = threading.Event() | |
| 22 | ||
| 23 | def do_GET(self): | |
| 24 | self.rfile.read(int(self.headers.get("Content-Length", 0))) | |
| 25 | key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" | |
| 26 | if self.headers.get("Authorization") != "Bearer " + key: | |
| 27 | status, body = 401, b'{"error":"unauthorized"}' | |
| 28 | elif self.path.startswith("/v1/slow"): | |
| 29 | type(self).started.set() | |
| 30 | time.sleep(12) | |
| 31 | status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n" | |
| 32 | elif self.path.startswith("/v1/cancel"): | |
| 33 | type(self).started.set() | |
| 34 | time.sleep(1) | |
| 35 | status, body = 200, b"finished" | |
| 36 | elif self.path.startswith("/v1/broken"): | |
| 37 | self.close_connection = True | |
| 38 | return | |
| 39 | else: | |
| 40 | status, body = 200, b'{"object":"list"}' | |
| 41 | try: | |
| 42 | self.send_response(status) | |
| 43 | self.send_header("Content-Length", str(len(body))) | |
| 44 | self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json") | |
| 45 | self.send_header("Set-Cookie", "response-cookie-private-fixture") | |
| 46 | self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) | |
| 47 | self.send_header("X-Request-ID", "upstream-private-fixture") | |
| 48 | self.end_headers() | |
| 49 | self.wfile.write(body) | |
| 50 | except (BrokenPipeError, ConnectionResetError): | |
| 51 | pass | |
| 52 | ||
| 53 | do_POST = do_GET | |
| 54 | ||
| 55 | def log_message(self, *_): | |
| 56 | pass | |
| 57 | ||
| 58 | ||
| 59 | class Login(BaseHTTPRequestHandler): | |
| 60 | def do_GET(self): | |
| 61 | self.send_response(302) | |
| 62 | self.send_header("Location", "/snow.oauth2/start") | |
| 63 | self.end_headers() | |
| 64 | ||
| 65 | def log_message(self, *_): | |
| 66 | pass | |
| 67 | ||
| 68 | ||
| 69 | def main(): | |
| 70 | parser = argparse.ArgumentParser() | |
| 71 | parser.add_argument("--caddy", required=True) | |
| 72 | args = parser.parse_args() | |
| 73 | with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login: | |
| 74 | root = Path(temporary) | |
| 75 | routes = root / "routes" | |
| 76 | routes.mkdir() | |
| 77 | proof = root / "proof" | |
| 78 | proof.write_text("a" * 64) | |
| 79 | service = "substation-preview-1234abcd" | |
| 80 | host = service + ".snowglobe.test" | |
| 81 | paths = {"/v1/*": "hub", "/v0/management/*": "hub"} | |
| 82 | data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False, | |
| 83 | "uid": os.getuid(), "secrets": {}, "requiredSecrets": [], | |
| 84 | "containers": {"app": {"rootGroup": False, "volumes": {}, "http": { | |
| 85 | "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}}, | |
| 86 | "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}} | |
| 87 | with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0): | |
| 88 | snowglobe.provision(data) | |
| 89 | manifest = json.loads((routes / (service + ".json")).read_text()) | |
| 90 | assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths} | |
| 91 | instances = { | |
| 92 | service: {"ServiceName": service, "JobID": service, "AllocID": service, | |
| 93 | "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]}, | |
| 94 | service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service, | |
| 95 | "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []}, | |
| 96 | "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria", | |
| 97 | "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]}, | |
| 98 | } | |
| 99 | ||
| 100 | def nomad(path, _): | |
| 101 | if path == "/v1/services": | |
| 102 | return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}] | |
| 103 | if path.startswith("/v1/service/"): | |
| 104 | item = instances.get(path.rsplit("/", 1)[1]) | |
| 105 | return [item] if item else [] | |
| 106 | return {"http": {"Status": "success"}} | |
| 107 | ||
| 108 | environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port), | |
| 109 | "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary, | |
| 110 | "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")} | |
| 111 | environment.pop("SNOWGLOBE_INTERNAL_PORT", None) | |
| 112 | with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)): | |
| 113 | rendered = router.render("fixture") | |
| 114 | assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered | |
| 115 | instances[service + "-hub"]["JobID"] = "substation" | |
| 116 | try: | |
| 117 | router.render("fixture") | |
| 118 | except ValueError: | |
| 119 | pass | |
| 120 | else: | |
| 121 | raise AssertionError("preview routed to production") | |
| 122 | instances[service + "-hub"]["JobID"] = service | |
| 123 | missing = instances.pop(service + "-hub") | |
| 124 | assert "respond 503" in router.render("fixture") | |
| 125 | instances[service + "-hub"] = missing | |
| 126 | ports = [] | |
| 127 | for _ in range(2): | |
| 128 | with socket.socket() as listener: | |
| 129 | listener.bind(("127.0.0.1", 0)) | |
| 130 | ports.append(listener.getsockname()[1]) | |
| 131 | port, admin = ports | |
| 132 | site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0] | |
| 133 | site = site.replace(" tls internal\n", "") | |
| 134 | config = root / "Caddyfile" | |
| 135 | content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n" | |
| 136 | config.write_text(content) | |
| 137 | for backend in (hub, login): | |
| 138 | threading.Thread(target=backend.serve_forever, daemon=True).start() | |
| 139 | log_path = root / "caddy.log" | |
| 140 | with log_path.open("w") as log: | |
| 141 | process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment) | |
| 142 | try: | |
| 143 | def request(path, key=None, method="GET", body=None, **headers): | |
| 144 | if key: | |
| 145 | headers["Authorization"] = "Bearer " + key | |
| 146 | connection = HTTPConnection("127.0.0.1", port, timeout=20) | |
| 147 | connection.request(method, path, body=body, headers=headers) | |
| 148 | response = connection.getresponse() | |
| 149 | result = response.status, dict(response.getheaders()), response.read() | |
| 150 | connection.close() | |
| 151 | return result | |
| 152 | ||
| 153 | deadline = time.monotonic() + 10 | |
| 154 | while True: | |
| 155 | try: | |
| 156 | assert request("/")[0] == 302 | |
| 157 | break | |
| 158 | except OSError: | |
| 159 | assert process.poll() is None and time.monotonic() < deadline, log_path.read_text() | |
| 160 | time.sleep(.05) | |
| 161 | for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401), | |
| 162 | ("/v0/management/version", "client-fixture", 401), | |
| 163 | ("/v1/models", "client-fixture", 200), | |
| 164 | ("/v0/management/version", "management-fixture", 200)]: | |
| 165 | result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"}) | |
| 166 | assert result == status, (path, result) | |
| 167 | assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"] | |
| 168 | assert headers["X-Request-Id"] != "spoof-private-fixture" | |
| 169 | assert request("/api/summary", "client-fixture")[0] == 302 | |
| 170 | assert request("/v1", "client-fixture")[0] == 302 | |
| 171 | Hub.started.clear() | |
| 172 | outcome = [] | |
| 173 | worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture"))) | |
| 174 | worker.start() | |
| 175 | assert Hub.started.wait(5) | |
| 176 | config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0")) | |
| 177 | subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment) | |
| 178 | worker.join(timeout=20) | |
| 179 | assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome | |
| 180 | assert request("/v1/broken", "client-fixture")[0] == 502 | |
| 181 | Hub.started.clear() | |
| 182 | client = socket.create_connection(("127.0.0.1", port)) | |
| 183 | client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) | |
| 184 | assert Hub.started.wait(5) | |
| 185 | client.close() | |
| 186 | deadline = time.monotonic() + 5 | |
| 187 | while True: | |
| 188 | entries = [] | |
| 189 | for line in log_path.read_text().splitlines(): | |
| 190 | try: | |
| 191 | entry = json.loads(line) | |
| 192 | except ValueError: | |
| 193 | continue | |
| 194 | if entry.get("service") == service: | |
| 195 | entries.append(entry) | |
| 196 | if any(entry.get("status") == 499 for entry in entries): | |
| 197 | break | |
| 198 | assert time.monotonic() < deadline, entries | |
| 199 | time.sleep(.05) | |
| 200 | assert all("request" not in entry and "resp_headers" not in entry for entry in entries) | |
| 201 | assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) | |
| 202 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) | |
| 203 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) | |
| 204 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) | |
| 205 | assert "private-fixture" not in log_path.read_text() | |
| 206 | assert "client-fixture" not in log_path.read_text() | |
| 207 | assert "management-fixture" not in log_path.read_text() | |
| 208 | finally: | |
| 209 | process.terminate() | |
| 210 | process.wait(timeout=10) | |
| 211 | for backend in (hub, login): | |
| 212 | backend.shutdown() | |
| 213 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") | |
| 214 | ||
| 215 | ||
| 216 | if __name__ == "__main__": | |
| 217 | main() |
tools/router.py+28-2| ... | ... | @@ -359,6 +359,32 @@ def render(token): |
| 359 | 359 | if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path): |
| 360 | 360 | raise ValueError(f"invalid metrics path: {host}") |
| 361 | 361 | lines += [f" handle {metrics_path} {{", " respond 404", " }"] |
| 362 | api_paths = assets.get("apiPaths", {}).get(host, {}) | |
| 363 | if api_paths: | |
| 364 | if any(not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) or not re.fullmatch(r"[a-z][a-z0-9-]*", target) | |
| 365 | or not (target == service or target.startswith(service + "-")) for path, target in api_paths.items()): | |
| 366 | raise ValueError(f"invalid API routes: {host}") | |
| 367 | lines += [f" log {service} {{", " output stdout", " format filter {", | |
| 368 | " request delete", " resp_headers delete", " user_id delete", " }", " }", | |
| 369 | " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", | |
| 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", | |
| 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", | |
| 372 | " log_append upstream {http.reverse_proxy.upstream.hostport}", | |
| 373 | " log_append upstream_status {http.reverse_proxy.status_code}", | |
| 374 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", | |
| 375 | " log_append proxy_ms {http.reverse_proxy.duration_ms}"] | |
| 376 | for index, target in enumerate(sorted(set(api_paths.values()))): | |
| 377 | upstreams = set() | |
| 378 | for instance in nomad("/v1/service/" + target, token): | |
| 379 | address, port = instance["Address"], instance["Port"] | |
| 380 | if instance.get("JobID") != service or not re.fullmatch(r"127\.0\.0\.1|::1", address) or type(port) is not int or not 1 <= port <= 65535: | |
| 381 | raise ValueError(f"API upstream must belong to {service} on loopback: {target}") | |
| 382 | upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") | |
| 383 | paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) | |
| 384 | lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", | |
| 385 | " request_header X-Request-ID {http.request.uuid}"] | |
| 386 | lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] | |
| 387 | lines.append(" }") | |
| 362 | 388 | upstreams = " ".join(sorted(route["upstreams"])) |
| 363 | 389 | if role: |
| 364 | 390 | auth = " ".join(sorted(auth_upstreams)) |
| ... | ... | @@ -438,10 +464,10 @@ def render(token): |
| 438 | 464 | raise ValueError("invalid login domain") |
| 439 | 465 | lines += [f"{auth_host} {{", *([" tls internal"] if auth_host.endswith(".test") else []), |
| 440 | 466 | *webauthn, " handle {", f" redir https://{dashboard_host}/ 308", " }", "}"] |
| 441 | traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None) | |
| 467 | traces = internal_services.get("victoria-traces") | |
| 442 | 468 | if traces: |
| 443 | 469 | lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1", |
| 444 | *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"] | |
| 470 | *proxy(" ".join(sorted(traces)), " "), "}"] | |
| 445 | 471 | dashboard_port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) |
| 446 | 472 | if not 1 <= dashboard_port <= 65535: |
| 447 | 473 | raise ValueError("invalid dashboard route") |
tools/snowglobe.py+11-1| ... | ... | @@ -558,6 +558,7 @@ def provision(data): |
| 558 | 558 | identity = {} |
| 559 | 559 | head_html = {} |
| 560 | 560 | metrics_paths = {} |
| 561 | api_paths = {} | |
| 561 | 562 | for task in data["containers"].values(): |
| 562 | 563 | if task.get("http"): |
| 563 | 564 | http = task["http"] |
| ... | ... | @@ -567,6 +568,15 @@ def provision(data): |
| 567 | 568 | head_html[http["hostname"]] = http["headHtml"] |
| 568 | 569 | if http.get("metricsPath") and http["hostname"]: |
| 569 | 570 | metrics_paths[http["hostname"]] = http["metricsPath"] |
| 571 | if http.get("apiPaths"): | |
| 572 | targets = {} | |
| 573 | for path, task_name in http["apiPaths"].items(): | |
| 574 | target = data["containers"].get(task_name) | |
| 575 | if (not http["hostname"] or not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) | |
| 576 | or not target or not (target.get("http") or target.get("tcp"))): | |
| 577 | raise ValueError(f"invalid API route for {data['id']}: {path}") | |
| 578 | targets[path] = data["id"] if task_name == "app" else data["id"] + "-" + task_name | |
| 579 | api_paths[http["hostname"]] = targets | |
| 570 | 580 | for request, rel in http["overrideFiles"].items(): |
| 571 | 581 | origin = assets[rel] |
| 572 | 582 | target = directories if origin.is_dir() else static |
| ... | ... | @@ -625,7 +635,7 @@ def provision(data): |
| 625 | 635 | proof[http["hostname"]] = "X-Snowglobe-Idp-" + values[name] |
| 626 | 636 | write_private(route_dir / f"{data['id']}.json", json.dumps({ |
| 627 | 637 | "files": static, "dirs": directories, "identity": identity, |
| 628 | "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, | |
| 638 | "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, "apiPaths": api_paths, | |
| 629 | 639 | })) |
| 630 | 640 | |
| 631 | 641 |
tools/substation-monitor-test.js created+35| ... | ... | @@ -0,0 +1,35 @@ |
| 1 | import assert from "node:assert/strict"; | |
| 2 | import { probe } from "../service/substation/monitor.js"; | |
| 3 | ||
| 4 | const requestId = "2c42c339-cd80-4f22-a4d7-61a411bba5e8"; | |
| 5 | const replies = [ | |
| 6 | [401, { error: "unauthorized" }], | |
| 7 | [200, { object: "list", data: [] }], | |
| 8 | [200, { version: "fixture" }], | |
| 9 | ]; | |
| 10 | let calls = []; | |
| 11 | let scenario = "healthy"; | |
| 12 | globalThis.fetch = async (url, options) => { | |
| 13 | const index = calls.length; | |
| 14 | calls.push({ url: String(url), options }); | |
| 15 | if (scenario === "network") throw new Error("client-private-fixture"); | |
| 16 | const [status, body] = replies[index]; | |
| 17 | return new Response(JSON.stringify(scenario === "body" ? { private: "client-private-fixture" } : body), { | |
| 18 | status: scenario === "redirect" ? 302 : status, | |
| 19 | headers: { "X-Request-ID": scenario === "id" ? "client-private-fixture" : requestId }, | |
| 20 | }); | |
| 21 | }; | |
| 22 | for (scenario of ["healthy", "redirect", "body", "id", "network"]) { | |
| 23 | calls = []; | |
| 24 | const checks = await probe("https://substation.invalid", "client-private-fixture", "management-private-fixture"); | |
| 25 | assert.equal(checks.length, 3); | |
| 26 | assert.equal(checks.every(check => check.ok), scenario === "healthy"); | |
| 27 | assert.ok(calls.every(call => call.options.redirect === "manual" && call.options.signal instanceof AbortSignal)); | |
| 28 | assert.equal(calls[0].options.headers.Authorization, undefined); | |
| 29 | assert.equal(calls[1].options.headers.Authorization, "Bearer client-private-fixture"); | |
| 30 | assert.equal(calls[2].options.headers.Authorization, "Bearer management-private-fixture"); | |
| 31 | assert.ok(!JSON.stringify(checks).includes("private-fixture")); | |
| 32 | if (scenario === "healthy") assert.ok(checks.every(check => check.requestId === requestId)); | |
| 33 | if (scenario === "id" || scenario === "network") assert.ok(checks.every(check => check.requestId === null)); | |
| 34 | } | |
| 35 | console.log("Substation monitor: authentication, redirect rejection, response validation, request IDs and private failures passed"); |
tools/substation.md+22-1| ... | ... | @@ -3,7 +3,8 @@ |
| 3 | 3 | `https://substation.ai.paperclover.net` serves the Substation dashboard and its |
| 4 | 4 | native APIs. Snow OAuth protects the dashboard and requires `infra-admin`. |
| 5 | 5 | Substation's trusted-proxy mode accepts the identity header from the local |
| 6 | OAuth proxy. Hub requests authenticate with Substation's own keys. | |
| 6 | OAuth proxy. Caddy sends `/v1/*` and `/v0/management/*` directly to the hub, | |
| 7 | which checks its own client and management keys. | |
| 7 | 8 | |
| 8 | 9 | ## Set up accounts |
| 9 | 10 | |
| ... | ... | @@ -94,3 +95,23 @@ The in-app updater is unavailable inside the immutable image. |
| 94 | 95 | CPA's live dataset, preview, keys, and routing are retired after the replacement |
| 95 | 96 | passes production checks. Restricted off-server recovery copies retain the old |
| 96 | 97 | store; restoring them requires a separate explicit request. |
| 98 | ||
| 99 | ## Request diagnostics | |
| 100 | ||
| 101 | API responses include a Caddy-generated `X-Request-ID`. API access records in | |
| 102 | Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, | |
| 103 | duration, upstream address, upstream status, and time to upstream headers. | |
| 104 | Headers, query strings, cookies, and request/response bodies are excluded from | |
| 105 | these access records. The log shipper retains them in Victoria Logs. | |
| 106 | ||
| 107 | The `monitor` task checks anonymous rejection, client-key model access, and | |
| 108 | management-key version access through HTTPS every minute. It requires the edge | |
| 109 | request ID and rejects redirects, so an OAuth login page cannot pass as a healthy | |
| 110 | API. It sends no generation requests and logs only check outcomes and IDs. | |
| 111 | ||
| 112 | Inspect `substation` task `monitor` in the service logs, and search Caddy's | |
| 113 | journal for the response's request ID. A 499 with no upstream status marks a | |
| 114 | cancellation before upstream headers; an upstream 499 is recorded separately. | |
| 115 | A 200 followed by a truncated stream needs the edge byte count and the hub's | |
| 116 | stream outcome. A successful probe does not establish that every generation | |
| 117 | stream completed. |