authoravatarclover caruso<git@paperclover.net>2026-11-11 12:36:39-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log665ddec25df72359c856c0603d356d5ce7b2831f
treec0fadfaa96f627e0c26dd89cf927a080af5850ad
parent97b5aceb6cd7fb036fe8b18c8cee7b24065b0b82
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Bypass OAuth for Substation APIs and monitor request failures

Route declared API paths to sibling tasks over loopback. Substation keeps client and management key checks in the hub; OAuth protects its dashboard. Generate edge request IDs and record metadata without headers, queries or bodies, including upstream status and header latency for cancellation diagnosis. Probe anonymous rejection and both key roles through HTTPS every minute. Restore the trace ingestion route by selecting the registered task name. Verified actual staging hub, OAuth and monitor containers behind isolated Caddy; key separation and preview isolation; slow headers across reload; client cancellation 499 and upstream failure 502; private logs; existing WebSocket, cookie, Shale and Zig routes; ten deployment export tests. Assisted-by: gpt-6.1-sol

8 files changed, 383 insertions(+), 6 deletions(-)

config/Service.pkl+2
......@@ -29,6 +29,8 @@ class Http {
2929 /// Internal Prometheus endpoint collected by the home server dashboard.
3030 metricsPath: String?
3131 checkHeaders: Mapping<String, String> = new {}
32 /// API paths authenticated by a sibling task, bypassing the browser login proxy.
33 apiPaths: Mapping<String, String> = new {}
3234 /// Request path to a file or directory in this service's folder.
3335 overrideFiles: Mapping<String, String> = new {}
3436 /// HTML inserted before </head> for the listed page paths.
service/substation/monitor.js created+47
......@@ -0,0 +1,47 @@
1export async function probe(url, clientKey, managementKey) {
2 const checks = [];
3 for (const [path, key, expected] of [
4 ["/v1/models", null, 401],
5 ["/v1/models", clientKey, 200],
6 ["/v0/management/version", managementKey, 200],
7 ]) {
8 const started = performance.now();
9 let status = null;
10 let requestId = null;
11 let ok = false;
12 try {
13 const response = await fetch(new URL(path, url), {
14 headers: key ? { Authorization: `Bearer ${key}` } : {},
15 redirect: "manual",
16 signal: AbortSignal.timeout(10_000),
17 });
18 status = response.status;
19 const id = response.headers.get("x-request-id");
20 if (/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(id ?? "")) requestId = id;
21 const body = await response.json();
22 ok = status === expected && requestId !== null && (expected === 401
23 ? typeof body.error === "string"
24 : path === "/v1/models"
25 ? body.object === "list" && Array.isArray(body.data)
26 : typeof body.version === "string");
27 } catch {}
28 checks.push({ path, authenticated: key !== null, status, requestId, ok,
29 elapsedMs: Math.round(performance.now() - started) });
30 }
31 return checks;
32}
33
34if (import.meta.main) {
35 const { SUBSTATION_URL: url, SUBSTATION_CLIENT_KEY: clientKey, SUBSTATION_MANAGEMENT_KEY: managementKey } = process.env;
36 if (!url || !clientKey || !managementKey) throw new Error("Substation monitor configuration is incomplete");
37 for (;;) {
38 const checks = await probe(url, clientKey, managementKey);
39 const ok = checks.every(check => check.ok);
40 console.log(JSON.stringify({ event: "substation-probe", level: ok ? "info" : "error", checks }));
41 if (process.argv.includes("--once")) {
42 process.exitCode = ok ? 0 : 1;
43 break;
44 }
45 await Bun.sleep(60_000);
46 }
47}
service/substation/service.pkl+21-2
......@@ -88,10 +88,14 @@ containers {
8888 containerPort = 4180
8989 loopback = true
9090 subdomain = "substation.ai"
91 apiPaths {
92 ["/v1/*"] = "hub"
93 ["/v0/management/*"] = "hub"
94 }
9195 }
9296 envTemplate = """
9397 OAUTH2_PROXY_HTTP_ADDRESS=127.0.0.1:{{ env "NOMAD_PORT_app_http" }}
94 OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v1/,http://127.0.0.1:{{ env "NOMAD_PORT_hub_api" }}/v0/management/,http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/
98 OAUTH2_PROXY_UPSTREAMS=http://127.0.0.1:{{ env "NOMAD_PORT_dashboard_web" }}/
9599 """
96100 env {
97101 ["OAUTH2_PROXY_COOKIE_NAME"] = "_snow_\(module.id)"
......@@ -105,7 +109,22 @@ containers {
105109 ["OAUTH2_PROXY_PASS_ACCESS_TOKEN"] = "false"
106110 ["OAUTH2_PROXY_PASS_BASIC_AUTH"] = "false"
107111 ["OAUTH2_PROXY_UPSTREAM_TIMEOUT"] = "0s"
108 ["OAUTH2_PROXY_SKIP_AUTH_ROUTES"] = "^/v1/,^/v0/management/"
112 }
113 }
114 ["monitor"] = (runtime) {
115 entrypoint = "/usr/local/bin/bun"
116 args { "/monitor.js" }
117 cpu = 25
118 memory = 64
119 volumes = new {
120 ["/monitor.js"] { config = "monitor.js" }
121 ["/etc/ssl/certs/ca-certificates.crt"] { src = "/var/lib/snowglobe/ca-bundle.crt"; readOnly = true }
122 }
123 extraHosts { "\(module.containers["app"].http.hostname):127.0.0.1" }
124 env = new {
125 ["SUBSTATION_URL"] = "https://\(module.containers["app"].http.hostname)"
126 ["SUBSTATION_CLIENT_KEY"] = "${secret.own.api_key}"
127 ["SUBSTATION_MANAGEMENT_KEY"] = "${secret.own.management_key}"
109128 }
110129 }
111130}
tools/router-api-test.py created+217
......@@ -0,0 +1,217 @@
1#!/usr/bin/env python3
2import argparse
3from http.client import HTTPConnection
4from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5import json
6import os
7from pathlib import Path
8import socket
9import subprocess
10import tempfile
11import threading
12import time
13from unittest.mock import patch
14
15import router
16import snowglobe
17
18
19class Hub(BaseHTTPRequestHandler):
20 protocol_version = "HTTP/1.1"
21 started = threading.Event()
22
23 def do_GET(self):
24 self.rfile.read(int(self.headers.get("Content-Length", 0)))
25 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
26 if self.headers.get("Authorization") != "Bearer " + key:
27 status, body = 401, b'{"error":"unauthorized"}'
28 elif self.path.startswith("/v1/slow"):
29 type(self).started.set()
30 time.sleep(12)
31 status, body = 200, b"data: response-private-fixture\n\ndata: complete\n\n"
32 elif self.path.startswith("/v1/cancel"):
33 type(self).started.set()
34 time.sleep(1)
35 status, body = 200, b"finished"
36 elif self.path.startswith("/v1/broken"):
37 self.close_connection = True
38 return
39 else:
40 status, body = 200, b'{"object":"list"}'
41 try:
42 self.send_response(status)
43 self.send_header("Content-Length", str(len(body)))
44 self.send_header("Content-Type", "text/event-stream" if self.path.startswith("/v1/slow") else "application/json")
45 self.send_header("Set-Cookie", "response-cookie-private-fixture")
46 self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", ""))
47 self.send_header("X-Request-ID", "upstream-private-fixture")
48 self.end_headers()
49 self.wfile.write(body)
50 except (BrokenPipeError, ConnectionResetError):
51 pass
52
53 do_POST = do_GET
54
55 def log_message(self, *_):
56 pass
57
58
59class Login(BaseHTTPRequestHandler):
60 def do_GET(self):
61 self.send_response(302)
62 self.send_header("Location", "/snow.oauth2/start")
63 self.end_headers()
64
65 def log_message(self, *_):
66 pass
67
68
69def main():
70 parser = argparse.ArgumentParser()
71 parser.add_argument("--caddy", required=True)
72 args = parser.parse_args()
73 with tempfile.TemporaryDirectory(prefix="snowglobe-api-route-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Hub) as hub, ThreadingHTTPServer(("127.0.0.1", 0), Login) as login:
74 root = Path(temporary)
75 routes = root / "routes"
76 routes.mkdir()
77 proof = root / "proof"
78 proof.write_text("a" * 64)
79 service = "substation-preview-1234abcd"
80 host = service + ".snowglobe.test"
81 paths = {"/v1/*": "hub", "/v0/management/*": "hub"}
82 data = {"id": service, "sourceId": "substation", "hostRoot": str(root), "hasManagedVolumes": False,
83 "uid": os.getuid(), "secrets": {}, "requiredSecrets": [],
84 "containers": {"app": {"rootGroup": False, "volumes": {}, "http": {
85 "hostname": host, "identityHeaders": {}, "headHtml": {}, "overrideFiles": {}, "apiPaths": paths}},
86 "hub": {"rootGroup": False, "volumes": {}, "tcp": [{"name": "api"}]}}}
87 with patch.object(snowglobe, "STATE", root), patch.object(snowglobe, "ASSETS", root / "assets"), patch.object(snowglobe.os, "geteuid", return_value=0):
88 snowglobe.provision(data)
89 manifest = json.loads((routes / (service + ".json")).read_text())
90 assert manifest["apiPaths"][host] == {path: service + "-hub" for path in paths}
91 instances = {
92 service: {"ServiceName": service, "JobID": service, "AllocID": service,
93 "Address": "127.0.0.1", "Port": login.server_port, "Tags": ["caddy-host=" + host]},
94 service + "-hub": {"ServiceName": service + "-hub", "JobID": service, "AllocID": service,
95 "Address": "127.0.0.1", "Port": hub.server_port, "Tags": []},
96 "victoria-traces": {"ServiceName": "victoria-traces", "JobID": "victoria", "AllocID": "victoria",
97 "Address": "127.0.0.1", "Port": hub.server_port, "Tags": ["caddy-host=traces.snowglobe.test"]},
98 }
99
100 def nomad(path, _):
101 if path == "/v1/services":
102 return [{"Namespace": "default", "Services": [{"ServiceName": name} for name in instances]}]
103 if path.startswith("/v1/service/"):
104 item = instances.get(path.rsplit("/", 1)[1])
105 return [item] if item else []
106 return {"http": {"Status": "success"}}
107
108 environment = {**os.environ, "SNOWGLOBE_DOMAIN": "snowglobe.test", "SNOWGLOBE_DASHBOARD_PORT": str(login.server_port),
109 "SNOWGLOBE_PROXY_TOKEN_FILE": str(proof), "SNOWGLOBE_STATE": temporary,
110 "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")}
111 environment.pop("SNOWGLOBE_INTERNAL_PORT", None)
112 with patch.dict(os.environ, environment, clear=True), patch.object(router, "nomad", nomad), patch.object(router, "ROUTE_DIR", str(routes)):
113 rendered = router.render("fixture")
114 assert f"http://127.0.0.1:10428 {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{hub.server_port}" in rendered
115 instances[service + "-hub"]["JobID"] = "substation"
116 try:
117 router.render("fixture")
118 except ValueError:
119 pass
120 else:
121 raise AssertionError("preview routed to production")
122 instances[service + "-hub"]["JobID"] = service
123 missing = instances.pop(service + "-hub")
124 assert "respond 503" in router.render("fixture")
125 instances[service + "-hub"] = missing
126 ports = []
127 for _ in range(2):
128 with socket.socket() as listener:
129 listener.bind(("127.0.0.1", 0))
130 ports.append(listener.getsockname()[1])
131 port, admin = ports
132 site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0]
133 site = site.replace(" tls internal\n", "")
134 config = root / "Caddyfile"
135 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n"
136 config.write_text(content)
137 for backend in (hub, login):
138 threading.Thread(target=backend.serve_forever, daemon=True).start()
139 log_path = root / "caddy.log"
140 with log_path.open("w") as log:
141 process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment)
142 try:
143 def request(path, key=None, method="GET", body=None, **headers):
144 if key:
145 headers["Authorization"] = "Bearer " + key
146 connection = HTTPConnection("127.0.0.1", port, timeout=20)
147 connection.request(method, path, body=body, headers=headers)
148 response = connection.getresponse()
149 result = response.status, dict(response.getheaders()), response.read()
150 connection.close()
151 return result
152
153 deadline = time.monotonic() + 10
154 while True:
155 try:
156 assert request("/")[0] == 302
157 break
158 except OSError:
159 assert process.poll() is None and time.monotonic() < deadline, log_path.read_text()
160 time.sleep(.05)
161 for path, key, status in [("/v1/models", None, 401), ("/v1/models", "management-fixture", 401),
162 ("/v0/management/version", "client-fixture", 401),
163 ("/v1/models", "client-fixture", 200),
164 ("/v0/management/version", "management-fixture", 200)]:
165 result, headers, _ = request(path, key, **{"X-Forwarded-User": "spoof-private-fixture", "X-Request-ID": "spoof-private-fixture"})
166 assert result == status, (path, result)
167 assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"]
168 assert headers["X-Request-Id"] != "spoof-private-fixture"
169 assert request("/api/summary", "client-fixture")[0] == 302
170 assert request("/v1", "client-fixture")[0] == 302
171 Hub.started.clear()
172 outcome = []
173 worker = threading.Thread(target=lambda: outcome.append(request("/v1/slow?query-private-fixture", "client-fixture", "POST", b"body-private-fixture", Cookie="cookie-private-fixture")))
174 worker.start()
175 assert Hub.started.wait(5)
176 config.write_text(content.replace(" handle @api_0", " header X-Reload verified\n handle @api_0"))
177 subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment)
178 worker.join(timeout=20)
179 assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome
180 assert request("/v1/broken", "client-fixture")[0] == 502
181 Hub.started.clear()
182 client = socket.create_connection(("127.0.0.1", port))
183 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
184 assert Hub.started.wait(5)
185 client.close()
186 deadline = time.monotonic() + 5
187 while True:
188 entries = []
189 for line in log_path.read_text().splitlines():
190 try:
191 entry = json.loads(line)
192 except ValueError:
193 continue
194 if entry.get("service") == service:
195 entries.append(entry)
196 if any(entry.get("status") == 499 for entry in entries):
197 break
198 assert time.monotonic() < deadline, entries
199 time.sleep(.05)
200 assert all("request" not in entry and "resp_headers" not in entry for entry in entries)
201 assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries)
202 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
203 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
204 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
205 assert "private-fixture" not in log_path.read_text()
206 assert "client-fixture" not in log_path.read_text()
207 assert "management-fixture" not in log_path.read_text()
208 finally:
209 process.terminate()
210 process.wait(timeout=10)
211 for backend in (hub, login):
212 backend.shutdown()
213 print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed")
214
215
216if __name__ == "__main__":
217 main()
tools/router.py+28-2
......@@ -359,6 +359,32 @@ def render(token):
359359 if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path):
360360 raise ValueError(f"invalid metrics path: {host}")
361361 lines += [f" handle {metrics_path} {{", " respond 404", " }"]
362 api_paths = assets.get("apiPaths", {}).get(host, {})
363 if api_paths:
364 if any(not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) or not re.fullmatch(r"[a-z][a-z0-9-]*", target)
365 or not (target == service or target.startswith(service + "-")) for path, target in api_paths.items()):
366 raise ValueError(f"invalid API routes: {host}")
367 lines += [f" log {service} {{", " output stdout", " format filter {",
368 " request delete", " resp_headers delete", " user_id delete", " }", " }",
369 " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api",
370 f" log_append service {service}", " log_append request_id {http.request.uuid}",
371 " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}",
372 " log_append upstream {http.reverse_proxy.upstream.hostport}",
373 " log_append upstream_status {http.reverse_proxy.status_code}",
374 " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}",
375 " log_append proxy_ms {http.reverse_proxy.duration_ms}"]
376 for index, target in enumerate(sorted(set(api_paths.values()))):
377 upstreams = set()
378 for instance in nomad("/v1/service/" + target, token):
379 address, port = instance["Address"], instance["Port"]
380 if instance.get("JobID") != service or not re.fullmatch(r"127\.0\.0\.1|::1", address) or type(port) is not int or not 1 <= port <= 65535:
381 raise ValueError(f"API upstream must belong to {service} on loopback: {target}")
382 upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}")
383 paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target))
384 lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}",
385 " request_header X-Request-ID {http.request.uuid}"]
386 lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"]
387 lines.append(" }")
362388 upstreams = " ".join(sorted(route["upstreams"]))
363389 if role:
364390 auth = " ".join(sorted(auth_upstreams))
......@@ -438,10 +464,10 @@ def render(token):
438464 raise ValueError("invalid login domain")
439465 lines += [f"{auth_host} {{", *([" tls internal"] if auth_host.endswith(".test") else []),
440466 *webauthn, " handle {", f" redir https://{dashboard_host}/ 308", " }", "}"]
441 traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None)
467 traces = internal_services.get("victoria-traces")
442468 if traces:
443469 lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1",
444 *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"]
470 *proxy(" ".join(sorted(traces)), " "), "}"]
445471 dashboard_port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"])
446472 if not 1 <= dashboard_port <= 65535:
447473 raise ValueError("invalid dashboard route")
tools/snowglobe.py+11-1
......@@ -558,6 +558,7 @@ def provision(data):
558558 identity = {}
559559 head_html = {}
560560 metrics_paths = {}
561 api_paths = {}
561562 for task in data["containers"].values():
562563 if task.get("http"):
563564 http = task["http"]
......@@ -567,6 +568,15 @@ def provision(data):
567568 head_html[http["hostname"]] = http["headHtml"]
568569 if http.get("metricsPath") and http["hostname"]:
569570 metrics_paths[http["hostname"]] = http["metricsPath"]
571 if http.get("apiPaths"):
572 targets = {}
573 for path, task_name in http["apiPaths"].items():
574 target = data["containers"].get(task_name)
575 if (not http["hostname"] or not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path)
576 or not target or not (target.get("http") or target.get("tcp"))):
577 raise ValueError(f"invalid API route for {data['id']}: {path}")
578 targets[path] = data["id"] if task_name == "app" else data["id"] + "-" + task_name
579 api_paths[http["hostname"]] = targets
570580 for request, rel in http["overrideFiles"].items():
571581 origin = assets[rel]
572582 target = directories if origin.is_dir() else static
......@@ -625,7 +635,7 @@ def provision(data):
625635 proof[http["hostname"]] = "X-Snowglobe-Idp-" + values[name]
626636 write_private(route_dir / f"{data['id']}.json", json.dumps({
627637 "files": static, "dirs": directories, "identity": identity,
628 "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths,
638 "identityProof": proof, "headHtml": head_html, "metricsPaths": metrics_paths, "apiPaths": api_paths,
629639 }))
630640
631641
tools/substation-monitor-test.js created+35
......@@ -0,0 +1,35 @@
1import assert from "node:assert/strict";
2import { probe } from "../service/substation/monitor.js";
3
4const requestId = "2c42c339-cd80-4f22-a4d7-61a411bba5e8";
5const replies = [
6 [401, { error: "unauthorized" }],
7 [200, { object: "list", data: [] }],
8 [200, { version: "fixture" }],
9];
10let calls = [];
11let scenario = "healthy";
12globalThis.fetch = async (url, options) => {
13 const index = calls.length;
14 calls.push({ url: String(url), options });
15 if (scenario === "network") throw new Error("client-private-fixture");
16 const [status, body] = replies[index];
17 return new Response(JSON.stringify(scenario === "body" ? { private: "client-private-fixture" } : body), {
18 status: scenario === "redirect" ? 302 : status,
19 headers: { "X-Request-ID": scenario === "id" ? "client-private-fixture" : requestId },
20 });
21};
22for (scenario of ["healthy", "redirect", "body", "id", "network"]) {
23 calls = [];
24 const checks = await probe("https://substation.invalid", "client-private-fixture", "management-private-fixture");
25 assert.equal(checks.length, 3);
26 assert.equal(checks.every(check => check.ok), scenario === "healthy");
27 assert.ok(calls.every(call => call.options.redirect === "manual" && call.options.signal instanceof AbortSignal));
28 assert.equal(calls[0].options.headers.Authorization, undefined);
29 assert.equal(calls[1].options.headers.Authorization, "Bearer client-private-fixture");
30 assert.equal(calls[2].options.headers.Authorization, "Bearer management-private-fixture");
31 assert.ok(!JSON.stringify(checks).includes("private-fixture"));
32 if (scenario === "healthy") assert.ok(checks.every(check => check.requestId === requestId));
33 if (scenario === "id" || scenario === "network") assert.ok(checks.every(check => check.requestId === null));
34}
35console.log("Substation monitor: authentication, redirect rejection, response validation, request IDs and private failures passed");
tools/substation.md+22-1
......@@ -3,7 +3,8 @@
33`https://substation.ai.paperclover.net` serves the Substation dashboard and its
44native APIs. Snow OAuth protects the dashboard and requires `infra-admin`.
55Substation's trusted-proxy mode accepts the identity header from the local
6OAuth proxy. Hub requests authenticate with Substation's own keys.
6OAuth proxy. Caddy sends `/v1/*` and `/v0/management/*` directly to the hub,
7which checks its own client and management keys.
78
89## Set up accounts
910
......@@ -94,3 +95,23 @@ The in-app updater is unavailable inside the immutable image.
9495CPA's live dataset, preview, keys, and routing are retired after the replacement
9596passes production checks. Restricted off-server recovery copies retain the old
9697store; restoring them requires a separate explicit request.
98
99## Request diagnostics
100
101API responses include a Caddy-generated `X-Request-ID`. API access records in
102Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes,
103duration, upstream address, upstream status, and time to upstream headers.
104Headers, query strings, cookies, and request/response bodies are excluded from
105these access records. The log shipper retains them in Victoria Logs.
106
107The `monitor` task checks anonymous rejection, client-key model access, and
108management-key version access through HTTPS every minute. It requires the edge
109request ID and rejects redirects, so an OAuth login page cannot pass as a healthy
110API. It sends no generation requests and logs only check outcomes and IDs.
111
112Inspect `substation` task `monitor` in the service logs, and search Caddy's
113journal for the response's request ID. A 499 with no upstream status marks a
114cancellation before upstream headers; an upstream 499 is recorded separately.
115A 200 followed by a truncated stream needs the edge byte count and the hub's
116stream outcome. A successful probe does not establish that every generation
117stream completed.