| author | |
| committer | |
| log | 53fd9c969ec080da95877749caebd841358fe8c7 |
| tree | 0d7dba6fc9295e7badb50705abc049b76280f8b7 |
| parent | 665ddec25df72359c856c0603d356d5ce7b2831f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
The recurring Substation error is a Caddy cancellation after 60.045 seconds,
with no upstream response headers. Two other failures hit a client-upload
read timeout at 60 seconds. Full uploads with 65-second upstream header
delays pass against the deployed Caddy binary, so response latency alone
does not reproduce the failure.
Record declared Content-Length beside bytes received to distinguish an
incomplete upload on recurrence. Preserve header and body privacy.
Validated complete and interrupted upload accounting, key separation,
preview isolation, cancellations, slow headers across reload, and private
logs with the deployed Caddy binary. Substation staging remains healthy.
Assisted-by: gpt-6.1-sol3 files changed, 19 insertions(+), 4 deletions(-)
tools/router-api-test.py+12-1| ... | ... | @@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler): |
| 21 | 21 | started = threading.Event() |
| 22 | 22 | |
| 23 | 23 | def do_GET(self): |
| 24 | if self.path == "/v1/upload-canceled": | |
| 25 | type(self).started.set() | |
| 24 | 26 | self.rfile.read(int(self.headers.get("Content-Length", 0))) |
| 25 | 27 | key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" |
| 26 | 28 | if self.headers.get("Authorization") != "Bearer " + key: |
| ... | ... | @@ -180,6 +182,11 @@ def main(): |
| 180 | 182 | assert request("/v1/broken", "client-fixture")[0] == 502 |
| 181 | 183 | Hub.started.clear() |
| 182 | 184 | client = socket.create_connection(("127.0.0.1", port)) |
| 185 | client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) | |
| 186 | assert Hub.started.wait(5) | |
| 187 | client.close() | |
| 188 | Hub.started.clear() | |
| 189 | client = socket.create_connection(("127.0.0.1", port)) | |
| 183 | 190 | client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) |
| 184 | 191 | assert Hub.started.wait(5) |
| 185 | 192 | client.close() |
| ... | ... | @@ -202,6 +209,10 @@ def main(): |
| 202 | 209 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) |
| 203 | 210 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) |
| 204 | 211 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) |
| 212 | upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") | |
| 213 | assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 | |
| 214 | complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") | |
| 215 | assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture") | |
| 205 | 216 | assert "private-fixture" not in log_path.read_text() |
| 206 | 217 | assert "client-fixture" not in log_path.read_text() |
| 207 | 218 | assert "management-fixture" not in log_path.read_text() |
| ... | ... | @@ -210,7 +221,7 @@ def main(): |
| 210 | 221 | process.wait(timeout=10) |
| 211 | 222 | for backend in (hub, login): |
| 212 | 223 | backend.shutdown() |
| 213 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") | |
| 224 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed") | |
| 214 | 225 | |
| 215 | 226 | |
| 216 | 227 | if __name__ == "__main__": |
tools/router.py+1| ... | ... | @@ -369,6 +369,7 @@ def render(token): |
| 369 | 369 | " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", |
| 370 | 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", |
| 371 | 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", |
| 372 | " log_append content_length {http.request.header.Content-Length}", | |
| 372 | 373 | " log_append upstream {http.reverse_proxy.upstream.hostport}", |
| 373 | 374 | " log_append upstream_status {http.reverse_proxy.status_code}", |
| 374 | 375 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", |
tools/substation.md+6-3| ... | ... | @@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request. |
| 100 | 100 | |
| 101 | 101 | API responses include a Caddy-generated `X-Request-ID`. API access records in |
| 102 | 102 | Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, |
| 103 | duration, upstream address, upstream status, and time to upstream headers. | |
| 104 | Headers, query strings, cookies, and request/response bodies are excluded from | |
| 105 | these access records. The log shipper retains them in Victoria Logs. | |
| 103 | duration, declared content length, upstream address, upstream status, and time | |
| 104 | to upstream headers. Other headers, query strings, cookies, and request/response | |
| 105 | bodies are excluded from these access records. The log shipper retains them in | |
| 106 | Victoria Logs. | |
| 106 | 107 | |
| 107 | 108 | The `monitor` task checks anonymous rejection, client-key model access, and |
| 108 | 109 | management-key version access through HTTPS every minute. It requires the edge |
| ... | ... | @@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs. |
| 112 | 113 | Inspect `substation` task `monitor` in the service logs, and search Caddy's |
| 113 | 114 | journal for the response's request ID. A 499 with no upstream status marks a |
| 114 | 115 | cancellation before upstream headers; an upstream 499 is recorded separately. |
| 116 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length | |
| 117 | uploads. A missing length can mean a chunked request; it does not prove completion. | |
| 115 | 118 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| 116 | 119 | stream outcome. A successful probe does not establish that every generation |
| 117 | 120 | stream completed. |