From 53fd9c969ec080da95877749caebd841358fe8c7 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 15:00:14 -0800 Subject: [PATCH] Record declared API upload size for cancellation diagnosis The recurring Substation error is a Caddy cancellation after 60.045 seconds, with no upstream response headers. Two other failures hit a client-upload read timeout at 60 seconds. Full uploads with 65-second upstream header delays pass against the deployed Caddy binary, so response latency alone does not reproduce the failure. Record declared Content-Length beside bytes received to distinguish an incomplete upload on recurrence. Preserve header and body privacy. Validated complete and interrupted upload accounting, key separation, preview isolation, cancellations, slow headers across reload, and private logs with the deployed Caddy binary. Substation staging remains healthy. Assisted-by: gpt-6.1-sol --- tools/router-api-test.py | 13 ++++++++++++- tools/router.py | 1 + tools/substation.md | 9 ++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/router-api-test.py b/tools/router-api-test.py index ce2a6c2bfec9e33a814f2987671e85a682f0b3bf..0920e6804e0c058867dd1fb9bfe66916a4beb76b 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler): started = threading.Event() def do_GET(self): + if self.path == "/v1/upload-canceled": + type(self).started.set() self.rfile.read(int(self.headers.get("Content-Length", 0))) key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" if self.headers.get("Authorization") != "Bearer " + key: @@ -180,6 +182,11 @@ def main(): assert request("/v1/broken", "client-fixture")[0] == 502 Hub.started.clear() client = socket.create_connection(("127.0.0.1", port)) + client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) + assert Hub.started.wait(5) + client.close() + Hub.started.clear() + client = socket.create_connection(("127.0.0.1", port)) client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) assert Hub.started.wait(5) client.close() @@ -202,6 +209,10 @@ def main(): assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") + assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 + complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") + assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture") assert "private-fixture" not in log_path.read_text() assert "client-fixture" not in log_path.read_text() assert "management-fixture" not in log_path.read_text() @@ -210,7 +221,7 @@ def main(): process.wait(timeout=10) for backend in (hub, login): backend.shutdown() - print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") + print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed") if __name__ == "__main__": diff --git a/tools/router.py b/tools/router.py index ecdf535ab8471e80b5079be7f8e60d64017f61fc..958afc20c75bf7be06a3d9d6dd13e740ab0d49d9 100644 --- a/tools/router.py +++ b/tools/router.py @@ -369,6 +369,7 @@ def render(token): " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", f" log_append service {service}", " log_append request_id {http.request.uuid}", " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", + " log_append content_length {http.request.header.Content-Length}", " log_append upstream {http.reverse_proxy.upstream.hostport}", " log_append upstream_status {http.reverse_proxy.status_code}", " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", diff --git a/tools/substation.md b/tools/substation.md index 9d66d714533c47c46e37cd14301b46049ceca3fe..701dd0df289bae4ff05b2360d6eaaaf24c750083 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request. API responses include a Caddy-generated `X-Request-ID`. API access records in Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, -duration, upstream address, upstream status, and time to upstream headers. -Headers, query strings, cookies, and request/response bodies are excluded from -these access records. The log shipper retains them in Victoria Logs. +duration, declared content length, upstream address, upstream status, and time +to upstream headers. Other headers, query strings, cookies, and request/response +bodies are excluded from these access records. The log shipper retains them in +Victoria Logs. The `monitor` task checks anonymous rejection, client-key model access, and management-key version access through HTTPS every minute. It requires the edge @@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs. Inspect `substation` task `monitor` in the service logs, and search Caddy's journal for the response's request ID. A 499 with no upstream status marks a cancellation before upstream headers; an upstream 499 is recorded separately. +Compare `bytes_read` with `content_length` to identify incomplete fixed-length +uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's stream outcome. A successful probe does not establish that every generation stream completed. -- 2.54.1