diff --git a/tools/router-api-test.py b/tools/router-api-test.py index ce2a6c2bfec9e33a814f2987671e85a682f0b3bf..0920e6804e0c058867dd1fb9bfe66916a4beb76b 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler): started = threading.Event() def do_GET(self): + if self.path == "/v1/upload-canceled": + type(self).started.set() self.rfile.read(int(self.headers.get("Content-Length", 0))) key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" if self.headers.get("Authorization") != "Bearer " + key: @@ -180,6 +182,11 @@ def main(): assert request("/v1/broken", "client-fixture")[0] == 502 Hub.started.clear() client = socket.create_connection(("127.0.0.1", port)) + client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) + assert Hub.started.wait(5) + client.close() + Hub.started.clear() + client = socket.create_connection(("127.0.0.1", port)) client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) assert Hub.started.wait(5) client.close() @@ -202,6 +209,10 @@ def main(): assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) + upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") + assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 + complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") + assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture") assert "private-fixture" not in log_path.read_text() assert "client-fixture" not in log_path.read_text() assert "management-fixture" not in log_path.read_text() @@ -210,7 +221,7 @@ def main(): process.wait(timeout=10) for backend in (hub, login): backend.shutdown() - print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") + print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed") if __name__ == "__main__": diff --git a/tools/router.py b/tools/router.py index ecdf535ab8471e80b5079be7f8e60d64017f61fc..958afc20c75bf7be06a3d9d6dd13e740ab0d49d9 100644 --- a/tools/router.py +++ b/tools/router.py @@ -369,6 +369,7 @@ def render(token): " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", f" log_append service {service}", " log_append request_id {http.request.uuid}", " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", + " log_append content_length {http.request.header.Content-Length}", " log_append upstream {http.reverse_proxy.upstream.hostport}", " log_append upstream_status {http.reverse_proxy.status_code}", " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", diff --git a/tools/substation.md b/tools/substation.md index 9d66d714533c47c46e37cd14301b46049ceca3fe..701dd0df289bae4ff05b2360d6eaaaf24c750083 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request. API responses include a Caddy-generated `X-Request-ID`. API access records in Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, -duration, upstream address, upstream status, and time to upstream headers. -Headers, query strings, cookies, and request/response bodies are excluded from -these access records. The log shipper retains them in Victoria Logs. +duration, declared content length, upstream address, upstream status, and time +to upstream headers. Other headers, query strings, cookies, and request/response +bodies are excluded from these access records. The log shipper retains them in +Victoria Logs. The `monitor` task checks anonymous rejection, client-key model access, and management-key version access through HTTPS every minute. It requires the edge @@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs. Inspect `substation` task `monitor` in the service logs, and search Caddy's journal for the response's request ID. A 499 with no upstream status marks a cancellation before upstream headers; an upstream 499 is recorded separately. +Compare `bytes_read` with `content_length` to identify incomplete fixed-length +uploads. A missing length can mean a chunked request; it does not prove completion. A 200 followed by a truncated stream needs the edge byte count and the hub's stream outcome. A successful probe does not establish that every generation stream completed.