| author | |
| committer | |
| log | 53fd9c969ec080da95877749caebd841358fe8c7 |
| tree | 0d7dba6fc9295e7badb50705abc049b76280f8b7 |
| parent | 665ddec25df72359c856c0603d356d5ce7b2831f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
The recurring Substation error is a Caddy cancellation after 60.045 seconds,
with no upstream response headers. Two other failures hit a client-upload
read timeout at 60 seconds. Full uploads with 65-second upstream header
delays pass against the deployed Caddy binary, so response latency alone
does not reproduce the failure.
Record declared Content-Length beside bytes received to distinguish an
incomplete upload on recurrence. Preserve header and body privacy.
Validated complete and interrupted upload accounting, key separation,
preview isolation, cancellations, slow headers across reload, and private
logs with the deployed Caddy binary. Substation staging remains healthy.
Assisted-by: gpt-6.1-sol3 files changed, 19 insertions(+), 4 deletions(-)
tools/router-api-test.py+12-1| ... | @@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler): | ... | @@ -21,6 +21,8 @@ class Hub(BaseHTTPRequestHandler): |
| 21 | started = threading.Event() | 21 | started = threading.Event() |
| 22 | 22 | ||
| 23 | def do_GET(self): | 23 | def do_GET(self): |
| 24 | if self.path == "/v1/upload-canceled": | ||
| 25 | type(self).started.set() | ||
| 24 | self.rfile.read(int(self.headers.get("Content-Length", 0))) | 26 | self.rfile.read(int(self.headers.get("Content-Length", 0))) |
| 25 | key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" | 27 | key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture" |
| 26 | if self.headers.get("Authorization") != "Bearer " + key: | 28 | if self.headers.get("Authorization") != "Bearer " + key: |
| ... | @@ -180,6 +182,11 @@ def main(): | ... | @@ -180,6 +182,11 @@ def main(): |
| 180 | assert request("/v1/broken", "client-fixture")[0] == 502 | 182 | assert request("/v1/broken", "client-fixture")[0] == 502 |
| 181 | Hub.started.clear() | 183 | Hub.started.clear() |
| 182 | client = socket.create_connection(("127.0.0.1", port)) | 184 | client = socket.create_connection(("127.0.0.1", port)) |
| 185 | client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000) | ||
| 186 | assert Hub.started.wait(5) | ||
| 187 | client.close() | ||
| 188 | Hub.started.clear() | ||
| 189 | client = socket.create_connection(("127.0.0.1", port)) | ||
| 183 | client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) | 190 | client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode()) |
| 184 | assert Hub.started.wait(5) | 191 | assert Hub.started.wait(5) |
| 185 | client.close() | 192 | client.close() |
| ... | @@ -202,6 +209,10 @@ def main(): | ... | @@ -202,6 +209,10 @@ def main(): |
| 202 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) | 209 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) |
| 203 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) | 210 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) |
| 204 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) | 211 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) |
| 212 | upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled") | ||
| 213 | assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000 | ||
| 214 | complete = next(entry for entry in entries if entry.get("path") == "/v1/slow") | ||
| 215 | assert int(complete["content_length"]) == complete["bytes_read"] == len(b"body-private-fixture") | ||
| 205 | assert "private-fixture" not in log_path.read_text() | 216 | assert "private-fixture" not in log_path.read_text() |
| 206 | assert "client-fixture" not in log_path.read_text() | 217 | assert "client-fixture" not in log_path.read_text() |
| 207 | assert "management-fixture" not in log_path.read_text() | 218 | assert "management-fixture" not in log_path.read_text() |
| ... | @@ -210,7 +221,7 @@ def main(): | ... | @@ -210,7 +221,7 @@ def main(): |
| 210 | process.wait(timeout=10) | 221 | process.wait(timeout=10) |
| 211 | for backend in (hub, login): | 222 | for backend in (hub, login): |
| 212 | backend.shutdown() | 223 | backend.shutdown() |
| 213 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, disconnect 499, upstream 502, private logs and trace discovery passed") | 224 | print("API routes: key separation, browser login, preview isolation, slow headers through reload, cancellation 499, incomplete upload diagnostics, upstream 502, private logs and trace discovery passed") |
| 214 | 225 | ||
| 215 | 226 | ||
| 216 | if __name__ == "__main__": | 227 | if __name__ == "__main__": |
tools/router.py+1| ... | @@ -369,6 +369,7 @@ def render(token): | ... | @@ -369,6 +369,7 @@ def render(token): |
| 369 | " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", | 369 | " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", |
| 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", | 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", |
| 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", | 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", |
| 372 | " log_append content_length {http.request.header.Content-Length}", | ||
| 372 | " log_append upstream {http.reverse_proxy.upstream.hostport}", | 373 | " log_append upstream {http.reverse_proxy.upstream.hostport}", |
| 373 | " log_append upstream_status {http.reverse_proxy.status_code}", | 374 | " log_append upstream_status {http.reverse_proxy.status_code}", |
| 374 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", | 375 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", |
tools/substation.md+6-3| ... | @@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request. | ... | @@ -100,9 +100,10 @@ store; restoring them requires a separate explicit request. |
| 100 | 100 | ||
| 101 | API responses include a Caddy-generated `X-Request-ID`. API access records in | 101 | API responses include a Caddy-generated `X-Request-ID`. API access records in |
| 102 | Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, | 102 | Caddy's journal contain that ID, trace ID, path, method, protocol, status, bytes, |
| 103 | duration, upstream address, upstream status, and time to upstream headers. | 103 | duration, declared content length, upstream address, upstream status, and time |
| 104 | Headers, query strings, cookies, and request/response bodies are excluded from | 104 | to upstream headers. Other headers, query strings, cookies, and request/response |
| 105 | these access records. The log shipper retains them in Victoria Logs. | 105 | bodies are excluded from these access records. The log shipper retains them in |
| 106 | Victoria Logs. | ||
| 106 | 107 | ||
| 107 | The `monitor` task checks anonymous rejection, client-key model access, and | 108 | The `monitor` task checks anonymous rejection, client-key model access, and |
| 108 | management-key version access through HTTPS every minute. It requires the edge | 109 | management-key version access through HTTPS every minute. It requires the edge |
| ... | @@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs. | ... | @@ -112,6 +113,8 @@ API. It sends no generation requests and logs only check outcomes and IDs. |
| 112 | Inspect `substation` task `monitor` in the service logs, and search Caddy's | 113 | Inspect `substation` task `monitor` in the service logs, and search Caddy's |
| 113 | journal for the response's request ID. A 499 with no upstream status marks a | 114 | journal for the response's request ID. A 499 with no upstream status marks a |
| 114 | cancellation before upstream headers; an upstream 499 is recorded separately. | 115 | cancellation before upstream headers; an upstream 499 is recorded separately. |
| 116 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length | ||
| 117 | uploads. A missing length can mean a chunked request; it does not prove completion. | ||
| 115 | A 200 followed by a truncated stream needs the edge byte count and the hub's | 118 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| 116 | stream outcome. A successful probe does not establish that every generation | 119 | stream outcome. A successful probe does not establish that every generation |
| 117 | stream completed. | 120 | stream completed. |