| author | |
| committer | |
| log | fc19eea96637a4f7f3b9716a4d5d2ed01cff0163 |
| tree | e58455f0472b558440581f2e8156890e6fa639e4 |
| parent | 9f7ec5c28c4d001c4d36e7c68550acb86e533950 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A stalled upload can leave Caddy 2.11.7 parsing queued JSON as the next request,
returning an unlogged 431. Clearing the failed read deadline also lets net/http
block draining a small unfinished body before the error handler runs.
Patch Caddy to retain failed-body deadlines through transport closure and close
HTTP/1 when its direct cancellation path writes 499. API error routes close
499/504 connections and attach the request ID and retry hint. Normal requests,
upstream retry decisions, and HTTP/2 connection behavior remain intact.
The deployed binary reproduced both the 431 and the small-body drain hang.
The rebuilt binary passed real-Caddy regressions for 100 KB and 2 MB incomplete
uploads, late body bytes, keep-alive, upstream retry decisions, auth separation,
streaming through reload, private logs and tracing. An HTTP/2 error control
retained retry metadata without Connection headers. Ten deployment tests passed.
Substation staging health and the complete NixOS build passed.
Paired captures isolated the remaining stalls to Worktop TCP loss recovery. Five
fresh RACK-off Bun uploads recovered after the original and first repair were
dropped; a contiguous-loss control and native production canary also completed.
A root LaunchDaemon reapplies RACK-off at boot. Eight client watcher tests pass.
Assisted-by: gpt-sol-6.1
Assisted-by: gpt-sol-6:daybreak6 files changed, 104 insertions(+), 13 deletions(-)
nixos/caddy-body-timeout.patch created+35| ... | ... | @@ -0,0 +1,35 @@ |
| 1 | --- a/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/idletimeout.go | |
| 2 | +++ b/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/idletimeout.go | |
| 3 | @@ -69,8 +69,8 @@ | |
| 4 | // IdleTimeoutReader wraps a request body with IdleDeadline, resetting | |
| 5 | // the read deadline before every Read call instead of bounding the | |
| 6 | // whole body transfer with a single hard deadline. | |
| 7 | -// The deadline is cleared after an error (most likely io.EOF) is encountered when reading the body | |
| 8 | -// to prevent context canceled error for h1 requests. | |
| 9 | +// EOF clears the deadline to permit HTTP/1 keep-alive. | |
| 10 | +// Failed reads keep it armed so net/http cannot block draining the body in Close. | |
| 11 | // see: https://github.com/caddyserver/caddy/issues/8103 | |
| 12 | type IdleTimeoutReader struct { | |
| 13 | 	io.ReadCloser | |
| 14 | @@ -108,7 +108,7 @@ | |
| 15 | 	r.Deadline.transferred += int64(n) | |
| 16 | 	if err != nil { | |
| 17 | 		r.terminalErr = err | |
| 18 | -		if !r.finished { | |
| 19 | +		if err == io.EOF && !r.finished { | |
| 20 | 			r.clearDeadlineLocked() | |
| 21 | 		} | |
| 22 | 	} | |
| 23 | --- a/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go | |
| 24 | +++ b/vendor/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go | |
| 25 | @@ -735,6 +735,10 @@ | |
| 26 | 		// request, which is not our failure; don't retry or ding the upstream. | |
| 27 | 		// Record a 499 (client closed request) so the access log reflects the | |
| 28 | 		// disconnect instead of a misleading 0 status (see #7396). | |
| 29 | +		// Cancellation can leave body bytes queued on a still-open client socket. | |
| 30 | +		if origReq.ProtoMajor == 1 { | |
| 31 | +			w.Header().Set("Connection", "close") | |
| 32 | +		} | |
| 33 | 		w.WriteHeader(499) | |
| 34 | 		return true, nil | |
| 35 | 	} |
nixos/configuration.nix+4-2| ... | ... | @@ -83,10 +83,12 @@ in |
| 83 | 83 | services.openssh.enable = true; |
| 84 | 84 | services.caddy = { |
| 85 | 85 | enable = true; |
| 86 | package = pkgs.caddy.withPlugins { | |
| 86 | package = (pkgs.caddy.withPlugins { | |
| 87 | 87 | plugins = [ "github.com/caddyserver/replace-response@v0.0.0-20250618171559-80962887e4c6" ]; |
| 88 | 88 | hash = "sha256-iWtaVD4LrOgCFZc90crAlHYbHHxz8zCFnvD2ZLRkO0Q="; |
| 89 | }; | |
| 89 | }).overrideAttrs (old: { | |
| 90 | patches = (old.patches or []) ++ [ ./caddy-body-timeout.patch ]; | |
| 91 | }); | |
| 90 | 92 | extraConfig = "import /var/lib/caddy/routes.caddy"; |
| 91 | 93 | virtualHosts = { |
| 92 | 94 | ":8082".extraConfig = "reverse_proxy 127.0.0.1:4646"; |
tools/net.paperclover.tcp-rack-off.plist created+16| ... | ... | @@ -0,0 +1,16 @@ |
| 1 | <?xml version="1.0" encoding="UTF-8"?> | |
| 2 | <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| 3 | <plist version="1.0"> | |
| 4 | <dict> | |
| 5 | <key>Label</key> | |
| 6 | <string>net.paperclover.tcp-rack-off</string> | |
| 7 | <key>ProgramArguments</key> | |
| 8 | <array> | |
| 9 | <string>/usr/sbin/sysctl</string> | |
| 10 | <string>-w</string> | |
| 11 | <string>net.inet.tcp.rack=0</string> | |
| 12 | </array> | |
| 13 | <key>RunAtLoad</key> | |
| 14 | <true/> | |
| 15 | </dict> | |
| 16 | </plist> |
tools/router-api-test.py+31-11| ... | ... | @@ -1,6 +1,6 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | import argparse |
| 3 | from http.client import HTTPSConnection | |
| 3 | from http.client import HTTPResponse, HTTPSConnection | |
| 4 | 4 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 5 | 5 | import json |
| 6 | 6 | import os |
| ... | ... | @@ -31,6 +31,8 @@ class Hub(BaseHTTPRequestHandler): |
| 31 | 31 | status, body = 401, b'{"error":"unauthorized"}' |
| 32 | 32 | elif self.path.startswith("/v1/transport-canceled"): |
| 33 | 33 | status, body = 499, b"" |
| 34 | elif self.path.startswith("/v1/upstream-timeout"): | |
| 35 | status, body = 504, b"" | |
| 34 | 36 | elif self.path.startswith("/v1/slow"): |
| 35 | 37 | type(self).started.set() |
| 36 | 38 | time.sleep(12) |
| ... | ... | @@ -51,7 +53,7 @@ class Hub(BaseHTTPRequestHandler): |
| 51 | 53 | self.send_header("Set-Cookie", "response-cookie-private-fixture") |
| 52 | 54 | self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", "")) |
| 53 | 55 | self.send_header("X-Request-ID", "upstream-private-fixture") |
| 54 | if self.path == "/v1/transport-canceled-no-retry": | |
| 56 | if self.path in {"/v1/transport-canceled-no-retry", "/v1/upstream-timeout"}: | |
| 55 | 57 | self.send_header("X-Should-Retry", "false") |
| 56 | 58 | self.end_headers() |
| 57 | 59 | self.wfile.write(body) |
| ... | ... | @@ -180,16 +182,34 @@ def main(): |
| 180 | 182 | assert status == 499 and headers["X-Should-Retry"] == "true" |
| 181 | 183 | status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture") |
| 182 | 184 | assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers) |
| 183 | client = HTTPSConnection("localhost", port, timeout=5, context=tls) | |
| 184 | client.putrequest("POST", "/v1/upload-idle") | |
| 185 | client.putheader("Authorization", "Bearer client-fixture") | |
| 186 | client.putheader("Content-Length", "1000000") | |
| 187 | client.endheaders(b"x" * 40000) | |
| 188 | response = client.getresponse() | |
| 189 | assert response.status in (499, 504), response.status | |
| 190 | if response.status == 499: | |
| 185 | assert "Connection" not in headers | |
| 186 | status, headers, _ = request("/v1/upstream-timeout", "client-fixture") | |
| 187 | assert status == 504 and headers["X-Should-Retry"] == "false" and "Connection" not in headers | |
| 188 | for size in (100000, 2000000): | |
| 189 | wire = tls.wrap_socket(socket.create_connection(("127.0.0.1", port), timeout=5), server_hostname="localhost") | |
| 190 | wire.sendall(("POST /v1/upload-idle HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer client-fixture\r\n" | |
| 191 | f"Content-Length: {size}\r\n\r\n").encode() + b"x" * 40000) | |
| 192 | response = HTTPResponse(wire) | |
| 193 | response.begin() | |
| 194 | assert response.status in (499, 504), response.status | |
| 195 | response.read() | |
| 196 | assert response.headers["Connection"] == "close", dict(response.getheaders()) | |
| 191 | 197 | assert response.headers["X-Should-Retry"] == "true" |
| 192 | response.read() | |
| 198 | assert re.fullmatch(r"[0-9a-f-]{36}", response.headers["X-Request-ID"]) | |
| 199 | try: | |
| 200 | wire.sendall(b"x" * (size - 40000)) | |
| 201 | assert wire.recv(4096) == b"", "late request body became another response" | |
| 202 | except (BrokenPipeError, ConnectionResetError, ssl.SSLError): | |
| 203 | pass | |
| 204 | finally: | |
| 205 | response.close() | |
| 206 | wire.close() | |
| 207 | client = HTTPSConnection("localhost", port, timeout=5, context=tls) | |
| 208 | for _ in range(2): | |
| 209 | client.request("GET", "/v1/models", headers={"Authorization": "Bearer client-fixture"}) | |
| 210 | response = client.getresponse() | |
| 211 | assert response.status == 200 and response.headers.get("Connection") != "close" | |
| 212 | response.read() | |
| 193 | 213 | client.close() |
| 194 | 214 | assert request("/api/summary", "client-fixture")[0] == 302 |
| 195 | 215 | assert request("/v1", "client-fixture")[0] == 302 |
tools/router.py+5| ... | ... | @@ -375,6 +375,11 @@ def render(token): |
| 375 | 375 | " log_append upstream_status {http.reverse_proxy.status_code}", |
| 376 | 376 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", |
| 377 | 377 | " log_append proxy_ms {http.reverse_proxy.duration_ms}"] |
| 378 | lines += [" handle_errors 499 504 {", " @api_error path " + " ".join(sorted(api_paths)), | |
| 379 | " handle @api_error {", " @http1 not protocol http/2+", | |
| 380 | # Late body bytes must not become the next HTTP/1 request after a timeout. | |
| 381 | " header @http1 Connection close", " header X-Request-ID {http.request.uuid}", | |
| 382 | " header X-Should-Retry true", " }", ' respond "" {err.status_code}', " }"] | |
| 378 | 383 | for index, target in enumerate(sorted(set(api_paths.values()))): |
| 379 | 384 | upstreams = set() |
| 380 | 385 | for instance in nomad("/v1/service/" + target, token): |
tools/substation.md+13| ... | ... | @@ -116,6 +116,14 @@ client cancellation or Caddy's request-body idle timeout; an upstream 499 is |
| 116 | 116 | recorded separately. |
| 117 | 117 | API 499 responses default to `X-Should-Retry: true` so Claude can recover from |
| 118 | 118 | request-body timeouts; an upstream's explicit retry decision takes precedence. |
| 119 | Caddy-generated API 499/504 errors close HTTP/1 connections and include the | |
| 120 | request ID and retry hint. A timed-out upload can still have queued body bytes; | |
| 121 | reusing that connection lets those bytes become an invalid next request and | |
| 122 | can produce an unlogged 431 before routing. Upstream responses keep their | |
| 123 | retry decisions and connection behavior. | |
| 124 | The packaged [Caddy patch](../nixos/caddy-body-timeout.patch) preserves failed | |
| 125 | read deadlines while the upstream transport closes the body. Otherwise, | |
| 126 | net/http can block draining a small unfinished upload before error handling. | |
| 119 | 127 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length |
| 120 | 128 | uploads. A missing length can mean a chunked request; it does not prove completion. |
| 121 | 129 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| ... | ... | @@ -142,6 +150,11 @@ Worktop's `net.paperclover.substation-watch` LaunchAgent records matching |
| 142 | 150 | Substation sockets under `~/.local/state/substation-watch`. Minute files retain |
| 143 | 151 | 12 hours of process IDs, CPU time/state, TCP queues, byte counters, |
| 144 | 152 | retransmissions and windows. |
| 153 | Paired captures showed Worktop leave a SACK-reported hole unrepaired for 75–109 | |
| 154 | seconds. `net.paperclover.tcp-rack-off` disables RACK before login while retaining | |
| 155 | SACK recovery; it is a system-wide transport trade, not a Substation setting. | |
| 156 | The root-owned LaunchDaemon is installed from the exported | |
| 157 | `tools/net.paperclover.tcp-rack-off.plist` and should exit zero after each boot. | |
| 145 | 158 | The installed script is an exported revision of `tools/substation-client-watch.py`; |
| 146 | 159 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. |
| 147 | 160 | Match UTC timestamps to the server incident before comparing TCP progress. |