authoravatarclover caruso<git@paperclover.net>2026-11-11 16:07:30-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640
tree3b9faa611d878270804a80696fe449a765ce6bca
parentc65330b68ae0ff9f783e00a339d61006cf9c8ff0
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Let Claude retry stalled Substation uploads

A captured Worktop upload stopped at 1.70 MB of 4.19 MB with an open server receive window, an unfilled TCP sequence hole, repeated SACKs and 1.78 MB queued at the sender. Caddy can report its body idle timeout as 499; Claude then treats this transient transport failure as a terminal client error. Default API 499 responses to X-Should-Retry: true while preserving an upstream retry decision. Keep the original status and diagnostics. Validated the generated routes against the deployed Caddy 2.11.7 over TLS: idle upload timeout, retry hints and explicit refusal, key separation, browser login, preview isolation, stream completion through reload and log privacy. Worktop Claude 2.1.322 failed a bare-499 fixture in one attempt, then retried and completed OK in two attempts with the hint. Staging healthy. Assisted-by: gpt-6.1-sol

3 files changed, 36 insertions(+), 9 deletions(-)

tools/router-api-test.py+31-9
......@@ -1,12 +1,13 @@
11#!/usr/bin/env python3
22import argparse
3from http.client import HTTPConnection
3from http.client import HTTPSConnection
44from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
55import json
66import os
77from pathlib import Path
88import re
99import socket
10import ssl
1011import subprocess
1112import tempfile
1213import threading
......@@ -28,6 +29,8 @@ class Hub(BaseHTTPRequestHandler):
2829 key = "management-fixture" if self.path.startswith("/v0/") else "client-fixture"
2930 if self.headers.get("Authorization") != "Bearer " + key:
3031 status, body = 401, b'{"error":"unauthorized"}'
32 elif self.path.startswith("/v1/transport-canceled"):
33 status, body = 499, b""
3134 elif self.path.startswith("/v1/slow"):
3235 type(self).started.set()
3336 time.sleep(12)
......@@ -48,6 +51,8 @@ class Hub(BaseHTTPRequestHandler):
4851 self.send_header("Set-Cookie", "response-cookie-private-fixture")
4952 self.send_header("X-Fixture-Request-ID", self.headers.get("X-Request-ID", ""))
5053 self.send_header("X-Request-ID", "upstream-private-fixture")
54 if self.path == "/v1/transport-canceled-no-retry":
55 self.send_header("X-Should-Retry", "false")
5156 self.end_headers()
5257 self.wfile.write(body)
5358 except (BrokenPipeError, ConnectionResetError):
......@@ -135,18 +140,19 @@ def main():
135140 site = rendered.split(host + " {\n", 1)[1].split("\n}\n", 1)[0]
136141 site = site.replace(" tls internal\n", "")
137142 config = root / "Caddyfile"
138 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\nhttp://127.0.0.1:{port} {{\n{site}\n}}\n"
143 content = f"{{\n admin 127.0.0.1:{admin}\n auto_https disable_redirects\n skip_install_trust\n servers {{\n timeouts {{\n read_body_idle 1s\n }}\n }}\n}}\nhttps://localhost:{port} {{\n bind 127.0.0.1\n tls internal\n{site}\n}}\n"
139144 config.write_text(content)
140145 for backend in (hub, login):
141146 threading.Thread(target=backend.serve_forever, daemon=True).start()
142147 log_path = root / "caddy.log"
148 tls = ssl._create_unverified_context()
143149 with log_path.open("w") as log:
144150 process = subprocess.Popen([args.caddy, "run", "--config", str(config), "--adapter", "caddyfile"], stdout=log, stderr=log, env=environment)
145151 try:
146152 def request(path, key=None, method="GET", body=None, **headers):
147153 if key:
148154 headers["Authorization"] = "Bearer " + key
149 connection = HTTPConnection("127.0.0.1", port, timeout=20)
155 connection = HTTPSConnection("localhost", port, timeout=20, context=tls)
150156 connection.request(method, path, body=body, headers=headers)
151157 response = connection.getresponse()
152158 result = response.status, dict(response.getheaders()), response.read()
......@@ -169,6 +175,22 @@ def main():
169175 assert result == status, (path, result)
170176 assert headers["X-Request-Id"] == headers["X-Fixture-Request-Id"]
171177 assert headers["X-Request-Id"] != "spoof-private-fixture"
178 assert "X-Should-Retry" not in headers
179 status, headers, _ = request("/v1/transport-canceled", "client-fixture")
180 assert status == 499 and headers["X-Should-Retry"] == "true"
181 status, headers, _ = request("/v1/transport-canceled-no-retry", "client-fixture")
182 assert status == 499 and headers["X-Should-Retry"] == "false", (status, headers)
183 client = HTTPSConnection("localhost", port, timeout=5, context=tls)
184 client.putrequest("POST", "/v1/upload-idle")
185 client.putheader("Authorization", "Bearer client-fixture")
186 client.putheader("Content-Length", "1000000")
187 client.endheaders(b"x" * 40000)
188 response = client.getresponse()
189 assert response.status in (499, 504), response.status
190 if response.status == 499:
191 assert response.headers["X-Should-Retry"] == "true"
192 response.read()
193 client.close()
172194 assert request("/api/summary", "client-fixture")[0] == 302
173195 assert request("/v1", "client-fixture")[0] == 302
174196 Hub.started.clear()
......@@ -182,13 +204,13 @@ def main():
182204 assert outcome and outcome[0][0] == 200 and outcome[0][2].endswith(b"data: complete\n\n"), outcome
183205 assert request("/v1/broken", "client-fixture")[0] == 502
184206 Hub.started.clear()
185 client = socket.create_connection(("127.0.0.1", port))
186 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
207 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
208 client.sendall(f"POST /v1/upload-canceled HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\nContent-Length: 100000\r\n\r\n".encode() + b"x" * 40000)
187209 assert Hub.started.wait(5)
188210 client.close()
189211 Hub.started.clear()
190 client = socket.create_connection(("127.0.0.1", port))
191 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
212 client = tls.wrap_socket(socket.create_connection(("127.0.0.1", port)), server_hostname="localhost")
213 client.sendall(f"GET /v1/cancel HTTP/1.1\r\nHost: localhost:{port}\r\nAuthorization: Bearer client-fixture\r\n\r\n".encode())
192214 assert Hub.started.wait(5)
193215 client.close()
194216 deadline = time.monotonic() + 5
......@@ -201,7 +223,7 @@ def main():
201223 continue
202224 if entry.get("service") == service:
203225 entries.append(entry)
204 if any(entry.get("status") == 499 for entry in entries):
226 if any(entry.get("status") == 499 and entry.get("path") == "/v1/cancel" for entry in entries):
205227 break
206228 assert time.monotonic() < deadline, entries
207229 time.sleep(.05)
......@@ -210,7 +232,7 @@ def main():
210232 assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries)
211233 assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200)
212234 assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200)
213 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499)
235 assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499 and entry.get("path") == "/v1/cancel")
214236 upload = next(entry for entry in entries if entry.get("path") == "/v1/upload-canceled")
215237 assert upload["status"] in (499, 502) and upload["content_length"] == "100000" and upload["bytes_read"] == 40000
216238 complete = next(entry for entry in entries if entry.get("path") == "/v1/slow")
tools/router.py+3
......@@ -384,6 +384,9 @@ def render(token):
384384 upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}")
385385 paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target))
386386 lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}",
387 # Caddy can return 499 to a connected client when its request-body idle timer expires.
388 " header X-Should-Retry true {", " match {", " status 499",
389 " header !X-Should-Retry", " }", " }",
387390 " request_header X-Request-ID {http.request.uuid}"]
388391 lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"]
389392 lines.append(" }")
tools/substation.md+2
......@@ -114,6 +114,8 @@ Inspect `substation` task `monitor` in the service logs, and search Caddy's
114114journal for the response's request ID. A 499 with no upstream status can mean
115115client cancellation or Caddy's request-body idle timeout; an upstream 499 is
116116recorded separately.
117API 499 responses default to `X-Should-Retry: true` so Claude can recover from
118request-body timeouts; an upstream's explicit retry decision takes precedence.
117119Compare `bytes_read` with `content_length` to identify incomplete fixed-length
118120uploads. A missing length can mean a chunked request; it does not prove completion.
119121A 200 followed by a truncated stream needs the edge byte count and the hub's