From 3b735d29265d065eff0ee72b7943eb7795f87bbb Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 16:43:57 -0800 Subject: [PATCH] Retain Worktop upload stalls before edge timeouts Archive sender-side evidence when queued upload bytes make no acknowledgement progress for ten seconds. macOS txbytes includes queued bytes; derive ACK progress by subtracting Send-Q so newly buffered data cannot hide a stall. Keep the preceding 120 samples in a private atomic archive on each stall transition, bounded to 100 incidents independently of the rolling history. Four client-watcher tests passed locally and on Worktop. A disposable staging connection with scoped, automatically cleaned packet loss produced one private archive before its TCP connection closed; the archive survived watcher restart. Staging health passed. Separate native Claude controls confirmed recovery from an early empty-body 499 using the harness retry defaults and the production hint. Assisted-by: gpt-6.1-sol --- tools/substation-client-watch-test.py | 20 +++++++++++++++++ tools/substation-client-watch.py | 32 +++++++++++++++++++++++++++ tools/substation.md | 3 +++ 3 files changed, 55 insertions(+) diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index c4d71b804d392abb138fd42360d81aa118951808..b4de4225abebb5dd933395bc37561645da5c61e9 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase): self.assertEqual(rows[0]["tx_win"], "1000000") self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") + def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self): + row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200} + samples = [{"ts": 0, "connections": [row]}, + {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]}, + {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}] + self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)}) + self.assertEqual(watch.stalls(samples[:-1]), set()) + samples[-1]["connections"][0]["send_q"] = 190 + self.assertEqual(watch.stalls(samples), set()) + + def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self): + row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200} + samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}] + self.assertEqual(watch.stalls(samples), set()) + samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}] + samples[-1]["connections"] = [{**row, "send_q": 100}] + self.assertEqual(watch.stalls(samples), set()) + samples.insert(1, {"ts": 5, "connections": []}) + self.assertEqual(watch.stalls(samples), set()) + if __name__ == "__main__": unittest.main() diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 6e43f70c394b0e54df93506fd9fd77923361755c..067bc8be8f9235ae70018e525a2b8221680f04ae 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import csv +from collections import deque from datetime import datetime, timezone import ipaddress import json @@ -62,10 +63,31 @@ def counters(output, targets): return rows +def stalls(samples): + latest = samples[-1] + stalled = set() + for row in latest.get("connections", []): + if not row["pid"] or not row["send_q"]: + continue + key = (row["local"], row["peer"], row["pid"]) + acknowledged = row["bytes_sent"] - row["send_q"] + since = latest["ts"] + for sample in reversed(samples): + prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None) + if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged: + break + since = sample["ts"] + if latest["ts"] - since >= 10: + stalled.add(key) + return stalled + + def main(): os.umask(0o077) ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) targets = set() + samples = deque(maxlen=120) + stalled = set() resolved = measured = 0 while True: now = time.time() @@ -94,6 +116,16 @@ def main(): stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") with (ROOT / (stamp + ".jsonl")).open("a") as file: file.write(json.dumps(record, separators=(",", ":")) + "\n") + samples.append(record) + current = stalls(samples) + if added := current - stalled: + path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") + pending = path.with_suffix(".pending") + pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) + pending.replace(path) + for old in sorted(ROOT.glob("stall-*.json"))[:-100]: + old.unlink() + stalled = current for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: path.unlink() time.sleep(1) diff --git a/tools/substation.md b/tools/substation.md index b1e4f01b2b8aaca8c5d219dea54c372ced39837b..f6afb5a294c2c775dff20f29d7e346f3c34569e0 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -145,3 +145,6 @@ retransmissions and windows. The installed script is an exported revision of `tools/substation-client-watch.py`; the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. Match UTC timestamps to the server incident before comparing TCP progress. +An upload with queued bytes and no acknowledgement progress for 10 seconds +retains the preceding two minutes in a private `stall-.json` file. +The latest 100 stalls survive rolling-history expiry and watcher restarts. -- 2.54.1