authoravatarclover caruso<git@paperclover.net>2026-11-11 16:43:57-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log3b735d29265d065eff0ee72b7943eb7795f87bbb
tree67913067d74065aa840c9f184bd7267e193ec54a
parent9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain Worktop upload stalls before edge timeouts

Archive sender-side evidence when queued upload bytes make no acknowledgement progress for ten seconds. macOS txbytes includes queued bytes; derive ACK progress by subtracting Send-Q so newly buffered data cannot hide a stall. Keep the preceding 120 samples in a private atomic archive on each stall transition, bounded to 100 incidents independently of the rolling history. Four client-watcher tests passed locally and on Worktop. A disposable staging connection with scoped, automatically cleaned packet loss produced one private archive before its TCP connection closed; the archive survived watcher restart. Staging health passed. Separate native Claude controls confirmed recovery from an early empty-body 499 using the harness retry defaults and the production hint. Assisted-by: gpt-6.1-sol

3 files changed, 55 insertions(+), 0 deletions(-)

tools/substation-client-watch-test.py+20
...@@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase):...@@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase):
38 self.assertEqual(rows[0]["tx_win"], "1000000")38 self.assertEqual(rows[0]["tx_win"], "1000000")
39 self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1")39 self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1")
4040
41 def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self):
42 row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200}
43 samples = [{"ts": 0, "connections": [row]},
44 {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]},
45 {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}]
46 self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)})
47 self.assertEqual(watch.stalls(samples[:-1]), set())
48 samples[-1]["connections"][0]["send_q"] = 190
49 self.assertEqual(watch.stalls(samples), set())
50
51 def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self):
52 row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200}
53 samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}]
54 self.assertEqual(watch.stalls(samples), set())
55 samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}]
56 samples[-1]["connections"] = [{**row, "send_q": 100}]
57 self.assertEqual(watch.stalls(samples), set())
58 samples.insert(1, {"ts": 5, "connections": []})
59 self.assertEqual(watch.stalls(samples), set())
60
4161
42if __name__ == "__main__":62if __name__ == "__main__":
43 unittest.main()63 unittest.main()
tools/substation-client-watch.py+32
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import csv2import csv
3from collections import deque
3from datetime import datetime, timezone4from datetime import datetime, timezone
4import ipaddress5import ipaddress
5import json6import json
...@@ -62,10 +63,31 @@ def counters(output, targets):...@@ -62,10 +63,31 @@ def counters(output, targets):
62 return rows63 return rows
6364
6465
66def stalls(samples):
67 latest = samples[-1]
68 stalled = set()
69 for row in latest.get("connections", []):
70 if not row["pid"] or not row["send_q"]:
71 continue
72 key = (row["local"], row["peer"], row["pid"])
73 acknowledged = row["bytes_sent"] - row["send_q"]
74 since = latest["ts"]
75 for sample in reversed(samples):
76 prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None)
77 if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged:
78 break
79 since = sample["ts"]
80 if latest["ts"] - since >= 10:
81 stalled.add(key)
82 return stalled
83
84
65def main():85def main():
66 os.umask(0o077)86 os.umask(0o077)
67 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)87 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
68 targets = set()88 targets = set()
89 samples = deque(maxlen=120)
90 stalled = set()
69 resolved = measured = 091 resolved = measured = 0
70 while True:92 while True:
71 now = time.time()93 now = time.time()
...@@ -94,6 +116,16 @@ def main():...@@ -94,6 +116,16 @@ def main():
94 stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M")116 stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M")
95 with (ROOT / (stamp + ".jsonl")).open("a") as file:117 with (ROOT / (stamp + ".jsonl")).open("a") as file:
96 file.write(json.dumps(record, separators=(",", ":")) + "\n")118 file.write(json.dumps(record, separators=(",", ":")) + "\n")
119 samples.append(record)
120 current = stalls(samples)
121 if added := current - stalled:
122 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")
123 pending = path.with_suffix(".pending")
124 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":")))
125 pending.replace(path)
126 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
127 old.unlink()
128 stalled = current
97 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:129 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
98 path.unlink()130 path.unlink()
99 time.sleep(1)131 time.sleep(1)
tools/substation.md+3
...@@ -145,3 +145,6 @@ retransmissions and windows....@@ -145,3 +145,6 @@ retransmissions and windows.
145The installed script is an exported revision of `tools/substation-client-watch.py`;145The installed script is an exported revision of `tools/substation-client-watch.py`;
146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.
147Match UTC timestamps to the server incident before comparing TCP progress.147Match UTC timestamps to the server incident before comparing TCP progress.
148An upload with queued bytes and no acknowledgement progress for 10 seconds
149retains the preceding two minutes in a private `stall-<timestamp>.json` file.
150The latest 100 stalls survive rolling-history expiry and watcher restarts.