| author | |
| committer | |
| log | 3b735d29265d065eff0ee72b7943eb7795f87bbb |
| tree | 67913067d74065aa840c9f184bd7267e193ec54a |
| parent | 9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Archive sender-side evidence when queued upload bytes make no acknowledgement
progress for ten seconds. macOS txbytes includes queued bytes; derive ACK
progress by subtracting Send-Q so newly buffered data cannot hide a stall.
Keep the preceding 120 samples in a private atomic archive on each stall
transition, bounded to 100 incidents independently of the rolling history.
Four client-watcher tests passed locally and on Worktop. A disposable staging
connection with scoped, automatically cleaned packet loss produced one private
archive before its TCP connection closed; the archive survived watcher restart.
Staging health passed. Separate native Claude controls confirmed recovery from
an early empty-body 499 using the harness retry defaults and the production hint.
Assisted-by: gpt-6.1-sol3 files changed, 55 insertions(+), 0 deletions(-)
tools/substation-client-watch-test.py+20| ... | ... | @@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase): |
| 38 | 38 | self.assertEqual(rows[0]["tx_win"], "1000000") |
| 39 | 39 | self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") |
| 40 | 40 | |
| 41 | def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self): | |
| 42 | row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200} | |
| 43 | samples = [{"ts": 0, "connections": [row]}, | |
| 44 | {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]}, | |
| 45 | {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}] | |
| 46 | self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)}) | |
| 47 | self.assertEqual(watch.stalls(samples[:-1]), set()) | |
| 48 | samples[-1]["connections"][0]["send_q"] = 190 | |
| 49 | self.assertEqual(watch.stalls(samples), set()) | |
| 50 | ||
| 51 | def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self): | |
| 52 | row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200} | |
| 53 | samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}] | |
| 54 | self.assertEqual(watch.stalls(samples), set()) | |
| 55 | samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}] | |
| 56 | samples[-1]["connections"] = [{**row, "send_q": 100}] | |
| 57 | self.assertEqual(watch.stalls(samples), set()) | |
| 58 | samples.insert(1, {"ts": 5, "connections": []}) | |
| 59 | self.assertEqual(watch.stalls(samples), set()) | |
| 60 | ||
| 41 | 61 | |
| 42 | 62 | if __name__ == "__main__": |
| 43 | 63 | unittest.main() |
tools/substation-client-watch.py+32| ... | ... | @@ -1,5 +1,6 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | import csv |
| 3 | from collections import deque | |
| 3 | 4 | from datetime import datetime, timezone |
| 4 | 5 | import ipaddress |
| 5 | 6 | import json |
| ... | ... | @@ -62,10 +63,31 @@ def counters(output, targets): |
| 62 | 63 | return rows |
| 63 | 64 | |
| 64 | 65 | |
| 66 | def stalls(samples): | |
| 67 | latest = samples[-1] | |
| 68 | stalled = set() | |
| 69 | for row in latest.get("connections", []): | |
| 70 | if not row["pid"] or not row["send_q"]: | |
| 71 | continue | |
| 72 | key = (row["local"], row["peer"], row["pid"]) | |
| 73 | acknowledged = row["bytes_sent"] - row["send_q"] | |
| 74 | since = latest["ts"] | |
| 75 | for sample in reversed(samples): | |
| 76 | prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None) | |
| 77 | if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged: | |
| 78 | break | |
| 79 | since = sample["ts"] | |
| 80 | if latest["ts"] - since >= 10: | |
| 81 | stalled.add(key) | |
| 82 | return stalled | |
| 83 | ||
| 84 | ||
| 65 | 85 | def main(): |
| 66 | 86 | os.umask(0o077) |
| 67 | 87 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 68 | 88 | targets = set() |
| 89 | samples = deque(maxlen=120) | |
| 90 | stalled = set() | |
| 69 | 91 | resolved = measured = 0 |
| 70 | 92 | while True: |
| 71 | 93 | now = time.time() |
| ... | ... | @@ -94,6 +116,16 @@ def main(): |
| 94 | 116 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") |
| 95 | 117 | with (ROOT / (stamp + ".jsonl")).open("a") as file: |
| 96 | 118 | file.write(json.dumps(record, separators=(",", ":")) + "\n") |
| 119 | samples.append(record) | |
| 120 | current = stalls(samples) | |
| 121 | if added := current - stalled: | |
| 122 | path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") | |
| 123 | pending = path.with_suffix(".pending") | |
| 124 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) | |
| 125 | pending.replace(path) | |
| 126 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: | |
| 127 | old.unlink() | |
| 128 | stalled = current | |
| 97 | 129 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: |
| 98 | 130 | path.unlink() |
| 99 | 131 | time.sleep(1) |
tools/substation.md+3| ... | ... | @@ -145,3 +145,6 @@ retransmissions and windows. |
| 145 | 145 | The installed script is an exported revision of `tools/substation-client-watch.py`; |
| 146 | 146 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. |
| 147 | 147 | Match UTC timestamps to the server incident before comparing TCP progress. |
| 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds | |
| 149 | retains the preceding two minutes in a private `stall-<timestamp>.json` file. | |
| 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. |