| author | |
| committer | |
| log | 3b735d29265d065eff0ee72b7943eb7795f87bbb |
| tree | 67913067d74065aa840c9f184bd7267e193ec54a |
| parent | 9ffb8cb4f4f5c06bc6240d3f27ef6e0b71e54640 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Archive sender-side evidence when queued upload bytes make no acknowledgement
progress for ten seconds. macOS txbytes includes queued bytes; derive ACK
progress by subtracting Send-Q so newly buffered data cannot hide a stall.
Keep the preceding 120 samples in a private atomic archive on each stall
transition, bounded to 100 incidents independently of the rolling history.
Four client-watcher tests passed locally and on Worktop. A disposable staging
connection with scoped, automatically cleaned packet loss produced one private
archive before its TCP connection closed; the archive survived watcher restart.
Staging health passed. Separate native Claude controls confirmed recovery from
an early empty-body 499 using the harness retry defaults and the production hint.
Assisted-by: gpt-6.1-sol3 files changed, 55 insertions(+), 0 deletions(-)
tools/substation-client-watch-test.py+20| ... | @@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase): | ... | @@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase): |
| 38 | self.assertEqual(rows[0]["tx_win"], "1000000") | 38 | self.assertEqual(rows[0]["tx_win"], "1000000") |
| 39 | self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") | 39 | self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") |
| 40 | 40 | ||
| 41 | def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self): | ||
| 42 | row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200} | ||
| 43 | samples = [{"ts": 0, "connections": [row]}, | ||
| 44 | {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]}, | ||
| 45 | {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}] | ||
| 46 | self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)}) | ||
| 47 | self.assertEqual(watch.stalls(samples[:-1]), set()) | ||
| 48 | samples[-1]["connections"][0]["send_q"] = 190 | ||
| 49 | self.assertEqual(watch.stalls(samples), set()) | ||
| 50 | |||
| 51 | def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self): | ||
| 52 | row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200} | ||
| 53 | samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}] | ||
| 54 | self.assertEqual(watch.stalls(samples), set()) | ||
| 55 | samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}] | ||
| 56 | samples[-1]["connections"] = [{**row, "send_q": 100}] | ||
| 57 | self.assertEqual(watch.stalls(samples), set()) | ||
| 58 | samples.insert(1, {"ts": 5, "connections": []}) | ||
| 59 | self.assertEqual(watch.stalls(samples), set()) | ||
| 60 | |||
| 41 | 61 | ||
| 42 | if __name__ == "__main__": | 62 | if __name__ == "__main__": |
| 43 | unittest.main() | 63 | unittest.main() |
tools/substation-client-watch.py+32| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | import csv | 2 | import csv |
| 3 | from collections import deque | ||
| 3 | from datetime import datetime, timezone | 4 | from datetime import datetime, timezone |
| 4 | import ipaddress | 5 | import ipaddress |
| 5 | import json | 6 | import json |
| ... | @@ -62,10 +63,31 @@ def counters(output, targets): | ... | @@ -62,10 +63,31 @@ def counters(output, targets): |
| 62 | return rows | 63 | return rows |
| 63 | 64 | ||
| 64 | 65 | ||
| 66 | def stalls(samples): | ||
| 67 | latest = samples[-1] | ||
| 68 | stalled = set() | ||
| 69 | for row in latest.get("connections", []): | ||
| 70 | if not row["pid"] or not row["send_q"]: | ||
| 71 | continue | ||
| 72 | key = (row["local"], row["peer"], row["pid"]) | ||
| 73 | acknowledged = row["bytes_sent"] - row["send_q"] | ||
| 74 | since = latest["ts"] | ||
| 75 | for sample in reversed(samples): | ||
| 76 | prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None) | ||
| 77 | if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged: | ||
| 78 | break | ||
| 79 | since = sample["ts"] | ||
| 80 | if latest["ts"] - since >= 10: | ||
| 81 | stalled.add(key) | ||
| 82 | return stalled | ||
| 83 | |||
| 84 | |||
| 65 | def main(): | 85 | def main(): |
| 66 | os.umask(0o077) | 86 | os.umask(0o077) |
| 67 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) | 87 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 68 | targets = set() | 88 | targets = set() |
| 89 | samples = deque(maxlen=120) | ||
| 90 | stalled = set() | ||
| 69 | resolved = measured = 0 | 91 | resolved = measured = 0 |
| 70 | while True: | 92 | while True: |
| 71 | now = time.time() | 93 | now = time.time() |
| ... | @@ -94,6 +116,16 @@ def main(): | ... | @@ -94,6 +116,16 @@ def main(): |
| 94 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") | 116 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") |
| 95 | with (ROOT / (stamp + ".jsonl")).open("a") as file: | 117 | with (ROOT / (stamp + ".jsonl")).open("a") as file: |
| 96 | file.write(json.dumps(record, separators=(",", ":")) + "\n") | 118 | file.write(json.dumps(record, separators=(",", ":")) + "\n") |
| 119 | samples.append(record) | ||
| 120 | current = stalls(samples) | ||
| 121 | if added := current - stalled: | ||
| 122 | path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") | ||
| 123 | pending = path.with_suffix(".pending") | ||
| 124 | pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) | ||
| 125 | pending.replace(path) | ||
| 126 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: | ||
| 127 | old.unlink() | ||
| 128 | stalled = current | ||
| 97 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: | 129 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: |
| 98 | path.unlink() | 130 | path.unlink() |
| 99 | time.sleep(1) | 131 | time.sleep(1) |
tools/substation.md+3| ... | @@ -145,3 +145,6 @@ retransmissions and windows. | ... | @@ -145,3 +145,6 @@ retransmissions and windows. |
| 145 | The installed script is an exported revision of `tools/substation-client-watch.py`; | 145 | The installed script is an exported revision of `tools/substation-client-watch.py`; |
| 146 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. | 146 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. |
| 147 | Match UTC timestamps to the server incident before comparing TCP progress. | 147 | Match UTC timestamps to the server incident before comparing TCP progress. |
| 148 | An upload with queued bytes and no acknowledgement progress for 10 seconds | ||
| 149 | retains the preceding two minutes in a private `stall-<timestamp>.json` file. | ||
| 150 | The latest 100 stalls survive rolling-history expiry and watcher restarts. |