diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index c4d71b804d392abb138fd42360d81aa118951808..b4de4225abebb5dd933395bc37561645da5c61e9 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -38,6 +38,26 @@ class ClientEvidence(unittest.TestCase): self.assertEqual(rows[0]["tx_win"], "1000000") self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") + def test_stall_retains_unacknowledged_upload_despite_new_buffered_bytes(self): + row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 100, "bytes_sent": 200} + samples = [{"ts": 0, "connections": [row]}, + {"ts": 5, "connections": [{**row, "send_q": 150, "bytes_sent": 250}]}, + {"ts": 10, "connections": [{**row, "send_q": 200, "bytes_sent": 300}]}] + self.assertEqual(watch.stalls(samples), {(row["local"], row["peer"], 42)}) + self.assertEqual(watch.stalls(samples[:-1]), set()) + samples[-1]["connections"][0]["send_q"] = 190 + self.assertEqual(watch.stalls(samples), set()) + + def test_idle_socket_and_reused_connection_do_not_count_as_stalls(self): + row = {"local": "192.0.2.1.50000", "peer": "198.51.100.1.443", "pid": 42, "send_q": 0, "bytes_sent": 200} + samples = [{"ts": 0, "connections": [row]}, {"ts": 10, "connections": [row]}] + self.assertEqual(watch.stalls(samples), set()) + samples[0]["connections"] = [{**row, "send_q": 100, "pid": 41}] + samples[-1]["connections"] = [{**row, "send_q": 100}] + self.assertEqual(watch.stalls(samples), set()) + samples.insert(1, {"ts": 5, "connections": []}) + self.assertEqual(watch.stalls(samples), set()) + if __name__ == "__main__": unittest.main() diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 6e43f70c394b0e54df93506fd9fd77923361755c..067bc8be8f9235ae70018e525a2b8221680f04ae 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import csv +from collections import deque from datetime import datetime, timezone import ipaddress import json @@ -62,10 +63,31 @@ def counters(output, targets): return rows +def stalls(samples): + latest = samples[-1] + stalled = set() + for row in latest.get("connections", []): + if not row["pid"] or not row["send_q"]: + continue + key = (row["local"], row["peer"], row["pid"]) + acknowledged = row["bytes_sent"] - row["send_q"] + since = latest["ts"] + for sample in reversed(samples): + prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None) + if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged: + break + since = sample["ts"] + if latest["ts"] - since >= 10: + stalled.add(key) + return stalled + + def main(): os.umask(0o077) ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) targets = set() + samples = deque(maxlen=120) + stalled = set() resolved = measured = 0 while True: now = time.time() @@ -94,6 +116,16 @@ def main(): stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") with (ROOT / (stamp + ".jsonl")).open("a") as file: file.write(json.dumps(record, separators=(",", ":")) + "\n") + samples.append(record) + current = stalls(samples) + if added := current - stalled: + path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") + pending = path.with_suffix(".pending") + pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) + pending.replace(path) + for old in sorted(ROOT.glob("stall-*.json"))[:-100]: + old.unlink() + stalled = current for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: path.unlink() time.sleep(1) diff --git a/tools/substation.md b/tools/substation.md index b1e4f01b2b8aaca8c5d219dea54c372ced39837b..f6afb5a294c2c775dff20f29d7e346f3c34569e0 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -145,3 +145,6 @@ retransmissions and windows. The installed script is an exported revision of `tools/substation-client-watch.py`; the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. Match UTC timestamps to the server incident before comparing TCP progress. +An upload with queued bytes and no acknowledgement progress for 10 seconds +retains the preceding two minutes in a private `stall-.json` file. +The latest 100 stalls survive rolling-history expiry and watcher restarts.