From 9f7ec5c28c4d001c4d36e7c68550acb86e533950 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 12 Nov 2026 11:00:46 -0800 Subject: [PATCH] Retain complete Worktop upload stall episodes Give each stalled socket one archive, then update it after stall detection ends and packet ingestion catches up. Retain late retransmissions and connection closure instead of only the first ten seconds. Skip expired packet records before parsing so restart catch-up does not spend CPU on discarded history. Thirteen watcher tests passed locally and eight client tests passed on Worktop. A disposable staging connection with scoped packet loss updated the same archive through its reset, retaining 2,473 matching packets with zero backlog at finalization. The firewall fixture was removed and private evidence retained. Staging health passed. Worktop runs the exported client revision. Assisted-by: gpt-6.1-sol --- tools/substation-client-watch-test.py | 13 +++++++++++++ tools/substation-client-watch.py | 28 ++++++++++++++++++++------- tools/substation.md | 7 +++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index 6759a0945f403cb09a8dce50fa03a501dc8d4899..57d5dc2f1e8fdc50800489e909c21b6a356734f2 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -13,6 +13,19 @@ spec.loader.exec_module(watch) class ClientEvidence(unittest.TestCase): + def setUp(self): + clock = patch.object(watch.time, "time", return_value=200) + clock.start() + self.addCleanup(clock.stop) + + def test_capture_skips_expired_metadata_and_still_advances(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + path.write_text("invalid line\n1.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") + rows, offset, _ = watch.capture(path, 0) + self.assertEqual(rows, []) + self.assertEqual(offset, path.stat().st_size) + def test_capture_keeps_partial_lines_and_excludes_payload(self): with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "capture.log" diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index c50a053e69c98dfa2b9340d77d4dea1dc0525455..e97cdcdadacc8468238a2b4ffedad60d24b670b2 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -102,7 +102,13 @@ def capture(path, offset): file.truncate() offset = 0 truncated_at = time.time() + cutoff = time.time() - 120 for line in data[:end].splitlines(): + try: + if float(line.partition(b" ")[0]) < cutoff: + continue + except ValueError: + continue if row := packet(line.decode("ascii", errors="replace")): rows.append(row) return rows, offset, truncated_at @@ -114,7 +120,7 @@ def main(): targets = set() samples = deque(maxlen=120) packets = deque(maxlen=100000) - stalled = set() + stalled = {} capture_state = {"offset": 0, "busy": False} capture_gaps = deque(maxlen=100) capture_path = ROOT / "tcp-headers.log" @@ -173,17 +179,25 @@ def main(): file.write(json.dumps(record, separators=(",", ":")) + "\n") samples.append(record) current = stalls(samples) - if added := current - stalled: - path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") + for key in current | stalled.keys(): + if key in current: + if key in stalled: + continue + path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json" + stalled[key] = path + else: + if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")): + continue + path = stalled.pop(key) pending = path.with_suffix(".pending") - endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} - pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), + local, _, port = key[0].rpartition(".") + endpoint = f"[{local}]:{port}" + pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples), "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], - "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) + "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() - stalled = current for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: path.unlink() time.sleep(1) diff --git a/tools/substation.md b/tools/substation.md index 42b1ada27b10ff2682ca559eff897142468b2afa..3894ff356393326752ea42b6678f5ad061ec22cd 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. Match UTC timestamps to the server incident before comparing TCP progress. An upload with queued bytes and no acknowledgement progress for 10 seconds -retains the preceding two minutes in a private `stall-.json` file. +retains the preceding 120 socket samples in a private `stall-.json` file. The latest 100 stalls survive rolling-history expiry and watcher restarts. +Each stalled socket has one archive, updated after the stall and after packet +ingestion catches up, retaining later retransmissions and closure. When an administrator starts a TCP summary capture into `tcp-headers.log`, the -client watcher retains matching sequence/ACK/SACK metadata with each stall. +client watcher retains up to two minutes of matching sequence/ACK/SACK metadata +with each stall, bounded to 100,000 packets. It consumes and truncates that append-only log at 64 MiB without restarting tcpdump. Archives mark each truncation; exclude that instant from packet-loss conclusions because concurrent writes can be lost during truncation. -- 2.54.1