diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index 6759a0945f403cb09a8dce50fa03a501dc8d4899..57d5dc2f1e8fdc50800489e909c21b6a356734f2 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -13,6 +13,19 @@ spec.loader.exec_module(watch) class ClientEvidence(unittest.TestCase): + def setUp(self): + clock = patch.object(watch.time, "time", return_value=200) + clock.start() + self.addCleanup(clock.stop) + + def test_capture_skips_expired_metadata_and_still_advances(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + path.write_text("invalid line\n1.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") + rows, offset, _ = watch.capture(path, 0) + self.assertEqual(rows, []) + self.assertEqual(offset, path.stat().st_size) + def test_capture_keeps_partial_lines_and_excludes_payload(self): with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "capture.log" diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index c50a053e69c98dfa2b9340d77d4dea1dc0525455..e97cdcdadacc8468238a2b4ffedad60d24b670b2 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -102,7 +102,13 @@ def capture(path, offset): file.truncate() offset = 0 truncated_at = time.time() + cutoff = time.time() - 120 for line in data[:end].splitlines(): + try: + if float(line.partition(b" ")[0]) < cutoff: + continue + except ValueError: + continue if row := packet(line.decode("ascii", errors="replace")): rows.append(row) return rows, offset, truncated_at @@ -114,7 +120,7 @@ def main(): targets = set() samples = deque(maxlen=120) packets = deque(maxlen=100000) - stalled = set() + stalled = {} capture_state = {"offset": 0, "busy": False} capture_gaps = deque(maxlen=100) capture_path = ROOT / "tcp-headers.log" @@ -173,17 +179,25 @@ def main(): file.write(json.dumps(record, separators=(",", ":")) + "\n") samples.append(record) current = stalls(samples) - if added := current - stalled: - path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") + for key in current | stalled.keys(): + if key in current: + if key in stalled: + continue + path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json" + stalled[key] = path + else: + if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")): + continue + path = stalled.pop(key) pending = path.with_suffix(".pending") - endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} - pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), + local, _, port = key[0].rpartition(".") + endpoint = f"[{local}]:{port}" + pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples), "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], - "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) + "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() - stalled = current for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: path.unlink() time.sleep(1) diff --git a/tools/substation.md b/tools/substation.md index 42b1ada27b10ff2682ca559eff897142468b2afa..3894ff356393326752ea42b6678f5ad061ec22cd 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. Match UTC timestamps to the server incident before comparing TCP progress. An upload with queued bytes and no acknowledgement progress for 10 seconds -retains the preceding two minutes in a private `stall-.json` file. +retains the preceding 120 socket samples in a private `stall-.json` file. The latest 100 stalls survive rolling-history expiry and watcher restarts. +Each stalled socket has one archive, updated after the stall and after packet +ingestion catches up, retaining later retransmissions and closure. When an administrator starts a TCP summary capture into `tcp-headers.log`, the -client watcher retains matching sequence/ACK/SACK metadata with each stall. +client watcher retains up to two minutes of matching sequence/ACK/SACK metadata +with each stall, bounded to 100,000 packets. It consumes and truncates that append-only log at 64 MiB without restarting tcpdump. Archives mark each truncation; exclude that instant from packet-loss conclusions because concurrent writes can be lost during truncation.