diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index f62593af11ff23e4b53b2ea2a00f6051d9a22f5e..6759a0945f403cb09a8dce50fa03a501dc8d4899 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -4,6 +4,7 @@ import json from pathlib import Path import tempfile import unittest +from unittest.mock import patch spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) @@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase): with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "capture.log" line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" - self.assertEqual(watch.capture(path, 0), ([], 0, False)) + self.assertEqual(watch.capture(path, 0), ([], 0, None)) path.write_text(line) - self.assertEqual(watch.capture(path, 0), ([], 0, False)) + self.assertEqual(watch.capture(path, 0), ([], 0, None)) with path.open("a") as writer: writer.write("\n") rows, offset, rotated = watch.capture(path, 0) self.assertEqual(rows[0]["seq"], "10:20") self.assertNotIn("private", json.dumps(rows)) self.assertFalse(rotated) - self.assertEqual(watch.capture(path, offset), ([], offset, False)) + self.assertEqual(watch.capture(path, offset), ([], offset, None)) def test_capture_rotation_preserves_the_open_append_writer(self): with tempfile.TemporaryDirectory() as temporary: @@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase): self.assertEqual(rows[0]["ack"], "20") self.assertFalse(rotated) + def test_capture_rotates_before_parsing_while_the_writer_remains_active(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + first = "200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n" + second = first.replace("200.0", "201.0") + with path.open("a") as writer: + writer.truncate(64 * 1024 * 1024) + writer.write(first) + writer.flush() + original = watch.packet + + def parse(line): + writer.write(second) + writer.flush() + return original(line) + + with patch.object(watch, "packet", side_effect=parse): + rows, offset, rotated = watch.capture(path, 64 * 1024 * 1024) + self.assertTrue(rotated) + self.assertEqual(rows[0]["ts"], 200.0) + rows, _, _ = watch.capture(path, offset) + self.assertEqual(rows[0]["ts"], 201.0) + def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): text = "\n".join([ "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 32e682f7579f3ea5e907af58249cea4fe2349f77..c50a053e69c98dfa2b9340d77d4dea1dc0525455 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -9,6 +9,7 @@ from pathlib import Path import re import socket import subprocess +import threading import time from substation_packets import packet @@ -86,7 +87,7 @@ def stalls(samples): def capture(path, offset): rows = [] - rotated = False + truncated_at = None if path.exists(): with path.open("r+b") as file: if path.stat().st_size < offset: @@ -95,16 +96,16 @@ def capture(path, offset): data = file.read(8 * 1024 * 1024) end = data.rfind(b"\n") + 1 offset += end - for line in data[:end].splitlines(): - if row := packet(line.decode("ascii", errors="replace")): - rows.append(row) if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: # tcpdump holds an append descriptor; truncation preserves its capture privileges. file.seek(0) file.truncate() offset = 0 - rotated = True - return rows, offset, rotated + truncated_at = time.time() + for line in data[:end].splitlines(): + if row := packet(line.decode("ascii", errors="replace")): + rows.append(row) + return rows, offset, truncated_at def main(): @@ -114,7 +115,29 @@ def main(): samples = deque(maxlen=120) packets = deque(maxlen=100000) stalled = set() - capture_offset = 0 + capture_state = {"offset": 0, "busy": False} + capture_gaps = deque(maxlen=100) + capture_path = ROOT / "tcp-headers.log" + + def read_capture(): + while True: + capture_state["busy"] = True + try: + rows, offset, truncated_at = capture(capture_path, capture_state["offset"]) + packets.extend(rows) + while packets and packets[0]["ts"] < time.time() - 120: + packets.popleft() + capture_state["offset"] = offset + capture_state.pop("error", None) + if truncated_at is not None: + capture_gaps.append(truncated_at) + except Exception as error: + capture_state["error"] = type(error).__name__ + finally: + capture_state["busy"] = False + time.sleep(0.1) + + threading.Thread(target=read_capture, daemon=True).start() resolved = measured = 0 while True: now = time.time() @@ -126,12 +149,11 @@ def main(): result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) rows = sockets(result.stdout, targets) record["connections"] = rows - captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) - packets.extend(captured) - while packets and packets[0]["ts"] < now - 120: - packets.popleft() - if rotated: - record["capture_rotated"] = True + if capture_path.exists(): + record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"]) + record["capture_busy"] = capture_state["busy"] + if "error" in capture_state: + record["capture_error"] = capture_state["error"] if now - measured >= 5: pids = sorted({row["pid"] for row in rows if row["pid"]}) if pids: @@ -156,7 +178,8 @@ def main(): pending = path.with_suffix(".pending") endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), - "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) + "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], + "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() diff --git a/tools/substation.md b/tools/substation.md index 7639857df27be0c30aeb76d17d07de0dadc3a03a..42b1ada27b10ff2682ca559eff897142468b2afa 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts. When an administrator starts a TCP summary capture into `tcp-headers.log`, the client watcher retains matching sequence/ACK/SACK metadata with each stall. It consumes and truncates that append-only log at 64 MiB without restarting -tcpdump. Samples mark each truncation; exclude that instant from packet-loss +tcpdump. Archives mark each truncation; exclude that instant from packet-loss conclusions because concurrent writes can be lost during truncation. +Packet ingestion runs independently of socket sampling. Samples record unread +capture bytes, active ingestion and errors. Infer missing sender packets only +after ingestion finishes and the backlog clears, with no capture error or +truncation gap in that interval. The LaunchAgent uses `ProcessType: Standard`.