From 029fe21eabd885646110ce6c0991a7dc1be99bfe Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 12 Nov 2026 10:41:10 -0800 Subject: [PATCH] Keep Worktop socket sampling independent of packet ingestion Read and parse the capture log on a separate thread so upload bursts and macOS resource limits cannot delay TCP queue samples. Record ingestion backlog, busy state and errors so absent packets are not interpreted without capture coverage. Truncate before parsing and retain the truncation timestamp with stall evidence. The installed LaunchAgent now uses the Standard process type. Twelve watcher tests passed locally and seven client tests passed on Worktop. Live sampling stayed within 2.2 seconds while clearing a 55 MB packet backlog. Staging health passed. Disposable persistent TLS controls sent 198 MB each through Python and Bun 1.3.5 under 143 and 359 scoped packet drops; all 32 bodies were read fully and returned 200 on their original connections. Those controls did not reproduce the natural fault. The fixture and drop rules were removed; private diagnostic metadata remains on Snow Globe. Assisted-by: gpt-6.1-sol --- tools/substation-client-watch-test.py | 30 ++++++++++++++-- tools/substation-client-watch.py | 51 +++++++++++++++++++-------- tools/substation.md | 6 +++- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index f62593af11ff23e4b53b2ea2a00f6051d9a22f5e..6759a0945f403cb09a8dce50fa03a501dc8d4899 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -4,6 +4,7 @@ import json from pathlib import Path import tempfile import unittest +from unittest.mock import patch spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) @@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase): with tempfile.TemporaryDirectory() as temporary: path = Path(temporary) / "capture.log" line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" - self.assertEqual(watch.capture(path, 0), ([], 0, False)) + self.assertEqual(watch.capture(path, 0), ([], 0, None)) path.write_text(line) - self.assertEqual(watch.capture(path, 0), ([], 0, False)) + self.assertEqual(watch.capture(path, 0), ([], 0, None)) with path.open("a") as writer: writer.write("\n") rows, offset, rotated = watch.capture(path, 0) self.assertEqual(rows[0]["seq"], "10:20") self.assertNotIn("private", json.dumps(rows)) self.assertFalse(rotated) - self.assertEqual(watch.capture(path, offset), ([], offset, False)) + self.assertEqual(watch.capture(path, offset), ([], offset, None)) def test_capture_rotation_preserves_the_open_append_writer(self): with tempfile.TemporaryDirectory() as temporary: @@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase): self.assertEqual(rows[0]["ack"], "20") self.assertFalse(rotated) + def test_capture_rotates_before_parsing_while_the_writer_remains_active(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + first = "200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n" + second = first.replace("200.0", "201.0") + with path.open("a") as writer: + writer.truncate(64 * 1024 * 1024) + writer.write(first) + writer.flush() + original = watch.packet + + def parse(line): + writer.write(second) + writer.flush() + return original(line) + + with patch.object(watch, "packet", side_effect=parse): + rows, offset, rotated = watch.capture(path, 64 * 1024 * 1024) + self.assertTrue(rotated) + self.assertEqual(rows[0]["ts"], 200.0) + rows, _, _ = watch.capture(path, offset) + self.assertEqual(rows[0]["ts"], 201.0) + def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): text = "\n".join([ "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 32e682f7579f3ea5e907af58249cea4fe2349f77..c50a053e69c98dfa2b9340d77d4dea1dc0525455 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -9,6 +9,7 @@ from pathlib import Path import re import socket import subprocess +import threading import time from substation_packets import packet @@ -86,7 +87,7 @@ def stalls(samples): def capture(path, offset): rows = [] - rotated = False + truncated_at = None if path.exists(): with path.open("r+b") as file: if path.stat().st_size < offset: @@ -95,16 +96,16 @@ def capture(path, offset): data = file.read(8 * 1024 * 1024) end = data.rfind(b"\n") + 1 offset += end - for line in data[:end].splitlines(): - if row := packet(line.decode("ascii", errors="replace")): - rows.append(row) if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: # tcpdump holds an append descriptor; truncation preserves its capture privileges. file.seek(0) file.truncate() offset = 0 - rotated = True - return rows, offset, rotated + truncated_at = time.time() + for line in data[:end].splitlines(): + if row := packet(line.decode("ascii", errors="replace")): + rows.append(row) + return rows, offset, truncated_at def main(): @@ -114,7 +115,29 @@ def main(): samples = deque(maxlen=120) packets = deque(maxlen=100000) stalled = set() - capture_offset = 0 + capture_state = {"offset": 0, "busy": False} + capture_gaps = deque(maxlen=100) + capture_path = ROOT / "tcp-headers.log" + + def read_capture(): + while True: + capture_state["busy"] = True + try: + rows, offset, truncated_at = capture(capture_path, capture_state["offset"]) + packets.extend(rows) + while packets and packets[0]["ts"] < time.time() - 120: + packets.popleft() + capture_state["offset"] = offset + capture_state.pop("error", None) + if truncated_at is not None: + capture_gaps.append(truncated_at) + except Exception as error: + capture_state["error"] = type(error).__name__ + finally: + capture_state["busy"] = False + time.sleep(0.1) + + threading.Thread(target=read_capture, daemon=True).start() resolved = measured = 0 while True: now = time.time() @@ -126,12 +149,11 @@ def main(): result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) rows = sockets(result.stdout, targets) record["connections"] = rows - captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) - packets.extend(captured) - while packets and packets[0]["ts"] < now - 120: - packets.popleft() - if rotated: - record["capture_rotated"] = True + if capture_path.exists(): + record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"]) + record["capture_busy"] = capture_state["busy"] + if "error" in capture_state: + record["capture_error"] = capture_state["error"] if now - measured >= 5: pids = sorted({row["pid"] for row in rows if row["pid"]}) if pids: @@ -156,7 +178,8 @@ def main(): pending = path.with_suffix(".pending") endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), - "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) + "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], + "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() diff --git a/tools/substation.md b/tools/substation.md index 7639857df27be0c30aeb76d17d07de0dadc3a03a..42b1ada27b10ff2682ca559eff897142468b2afa 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts. When an administrator starts a TCP summary capture into `tcp-headers.log`, the client watcher retains matching sequence/ACK/SACK metadata with each stall. It consumes and truncates that append-only log at 64 MiB without restarting -tcpdump. Samples mark each truncation; exclude that instant from packet-loss +tcpdump. Archives mark each truncation; exclude that instant from packet-loss conclusions because concurrent writes can be lost during truncation. +Packet ingestion runs independently of socket sampling. Samples record unread +capture bytes, active ingestion and errors. Infer missing sender packets only +after ingestion finishes and the backlog clears, with no capture error or +truncation gap in that interval. The LaunchAgent uses `ProcessType: Standard`. -- 2.54.1