authoravatarclover caruso<git@paperclover.net>2026-11-12 10:41:10-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log029fe21eabd885646110ce6c0991a7dc1be99bfe
tree43af4de3fdd388f87ca7420430f2d9501a4515df
parent42fe3bf6420ef43eef3bd23ca7093532cf255c1b
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Keep Worktop socket sampling independent of packet ingestion

Read and parse the capture log on a separate thread so upload bursts and macOS resource limits cannot delay TCP queue samples. Record ingestion backlog, busy state and errors so absent packets are not interpreted without capture coverage. Truncate before parsing and retain the truncation timestamp with stall evidence. The installed LaunchAgent now uses the Standard process type. Twelve watcher tests passed locally and seven client tests passed on Worktop. Live sampling stayed within 2.2 seconds while clearing a 55 MB packet backlog. Staging health passed. Disposable persistent TLS controls sent 198 MB each through Python and Bun 1.3.5 under 143 and 359 scoped packet drops; all 32 bodies were read fully and returned 200 on their original connections. Those controls did not reproduce the natural fault. The fixture and drop rules were removed; private diagnostic metadata remains on Snow Globe. Assisted-by: gpt-6.1-sol

3 files changed, 69 insertions(+), 18 deletions(-)

tools/substation-client-watch-test.py+27-3
...@@ -4,6 +4,7 @@ import json...@@ -4,6 +4,7 @@ import json
4from pathlib import Path4from pathlib import Path
5import tempfile5import tempfile
6import unittest6import unittest
7from unittest.mock import patch
78
89
9spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py"))10spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py"))
...@@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase):...@@ -16,16 +17,16 @@ class ClientEvidence(unittest.TestCase):
16 with tempfile.TemporaryDirectory() as temporary:17 with tempfile.TemporaryDirectory() as temporary:
17 path = Path(temporary) / "capture.log"18 path = Path(temporary) / "capture.log"
18 line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload"19 line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload"
19 self.assertEqual(watch.capture(path, 0), ([], 0, False))20 self.assertEqual(watch.capture(path, 0), ([], 0, None))
20 path.write_text(line)21 path.write_text(line)
21 self.assertEqual(watch.capture(path, 0), ([], 0, False))22 self.assertEqual(watch.capture(path, 0), ([], 0, None))
22 with path.open("a") as writer:23 with path.open("a") as writer:
23 writer.write("\n")24 writer.write("\n")
24 rows, offset, rotated = watch.capture(path, 0)25 rows, offset, rotated = watch.capture(path, 0)
25 self.assertEqual(rows[0]["seq"], "10:20")26 self.assertEqual(rows[0]["seq"], "10:20")
26 self.assertNotIn("private", json.dumps(rows))27 self.assertNotIn("private", json.dumps(rows))
27 self.assertFalse(rotated)28 self.assertFalse(rotated)
28 self.assertEqual(watch.capture(path, offset), ([], offset, False))29 self.assertEqual(watch.capture(path, offset), ([], offset, None))
2930
30 def test_capture_rotation_preserves_the_open_append_writer(self):31 def test_capture_rotation_preserves_the_open_append_writer(self):
31 with tempfile.TemporaryDirectory() as temporary:32 with tempfile.TemporaryDirectory() as temporary:
...@@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase):...@@ -43,6 +44,29 @@ class ClientEvidence(unittest.TestCase):
43 self.assertEqual(rows[0]["ack"], "20")44 self.assertEqual(rows[0]["ack"], "20")
44 self.assertFalse(rotated)45 self.assertFalse(rotated)
4546
47 def test_capture_rotates_before_parsing_while_the_writer_remains_active(self):
48 with tempfile.TemporaryDirectory() as temporary:
49 path = Path(temporary) / "capture.log"
50 first = "200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n"
51 second = first.replace("200.0", "201.0")
52 with path.open("a") as writer:
53 writer.truncate(64 * 1024 * 1024)
54 writer.write(first)
55 writer.flush()
56 original = watch.packet
57
58 def parse(line):
59 writer.write(second)
60 writer.flush()
61 return original(line)
62
63 with patch.object(watch, "packet", side_effect=parse):
64 rows, offset, rotated = watch.capture(path, 64 * 1024 * 1024)
65 self.assertTrue(rotated)
66 self.assertEqual(rows[0]["ts"], 200.0)
67 rows, _, _ = watch.capture(path, offset)
68 self.assertEqual(rows[0]["ts"], 201.0)
69
46 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):70 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):
47 text = "\n".join([71 text = "\n".join([
48 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",72 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",
tools/substation-client-watch.py+37-14
...@@ -9,6 +9,7 @@ from pathlib import Path...@@ -9,6 +9,7 @@ from pathlib import Path
9import re9import re
10import socket10import socket
11import subprocess11import subprocess
12import threading
12import time13import time
1314
14from substation_packets import packet15from substation_packets import packet
...@@ -86,7 +87,7 @@ def stalls(samples):...@@ -86,7 +87,7 @@ def stalls(samples):
8687
87def capture(path, offset):88def capture(path, offset):
88 rows = []89 rows = []
89 rotated = False90 truncated_at = None
90 if path.exists():91 if path.exists():
91 with path.open("r+b") as file:92 with path.open("r+b") as file:
92 if path.stat().st_size < offset:93 if path.stat().st_size < offset:
...@@ -95,16 +96,16 @@ def capture(path, offset):...@@ -95,16 +96,16 @@ def capture(path, offset):
95 data = file.read(8 * 1024 * 1024)96 data = file.read(8 * 1024 * 1024)
96 end = data.rfind(b"\n") + 197 end = data.rfind(b"\n") + 1
97 offset += end98 offset += end
98 for line in data[:end].splitlines():
99 if row := packet(line.decode("ascii", errors="replace")):
100 rows.append(row)
101 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:99 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:
102 # tcpdump holds an append descriptor; truncation preserves its capture privileges.100 # tcpdump holds an append descriptor; truncation preserves its capture privileges.
103 file.seek(0)101 file.seek(0)
104 file.truncate()102 file.truncate()
105 offset = 0103 offset = 0
106 rotated = True104 truncated_at = time.time()
107 return rows, offset, rotated105 for line in data[:end].splitlines():
106 if row := packet(line.decode("ascii", errors="replace")):
107 rows.append(row)
108 return rows, offset, truncated_at
108109
109110
110def main():111def main():
...@@ -114,7 +115,29 @@ def main():...@@ -114,7 +115,29 @@ def main():
114 samples = deque(maxlen=120)115 samples = deque(maxlen=120)
115 packets = deque(maxlen=100000)116 packets = deque(maxlen=100000)
116 stalled = set()117 stalled = set()
117 capture_offset = 0118 capture_state = {"offset": 0, "busy": False}
119 capture_gaps = deque(maxlen=100)
120 capture_path = ROOT / "tcp-headers.log"
121
122 def read_capture():
123 while True:
124 capture_state["busy"] = True
125 try:
126 rows, offset, truncated_at = capture(capture_path, capture_state["offset"])
127 packets.extend(rows)
128 while packets and packets[0]["ts"] < time.time() - 120:
129 packets.popleft()
130 capture_state["offset"] = offset
131 capture_state.pop("error", None)
132 if truncated_at is not None:
133 capture_gaps.append(truncated_at)
134 except Exception as error:
135 capture_state["error"] = type(error).__name__
136 finally:
137 capture_state["busy"] = False
138 time.sleep(0.1)
139
140 threading.Thread(target=read_capture, daemon=True).start()
118 resolved = measured = 0141 resolved = measured = 0
119 while True:142 while True:
120 now = time.time()143 now = time.time()
...@@ -126,12 +149,11 @@ def main():...@@ -126,12 +149,11 @@ def main():
126 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)149 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
127 rows = sockets(result.stdout, targets)150 rows = sockets(result.stdout, targets)
128 record["connections"] = rows151 record["connections"] = rows
129 captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset)152 if capture_path.exists():
130 packets.extend(captured)153 record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"])
131 while packets and packets[0]["ts"] < now - 120:154 record["capture_busy"] = capture_state["busy"]
132 packets.popleft()155 if "error" in capture_state:
133 if rotated:156 record["capture_error"] = capture_state["error"]
134 record["capture_rotated"] = True
135 if now - measured >= 5:157 if now - measured >= 5:
136 pids = sorted({row["pid"] for row in rows if row["pid"]})158 pids = sorted({row["pid"] for row in rows if row["pid"]})
137 if pids:159 if pids:
...@@ -156,7 +178,8 @@ def main():...@@ -156,7 +178,8 @@ def main():
156 pending = path.with_suffix(".pending")178 pending = path.with_suffix(".pending")
157 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}179 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}
158 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),180 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),
159 "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))181 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],
182 "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
160 pending.replace(path)183 pending.replace(path)
161 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:184 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
162 old.unlink()185 old.unlink()
tools/substation.md+5-1
...@@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts....@@ -151,5 +151,9 @@ The latest 100 stalls survive rolling-history expiry and watcher restarts.
151When an administrator starts a TCP summary capture into `tcp-headers.log`, the151When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152client watcher retains matching sequence/ACK/SACK metadata with each stall.152client watcher retains matching sequence/ACK/SACK metadata with each stall.
153It consumes and truncates that append-only log at 64 MiB without restarting153It consumes and truncates that append-only log at 64 MiB without restarting
154tcpdump. Samples mark each truncation; exclude that instant from packet-loss154tcpdump. Archives mark each truncation; exclude that instant from packet-loss
155conclusions because concurrent writes can be lost during truncation.155conclusions because concurrent writes can be lost during truncation.
156Packet ingestion runs independently of socket sampling. Samples record unread
157capture bytes, active ingestion and errors. Infer missing sender packets only
158after ingestion finishes and the backlog clears, with no capture error or
159truncation gap in that interval. The LaunchAgent uses `ProcessType: Standard`.