authoravatarclover caruso<git@paperclover.net>2026-11-12 10:23:02-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log42fe3bf6420ef43eef3bd23ca7093532cf255c1b
tree31dc6ce8271413a4093ad4761add9d4c8ed6151c
parent3b735d29265d065eff0ee72b7943eb7795f87bbb
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain client packet evidence for stalled Substation uploads

Share the transport-only tcpdump parser between the server and Worktop watchers. Accept macOS capture summaries and retain matching packet metadata in each client stall archive. Consume the append log in bounded chunks and truncate it at 64 MiB; mark truncation instants because concurrent writes can be lost. Eleven watcher tests passed locally and six client tests passed on Worktop. A disposable staging connection with scoped packet loss retained 1,745 client packet records with its stall before the connection aborted. The temporary firewall rule and capture were removed. Staging client/management auth checks passed. The Worktop LaunchAgent runs the exported client revision. Assisted-by: gpt-6.1-sol

6 files changed, 102 insertions(+), 22 deletions(-)

tools/substation-client-watch-test.py+32
...@@ -2,6 +2,7 @@...@@ -2,6 +2,7 @@
2import importlib.util2import importlib.util
3import json3import json
4from pathlib import Path4from pathlib import Path
5import tempfile
5import unittest6import unittest
67
78
...@@ -11,6 +12,37 @@ spec.loader.exec_module(watch)...@@ -11,6 +12,37 @@ spec.loader.exec_module(watch)
1112
1213
13class ClientEvidence(unittest.TestCase):14class ClientEvidence(unittest.TestCase):
15 def test_capture_keeps_partial_lines_and_excludes_payload(self):
16 with tempfile.TemporaryDirectory() as temporary:
17 path = Path(temporary) / "capture.log"
18 line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload"
19 self.assertEqual(watch.capture(path, 0), ([], 0, False))
20 path.write_text(line)
21 self.assertEqual(watch.capture(path, 0), ([], 0, False))
22 with path.open("a") as writer:
23 writer.write("\n")
24 rows, offset, rotated = watch.capture(path, 0)
25 self.assertEqual(rows[0]["seq"], "10:20")
26 self.assertNotIn("private", json.dumps(rows))
27 self.assertFalse(rotated)
28 self.assertEqual(watch.capture(path, offset), ([], offset, False))
29
30 def test_capture_rotation_preserves_the_open_append_writer(self):
31 with tempfile.TemporaryDirectory() as temporary:
32 path = Path(temporary) / "capture.log"
33 with path.open("a") as writer:
34 writer.truncate(64 * 1024 * 1024)
35 writer.write("\n")
36 writer.flush()
37 _, offset, rotated = watch.capture(path, 64 * 1024 * 1024)
38 self.assertTrue(rotated)
39 self.assertEqual(offset, 0)
40 writer.write("200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n")
41 writer.flush()
42 rows, _, rotated = watch.capture(path, offset)
43 self.assertEqual(rows[0]["ack"], "20")
44 self.assertFalse(rotated)
45
14 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):46 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):
15 text = "\n".join([47 text = "\n".join([
16 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",48 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",
tools/substation-client-watch.py+36-1
...@@ -11,6 +11,8 @@ import socket...@@ -11,6 +11,8 @@ import socket
11import subprocess11import subprocess
12import time12import time
1313
14from substation_packets import packet
15
1416
15ROOT = Path.home() / ".local/state/substation-watch"17ROOT = Path.home() / ".local/state/substation-watch"
16COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win"18COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win"
...@@ -82,12 +84,37 @@ def stalls(samples):...@@ -82,12 +84,37 @@ def stalls(samples):
82 return stalled84 return stalled
8385
8486
87def capture(path, offset):
88 rows = []
89 rotated = False
90 if path.exists():
91 with path.open("r+b") as file:
92 if path.stat().st_size < offset:
93 offset = 0
94 file.seek(offset)
95 data = file.read(8 * 1024 * 1024)
96 end = data.rfind(b"\n") + 1
97 offset += end
98 for line in data[:end].splitlines():
99 if row := packet(line.decode("ascii", errors="replace")):
100 rows.append(row)
101 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:
102 # tcpdump holds an append descriptor; truncation preserves its capture privileges.
103 file.seek(0)
104 file.truncate()
105 offset = 0
106 rotated = True
107 return rows, offset, rotated
108
109
85def main():110def main():
86 os.umask(0o077)111 os.umask(0o077)
87 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)112 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
88 targets = set()113 targets = set()
89 samples = deque(maxlen=120)114 samples = deque(maxlen=120)
115 packets = deque(maxlen=100000)
90 stalled = set()116 stalled = set()
117 capture_offset = 0
91 resolved = measured = 0118 resolved = measured = 0
92 while True:119 while True:
93 now = time.time()120 now = time.time()
...@@ -99,6 +126,12 @@ def main():...@@ -99,6 +126,12 @@ def main():
99 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)126 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
100 rows = sockets(result.stdout, targets)127 rows = sockets(result.stdout, targets)
101 record["connections"] = rows128 record["connections"] = rows
129 captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset)
130 packets.extend(captured)
131 while packets and packets[0]["ts"] < now - 120:
132 packets.popleft()
133 if rotated:
134 record["capture_rotated"] = True
102 if now - measured >= 5:135 if now - measured >= 5:
103 pids = sorted({row["pid"] for row in rows if row["pid"]})136 pids = sorted({row["pid"] for row in rows if row["pid"]})
104 if pids:137 if pids:
...@@ -121,7 +154,9 @@ def main():...@@ -121,7 +154,9 @@ def main():
121 if added := current - stalled:154 if added := current - stalled:
122 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")155 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")
123 pending = path.with_suffix(".pending")156 pending = path.with_suffix(".pending")
124 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":")))157 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}
158 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),
159 "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))
125 pending.replace(path)160 pending.replace(path)
126 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:161 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
127 old.unlink()162 old.unlink()
tools/substation-watch-test.py+4
...@@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase):...@@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase):
56 self.assertEqual(row["flags"], "R.")56 self.assertEqual(row["flags"], "R.")
57 self.assertEqual(row["sack"], ["50:60"])57 self.assertEqual(row["sack"], ["50:60"])
58 self.assertNotIn("private", json.dumps(row))58 self.assertNotIn("private", json.dumps(row))
59 mac = watch.packet("190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10")
60 self.assertEqual(mac, row)
61 ipv6 = watch.packet("190.123 IP6 2001:db8::1.50000 > 2001:db8::5.443: Flags [.], ack 30, win 40, length 0")
62 self.assertEqual(ipv6["source"], "[2001:db8::1]:50000")
59 edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"}63 edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"}
60 self.assertEqual(watch.incident(edge, [], [row])["packets"], [row])64 self.assertEqual(watch.incident(edge, [], [row])["packets"], [row])
61 edge["client"] = "192.0.2.1:50001"65 edge["client"] = "192.0.2.1:50001"
tools/substation-watch.py+2-21
...@@ -10,6 +10,8 @@ import threading...@@ -10,6 +10,8 @@ import threading
10import time10import time
11from urllib.request import Request, urlopen11from urllib.request import Request, urlopen
1212
13from substation_packets import packet
14
1315
14ROOT = Path("/var/lib/snowglobe/substation-watch")16ROOT = Path("/var/lib/snowglobe/substation-watch")
15EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",17EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id",
...@@ -68,27 +70,6 @@ def event(entry, host):...@@ -68,27 +70,6 @@ def event(entry, host):
68 return None, None70 return None, None
6971
7072
71def packet(line):
72 match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
73 if not match:
74 return None
75 stamp, source, destination, flags, details = match.groups()
76 endpoints = {}
77 try:
78 for key, value in (("source", source), ("destination", destination)):
79 host, _, port = value.rpartition(".")
80 host = ipaddress.ip_address(host)
81 endpoints[key] = f"[{host}]:{int(port)}"
82 except ValueError:
83 return None
84 row = {"ts": float(stamp), **endpoints, "flags": flags}
85 for key in ("seq", "ack", "win", "length"):
86 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
87 row[key] = value[1]
88 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
89 return row
90
91
92def incident(edge, samples, packets):73def incident(edge, samples, packets):
93 length = edge.get("content_length")74 length = edge.get("content_length")
94 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None75 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
tools/substation.md+5
...@@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress....@@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress.
148An upload with queued bytes and no acknowledgement progress for 10 seconds148An upload with queued bytes and no acknowledgement progress for 10 seconds
149retains the preceding two minutes in a private `stall-<timestamp>.json` file.149retains the preceding two minutes in a private `stall-<timestamp>.json` file.
150The latest 100 stalls survive rolling-history expiry and watcher restarts.150The latest 100 stalls survive rolling-history expiry and watcher restarts.
151When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152client watcher retains matching sequence/ACK/SACK metadata with each stall.
153It consumes and truncates that append-only log at 64 MiB without restarting
154tcpdump. Samples mark each truncation; exclude that instant from packet-loss
155conclusions because concurrent writes can be lost during truncation.
tools/substation_packets.py created+23
...@@ -0,0 +1,23 @@
1import ipaddress
2import re
3
4
5def packet(line):
6 match = re.fullmatch(r"([0-9.]+) (?:\S+\s+(?:In|Out)\s+)?IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
7 if not match:
8 return None
9 stamp, source, destination, flags, details = match.groups()
10 endpoints = {}
11 try:
12 for key, value in (("source", source), ("destination", destination)):
13 host, _, port = value.rpartition(".")
14 host = ipaddress.ip_address(host)
15 endpoints[key] = f"[{host}]:{int(port)}"
16 except ValueError:
17 return None
18 row = {"ts": float(stamp), **endpoints, "flags": flags}
19 for key in ("seq", "ack", "win", "length"):
20 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
21 row[key] = value[1]
22 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
23 return row