From 42fe3bf6420ef43eef3bd23ca7093532cf255c1b Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 12 Nov 2026 10:23:02 -0800 Subject: [PATCH] Retain client packet evidence for stalled Substation uploads Share the transport-only tcpdump parser between the server and Worktop watchers. Accept macOS capture summaries and retain matching packet metadata in each client stall archive. Consume the append log in bounded chunks and truncate it at 64 MiB; mark truncation instants because concurrent writes can be lost. Eleven watcher tests passed locally and six client tests passed on Worktop. A disposable staging connection with scoped packet loss retained 1,745 client packet records with its stall before the connection aborted. The temporary firewall rule and capture were removed. Staging client/management auth checks passed. The Worktop LaunchAgent runs the exported client revision. Assisted-by: gpt-6.1-sol --- tools/substation-client-watch-test.py | 32 +++++++++++++++++++++++ tools/substation-client-watch.py | 37 ++++++++++++++++++++++++++- tools/substation-watch-test.py | 4 +++ tools/substation-watch.py | 23 ++--------------- tools/substation.md | 5 ++++ tools/substation_packets.py | 23 +++++++++++++++++ 6 files changed, 102 insertions(+), 22 deletions(-) create mode 100644 tools/substation_packets.py diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index b4de4225abebb5dd933395bc37561645da5c61e9..f62593af11ff23e4b53b2ea2a00f6051d9a22f5e 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -2,6 +2,7 @@ import importlib.util import json from pathlib import Path +import tempfile import unittest @@ -11,6 +12,37 @@ spec.loader.exec_module(watch) class ClientEvidence(unittest.TestCase): + def test_capture_keeps_partial_lines_and_excludes_payload(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" + self.assertEqual(watch.capture(path, 0), ([], 0, False)) + path.write_text(line) + self.assertEqual(watch.capture(path, 0), ([], 0, False)) + with path.open("a") as writer: + writer.write("\n") + rows, offset, rotated = watch.capture(path, 0) + self.assertEqual(rows[0]["seq"], "10:20") + self.assertNotIn("private", json.dumps(rows)) + self.assertFalse(rotated) + self.assertEqual(watch.capture(path, offset), ([], offset, False)) + + def test_capture_rotation_preserves_the_open_append_writer(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + with path.open("a") as writer: + writer.truncate(64 * 1024 * 1024) + writer.write("\n") + writer.flush() + _, offset, rotated = watch.capture(path, 64 * 1024 * 1024) + self.assertTrue(rotated) + self.assertEqual(offset, 0) + writer.write("200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") + writer.flush() + rows, _, rotated = watch.capture(path, offset) + self.assertEqual(rows[0]["ack"], "20") + self.assertFalse(rotated) + def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): text = "\n".join([ "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 067bc8be8f9235ae70018e525a2b8221680f04ae..32e682f7579f3ea5e907af58249cea4fe2349f77 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -11,6 +11,8 @@ import socket import subprocess import time +from substation_packets import packet + ROOT = Path.home() / ".local/state/substation-watch" COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" @@ -82,12 +84,37 @@ def stalls(samples): return stalled +def capture(path, offset): + rows = [] + rotated = False + if path.exists(): + with path.open("r+b") as file: + if path.stat().st_size < offset: + offset = 0 + file.seek(offset) + data = file.read(8 * 1024 * 1024) + end = data.rfind(b"\n") + 1 + offset += end + for line in data[:end].splitlines(): + if row := packet(line.decode("ascii", errors="replace")): + rows.append(row) + if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: + # tcpdump holds an append descriptor; truncation preserves its capture privileges. + file.seek(0) + file.truncate() + offset = 0 + rotated = True + return rows, offset, rotated + + def main(): os.umask(0o077) ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) targets = set() samples = deque(maxlen=120) + packets = deque(maxlen=100000) stalled = set() + capture_offset = 0 resolved = measured = 0 while True: now = time.time() @@ -99,6 +126,12 @@ def main(): result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) rows = sockets(result.stdout, targets) record["connections"] = rows + captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) + packets.extend(captured) + while packets and packets[0]["ts"] < now - 120: + packets.popleft() + if rotated: + record["capture_rotated"] = True if now - measured >= 5: pids = sorted({row["pid"] for row in rows if row["pid"]}) if pids: @@ -121,7 +154,9 @@ def main(): if added := current - stalled: path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") pending = path.with_suffix(".pending") - pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) + endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} + pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), + "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() diff --git a/tools/substation-watch-test.py b/tools/substation-watch-test.py index 249eed41a48832c572c002241b474e0c35a92c25..b362f375ba2696aec26182b2b2f86dbcf9784cc5 100644 --- a/tools/substation-watch-test.py +++ b/tools/substation-watch-test.py @@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase): self.assertEqual(row["flags"], "R.") self.assertEqual(row["sack"], ["50:60"]) self.assertNotIn("private", json.dumps(row)) + mac = watch.packet("190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10") + self.assertEqual(mac, row) + ipv6 = watch.packet("190.123 IP6 2001:db8::1.50000 > 2001:db8::5.443: Flags [.], ack 30, win 40, length 0") + self.assertEqual(ipv6["source"], "[2001:db8::1]:50000") edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"} self.assertEqual(watch.incident(edge, [], [row])["packets"], [row]) edge["client"] = "192.0.2.1:50001" diff --git a/tools/substation-watch.py b/tools/substation-watch.py index d4e9d630a41bd9e3f76a8288200873c5f2787bb9..d2bb8c93f2fff0fdfc407b393c0aae6901a8a318 100644 --- a/tools/substation-watch.py +++ b/tools/substation-watch.py @@ -10,6 +10,8 @@ import threading import time from urllib.request import Request, urlopen +from substation_packets import packet + ROOT = Path("/var/lib/snowglobe/substation-watch") EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", @@ -68,27 +70,6 @@ def event(entry, host): return None, None -def packet(line): - match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) - if not match: - return None - stamp, source, destination, flags, details = match.groups() - endpoints = {} - try: - for key, value in (("source", source), ("destination", destination)): - host, _, port = value.rpartition(".") - host = ipaddress.ip_address(host) - endpoints[key] = f"[{host}]:{int(port)}" - except ValueError: - return None - row = {"ts": float(stamp), **endpoints, "flags": flags} - for key in ("seq", "ack", "win", "length"): - if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): - row[key] = value[1] - row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) - return row - - def incident(edge, samples, packets): length = edge.get("content_length") complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None diff --git a/tools/substation.md b/tools/substation.md index f6afb5a294c2c775dff20f29d7e346f3c34569e0..7639857df27be0c30aeb76d17d07de0dadc3a03a 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress. An upload with queued bytes and no acknowledgement progress for 10 seconds retains the preceding two minutes in a private `stall-.json` file. The latest 100 stalls survive rolling-history expiry and watcher restarts. +When an administrator starts a TCP summary capture into `tcp-headers.log`, the +client watcher retains matching sequence/ACK/SACK metadata with each stall. +It consumes and truncates that append-only log at 64 MiB without restarting +tcpdump. Samples mark each truncation; exclude that instant from packet-loss +conclusions because concurrent writes can be lost during truncation. diff --git a/tools/substation_packets.py b/tools/substation_packets.py new file mode 100644 index 0000000000000000000000000000000000000000..ea54c3bcfaffb73f999660be85813753144d37b2 --- /dev/null +++ b/tools/substation_packets.py @@ -0,0 +1,23 @@ +import ipaddress +import re + + +def packet(line): + match = re.fullmatch(r"([0-9.]+) (?:\S+\s+(?:In|Out)\s+)?IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) + if not match: + return None + stamp, source, destination, flags, details = match.groups() + endpoints = {} + try: + for key, value in (("source", source), ("destination", destination)): + host, _, port = value.rpartition(".") + host = ipaddress.ip_address(host) + endpoints[key] = f"[{host}]:{int(port)}" + except ValueError: + return None + row = {"ts": float(stamp), **endpoints, "flags": flags} + for key in ("seq", "ack", "win", "length"): + if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): + row[key] = value[1] + row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) + return row -- 2.54.1