diff --git a/tools/substation-client-watch-test.py b/tools/substation-client-watch-test.py index b4de4225abebb5dd933395bc37561645da5c61e9..f62593af11ff23e4b53b2ea2a00f6051d9a22f5e 100644 --- a/tools/substation-client-watch-test.py +++ b/tools/substation-client-watch-test.py @@ -2,6 +2,7 @@ import importlib.util import json from pathlib import Path +import tempfile import unittest @@ -11,6 +12,37 @@ spec.loader.exec_module(watch) class ClientEvidence(unittest.TestCase): + def test_capture_keeps_partial_lines_and_excludes_payload(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + line = "190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [P.], seq 10:20, ack 30, win 40, length 10 private-payload" + self.assertEqual(watch.capture(path, 0), ([], 0, False)) + path.write_text(line) + self.assertEqual(watch.capture(path, 0), ([], 0, False)) + with path.open("a") as writer: + writer.write("\n") + rows, offset, rotated = watch.capture(path, 0) + self.assertEqual(rows[0]["seq"], "10:20") + self.assertNotIn("private", json.dumps(rows)) + self.assertFalse(rotated) + self.assertEqual(watch.capture(path, offset), ([], offset, False)) + + def test_capture_rotation_preserves_the_open_append_writer(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "capture.log" + with path.open("a") as writer: + writer.truncate(64 * 1024 * 1024) + writer.write("\n") + writer.flush() + _, offset, rotated = watch.capture(path, 64 * 1024 * 1024) + self.assertTrue(rotated) + self.assertEqual(offset, 0) + writer.write("200.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n") + writer.flush() + rows, _, rotated = watch.capture(path, offset) + self.assertEqual(rows[0]["ack"], "20") + self.assertFalse(rotated) + def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): text = "\n".join([ "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", diff --git a/tools/substation-client-watch.py b/tools/substation-client-watch.py index 067bc8be8f9235ae70018e525a2b8221680f04ae..32e682f7579f3ea5e907af58249cea4fe2349f77 100644 --- a/tools/substation-client-watch.py +++ b/tools/substation-client-watch.py @@ -11,6 +11,8 @@ import socket import subprocess import time +from substation_packets import packet + ROOT = Path.home() / ".local/state/substation-watch" COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" @@ -82,12 +84,37 @@ def stalls(samples): return stalled +def capture(path, offset): + rows = [] + rotated = False + if path.exists(): + with path.open("r+b") as file: + if path.stat().st_size < offset: + offset = 0 + file.seek(offset) + data = file.read(8 * 1024 * 1024) + end = data.rfind(b"\n") + 1 + offset += end + for line in data[:end].splitlines(): + if row := packet(line.decode("ascii", errors="replace")): + rows.append(row) + if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: + # tcpdump holds an append descriptor; truncation preserves its capture privileges. + file.seek(0) + file.truncate() + offset = 0 + rotated = True + return rows, offset, rotated + + def main(): os.umask(0o077) ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) targets = set() samples = deque(maxlen=120) + packets = deque(maxlen=100000) stalled = set() + capture_offset = 0 resolved = measured = 0 while True: now = time.time() @@ -99,6 +126,12 @@ def main(): result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) rows = sockets(result.stdout, targets) record["connections"] = rows + captured, capture_offset, rotated = capture(ROOT / "tcp-headers.log", capture_offset) + packets.extend(captured) + while packets and packets[0]["ts"] < now - 120: + packets.popleft() + if rotated: + record["capture_rotated"] = True if now - measured >= 5: pids = sorted({row["pid"] for row in rows if row["pid"]}) if pids: @@ -121,7 +154,9 @@ def main(): if added := current - stalled: path = ROOT / ("stall-" + str(int(now * 1000)) + ".json") pending = path.with_suffix(".pending") - pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples)}, separators=(",", ":"))) + endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added} + pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples), + "packets": [row for row in packets if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":"))) pending.replace(path) for old in sorted(ROOT.glob("stall-*.json"))[:-100]: old.unlink() diff --git a/tools/substation-watch-test.py b/tools/substation-watch-test.py index 249eed41a48832c572c002241b474e0c35a92c25..b362f375ba2696aec26182b2b2f86dbcf9784cc5 100644 --- a/tools/substation-watch-test.py +++ b/tools/substation-watch-test.py @@ -56,6 +56,10 @@ class TransportEvidence(unittest.TestCase): self.assertEqual(row["flags"], "R.") self.assertEqual(row["sack"], ["50:60"]) self.assertNotIn("private", json.dumps(row)) + mac = watch.packet("190.123 IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10") + self.assertEqual(mac, row) + ipv6 = watch.packet("190.123 IP6 2001:db8::1.50000 > 2001:db8::5.443: Flags [.], ack 30, win 40, length 0") + self.assertEqual(ipv6["source"], "[2001:db8::1]:50000") edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"} self.assertEqual(watch.incident(edge, [], [row])["packets"], [row]) edge["client"] = "192.0.2.1:50001" diff --git a/tools/substation-watch.py b/tools/substation-watch.py index d4e9d630a41bd9e3f76a8288200873c5f2787bb9..d2bb8c93f2fff0fdfc407b393c0aae6901a8a318 100644 --- a/tools/substation-watch.py +++ b/tools/substation-watch.py @@ -10,6 +10,8 @@ import threading import time from urllib.request import Request, urlopen +from substation_packets import packet + ROOT = Path("/var/lib/snowglobe/substation-watch") EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", @@ -68,27 +70,6 @@ def event(entry, host): return None, None -def packet(line): - match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) - if not match: - return None - stamp, source, destination, flags, details = match.groups() - endpoints = {} - try: - for key, value in (("source", source), ("destination", destination)): - host, _, port = value.rpartition(".") - host = ipaddress.ip_address(host) - endpoints[key] = f"[{host}]:{int(port)}" - except ValueError: - return None - row = {"ts": float(stamp), **endpoints, "flags": flags} - for key in ("seq", "ack", "win", "length"): - if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): - row[key] = value[1] - row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) - return row - - def incident(edge, samples, packets): length = edge.get("content_length") complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None diff --git a/tools/substation.md b/tools/substation.md index f6afb5a294c2c775dff20f29d7e346f3c34569e0..7639857df27be0c30aeb76d17d07de0dadc3a03a 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -148,3 +148,8 @@ Match UTC timestamps to the server incident before comparing TCP progress. An upload with queued bytes and no acknowledgement progress for 10 seconds retains the preceding two minutes in a private `stall-.json` file. The latest 100 stalls survive rolling-history expiry and watcher restarts. +When an administrator starts a TCP summary capture into `tcp-headers.log`, the +client watcher retains matching sequence/ACK/SACK metadata with each stall. +It consumes and truncates that append-only log at 64 MiB without restarting +tcpdump. Samples mark each truncation; exclude that instant from packet-loss +conclusions because concurrent writes can be lost during truncation. diff --git a/tools/substation_packets.py b/tools/substation_packets.py new file mode 100644 index 0000000000000000000000000000000000000000..ea54c3bcfaffb73f999660be85813753144d37b2 --- /dev/null +++ b/tools/substation_packets.py @@ -0,0 +1,23 @@ +import ipaddress +import re + + +def packet(line): + match = re.fullmatch(r"([0-9.]+) (?:\S+\s+(?:In|Out)\s+)?IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) + if not match: + return None + stamp, source, destination, flags, details = match.groups() + endpoints = {} + try: + for key, value in (("source", source), ("destination", destination)): + host, _, port = value.rpartition(".") + host = ipaddress.ip_address(host) + endpoints[key] = f"[{host}]:{int(port)}" + except ValueError: + return None + row = {"ts": float(stamp), **endpoints, "flags": flags} + for key in ("seq", "ack", "win", "length"): + if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): + row[key] = value[1] + row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) + return row