From c65330b68ae0ff9f783e00a339d61006cf9c8ff0 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 15:56:04 -0800 Subject: [PATCH] Retain TCP close and acknowledgement metadata for Substation Keep bounded transport-only packet metadata for known API clients. Attach matching sequence numbers, ACK/SACK ranges, windows and FIN/RST timing to failed-request evidence so sender loss and server cancellation can be told apart. Validated parser privacy/filtering, five watcher tests, ten deployment tests, staging health and an isolated transient service on Snow Globe. A disposable partial upload retained the client RST before its Caddy error. Assisted-by: gpt-6.1-sol --- nixos/configuration.nix | 2 +- tools/substation-watch-test.py | 17 +++++++++-- tools/substation-watch.py | 55 ++++++++++++++++++++++++++++++---- tools/substation.md | 3 ++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 6c90198680f9e2604d10569a88d14cddd9672151..bccf9b25f415a4e75d26e77da9049251486d1396 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -156,7 +156,7 @@ in after = [ "nomad.service" "caddy.service" ]; unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ]; environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN; - path = [ pkgs.systemd pkgs.iproute2 ]; + path = [ pkgs.systemd pkgs.iproute2 pkgs.tcpdump ]; serviceConfig = { ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py"; Restart = "always"; diff --git a/tools/substation-watch-test.py b/tools/substation-watch-test.py index 283096efbdeae2d29ebb848a585dc0992bbe091b..249eed41a48832c572c002241b474e0c35a92c25 100644 --- a/tools/substation-watch-test.py +++ b/tools/substation-watch-test.py @@ -43,13 +43,24 @@ class TransportEvidence(unittest.TestCase): def test_completion_and_window(self): edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40} samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}] - record = watch.incident(edge, samples) + record = watch.incident(edge, samples, []) self.assertFalse(record["upload_complete"]) self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}]) edge["bytes_read"] = 100 - self.assertTrue(watch.incident(edge, samples)["upload_complete"]) + self.assertTrue(watch.incident(edge, samples, [])["upload_complete"]) edge["content_length"] = "" - self.assertIsNone(watch.incident(edge, samples)["upload_complete"]) + self.assertIsNone(watch.incident(edge, samples, [])["upload_complete"]) + + def test_packet_metadata_excludes_payload_and_other_connections(self): + row = watch.packet("190.123 lo In IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10 private-payload") + self.assertEqual(row["flags"], "R.") + self.assertEqual(row["sack"], ["50:60"]) + self.assertNotIn("private", json.dumps(row)) + edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"} + self.assertEqual(watch.incident(edge, [], [row])["packets"], [row]) + edge["client"] = "192.0.2.1:50001" + self.assertEqual(watch.incident(edge, [], [row])["packets"], []) + self.assertIsNone(watch.packet("private application payload")) def test_private_atomic_state_survives_reload(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/tools/substation-watch.py b/tools/substation-watch.py index 148fa8ddc158f539534f8fa8eb99097e02d4c213..d4e9d630a41bd9e3f76a8288200873c5f2787bb9 100644 --- a/tools/substation-watch.py +++ b/tools/substation-watch.py @@ -6,6 +6,7 @@ import os from pathlib import Path import re import subprocess +import threading import time from urllib.request import Request, urlopen @@ -67,11 +68,35 @@ def event(entry, host): return None, None -def incident(edge, samples): +def packet(line): + match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip()) + if not match: + return None + stamp, source, destination, flags, details = match.groups() + endpoints = {} + try: + for key, value in (("source", source), ("destination", destination)): + host, _, port = value.rpartition(".") + host = ipaddress.ip_address(host) + endpoints[key] = f"[{host}]:{int(port)}" + except ValueError: + return None + row = {"ts": float(stamp), **endpoints, "flags": flags} + for key in ("seq", "ack", "win", "length"): + if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details): + row[key] = value[1] + row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details) + return row + + +def incident(edge, samples, packets): length = edge.get("content_length") complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None + peer = endpoint(edge["client"]) if edge.get("client") else None return {"edge": edge, "upload_complete": complete, - "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]} + "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5], + "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5 + and peer in (endpoint(row["source"]), endpoint(row["destination"]))]} def main(): @@ -81,12 +106,32 @@ def main(): state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} recent = ROOT / "recent.json" samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) + packets = deque(maxlen=100000) + capture = None + captured_clients = None + + def read_packets(stream): + for line in stream: + row = packet(line) + if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")): + packets.append(row) + ports = set() refreshed = saved = reported = 0 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] while True: try: now = time.time() + clients = tuple(sorted(state["clients"])) + if capture is None or capture.poll() is not None or clients != captured_clients: + if capture is not None: + capture.terminate() + capture.wait(timeout=2) + filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")" + capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start() + captured_clients = clients if now - refreshed >= 10: token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) @@ -109,11 +154,11 @@ def main(): if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): path = ROOT / (edge["request_id"] + ".json") if not path.exists(): - record = incident(edge, samples) + record = incident(edge, samples, packets) save(path, record) print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], "status": edge["status"], "upload_complete": record["upload_complete"], - "samples": len(record["tcp"])}), flush=True) + "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True) state["cursor"] = entry["__CURSOR"] state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} if ports: @@ -130,7 +175,7 @@ def main(): path.unlink() if now - reported >= 60: print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), - "clients": len(state["clients"]), "samples": len(samples)}), flush=True) + "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True) reported = now except Exception as error: print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) diff --git a/tools/substation.md b/tools/substation.md index cb0cc729943dedbfa5a3b3f539f99f7fb39cf9e6..c985e9e3e094844669aad8b2b34cc8471f480418 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -132,6 +132,9 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria Logs. `upload_complete: false` proves an incomplete fixed-length upload; the saved receive/send queues, byte counters, receive windows, and retransmits distinguish stalled sending from server backpressure and network loss. +Incident files also retain matching TCP sequence numbers, acknowledgements, +SACK ranges, windows and flags. These establish packet loss and which side +sent FIN/RST first. Only parsed transport fields are retained. Worktop's `net.paperclover.substation-watch` LaunchAgent records matching Substation sockets under `~/.local/state/substation-watch`. Minute files retain -- 2.54.1