authoravatarclover caruso<git@paperclover.net>2026-11-11 15:56:04-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
logc65330b68ae0ff9f783e00a339d61006cf9c8ff0
treec86163368fa3946cb6c0fcb3f238d1cceef8d949
parent3bc44ef75c76cdead70fd5f67b61325c5618f368
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain TCP close and acknowledgement metadata for Substation

Keep bounded transport-only packet metadata for known API clients. Attach matching sequence numbers, ACK/SACK ranges, windows and FIN/RST timing to failed-request evidence so sender loss and server cancellation can be told apart. Validated parser privacy/filtering, five watcher tests, ten deployment tests, staging health and an isolated transient service on Snow Globe. A disposable partial upload retained the client RST before its Caddy error. Assisted-by: gpt-6.1-sol

4 files changed, 68 insertions(+), 9 deletions(-)

nixos/configuration.nix+1-1
...@@ -156,7 +156,7 @@ in...@@ -156,7 +156,7 @@ in
156 after = [ "nomad.service" "caddy.service" ];156 after = [ "nomad.service" "caddy.service" ];
157 unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ];157 unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ];
158 environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN;158 environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN;
159 path = [ pkgs.systemd pkgs.iproute2 ];159 path = [ pkgs.systemd pkgs.iproute2 pkgs.tcpdump ];
160 serviceConfig = {160 serviceConfig = {
161 ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py";161 ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py";
162 Restart = "always";162 Restart = "always";
tools/substation-watch-test.py+14-3
...@@ -43,13 +43,24 @@ class TransportEvidence(unittest.TestCase):...@@ -43,13 +43,24 @@ class TransportEvidence(unittest.TestCase):
43 def test_completion_and_window(self):43 def test_completion_and_window(self):
44 edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40}44 edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40}
45 samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}]45 samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}]
46 record = watch.incident(edge, samples)46 record = watch.incident(edge, samples, [])
47 self.assertFalse(record["upload_complete"])47 self.assertFalse(record["upload_complete"])
48 self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}])48 self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}])
49 edge["bytes_read"] = 10049 edge["bytes_read"] = 100
50 self.assertTrue(watch.incident(edge, samples)["upload_complete"])50 self.assertTrue(watch.incident(edge, samples, [])["upload_complete"])
51 edge["content_length"] = ""51 edge["content_length"] = ""
52 self.assertIsNone(watch.incident(edge, samples)["upload_complete"])52 self.assertIsNone(watch.incident(edge, samples, [])["upload_complete"])
53
54 def test_packet_metadata_excludes_payload_and_other_connections(self):
55 row = watch.packet("190.123 lo In IP 192.0.2.1.50000 > 192.0.2.5.443: Flags [R.], seq 10:20, ack 30, win 40, options [sack 1 {50:60}], length 10 private-payload")
56 self.assertEqual(row["flags"], "R.")
57 self.assertEqual(row["sack"], ["50:60"])
58 self.assertNotIn("private", json.dumps(row))
59 edge = {"ts": 200, "duration": 60, "client": "192.0.2.1:50000"}
60 self.assertEqual(watch.incident(edge, [], [row])["packets"], [row])
61 edge["client"] = "192.0.2.1:50001"
62 self.assertEqual(watch.incident(edge, [], [row])["packets"], [])
63 self.assertIsNone(watch.packet("private application payload"))
5364
54 def test_private_atomic_state_survives_reload(self):65 def test_private_atomic_state_survives_reload(self):
55 with tempfile.TemporaryDirectory() as temporary:66 with tempfile.TemporaryDirectory() as temporary:
tools/substation-watch.py+50-5
...@@ -6,6 +6,7 @@ import os...@@ -6,6 +6,7 @@ import os
6from pathlib import Path6from pathlib import Path
7import re7import re
8import subprocess8import subprocess
9import threading
9import time10import time
10from urllib.request import Request, urlopen11from urllib.request import Request, urlopen
1112
...@@ -67,11 +68,35 @@ def event(entry, host):...@@ -67,11 +68,35 @@ def event(entry, host):
67 return None, None68 return None, None
6869
6970
70def incident(edge, samples):71def packet(line):
72 match = re.fullmatch(r"([0-9.]+) \S+\s+(?:In|Out)\s+IP6? (\S+) > (\S+): Flags \[([A-Za-z.*]+)\], (.*)", line.strip())
73 if not match:
74 return None
75 stamp, source, destination, flags, details = match.groups()
76 endpoints = {}
77 try:
78 for key, value in (("source", source), ("destination", destination)):
79 host, _, port = value.rpartition(".")
80 host = ipaddress.ip_address(host)
81 endpoints[key] = f"[{host}]:{int(port)}"
82 except ValueError:
83 return None
84 row = {"ts": float(stamp), **endpoints, "flags": flags}
85 for key in ("seq", "ack", "win", "length"):
86 if value := re.search(r"\b" + key + r" ([0-9]+(?::[0-9]+)?)", details):
87 row[key] = value[1]
88 row["sack"] = re.findall(r"\{([0-9]+:[0-9]+)\}", details)
89 return row
90
91
92def incident(edge, samples, packets):
71 length = edge.get("content_length")93 length = edge.get("content_length")
72 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None94 complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None
95 peer = endpoint(edge["client"]) if edge.get("client") else None
73 return {"edge": edge, "upload_complete": complete,96 return {"edge": edge, "upload_complete": complete,
74 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]}97 "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5],
98 "packets": [row for row in list(packets) if edge["ts"] - edge["duration"] - 5 <= row["ts"] <= edge["ts"] + 5
99 and peer in (endpoint(row["source"]), endpoint(row["destination"]))]}
75100
76101
77def main():102def main():
...@@ -81,12 +106,32 @@ def main():...@@ -81,12 +106,32 @@ def main():
81 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}106 state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}}
82 recent = ROOT / "recent.json"107 recent = ROOT / "recent.json"
83 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)108 samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120)
109 packets = deque(maxlen=100000)
110 capture = None
111 captured_clients = None
112
113 def read_packets(stream):
114 for line in stream:
115 row = packet(line)
116 if row and any(endpoint(row[key])[0] in state["clients"] for key in ("source", "destination")):
117 packets.append(row)
118
84 ports = set()119 ports = set()
85 refreshed = saved = reported = 0120 refreshed = saved = reported = 0
86 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]121 host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"]
87 while True:122 while True:
88 try:123 try:
89 now = time.time()124 now = time.time()
125 clients = tuple(sorted(state["clients"]))
126 if capture is None or capture.poll() is not None or clients != captured_clients:
127 if capture is not None:
128 capture.terminate()
129 capture.wait(timeout=2)
130 filters = "tcp port 443 and (" + " or ".join("host " + client for client in (clients or ("127.0.0.1",))) + ")"
131 capture = subprocess.Popen(["tcpdump", "-l", "-tt", "-nn", "-S", "-i", "any", "-s", "128", filters],
132 stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
133 threading.Thread(target=read_packets, args=(capture.stdout,), daemon=True).start()
134 captured_clients = clients
90 if now - refreshed >= 10:135 if now - refreshed >= 10:
91 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()136 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
92 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})137 request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token})
...@@ -109,11 +154,11 @@ def main():...@@ -109,11 +154,11 @@ def main():
109 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):154 if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))):
110 path = ROOT / (edge["request_id"] + ".json")155 path = ROOT / (edge["request_id"] + ".json")
111 if not path.exists():156 if not path.exists():
112 record = incident(edge, samples)157 record = incident(edge, samples, packets)
113 save(path, record)158 save(path, record)
114 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],159 print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"],
115 "status": edge["status"], "upload_complete": record["upload_complete"],160 "status": edge["status"], "upload_complete": record["upload_complete"],
116 "samples": len(record["tcp"])}), flush=True)161 "samples": len(record["tcp"]), "packets": len(record["packets"])}), flush=True)
117 state["cursor"] = entry["__CURSOR"]162 state["cursor"] = entry["__CURSOR"]
118 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}163 state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600}
119 if ports:164 if ports:
...@@ -130,7 +175,7 @@ def main():...@@ -130,7 +175,7 @@ def main():
130 path.unlink()175 path.unlink()
131 if now - reported >= 60:176 if now - reported >= 60:
132 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),177 print(json.dumps({"event": "substation-watcher", "ports": sorted(ports),
133 "clients": len(state["clients"]), "samples": len(samples)}), flush=True)178 "clients": len(state["clients"]), "samples": len(samples), "packets": len(packets)}), flush=True)
134 reported = now179 reported = now
135 except Exception as error:180 except Exception as error:
136 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)181 print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True)
tools/substation.md+3
...@@ -132,6 +132,9 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria...@@ -132,6 +132,9 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria
132Logs. `upload_complete: false` proves an incomplete fixed-length upload;132Logs. `upload_complete: false` proves an incomplete fixed-length upload;
133the saved receive/send queues, byte counters, receive windows, and retransmits133the saved receive/send queues, byte counters, receive windows, and retransmits
134distinguish stalled sending from server backpressure and network loss.134distinguish stalled sending from server backpressure and network loss.
135Incident files also retain matching TCP sequence numbers, acknowledgements,
136SACK ranges, windows and flags. These establish packet loss and which side
137sent FIN/RST first. Only parsed transport fields are retained.
135138
136Worktop's `net.paperclover.substation-watch` LaunchAgent records matching139Worktop's `net.paperclover.substation-watch` LaunchAgent records matching
137Substation sockets under `~/.local/state/substation-watch`. Minute files retain140Substation sockets under `~/.local/state/substation-watch`. Minute files retain