1#!/usr/bin/env python3
2import csv
3from collections import deque
4from datetime import datetime, timezone
5import ipaddress
6import json
7import os
8from pathlib import Path
9import re
10import socket
11import subprocess
12import threading
13import time
14
15from substation_packets import packet
16
17
18ROOT = Path.home() / ".local/state/substation-watch"
19COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win"
20
21
22def address(value):
23 parsed = ipaddress.ip_address(value.strip("[]"))
24 if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped:
25 parsed = parsed.ipv4_mapped
26 return str(parsed)
27
28
29def sockets(output, targets):
30 rows = []
31 for line in output.splitlines():
32 parts = line.split()
33 if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}:
34 continue
35 peer, _, port = parts[4].rpartition(".")
36 if port != "443" or address(peer) not in targets:
37 continue
38 pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None)
39 rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid,
40 **{key: int(parts[index]) for key, index in
41 (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}})
42 return rows
43
44
45def counters(output, targets):
46 reader = csv.reader(output.splitlines())
47 header = next(reader, [])
48 rows = []
49 pid = None
50 for values in reader:
51 if not values:
52 continue
53 if not values[0].startswith(("tcp4 ", "tcp6 ")):
54 last = values[0].rpartition(".")[2]
55 pid = int(last) if last.isdigit() else None
56 continue
57 peer = values[0].partition("<->")[2]
58 host, _, port = peer.rpartition(":")
59 if port != "443" or address(host) not in targets:
60 continue
61 row = {"connection": values[0], "pid": pid}
62 for key, value in zip(header[1:], values[1:]):
63 if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value):
64 row[key] = value
65 rows.append(row)
66 return rows
67
68
69def stalls(samples):
70 latest = samples[-1]
71 stalled = set()
72 for row in latest.get("connections", []):
73 if not row["pid"] or not row["send_q"]:
74 continue
75 key = (row["local"], row["peer"], row["pid"])
76 acknowledged = row["bytes_sent"] - row["send_q"]
77 since = latest["ts"]
78 for sample in reversed(samples):
79 prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None)
80 if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged:
81 break
82 since = sample["ts"]
83 if latest["ts"] - since >= 10:
84 stalled.add(key)
85 return stalled
86
87
88def capture(path, offset):
89 rows = []
90 truncated_at = None
91 if path.exists():
92 with path.open("r+b") as file:
93 if path.stat().st_size < offset:
94 offset = 0
95 file.seek(offset)
96 data = file.read(8 * 1024 * 1024)
97 end = data.rfind(b"\n") + 1
98 offset += end
99 if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size:
100 # tcpdump holds an append descriptor; truncation preserves its capture privileges.
101 file.seek(0)
102 file.truncate()
103 offset = 0
104 truncated_at = time.time()
105 cutoff = time.time() - 120
106 for line in data[:end].splitlines():
107 try:
108 if float(line.partition(b" ")[0]) < cutoff:
109 continue
110 except ValueError:
111 continue
112 if row := packet(line.decode("ascii", errors="replace")):
113 rows.append(row)
114 return rows, offset, truncated_at
115
116
117def main():
118 os.umask(0o077)
119 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
120 targets = set()
121 samples = deque(maxlen=120)
122 packets = deque(maxlen=100000)
123 stalled = {}
124 capture_state = {"offset": 0, "busy": False}
125 capture_gaps = deque(maxlen=100)
126 capture_path = ROOT / "tcp-headers.log"
127
128 def read_capture():
129 while True:
130 capture_state["busy"] = True
131 try:
132 rows, offset, truncated_at = capture(capture_path, capture_state["offset"])
133 packets.extend(rows)
134 while packets and packets[0]["ts"] < time.time() - 120:
135 packets.popleft()
136 capture_state["offset"] = offset
137 capture_state.pop("error", None)
138 if truncated_at is not None:
139 capture_gaps.append(truncated_at)
140 except Exception as error:
141 capture_state["error"] = type(error).__name__
142 finally:
143 capture_state["busy"] = False
144 time.sleep(0.1)
145
146 threading.Thread(target=read_capture, daemon=True).start()
147 resolved = measured = 0
148 while True:
149 now = time.time()
150 record = {"ts": now}
151 try:
152 if now - resolved > 300:
153 targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)}
154 resolved = now
155 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
156 rows = sockets(result.stdout, targets)
157 record["connections"] = rows
158 if capture_path.exists():
159 record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"])
160 record["capture_busy"] = capture_state["busy"]
161 if "error" in capture_state:
162 record["capture_error"] = capture_state["error"]
163 if now - measured >= 5:
164 pids = sorted({row["pid"] for row in rows if row["pid"]})
165 if pids:
166 result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="],
167 capture_output=True, text=True, timeout=5)
168 record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]}
169 for line in result.stdout.splitlines() if len(parts := line.split()) == 4]
170 result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS,
171 *(part for pid in pids for part in ("-p", str(pid)))],
172 check=True, capture_output=True, text=True, timeout=10)
173 record["tcp"] = counters(result.stdout, targets)
174 measured = now
175 except Exception as error:
176 record["error"] = type(error).__name__
177 stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M")
178 with (ROOT / (stamp + ".jsonl")).open("a") as file:
179 file.write(json.dumps(record, separators=(",", ":")) + "\n")
180 samples.append(record)
181 current = stalls(samples)
182 for key in current | stalled.keys():
183 if key in current:
184 if key in stalled:
185 continue
186 path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json"
187 stalled[key] = path
188 else:
189 if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")):
190 continue
191 path = stalled.pop(key)
192 pending = path.with_suffix(".pending")
193 local, _, port = key[0].rpartition(".")
194 endpoint = f"[{local}]:{port}"
195 pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples),
196 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],
197 "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":")))
198 pending.replace(path)
199 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
200 old.unlink()
201 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
202 path.unlink()
203 time.sleep(1)
204
205
206if __name__ == "__main__":
207 main()