| 1 | #!/usr/bin/env python3 |
| 2 | import csv |
| 3 | from collections import deque |
| 4 | from datetime import datetime, timezone |
| 5 | import ipaddress |
| 6 | import json |
| 7 | import os |
| 8 | from pathlib import Path |
| 9 | import re |
| 10 | import socket |
| 11 | import subprocess |
| 12 | import threading |
| 13 | import time |
| 14 | |
| 15 | from substation_packets import packet |
| 16 | |
| 17 | |
| 18 | ROOT = Path.home() / ".local/state/substation-watch" |
| 19 | COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" |
| 20 | |
| 21 | |
| 22 | def address(value): |
| 23 | parsed = ipaddress.ip_address(value.strip("[]")) |
| 24 | if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped: |
| 25 | parsed = parsed.ipv4_mapped |
| 26 | return str(parsed) |
| 27 | |
| 28 | |
| 29 | def sockets(output, targets): |
| 30 | rows = [] |
| 31 | for line in output.splitlines(): |
| 32 | parts = line.split() |
| 33 | if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}: |
| 34 | continue |
| 35 | peer, _, port = parts[4].rpartition(".") |
| 36 | if port != "443" or address(peer) not in targets: |
| 37 | continue |
| 38 | pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None) |
| 39 | rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid, |
| 40 | **{key: int(parts[index]) for key, index in |
| 41 | (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}}) |
| 42 | return rows |
| 43 | |
| 44 | |
| 45 | def counters(output, targets): |
| 46 | reader = csv.reader(output.splitlines()) |
| 47 | header = next(reader, []) |
| 48 | rows = [] |
| 49 | pid = None |
| 50 | for values in reader: |
| 51 | if not values: |
| 52 | continue |
| 53 | if not values[0].startswith(("tcp4 ", "tcp6 ")): |
| 54 | last = values[0].rpartition(".")[2] |
| 55 | pid = int(last) if last.isdigit() else None |
| 56 | continue |
| 57 | peer = values[0].partition("<->")[2] |
| 58 | host, _, port = peer.rpartition(":") |
| 59 | if port != "443" or address(host) not in targets: |
| 60 | continue |
| 61 | row = {"connection": values[0], "pid": pid} |
| 62 | for key, value in zip(header[1:], values[1:]): |
| 63 | if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value): |
| 64 | row[key] = value |
| 65 | rows.append(row) |
| 66 | return rows |
| 67 | |
| 68 | |
| 69 | def stalls(samples): |
| 70 | latest = samples[-1] |
| 71 | stalled = set() |
| 72 | for row in latest.get("connections", []): |
| 73 | if not row["pid"] or not row["send_q"]: |
| 74 | continue |
| 75 | key = (row["local"], row["peer"], row["pid"]) |
| 76 | acknowledged = row["bytes_sent"] - row["send_q"] |
| 77 | since = latest["ts"] |
| 78 | for sample in reversed(samples): |
| 79 | prior = next((r for r in sample.get("connections", []) if (r["local"], r["peer"], r["pid"]) == key), None) |
| 80 | if not prior or not prior["send_q"] or prior["bytes_sent"] - prior["send_q"] != acknowledged: |
| 81 | break |
| 82 | since = sample["ts"] |
| 83 | if latest["ts"] - since >= 10: |
| 84 | stalled.add(key) |
| 85 | return stalled |
| 86 | |
| 87 | |
| 88 | def capture(path, offset): |
| 89 | rows = [] |
| 90 | truncated_at = None |
| 91 | if path.exists(): |
| 92 | with path.open("r+b") as file: |
| 93 | if path.stat().st_size < offset: |
| 94 | offset = 0 |
| 95 | file.seek(offset) |
| 96 | data = file.read(8 * 1024 * 1024) |
| 97 | end = data.rfind(b"\n") + 1 |
| 98 | offset += end |
| 99 | if offset >= 64 * 1024 * 1024 and offset == path.stat().st_size: |
| 100 | # tcpdump holds an append descriptor; truncation preserves its capture privileges. |
| 101 | file.seek(0) |
| 102 | file.truncate() |
| 103 | offset = 0 |
| 104 | truncated_at = time.time() |
| 105 | cutoff = time.time() - 120 |
| 106 | for line in data[:end].splitlines(): |
| 107 | try: |
| 108 | if float(line.partition(b" ")[0]) < cutoff: |
| 109 | continue |
| 110 | except ValueError: |
| 111 | continue |
| 112 | if row := packet(line.decode("ascii", errors="replace")): |
| 113 | rows.append(row) |
| 114 | return rows, offset, truncated_at |
| 115 | |
| 116 | |
| 117 | def main(): |
| 118 | os.umask(0o077) |
| 119 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 120 | targets = set() |
| 121 | samples = deque(maxlen=120) |
| 122 | packets = deque(maxlen=100000) |
| 123 | stalled = {} |
| 124 | capture_state = {"offset": 0, "busy": False} |
| 125 | capture_gaps = deque(maxlen=100) |
| 126 | capture_path = ROOT / "tcp-headers.log" |
| 127 | |
| 128 | def read_capture(): |
| 129 | while True: |
| 130 | capture_state["busy"] = True |
| 131 | try: |
| 132 | rows, offset, truncated_at = capture(capture_path, capture_state["offset"]) |
| 133 | packets.extend(rows) |
| 134 | while packets and packets[0]["ts"] < time.time() - 120: |
| 135 | packets.popleft() |
| 136 | capture_state["offset"] = offset |
| 137 | capture_state.pop("error", None) |
| 138 | if truncated_at is not None: |
| 139 | capture_gaps.append(truncated_at) |
| 140 | except Exception as error: |
| 141 | capture_state["error"] = type(error).__name__ |
| 142 | finally: |
| 143 | capture_state["busy"] = False |
| 144 | time.sleep(0.1) |
| 145 | |
| 146 | threading.Thread(target=read_capture, daemon=True).start() |
| 147 | resolved = measured = 0 |
| 148 | while True: |
| 149 | now = time.time() |
| 150 | record = {"ts": now} |
| 151 | try: |
| 152 | if now - resolved > 300: |
| 153 | targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)} |
| 154 | resolved = now |
| 155 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) |
| 156 | rows = sockets(result.stdout, targets) |
| 157 | record["connections"] = rows |
| 158 | if capture_path.exists(): |
| 159 | record["capture_backlog"] = max(0, capture_path.stat().st_size - capture_state["offset"]) |
| 160 | record["capture_busy"] = capture_state["busy"] |
| 161 | if "error" in capture_state: |
| 162 | record["capture_error"] = capture_state["error"] |
| 163 | if now - measured >= 5: |
| 164 | pids = sorted({row["pid"] for row in rows if row["pid"]}) |
| 165 | if pids: |
| 166 | result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="], |
| 167 | capture_output=True, text=True, timeout=5) |
| 168 | record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]} |
| 169 | for line in result.stdout.splitlines() if len(parts := line.split()) == 4] |
| 170 | result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS, |
| 171 | *(part for pid in pids for part in ("-p", str(pid)))], |
| 172 | check=True, capture_output=True, text=True, timeout=10) |
| 173 | record["tcp"] = counters(result.stdout, targets) |
| 174 | measured = now |
| 175 | except Exception as error: |
| 176 | record["error"] = type(error).__name__ |
| 177 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") |
| 178 | with (ROOT / (stamp + ".jsonl")).open("a") as file: |
| 179 | file.write(json.dumps(record, separators=(",", ":")) + "\n") |
| 180 | samples.append(record) |
| 181 | current = stalls(samples) |
| 182 | for key in current | stalled.keys(): |
| 183 | if key in current: |
| 184 | if key in stalled: |
| 185 | continue |
| 186 | path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json" |
| 187 | stalled[key] = path |
| 188 | else: |
| 189 | if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")): |
| 190 | continue |
| 191 | path = stalled.pop(key) |
| 192 | pending = path.with_suffix(".pending") |
| 193 | local, _, port = key[0].rpartition(".") |
| 194 | endpoint = f"[{local}]:{port}" |
| 195 | pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples), |
| 196 | "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120], |
| 197 | "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":"))) |
| 198 | pending.replace(path) |
| 199 | for old in sorted(ROOT.glob("stall-*.json"))[:-100]: |
| 200 | old.unlink() |
| 201 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: |
| 202 | path.unlink() |
| 203 | time.sleep(1) |
| 204 | |
| 205 | |
| 206 | if __name__ == "__main__": |
| 207 | main() |