| author | |
| committer | |
| log | 3bc44ef75c76cdead70fd5f67b61325c5618f368 |
| tree | dcbb4a790569e1584d16c085bdb7a00ad97cd4d9 |
| parent | 9cd6ce8c7d9ad4395f6deb4ef710c060029c077f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Retain sender-side socket counters, process IDs, retransmissions and windows
for Substation connections, without command lines or application payloads.
Bound private minute archives to 12 hours and supervise the exported script
with a Worktop LaunchAgent so captures survive app and agent restarts.
Validated parser privacy and filtering, PID handling, real netstat and nettop
output, source installation, LaunchAgent startup and ongoing private capture.
Assisted-by: gpt-6.1-sol3 files changed, 157 insertions(+), 2 deletions(-)
tools/substation-client-watch-test.py created+43| ... | ... | @@ -0,0 +1,43 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import importlib.util | |
| 3 | import json | |
| 4 | from pathlib import Path | |
| 5 | import unittest | |
| 6 | ||
| 7 | ||
| 8 | spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py")) | |
| 9 | watch = importlib.util.module_from_spec(spec) | |
| 10 | spec.loader.exec_module(watch) | |
| 11 | ||
| 12 | ||
| 13 | class ClientEvidence(unittest.TestCase): | |
| 14 | def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self): | |
| 15 | text = "\n".join([ | |
| 16 | "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0", | |
| 17 | "tcp4 0 0 192.0.2.1.50001 198.51.100.2.443 ESTABLISHED 10 20 1000 2000 private-name:43 0 0", | |
| 18 | "tcp4 0 0 192.0.2.1.50002 198.51.100.1.443 ESTABLISHED 30 40 1000 2000 The Knife:44 0 0", | |
| 19 | ]) | |
| 20 | rows = watch.sockets(text, {"198.51.100.1"}) | |
| 21 | self.assertEqual(len(rows), 2) | |
| 22 | self.assertEqual(rows[0]["send_q"], 1234) | |
| 23 | self.assertEqual(rows[0]["pid"], 42) | |
| 24 | self.assertEqual(rows[1]["pid"], 44) | |
| 25 | self.assertNotIn("private-name", json.dumps(rows)) | |
| 26 | ||
| 27 | def test_nettop_retransmissions_windows_and_owner(self): | |
| 28 | text = "\n".join([ | |
| 29 | ",bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win,", | |
| 30 | "2.1.322.42,10,20,0,0,0,,,,", | |
| 31 | "tcp4 192.0.2.1:50000<->198.51.100.1:443,10,2000000,0,0,123456,20.5 ms,131072,1000000,", | |
| 32 | "tcp4 192.0.2.1:50001<->198.51.100.2:443,10,20,0,0,0,2 ms,131072,1000000,", | |
| 33 | ]) | |
| 34 | rows = watch.counters(text, {"198.51.100.1"}) | |
| 35 | self.assertEqual(len(rows), 1) | |
| 36 | self.assertEqual(rows[0]["pid"], 42) | |
| 37 | self.assertEqual(rows[0]["re-tx"], "123456") | |
| 38 | self.assertEqual(rows[0]["tx_win"], "1000000") | |
| 39 | self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1") | |
| 40 | ||
| 41 | ||
| 42 | if __name__ == "__main__": | |
| 43 | unittest.main() |
tools/substation-client-watch.py created+103| ... | ... | @@ -0,0 +1,103 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import csv | |
| 3 | from datetime import datetime, timezone | |
| 4 | import ipaddress | |
| 5 | import json | |
| 6 | import os | |
| 7 | from pathlib import Path | |
| 8 | import re | |
| 9 | import socket | |
| 10 | import subprocess | |
| 11 | import time | |
| 12 | ||
| 13 | ||
| 14 | ROOT = Path.home() / ".local/state/substation-watch" | |
| 15 | COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win" | |
| 16 | ||
| 17 | ||
| 18 | def address(value): | |
| 19 | parsed = ipaddress.ip_address(value.strip("[]")) | |
| 20 | if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped: | |
| 21 | parsed = parsed.ipv4_mapped | |
| 22 | return str(parsed) | |
| 23 | ||
| 24 | ||
| 25 | def sockets(output, targets): | |
| 26 | rows = [] | |
| 27 | for line in output.splitlines(): | |
| 28 | parts = line.split() | |
| 29 | if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}: | |
| 30 | continue | |
| 31 | peer, _, port = parts[4].rpartition(".") | |
| 32 | if port != "443" or address(peer) not in targets: | |
| 33 | continue | |
| 34 | pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None) | |
| 35 | rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid, | |
| 36 | **{key: int(parts[index]) for key, index in | |
| 37 | (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}}) | |
| 38 | return rows | |
| 39 | ||
| 40 | ||
| 41 | def counters(output, targets): | |
| 42 | reader = csv.reader(output.splitlines()) | |
| 43 | header = next(reader, []) | |
| 44 | rows = [] | |
| 45 | pid = None | |
| 46 | for values in reader: | |
| 47 | if not values: | |
| 48 | continue | |
| 49 | if not values[0].startswith(("tcp4 ", "tcp6 ")): | |
| 50 | last = values[0].rpartition(".")[2] | |
| 51 | pid = int(last) if last.isdigit() else None | |
| 52 | continue | |
| 53 | peer = values[0].partition("<->")[2] | |
| 54 | host, _, port = peer.rpartition(":") | |
| 55 | if port != "443" or address(host) not in targets: | |
| 56 | continue | |
| 57 | row = {"connection": values[0], "pid": pid} | |
| 58 | for key, value in zip(header[1:], values[1:]): | |
| 59 | if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value): | |
| 60 | row[key] = value | |
| 61 | rows.append(row) | |
| 62 | return rows | |
| 63 | ||
| 64 | ||
| 65 | def main(): | |
| 66 | os.umask(0o077) | |
| 67 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) | |
| 68 | targets = set() | |
| 69 | resolved = measured = 0 | |
| 70 | while True: | |
| 71 | now = time.time() | |
| 72 | record = {"ts": now} | |
| 73 | try: | |
| 74 | if now - resolved > 300: | |
| 75 | targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)} | |
| 76 | resolved = now | |
| 77 | result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5) | |
| 78 | rows = sockets(result.stdout, targets) | |
| 79 | record["connections"] = rows | |
| 80 | if now - measured >= 5: | |
| 81 | pids = sorted({row["pid"] for row in rows if row["pid"]}) | |
| 82 | if pids: | |
| 83 | result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="], | |
| 84 | capture_output=True, text=True, timeout=5) | |
| 85 | record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]} | |
| 86 | for line in result.stdout.splitlines() if len(parts := line.split()) == 4] | |
| 87 | result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS, | |
| 88 | *(part for pid in pids for part in ("-p", str(pid)))], | |
| 89 | check=True, capture_output=True, text=True, timeout=10) | |
| 90 | record["tcp"] = counters(result.stdout, targets) | |
| 91 | measured = now | |
| 92 | except Exception as error: | |
| 93 | record["error"] = type(error).__name__ | |
| 94 | stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M") | |
| 95 | with (ROOT / (stamp + ".jsonl")).open("a") as file: | |
| 96 | file.write(json.dumps(record, separators=(",", ":")) + "\n") | |
| 97 | for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]: | |
| 98 | path.unlink() | |
| 99 | time.sleep(1) | |
| 100 | ||
| 101 | ||
| 102 | if __name__ == "__main__": | |
| 103 | main() |
tools/substation.md+11-2| ... | ... | @@ -111,8 +111,9 @@ request ID and rejects redirects, so an OAuth login page cannot pass as a health |
| 111 | 111 | API. It sends no generation requests and logs only check outcomes and IDs. |
| 112 | 112 | |
| 113 | 113 | Inspect `substation` task `monitor` in the service logs, and search Caddy's |
| 114 | journal for the response's request ID. A 499 with no upstream status marks a | |
| 115 | cancellation before upstream headers; an upstream 499 is recorded separately. | |
| 114 | journal for the response's request ID. A 499 with no upstream status can mean | |
| 115 | client cancellation or Caddy's request-body idle timeout; an upstream 499 is | |
| 116 | recorded separately. | |
| 116 | 117 | Compare `bytes_read` with `content_length` to identify incomplete fixed-length |
| 117 | 118 | uploads. A missing length can mean a chunked request; it does not prove completion. |
| 118 | 119 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| ... | ... | @@ -131,3 +132,11 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria |
| 131 | 132 | Logs. `upload_complete: false` proves an incomplete fixed-length upload; |
| 132 | 133 | the saved receive/send queues, byte counters, receive windows, and retransmits |
| 133 | 134 | distinguish stalled sending from server backpressure and network loss. |
| 135 | ||
| 136 | Worktop's `net.paperclover.substation-watch` LaunchAgent records matching | |
| 137 | Substation sockets under `~/.local/state/substation-watch`. Minute files retain | |
| 138 | 12 hours of process IDs, CPU time/state, TCP queues, byte counters, | |
| 139 | retransmissions and windows. | |
| 140 | The installed script is an exported revision of `tools/substation-client-watch.py`; | |
| 141 | the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link. | |
| 142 | Match UTC timestamps to the server incident before comparing TCP progress. |