authoravatarclover caruso<git@paperclover.net>2026-11-11 15:41:21-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log3bc44ef75c76cdead70fd5f67b61325c5618f368
treedcbb4a790569e1584d16c085bdb7a00ad97cd4d9
parent9cd6ce8c7d9ad4395f6deb4ef710c060029c077f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Capture Worktop TCP upload progress for Substation

Retain sender-side socket counters, process IDs, retransmissions and windows for Substation connections, without command lines or application payloads. Bound private minute archives to 12 hours and supervise the exported script with a Worktop LaunchAgent so captures survive app and agent restarts. Validated parser privacy and filtering, PID handling, real netstat and nettop output, source installation, LaunchAgent startup and ongoing private capture. Assisted-by: gpt-6.1-sol

3 files changed, 157 insertions(+), 2 deletions(-)

tools/substation-client-watch-test.py created+43
......@@ -0,0 +1,43 @@
1#!/usr/bin/env python3
2import importlib.util
3import json
4from pathlib import Path
5import unittest
6
7
8spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-client-watch.py"))
9watch = importlib.util.module_from_spec(spec)
10spec.loader.exec_module(watch)
11
12
13class ClientEvidence(unittest.TestCase):
14 def test_socket_counters_and_pids_exclude_other_destinations_and_program_names(self):
15 text = "\n".join([
16 "tcp4 0 1234 192.0.2.1.50000 198.51.100.1.443 ESTABLISHED 10 20 1000 2000 private-name:42 0 0",
17 "tcp4 0 0 192.0.2.1.50001 198.51.100.2.443 ESTABLISHED 10 20 1000 2000 private-name:43 0 0",
18 "tcp4 0 0 192.0.2.1.50002 198.51.100.1.443 ESTABLISHED 30 40 1000 2000 The Knife:44 0 0",
19 ])
20 rows = watch.sockets(text, {"198.51.100.1"})
21 self.assertEqual(len(rows), 2)
22 self.assertEqual(rows[0]["send_q"], 1234)
23 self.assertEqual(rows[0]["pid"], 42)
24 self.assertEqual(rows[1]["pid"], 44)
25 self.assertNotIn("private-name", json.dumps(rows))
26
27 def test_nettop_retransmissions_windows_and_owner(self):
28 text = "\n".join([
29 ",bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win,",
30 "2.1.322.42,10,20,0,0,0,,,,",
31 "tcp4 192.0.2.1:50000<->198.51.100.1:443,10,2000000,0,0,123456,20.5 ms,131072,1000000,",
32 "tcp4 192.0.2.1:50001<->198.51.100.2:443,10,20,0,0,0,2 ms,131072,1000000,",
33 ])
34 rows = watch.counters(text, {"198.51.100.1"})
35 self.assertEqual(len(rows), 1)
36 self.assertEqual(rows[0]["pid"], 42)
37 self.assertEqual(rows[0]["re-tx"], "123456")
38 self.assertEqual(rows[0]["tx_win"], "1000000")
39 self.assertEqual(watch.address("::ffff:198.51.100.1"), "198.51.100.1")
40
41
42if __name__ == "__main__":
43 unittest.main()
tools/substation-client-watch.py created+103
......@@ -0,0 +1,103 @@
1#!/usr/bin/env python3
2import csv
3from datetime import datetime, timezone
4import ipaddress
5import json
6import os
7from pathlib import Path
8import re
9import socket
10import subprocess
11import time
12
13
14ROOT = Path.home() / ".local/state/substation-watch"
15COLUMNS = "bytes_in,bytes_out,rx_dupe,rx_ooo,re-tx,rtt_avg,rcvsize,tx_win"
16
17
18def address(value):
19 parsed = ipaddress.ip_address(value.strip("[]"))
20 if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped:
21 parsed = parsed.ipv4_mapped
22 return str(parsed)
23
24
25def sockets(output, targets):
26 rows = []
27 for line in output.splitlines():
28 parts = line.split()
29 if len(parts) < 11 or parts[0] not in {"tcp4", "tcp6"}:
30 continue
31 peer, _, port = parts[4].rpartition(".")
32 if port != "443" or address(peer) not in targets:
33 continue
34 pid = next((int(match[1]) for part in parts[10:] if (match := re.fullmatch(r".*:([0-9]+)", part))), None)
35 rows.append({"local": parts[3], "peer": parts[4], "state": parts[5], "pid": pid,
36 **{key: int(parts[index]) for key, index in
37 (("recv_q", 1), ("send_q", 2), ("bytes_received", 6), ("bytes_sent", 7), ("recv_hiwat", 8), ("send_hiwat", 9))}})
38 return rows
39
40
41def counters(output, targets):
42 reader = csv.reader(output.splitlines())
43 header = next(reader, [])
44 rows = []
45 pid = None
46 for values in reader:
47 if not values:
48 continue
49 if not values[0].startswith(("tcp4 ", "tcp6 ")):
50 last = values[0].rpartition(".")[2]
51 pid = int(last) if last.isdigit() else None
52 continue
53 peer = values[0].partition("<->")[2]
54 host, _, port = peer.rpartition(":")
55 if port != "443" or address(host) not in targets:
56 continue
57 row = {"connection": values[0], "pid": pid}
58 for key, value in zip(header[1:], values[1:]):
59 if key in COLUMNS.split(",") and re.fullmatch(r"[0-9. ]+(?:ms)?", value):
60 row[key] = value
61 rows.append(row)
62 return rows
63
64
65def main():
66 os.umask(0o077)
67 ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
68 targets = set()
69 resolved = measured = 0
70 while True:
71 now = time.time()
72 record = {"ts": now}
73 try:
74 if now - resolved > 300:
75 targets = {address(item[4][0]) for item in socket.getaddrinfo("substation.ai.paperclover.net", 443, type=socket.SOCK_STREAM)}
76 resolved = now
77 result = subprocess.run(["netstat", "-anv", "-p", "tcp"], check=True, capture_output=True, text=True, timeout=5)
78 rows = sockets(result.stdout, targets)
79 record["connections"] = rows
80 if now - measured >= 5:
81 pids = sorted({row["pid"] for row in rows if row["pid"]})
82 if pids:
83 result = subprocess.run(["ps", "-p", ",".join(map(str, pids)), "-o", "pid=,pcpu=,time=,state="],
84 capture_output=True, text=True, timeout=5)
85 record["processes"] = [{"pid": int(parts[0]), "cpu_percent": float(parts[1]), "cpu_time": parts[2], "state": parts[3]}
86 for line in result.stdout.splitlines() if len(parts := line.split()) == 4]
87 result = subprocess.run(["nettop", "-L", "1", "-n", "-x", "-m", "tcp", "-J", COLUMNS,
88 *(part for pid in pids for part in ("-p", str(pid)))],
89 check=True, capture_output=True, text=True, timeout=10)
90 record["tcp"] = counters(result.stdout, targets)
91 measured = now
92 except Exception as error:
93 record["error"] = type(error).__name__
94 stamp = datetime.fromtimestamp(now, timezone.utc).strftime("%Y%m%dT%H%M")
95 with (ROOT / (stamp + ".jsonl")).open("a") as file:
96 file.write(json.dumps(record, separators=(",", ":")) + "\n")
97 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
98 path.unlink()
99 time.sleep(1)
100
101
102if __name__ == "__main__":
103 main()
tools/substation.md+11-2
......@@ -111,8 +111,9 @@ request ID and rejects redirects, so an OAuth login page cannot pass as a health
111111API. It sends no generation requests and logs only check outcomes and IDs.
112112
113113Inspect `substation` task `monitor` in the service logs, and search Caddy's
114journal for the response's request ID. A 499 with no upstream status marks a
115cancellation before upstream headers; an upstream 499 is recorded separately.
114journal for the response's request ID. A 499 with no upstream status can mean
115client cancellation or Caddy's request-body idle timeout; an upstream 499 is
116recorded separately.
116117Compare `bytes_read` with `content_length` to identify incomplete fixed-length
117118uploads. A missing length can mean a chunked request; it does not prove completion.
118119A 200 followed by a truncated stream needs the edge byte count and the hub's
......@@ -131,3 +132,11 @@ Watcher heartbeat and incident summaries appear in its journal and Victoria
131132Logs. `upload_complete: false` proves an incomplete fixed-length upload;
132133the saved receive/send queues, byte counters, receive windows, and retransmits
133134distinguish stalled sending from server backpressure and network loss.
135
136Worktop's `net.paperclover.substation-watch` LaunchAgent records matching
137Substation sockets under `~/.local/state/substation-watch`. Minute files retain
13812 hours of process IDs, CPU time/state, TCP queues, byte counters,
139retransmissions and windows.
140The installed script is an exported revision of `tools/substation-client-watch.py`;
141the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.
142Match UTC timestamps to the server incident before comparing TCP progress.