authoravatarclover caruso<git@paperclover.net>2026-11-12 11:00:46-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
log9f7ec5c28c4d001c4d36e7c68550acb86e533950
treec3791dc164947efd2c354c648f2bdd12a9ba7deb
parent029fe21eabd885646110ce6c0991a7dc1be99bfe
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Retain complete Worktop upload stall episodes

Give each stalled socket one archive, then update it after stall detection ends and packet ingestion catches up. Retain late retransmissions and connection closure instead of only the first ten seconds. Skip expired packet records before parsing so restart catch-up does not spend CPU on discarded history. Thirteen watcher tests passed locally and eight client tests passed on Worktop. A disposable staging connection with scoped packet loss updated the same archive through its reset, retaining 2,473 matching packets with zero backlog at finalization. The firewall fixture was removed and private evidence retained. Staging health passed. Worktop runs the exported client revision. Assisted-by: gpt-6.1-sol

3 files changed, 39 insertions(+), 9 deletions(-)

tools/substation-client-watch-test.py+13
...@@ -13,6 +13,19 @@ spec.loader.exec_module(watch)...@@ -13,6 +13,19 @@ spec.loader.exec_module(watch)
1313
1414
15class ClientEvidence(unittest.TestCase):15class ClientEvidence(unittest.TestCase):
16 def setUp(self):
17 clock = patch.object(watch.time, "time", return_value=200)
18 clock.start()
19 self.addCleanup(clock.stop)
20
21 def test_capture_skips_expired_metadata_and_still_advances(self):
22 with tempfile.TemporaryDirectory() as temporary:
23 path = Path(temporary) / "capture.log"
24 path.write_text("invalid line\n1.0 IP 192.0.2.5.443 > 192.0.2.1.50000: Flags [.], ack 20, win 40, length 0\n")
25 rows, offset, _ = watch.capture(path, 0)
26 self.assertEqual(rows, [])
27 self.assertEqual(offset, path.stat().st_size)
28
16 def test_capture_keeps_partial_lines_and_excludes_payload(self):29 def test_capture_keeps_partial_lines_and_excludes_payload(self):
17 with tempfile.TemporaryDirectory() as temporary:30 with tempfile.TemporaryDirectory() as temporary:
18 path = Path(temporary) / "capture.log"31 path = Path(temporary) / "capture.log"
tools/substation-client-watch.py+21-7
...@@ -102,7 +102,13 @@ def capture(path, offset):...@@ -102,7 +102,13 @@ def capture(path, offset):
102 file.truncate()102 file.truncate()
103 offset = 0103 offset = 0
104 truncated_at = time.time()104 truncated_at = time.time()
105 cutoff = time.time() - 120
105 for line in data[:end].splitlines():106 for line in data[:end].splitlines():
107 try:
108 if float(line.partition(b" ")[0]) < cutoff:
109 continue
110 except ValueError:
111 continue
106 if row := packet(line.decode("ascii", errors="replace")):112 if row := packet(line.decode("ascii", errors="replace")):
107 rows.append(row)113 rows.append(row)
108 return rows, offset, truncated_at114 return rows, offset, truncated_at
...@@ -114,7 +120,7 @@ def main():...@@ -114,7 +120,7 @@ def main():
114 targets = set()120 targets = set()
115 samples = deque(maxlen=120)121 samples = deque(maxlen=120)
116 packets = deque(maxlen=100000)122 packets = deque(maxlen=100000)
117 stalled = set()123 stalled = {}
118 capture_state = {"offset": 0, "busy": False}124 capture_state = {"offset": 0, "busy": False}
119 capture_gaps = deque(maxlen=100)125 capture_gaps = deque(maxlen=100)
120 capture_path = ROOT / "tcp-headers.log"126 capture_path = ROOT / "tcp-headers.log"
...@@ -173,17 +179,25 @@ def main():...@@ -173,17 +179,25 @@ def main():
173 file.write(json.dumps(record, separators=(",", ":")) + "\n")179 file.write(json.dumps(record, separators=(",", ":")) + "\n")
174 samples.append(record)180 samples.append(record)
175 current = stalls(samples)181 current = stalls(samples)
176 if added := current - stalled:182 for key in current | stalled.keys():
177 path = ROOT / ("stall-" + str(int(now * 1000)) + ".json")183 if key in current:
184 if key in stalled:
185 continue
186 path = ROOT / f"stall-{int(now * 1000)}-{key[2]}-{key[0].rpartition('.')[2]}.json"
187 stalled[key] = path
188 else:
189 if any(record.get(field) for field in ("error", "capture_error", "capture_busy", "capture_backlog")):
190 continue
191 path = stalled.pop(key)
178 pending = path.with_suffix(".pending")192 pending = path.with_suffix(".pending")
179 endpoints = {f"[{local.rpartition('.')[0]}]:{local.rpartition('.')[2]}" for local, _, _ in added}193 local, _, port = key[0].rpartition(".")
180 pending.write_text(json.dumps({"ts": now, "connections": sorted(added), "samples": list(samples),194 endpoint = f"[{local}]:{port}"
195 pending.write_text(json.dumps({"ts": now, "connections": [key], "samples": list(samples),
181 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],196 "capture_gaps": [ts for ts in list(capture_gaps) if ts >= now - 120],
182 "packets": [row for row in list(packets) if endpoints.intersection((row["source"], row["destination"]))]}, separators=(",", ":")))197 "packets": [row for row in list(packets) if endpoint in (row["source"], row["destination"])]}, separators=(",", ":")))
183 pending.replace(path)198 pending.replace(path)
184 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:199 for old in sorted(ROOT.glob("stall-*.json"))[:-100]:
185 old.unlink()200 old.unlink()
186 stalled = current
187 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:201 for path in sorted(ROOT.glob("????????T????.jsonl"))[:-720]:
188 path.unlink()202 path.unlink()
189 time.sleep(1)203 time.sleep(1)
tools/substation.md+5-2
...@@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p...@@ -146,10 +146,13 @@ The installed script is an exported revision of `tools/substation-client-watch.p
146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.146the LaunchAgent follows its `~/.local/share/substation-watch/current.py` link.
147Match UTC timestamps to the server incident before comparing TCP progress.147Match UTC timestamps to the server incident before comparing TCP progress.
148An upload with queued bytes and no acknowledgement progress for 10 seconds148An upload with queued bytes and no acknowledgement progress for 10 seconds
149retains the preceding two minutes in a private `stall-<timestamp>.json` file.149retains the preceding 120 socket samples in a private `stall-<timestamp>.json` file.
150The latest 100 stalls survive rolling-history expiry and watcher restarts.150The latest 100 stalls survive rolling-history expiry and watcher restarts.
151Each stalled socket has one archive, updated after the stall and after packet
152ingestion catches up, retaining later retransmissions and closure.
151When an administrator starts a TCP summary capture into `tcp-headers.log`, the153When an administrator starts a TCP summary capture into `tcp-headers.log`, the
152client watcher retains matching sequence/ACK/SACK metadata with each stall.154client watcher retains up to two minutes of matching sequence/ACK/SACK metadata
155with each stall, bounded to 100,000 packets.
153It consumes and truncates that append-only log at 64 MiB without restarting156It consumes and truncates that append-only log at 64 MiB without restarting
154tcpdump. Archives mark each truncation; exclude that instant from packet-loss157tcpdump. Archives mark each truncation; exclude that instant from packet-loss
155conclusions because concurrent writes can be lost during truncation.158conclusions because concurrent writes can be lost during truncation.