From 9cd6ce8c7d9ad4395f6deb4ef710c060029c077f Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 11 Nov 2026 15:26:19 -0800 Subject: [PATCH] Retain Substation transport incidents with TCP progress A recurring 60-second failure received only 965720 of 2814027 upload bytes. Keep a restart-safe journal watcher and two minutes of TCP counters so the next failure identifies stalled sending, network loss or server backpressure. Record the API client endpoint and retain each edge failure with upload completion and surrounding connection counters in restricted state. Persist the journal cursor, bound incident retention, recover cursor gaps visibly, and run the watcher under systemd across deployments and reboots. Validated metadata filtering, connection selection, upload accounting, atomic state permissions, actual isolated systemd restart and cursor resume, API routing and privacy, release activation tests, deployment export tests, and healthy Substation staging. Assisted-by: gpt-6.1-sol --- nixos/configuration.nix | 15 ++++ tools/release.py | 2 + tools/router-api-test.py | 2 + tools/router.py | 1 + tools/substation-watch-test.py | 65 +++++++++++++++ tools/substation-watch.py | 141 +++++++++++++++++++++++++++++++++ tools/substation.md | 13 +++ 7 files changed, 239 insertions(+) create mode 100644 tools/substation-watch-test.py create mode 100644 tools/substation-watch.py diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 0cf7981f6f2a844c1e940547191be2ba4d7b38fd..6c90198680f9e2604d10569a88d14cddd9672151 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -150,6 +150,21 @@ in requires = [ "podman.socket" ]; }; + systemd.services.snowglobe-substation-watch = { + wantedBy = [ "multi-user.target" ]; + wants = [ "nomad.service" "caddy.service" ]; + after = [ "nomad.service" "caddy.service" ]; + unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ]; + environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN; + path = [ pkgs.systemd pkgs.iproute2 ]; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py"; + Restart = "always"; + RestartSec = 3; + UMask = "0077"; + }; + }; + systemd.services.snowglobe-router = { wantedBy = [ "multi-user.target" ]; wants = [ "nomad.service" "caddy.service" ]; diff --git a/tools/release.py b/tools/release.py index b2cb104a5944a4a705d760778d1d138a08dfe71a..4d7ab541dd7f79bedd4317d2a1d15bc72aa0f965 100644 --- a/tools/release.py +++ b/tools/release.py @@ -230,6 +230,8 @@ def activate(release, initial=False): pending.replace(managed) if (path / "tools/log-shipper.py").exists(): subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True) + if (path / "tools/substation-watch.py").exists(): + subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True) manifest = path / ".snowglobe-release.json" if json.loads(manifest.read_text()).get("version", 1) >= 2: subprocess.run(["python3", str(script), "check"], check=True, env=environment) diff --git a/tools/router-api-test.py b/tools/router-api-test.py index 0920e6804e0c058867dd1fb9bfe66916a4beb76b..6a4c24dfcdfc42b8e10cb92c416fbacab892683e 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path +import re import socket import subprocess import tempfile @@ -206,6 +207,7 @@ def main(): time.sleep(.05) assert all("request" not in entry and "resp_headers" not in entry for entry in entries) assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) + assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) diff --git a/tools/router.py b/tools/router.py index 958afc20c75bf7be06a3d9d6dd13e740ab0d49d9..5e28dd9564aaf5c442fd376e3e7596d389b91264 100644 --- a/tools/router.py +++ b/tools/router.py @@ -370,6 +370,7 @@ def render(token): f" log_append service {service}", " log_append request_id {http.request.uuid}", " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", " log_append content_length {http.request.header.Content-Length}", + " log_append client {remote}", " log_append upstream {http.reverse_proxy.upstream.hostport}", " log_append upstream_status {http.reverse_proxy.status_code}", " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", diff --git a/tools/substation-watch-test.py b/tools/substation-watch-test.py new file mode 100644 index 0000000000000000000000000000000000000000..283096efbdeae2d29ebb848a585dc0992bbe091b --- /dev/null +++ b/tools/substation-watch-test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-watch.py")) +watch = importlib.util.module_from_spec(spec) +spec.loader.exec_module(watch) + + +class TransportEvidence(unittest.TestCase): + def test_filters_private_headers_bodies_and_other_services(self): + original = {"service": "substation", "status": 504, "request_id": "fixture", "content_length": "100", "bytes_read": 40, + "client": "192.0.2.1:1234", "request": {"headers": {"X-Api-Key": "private"}}, "body": "private"} + edge, client = watch.event({"MESSAGE": json.dumps(original)}, "substation.example") + self.assertEqual(client, "192.0.2.1:1234") + self.assertNotIn("private", json.dumps(edge)) + self.assertEqual(watch.event({"MESSAGE": json.dumps({"service": "another"})}, "substation.example"), (None, None)) + self.assertEqual(watch.event({"MESSAGE": "not json"}, "substation.example"), (None, None)) + peer = {"request": {"host": "substation.example", "remote_ip": "2001:db8::1", "remote_port": "44300", "headers": {"Authorization": "private"}}} + self.assertEqual(watch.event({"MESSAGE": json.dumps(peer)}, "substation.example"), (None, "[2001:db8::1]:44300")) + + def test_connection_selection_and_counters(self): + output = "\n".join([ + "0 100 127.0.0.1:24000 127.0.0.1:45000 cubic bytes_received:40000 rtt:0.1/0.2 snd_wnd:0 ignored:private", + "0 0 192.0.2.5:443 192.0.2.1:1234 cubic bytes_received:41000 lastrcv:60000", + "0 0 192.0.2.5:443 192.0.2.2:1234 cubic bytes_received:99999", + "0 0 [::ffff:192.0.2.5]:443 [::ffff:192.0.2.1]:5678 cubic bytes_received:42000", + ]) + rows = watch.connections(output, {24000}, {"192.0.2.1"}) + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0]["recv_q"], 0) + self.assertEqual(rows[0]["send_q"], 100) + self.assertEqual(rows[0]["snd_wnd"], "0") + self.assertEqual(rows[1]["lastrcv"], "60000") + self.assertNotIn("private", json.dumps(rows)) + self.assertEqual(watch.endpoint("[2001:db8::1]:1234"), ("2001:db8::1", 1234)) + self.assertEqual(watch.endpoint("[::ffff:192.0.2.1]:1234"), ("192.0.2.1", 1234)) + + def test_completion_and_window(self): + edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40} + samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}] + record = watch.incident(edge, samples) + self.assertFalse(record["upload_complete"]) + self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}]) + edge["bytes_read"] = 100 + self.assertTrue(watch.incident(edge, samples)["upload_complete"]) + edge["content_length"] = "" + self.assertIsNone(watch.incident(edge, samples)["upload_complete"]) + + def test_private_atomic_state_survives_reload(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "state.json" + watch.save(path, {"cursor": "first"}) + watch.save(path, {"cursor": "second"}) + self.assertEqual(json.loads(path.read_text()), {"cursor": "second"}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertFalse(path.with_suffix(".pending").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/substation-watch.py b/tools/substation-watch.py new file mode 100644 index 0000000000000000000000000000000000000000..148fa8ddc158f539534f8fa8eb99097e02d4c213 --- /dev/null +++ b/tools/substation-watch.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +from collections import deque +import ipaddress +import json +import os +from pathlib import Path +import re +import subprocess +import time +from urllib.request import Request, urlopen + + +ROOT = Path("/var/lib/snowglobe/substation-watch") +EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", + "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client") +TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack", + "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"} + + +def save(path, value): + pending = path.with_suffix(".pending") + with pending.open("w") as file: + json.dump(value, file, separators=(",", ":")) + pending.chmod(0o600) + pending.replace(path) + + +def endpoint(value): + address, _, port = value.rpartition(":") + address = ipaddress.ip_address(address.strip("[]")) + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped + return str(address), int(port) + + +def connections(output, ports, clients): + rows = [] + for line in output.splitlines(): + parts = line.split() + if len(parts) < 4: + continue + local, peer = endpoint(parts[2]), endpoint(parts[3]) + if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients): + continue + row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])} + for part in parts[4:]: + key, _, value = part.partition(":") + if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value): + row[key] = value + rows.append(row) + return rows + + +def event(entry, host): + try: + message = json.loads(entry.get("MESSAGE", "")) + except (ValueError, TypeError): + return None, None + if not isinstance(message, dict): + return None, None + if message.get("service") == "substation": + return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client") + request = message.get("request", {}) + if request.get("host") == host and request.get("remote_ip"): + address = str(ipaddress.ip_address(request["remote_ip"])) + return None, f"[{address}]:{int(request['remote_port'])}" + return None, None + + +def incident(edge, samples): + length = edge.get("content_length") + complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None + return {"edge": edge, "upload_complete": complete, + "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]} + + +def main(): + os.umask(0o077) + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + state_file = ROOT / "state.json" + state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} + recent = ROOT / "recent.json" + samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) + ports = set() + refreshed = saved = reported = 0 + host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] + while True: + try: + now = time.time() + if now - refreshed >= 10: + token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() + request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) + with urlopen(request, timeout=5) as response: + ports = {item["Port"] for item in json.load(response) + if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback} + refreshed = now + command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"] + command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] + journal = subprocess.run(command, capture_output=True, text=True, timeout=10) + if journal.returncode: + if "cursor" in journal.stderr.lower() and state.pop("cursor", None): + print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True) + journal.check_returncode() + for line in journal.stdout.splitlines(): + entry = json.loads(line) + edge, client = event(entry, host) + if client: + state["clients"][endpoint(client)[0]] = now + if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): + path = ROOT / (edge["request_id"] + ".json") + if not path.exists(): + record = incident(edge, samples) + save(path, record) + print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], + "status": edge["status"], "upload_complete": record["upload_complete"], + "samples": len(record["tcp"])}), flush=True) + state["cursor"] = entry["__CURSOR"] + state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} + if ports: + filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )" + result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5) + rows = connections(result.stdout, ports, state["clients"]) + samples.append({"ts": now, "connections": rows}) + save(state_file, state) + if now - saved >= 5: + save(recent, list(samples)) + saved = now + old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime) + for path in old[:-100]: + path.unlink() + if now - reported >= 60: + print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), + "clients": len(state["clients"]), "samples": len(samples)}), flush=True) + reported = now + except Exception as error: + print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/tools/substation.md b/tools/substation.md index 701dd0df289bae4ff05b2360d6eaaaf24c750083..7fbd7a8141983fd8b23b585ee7511c919d793280 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi A 200 followed by a truncated stream needs the edge byte count and the hub's stream outcome. A successful probe does not establish that every generation stream completed. + +`snowglobe-substation-watch.service` follows Caddy with a persisted journal +cursor and samples TCP counters once per second for the hub and known API +clients. It survives service restarts and reboots. The latest two minutes are +retained in `/var/lib/snowglobe/substation-watch/recent.json`; each 499 or 5xx +retains its request metadata and surrounding TCP progress in a private file +named by request ID. The directory keeps the latest 100 incidents. No packet +payloads, request bodies, cookies, or credentials are collected. + +Watcher heartbeat and incident summaries appear in its journal and Victoria +Logs. `upload_complete: false` proves an incomplete fixed-length upload; +the saved receive/send queues, byte counters, receive windows, and retransmits +distinguish stalled sending from server backpressure and network loss. -- 2.54.1