| author | |
| committer | |
| log | 9cd6ce8c7d9ad4395f6deb4ef710c060029c077f |
| tree | 1122679175f96e3ddd5ac542b9413b7ce17c0692 |
| parent | 53fd9c969ec080da95877749caebd841358fe8c7 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A recurring 60-second failure received only 965720 of 2814027 upload bytes.
Keep a restart-safe journal watcher and two minutes of TCP counters so the
next failure identifies stalled sending, network loss or server backpressure.
Record the API client endpoint and retain each edge failure with upload
completion and surrounding connection counters in restricted state. Persist
the journal cursor, bound incident retention, recover cursor gaps visibly,
and run the watcher under systemd across deployments and reboots.
Validated metadata filtering, connection selection, upload accounting,
atomic state permissions, actual isolated systemd restart and cursor resume,
API routing and privacy, release activation tests, deployment export tests,
and healthy Substation staging.
Assisted-by: gpt-6.1-sol7 files changed, 239 insertions(+), 0 deletions(-)
nixos/configuration.nix+15| ... | ... | @@ -150,6 +150,21 @@ in |
| 150 | 150 | requires = [ "podman.socket" ]; |
| 151 | 151 | }; |
| 152 | 152 | |
| 153 | systemd.services.snowglobe-substation-watch = { | |
| 154 | wantedBy = [ "multi-user.target" ]; | |
| 155 | wants = [ "nomad.service" "caddy.service" ]; | |
| 156 | after = [ "nomad.service" "caddy.service" ]; | |
| 157 | unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ]; | |
| 158 | environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN; | |
| 159 | path = [ pkgs.systemd pkgs.iproute2 ]; | |
| 160 | serviceConfig = { | |
| 161 | ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py"; | |
| 162 | Restart = "always"; | |
| 163 | RestartSec = 3; | |
| 164 | UMask = "0077"; | |
| 165 | }; | |
| 166 | }; | |
| 167 | ||
| 153 | 168 | systemd.services.snowglobe-router = { |
| 154 | 169 | wantedBy = [ "multi-user.target" ]; |
| 155 | 170 | wants = [ "nomad.service" "caddy.service" ]; |
tools/release.py+2| ... | ... | @@ -230,6 +230,8 @@ def activate(release, initial=False): |
| 230 | 230 | pending.replace(managed) |
| 231 | 231 | if (path / "tools/log-shipper.py").exists(): |
| 232 | 232 | subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True) |
| 233 | if (path / "tools/substation-watch.py").exists(): | |
| 234 | subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True) | |
| 233 | 235 | manifest = path / ".snowglobe-release.json" |
| 234 | 236 | if json.loads(manifest.read_text()).get("version", 1) >= 2: |
| 235 | 237 | subprocess.run(["python3", str(script), "check"], check=True, env=environment) |
tools/router-api-test.py+2| ... | ... | @@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 5 | 5 | import json |
| 6 | 6 | import os |
| 7 | 7 | from pathlib import Path |
| 8 | import re | |
| 8 | 9 | import socket |
| 9 | 10 | import subprocess |
| 10 | 11 | import tempfile |
| ... | ... | @@ -206,6 +207,7 @@ def main(): |
| 206 | 207 | time.sleep(.05) |
| 207 | 208 | assert all("request" not in entry and "resp_headers" not in entry for entry in entries) |
| 208 | 209 | assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) |
| 210 | assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) | |
| 209 | 211 | assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) |
| 210 | 212 | assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) |
| 211 | 213 | assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) |
tools/router.py+1| ... | ... | @@ -370,6 +370,7 @@ def render(token): |
| 370 | 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", |
| 371 | 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", |
| 372 | 372 | " log_append content_length {http.request.header.Content-Length}", |
| 373 | " log_append client {remote}", | |
| 373 | 374 | " log_append upstream {http.reverse_proxy.upstream.hostport}", |
| 374 | 375 | " log_append upstream_status {http.reverse_proxy.status_code}", |
| 375 | 376 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", |
tools/substation-watch-test.py created+65| ... | ... | @@ -0,0 +1,65 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import importlib.util | |
| 3 | import json | |
| 4 | from pathlib import Path | |
| 5 | import tempfile | |
| 6 | import unittest | |
| 7 | ||
| 8 | ||
| 9 | spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-watch.py")) | |
| 10 | watch = importlib.util.module_from_spec(spec) | |
| 11 | spec.loader.exec_module(watch) | |
| 12 | ||
| 13 | ||
| 14 | class TransportEvidence(unittest.TestCase): | |
| 15 | def test_filters_private_headers_bodies_and_other_services(self): | |
| 16 | original = {"service": "substation", "status": 504, "request_id": "fixture", "content_length": "100", "bytes_read": 40, | |
| 17 | "client": "192.0.2.1:1234", "request": {"headers": {"X-Api-Key": "private"}}, "body": "private"} | |
| 18 | edge, client = watch.event({"MESSAGE": json.dumps(original)}, "substation.example") | |
| 19 | self.assertEqual(client, "192.0.2.1:1234") | |
| 20 | self.assertNotIn("private", json.dumps(edge)) | |
| 21 | self.assertEqual(watch.event({"MESSAGE": json.dumps({"service": "another"})}, "substation.example"), (None, None)) | |
| 22 | self.assertEqual(watch.event({"MESSAGE": "not json"}, "substation.example"), (None, None)) | |
| 23 | peer = {"request": {"host": "substation.example", "remote_ip": "2001:db8::1", "remote_port": "44300", "headers": {"Authorization": "private"}}} | |
| 24 | self.assertEqual(watch.event({"MESSAGE": json.dumps(peer)}, "substation.example"), (None, "[2001:db8::1]:44300")) | |
| 25 | ||
| 26 | def test_connection_selection_and_counters(self): | |
| 27 | output = "\n".join([ | |
| 28 | "0 100 127.0.0.1:24000 127.0.0.1:45000 cubic bytes_received:40000 rtt:0.1/0.2 snd_wnd:0 ignored:private", | |
| 29 | "0 0 192.0.2.5:443 192.0.2.1:1234 cubic bytes_received:41000 lastrcv:60000", | |
| 30 | "0 0 192.0.2.5:443 192.0.2.2:1234 cubic bytes_received:99999", | |
| 31 | "0 0 [::ffff:192.0.2.5]:443 [::ffff:192.0.2.1]:5678 cubic bytes_received:42000", | |
| 32 | ]) | |
| 33 | rows = watch.connections(output, {24000}, {"192.0.2.1"}) | |
| 34 | self.assertEqual(len(rows), 3) | |
| 35 | self.assertEqual(rows[0]["recv_q"], 0) | |
| 36 | self.assertEqual(rows[0]["send_q"], 100) | |
| 37 | self.assertEqual(rows[0]["snd_wnd"], "0") | |
| 38 | self.assertEqual(rows[1]["lastrcv"], "60000") | |
| 39 | self.assertNotIn("private", json.dumps(rows)) | |
| 40 | self.assertEqual(watch.endpoint("[2001:db8::1]:1234"), ("2001:db8::1", 1234)) | |
| 41 | self.assertEqual(watch.endpoint("[::ffff:192.0.2.1]:1234"), ("192.0.2.1", 1234)) | |
| 42 | ||
| 43 | def test_completion_and_window(self): | |
| 44 | edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40} | |
| 45 | samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}] | |
| 46 | record = watch.incident(edge, samples) | |
| 47 | self.assertFalse(record["upload_complete"]) | |
| 48 | self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}]) | |
| 49 | edge["bytes_read"] = 100 | |
| 50 | self.assertTrue(watch.incident(edge, samples)["upload_complete"]) | |
| 51 | edge["content_length"] = "" | |
| 52 | self.assertIsNone(watch.incident(edge, samples)["upload_complete"]) | |
| 53 | ||
| 54 | def test_private_atomic_state_survives_reload(self): | |
| 55 | with tempfile.TemporaryDirectory() as temporary: | |
| 56 | path = Path(temporary) / "state.json" | |
| 57 | watch.save(path, {"cursor": "first"}) | |
| 58 | watch.save(path, {"cursor": "second"}) | |
| 59 | self.assertEqual(json.loads(path.read_text()), {"cursor": "second"}) | |
| 60 | self.assertEqual(path.stat().st_mode & 0o777, 0o600) | |
| 61 | self.assertFalse(path.with_suffix(".pending").exists()) | |
| 62 | ||
| 63 | ||
| 64 | if __name__ == "__main__": | |
| 65 | unittest.main() |
tools/substation-watch.py created+141| ... | ... | @@ -0,0 +1,141 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | from collections import deque | |
| 3 | import ipaddress | |
| 4 | import json | |
| 5 | import os | |
| 6 | from pathlib import Path | |
| 7 | import re | |
| 8 | import subprocess | |
| 9 | import time | |
| 10 | from urllib.request import Request, urlopen | |
| 11 | ||
| 12 | ||
| 13 | ROOT = Path("/var/lib/snowglobe/substation-watch") | |
| 14 | EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", | |
| 15 | "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client") | |
| 16 | TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack", | |
| 17 | "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"} | |
| 18 | ||
| 19 | ||
| 20 | def save(path, value): | |
| 21 | pending = path.with_suffix(".pending") | |
| 22 | with pending.open("w") as file: | |
| 23 | json.dump(value, file, separators=(",", ":")) | |
| 24 | pending.chmod(0o600) | |
| 25 | pending.replace(path) | |
| 26 | ||
| 27 | ||
| 28 | def endpoint(value): | |
| 29 | address, _, port = value.rpartition(":") | |
| 30 | address = ipaddress.ip_address(address.strip("[]")) | |
| 31 | if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: | |
| 32 | address = address.ipv4_mapped | |
| 33 | return str(address), int(port) | |
| 34 | ||
| 35 | ||
| 36 | def connections(output, ports, clients): | |
| 37 | rows = [] | |
| 38 | for line in output.splitlines(): | |
| 39 | parts = line.split() | |
| 40 | if len(parts) < 4: | |
| 41 | continue | |
| 42 | local, peer = endpoint(parts[2]), endpoint(parts[3]) | |
| 43 | if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients): | |
| 44 | continue | |
| 45 | row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])} | |
| 46 | for part in parts[4:]: | |
| 47 | key, _, value = part.partition(":") | |
| 48 | if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value): | |
| 49 | row[key] = value | |
| 50 | rows.append(row) | |
| 51 | return rows | |
| 52 | ||
| 53 | ||
| 54 | def event(entry, host): | |
| 55 | try: | |
| 56 | message = json.loads(entry.get("MESSAGE", "")) | |
| 57 | except (ValueError, TypeError): | |
| 58 | return None, None | |
| 59 | if not isinstance(message, dict): | |
| 60 | return None, None | |
| 61 | if message.get("service") == "substation": | |
| 62 | return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client") | |
| 63 | request = message.get("request", {}) | |
| 64 | if request.get("host") == host and request.get("remote_ip"): | |
| 65 | address = str(ipaddress.ip_address(request["remote_ip"])) | |
| 66 | return None, f"[{address}]:{int(request['remote_port'])}" | |
| 67 | return None, None | |
| 68 | ||
| 69 | ||
| 70 | def incident(edge, samples): | |
| 71 | length = edge.get("content_length") | |
| 72 | complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None | |
| 73 | return {"edge": edge, "upload_complete": complete, | |
| 74 | "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]} | |
| 75 | ||
| 76 | ||
| 77 | def main(): | |
| 78 | os.umask(0o077) | |
| 79 | ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) | |
| 80 | state_file = ROOT / "state.json" | |
| 81 | state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} | |
| 82 | recent = ROOT / "recent.json" | |
| 83 | samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) | |
| 84 | ports = set() | |
| 85 | refreshed = saved = reported = 0 | |
| 86 | host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] | |
| 87 | while True: | |
| 88 | try: | |
| 89 | now = time.time() | |
| 90 | if now - refreshed >= 10: | |
| 91 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() | |
| 92 | request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) | |
| 93 | with urlopen(request, timeout=5) as response: | |
| 94 | ports = {item["Port"] for item in json.load(response) | |
| 95 | if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback} | |
| 96 | refreshed = now | |
| 97 | command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"] | |
| 98 | command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] | |
| 99 | journal = subprocess.run(command, capture_output=True, text=True, timeout=10) | |
| 100 | if journal.returncode: | |
| 101 | if "cursor" in journal.stderr.lower() and state.pop("cursor", None): | |
| 102 | print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True) | |
| 103 | journal.check_returncode() | |
| 104 | for line in journal.stdout.splitlines(): | |
| 105 | entry = json.loads(line) | |
| 106 | edge, client = event(entry, host) | |
| 107 | if client: | |
| 108 | state["clients"][endpoint(client)[0]] = now | |
| 109 | if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): | |
| 110 | path = ROOT / (edge["request_id"] + ".json") | |
| 111 | if not path.exists(): | |
| 112 | record = incident(edge, samples) | |
| 113 | save(path, record) | |
| 114 | print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], | |
| 115 | "status": edge["status"], "upload_complete": record["upload_complete"], | |
| 116 | "samples": len(record["tcp"])}), flush=True) | |
| 117 | state["cursor"] = entry["__CURSOR"] | |
| 118 | state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} | |
| 119 | if ports: | |
| 120 | filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )" | |
| 121 | result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5) | |
| 122 | rows = connections(result.stdout, ports, state["clients"]) | |
| 123 | samples.append({"ts": now, "connections": rows}) | |
| 124 | save(state_file, state) | |
| 125 | if now - saved >= 5: | |
| 126 | save(recent, list(samples)) | |
| 127 | saved = now | |
| 128 | old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime) | |
| 129 | for path in old[:-100]: | |
| 130 | path.unlink() | |
| 131 | if now - reported >= 60: | |
| 132 | print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), | |
| 133 | "clients": len(state["clients"]), "samples": len(samples)}), flush=True) | |
| 134 | reported = now | |
| 135 | except Exception as error: | |
| 136 | print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) | |
| 137 | time.sleep(1) | |
| 138 | ||
| 139 | ||
| 140 | if __name__ == "__main__": | |
| 141 | main() |
tools/substation.md+13| ... | ... | @@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi |
| 118 | 118 | A 200 followed by a truncated stream needs the edge byte count and the hub's |
| 119 | 119 | stream outcome. A successful probe does not establish that every generation |
| 120 | 120 | stream completed. |
| 121 | ||
| 122 | `snowglobe-substation-watch.service` follows Caddy with a persisted journal | |
| 123 | cursor and samples TCP counters once per second for the hub and known API | |
| 124 | clients. It survives service restarts and reboots. The latest two minutes are | |
| 125 | retained in `/var/lib/snowglobe/substation-watch/recent.json`; each 499 or 5xx | |
| 126 | retains its request metadata and surrounding TCP progress in a private file | |
| 127 | named by request ID. The directory keeps the latest 100 incidents. No packet | |
| 128 | payloads, request bodies, cookies, or credentials are collected. | |
| 129 | ||
| 130 | Watcher heartbeat and incident summaries appear in its journal and Victoria | |
| 131 | Logs. `upload_complete: false` proves an incomplete fixed-length upload; | |
| 132 | the saved receive/send queues, byte counters, receive windows, and retransmits | |
| 133 | distinguish stalled sending from server backpressure and network loss. |