diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 0cf7981f6f2a844c1e940547191be2ba4d7b38fd..6c90198680f9e2604d10569a88d14cddd9672151 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -150,6 +150,21 @@ in requires = [ "podman.socket" ]; }; + systemd.services.snowglobe-substation-watch = { + wantedBy = [ "multi-user.target" ]; + wants = [ "nomad.service" "caddy.service" ]; + after = [ "nomad.service" "caddy.service" ]; + unitConfig.ConditionPathExists = [ "/var/lib/snowglobe/nomad.token" "/opt/snowglobe/current/tools/substation-watch.py" ]; + environment.SNOWGLOBE_DOMAIN = config.environment.variables.SNOWGLOBE_DOMAIN; + path = [ pkgs.systemd pkgs.iproute2 ]; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 /opt/snowglobe/current/tools/substation-watch.py"; + Restart = "always"; + RestartSec = 3; + UMask = "0077"; + }; + }; + systemd.services.snowglobe-router = { wantedBy = [ "multi-user.target" ]; wants = [ "nomad.service" "caddy.service" ]; diff --git a/tools/release.py b/tools/release.py index b2cb104a5944a4a705d760778d1d138a08dfe71a..4d7ab541dd7f79bedd4317d2a1d15bc72aa0f965 100644 --- a/tools/release.py +++ b/tools/release.py @@ -230,6 +230,8 @@ def activate(release, initial=False): pending.replace(managed) if (path / "tools/log-shipper.py").exists(): subprocess.run(["systemctl", "restart", "snowglobe-log-shipper.service"], check=True) + if (path / "tools/substation-watch.py").exists(): + subprocess.run(["systemctl", "restart", "snowglobe-substation-watch.service"], check=True) manifest = path / ".snowglobe-release.json" if json.loads(manifest.read_text()).get("version", 1) >= 2: subprocess.run(["python3", str(script), "check"], check=True, env=environment) diff --git a/tools/router-api-test.py b/tools/router-api-test.py index 0920e6804e0c058867dd1fb9bfe66916a4beb76b..6a4c24dfcdfc42b8e10cb92c416fbacab892683e 100644 --- a/tools/router-api-test.py +++ b/tools/router-api-test.py @@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path +import re import socket import subprocess import tempfile @@ -206,6 +207,7 @@ def main(): time.sleep(.05) assert all("request" not in entry and "resp_headers" not in entry for entry in entries) assert all(entry.get("request_id") and entry.get("path", "").startswith(("/v1/", "/v0/management/")) for entry in entries) + assert all(re.fullmatch(r"127\.0\.0\.1:[0-9]+", entry["client"]) for entry in entries) assert any(entry.get("upstream_header_ms") for entry in entries if entry.get("status") == 200) assert all(entry.get("upstream_status") == 200 for entry in entries if entry.get("status") == 200) assert all(not entry.get("upstream_status") for entry in entries if entry.get("status") == 499) diff --git a/tools/router.py b/tools/router.py index 958afc20c75bf7be06a3d9d6dd13e740ab0d49d9..5e28dd9564aaf5c442fd376e3e7596d389b91264 100644 --- a/tools/router.py +++ b/tools/router.py @@ -370,6 +370,7 @@ def render(token): f" log_append service {service}", " log_append request_id {http.request.uuid}", " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", " log_append content_length {http.request.header.Content-Length}", + " log_append client {remote}", " log_append upstream {http.reverse_proxy.upstream.hostport}", " log_append upstream_status {http.reverse_proxy.status_code}", " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", diff --git a/tools/substation-watch-test.py b/tools/substation-watch-test.py new file mode 100644 index 0000000000000000000000000000000000000000..283096efbdeae2d29ebb848a585dc0992bbe091b --- /dev/null +++ b/tools/substation-watch-test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +spec = importlib.util.spec_from_file_location("watch", Path(__file__).with_name("substation-watch.py")) +watch = importlib.util.module_from_spec(spec) +spec.loader.exec_module(watch) + + +class TransportEvidence(unittest.TestCase): + def test_filters_private_headers_bodies_and_other_services(self): + original = {"service": "substation", "status": 504, "request_id": "fixture", "content_length": "100", "bytes_read": 40, + "client": "192.0.2.1:1234", "request": {"headers": {"X-Api-Key": "private"}}, "body": "private"} + edge, client = watch.event({"MESSAGE": json.dumps(original)}, "substation.example") + self.assertEqual(client, "192.0.2.1:1234") + self.assertNotIn("private", json.dumps(edge)) + self.assertEqual(watch.event({"MESSAGE": json.dumps({"service": "another"})}, "substation.example"), (None, None)) + self.assertEqual(watch.event({"MESSAGE": "not json"}, "substation.example"), (None, None)) + peer = {"request": {"host": "substation.example", "remote_ip": "2001:db8::1", "remote_port": "44300", "headers": {"Authorization": "private"}}} + self.assertEqual(watch.event({"MESSAGE": json.dumps(peer)}, "substation.example"), (None, "[2001:db8::1]:44300")) + + def test_connection_selection_and_counters(self): + output = "\n".join([ + "0 100 127.0.0.1:24000 127.0.0.1:45000 cubic bytes_received:40000 rtt:0.1/0.2 snd_wnd:0 ignored:private", + "0 0 192.0.2.5:443 192.0.2.1:1234 cubic bytes_received:41000 lastrcv:60000", + "0 0 192.0.2.5:443 192.0.2.2:1234 cubic bytes_received:99999", + "0 0 [::ffff:192.0.2.5]:443 [::ffff:192.0.2.1]:5678 cubic bytes_received:42000", + ]) + rows = watch.connections(output, {24000}, {"192.0.2.1"}) + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0]["recv_q"], 0) + self.assertEqual(rows[0]["send_q"], 100) + self.assertEqual(rows[0]["snd_wnd"], "0") + self.assertEqual(rows[1]["lastrcv"], "60000") + self.assertNotIn("private", json.dumps(rows)) + self.assertEqual(watch.endpoint("[2001:db8::1]:1234"), ("2001:db8::1", 1234)) + self.assertEqual(watch.endpoint("[::ffff:192.0.2.1]:1234"), ("192.0.2.1", 1234)) + + def test_completion_and_window(self): + edge = {"ts": 200, "duration": 60, "content_length": "100", "bytes_read": 40} + samples = [{"ts": 130}, {"ts": 140}, {"ts": 195}, {"ts": 210}] + record = watch.incident(edge, samples) + self.assertFalse(record["upload_complete"]) + self.assertEqual(record["tcp"], [{"ts": 140}, {"ts": 195}]) + edge["bytes_read"] = 100 + self.assertTrue(watch.incident(edge, samples)["upload_complete"]) + edge["content_length"] = "" + self.assertIsNone(watch.incident(edge, samples)["upload_complete"]) + + def test_private_atomic_state_survives_reload(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "state.json" + watch.save(path, {"cursor": "first"}) + watch.save(path, {"cursor": "second"}) + self.assertEqual(json.loads(path.read_text()), {"cursor": "second"}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertFalse(path.with_suffix(".pending").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/substation-watch.py b/tools/substation-watch.py new file mode 100644 index 0000000000000000000000000000000000000000..148fa8ddc158f539534f8fa8eb99097e02d4c213 --- /dev/null +++ b/tools/substation-watch.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +from collections import deque +import ipaddress +import json +import os +from pathlib import Path +import re +import subprocess +import time +from urllib.request import Request, urlopen + + +ROOT = Path("/var/lib/snowglobe/substation-watch") +EDGE_FIELDS = ("ts", "status", "bytes_read", "content_length", "duration", "request_id", + "method", "path", "protocol", "upstream", "upstream_status", "upstream_header_ms", "client") +TCP_FIELDS = {"bytes_sent", "bytes_acked", "bytes_received", "bytes_retrans", "lastsnd", "lastrcv", "lastack", + "rtt", "rto", "retrans", "snd_wnd", "rcv_wnd", "rcv_space", "rcv_ssthresh", "cwnd", "unacked"} + + +def save(path, value): + pending = path.with_suffix(".pending") + with pending.open("w") as file: + json.dump(value, file, separators=(",", ":")) + pending.chmod(0o600) + pending.replace(path) + + +def endpoint(value): + address, _, port = value.rpartition(":") + address = ipaddress.ip_address(address.strip("[]")) + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped + return str(address), int(port) + + +def connections(output, ports, clients): + rows = [] + for line in output.splitlines(): + parts = line.split() + if len(parts) < 4: + continue + local, peer = endpoint(parts[2]), endpoint(parts[3]) + if not (local[1] in ports or peer[1] in ports or local[1] == 443 and peer[0] in clients): + continue + row = {"local": parts[2], "peer": parts[3], "recv_q": int(parts[0]), "send_q": int(parts[1])} + for part in parts[4:]: + key, _, value = part.partition(":") + if key in TCP_FIELDS and re.fullmatch(r"[0-9./]+", value): + row[key] = value + rows.append(row) + return rows + + +def event(entry, host): + try: + message = json.loads(entry.get("MESSAGE", "")) + except (ValueError, TypeError): + return None, None + if not isinstance(message, dict): + return None, None + if message.get("service") == "substation": + return {key: message.get(key) for key in EDGE_FIELDS}, message.get("client") + request = message.get("request", {}) + if request.get("host") == host and request.get("remote_ip"): + address = str(ipaddress.ip_address(request["remote_ip"])) + return None, f"[{address}]:{int(request['remote_port'])}" + return None, None + + +def incident(edge, samples): + length = edge.get("content_length") + complete = int(length) == edge["bytes_read"] if str(length).isdigit() else None + return {"edge": edge, "upload_complete": complete, + "tcp": [sample for sample in samples if edge["ts"] - edge["duration"] - 5 <= sample["ts"] <= edge["ts"] + 5]} + + +def main(): + os.umask(0o077) + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + state_file = ROOT / "state.json" + state = json.loads(state_file.read_text()) if state_file.exists() else {"clients": {}} + recent = ROOT / "recent.json" + samples = deque(json.loads(recent.read_text()) if recent.exists() and time.time() - recent.stat().st_mtime < 120 else [], maxlen=120) + ports = set() + refreshed = saved = reported = 0 + host = "substation.ai." + os.environ["SNOWGLOBE_DOMAIN"] + while True: + try: + now = time.time() + if now - refreshed >= 10: + token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() + request = Request("http://127.0.0.1:4646/v1/service/substation-hub", headers={"X-Nomad-Token": token}) + with urlopen(request, timeout=5) as response: + ports = {item["Port"] for item in json.load(response) + if item.get("JobID") == "substation" and ipaddress.ip_address(item["Address"]).is_loopback} + refreshed = now + command = ["journalctl", "-u", "caddy", "--no-pager", "--output=json"] + command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] + journal = subprocess.run(command, capture_output=True, text=True, timeout=10) + if journal.returncode: + if "cursor" in journal.stderr.lower() and state.pop("cursor", None): + print(json.dumps({"event": "substation-watcher-journal-gap"}), flush=True) + journal.check_returncode() + for line in journal.stdout.splitlines(): + entry = json.loads(line) + edge, client = event(entry, host) + if client: + state["clients"][endpoint(client)[0]] = now + if edge and edge.get("status", 0) >= 499 and re.fullmatch(r"[0-9a-f-]{36}", str(edge.get("request_id", ""))): + path = ROOT / (edge["request_id"] + ".json") + if not path.exists(): + record = incident(edge, samples) + save(path, record) + print(json.dumps({"event": "substation-incident", "request_id": edge["request_id"], + "status": edge["status"], "upload_complete": record["upload_complete"], + "samples": len(record["tcp"])}), flush=True) + state["cursor"] = entry["__CURSOR"] + state["clients"] = {client: stamp for client, stamp in state["clients"].items() if now - stamp < 3600} + if ports: + filters = "( sport = :443 or " + " or ".join(f"sport = :{port} or dport = :{port}" for port in sorted(ports)) + " )" + result = subprocess.run(["ss", "-HtinO", "state", "established", filters], check=True, capture_output=True, text=True, timeout=5) + rows = connections(result.stdout, ports, state["clients"]) + samples.append({"ts": now, "connections": rows}) + save(state_file, state) + if now - saved >= 5: + save(recent, list(samples)) + saved = now + old = sorted((p for p in ROOT.glob("*.json") if re.fullmatch(r"[0-9a-f-]{36}", p.stem)), key=lambda p: p.stat().st_mtime) + for path in old[:-100]: + path.unlink() + if now - reported >= 60: + print(json.dumps({"event": "substation-watcher", "ports": sorted(ports), + "clients": len(state["clients"]), "samples": len(samples)}), flush=True) + reported = now + except Exception as error: + print(json.dumps({"event": "substation-watcher-error", "type": type(error).__name__}), flush=True) + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/tools/substation.md b/tools/substation.md index 701dd0df289bae4ff05b2360d6eaaaf24c750083..7fbd7a8141983fd8b23b585ee7511c919d793280 100644 --- a/tools/substation.md +++ b/tools/substation.md @@ -118,3 +118,16 @@ uploads. A missing length can mean a chunked request; it does not prove completi A 200 followed by a truncated stream needs the edge byte count and the hub's stream outcome. A successful probe does not establish that every generation stream completed. + +`snowglobe-substation-watch.service` follows Caddy with a persisted journal +cursor and samples TCP counters once per second for the hub and known API +clients. It survives service restarts and reboots. The latest two minutes are +retained in `/var/lib/snowglobe/substation-watch/recent.json`; each 499 or 5xx +retains its request metadata and surrounding TCP progress in a private file +named by request ID. The directory keeps the latest 100 incidents. No packet +payloads, request bodies, cookies, or credentials are collected. + +Watcher heartbeat and incident summaries appear in its journal and Victoria +Logs. `upload_complete: false` proves an incomplete fixed-length upload; +the saved receive/send queues, byte counters, receive windows, and retransmits +distinguish stalled sending from server backpressure and network loss.