1#!/usr/bin/env python3
2import datetime
3import json
4import math
5import os
6from pathlib import Path
7import re
8import subprocess
9import time
10from urllib.parse import urlencode
11from urllib.request import Request, urlopen
12
13
14STATE = Path("/var/lib/snowglobe/log-shipper.json")
15LOGS = Path("/var/lib/nomad/alloc")
16CRI = re.compile(r"^(\S+) (stdout|stderr) [FP] ?(.*)$")
17
18
19def nomad(path):
20 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
21 with urlopen(Request("http://127.0.0.1:4646" + path, headers={"X-Nomad-Token": token}), timeout=10) as response:
22 return json.load(response)
23
24
25def send(base, rows, fields):
26 if not rows:
27 return
28 data = b"\n".join(json.dumps(row, ensure_ascii=False).encode() for row in rows) + b"\n"
29 request = Request(base + "/insert/jsonline", data=data, headers={
30 "Content-Type": "application/stream+json", "VL-Stream-Fields": fields,
31 })
32 with urlopen(request, timeout=30) as response:
33 response.read()
34
35
36def retention_bounds(base):
37 with urlopen(base + "/metrics", timeout=10) as response:
38 metrics = response.read().decode()
39 flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics))
40 durations = {}
41 units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000}
42 for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"):
43 value = flags[name]
44 if re.fullmatch(r"[0-9.]+M?", value):
45 seconds = float(value.rstrip("M")) * 2678400
46 else:
47 parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower())
48 if "".join(number + unit for number, unit in parts) != value.lower():
49 raise ValueError("unsupported log retention duration")
50 seconds = sum(float(number) * units[unit] for number, unit in parts)
51 if not math.isfinite(seconds) or seconds < 0:
52 raise ValueError("invalid log retention duration")
53 durations[name] = int(seconds * 1000) / 1000
54 retention = max(86400, durations["retentionPeriod"])
55 backfill = durations["maxBackfillAge"]
56 backfill = backfill if 0 < backfill <= retention else retention
57 current = time.time()
58 return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400
59
60
61def save(state):
62 pending_state = STATE.with_suffix(".pending")
63 with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file:
64 json.dump(state, file)
65 os.replace(pending_state, STATE)
66
67
68def once(state):
69 services = nomad("/v1/service/victoria-logs")
70 if not services:
71 return
72 target = services[0]
73 base = f"http://{target['Address']}:{target['Port']}"
74 if "pending" not in state:
75 allocations = nomad("/v1/allocations")
76 jobs = {item["ID"]: item["JobID"] for item in allocations}
77 offsets = state.get("files", {}).copy()
78 rows = []
79 pending = {}
80 for file in sorted(LOGS.glob("*/alloc/logs/*")):
81 match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name)
82 job = jobs.get(file.parent.parent.parent.name)
83 if not match or not job:
84 continue
85 info = file.stat()
86 old = offsets.get(str(file), {})
87 offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0
88 with file.open("rb") as stream:
89 stream.seek(offset)
90 chunk = stream.read(262144)
91 if not chunk:
92 continue
93 if not chunk.endswith(b"\n"):
94 chunk = chunk[:chunk.rfind(b"\n") + 1]
95 if not chunk:
96 continue
97 pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)}
98 position = offset
99 for raw in chunk.splitlines(keepends=True):
100 identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}"
101 position += len(raw)
102 parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n"))
103 if not parsed:
104 continue
105 stamp, stream, message = parsed.groups()
106 level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info"
107 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job,
108 "task": match.group(1), "stream": stream, "level": level,
109 "id": identity, "allocation": file.parent.parent.parent.name})
110 events = {}
111 for allocation in allocations:
112 seen = set(state.get("events", {}).get(allocation["ID"], []))
113 current = events[allocation["ID"]] = []
114 for task, task_state in (allocation.get("TaskStates") or {}).items():
115 for event in task_state.get("Events") or []:
116 kind = event["Type"]
117 identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}"
118 current.append(identity)
119 if identity in seen:
120 continue
121 oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true"
122 message = event.get("DisplayMessage") or event.get("Message") or kind
123 if oom:
124 message = "Out of memory (OOM): " + message
125 stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat()
126 level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info"
127 rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"],
128 "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind,
129 "id": identity, "allocation": allocation["ID"]})
130 sequence = max(state.get("sequence", 0), time.time_ns() // 1000)
131 for row in rows:
132 sequence += 1
133 row["sequence"] = sequence
134 offsets.update(pending)
135 state["sequence"] = sequence
136 state["pending"] = {"rows": rows, "files": offsets, "events": events}
137 save(state)
138 batch = state["pending"]
139 rows = batch["rows"]
140 sequence = state["sequence"]
141 if rows:
142 minimum, maximum = retention_bounds(base)
143 rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum]
144 if rows:
145 send(base, rows, "source,job,task,stream")
146 query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows"
147 deadline = time.monotonic() + 30
148 while True:
149 request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode())
150 with urlopen(request, timeout=10) as response:
151 visible = json.loads(response.read() or b"{}")
152 if int(visible.get("rows", 0)) == len(rows):
153 break
154 if time.monotonic() >= deadline:
155 raise TimeoutError("log batch is not searchable yet")
156 time.sleep(.25)
157 send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source")
158 state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()}
159 state["events"] = batch["events"]
160 del state["pending"]
161 save(state)
162
163 command = ["journalctl", "--no-pager", "--output=json"]
164 command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"]
165 system = []
166 cursor = None
167 process = subprocess.Popen(command, stdout=subprocess.PIPE, text=True)
168 try:
169 for index, line in enumerate(process.stdout):
170 if index == 5000:
171 process.terminate()
172 break
173 entry = json.loads(line)
174 cursor = entry.get("__CURSOR", cursor)
175 message = entry.get("MESSAGE")
176 if not isinstance(message, str):
177 continue
178 stamp = int(entry["__REALTIME_TIMESTAMP"]) / 1_000_000
179 system.append({"_time": datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).isoformat(),
180 "_msg": message, "source": "journald", "unit": entry.get("_SYSTEMD_UNIT", "system"),
181 "identifier": entry.get("SYSLOG_IDENTIFIER", ""), "priority": entry.get("PRIORITY", "")})
182 finally:
183 process.stdout.close()
184 process.wait()
185 if process.returncode not in (0, -15):
186 raise RuntimeError(f"journalctl exited {process.returncode}")
187 send(base, system, "source,unit")
188 if cursor:
189 state["cursor"] = cursor
190 save(state)
191
192
193def main():
194 STATE.parent.mkdir(parents=True, exist_ok=True)
195 while True:
196 try:
197 state = json.loads(STATE.read_text()) if STATE.exists() else {}
198 once(state)
199 except Exception as error:
200 print(f"log shipping paused: {error}", flush=True)
201 time.sleep(1)
202
203
204if __name__ == "__main__":
205 main()