| 1 | #!/usr/bin/env python3 |
| 2 | import datetime |
| 3 | import json |
| 4 | import math |
| 5 | import os |
| 6 | from pathlib import Path |
| 7 | import re |
| 8 | import subprocess |
| 9 | import time |
| 10 | from urllib.parse import urlencode |
| 11 | from urllib.request import Request, urlopen |
| 12 | |
| 13 | |
| 14 | STATE = Path("/var/lib/snowglobe/log-shipper.json") |
| 15 | LOGS = Path("/var/lib/nomad/alloc") |
| 16 | CRI = re.compile(r"^(\S+) (stdout|stderr) [FP] ?(.*)$") |
| 17 | |
| 18 | |
| 19 | def nomad(path): |
| 20 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() |
| 21 | with urlopen(Request("http://127.0.0.1:4646" + path, headers={"X-Nomad-Token": token}), timeout=10) as response: |
| 22 | return json.load(response) |
| 23 | |
| 24 | |
| 25 | def send(base, rows, fields): |
| 26 | if not rows: |
| 27 | return |
| 28 | data = b"\n".join(json.dumps(row, ensure_ascii=False).encode() for row in rows) + b"\n" |
| 29 | request = Request(base + "/insert/jsonline", data=data, headers={ |
| 30 | "Content-Type": "application/stream+json", "VL-Stream-Fields": fields, |
| 31 | }) |
| 32 | with urlopen(request, timeout=30) as response: |
| 33 | response.read() |
| 34 | |
| 35 | |
| 36 | def retention_bounds(base): |
| 37 | with urlopen(base + "/metrics", timeout=10) as response: |
| 38 | metrics = response.read().decode() |
| 39 | flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics)) |
| 40 | durations = {} |
| 41 | units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000} |
| 42 | for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"): |
| 43 | value = flags[name] |
| 44 | if re.fullmatch(r"[0-9.]+M?", value): |
| 45 | seconds = float(value.rstrip("M")) * 2678400 |
| 46 | else: |
| 47 | parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower()) |
| 48 | if "".join(number + unit for number, unit in parts) != value.lower(): |
| 49 | raise ValueError("unsupported log retention duration") |
| 50 | seconds = sum(float(number) * units[unit] for number, unit in parts) |
| 51 | if not math.isfinite(seconds) or seconds < 0: |
| 52 | raise ValueError("invalid log retention duration") |
| 53 | durations[name] = int(seconds * 1000) / 1000 |
| 54 | retention = max(86400, durations["retentionPeriod"]) |
| 55 | backfill = durations["maxBackfillAge"] |
| 56 | backfill = backfill if 0 < backfill <= retention else retention |
| 57 | current = time.time() |
| 58 | return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400 |
| 59 | |
| 60 | |
| 61 | def save(state): |
| 62 | pending_state = STATE.with_suffix(".pending") |
| 63 | with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file: |
| 64 | json.dump(state, file) |
| 65 | os.replace(pending_state, STATE) |
| 66 | |
| 67 | |
| 68 | def once(state): |
| 69 | services = nomad("/v1/service/victoria-logs") |
| 70 | if not services: |
| 71 | return |
| 72 | target = services[0] |
| 73 | base = f"http://{target['Address']}:{target['Port']}" |
| 74 | if "pending" not in state: |
| 75 | allocations = nomad("/v1/allocations") |
| 76 | jobs = {item["ID"]: item["JobID"] for item in allocations} |
| 77 | offsets = state.get("files", {}).copy() |
| 78 | rows = [] |
| 79 | pending = {} |
| 80 | for file in sorted(LOGS.glob("*/alloc/logs/*")): |
| 81 | match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name) |
| 82 | job = jobs.get(file.parent.parent.parent.name) |
| 83 | if not match or not job: |
| 84 | continue |
| 85 | info = file.stat() |
| 86 | old = offsets.get(str(file), {}) |
| 87 | offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0 |
| 88 | with file.open("rb") as stream: |
| 89 | stream.seek(offset) |
| 90 | chunk = stream.read(262144) |
| 91 | if not chunk: |
| 92 | continue |
| 93 | if not chunk.endswith(b"\n"): |
| 94 | chunk = chunk[:chunk.rfind(b"\n") + 1] |
| 95 | if not chunk: |
| 96 | continue |
| 97 | pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)} |
| 98 | position = offset |
| 99 | for raw in chunk.splitlines(keepends=True): |
| 100 | identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}" |
| 101 | position += len(raw) |
| 102 | parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n")) |
| 103 | if not parsed: |
| 104 | continue |
| 105 | stamp, stream, message = parsed.groups() |
| 106 | level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info" |
| 107 | rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job, |
| 108 | "task": match.group(1), "stream": stream, "level": level, |
| 109 | "id": identity, "allocation": file.parent.parent.parent.name}) |
| 110 | events = {} |
| 111 | for allocation in allocations: |
| 112 | seen = set(state.get("events", {}).get(allocation["ID"], [])) |
| 113 | current = events[allocation["ID"]] = [] |
| 114 | for task, task_state in (allocation.get("TaskStates") or {}).items(): |
| 115 | for event in task_state.get("Events") or []: |
| 116 | kind = event["Type"] |
| 117 | identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}" |
| 118 | current.append(identity) |
| 119 | if identity in seen: |
| 120 | continue |
| 121 | oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true" |
| 122 | message = event.get("DisplayMessage") or event.get("Message") or kind |
| 123 | if oom: |
| 124 | message = "Out of memory (OOM): " + message |
| 125 | stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat() |
| 126 | level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info" |
| 127 | rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"], |
| 128 | "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind, |
| 129 | "id": identity, "allocation": allocation["ID"]}) |
| 130 | sequence = max(state.get("sequence", 0), time.time_ns() // 1000) |
| 131 | for row in rows: |
| 132 | sequence += 1 |
| 133 | row["sequence"] = sequence |
| 134 | offsets.update(pending) |
| 135 | state["sequence"] = sequence |
| 136 | state["pending"] = {"rows": rows, "files": offsets, "events": events} |
| 137 | save(state) |
| 138 | batch = state["pending"] |
| 139 | rows = batch["rows"] |
| 140 | sequence = state["sequence"] |
| 141 | if rows: |
| 142 | minimum, maximum = retention_bounds(base) |
| 143 | rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum] |
| 144 | if rows: |
| 145 | send(base, rows, "source,job,task,stream") |
| 146 | query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows" |
| 147 | deadline = time.monotonic() + 30 |
| 148 | while True: |
| 149 | request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode()) |
| 150 | with urlopen(request, timeout=10) as response: |
| 151 | visible = json.loads(response.read() or b"{}") |
| 152 | if int(visible.get("rows", 0)) == len(rows): |
| 153 | break |
| 154 | if time.monotonic() >= deadline: |
| 155 | raise TimeoutError("log batch is not searchable yet") |
| 156 | time.sleep(.25) |
| 157 | send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source") |
| 158 | state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()} |
| 159 | state["events"] = batch["events"] |
| 160 | del state["pending"] |
| 161 | save(state) |
| 162 | |
| 163 | command = ["journalctl", "--no-pager", "--output=json"] |
| 164 | command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] |
| 165 | system = [] |
| 166 | cursor = None |
| 167 | process = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) |
| 168 | try: |
| 169 | for index, line in enumerate(process.stdout): |
| 170 | if index == 5000: |
| 171 | process.terminate() |
| 172 | break |
| 173 | entry = json.loads(line) |
| 174 | cursor = entry.get("__CURSOR", cursor) |
| 175 | message = entry.get("MESSAGE") |
| 176 | if not isinstance(message, str): |
| 177 | continue |
| 178 | stamp = int(entry["__REALTIME_TIMESTAMP"]) / 1_000_000 |
| 179 | system.append({"_time": datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).isoformat(), |
| 180 | "_msg": message, "source": "journald", "unit": entry.get("_SYSTEMD_UNIT", "system"), |
| 181 | "identifier": entry.get("SYSLOG_IDENTIFIER", ""), "priority": entry.get("PRIORITY", "")}) |
| 182 | finally: |
| 183 | process.stdout.close() |
| 184 | process.wait() |
| 185 | if process.returncode not in (0, -15): |
| 186 | raise RuntimeError(f"journalctl exited {process.returncode}") |
| 187 | send(base, system, "source,unit") |
| 188 | if cursor: |
| 189 | state["cursor"] = cursor |
| 190 | save(state) |
| 191 | |
| 192 | |
| 193 | def main(): |
| 194 | STATE.parent.mkdir(parents=True, exist_ok=True) |
| 195 | while True: |
| 196 | try: |
| 197 | state = json.loads(STATE.read_text()) if STATE.exists() else {} |
| 198 | once(state) |
| 199 | except Exception as error: |
| 200 | print(f"log shipping paused: {error}", flush=True) |
| 201 | time.sleep(1) |
| 202 | |
| 203 | |
| 204 | if __name__ == "__main__": |
| 205 | main() |