| 1 | #!/usr/bin/env python3 |
| 2 | """Files logs and metrics pushed by the VPS agents in Victoria under source=vps.""" |
| 3 | from compression import zstd |
| 4 | import gzip |
| 5 | import hmac |
| 6 | import json |
| 7 | import os |
| 8 | import re |
| 9 | import threading |
| 10 | import time |
| 11 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 12 | from urllib.error import HTTPError |
| 13 | from urllib.request import Request, urlopen |
| 14 | |
| 15 | TOKEN = os.environ["INGEST_TOKEN"].encode() |
| 16 | LOGS = os.environ["LOGS_URL"] |
| 17 | METRICS = os.environ["METRICS_URL"] |
| 18 | PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log") |
| 19 | ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I) |
| 20 | WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I) |
| 21 | LIMIT = 32 << 20 |
| 22 | JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY") |
| 23 | |
| 24 | sequence_lock = threading.Lock() |
| 25 | last_sequence = 0 |
| 26 | |
| 27 | |
| 28 | def next_sequence(): |
| 29 | """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow.""" |
| 30 | global last_sequence |
| 31 | with sequence_lock: |
| 32 | last_sequence = max(last_sequence + 1, time.time_ns() // 1000) |
| 33 | return last_sequence |
| 34 | |
| 35 | |
| 36 | def shape(row): |
| 37 | if match := PM2.fullmatch(row.get("file", "")): |
| 38 | job, stream = match[1], "stdout" if match[2] == "out" else "stderr" |
| 39 | message = row.get("_msg", "") |
| 40 | level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info" |
| 41 | else: |
| 42 | job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal" |
| 43 | priority = int(row.get("PRIORITY") or 6) |
| 44 | row = {key: row[key] for key in JOURNAL if key in row} |
| 45 | level = "error" if priority <= 3 else "warn" if priority == 4 else "info" |
| 46 | row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream, |
| 47 | level=level, sequence=next_sequence()) |
| 48 | return row |
| 49 | |
| 50 | |
| 51 | def forward(url, body, headers): |
| 52 | with urlopen(Request(url, data=body, headers=headers), timeout=30) as response: |
| 53 | response.read() |
| 54 | |
| 55 | |
| 56 | class Handler(BaseHTTPRequestHandler): |
| 57 | protocol_version = "HTTP/1.1" |
| 58 | |
| 59 | def reply(self, status, text=""): |
| 60 | body = text.encode() |
| 61 | self.send_response(status) |
| 62 | self.send_header("Content-Length", str(len(body))) |
| 63 | self.end_headers() |
| 64 | self.wfile.write(body) |
| 65 | |
| 66 | def do_GET(self): |
| 67 | self.reply(204 if self.path == "/health" else 404) |
| 68 | |
| 69 | def do_POST(self): |
| 70 | length = int(self.headers.get("Content-Length") or 0) |
| 71 | if length > LIMIT: |
| 72 | self.close_connection = True |
| 73 | return self.reply(413, "batch too large") |
| 74 | body = self.rfile.read(length) |
| 75 | if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN): |
| 76 | return self.reply(401, "bad or missing token") |
| 77 | try: |
| 78 | if self.path == "/insert/jsonline": |
| 79 | encoding = self.headers.get("Content-Encoding", "") |
| 80 | text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body |
| 81 | rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()] |
| 82 | forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows), |
| 83 | {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"}) |
| 84 | forward(LOGS + "/internal/force_flush", b"", {}) |
| 85 | elif self.path == "/api/v1/write": |
| 86 | forward(METRICS + "/api/v1/write?extra_label=source=vps", body, |
| 87 | {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers}) |
| 88 | else: |
| 89 | return self.reply(404) |
| 90 | except HTTPError as error: |
| 91 | return self.reply(502, f"victoria answered {error.code}") |
| 92 | except (OSError, ValueError, zstd.ZstdError) as error: |
| 93 | return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error)) |
| 94 | self.reply(204) |
| 95 | |
| 96 | def log_request(self, code="-", size="-"): |
| 97 | if not str(int(code)).startswith("2"): |
| 98 | super().log_request(code, size) |
| 99 | |
| 100 | |
| 101 | ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() |