1#!/usr/bin/env python3
2"""Files logs and metrics pushed by the VPS agents in Victoria under source=vps."""
3from compression import zstd
4import gzip
5import hmac
6import json
7import os
8import re
9import threading
10import time
11from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
12from urllib.error import HTTPError
13from urllib.request import Request, urlopen
14
15TOKEN = os.environ["INGEST_TOKEN"].encode()
16LOGS = os.environ["LOGS_URL"]
17METRICS = os.environ["METRICS_URL"]
18PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log")
19ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I)
20WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I)
21LIMIT = 32 << 20
22JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY")
23
24sequence_lock = threading.Lock()
25last_sequence = 0
26
27
28def next_sequence():
29 """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow."""
30 global last_sequence
31 with sequence_lock:
32 last_sequence = max(last_sequence + 1, time.time_ns() // 1000)
33 return last_sequence
34
35
36def shape(row):
37 if match := PM2.fullmatch(row.get("file", "")):
38 job, stream = match[1], "stdout" if match[2] == "out" else "stderr"
39 message = row.get("_msg", "")
40 level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info"
41 else:
42 job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal"
43 priority = int(row.get("PRIORITY") or 6)
44 row = {key: row[key] for key in JOURNAL if key in row}
45 level = "error" if priority <= 3 else "warn" if priority == 4 else "info"
46 row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream,
47 level=level, sequence=next_sequence())
48 return row
49
50
51def forward(url, body, headers):
52 with urlopen(Request(url, data=body, headers=headers), timeout=30) as response:
53 response.read()
54
55
56class Handler(BaseHTTPRequestHandler):
57 protocol_version = "HTTP/1.1"
58
59 def reply(self, status, text=""):
60 body = text.encode()
61 self.send_response(status)
62 self.send_header("Content-Length", str(len(body)))
63 self.end_headers()
64 self.wfile.write(body)
65
66 def do_GET(self):
67 self.reply(204 if self.path == "/health" else 404)
68
69 def do_POST(self):
70 length = int(self.headers.get("Content-Length") or 0)
71 if length > LIMIT:
72 self.close_connection = True
73 return self.reply(413, "batch too large")
74 body = self.rfile.read(length)
75 if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN):
76 return self.reply(401, "bad or missing token")
77 try:
78 if self.path == "/insert/jsonline":
79 encoding = self.headers.get("Content-Encoding", "")
80 text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body
81 rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()]
82 forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows),
83 {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"})
84 forward(LOGS + "/internal/force_flush", b"", {})
85 elif self.path == "/api/v1/write":
86 forward(METRICS + "/api/v1/write?extra_label=source=vps", body,
87 {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers})
88 else:
89 return self.reply(404)
90 except HTTPError as error:
91 return self.reply(502, f"victoria answered {error.code}")
92 except (OSError, ValueError, zstd.ZstdError) as error:
93 return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error))
94 self.reply(204)
95
96 def log_request(self, code="-", size="-"):
97 if not str(int(code)).startswith("2"):
98 super().log_request(code, size)
99
100
101ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever()