From ca421c2098f96de8070bb7d17dd1d3aab3b47487 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 24 Oct 2026 19:43:48 -0700 Subject: [PATCH] Ship VPS logs and metrics to Snow Globe A public ingest service takes vlagent and vmagent pushes with a bearer token and files them in Victoria under source=vps, naming each log line after its pm2 app or systemd unit. On the VPS, vlagent tails the pm2 logs and the journal, vmagent scrapes the iroh relay, and a small script reports pm2 state as service metrics. The dashboard lists these under "vps services" with the same logs, cpu, memory, and metrics views as Nomad services, keyed vps.. Victoria logs gets a host port so containers on the bridge can reach it. Assisted-by: claude-opus-5.5 --- dashboard/src/core.rs | 12 +-- dashboard/src/main.rs | 2 +- dashboard/src/telemetry.rs | 56 +++++++++++++- dashboard/web/api.contract.ts | 5 +- dashboard/web/api.ts | 1 + dashboard/web/components/Sidebar.tsx | 33 +++++++++ dashboard/web/main.tsx | 4 +- dashboard/web/pages/Service.tsx | 4 +- dashboard/web/pages/Vps.tsx | 90 +++++++++++++++++++++++ dashboard/web/types/model.ts | 12 +++ service/ingest/build/Dockerfile | 3 + service/ingest/build/app.py | 101 ++++++++++++++++++++++++++ service/ingest/service.pkl | 30 ++++++++ service/iroh-relay/iroh-relay.service | 1 + service/iroh-relay/run.py | 4 +- service/victoria/service.pkl | 1 + tools/iroh-relay.md | 3 + tools/vps.md | 39 ++++++++++ tools/vps/journal-upload.conf | 2 + tools/vps/pm2-metrics.py | 22 ++++++ tools/vps/pm2-metrics.service | 14 ++++ tools/vps/vlagent.service | 23 ++++++ tools/vps/vmagent.service | 25 +++++++ tools/vps/vmagent.yml | 7 ++ 24 files changed, 480 insertions(+), 14 deletions(-) create mode 100644 dashboard/web/pages/Vps.tsx create mode 100644 service/ingest/build/Dockerfile create mode 100644 service/ingest/build/app.py create mode 100644 service/ingest/service.pkl create mode 100644 tools/vps.md create mode 100644 tools/vps/journal-upload.conf create mode 100644 tools/vps/pm2-metrics.py create mode 100644 tools/vps/pm2-metrics.service create mode 100644 tools/vps/vlagent.service create mode 100644 tools/vps/vmagent.service create mode 100644 tools/vps/vmagent.yml diff --git a/dashboard/src/core.rs b/dashboard/src/core.rs index d5fe219239a7a8beca4f095f67a5e5b5b1cf130b..3cdc741d67baeb20cb601acf2956cf4123efb9e9 100644 --- a/dashboard/src/core.rs +++ b/dashboard/src/core.rs @@ -622,6 +622,7 @@ pub async fn route( json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null}) } ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()), + ["vps"] if method == Method::GET => telemetry::vps(app).await?, ["launcher"] if method == Method::GET => { let found = launcher(app.clone()).await?; let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10)); @@ -723,11 +724,12 @@ pub async fn route( .response()); } ["services", id, "metrics", name] if method == Method::GET => { - let detail = service(app.clone(), id).await?; - if !array(&detail.value["applicationMetrics"]) - .iter() - .any(|v| v == *name) - { + let names = if id.starts_with("vps.") { + telemetry::metric_names(app.clone(), id).await? + } else { + service(app.clone(), id).await?.value["applicationMetrics"].clone() + }; + if !array(&names).iter().any(|v| v == *name) { return Err(Error::new(404, "Metric unavailable for this service.")); } return Ok(telemetry::metrics(app, name, query, Some(id), None) diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index 000fcd65aad75acc5c49626d195914aed20b1d83..0a2a034c898c523f503826a55dd1b154476bae4d 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -279,7 +279,7 @@ async fn api(State(app): State>, request: Request) -> Result "services" if parts.len() == 1 => Some("metrics"), "storage" if parts.len() == 1 => Some("metrics"), "ai" | "mcp" => Some("ai"), - "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { + "services" | "vps" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { Some("admin") } "media" | "seedbox" | "youtube" => Some("media"), diff --git a/dashboard/src/telemetry.rs b/dashboard/src/telemetry.rs index 88b1a8376edc3b020ea9b7843fe1f61714be0edb..07357542ae5f06123132e39c5ba36dec22127f06 100644 --- a/dashboard/src/telemetry.rs +++ b/dashboard/src/telemetry.rs @@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc, id: &str) -> Result { ( "match[]", format!( - "{{{label}={},__name__!~\"snowglobe_.*\"}}", + "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}", serde_json::to_string(id).unwrap() ), ) @@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc, id: &str) -> Result { Ok(json!(values)) } +/// Services the VPS agents report, newest state first; quiet for two minutes counts as down. +pub async fn vps(app: Arc) -> Result { + let base = endpoint(app.clone(), "victoria-metrics").await?; + let instant = |query: &str| { + let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())])); + let app = app.clone(); + async move { + let response = read(app, url, None).await?; + Ok::<_, Error>( + array(&response.value["data"]["result"]) + .iter() + .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1]))) + .collect::>(), + ) + } + }; + let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#; + let (known, current, cpu, memory, restarts, started) = futures::try_join!( + instant(&format!("max by (service) (max_over_time({up}[7d]))")), + instant(&format!("max by (service) (last_over_time({up}[2m]))")), + instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#), + instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#), + )?; + let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect(); + ids.sort(); + let mut services = Vec::new(); + for id in ids { + services.push(json!({ + "id": id, + "name": &id[4..], + "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" }, + "cpu": cpu.get(&id), + "memory": memory.get(&id), + "restarts": restarts.get(&id), + "startedAt": started.get(&id).filter(|v| **v > 0.0), + "applicationMetrics": metric_names(app.clone(), &id).await?, + })); + } + Ok(json!(services)) +} + #[derive(Debug, PartialEq)] struct Term { field: Option, @@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result> { Ok(found) } fn log_query(id: &str, query: &HashMap) -> Result { + let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job)); let mut filters = vec![ - "source:=nomad".to_owned(), - format!("job:={}", serde_json::to_string(id)?), + format!("source:={source}"), + format!("job:={}", serde_json::to_string(job)?), ]; if let Some(level) = query.get("level") { filters.push( @@ -1001,6 +1045,12 @@ pub fn start(app: Arc) { mod tests { use super::*; #[test] + fn vps_ids_read_the_vps_source() { + let none = HashMap::new(); + assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#); + assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#); + } + #[test] fn search_fields_comparisons_exclusions_and_unknown_fields() { let parsed = terms( r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#, diff --git a/dashboard/web/api.contract.ts b/dashboard/web/api.contract.ts index 98834fa0df4b2e4429d6bfcbdbe92b2d01a34f6d..a82e8a50d83b4f637e9f754bd36c919889d2eaac 100644 --- a/dashboard/web/api.contract.ts +++ b/dashboard/web/api.contract.ts @@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts"; import type { Hono } from "hono"; import type { Health } from "./types/model.ts"; import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; -import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; +import type { Me, ServiceSummary, ServiceDetail, VpsService, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; import type { MapNode } from "./types/storage.index.ts"; import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; @@ -102,6 +102,9 @@ export type Api = Hono<{}, { "/services": { $get: Endpoint; }; + "/vps": { + $get: Endpoint; + }; "/services/:id": { $get: Endpoint; }; diff --git a/dashboard/web/api.ts b/dashboard/web/api.ts index c3c99c3f76df045bfb6996ba1f04911aa3b9a32e..a249588a9408147d37842ea6e186041d8438fccc 100644 --- a/dashboard/web/api.ts +++ b/dashboard/web/api.ts @@ -80,6 +80,7 @@ export const queries = { storage: query("storage", () => parseResponse(api.storage.$get())), deploys: query("deploys", () => parseResponse(api.deploys.$get())), vms: query("vms", () => parseResponse(api.vms.$get())), + vps: query("vps", () => parseResponse(api.vps.$get())), metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) => parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))), seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())), diff --git a/dashboard/web/components/Sidebar.tsx b/dashboard/web/components/Sidebar.tsx index 3af6518a3640544a1c895125e313ad2f034d22fb..94f24296a324c1ca67be5cd1149a95ad8d982fc0 100644 --- a/dashboard/web/components/Sidebar.tsx +++ b/dashboard/web/components/Sidebar.tsx @@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house"; import LogOut from "lucide-solid/icons/log-out"; import Monitor from "lucide-solid/icons/monitor"; import Rocket from "lucide-solid/icons/rocket"; +import Server from "lucide-solid/icons/server"; import SquarePlay from "lucide-solid/icons/square-play"; import UserRound from "lucide-solid/icons/user-round"; import Plug from "lucide-solid/icons/plug"; @@ -247,6 +248,37 @@ function Services() { ); } +function VpsServices() { + const [open, toggle] = remembered("sidebar.vps", true); + const [services, { refetch }] = queries.vps.use(); + const timer = setInterval(() => !services.loading && refetch(), 15_000); + onCleanup(() => clearInterval(timer)); + const loaded = lastGood(services); + const down = () => (loaded() ?? []).filter((service) => service.health !== "healthy").map((service) => service.name); + return ( + <> + + + + ); +} + /** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */ export function viewAs(groups: string[] | null) { document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`; @@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) { {(page) => } + {(page) => page.section === "vms" ? : } diff --git a/dashboard/web/main.tsx b/dashboard/web/main.tsx index e77ab74909898f9298d24a020b58662b6a953bfc..ce72d7deafe826c1f5fc949c53f60b882fb444d3 100644 --- a/dashboard/web/main.tsx +++ b/dashboard/web/main.tsx @@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx"; import { Deploys } from "./pages/Deploys.tsx"; import { Deploy } from "./pages/Deploy.tsx"; import { VMs } from "./pages/VMs.tsx"; +import { Vps } from "./pages/Vps.tsx"; import { MCP } from "./pages/MCP.tsx"; import { MCPConsent } from "./pages/MCPConsent.tsx"; import { SignIn } from "./pages/SignIn.tsx"; @@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) = function Shell(props: RouteSectionProps) { const location = useLocation(); const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request"); - const section = () => location.pathname.startsWith("/services/") ? "admin" + const section = () => /^\/(services|vps)\//.test(location.pathname) ? "admin" : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section; return ( <> @@ -78,6 +79,7 @@ render(() => ( queries.service.preload(params.id!); queries.deploys.preload(); }} /> + diff --git a/dashboard/web/pages/Service.tsx b/dashboard/web/pages/Service.tsx index 1b6b1343514b7115a14270a5e4724784fe5d9505..d84933b6baf07882917a2ee4cc21b94ee33da2c5 100644 --- a/dashboard/web/pages/Service.tsx +++ b/dashboard/web/pages/Service.tsx @@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ -function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { +export function Logs(props: { service: Pick; tools: HTMLElement }) { const [params, setParams] = useSearchParams(); const at = () => (params.at ? Number(params.at) : null); const bounded = () => at() !== null || !!params.to; @@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } }))); const PAGE = 50; -function AppMetrics(props: { service: ServiceDetail; range: number }) { +export function AppMetrics(props: { service: Pick; range: number }) { const [name, setName] = createSignal(props.service.applicationMetrics[0]!); const [data, { refetch }] = createResource( () => [props.service.id, name(), props.range] as const, diff --git a/dashboard/web/pages/Vps.tsx b/dashboard/web/pages/Vps.tsx new file mode 100644 index 0000000000000000000000000000000000000000..79c7f646158c70eddabc2b59be06afcf6e36aa67 --- /dev/null +++ b/dashboard/web/pages/Vps.tsx @@ -0,0 +1,90 @@ +import { A, useParams } from "@solidjs/router"; +import RotateCcw from "lucide-solid/icons/rotate-ccw"; +import Server from "lucide-solid/icons/server"; +import { createSignal, Match, onCleanup, Show, Switch } from "solid-js"; +import { queries } from "../api.ts"; +import { AppIcon } from "../components/AppIcon.tsx"; +import { ListPage } from "../components/ListPage.tsx"; +import { lastGood, Loaded } from "../components/Loaded.tsx"; +import { RangePicker } from "../components/Metric.tsx"; +import { StatusLabel } from "../components/Status.tsx"; +import { Strip } from "../components/Strip.tsx"; +import { TabBar } from "../components/TabBar.tsx"; +import { bytes, cores, datetime, duration, plural } from "../format.ts"; +import { AppMetrics, Logs } from "./Service.tsx"; + +/** Remounts per app, so switching never shows the last one's logs. */ +export function Vps() { + const params = useParams<{ name: string }>(); + return ; +} + +function VpsPage() { + const params = useParams<{ name: string; tab?: string }>(); + const [services, { refetch }] = queries.vps.use(); + const timer = setInterval(refetch, 15_000); + onCleanup(() => clearInterval(timer)); + const loaded = lastGood(services); + const [range, setRange] = createSignal(3600); + return ( +
}> + {() => ( + service.name === params.name)} fallback={ +
The VPS hasn't reported {params.name} in the last week.
+ }> + {(data) => { + const tabHref = (tab: string) => `/vps/${data().name}${tab ? "/" + tab : ""}`; + let tools!: HTMLDivElement; + return ( + + +

{data().name}

+
+ + }> + {(t) => ( + + up {duration(Date.now() / 1000 - t())} + + )} + + + + {(restarts) => } + +
+ + } summary={ + +
+ } + now={() => ({ value: cores(data().cpu ?? 0) })} /> + ({ value: bytes(data().memory ?? 0) })} /> +
+
+ }> +
+ + logs + metrics + +
+ + + + + + +
+ ); + }} +
+ )} +
+ ); +} + diff --git a/dashboard/web/types/model.ts b/dashboard/web/types/model.ts index f8540bca4367e954c2ba9a1d492893cbca4e84c4..5b1e6a89d6325356ff985e7983816d919bb48584 100644 --- a/dashboard/web/types/model.ts +++ b/dashboard/web/types/model.ts @@ -51,6 +51,18 @@ export interface ServiceSummary { memoryLimit: number; } +/** An app the VPS agents report, keyed `vps.`; usage, restarts and start time are pm2's. */ +export interface VpsService { + id: string; + name: string; + health: Health; + cpu: number | null; + memory: number | null; + restarts: number | null; + startedAt: number | null; + applicationMetrics: string[]; +} + /** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */ export const trouble = (health: Health) => health === "down" || health === "degraded"; diff --git a/service/ingest/build/Dockerfile b/service/ingest/build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..2a7f015843d4641a859c9f317f25aad2afa51acd --- /dev/null +++ b/service/ingest/build/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.io/library/python:3.14-slim@sha256:0214bda9af2652a418f2bf34373a4d3ca00a5f4784081e040049433ac9cc79d5 +COPY app.py /app/app.py +CMD ["python3", "-u", "/app/app.py"] diff --git a/service/ingest/build/app.py b/service/ingest/build/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0c3cb614aeec725421ab104036331ade7214a15e --- /dev/null +++ b/service/ingest/build/app.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Files logs and metrics pushed by the VPS agents in Victoria under source=vps.""" +from compression import zstd +import gzip +import hmac +import json +import os +import re +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +TOKEN = os.environ["INGEST_TOKEN"].encode() +LOGS = os.environ["LOGS_URL"] +METRICS = os.environ["METRICS_URL"] +PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log") +ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I) +WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I) +LIMIT = 32 << 20 +JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY") + +sequence_lock = threading.Lock() +last_sequence = 0 + + +def next_sequence(): + """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow.""" + global last_sequence + with sequence_lock: + last_sequence = max(last_sequence + 1, time.time_ns() // 1000) + return last_sequence + + +def shape(row): + if match := PM2.fullmatch(row.get("file", "")): + job, stream = match[1], "stdout" if match[2] == "out" else "stderr" + message = row.get("_msg", "") + level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info" + else: + job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal" + priority = int(row.get("PRIORITY") or 6) + row = {key: row[key] for key in JOURNAL if key in row} + level = "error" if priority <= 3 else "warn" if priority == 4 else "info" + row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream, + level=level, sequence=next_sequence()) + return row + + +def forward(url, body, headers): + with urlopen(Request(url, data=body, headers=headers), timeout=30) as response: + response.read() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def reply(self, status, text=""): + body = text.encode() + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.reply(204 if self.path == "/health" else 404) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + if length > LIMIT: + self.close_connection = True + return self.reply(413, "batch too large") + body = self.rfile.read(length) + if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN): + return self.reply(401, "bad or missing token") + try: + if self.path == "/insert/jsonline": + encoding = self.headers.get("Content-Encoding", "") + text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body + rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()] + forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows), + {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"}) + forward(LOGS + "/internal/force_flush", b"", {}) + elif self.path == "/api/v1/write": + forward(METRICS + "/api/v1/write?extra_label=source=vps", body, + {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers}) + else: + return self.reply(404) + except HTTPError as error: + return self.reply(502, f"victoria answered {error.code}") + except (OSError, ValueError, zstd.ZstdError) as error: + return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error)) + self.reply(204) + + def log_request(self, code="-", size="-"): + if not str(int(code)).startswith("2"): + super().log_request(code, size) + + +ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() diff --git a/service/ingest/service.pkl b/service/ingest/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..220c67dcde41236724c792233b201e780f1279f7 --- /dev/null +++ b/service/ingest/service.pkl @@ -0,0 +1,30 @@ +amends "../../config/Service.pkl" + +name = "Ingest" +tagline = "Logs and metrics pushed from the VPS" +launcher = false +dependsOn { "victoria" } + +secrets { ["token"] {} } + +container { + build = "build" + readOnlyRoot = true + cpu = 100 + memory = 128 + + http { + containerPort = 8000 + subdomain = "ingest" + checkPath = "/health" + } + + env { + ["INGEST_TOKEN"] = "${secret.own.token}" + } + + envTemplate = """ + {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-logs" }}LOGS_URL=http://\(module.nomadHostPort){{ end }} + {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-metrics" }}METRICS_URL=http://\(module.nomadHostPort){{ end }} + """ +} diff --git a/service/iroh-relay/iroh-relay.service b/service/iroh-relay/iroh-relay.service index d16095a145268f543996fbae48ac580f6e195ba8..2440608b80387bd8f0a7a7d547335636f0c857bb 100644 --- a/service/iroh-relay/iroh-relay.service +++ b/service/iroh-relay/iroh-relay.service @@ -6,6 +6,7 @@ After=network-online.target [Service] DynamicUser=true Environment=HTTP_PORT=3340 +Environment=METRICS_ADDR=127.0.0.1:9091 LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay diff --git a/service/iroh-relay/run.py b/service/iroh-relay/run.py index 6140475cd0eb5d3fe6ff2d460ba34f3f5dffea8b..443bbe93511471b4edb3fa2a6cced51b3a7108f7 100644 --- a/service/iroh-relay/run.py +++ b/service/iroh-relay/run.py @@ -36,8 +36,10 @@ if mode == "relay": raise RuntimeError("Relay certificate was not issued within three minutes") quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( Path(__file__).with_name("network.json").read_text())["quic"] + metrics = os.environ.get("METRICS_ADDR") config.write_text( - 'enable_metrics = false\nenable_quic_addr_discovery = true\n' + f'enable_metrics = {json.dumps(bool(metrics))}\nenable_quic_addr_discovery = true\n' + + (f'metrics_bind_addr = {json.dumps(metrics)}\n' if metrics else '') + f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' '[tls]\ncert_mode = "Reloading"\n' f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' diff --git a/service/victoria/service.pkl b/service/victoria/service.pkl index 783ada6df75137b36d30d6527820b9c29acefa60..5a4ae3c9bbde47a418d5502182387648b69f6dfc 100644 --- a/service/victoria/service.pkl +++ b/service/victoria/service.pkl @@ -11,6 +11,7 @@ containers { http { containerPort = 9428 + hostPort = 19428 subdomain = "logs" authRole = "infra-admin" checkPath = "/health" diff --git a/tools/iroh-relay.md b/tools/iroh-relay.md index e3124e0d479b84d1b6dfe0783704a07c04ceb67c..0c4926e116746480a8dff0b10c038aba3530aa58 100644 --- a/tools/iroh-relay.md +++ b/tools/iroh-relay.md @@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into `/etc/systemd/system/`. +The unit sets `METRICS_ADDR`, so the relay serves metrics on `127.0.0.1:9091` for +[vmagent](vps.md). + [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main Caddyfile. Validate before reloading Caddy. The ACME webroot is diff --git a/tools/vps.md b/tools/vps.md new file mode 100644 index 0000000000000000000000000000000000000000..ae1f0a366e926f0ba11485255c0f6f18e798ab70 --- /dev/null +++ b/tools/vps.md @@ -0,0 +1,39 @@ +# VPS logs and metrics + +The VPS (`ssh root@paperclover.net`, hostname `usw-server-1`) pushes its logs and +metrics to Snow Globe's [ingest](../service/ingest/service.pkl) service at +`https://ingest.paperclover.net`, which files them in Victoria under `source=vps`. +The dashboard lists them under **vps services**; their ids are `vps.`. + +| Agent | Collects | Sends | +| --- | --- | --- | +| [vlagent](vps/vlagent.service) | `~clo/.pm2/logs/*.log`, plus the journal through `systemd-journal-upload` | `/insert/jsonline` | +| [vmagent](vps/vmagent.service) | the iroh relay's metrics on `127.0.0.1:9091`, plus [pm2-metrics.py](vps/pm2-metrics.py) pushes | `/api/v1/write` | + +Ingest names each log line's `job` from its pm2 file (`site-out.log` is `site` on +stdout) or its systemd unit. pm2 writes no timestamps, so a line's time is when +vlagent read it. Both agents buffer up to 1 GB on disk while Snow Globe is +unreachable. + +```logsql +source:=vps job:=site level:=error +``` + +## Install + +Agents are the VictoriaMetrics release binaries, checksum-verified: + +| Binary | Release tarball | SHA256 | +| --- | --- | --- | +| `/usr/local/bin/vlagent` | `vlutils-linux-amd64-v1.52.0.tar.gz` | `caa315764df0d11a77ba61d4ee96b783ddb98455b167da8326940dda62914bc9` | +| `/usr/local/bin/vmagent` | `vmutils-linux-amd64-v1.152.0.tar.gz` | `8eee4a98ff1665c60682475e8a8b292b8d718b63a2f023124384dd2f6a220c79` | + +The ingest service's generated `token` secret lives in `/etc/snowglobe-ingest.token` +(root, mode 600). `vmagent.yml` and `pm2-metrics.py` go in +`/usr/local/lib/snowglobe-agents/`, the units in `/etc/systemd/system/`, and +[journal-upload.conf](vps/journal-upload.conf) in +`/etc/systemd/journal-upload.conf.d/snowglobe.conf` (package +`systemd-journal-remote`). Seed `/var/lib/systemd/journal-upload/state` with +`LAST_CURSOR=` and the current cursor, and run vlagent once against a sink until +its checkpoints reach the ends of the pm2 files; otherwise both send their whole +history, and the pm2 lines all get the same time. diff --git a/tools/vps/journal-upload.conf b/tools/vps/journal-upload.conf new file mode 100644 index 0000000000000000000000000000000000000000..008a25f46bf95fc032e8670f6c35fd88dac60cc5 --- /dev/null +++ b/tools/vps/journal-upload.conf @@ -0,0 +1,2 @@ +[Upload] +URL=http://127.0.0.1:9429/insert/journald diff --git a/tools/vps/pm2-metrics.py b/tools/vps/pm2-metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..b41cd407e87aa3973385ac385241c6e7cd048699 --- /dev/null +++ b/tools/vps/pm2-metrics.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Pushes each pm2 app's state to vmagent as Snow Globe service metrics.""" +import json +import subprocess +import time +from urllib.request import Request, urlopen + +while True: + apps = json.loads(subprocess.run(["pm2", "jlist"], capture_output=True, check=True, text=True).stdout) + lines = [] + for app in apps: + env, monit = app["pm2_env"], app.get("monit") or {} + labels = "{service=%s}" % json.dumps("vps." + app["name"]) + lines += [ + f"snowglobe_service_up{labels} {int(env['status'] == 'online')}", + f"snowglobe_service_cpu_cores{labels} {monit.get('cpu', 0) / 100}", + f"snowglobe_service_memory_bytes{labels} {monit.get('memory', 0)}", + f"snowglobe_service_restarts_total{labels} {env.get('restart_time', 0)}", + f"snowglobe_service_started_seconds{labels} {env.get('pm_uptime', 0) / 1000}", + ] + urlopen(Request("http://127.0.0.1:8429/api/v1/import/prometheus", data="\n".join(lines).encode() + b"\n"), timeout=10).read() + time.sleep(15) diff --git a/tools/vps/pm2-metrics.service b/tools/vps/pm2-metrics.service new file mode 100644 index 0000000000000000000000000000000000000000..fc499fd40656888e151186105f5777906cf9b7d2 --- /dev/null +++ b/tools/vps/pm2-metrics.service @@ -0,0 +1,14 @@ +[Unit] +Description=Report pm2 apps to vmagent +After=vmagent.service + +[Service] +User=clo +Environment=PATH=/home/clo/.local/bin:/usr/bin +ExecStart=/usr/bin/python3 /usr/local/lib/snowglobe-agents/pm2-metrics.py +Restart=always +RestartSec=15 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vlagent.service b/tools/vps/vlagent.service new file mode 100644 index 0000000000000000000000000000000000000000..a3aca78fade0e5684c43fa22867e25c8ff3759e7 --- /dev/null +++ b/tools/vps/vlagent.service @@ -0,0 +1,23 @@ +[Unit] +Description=Ship VPS logs to Snow Globe +Wants=network-online.target +After=network-online.target + +[Service] +LoadCredential=token:/etc/snowglobe-ingest.token +ExecStart=/usr/local/bin/vlagent -httpListenAddr=127.0.0.1:9429 -tmpDataPath=/var/lib/vlagent \ + -fileCollector.glob=/home/clo/.pm2/logs/*.log \ + -remoteWrite.url=https://ingest.paperclover.net/insert/jsonline -remoteWrite.format=jsonline \ + -remoteWrite.bearerTokenFile=%d/token -remoteWrite.maxDiskUsagePerURL=1GB +StateDirectory=vlagent +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +PrivateTmp=true +PrivateDevices=true +MemoryMax=256M + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vmagent.service b/tools/vps/vmagent.service new file mode 100644 index 0000000000000000000000000000000000000000..b50ea67a51bc35860f47db656d0d5392902ea2fd --- /dev/null +++ b/tools/vps/vmagent.service @@ -0,0 +1,25 @@ +[Unit] +Description=Ship VPS metrics to Snow Globe +Wants=network-online.target +After=network-online.target + +[Service] +DynamicUser=true +LoadCredential=token:/etc/snowglobe-ingest.token +ExecStart=/usr/local/bin/vmagent -httpListenAddr=127.0.0.1:8429 \ + -promscrape.config=/usr/local/lib/snowglobe-agents/vmagent.yml \ + -remoteWrite.url=https://ingest.paperclover.net/api/v1/write -remoteWrite.label=host=usw-server-1 \ + -remoteWrite.bearerTokenFile=%d/token \ + -remoteWrite.tmpDataPath=/var/lib/vmagent -remoteWrite.maxDiskUsagePerURL=1GB +StateDirectory=vmagent +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +MemoryMax=256M + +[Install] +WantedBy=multi-user.target diff --git a/tools/vps/vmagent.yml b/tools/vps/vmagent.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8baa0dc9b785046e86a2d7d6128b3f871c73677 --- /dev/null +++ b/tools/vps/vmagent.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 30s +scrape_configs: + - job_name: iroh-relay + static_configs: + - targets: ["127.0.0.1:9091"] + labels: { service: "vps.iroh-relay" } -- 2.54.1