authoravatarclover caruso<git@paperclover.net>2026-10-24 19:43:48-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:43:59-07:00
logca421c2098f96de8070bb7d17dd1d3aab3b47487
tree1e6e44c18413ce7e2bd6932558ef3213d8e4775d
parentdc8464162f442c4e404063c93a5221ae4d56b122
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Ship VPS logs and metrics to Snow Globe

A public ingest service takes vlagent and vmagent pushes with a bearer token and files them in Victoria under source=vps, naming each log line after its pm2 app or systemd unit. On the VPS, vlagent tails the pm2 logs and the journal, vmagent scrapes the iroh relay, and a small script reports pm2 state as service metrics. The dashboard lists these under "vps services" with the same logs, cpu, memory, and metrics views as Nomad services, keyed vps.<name>. Victoria logs gets a host port so containers on the bridge can reach it. Assisted-by: claude-opus-5.5

24 files changed, 480 insertions(+), 14 deletions(-)

dashboard/src/core.rs+7-5
......@@ -622,6 +622,7 @@ pub async fn route(
622622 json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null})
623623 }
624624 ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()),
625 ["vps"] if method == Method::GET => telemetry::vps(app).await?,
625626 ["launcher"] if method == Method::GET => {
626627 let found = launcher(app.clone()).await?;
627628 let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10));
......@@ -723,11 +724,12 @@ pub async fn route(
723724 .response());
724725 }
725726 ["services", id, "metrics", name] if method == Method::GET => {
726 let detail = service(app.clone(), id).await?;
727 if !array(&detail.value["applicationMetrics"])
728 .iter()
729 .any(|v| v == *name)
730 {
727 let names = if id.starts_with("vps.") {
728 telemetry::metric_names(app.clone(), id).await?
729 } else {
730 service(app.clone(), id).await?.value["applicationMetrics"].clone()
731 };
732 if !array(&names).iter().any(|v| v == *name) {
731733 return Err(Error::new(404, "Metric unavailable for this service."));
732734 }
733735 return Ok(telemetry::metrics(app, name, query, Some(id), None)
dashboard/src/main.rs+1-1
......@@ -279,7 +279,7 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response>
279279 "services" if parts.len() == 1 => Some("metrics"),
280280 "storage" if parts.len() == 1 => Some("metrics"),
281281 "ai" | "mcp" => Some("ai"),
282 "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => {
282 "services" | "vps" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => {
283283 Some("admin")
284284 }
285285 "media" | "seedbox" | "youtube" => Some("media"),
dashboard/src/telemetry.rs+53-3
......@@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> {
228228 (
229229 "match[]",
230230 format!(
231 "{{{label}={},__name__!~\"snowglobe_.*\"}}",
231 "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}",
232232 serde_json::to_string(id).unwrap()
233233 ),
234234 )
......@@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> {
248248 Ok(json!(values))
249249}
250250
251/// Services the VPS agents report, newest state first; quiet for two minutes counts as down.
252pub async fn vps(app: Arc<App>) -> Result<Value> {
253 let base = endpoint(app.clone(), "victoria-metrics").await?;
254 let instant = |query: &str| {
255 let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())]));
256 let app = app.clone();
257 async move {
258 let response = read(app, url, None).await?;
259 Ok::<_, Error>(
260 array(&response.value["data"]["result"])
261 .iter()
262 .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1])))
263 .collect::<HashMap<_, _>>(),
264 )
265 }
266 };
267 let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#;
268 let (known, current, cpu, memory, restarts, started) = futures::try_join!(
269 instant(&format!("max by (service) (max_over_time({up}[7d]))")),
270 instant(&format!("max by (service) (last_over_time({up}[2m]))")),
271 instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#),
272 instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#),
273 instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#),
274 instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#),
275 )?;
276 let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect();
277 ids.sort();
278 let mut services = Vec::new();
279 for id in ids {
280 services.push(json!({
281 "id": id,
282 "name": &id[4..],
283 "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" },
284 "cpu": cpu.get(&id),
285 "memory": memory.get(&id),
286 "restarts": restarts.get(&id),
287 "startedAt": started.get(&id).filter(|v| **v > 0.0),
288 "applicationMetrics": metric_names(app.clone(), &id).await?,
289 }));
290 }
291 Ok(json!(services))
292}
293
251294#[derive(Debug, PartialEq)]
252295struct Term {
253296 field: Option<String>,
......@@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> {
299342 Ok(found)
300343}
301344fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> {
345 let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job));
302346 let mut filters = vec![
303 "source:=nomad".to_owned(),
304 format!("job:={}", serde_json::to_string(id)?),
347 format!("source:={source}"),
348 format!("job:={}", serde_json::to_string(job)?),
305349 ];
306350 if let Some(level) = query.get("level") {
307351 filters.push(
......@@ -1001,6 +1045,12 @@ pub fn start(app: Arc<App>) {
10011045mod tests {
10021046 use super::*;
10031047 #[test]
1048 fn vps_ids_read_the_vps_source() {
1049 let none = HashMap::new();
1050 assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#);
1051 assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#);
1052 }
1053 #[test]
10041054 fn search_fields_comparisons_exclusions_and_unknown_fields() {
10051055 let parsed = terms(
10061056 r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#,
dashboard/web/api.contract.ts+4-1
......@@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts";
66import type { Hono } from "hono";
77import type { Health } from "./types/model.ts";
88import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts";
9import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts";
9import type { Me, ServiceSummary, ServiceDetail, VpsService, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts";
1010import type { MapNode } from "./types/storage.index.ts";
1111import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts";
1212import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts";
......@@ -102,6 +102,9 @@ export type Api = Hono<{}, {
102102 "/services": {
103103 $get: Endpoint<ServiceSummary[]>;
104104 };
105 "/vps": {
106 $get: Endpoint<VpsService[]>;
107 };
105108 "/services/:id": {
106109 $get: Endpoint<ServiceDetail, { param: { id: string; }; }>;
107110 };
dashboard/web/api.ts+1
......@@ -80,6 +80,7 @@ export const queries = {
8080 storage: query("storage", () => parseResponse(api.storage.$get())),
8181 deploys: query("deploys", () => parseResponse(api.deploys.$get())),
8282 vms: query("vms", () => parseResponse(api.vms.$get())),
83 vps: query("vps", () => parseResponse(api.vps.$get())),
8384 metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) =>
8485 parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))),
8586 seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())),
dashboard/web/components/Sidebar.tsx+33
......@@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house";
1010import LogOut from "lucide-solid/icons/log-out";
1111import Monitor from "lucide-solid/icons/monitor";
1212import Rocket from "lucide-solid/icons/rocket";
13import Server from "lucide-solid/icons/server";
1314import SquarePlay from "lucide-solid/icons/square-play";
1415import UserRound from "lucide-solid/icons/user-round";
1516import Plug from "lucide-solid/icons/plug";
......@@ -247,6 +248,37 @@ function Services() {
247248 );
248249}
249250
251function VpsServices() {
252 const [open, toggle] = remembered("sidebar.vps", true);
253 const [services, { refetch }] = queries.vps.use();
254 const timer = setInterval(() => !services.loading && refetch(), 15_000);
255 onCleanup(() => clearInterval(timer));
256 const loaded = lastGood(services);
257 const down = () => (loaded() ?? []).filter((service) => service.health !== "healthy").map((service) => service.name);
258 return (
259 <>
260 <button class="nav-item" onClick={toggle} aria-expanded={open()}>
261 <Server class="icon" />
262 <span class="label">vps services</span>
263 <span class="badge">
264 <Show when={down().length}>
265 <span class="tally" data-tip={down().join(", ")}><StatusLabel health="down">{down().length}</StatusLabel></span>
266 </Show>
267 </span>
268 <ChevronRight class={`chevron ${open() ? "open" : ""}`} />
269 </button>
270 <div class="nav-group" classList={{ open: open() }} inert={!open()}><div>
271 <For each={loaded()}>{(service) => (
272 <A href={`/vps/${service.name}`} class="nav-sub" activeClass="active">
273 <Status health={service.health} /><span class="label">{service.name}</span>
274 </A>
275 )}</For>
276 <Show when={services.error && !loaded()}><span class="nav-sub muted">VPS unavailable</span></Show>
277 </div></div>
278 </>
279 );
280}
281
250282/** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */
251283export function viewAs(groups: string[] | null) {
252284 document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`;
......@@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) {
310342 <For each={BEFORE.filter(allowed)}>{(page) => <NavLink page={page}><IssueCount /></NavLink>}</For>
311343 <Show when={props.me.sections.includes("admin")}>
312344 <Services />
345 <VpsServices />
313346 </Show>
314347 <For each={AFTER.filter(allowed)}>{(page) => page.section === "vms" ? <VMNav /> : <NavLink page={page} />}</For>
315348 </div>
dashboard/web/main.tsx+3-1
......@@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx";
2020import { Deploys } from "./pages/Deploys.tsx";
2121import { Deploy } from "./pages/Deploy.tsx";
2222import { VMs } from "./pages/VMs.tsx";
23import { Vps } from "./pages/Vps.tsx";
2324import { MCP } from "./pages/MCP.tsx";
2425import { MCPConsent } from "./pages/MCPConsent.tsx";
2526import { SignIn } from "./pages/SignIn.tsx";
......@@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) =
3435function Shell(props: RouteSectionProps) {
3536 const location = useLocation();
3637 const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request");
37 const section = () => location.pathname.startsWith("/services/") ? "admin"
38 const section = () => /^\/(services|vps)\//.test(location.pathname) ? "admin"
3839 : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section;
3940 return (
4041 <>
......@@ -78,6 +79,7 @@ render(() => (
7879 queries.service.preload(params.id!);
7980 queries.deploys.preload();
8081 }} />
82 <Route path="/vps/:name/:tab?" component={Vps} preload={preload(queries.vps)} />
8183 <Route path="/storage" component={Storage} preload={preload(queries.storage, queries.launcher)} />
8284 <Route path="/media" component={Media} preload={preload(queries.launcher)} />
8385 <Route path="/seedbox" component={Seedbox} preload={preload(queries.seedbox, queries.seedboxHistory, queries.launcher)} />
dashboard/web/pages/Service.tsx+2-2
......@@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con
4444type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string };
4545
4646/** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */
47function Logs(props: { service: ServiceDetail; tools: HTMLElement }) {
47export function Logs(props: { service: Pick<ServiceDetail, "id" | "containers" | "logs">; tools: HTMLElement }) {
4848 const [params, setParams] = useSearchParams<Query>();
4949 const at = () => (params.at ? Number(params.at) : null);
5050 const bounded = () => at() !== null || !!params.to;
......@@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er
272272const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } })));
273273const PAGE = 50;
274274
275function AppMetrics(props: { service: ServiceDetail; range: number }) {
275export function AppMetrics(props: { service: Pick<ServiceDetail, "id" | "applicationMetrics">; range: number }) {
276276 const [name, setName] = createSignal(props.service.applicationMetrics[0]!);
277277 const [data, { refetch }] = createResource(
278278 () => [props.service.id, name(), props.range] as const,
dashboard/web/pages/Vps.tsx created+90
......@@ -0,0 +1,90 @@
1import { A, useParams } from "@solidjs/router";
2import RotateCcw from "lucide-solid/icons/rotate-ccw";
3import Server from "lucide-solid/icons/server";
4import { createSignal, Match, onCleanup, Show, Switch } from "solid-js";
5import { queries } from "../api.ts";
6import { AppIcon } from "../components/AppIcon.tsx";
7import { ListPage } from "../components/ListPage.tsx";
8import { lastGood, Loaded } from "../components/Loaded.tsx";
9import { RangePicker } from "../components/Metric.tsx";
10import { StatusLabel } from "../components/Status.tsx";
11import { Strip } from "../components/Strip.tsx";
12import { TabBar } from "../components/TabBar.tsx";
13import { bytes, cores, datetime, duration, plural } from "../format.ts";
14import { AppMetrics, Logs } from "./Service.tsx";
15
16/** Remounts per app, so switching never shows the last one's logs. */
17export function Vps() {
18 const params = useParams<{ name: string }>();
19 return <Show when={params.name} keyed><VpsPage /></Show>;
20}
21
22function VpsPage() {
23 const params = useParams<{ name: string; tab?: string }>();
24 const [services, { refetch }] = queries.vps.use();
25 const timer = setInterval(refetch, 15_000);
26 onCleanup(() => clearInterval(timer));
27 const loaded = lastGood(services);
28 const [range, setRange] = createSignal(3600);
29 return (
30 <Loaded data={services} what="VPS services" retry={refetch} skeleton={<div class="page"><div class="skeleton" style={{ height: "112px" }} /></div>}>
31 {() => (
32 <Show when={loaded()?.find((service) => service.name === params.name)} fallback={
33 <div class="page empty">The VPS hasn't reported {params.name} in the last week.</div>
34 }>
35 {(data) => {
36 const tabHref = (tab: string) => `/vps/${data().name}${tab ? "/" + tab : ""}`;
37 let tools!: HTMLDivElement;
38 return (
39 <ListPage id="service" class="service-page" flush head={
40 <div class="page-head">
41 <span class="service-icon"><AppIcon id={data().id} name={data().name} icon={null} /></span>
42 <h1>{data().name}</h1>
43 <div class="stats">
44 <Show when={data().health === "healthy" && data().startedAt} fallback={
45 <span class="stat" role="status"><StatusLabel health={data().health} /></span>
46 }>
47 {(t) => (
48 <span class="stat" role="status" tabIndex={0} data-tip={`since ${datetime(t())}`}>
49 <StatusLabel health="healthy">up {duration(Date.now() / 1000 - t())}</StatusLabel>
50 </span>
51 )}
52 </Show>
53 <span class="stat"><Server size={13} aria-hidden="true" /><b>usw-server-1</b></span>
54 <Show when={data().restarts}>
55 {(restarts) => <span class="stat"><RotateCcw size={13} aria-hidden="true" />{plural(restarts(), "restart")}</span>}
56 </Show>
57 </div>
58 </div>
59 } summary={
60 <Show when={data().cpu !== null}>
61 <div class="service-strips">
62 <Strip title="cpu" metric="service.cpu" service={data().id} range={range()} format={cores} height={64}
63 tabs={<RangePicker value={range()} onChange={setRange} />}
64 now={() => ({ value: cores(data().cpu ?? 0) })} />
65 <Strip title="memory" metric="service.memory" service={data().id} range={range()} format={bytes}
66 height={64 + 24} last now={() => ({ value: bytes(data().memory ?? 0) })} />
67 </div>
68 </Show>
69 }>
70 <div ref={tools} class="tab-row">
71 <TabBar label="View">
72 <A href={tabHref("")} end>logs</A>
73 <Show when={data().applicationMetrics.length}><A href={tabHref("metrics")} end>metrics</A></Show>
74 </TabBar>
75 </div>
76 <Switch>
77 <Match when={!params.tab}><Logs service={{ id: data().id, containers: [], logs: null }} tools={tools} /></Match>
78 <Match when={params.tab === "metrics" && data().applicationMetrics.length}>
79 <AppMetrics service={data()} range={range()} />
80 </Match>
81 </Switch>
82 </ListPage>
83 );
84 }}
85 </Show>
86 )}
87 </Loaded>
88 );
89}
90
dashboard/web/types/model.ts+12
......@@ -51,6 +51,18 @@ export interface ServiceSummary {
5151 memoryLimit: number;
5252}
5353
54/** An app the VPS agents report, keyed `vps.<name>`; usage, restarts and start time are pm2's. */
55export interface VpsService {
56 id: string;
57 name: string;
58 health: Health;
59 cpu: number | null;
60 memory: number | null;
61 restarts: number | null;
62 startedAt: number | null;
63 applicationMetrics: string[];
64}
65
5466/** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */
5567export const trouble = (health: Health) => health === "down" || health === "degraded";
5668
service/ingest/build/Dockerfile created+3
......@@ -0,0 +1,3 @@
1FROM docker.io/library/python:3.14-slim@sha256:0214bda9af2652a418f2bf34373a4d3ca00a5f4784081e040049433ac9cc79d5
2COPY app.py /app/app.py
3CMD ["python3", "-u", "/app/app.py"]
service/ingest/build/app.py created+101
......@@ -0,0 +1,101 @@
1#!/usr/bin/env python3
2"""Files logs and metrics pushed by the VPS agents in Victoria under source=vps."""
3from compression import zstd
4import gzip
5import hmac
6import json
7import os
8import re
9import threading
10import time
11from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
12from urllib.error import HTTPError
13from urllib.request import Request, urlopen
14
15TOKEN = os.environ["INGEST_TOKEN"].encode()
16LOGS = os.environ["LOGS_URL"]
17METRICS = os.environ["METRICS_URL"]
18PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log")
19ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I)
20WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I)
21LIMIT = 32 << 20
22JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY")
23
24sequence_lock = threading.Lock()
25last_sequence = 0
26
27
28def next_sequence():
29 """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow."""
30 global last_sequence
31 with sequence_lock:
32 last_sequence = max(last_sequence + 1, time.time_ns() // 1000)
33 return last_sequence
34
35
36def shape(row):
37 if match := PM2.fullmatch(row.get("file", "")):
38 job, stream = match[1], "stdout" if match[2] == "out" else "stderr"
39 message = row.get("_msg", "")
40 level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info"
41 else:
42 job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal"
43 priority = int(row.get("PRIORITY") or 6)
44 row = {key: row[key] for key in JOURNAL if key in row}
45 level = "error" if priority <= 3 else "warn" if priority == 4 else "info"
46 row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream,
47 level=level, sequence=next_sequence())
48 return row
49
50
51def forward(url, body, headers):
52 with urlopen(Request(url, data=body, headers=headers), timeout=30) as response:
53 response.read()
54
55
56class Handler(BaseHTTPRequestHandler):
57 protocol_version = "HTTP/1.1"
58
59 def reply(self, status, text=""):
60 body = text.encode()
61 self.send_response(status)
62 self.send_header("Content-Length", str(len(body)))
63 self.end_headers()
64 self.wfile.write(body)
65
66 def do_GET(self):
67 self.reply(204 if self.path == "/health" else 404)
68
69 def do_POST(self):
70 length = int(self.headers.get("Content-Length") or 0)
71 if length > LIMIT:
72 self.close_connection = True
73 return self.reply(413, "batch too large")
74 body = self.rfile.read(length)
75 if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN):
76 return self.reply(401, "bad or missing token")
77 try:
78 if self.path == "/insert/jsonline":
79 encoding = self.headers.get("Content-Encoding", "")
80 text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body
81 rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()]
82 forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows),
83 {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"})
84 forward(LOGS + "/internal/force_flush", b"", {})
85 elif self.path == "/api/v1/write":
86 forward(METRICS + "/api/v1/write?extra_label=source=vps", body,
87 {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers})
88 else:
89 return self.reply(404)
90 except HTTPError as error:
91 return self.reply(502, f"victoria answered {error.code}")
92 except (OSError, ValueError, zstd.ZstdError) as error:
93 return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error))
94 self.reply(204)
95
96 def log_request(self, code="-", size="-"):
97 if not str(int(code)).startswith("2"):
98 super().log_request(code, size)
99
100
101ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever()
service/ingest/service.pkl created+30
......@@ -0,0 +1,30 @@
1amends "../../config/Service.pkl"
2
3name = "Ingest"
4tagline = "Logs and metrics pushed from the VPS"
5launcher = false
6dependsOn { "victoria" }
7
8secrets { ["token"] {} }
9
10container {
11 build = "build"
12 readOnlyRoot = true
13 cpu = 100
14 memory = 128
15
16 http {
17 containerPort = 8000
18 subdomain = "ingest"
19 checkPath = "/health"
20 }
21
22 env {
23 ["INGEST_TOKEN"] = "${secret.own.token}"
24 }
25
26 envTemplate = """
27 {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-logs" }}LOGS_URL=http://\(module.nomadHostPort){{ end }}
28 {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-metrics" }}METRICS_URL=http://\(module.nomadHostPort){{ end }}
29 """
30}
service/iroh-relay/iroh-relay.service+1
......@@ -6,6 +6,7 @@ After=network-online.target
66[Service]
77DynamicUser=true
88Environment=HTTP_PORT=3340
9Environment=METRICS_ADDR=127.0.0.1:9091
910LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem
1011LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem
1112ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay
service/iroh-relay/run.py+3-1
......@@ -36,8 +36,10 @@ if mode == "relay":
3636 raise RuntimeError("Relay certificate was not issued within three minutes")
3737 quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads(
3838 Path(__file__).with_name("network.json").read_text())["quic"]
39 metrics = os.environ.get("METRICS_ADDR")
3940 config.write_text(
40 'enable_metrics = false\nenable_quic_addr_discovery = true\n'
41 f'enable_metrics = {json.dumps(bool(metrics))}\nenable_quic_addr_discovery = true\n'
42 + (f'metrics_bind_addr = {json.dumps(metrics)}\n' if metrics else '') +
4143 f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n'
4244 '[tls]\ncert_mode = "Reloading"\n'
4345 f'quic_bind_addr = "0.0.0.0:{quic_port}"\n'
service/victoria/service.pkl+1
......@@ -11,6 +11,7 @@ containers {
1111
1212 http {
1313 containerPort = 9428
14 hostPort = 19428
1415 subdomain = "logs"
1516 authRole = "infra-admin"
1617 checkPath = "/health"
tools/iroh-relay.md+3
......@@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and
3333[`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into
3434`/etc/systemd/system/`.
3535
36The unit sets `METRICS_ADDR`, so the relay serves metrics on `127.0.0.1:9091` for
37[vmagent](vps.md).
38
3639[`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as
3740`/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main
3841Caddyfile. Validate before reloading Caddy. The ACME webroot is
tools/vps.md created+39
......@@ -0,0 +1,39 @@
1# VPS logs and metrics
2
3The VPS (`ssh root@paperclover.net`, hostname `usw-server-1`) pushes its logs and
4metrics to Snow Globe's [ingest](../service/ingest/service.pkl) service at
5`https://ingest.paperclover.net`, which files them in Victoria under `source=vps`.
6The dashboard lists them under **vps services**; their ids are `vps.<name>`.
7
8| Agent | Collects | Sends |
9| --- | --- | --- |
10| [vlagent](vps/vlagent.service) | `~clo/.pm2/logs/*.log`, plus the journal through `systemd-journal-upload` | `/insert/jsonline` |
11| [vmagent](vps/vmagent.service) | the iroh relay's metrics on `127.0.0.1:9091`, plus [pm2-metrics.py](vps/pm2-metrics.py) pushes | `/api/v1/write` |
12
13Ingest names each log line's `job` from its pm2 file (`site-out.log` is `site` on
14stdout) or its systemd unit. pm2 writes no timestamps, so a line's time is when
15vlagent read it. Both agents buffer up to 1 GB on disk while Snow Globe is
16unreachable.
17
18```logsql
19source:=vps job:=site level:=error
20```
21
22## Install
23
24Agents are the VictoriaMetrics release binaries, checksum-verified:
25
26| Binary | Release tarball | SHA256 |
27| --- | --- | --- |
28| `/usr/local/bin/vlagent` | `vlutils-linux-amd64-v1.52.0.tar.gz` | `caa315764df0d11a77ba61d4ee96b783ddb98455b167da8326940dda62914bc9` |
29| `/usr/local/bin/vmagent` | `vmutils-linux-amd64-v1.152.0.tar.gz` | `8eee4a98ff1665c60682475e8a8b292b8d718b63a2f023124384dd2f6a220c79` |
30
31The ingest service's generated `token` secret lives in `/etc/snowglobe-ingest.token`
32(root, mode 600). `vmagent.yml` and `pm2-metrics.py` go in
33`/usr/local/lib/snowglobe-agents/`, the units in `/etc/systemd/system/`, and
34[journal-upload.conf](vps/journal-upload.conf) in
35`/etc/systemd/journal-upload.conf.d/snowglobe.conf` (package
36`systemd-journal-remote`). Seed `/var/lib/systemd/journal-upload/state` with
37`LAST_CURSOR=` and the current cursor, and run vlagent once against a sink until
38its checkpoints reach the ends of the pm2 files; otherwise both send their whole
39history, and the pm2 lines all get the same time.
tools/vps/journal-upload.conf created+2
......@@ -0,0 +1,2 @@
1[Upload]
2URL=http://127.0.0.1:9429/insert/journald
tools/vps/pm2-metrics.py created+22
......@@ -0,0 +1,22 @@
1#!/usr/bin/env python3
2"""Pushes each pm2 app's state to vmagent as Snow Globe service metrics."""
3import json
4import subprocess
5import time
6from urllib.request import Request, urlopen
7
8while True:
9 apps = json.loads(subprocess.run(["pm2", "jlist"], capture_output=True, check=True, text=True).stdout)
10 lines = []
11 for app in apps:
12 env, monit = app["pm2_env"], app.get("monit") or {}
13 labels = "{service=%s}" % json.dumps("vps." + app["name"])
14 lines += [
15 f"snowglobe_service_up{labels} {int(env['status'] == 'online')}",
16 f"snowglobe_service_cpu_cores{labels} {monit.get('cpu', 0) / 100}",
17 f"snowglobe_service_memory_bytes{labels} {monit.get('memory', 0)}",
18 f"snowglobe_service_restarts_total{labels} {env.get('restart_time', 0)}",
19 f"snowglobe_service_started_seconds{labels} {env.get('pm_uptime', 0) / 1000}",
20 ]
21 urlopen(Request("http://127.0.0.1:8429/api/v1/import/prometheus", data="\n".join(lines).encode() + b"\n"), timeout=10).read()
22 time.sleep(15)
tools/vps/pm2-metrics.service created+14
......@@ -0,0 +1,14 @@
1[Unit]
2Description=Report pm2 apps to vmagent
3After=vmagent.service
4
5[Service]
6User=clo
7Environment=PATH=/home/clo/.local/bin:/usr/bin
8ExecStart=/usr/bin/python3 /usr/local/lib/snowglobe-agents/pm2-metrics.py
9Restart=always
10RestartSec=15
11NoNewPrivileges=true
12
13[Install]
14WantedBy=multi-user.target
tools/vps/vlagent.service created+23
......@@ -0,0 +1,23 @@
1[Unit]
2Description=Ship VPS logs to Snow Globe
3Wants=network-online.target
4After=network-online.target
5
6[Service]
7LoadCredential=token:/etc/snowglobe-ingest.token
8ExecStart=/usr/local/bin/vlagent -httpListenAddr=127.0.0.1:9429 -tmpDataPath=/var/lib/vlagent \
9 -fileCollector.glob=/home/clo/.pm2/logs/*.log \
10 -remoteWrite.url=https://ingest.paperclover.net/insert/jsonline -remoteWrite.format=jsonline \
11 -remoteWrite.bearerTokenFile=%d/token -remoteWrite.maxDiskUsagePerURL=1GB
12StateDirectory=vlagent
13Restart=always
14RestartSec=5
15NoNewPrivileges=true
16ProtectSystem=strict
17ProtectHome=read-only
18PrivateTmp=true
19PrivateDevices=true
20MemoryMax=256M
21
22[Install]
23WantedBy=multi-user.target
tools/vps/vmagent.service created+25
......@@ -0,0 +1,25 @@
1[Unit]
2Description=Ship VPS metrics to Snow Globe
3Wants=network-online.target
4After=network-online.target
5
6[Service]
7DynamicUser=true
8LoadCredential=token:/etc/snowglobe-ingest.token
9ExecStart=/usr/local/bin/vmagent -httpListenAddr=127.0.0.1:8429 \
10 -promscrape.config=/usr/local/lib/snowglobe-agents/vmagent.yml \
11 -remoteWrite.url=https://ingest.paperclover.net/api/v1/write -remoteWrite.label=host=usw-server-1 \
12 -remoteWrite.bearerTokenFile=%d/token \
13 -remoteWrite.tmpDataPath=/var/lib/vmagent -remoteWrite.maxDiskUsagePerURL=1GB
14StateDirectory=vmagent
15Restart=always
16RestartSec=5
17NoNewPrivileges=true
18ProtectSystem=strict
19ProtectHome=true
20PrivateTmp=true
21PrivateDevices=true
22MemoryMax=256M
23
24[Install]
25WantedBy=multi-user.target
tools/vps/vmagent.yml created+7
......@@ -0,0 +1,7 @@
1global:
2 scrape_interval: 30s
3scrape_configs:
4 - job_name: iroh-relay
5 static_configs:
6 - targets: ["127.0.0.1:9091"]
7 labels: { service: "vps.iroh-relay" }