| author | |
| committer | |
| log | ca421c2098f96de8070bb7d17dd1d3aab3b47487 |
| tree | 1e6e44c18413ce7e2bd6932558ef3213d8e4775d |
| parent | dc8464162f442c4e404063c93a5221ae4d56b122 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A public ingest service takes vlagent and vmagent pushes with a bearer token
and files them in Victoria under source=vps, naming each log line after its
pm2 app or systemd unit. On the VPS, vlagent tails the pm2 logs and the
journal, vmagent scrapes the iroh relay, and a small script reports pm2 state
as service metrics.
The dashboard lists these under "vps services" with the same logs, cpu,
memory, and metrics views as Nomad services, keyed vps.<name>.
Victoria logs gets a host port so containers on the bridge can reach it.
Assisted-by: claude-opus-5.524 files changed, 480 insertions(+), 14 deletions(-)
dashboard/src/core.rs+7-5| ... | ... | @@ -622,6 +622,7 @@ pub async fn route( |
| 622 | 622 | json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null}) |
| 623 | 623 | } |
| 624 | 624 | ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()), |
| 625 | ["vps"] if method == Method::GET => telemetry::vps(app).await?, | |
| 625 | 626 | ["launcher"] if method == Method::GET => { |
| 626 | 627 | let found = launcher(app.clone()).await?; |
| 627 | 628 | let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10)); |
| ... | ... | @@ -723,11 +724,12 @@ pub async fn route( |
| 723 | 724 | .response()); |
| 724 | 725 | } |
| 725 | 726 | ["services", id, "metrics", name] if method == Method::GET => { |
| 726 | let detail = service(app.clone(), id).await?; | |
| 727 | if !array(&detail.value["applicationMetrics"]) | |
| 728 | .iter() | |
| 729 | .any(|v| v == *name) | |
| 730 | { | |
| 727 | let names = if id.starts_with("vps.") { | |
| 728 | telemetry::metric_names(app.clone(), id).await? | |
| 729 | } else { | |
| 730 | service(app.clone(), id).await?.value["applicationMetrics"].clone() | |
| 731 | }; | |
| 732 | if !array(&names).iter().any(|v| v == *name) { | |
| 731 | 733 | return Err(Error::new(404, "Metric unavailable for this service.")); |
| 732 | 734 | } |
| 733 | 735 | return Ok(telemetry::metrics(app, name, query, Some(id), None) |
dashboard/src/main.rs+1-1| ... | ... | @@ -279,7 +279,7 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response> |
| 279 | 279 | "services" if parts.len() == 1 => Some("metrics"), |
| 280 | 280 | "storage" if parts.len() == 1 => Some("metrics"), |
| 281 | 281 | "ai" | "mcp" => Some("ai"), |
| 282 | "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { | |
| 282 | "services" | "vps" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { | |
| 283 | 283 | Some("admin") |
| 284 | 284 | } |
| 285 | 285 | "media" | "seedbox" | "youtube" => Some("media"), |
dashboard/src/telemetry.rs+53-3| ... | ... | @@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { |
| 228 | 228 | ( |
| 229 | 229 | "match[]", |
| 230 | 230 | format!( |
| 231 | "{{{label}={},__name__!~\"snowglobe_.*\"}}", | |
| 231 | "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}", | |
| 232 | 232 | serde_json::to_string(id).unwrap() |
| 233 | 233 | ), |
| 234 | 234 | ) |
| ... | ... | @@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { |
| 248 | 248 | Ok(json!(values)) |
| 249 | 249 | } |
| 250 | 250 | |
| 251 | /// Services the VPS agents report, newest state first; quiet for two minutes counts as down. | |
| 252 | pub async fn vps(app: Arc<App>) -> Result<Value> { | |
| 253 | let base = endpoint(app.clone(), "victoria-metrics").await?; | |
| 254 | let instant = |query: &str| { | |
| 255 | let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())])); | |
| 256 | let app = app.clone(); | |
| 257 | async move { | |
| 258 | let response = read(app, url, None).await?; | |
| 259 | Ok::<_, Error>( | |
| 260 | array(&response.value["data"]["result"]) | |
| 261 | .iter() | |
| 262 | .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1]))) | |
| 263 | .collect::<HashMap<_, _>>(), | |
| 264 | ) | |
| 265 | } | |
| 266 | }; | |
| 267 | let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#; | |
| 268 | let (known, current, cpu, memory, restarts, started) = futures::try_join!( | |
| 269 | instant(&format!("max by (service) (max_over_time({up}[7d]))")), | |
| 270 | instant(&format!("max by (service) (last_over_time({up}[2m]))")), | |
| 271 | instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#), | |
| 272 | instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#), | |
| 273 | instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#), | |
| 274 | instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#), | |
| 275 | )?; | |
| 276 | let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect(); | |
| 277 | ids.sort(); | |
| 278 | let mut services = Vec::new(); | |
| 279 | for id in ids { | |
| 280 | services.push(json!({ | |
| 281 | "id": id, | |
| 282 | "name": &id[4..], | |
| 283 | "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" }, | |
| 284 | "cpu": cpu.get(&id), | |
| 285 | "memory": memory.get(&id), | |
| 286 | "restarts": restarts.get(&id), | |
| 287 | "startedAt": started.get(&id).filter(|v| **v > 0.0), | |
| 288 | "applicationMetrics": metric_names(app.clone(), &id).await?, | |
| 289 | })); | |
| 290 | } | |
| 291 | Ok(json!(services)) | |
| 292 | } | |
| 293 | ||
| 251 | 294 | #[derive(Debug, PartialEq)] |
| 252 | 295 | struct Term { |
| 253 | 296 | field: Option<String>, |
| ... | ... | @@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> { |
| 299 | 342 | Ok(found) |
| 300 | 343 | } |
| 301 | 344 | fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> { |
| 345 | let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job)); | |
| 302 | 346 | let mut filters = vec![ |
| 303 | "source:=nomad".to_owned(), | |
| 304 | format!("job:={}", serde_json::to_string(id)?), | |
| 347 | format!("source:={source}"), | |
| 348 | format!("job:={}", serde_json::to_string(job)?), | |
| 305 | 349 | ]; |
| 306 | 350 | if let Some(level) = query.get("level") { |
| 307 | 351 | filters.push( |
| ... | ... | @@ -1001,6 +1045,12 @@ pub fn start(app: Arc<App>) { |
| 1001 | 1045 | mod tests { |
| 1002 | 1046 | use super::*; |
| 1003 | 1047 | #[test] |
| 1048 | fn vps_ids_read_the_vps_source() { | |
| 1049 | let none = HashMap::new(); | |
| 1050 | assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#); | |
| 1051 | assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#); | |
| 1052 | } | |
| 1053 | #[test] | |
| 1004 | 1054 | fn search_fields_comparisons_exclusions_and_unknown_fields() { |
| 1005 | 1055 | let parsed = terms( |
| 1006 | 1056 | r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#, |
dashboard/web/api.contract.ts+4-1| ... | ... | @@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts"; |
| 6 | 6 | import type { Hono } from "hono"; |
| 7 | 7 | import type { Health } from "./types/model.ts"; |
| 8 | 8 | import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; |
| 9 | import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; | |
| 9 | import type { Me, ServiceSummary, ServiceDetail, VpsService, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; | |
| 10 | 10 | import type { MapNode } from "./types/storage.index.ts"; |
| 11 | 11 | import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; |
| 12 | 12 | import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; |
| ... | ... | @@ -102,6 +102,9 @@ export type Api = Hono<{}, { |
| 102 | 102 | "/services": { |
| 103 | 103 | $get: Endpoint<ServiceSummary[]>; |
| 104 | 104 | }; |
| 105 | "/vps": { | |
| 106 | $get: Endpoint<VpsService[]>; | |
| 107 | }; | |
| 105 | 108 | "/services/:id": { |
| 106 | 109 | $get: Endpoint<ServiceDetail, { param: { id: string; }; }>; |
| 107 | 110 | }; |
dashboard/web/api.ts+1| ... | ... | @@ -80,6 +80,7 @@ export const queries = { |
| 80 | 80 | storage: query("storage", () => parseResponse(api.storage.$get())), |
| 81 | 81 | deploys: query("deploys", () => parseResponse(api.deploys.$get())), |
| 82 | 82 | vms: query("vms", () => parseResponse(api.vms.$get())), |
| 83 | vps: query("vps", () => parseResponse(api.vps.$get())), | |
| 83 | 84 | metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) => |
| 84 | 85 | parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))), |
| 85 | 86 | seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())), |
dashboard/web/components/Sidebar.tsx+33| ... | ... | @@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house"; |
| 10 | 10 | import LogOut from "lucide-solid/icons/log-out"; |
| 11 | 11 | import Monitor from "lucide-solid/icons/monitor"; |
| 12 | 12 | import Rocket from "lucide-solid/icons/rocket"; |
| 13 | import Server from "lucide-solid/icons/server"; | |
| 13 | 14 | import SquarePlay from "lucide-solid/icons/square-play"; |
| 14 | 15 | import UserRound from "lucide-solid/icons/user-round"; |
| 15 | 16 | import Plug from "lucide-solid/icons/plug"; |
| ... | ... | @@ -247,6 +248,37 @@ function Services() { |
| 247 | 248 | ); |
| 248 | 249 | } |
| 249 | 250 | |
| 251 | function VpsServices() { | |
| 252 | const [open, toggle] = remembered("sidebar.vps", true); | |
| 253 | const [services, { refetch }] = queries.vps.use(); | |
| 254 | const timer = setInterval(() => !services.loading && refetch(), 15_000); | |
| 255 | onCleanup(() => clearInterval(timer)); | |
| 256 | const loaded = lastGood(services); | |
| 257 | const down = () => (loaded() ?? []).filter((service) => service.health !== "healthy").map((service) => service.name); | |
| 258 | return ( | |
| 259 | <> | |
| 260 | <button class="nav-item" onClick={toggle} aria-expanded={open()}> | |
| 261 | <Server class="icon" /> | |
| 262 | <span class="label">vps services</span> | |
| 263 | <span class="badge"> | |
| 264 | <Show when={down().length}> | |
| 265 | <span class="tally" data-tip={down().join(", ")}><StatusLabel health="down">{down().length}</StatusLabel></span> | |
| 266 | </Show> | |
| 267 | </span> | |
| 268 | <ChevronRight class={`chevron ${open() ? "open" : ""}`} /> | |
| 269 | </button> | |
| 270 | <div class="nav-group" classList={{ open: open() }} inert={!open()}><div> | |
| 271 | <For each={loaded()}>{(service) => ( | |
| 272 | <A href={`/vps/${service.name}`} class="nav-sub" activeClass="active"> | |
| 273 | <Status health={service.health} /><span class="label">{service.name}</span> | |
| 274 | </A> | |
| 275 | )}</For> | |
| 276 | <Show when={services.error && !loaded()}><span class="nav-sub muted">VPS unavailable</span></Show> | |
| 277 | </div></div> | |
| 278 | </> | |
| 279 | ); | |
| 280 | } | |
| 281 | ||
| 250 | 282 | /** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */ |
| 251 | 283 | export function viewAs(groups: string[] | null) { |
| 252 | 284 | document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`; |
| ... | ... | @@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) { |
| 310 | 342 | <For each={BEFORE.filter(allowed)}>{(page) => <NavLink page={page}><IssueCount /></NavLink>}</For> |
| 311 | 343 | <Show when={props.me.sections.includes("admin")}> |
| 312 | 344 | <Services /> |
| 345 | <VpsServices /> | |
| 313 | 346 | </Show> |
| 314 | 347 | <For each={AFTER.filter(allowed)}>{(page) => page.section === "vms" ? <VMNav /> : <NavLink page={page} />}</For> |
| 315 | 348 | </div> |
dashboard/web/main.tsx+3-1| ... | ... | @@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx"; |
| 20 | 20 | import { Deploys } from "./pages/Deploys.tsx"; |
| 21 | 21 | import { Deploy } from "./pages/Deploy.tsx"; |
| 22 | 22 | import { VMs } from "./pages/VMs.tsx"; |
| 23 | import { Vps } from "./pages/Vps.tsx"; | |
| 23 | 24 | import { MCP } from "./pages/MCP.tsx"; |
| 24 | 25 | import { MCPConsent } from "./pages/MCPConsent.tsx"; |
| 25 | 26 | import { SignIn } from "./pages/SignIn.tsx"; |
| ... | ... | @@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) = |
| 34 | 35 | function Shell(props: RouteSectionProps) { |
| 35 | 36 | const location = useLocation(); |
| 36 | 37 | const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request"); |
| 37 | const section = () => location.pathname.startsWith("/services/") ? "admin" | |
| 38 | const section = () => /^\/(services|vps)\//.test(location.pathname) ? "admin" | |
| 38 | 39 | : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section; |
| 39 | 40 | return ( |
| 40 | 41 | <> |
| ... | ... | @@ -78,6 +79,7 @@ render(() => ( |
| 78 | 79 | queries.service.preload(params.id!); |
| 79 | 80 | queries.deploys.preload(); |
| 80 | 81 | }} /> |
| 82 | <Route path="/vps/:name/:tab?" component={Vps} preload={preload(queries.vps)} /> | |
| 81 | 83 | <Route path="/storage" component={Storage} preload={preload(queries.storage, queries.launcher)} /> |
| 82 | 84 | <Route path="/media" component={Media} preload={preload(queries.launcher)} /> |
| 83 | 85 | <Route path="/seedbox" component={Seedbox} preload={preload(queries.seedbox, queries.seedboxHistory, queries.launcher)} /> |
dashboard/web/pages/Service.tsx+2-2| ... | ... | @@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con |
| 44 | 44 | type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; |
| 45 | 45 | |
| 46 | 46 | /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ |
| 47 | function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | |
| 47 | export function Logs(props: { service: Pick<ServiceDetail, "id" | "containers" | "logs">; tools: HTMLElement }) { | |
| 48 | 48 | const [params, setParams] = useSearchParams<Query>(); |
| 49 | 49 | const at = () => (params.at ? Number(params.at) : null); |
| 50 | 50 | const bounded = () => at() !== null || !!params.to; |
| ... | ... | @@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er |
| 272 | 272 | const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } }))); |
| 273 | 273 | const PAGE = 50; |
| 274 | 274 | |
| 275 | function AppMetrics(props: { service: ServiceDetail; range: number }) { | |
| 275 | export function AppMetrics(props: { service: Pick<ServiceDetail, "id" | "applicationMetrics">; range: number }) { | |
| 276 | 276 | const [name, setName] = createSignal(props.service.applicationMetrics[0]!); |
| 277 | 277 | const [data, { refetch }] = createResource( |
| 278 | 278 | () => [props.service.id, name(), props.range] as const, |
dashboard/web/pages/Vps.tsx created+90| ... | ... | @@ -0,0 +1,90 @@ |
| 1 | import { A, useParams } from "@solidjs/router"; | |
| 2 | import RotateCcw from "lucide-solid/icons/rotate-ccw"; | |
| 3 | import Server from "lucide-solid/icons/server"; | |
| 4 | import { createSignal, Match, onCleanup, Show, Switch } from "solid-js"; | |
| 5 | import { queries } from "../api.ts"; | |
| 6 | import { AppIcon } from "../components/AppIcon.tsx"; | |
| 7 | import { ListPage } from "../components/ListPage.tsx"; | |
| 8 | import { lastGood, Loaded } from "../components/Loaded.tsx"; | |
| 9 | import { RangePicker } from "../components/Metric.tsx"; | |
| 10 | import { StatusLabel } from "../components/Status.tsx"; | |
| 11 | import { Strip } from "../components/Strip.tsx"; | |
| 12 | import { TabBar } from "../components/TabBar.tsx"; | |
| 13 | import { bytes, cores, datetime, duration, plural } from "../format.ts"; | |
| 14 | import { AppMetrics, Logs } from "./Service.tsx"; | |
| 15 | ||
| 16 | /** Remounts per app, so switching never shows the last one's logs. */ | |
| 17 | export function Vps() { | |
| 18 | const params = useParams<{ name: string }>(); | |
| 19 | return <Show when={params.name} keyed><VpsPage /></Show>; | |
| 20 | } | |
| 21 | ||
| 22 | function VpsPage() { | |
| 23 | const params = useParams<{ name: string; tab?: string }>(); | |
| 24 | const [services, { refetch }] = queries.vps.use(); | |
| 25 | const timer = setInterval(refetch, 15_000); | |
| 26 | onCleanup(() => clearInterval(timer)); | |
| 27 | const loaded = lastGood(services); | |
| 28 | const [range, setRange] = createSignal(3600); | |
| 29 | return ( | |
| 30 | <Loaded data={services} what="VPS services" retry={refetch} skeleton={<div class="page"><div class="skeleton" style={{ height: "112px" }} /></div>}> | |
| 31 | {() => ( | |
| 32 | <Show when={loaded()?.find((service) => service.name === params.name)} fallback={ | |
| 33 | <div class="page empty">The VPS hasn't reported {params.name} in the last week.</div> | |
| 34 | }> | |
| 35 | {(data) => { | |
| 36 | const tabHref = (tab: string) => `/vps/${data().name}${tab ? "/" + tab : ""}`; | |
| 37 | let tools!: HTMLDivElement; | |
| 38 | return ( | |
| 39 | <ListPage id="service" class="service-page" flush head={ | |
| 40 | <div class="page-head"> | |
| 41 | <span class="service-icon"><AppIcon id={data().id} name={data().name} icon={null} /></span> | |
| 42 | <h1>{data().name}</h1> | |
| 43 | <div class="stats"> | |
| 44 | <Show when={data().health === "healthy" && data().startedAt} fallback={ | |
| 45 | <span class="stat" role="status"><StatusLabel health={data().health} /></span> | |
| 46 | }> | |
| 47 | {(t) => ( | |
| 48 | <span class="stat" role="status" tabIndex={0} data-tip={`since ${datetime(t())}`}> | |
| 49 | <StatusLabel health="healthy">up {duration(Date.now() / 1000 - t())}</StatusLabel> | |
| 50 | </span> | |
| 51 | )} | |
| 52 | </Show> | |
| 53 | <span class="stat"><Server size={13} aria-hidden="true" /><b>usw-server-1</b></span> | |
| 54 | <Show when={data().restarts}> | |
| 55 | {(restarts) => <span class="stat"><RotateCcw size={13} aria-hidden="true" />{plural(restarts(), "restart")}</span>} | |
| 56 | </Show> | |
| 57 | </div> | |
| 58 | </div> | |
| 59 | } summary={ | |
| 60 | <Show when={data().cpu !== null}> | |
| 61 | <div class="service-strips"> | |
| 62 | <Strip title="cpu" metric="service.cpu" service={data().id} range={range()} format={cores} height={64} | |
| 63 | tabs={<RangePicker value={range()} onChange={setRange} />} | |
| 64 | now={() => ({ value: cores(data().cpu ?? 0) })} /> | |
| 65 | <Strip title="memory" metric="service.memory" service={data().id} range={range()} format={bytes} | |
| 66 | height={64 + 24} last now={() => ({ value: bytes(data().memory ?? 0) })} /> | |
| 67 | </div> | |
| 68 | </Show> | |
| 69 | }> | |
| 70 | <div ref={tools} class="tab-row"> | |
| 71 | <TabBar label="View"> | |
| 72 | <A href={tabHref("")} end>logs</A> | |
| 73 | <Show when={data().applicationMetrics.length}><A href={tabHref("metrics")} end>metrics</A></Show> | |
| 74 | </TabBar> | |
| 75 | </div> | |
| 76 | <Switch> | |
| 77 | <Match when={!params.tab}><Logs service={{ id: data().id, containers: [], logs: null }} tools={tools} /></Match> | |
| 78 | <Match when={params.tab === "metrics" && data().applicationMetrics.length}> | |
| 79 | <AppMetrics service={data()} range={range()} /> | |
| 80 | </Match> | |
| 81 | </Switch> | |
| 82 | </ListPage> | |
| 83 | ); | |
| 84 | }} | |
| 85 | </Show> | |
| 86 | )} | |
| 87 | </Loaded> | |
| 88 | ); | |
| 89 | } | |
| 90 |
dashboard/web/types/model.ts+12| ... | ... | @@ -51,6 +51,18 @@ export interface ServiceSummary { |
| 51 | 51 | memoryLimit: number; |
| 52 | 52 | } |
| 53 | 53 | |
| 54 | /** An app the VPS agents report, keyed `vps.<name>`; usage, restarts and start time are pm2's. */ | |
| 55 | export interface VpsService { | |
| 56 | id: string; | |
| 57 | name: string; | |
| 58 | health: Health; | |
| 59 | cpu: number | null; | |
| 60 | memory: number | null; | |
| 61 | restarts: number | null; | |
| 62 | startedAt: number | null; | |
| 63 | applicationMetrics: string[]; | |
| 64 | } | |
| 65 | ||
| 54 | 66 | /** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */ |
| 55 | 67 | export const trouble = (health: Health) => health === "down" || health === "degraded"; |
| 56 | 68 |
service/ingest/build/Dockerfile created+3| ... | ... | @@ -0,0 +1,3 @@ |
| 1 | FROM docker.io/library/python:3.14-slim@sha256:0214bda9af2652a418f2bf34373a4d3ca00a5f4784081e040049433ac9cc79d5 | |
| 2 | COPY app.py /app/app.py | |
| 3 | CMD ["python3", "-u", "/app/app.py"] |
service/ingest/build/app.py created+101| ... | ... | @@ -0,0 +1,101 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | """Files logs and metrics pushed by the VPS agents in Victoria under source=vps.""" | |
| 3 | from compression import zstd | |
| 4 | import gzip | |
| 5 | import hmac | |
| 6 | import json | |
| 7 | import os | |
| 8 | import re | |
| 9 | import threading | |
| 10 | import time | |
| 11 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| 12 | from urllib.error import HTTPError | |
| 13 | from urllib.request import Request, urlopen | |
| 14 | ||
| 15 | TOKEN = os.environ["INGEST_TOKEN"].encode() | |
| 16 | LOGS = os.environ["LOGS_URL"] | |
| 17 | METRICS = os.environ["METRICS_URL"] | |
| 18 | PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log") | |
| 19 | ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I) | |
| 20 | WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I) | |
| 21 | LIMIT = 32 << 20 | |
| 22 | JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY") | |
| 23 | ||
| 24 | sequence_lock = threading.Lock() | |
| 25 | last_sequence = 0 | |
| 26 | ||
| 27 | ||
| 28 | def next_sequence(): | |
| 29 | """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow.""" | |
| 30 | global last_sequence | |
| 31 | with sequence_lock: | |
| 32 | last_sequence = max(last_sequence + 1, time.time_ns() // 1000) | |
| 33 | return last_sequence | |
| 34 | ||
| 35 | ||
| 36 | def shape(row): | |
| 37 | if match := PM2.fullmatch(row.get("file", "")): | |
| 38 | job, stream = match[1], "stdout" if match[2] == "out" else "stderr" | |
| 39 | message = row.get("_msg", "") | |
| 40 | level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info" | |
| 41 | else: | |
| 42 | job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal" | |
| 43 | priority = int(row.get("PRIORITY") or 6) | |
| 44 | row = {key: row[key] for key in JOURNAL if key in row} | |
| 45 | level = "error" if priority <= 3 else "warn" if priority == 4 else "info" | |
| 46 | row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream, | |
| 47 | level=level, sequence=next_sequence()) | |
| 48 | return row | |
| 49 | ||
| 50 | ||
| 51 | def forward(url, body, headers): | |
| 52 | with urlopen(Request(url, data=body, headers=headers), timeout=30) as response: | |
| 53 | response.read() | |
| 54 | ||
| 55 | ||
| 56 | class Handler(BaseHTTPRequestHandler): | |
| 57 | protocol_version = "HTTP/1.1" | |
| 58 | ||
| 59 | def reply(self, status, text=""): | |
| 60 | body = text.encode() | |
| 61 | self.send_response(status) | |
| 62 | self.send_header("Content-Length", str(len(body))) | |
| 63 | self.end_headers() | |
| 64 | self.wfile.write(body) | |
| 65 | ||
| 66 | def do_GET(self): | |
| 67 | self.reply(204 if self.path == "/health" else 404) | |
| 68 | ||
| 69 | def do_POST(self): | |
| 70 | length = int(self.headers.get("Content-Length") or 0) | |
| 71 | if length > LIMIT: | |
| 72 | self.close_connection = True | |
| 73 | return self.reply(413, "batch too large") | |
| 74 | body = self.rfile.read(length) | |
| 75 | if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN): | |
| 76 | return self.reply(401, "bad or missing token") | |
| 77 | try: | |
| 78 | if self.path == "/insert/jsonline": | |
| 79 | encoding = self.headers.get("Content-Encoding", "") | |
| 80 | text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body | |
| 81 | rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()] | |
| 82 | forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows), | |
| 83 | {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"}) | |
| 84 | forward(LOGS + "/internal/force_flush", b"", {}) | |
| 85 | elif self.path == "/api/v1/write": | |
| 86 | forward(METRICS + "/api/v1/write?extra_label=source=vps", body, | |
| 87 | {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers}) | |
| 88 | else: | |
| 89 | return self.reply(404) | |
| 90 | except HTTPError as error: | |
| 91 | return self.reply(502, f"victoria answered {error.code}") | |
| 92 | except (OSError, ValueError, zstd.ZstdError) as error: | |
| 93 | return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error)) | |
| 94 | self.reply(204) | |
| 95 | ||
| 96 | def log_request(self, code="-", size="-"): | |
| 97 | if not str(int(code)).startswith("2"): | |
| 98 | super().log_request(code, size) | |
| 99 | ||
| 100 | ||
| 101 | ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() |
service/ingest/service.pkl created+30| ... | ... | @@ -0,0 +1,30 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | name = "Ingest" | |
| 4 | tagline = "Logs and metrics pushed from the VPS" | |
| 5 | launcher = false | |
| 6 | dependsOn { "victoria" } | |
| 7 | ||
| 8 | secrets { ["token"] {} } | |
| 9 | ||
| 10 | container { | |
| 11 | build = "build" | |
| 12 | readOnlyRoot = true | |
| 13 | cpu = 100 | |
| 14 | memory = 128 | |
| 15 | ||
| 16 | http { | |
| 17 | containerPort = 8000 | |
| 18 | subdomain = "ingest" | |
| 19 | checkPath = "/health" | |
| 20 | } | |
| 21 | ||
| 22 | env { | |
| 23 | ["INGEST_TOKEN"] = "${secret.own.token}" | |
| 24 | } | |
| 25 | ||
| 26 | envTemplate = """ | |
| 27 | {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-logs" }}LOGS_URL=http://\(module.nomadHostPort){{ end }} | |
| 28 | {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-metrics" }}METRICS_URL=http://\(module.nomadHostPort){{ end }} | |
| 29 | """ | |
| 30 | } |
service/iroh-relay/iroh-relay.service+1| ... | ... | @@ -6,6 +6,7 @@ After=network-online.target |
| 6 | 6 | [Service] |
| 7 | 7 | DynamicUser=true |
| 8 | 8 | Environment=HTTP_PORT=3340 |
| 9 | Environment=METRICS_ADDR=127.0.0.1:9091 | |
| 9 | 10 | LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem |
| 10 | 11 | LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem |
| 11 | 12 | ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay |
service/iroh-relay/run.py+3-1| ... | ... | @@ -36,8 +36,10 @@ if mode == "relay": |
| 36 | 36 | raise RuntimeError("Relay certificate was not issued within three minutes") |
| 37 | 37 | quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( |
| 38 | 38 | Path(__file__).with_name("network.json").read_text())["quic"] |
| 39 | metrics = os.environ.get("METRICS_ADDR") | |
| 39 | 40 | config.write_text( |
| 40 | 'enable_metrics = false\nenable_quic_addr_discovery = true\n' | |
| 41 | f'enable_metrics = {json.dumps(bool(metrics))}\nenable_quic_addr_discovery = true\n' | |
| 42 | + (f'metrics_bind_addr = {json.dumps(metrics)}\n' if metrics else '') + | |
| 41 | 43 | f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' |
| 42 | 44 | '[tls]\ncert_mode = "Reloading"\n' |
| 43 | 45 | f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' |
service/victoria/service.pkl+1| ... | ... | @@ -11,6 +11,7 @@ containers { |
| 11 | 11 | |
| 12 | 12 | http { |
| 13 | 13 | containerPort = 9428 |
| 14 | hostPort = 19428 | |
| 14 | 15 | subdomain = "logs" |
| 15 | 16 | authRole = "infra-admin" |
| 16 | 17 | checkPath = "/health" |
tools/iroh-relay.md+3| ... | ... | @@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and |
| 33 | 33 | [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into |
| 34 | 34 | `/etc/systemd/system/`. |
| 35 | 35 | |
| 36 | The unit sets `METRICS_ADDR`, so the relay serves metrics on `127.0.0.1:9091` for | |
| 37 | [vmagent](vps.md). | |
| 38 | ||
| 36 | 39 | [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as |
| 37 | 40 | `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main |
| 38 | 41 | Caddyfile. Validate before reloading Caddy. The ACME webroot is |
tools/vps.md created+39| ... | ... | @@ -0,0 +1,39 @@ |
| 1 | # VPS logs and metrics | |
| 2 | ||
| 3 | The VPS (`ssh root@paperclover.net`, hostname `usw-server-1`) pushes its logs and | |
| 4 | metrics to Snow Globe's [ingest](../service/ingest/service.pkl) service at | |
| 5 | `https://ingest.paperclover.net`, which files them in Victoria under `source=vps`. | |
| 6 | The dashboard lists them under **vps services**; their ids are `vps.<name>`. | |
| 7 | ||
| 8 | | Agent | Collects | Sends | | |
| 9 | | --- | --- | --- | | |
| 10 | | [vlagent](vps/vlagent.service) | `~clo/.pm2/logs/*.log`, plus the journal through `systemd-journal-upload` | `/insert/jsonline` | | |
| 11 | | [vmagent](vps/vmagent.service) | the iroh relay's metrics on `127.0.0.1:9091`, plus [pm2-metrics.py](vps/pm2-metrics.py) pushes | `/api/v1/write` | | |
| 12 | ||
| 13 | Ingest names each log line's `job` from its pm2 file (`site-out.log` is `site` on | |
| 14 | stdout) or its systemd unit. pm2 writes no timestamps, so a line's time is when | |
| 15 | vlagent read it. Both agents buffer up to 1 GB on disk while Snow Globe is | |
| 16 | unreachable. | |
| 17 | ||
| 18 | ```logsql | |
| 19 | source:=vps job:=site level:=error | |
| 20 | ``` | |
| 21 | ||
| 22 | ## Install | |
| 23 | ||
| 24 | Agents are the VictoriaMetrics release binaries, checksum-verified: | |
| 25 | ||
| 26 | | Binary | Release tarball | SHA256 | | |
| 27 | | --- | --- | --- | | |
| 28 | | `/usr/local/bin/vlagent` | `vlutils-linux-amd64-v1.52.0.tar.gz` | `caa315764df0d11a77ba61d4ee96b783ddb98455b167da8326940dda62914bc9` | | |
| 29 | | `/usr/local/bin/vmagent` | `vmutils-linux-amd64-v1.152.0.tar.gz` | `8eee4a98ff1665c60682475e8a8b292b8d718b63a2f023124384dd2f6a220c79` | | |
| 30 | ||
| 31 | The ingest service's generated `token` secret lives in `/etc/snowglobe-ingest.token` | |
| 32 | (root, mode 600). `vmagent.yml` and `pm2-metrics.py` go in | |
| 33 | `/usr/local/lib/snowglobe-agents/`, the units in `/etc/systemd/system/`, and | |
| 34 | [journal-upload.conf](vps/journal-upload.conf) in | |
| 35 | `/etc/systemd/journal-upload.conf.d/snowglobe.conf` (package | |
| 36 | `systemd-journal-remote`). Seed `/var/lib/systemd/journal-upload/state` with | |
| 37 | `LAST_CURSOR=` and the current cursor, and run vlagent once against a sink until | |
| 38 | its checkpoints reach the ends of the pm2 files; otherwise both send their whole | |
| 39 | history, and the pm2 lines all get the same time. |
tools/vps/journal-upload.conf created+2| ... | ... | @@ -0,0 +1,2 @@ |
| 1 | [Upload] | |
| 2 | URL=http://127.0.0.1:9429/insert/journald |
tools/vps/pm2-metrics.py created+22| ... | ... | @@ -0,0 +1,22 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | """Pushes each pm2 app's state to vmagent as Snow Globe service metrics.""" | |
| 3 | import json | |
| 4 | import subprocess | |
| 5 | import time | |
| 6 | from urllib.request import Request, urlopen | |
| 7 | ||
| 8 | while True: | |
| 9 | apps = json.loads(subprocess.run(["pm2", "jlist"], capture_output=True, check=True, text=True).stdout) | |
| 10 | lines = [] | |
| 11 | for app in apps: | |
| 12 | env, monit = app["pm2_env"], app.get("monit") or {} | |
| 13 | labels = "{service=%s}" % json.dumps("vps." + app["name"]) | |
| 14 | lines += [ | |
| 15 | f"snowglobe_service_up{labels} {int(env['status'] == 'online')}", | |
| 16 | f"snowglobe_service_cpu_cores{labels} {monit.get('cpu', 0) / 100}", | |
| 17 | f"snowglobe_service_memory_bytes{labels} {monit.get('memory', 0)}", | |
| 18 | f"snowglobe_service_restarts_total{labels} {env.get('restart_time', 0)}", | |
| 19 | f"snowglobe_service_started_seconds{labels} {env.get('pm_uptime', 0) / 1000}", | |
| 20 | ] | |
| 21 | urlopen(Request("http://127.0.0.1:8429/api/v1/import/prometheus", data="\n".join(lines).encode() + b"\n"), timeout=10).read() | |
| 22 | time.sleep(15) |
tools/vps/pm2-metrics.service created+14| ... | ... | @@ -0,0 +1,14 @@ |
| 1 | [Unit] | |
| 2 | Description=Report pm2 apps to vmagent | |
| 3 | After=vmagent.service | |
| 4 | ||
| 5 | [Service] | |
| 6 | User=clo | |
| 7 | Environment=PATH=/home/clo/.local/bin:/usr/bin | |
| 8 | ExecStart=/usr/bin/python3 /usr/local/lib/snowglobe-agents/pm2-metrics.py | |
| 9 | Restart=always | |
| 10 | RestartSec=15 | |
| 11 | NoNewPrivileges=true | |
| 12 | ||
| 13 | [Install] | |
| 14 | WantedBy=multi-user.target |
tools/vps/vlagent.service created+23| ... | ... | @@ -0,0 +1,23 @@ |
| 1 | [Unit] | |
| 2 | Description=Ship VPS logs to Snow Globe | |
| 3 | Wants=network-online.target | |
| 4 | After=network-online.target | |
| 5 | ||
| 6 | [Service] | |
| 7 | LoadCredential=token:/etc/snowglobe-ingest.token | |
| 8 | ExecStart=/usr/local/bin/vlagent -httpListenAddr=127.0.0.1:9429 -tmpDataPath=/var/lib/vlagent \ | |
| 9 | -fileCollector.glob=/home/clo/.pm2/logs/*.log \ | |
| 10 | -remoteWrite.url=https://ingest.paperclover.net/insert/jsonline -remoteWrite.format=jsonline \ | |
| 11 | -remoteWrite.bearerTokenFile=%d/token -remoteWrite.maxDiskUsagePerURL=1GB | |
| 12 | StateDirectory=vlagent | |
| 13 | Restart=always | |
| 14 | RestartSec=5 | |
| 15 | NoNewPrivileges=true | |
| 16 | ProtectSystem=strict | |
| 17 | ProtectHome=read-only | |
| 18 | PrivateTmp=true | |
| 19 | PrivateDevices=true | |
| 20 | MemoryMax=256M | |
| 21 | ||
| 22 | [Install] | |
| 23 | WantedBy=multi-user.target |
tools/vps/vmagent.service created+25| ... | ... | @@ -0,0 +1,25 @@ |
| 1 | [Unit] | |
| 2 | Description=Ship VPS metrics to Snow Globe | |
| 3 | Wants=network-online.target | |
| 4 | After=network-online.target | |
| 5 | ||
| 6 | [Service] | |
| 7 | DynamicUser=true | |
| 8 | LoadCredential=token:/etc/snowglobe-ingest.token | |
| 9 | ExecStart=/usr/local/bin/vmagent -httpListenAddr=127.0.0.1:8429 \ | |
| 10 | -promscrape.config=/usr/local/lib/snowglobe-agents/vmagent.yml \ | |
| 11 | -remoteWrite.url=https://ingest.paperclover.net/api/v1/write -remoteWrite.label=host=usw-server-1 \ | |
| 12 | -remoteWrite.bearerTokenFile=%d/token \ | |
| 13 | -remoteWrite.tmpDataPath=/var/lib/vmagent -remoteWrite.maxDiskUsagePerURL=1GB | |
| 14 | StateDirectory=vmagent | |
| 15 | Restart=always | |
| 16 | RestartSec=5 | |
| 17 | NoNewPrivileges=true | |
| 18 | ProtectSystem=strict | |
| 19 | ProtectHome=true | |
| 20 | PrivateTmp=true | |
| 21 | PrivateDevices=true | |
| 22 | MemoryMax=256M | |
| 23 | ||
| 24 | [Install] | |
| 25 | WantedBy=multi-user.target |
tools/vps/vmagent.yml created+7| ... | ... | @@ -0,0 +1,7 @@ |
| 1 | global: | |
| 2 | scrape_interval: 30s | |
| 3 | scrape_configs: | |
| 4 | - job_name: iroh-relay | |
| 5 | static_configs: | |
| 6 | - targets: ["127.0.0.1:9091"] | |
| 7 | labels: { service: "vps.iroh-relay" } |