| author | |
| committer | |
| log | ca421c2098f96de8070bb7d17dd1d3aab3b47487 |
| tree | 1e6e44c18413ce7e2bd6932558ef3213d8e4775d |
| parent | dc8464162f442c4e404063c93a5221ae4d56b122 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A public ingest service takes vlagent and vmagent pushes with a bearer token
and files them in Victoria under source=vps, naming each log line after its
pm2 app or systemd unit. On the VPS, vlagent tails the pm2 logs and the
journal, vmagent scrapes the iroh relay, and a small script reports pm2 state
as service metrics.
The dashboard lists these under "vps services" with the same logs, cpu,
memory, and metrics views as Nomad services, keyed vps.<name>.
Victoria logs gets a host port so containers on the bridge can reach it.
Assisted-by: claude-opus-5.524 files changed, 480 insertions(+), 14 deletions(-)
dashboard/src/core.rs+7-5| ... | @@ -622,6 +622,7 @@ pub async fn route( | ... | @@ -622,6 +622,7 @@ pub async fn route( |
| 622 | json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null}) | 622 | json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null}) |
| 623 | } | 623 | } |
| 624 | ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()), | 624 | ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()), |
| 625 | ["vps"] if method == Method::GET => telemetry::vps(app).await?, | ||
| 625 | ["launcher"] if method == Method::GET => { | 626 | ["launcher"] if method == Method::GET => { |
| 626 | let found = launcher(app.clone()).await?; | 627 | let found = launcher(app.clone()).await?; |
| 627 | let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10)); | 628 | let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10)); |
| ... | @@ -723,11 +724,12 @@ pub async fn route( | ... | @@ -723,11 +724,12 @@ pub async fn route( |
| 723 | .response()); | 724 | .response()); |
| 724 | } | 725 | } |
| 725 | ["services", id, "metrics", name] if method == Method::GET => { | 726 | ["services", id, "metrics", name] if method == Method::GET => { |
| 726 | let detail = service(app.clone(), id).await?; | 727 | let names = if id.starts_with("vps.") { |
| 727 | if !array(&detail.value["applicationMetrics"]) | 728 | telemetry::metric_names(app.clone(), id).await? |
| 728 | .iter() | 729 | } else { |
| 729 | .any(|v| v == *name) | 730 | service(app.clone(), id).await?.value["applicationMetrics"].clone() |
| 730 | { | 731 | }; |
| 732 | if !array(&names).iter().any(|v| v == *name) { | ||
| 731 | return Err(Error::new(404, "Metric unavailable for this service.")); | 733 | return Err(Error::new(404, "Metric unavailable for this service.")); |
| 732 | } | 734 | } |
| 733 | return Ok(telemetry::metrics(app, name, query, Some(id), None) | 735 | return Ok(telemetry::metrics(app, name, query, Some(id), None) |
dashboard/src/main.rs+1-1| ... | @@ -279,7 +279,7 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response> | ... | @@ -279,7 +279,7 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response> |
| 279 | "services" if parts.len() == 1 => Some("metrics"), | 279 | "services" if parts.len() == 1 => Some("metrics"), |
| 280 | "storage" if parts.len() == 1 => Some("metrics"), | 280 | "storage" if parts.len() == 1 => Some("metrics"), |
| 281 | "ai" | "mcp" => Some("ai"), | 281 | "ai" | "mcp" => Some("ai"), |
| 282 | "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { | 282 | "services" | "vps" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { |
| 283 | Some("admin") | 283 | Some("admin") |
| 284 | } | 284 | } |
| 285 | "media" | "seedbox" | "youtube" => Some("media"), | 285 | "media" | "seedbox" | "youtube" => Some("media"), |
dashboard/src/telemetry.rs+53-3| ... | @@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { | ... | @@ -228,7 +228,7 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { |
| 228 | ( | 228 | ( |
| 229 | "match[]", | 229 | "match[]", |
| 230 | format!( | 230 | format!( |
| 231 | "{{{label}={},__name__!~\"snowglobe_.*\"}}", | 231 | "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}", |
| 232 | serde_json::to_string(id).unwrap() | 232 | serde_json::to_string(id).unwrap() |
| 233 | ), | 233 | ), |
| 234 | ) | 234 | ) |
| ... | @@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { | ... | @@ -248,6 +248,49 @@ pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> { |
| 248 | Ok(json!(values)) | 248 | Ok(json!(values)) |
| 249 | } | 249 | } |
| 250 | 250 | ||
| 251 | /// Services the VPS agents report, newest state first; quiet for two minutes counts as down. | ||
| 252 | pub async fn vps(app: Arc<App>) -> Result<Value> { | ||
| 253 | let base = endpoint(app.clone(), "victoria-metrics").await?; | ||
| 254 | let instant = |query: &str| { | ||
| 255 | let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())])); | ||
| 256 | let app = app.clone(); | ||
| 257 | async move { | ||
| 258 | let response = read(app, url, None).await?; | ||
| 259 | Ok::<_, Error>( | ||
| 260 | array(&response.value["data"]["result"]) | ||
| 261 | .iter() | ||
| 262 | .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1]))) | ||
| 263 | .collect::<HashMap<_, _>>(), | ||
| 264 | ) | ||
| 265 | } | ||
| 266 | }; | ||
| 267 | let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#; | ||
| 268 | let (known, current, cpu, memory, restarts, started) = futures::try_join!( | ||
| 269 | instant(&format!("max by (service) (max_over_time({up}[7d]))")), | ||
| 270 | instant(&format!("max by (service) (last_over_time({up}[2m]))")), | ||
| 271 | instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#), | ||
| 272 | instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#), | ||
| 273 | instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#), | ||
| 274 | instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#), | ||
| 275 | )?; | ||
| 276 | let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect(); | ||
| 277 | ids.sort(); | ||
| 278 | let mut services = Vec::new(); | ||
| 279 | for id in ids { | ||
| 280 | services.push(json!({ | ||
| 281 | "id": id, | ||
| 282 | "name": &id[4..], | ||
| 283 | "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" }, | ||
| 284 | "cpu": cpu.get(&id), | ||
| 285 | "memory": memory.get(&id), | ||
| 286 | "restarts": restarts.get(&id), | ||
| 287 | "startedAt": started.get(&id).filter(|v| **v > 0.0), | ||
| 288 | "applicationMetrics": metric_names(app.clone(), &id).await?, | ||
| 289 | })); | ||
| 290 | } | ||
| 291 | Ok(json!(services)) | ||
| 292 | } | ||
| 293 | |||
| 251 | #[derive(Debug, PartialEq)] | 294 | #[derive(Debug, PartialEq)] |
| 252 | struct Term { | 295 | struct Term { |
| 253 | field: Option<String>, | 296 | field: Option<String>, |
| ... | @@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> { | ... | @@ -299,9 +342,10 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> { |
| 299 | Ok(found) | 342 | Ok(found) |
| 300 | } | 343 | } |
| 301 | fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> { | 344 | fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> { |
| 345 | let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job)); | ||
| 302 | let mut filters = vec![ | 346 | let mut filters = vec![ |
| 303 | "source:=nomad".to_owned(), | 347 | format!("source:={source}"), |
| 304 | format!("job:={}", serde_json::to_string(id)?), | 348 | format!("job:={}", serde_json::to_string(job)?), |
| 305 | ]; | 349 | ]; |
| 306 | if let Some(level) = query.get("level") { | 350 | if let Some(level) = query.get("level") { |
| 307 | filters.push( | 351 | filters.push( |
| ... | @@ -1001,6 +1045,12 @@ pub fn start(app: Arc<App>) { | ... | @@ -1001,6 +1045,12 @@ pub fn start(app: Arc<App>) { |
| 1001 | mod tests { | 1045 | mod tests { |
| 1002 | use super::*; | 1046 | use super::*; |
| 1003 | #[test] | 1047 | #[test] |
| 1048 | fn vps_ids_read_the_vps_source() { | ||
| 1049 | let none = HashMap::new(); | ||
| 1050 | assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#); | ||
| 1051 | assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#); | ||
| 1052 | } | ||
| 1053 | #[test] | ||
| 1004 | fn search_fields_comparisons_exclusions_and_unknown_fields() { | 1054 | fn search_fields_comparisons_exclusions_and_unknown_fields() { |
| 1005 | let parsed = terms( | 1055 | let parsed = terms( |
| 1006 | r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#, | 1056 | r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#, |
dashboard/web/api.contract.ts+4-1| ... | @@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts"; | ... | @@ -6,7 +6,7 @@ import type { User, Group, Session, Credential } from "./types/users.ts"; |
| 6 | import type { Hono } from "hono"; | 6 | import type { Hono } from "hono"; |
| 7 | import type { Health } from "./types/model.ts"; | 7 | import type { Health } from "./types/model.ts"; |
| 8 | import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; | 8 | import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; |
| 9 | import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; | 9 | import type { Me, ServiceSummary, ServiceDetail, VpsService, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; |
| 10 | import type { MapNode } from "./types/storage.index.ts"; | 10 | import type { MapNode } from "./types/storage.index.ts"; |
| 11 | import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; | 11 | import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; |
| 12 | import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; | 12 | import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; |
| ... | @@ -102,6 +102,9 @@ export type Api = Hono<{}, { | ... | @@ -102,6 +102,9 @@ export type Api = Hono<{}, { |
| 102 | "/services": { | 102 | "/services": { |
| 103 | $get: Endpoint<ServiceSummary[]>; | 103 | $get: Endpoint<ServiceSummary[]>; |
| 104 | }; | 104 | }; |
| 105 | "/vps": { | ||
| 106 | $get: Endpoint<VpsService[]>; | ||
| 107 | }; | ||
| 105 | "/services/:id": { | 108 | "/services/:id": { |
| 106 | $get: Endpoint<ServiceDetail, { param: { id: string; }; }>; | 109 | $get: Endpoint<ServiceDetail, { param: { id: string; }; }>; |
| 107 | }; | 110 | }; |
dashboard/web/api.ts+1| ... | @@ -80,6 +80,7 @@ export const queries = { | ... | @@ -80,6 +80,7 @@ export const queries = { |
| 80 | storage: query("storage", () => parseResponse(api.storage.$get())), | 80 | storage: query("storage", () => parseResponse(api.storage.$get())), |
| 81 | deploys: query("deploys", () => parseResponse(api.deploys.$get())), | 81 | deploys: query("deploys", () => parseResponse(api.deploys.$get())), |
| 82 | vms: query("vms", () => parseResponse(api.vms.$get())), | 82 | vms: query("vms", () => parseResponse(api.vms.$get())), |
| 83 | vps: query("vps", () => parseResponse(api.vps.$get())), | ||
| 83 | metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) => | 84 | metric: query("metric", ({ metric, range, service }: { metric: Metric; range: number; service?: string }) => |
| 84 | parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))), | 85 | parseResponse(api.metrics[":metric"].$get({ param: { metric }, query: { range: String(range), ...(service ? { service } : {}) } }))), |
| 85 | seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())), | 86 | seedbox: query("seedbox", () => parseResponse(api.seedbox.$get())), |
dashboard/web/components/Sidebar.tsx+33| ... | @@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house"; | ... | @@ -10,6 +10,7 @@ import House from "lucide-solid/icons/house"; |
| 10 | import LogOut from "lucide-solid/icons/log-out"; | 10 | import LogOut from "lucide-solid/icons/log-out"; |
| 11 | import Monitor from "lucide-solid/icons/monitor"; | 11 | import Monitor from "lucide-solid/icons/monitor"; |
| 12 | import Rocket from "lucide-solid/icons/rocket"; | 12 | import Rocket from "lucide-solid/icons/rocket"; |
| 13 | import Server from "lucide-solid/icons/server"; | ||
| 13 | import SquarePlay from "lucide-solid/icons/square-play"; | 14 | import SquarePlay from "lucide-solid/icons/square-play"; |
| 14 | import UserRound from "lucide-solid/icons/user-round"; | 15 | import UserRound from "lucide-solid/icons/user-round"; |
| 15 | import Plug from "lucide-solid/icons/plug"; | 16 | import Plug from "lucide-solid/icons/plug"; |
| ... | @@ -247,6 +248,37 @@ function Services() { | ... | @@ -247,6 +248,37 @@ function Services() { |
| 247 | ); | 248 | ); |
| 248 | } | 249 | } |
| 249 | 250 | ||
| 251 | function VpsServices() { | ||
| 252 | const [open, toggle] = remembered("sidebar.vps", true); | ||
| 253 | const [services, { refetch }] = queries.vps.use(); | ||
| 254 | const timer = setInterval(() => !services.loading && refetch(), 15_000); | ||
| 255 | onCleanup(() => clearInterval(timer)); | ||
| 256 | const loaded = lastGood(services); | ||
| 257 | const down = () => (loaded() ?? []).filter((service) => service.health !== "healthy").map((service) => service.name); | ||
| 258 | return ( | ||
| 259 | <> | ||
| 260 | <button class="nav-item" onClick={toggle} aria-expanded={open()}> | ||
| 261 | <Server class="icon" /> | ||
| 262 | <span class="label">vps services</span> | ||
| 263 | <span class="badge"> | ||
| 264 | <Show when={down().length}> | ||
| 265 | <span class="tally" data-tip={down().join(", ")}><StatusLabel health="down">{down().length}</StatusLabel></span> | ||
| 266 | </Show> | ||
| 267 | </span> | ||
| 268 | <ChevronRight class={`chevron ${open() ? "open" : ""}`} /> | ||
| 269 | </button> | ||
| 270 | <div class="nav-group" classList={{ open: open() }} inert={!open()}><div> | ||
| 271 | <For each={loaded()}>{(service) => ( | ||
| 272 | <A href={`/vps/${service.name}`} class="nav-sub" activeClass="active"> | ||
| 273 | <Status health={service.health} /><span class="label">{service.name}</span> | ||
| 274 | </A> | ||
| 275 | )}</For> | ||
| 276 | <Show when={services.error && !loaded()}><span class="nav-sub muted">VPS unavailable</span></Show> | ||
| 277 | </div></div> | ||
| 278 | </> | ||
| 279 | ); | ||
| 280 | } | ||
| 281 | |||
| 250 | /** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */ | 282 | /** Previews the dashboard as someone in fewer groups; the server only honors it for admins. */ |
| 251 | export function viewAs(groups: string[] | null) { | 283 | export function viewAs(groups: string[] | null) { |
| 252 | document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`; | 284 | document.cookie = groups ? `${VIEW_AS}=${encodeURIComponent(groups.join(","))}; path=/; samesite=strict` : `${VIEW_AS}=; path=/; max-age=0`; |
| ... | @@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) { | ... | @@ -310,6 +342,7 @@ export function Sidebar(props: { me: Me }) { |
| 310 | <For each={BEFORE.filter(allowed)}>{(page) => <NavLink page={page}><IssueCount /></NavLink>}</For> | 342 | <For each={BEFORE.filter(allowed)}>{(page) => <NavLink page={page}><IssueCount /></NavLink>}</For> |
| 311 | <Show when={props.me.sections.includes("admin")}> | 343 | <Show when={props.me.sections.includes("admin")}> |
| 312 | <Services /> | 344 | <Services /> |
| 345 | <VpsServices /> | ||
| 313 | </Show> | 346 | </Show> |
| 314 | <For each={AFTER.filter(allowed)}>{(page) => page.section === "vms" ? <VMNav /> : <NavLink page={page} />}</For> | 347 | <For each={AFTER.filter(allowed)}>{(page) => page.section === "vms" ? <VMNav /> : <NavLink page={page} />}</For> |
| 315 | </div> | 348 | </div> |
dashboard/web/main.tsx+3-1| ... | @@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx"; | ... | @@ -20,6 +20,7 @@ import { shares, Shares } from "./pages/Shares.tsx"; |
| 20 | import { Deploys } from "./pages/Deploys.tsx"; | 20 | import { Deploys } from "./pages/Deploys.tsx"; |
| 21 | import { Deploy } from "./pages/Deploy.tsx"; | 21 | import { Deploy } from "./pages/Deploy.tsx"; |
| 22 | import { VMs } from "./pages/VMs.tsx"; | 22 | import { VMs } from "./pages/VMs.tsx"; |
| 23 | import { Vps } from "./pages/Vps.tsx"; | ||
| 23 | import { MCP } from "./pages/MCP.tsx"; | 24 | import { MCP } from "./pages/MCP.tsx"; |
| 24 | import { MCPConsent } from "./pages/MCPConsent.tsx"; | 25 | import { MCPConsent } from "./pages/MCPConsent.tsx"; |
| 25 | import { SignIn } from "./pages/SignIn.tsx"; | 26 | import { SignIn } from "./pages/SignIn.tsx"; |
| ... | @@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) = | ... | @@ -34,7 +35,7 @@ const preload = (...list: { preload(): void }[]) => () => list.forEach((query) = |
| 34 | function Shell(props: RouteSectionProps) { | 35 | function Shell(props: RouteSectionProps) { |
| 35 | const location = useLocation(); | 36 | const location = useLocation(); |
| 36 | const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request"); | 37 | const consent = () => location.pathname.startsWith("/connect/") || location.pathname === "/mcp" && new URLSearchParams(location.search).has("request"); |
| 37 | const section = () => location.pathname.startsWith("/services/") ? "admin" | 38 | const section = () => /^\/(services|vps)\//.test(location.pathname) ? "admin" |
| 38 | : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section; | 39 | : PAGES.find((page) => page.href !== "/" && location.pathname.startsWith(page.href))?.section; |
| 39 | return ( | 40 | return ( |
| 40 | <> | 41 | <> |
| ... | @@ -78,6 +79,7 @@ render(() => ( | ... | @@ -78,6 +79,7 @@ render(() => ( |
| 78 | queries.service.preload(params.id!); | 79 | queries.service.preload(params.id!); |
| 79 | queries.deploys.preload(); | 80 | queries.deploys.preload(); |
| 80 | }} /> | 81 | }} /> |
| 82 | <Route path="/vps/:name/:tab?" component={Vps} preload={preload(queries.vps)} /> | ||
| 81 | <Route path="/storage" component={Storage} preload={preload(queries.storage, queries.launcher)} /> | 83 | <Route path="/storage" component={Storage} preload={preload(queries.storage, queries.launcher)} /> |
| 82 | <Route path="/media" component={Media} preload={preload(queries.launcher)} /> | 84 | <Route path="/media" component={Media} preload={preload(queries.launcher)} /> |
| 83 | <Route path="/seedbox" component={Seedbox} preload={preload(queries.seedbox, queries.seedboxHistory, queries.launcher)} /> | 85 | <Route path="/seedbox" component={Seedbox} preload={preload(queries.seedbox, queries.seedboxHistory, queries.launcher)} /> |
dashboard/web/pages/Service.tsx+2-2| ... | @@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con | ... | @@ -44,7 +44,7 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con |
| 44 | type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; | 44 | type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; |
| 45 | 45 | ||
| 46 | /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ | 46 | /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ |
| 47 | function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | 47 | export function Logs(props: { service: Pick<ServiceDetail, "id" | "containers" | "logs">; tools: HTMLElement }) { |
| 48 | const [params, setParams] = useSearchParams<Query>(); | 48 | const [params, setParams] = useSearchParams<Query>(); |
| 49 | const at = () => (params.at ? Number(params.at) : null); | 49 | const at = () => (params.at ? Number(params.at) : null); |
| 50 | const bounded = () => at() !== null || !!params.to; | 50 | const bounded = () => at() !== null || !!params.to; |
| ... | @@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er | ... | @@ -272,7 +272,7 @@ const traceList = query("traces", ({ id, ...query }: { id: string; q: string; er |
| 272 | const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } }))); | 272 | const oneTrace = query("trace", (id: string) => parseResponse(api.traces[":id"].$get({ param: { id } }))); |
| 273 | const PAGE = 50; | 273 | const PAGE = 50; |
| 274 | 274 | ||
| 275 | function AppMetrics(props: { service: ServiceDetail; range: number }) { | 275 | export function AppMetrics(props: { service: Pick<ServiceDetail, "id" | "applicationMetrics">; range: number }) { |
| 276 | const [name, setName] = createSignal(props.service.applicationMetrics[0]!); | 276 | const [name, setName] = createSignal(props.service.applicationMetrics[0]!); |
| 277 | const [data, { refetch }] = createResource( | 277 | const [data, { refetch }] = createResource( |
| 278 | () => [props.service.id, name(), props.range] as const, | 278 | () => [props.service.id, name(), props.range] as const, |
dashboard/web/pages/Vps.tsx created+90| ... | @@ -0,0 +1,90 @@ | ||
| 1 | import { A, useParams } from "@solidjs/router"; | ||
| 2 | import RotateCcw from "lucide-solid/icons/rotate-ccw"; | ||
| 3 | import Server from "lucide-solid/icons/server"; | ||
| 4 | import { createSignal, Match, onCleanup, Show, Switch } from "solid-js"; | ||
| 5 | import { queries } from "../api.ts"; | ||
| 6 | import { AppIcon } from "../components/AppIcon.tsx"; | ||
| 7 | import { ListPage } from "../components/ListPage.tsx"; | ||
| 8 | import { lastGood, Loaded } from "../components/Loaded.tsx"; | ||
| 9 | import { RangePicker } from "../components/Metric.tsx"; | ||
| 10 | import { StatusLabel } from "../components/Status.tsx"; | ||
| 11 | import { Strip } from "../components/Strip.tsx"; | ||
| 12 | import { TabBar } from "../components/TabBar.tsx"; | ||
| 13 | import { bytes, cores, datetime, duration, plural } from "../format.ts"; | ||
| 14 | import { AppMetrics, Logs } from "./Service.tsx"; | ||
| 15 | |||
| 16 | /** Remounts per app, so switching never shows the last one's logs. */ | ||
| 17 | export function Vps() { | ||
| 18 | const params = useParams<{ name: string }>(); | ||
| 19 | return <Show when={params.name} keyed><VpsPage /></Show>; | ||
| 20 | } | ||
| 21 | |||
| 22 | function VpsPage() { | ||
| 23 | const params = useParams<{ name: string; tab?: string }>(); | ||
| 24 | const [services, { refetch }] = queries.vps.use(); | ||
| 25 | const timer = setInterval(refetch, 15_000); | ||
| 26 | onCleanup(() => clearInterval(timer)); | ||
| 27 | const loaded = lastGood(services); | ||
| 28 | const [range, setRange] = createSignal(3600); | ||
| 29 | return ( | ||
| 30 | <Loaded data={services} what="VPS services" retry={refetch} skeleton={<div class="page"><div class="skeleton" style={{ height: "112px" }} /></div>}> | ||
| 31 | {() => ( | ||
| 32 | <Show when={loaded()?.find((service) => service.name === params.name)} fallback={ | ||
| 33 | <div class="page empty">The VPS hasn't reported {params.name} in the last week.</div> | ||
| 34 | }> | ||
| 35 | {(data) => { | ||
| 36 | const tabHref = (tab: string) => `/vps/${data().name}${tab ? "/" + tab : ""}`; | ||
| 37 | let tools!: HTMLDivElement; | ||
| 38 | return ( | ||
| 39 | <ListPage id="service" class="service-page" flush head={ | ||
| 40 | <div class="page-head"> | ||
| 41 | <span class="service-icon"><AppIcon id={data().id} name={data().name} icon={null} /></span> | ||
| 42 | <h1>{data().name}</h1> | ||
| 43 | <div class="stats"> | ||
| 44 | <Show when={data().health === "healthy" && data().startedAt} fallback={ | ||
| 45 | <span class="stat" role="status"><StatusLabel health={data().health} /></span> | ||
| 46 | }> | ||
| 47 | {(t) => ( | ||
| 48 | <span class="stat" role="status" tabIndex={0} data-tip={`since ${datetime(t())}`}> | ||
| 49 | <StatusLabel health="healthy">up {duration(Date.now() / 1000 - t())}</StatusLabel> | ||
| 50 | </span> | ||
| 51 | )} | ||
| 52 | </Show> | ||
| 53 | <span class="stat"><Server size={13} aria-hidden="true" /><b>usw-server-1</b></span> | ||
| 54 | <Show when={data().restarts}> | ||
| 55 | {(restarts) => <span class="stat"><RotateCcw size={13} aria-hidden="true" />{plural(restarts(), "restart")}</span>} | ||
| 56 | </Show> | ||
| 57 | </div> | ||
| 58 | </div> | ||
| 59 | } summary={ | ||
| 60 | <Show when={data().cpu !== null}> | ||
| 61 | <div class="service-strips"> | ||
| 62 | <Strip title="cpu" metric="service.cpu" service={data().id} range={range()} format={cores} height={64} | ||
| 63 | tabs={<RangePicker value={range()} onChange={setRange} />} | ||
| 64 | now={() => ({ value: cores(data().cpu ?? 0) })} /> | ||
| 65 | <Strip title="memory" metric="service.memory" service={data().id} range={range()} format={bytes} | ||
| 66 | height={64 + 24} last now={() => ({ value: bytes(data().memory ?? 0) })} /> | ||
| 67 | </div> | ||
| 68 | </Show> | ||
| 69 | }> | ||
| 70 | <div ref={tools} class="tab-row"> | ||
| 71 | <TabBar label="View"> | ||
| 72 | <A href={tabHref("")} end>logs</A> | ||
| 73 | <Show when={data().applicationMetrics.length}><A href={tabHref("metrics")} end>metrics</A></Show> | ||
| 74 | </TabBar> | ||
| 75 | </div> | ||
| 76 | <Switch> | ||
| 77 | <Match when={!params.tab}><Logs service={{ id: data().id, containers: [], logs: null }} tools={tools} /></Match> | ||
| 78 | <Match when={params.tab === "metrics" && data().applicationMetrics.length}> | ||
| 79 | <AppMetrics service={data()} range={range()} /> | ||
| 80 | </Match> | ||
| 81 | </Switch> | ||
| 82 | </ListPage> | ||
| 83 | ); | ||
| 84 | }} | ||
| 85 | </Show> | ||
| 86 | )} | ||
| 87 | </Loaded> | ||
| 88 | ); | ||
| 89 | } | ||
| 90 | |||
dashboard/web/types/model.ts+12| ... | @@ -51,6 +51,18 @@ export interface ServiceSummary { | ... | @@ -51,6 +51,18 @@ export interface ServiceSummary { |
| 51 | memoryLimit: number; | 51 | memoryLimit: number; |
| 52 | } | 52 | } |
| 53 | 53 | ||
| 54 | /** An app the VPS agents report, keyed `vps.<name>`; usage, restarts and start time are pm2's. */ | ||
| 55 | export interface VpsService { | ||
| 56 | id: string; | ||
| 57 | name: string; | ||
| 58 | health: Health; | ||
| 59 | cpu: number | null; | ||
| 60 | memory: number | null; | ||
| 61 | restarts: number | null; | ||
| 62 | startedAt: number | null; | ||
| 63 | applicationMetrics: string[]; | ||
| 64 | } | ||
| 65 | |||
| 54 | /** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */ | 66 | /** States that need someone to look, unlike stopped (on purpose) or the passing ones like restarting. */ |
| 55 | export const trouble = (health: Health) => health === "down" || health === "degraded"; | 67 | export const trouble = (health: Health) => health === "down" || health === "degraded"; |
| 56 | 68 |
service/ingest/build/Dockerfile created+3| ... | @@ -0,0 +1,3 @@ | ||
| 1 | FROM docker.io/library/python:3.14-slim@sha256:0214bda9af2652a418f2bf34373a4d3ca00a5f4784081e040049433ac9cc79d5 | ||
| 2 | COPY app.py /app/app.py | ||
| 3 | CMD ["python3", "-u", "/app/app.py"] | ||
service/ingest/build/app.py created+101| ... | @@ -0,0 +1,101 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Files logs and metrics pushed by the VPS agents in Victoria under source=vps.""" | ||
| 3 | from compression import zstd | ||
| 4 | import gzip | ||
| 5 | import hmac | ||
| 6 | import json | ||
| 7 | import os | ||
| 8 | import re | ||
| 9 | import threading | ||
| 10 | import time | ||
| 11 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ||
| 12 | from urllib.error import HTTPError | ||
| 13 | from urllib.request import Request, urlopen | ||
| 14 | |||
| 15 | TOKEN = os.environ["INGEST_TOKEN"].encode() | ||
| 16 | LOGS = os.environ["LOGS_URL"] | ||
| 17 | METRICS = os.environ["METRICS_URL"] | ||
| 18 | PM2 = re.compile(r".*/\.pm2/logs/(.+)-(out|error)\.log") | ||
| 19 | ERROR = re.compile(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", re.I) | ||
| 20 | WARN = re.compile(r"\b(WARN|WARNING|WRN)\b|level=warn", re.I) | ||
| 21 | LIMIT = 32 << 20 | ||
| 22 | JOURNAL = ("_time", "_msg", "_HOSTNAME", "_SYSTEMD_UNIT", "SYSLOG_IDENTIFIER", "_PID", "PRIORITY") | ||
| 23 | |||
| 24 | sequence_lock = threading.Lock() | ||
| 25 | last_sequence = 0 | ||
| 26 | |||
| 27 | |||
| 28 | def next_sequence(): | ||
| 29 | """Microsecond clock, so lines interleave with the Nomad shipper's sequence that live logs follow.""" | ||
| 30 | global last_sequence | ||
| 31 | with sequence_lock: | ||
| 32 | last_sequence = max(last_sequence + 1, time.time_ns() // 1000) | ||
| 33 | return last_sequence | ||
| 34 | |||
| 35 | |||
| 36 | def shape(row): | ||
| 37 | if match := PM2.fullmatch(row.get("file", "")): | ||
| 38 | job, stream = match[1], "stdout" if match[2] == "out" else "stderr" | ||
| 39 | message = row.get("_msg", "") | ||
| 40 | level = "error" if ERROR.search(message) else "warn" if WARN.search(message) else "info" | ||
| 41 | else: | ||
| 42 | job, stream = (row.get("_SYSTEMD_UNIT") or row.get("SYSLOG_IDENTIFIER") or "system").removesuffix(".service"), "journal" | ||
| 43 | priority = int(row.get("PRIORITY") or 6) | ||
| 44 | row = {key: row[key] for key in JOURNAL if key in row} | ||
| 45 | level = "error" if priority <= 3 else "warn" if priority == 4 else "info" | ||
| 46 | row.update(source="vps", host=row.pop("hostname", None) or row.pop("_HOSTNAME", ""), job=job, stream=stream, | ||
| 47 | level=level, sequence=next_sequence()) | ||
| 48 | return row | ||
| 49 | |||
| 50 | |||
| 51 | def forward(url, body, headers): | ||
| 52 | with urlopen(Request(url, data=body, headers=headers), timeout=30) as response: | ||
| 53 | response.read() | ||
| 54 | |||
| 55 | |||
| 56 | class Handler(BaseHTTPRequestHandler): | ||
| 57 | protocol_version = "HTTP/1.1" | ||
| 58 | |||
| 59 | def reply(self, status, text=""): | ||
| 60 | body = text.encode() | ||
| 61 | self.send_response(status) | ||
| 62 | self.send_header("Content-Length", str(len(body))) | ||
| 63 | self.end_headers() | ||
| 64 | self.wfile.write(body) | ||
| 65 | |||
| 66 | def do_GET(self): | ||
| 67 | self.reply(204 if self.path == "/health" else 404) | ||
| 68 | |||
| 69 | def do_POST(self): | ||
| 70 | length = int(self.headers.get("Content-Length") or 0) | ||
| 71 | if length > LIMIT: | ||
| 72 | self.close_connection = True | ||
| 73 | return self.reply(413, "batch too large") | ||
| 74 | body = self.rfile.read(length) | ||
| 75 | if not hmac.compare_digest(self.headers.get("Authorization", "").removeprefix("Bearer ").encode(), TOKEN): | ||
| 76 | return self.reply(401, "bad or missing token") | ||
| 77 | try: | ||
| 78 | if self.path == "/insert/jsonline": | ||
| 79 | encoding = self.headers.get("Content-Encoding", "") | ||
| 80 | text = zstd.decompress(body) if encoding == "zstd" else gzip.decompress(body) if encoding == "gzip" else body | ||
| 81 | rows = [shape(json.loads(line)) for line in text.splitlines() if line.strip()] | ||
| 82 | forward(LOGS + "/insert/jsonline", b"".join(json.dumps(row).encode() + b"\n" for row in rows), | ||
| 83 | {"Content-Type": "application/stream+json", "VL-Stream-Fields": "source,host,job,stream"}) | ||
| 84 | forward(LOGS + "/internal/force_flush", b"", {}) | ||
| 85 | elif self.path == "/api/v1/write": | ||
| 86 | forward(METRICS + "/api/v1/write?extra_label=source=vps", body, | ||
| 87 | {key: self.headers[key] for key in ("Content-Type", "Content-Encoding") if key in self.headers}) | ||
| 88 | else: | ||
| 89 | return self.reply(404) | ||
| 90 | except HTTPError as error: | ||
| 91 | return self.reply(502, f"victoria answered {error.code}") | ||
| 92 | except (OSError, ValueError, zstd.ZstdError) as error: | ||
| 93 | return self.reply(400 if isinstance(error, (ValueError, zstd.ZstdError)) else 502, str(error)) | ||
| 94 | self.reply(204) | ||
| 95 | |||
| 96 | def log_request(self, code="-", size="-"): | ||
| 97 | if not str(int(code)).startswith("2"): | ||
| 98 | super().log_request(code, size) | ||
| 99 | |||
| 100 | |||
| 101 | ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() | ||
service/ingest/service.pkl created+30| ... | @@ -0,0 +1,30 @@ | ||
| 1 | amends "../../config/Service.pkl" | ||
| 2 | |||
| 3 | name = "Ingest" | ||
| 4 | tagline = "Logs and metrics pushed from the VPS" | ||
| 5 | launcher = false | ||
| 6 | dependsOn { "victoria" } | ||
| 7 | |||
| 8 | secrets { ["token"] {} } | ||
| 9 | |||
| 10 | container { | ||
| 11 | build = "build" | ||
| 12 | readOnlyRoot = true | ||
| 13 | cpu = 100 | ||
| 14 | memory = 128 | ||
| 15 | |||
| 16 | http { | ||
| 17 | containerPort = 8000 | ||
| 18 | subdomain = "ingest" | ||
| 19 | checkPath = "/health" | ||
| 20 | } | ||
| 21 | |||
| 22 | env { | ||
| 23 | ["INGEST_TOKEN"] = "${secret.own.token}" | ||
| 24 | } | ||
| 25 | |||
| 26 | envTemplate = """ | ||
| 27 | {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-logs" }}LOGS_URL=http://\(module.nomadHostPort){{ end }} | ||
| 28 | {{ range nomadService 1 (env "NOMAD_ALLOC_ID") "victoria-metrics" }}METRICS_URL=http://\(module.nomadHostPort){{ end }} | ||
| 29 | """ | ||
| 30 | } | ||
service/iroh-relay/iroh-relay.service+1| ... | @@ -6,6 +6,7 @@ After=network-online.target | ... | @@ -6,6 +6,7 @@ After=network-online.target |
| 6 | [Service] | 6 | [Service] |
| 7 | DynamicUser=true | 7 | DynamicUser=true |
| 8 | Environment=HTTP_PORT=3340 | 8 | Environment=HTTP_PORT=3340 |
| 9 | Environment=METRICS_ADDR=127.0.0.1:9091 | ||
| 9 | LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem | 10 | LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem |
| 10 | LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem | 11 | LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem |
| 11 | ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay | 12 | ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay |
service/iroh-relay/run.py+3-1| ... | @@ -36,8 +36,10 @@ if mode == "relay": | ... | @@ -36,8 +36,10 @@ if mode == "relay": |
| 36 | raise RuntimeError("Relay certificate was not issued within three minutes") | 36 | raise RuntimeError("Relay certificate was not issued within three minutes") |
| 37 | quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( | 37 | quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( |
| 38 | Path(__file__).with_name("network.json").read_text())["quic"] | 38 | Path(__file__).with_name("network.json").read_text())["quic"] |
| 39 | metrics = os.environ.get("METRICS_ADDR") | ||
| 39 | config.write_text( | 40 | config.write_text( |
| 40 | 'enable_metrics = false\nenable_quic_addr_discovery = true\n' | 41 | f'enable_metrics = {json.dumps(bool(metrics))}\nenable_quic_addr_discovery = true\n' |
| 42 | + (f'metrics_bind_addr = {json.dumps(metrics)}\n' if metrics else '') + | ||
| 41 | f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' | 43 | f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' |
| 42 | '[tls]\ncert_mode = "Reloading"\n' | 44 | '[tls]\ncert_mode = "Reloading"\n' |
| 43 | f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' | 45 | f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' |
service/victoria/service.pkl+1| ... | @@ -11,6 +11,7 @@ containers { | ... | @@ -11,6 +11,7 @@ containers { |
| 11 | 11 | ||
| 12 | http { | 12 | http { |
| 13 | containerPort = 9428 | 13 | containerPort = 9428 |
| 14 | hostPort = 19428 | ||
| 14 | subdomain = "logs" | 15 | subdomain = "logs" |
| 15 | authRole = "infra-admin" | 16 | authRole = "infra-admin" |
| 16 | checkPath = "/health" | 17 | checkPath = "/health" |
tools/iroh-relay.md+3| ... | @@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and | ... | @@ -33,6 +33,9 @@ Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and |
| 33 | [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into | 33 | [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into |
| 34 | `/etc/systemd/system/`. | 34 | `/etc/systemd/system/`. |
| 35 | 35 | ||
| 36 | The unit sets `METRICS_ADDR`, so the relay serves metrics on `127.0.0.1:9091` for | ||
| 37 | [vmagent](vps.md). | ||
| 38 | |||
| 36 | [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as | 39 | [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as |
| 37 | `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main | 40 | `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main |
| 38 | Caddyfile. Validate before reloading Caddy. The ACME webroot is | 41 | Caddyfile. Validate before reloading Caddy. The ACME webroot is |
tools/vps.md created+39| ... | @@ -0,0 +1,39 @@ | ||
| 1 | # VPS logs and metrics | ||
| 2 | |||
| 3 | The VPS (`ssh root@paperclover.net`, hostname `usw-server-1`) pushes its logs and | ||
| 4 | metrics to Snow Globe's [ingest](../service/ingest/service.pkl) service at | ||
| 5 | `https://ingest.paperclover.net`, which files them in Victoria under `source=vps`. | ||
| 6 | The dashboard lists them under **vps services**; their ids are `vps.<name>`. | ||
| 7 | |||
| 8 | | Agent | Collects | Sends | | ||
| 9 | | --- | --- | --- | | ||
| 10 | | [vlagent](vps/vlagent.service) | `~clo/.pm2/logs/*.log`, plus the journal through `systemd-journal-upload` | `/insert/jsonline` | | ||
| 11 | | [vmagent](vps/vmagent.service) | the iroh relay's metrics on `127.0.0.1:9091`, plus [pm2-metrics.py](vps/pm2-metrics.py) pushes | `/api/v1/write` | | ||
| 12 | |||
| 13 | Ingest names each log line's `job` from its pm2 file (`site-out.log` is `site` on | ||
| 14 | stdout) or its systemd unit. pm2 writes no timestamps, so a line's time is when | ||
| 15 | vlagent read it. Both agents buffer up to 1 GB on disk while Snow Globe is | ||
| 16 | unreachable. | ||
| 17 | |||
| 18 | ```logsql | ||
| 19 | source:=vps job:=site level:=error | ||
| 20 | ``` | ||
| 21 | |||
| 22 | ## Install | ||
| 23 | |||
| 24 | Agents are the VictoriaMetrics release binaries, checksum-verified: | ||
| 25 | |||
| 26 | | Binary | Release tarball | SHA256 | | ||
| 27 | | --- | --- | --- | | ||
| 28 | | `/usr/local/bin/vlagent` | `vlutils-linux-amd64-v1.52.0.tar.gz` | `caa315764df0d11a77ba61d4ee96b783ddb98455b167da8326940dda62914bc9` | | ||
| 29 | | `/usr/local/bin/vmagent` | `vmutils-linux-amd64-v1.152.0.tar.gz` | `8eee4a98ff1665c60682475e8a8b292b8d718b63a2f023124384dd2f6a220c79` | | ||
| 30 | |||
| 31 | The ingest service's generated `token` secret lives in `/etc/snowglobe-ingest.token` | ||
| 32 | (root, mode 600). `vmagent.yml` and `pm2-metrics.py` go in | ||
| 33 | `/usr/local/lib/snowglobe-agents/`, the units in `/etc/systemd/system/`, and | ||
| 34 | [journal-upload.conf](vps/journal-upload.conf) in | ||
| 35 | `/etc/systemd/journal-upload.conf.d/snowglobe.conf` (package | ||
| 36 | `systemd-journal-remote`). Seed `/var/lib/systemd/journal-upload/state` with | ||
| 37 | `LAST_CURSOR=` and the current cursor, and run vlagent once against a sink until | ||
| 38 | its checkpoints reach the ends of the pm2 files; otherwise both send their whole | ||
| 39 | history, and the pm2 lines all get the same time. | ||
tools/vps/journal-upload.conf created+2| ... | @@ -0,0 +1,2 @@ | ||
| 1 | [Upload] | ||
| 2 | URL=http://127.0.0.1:9429/insert/journald | ||
tools/vps/pm2-metrics.py created+22| ... | @@ -0,0 +1,22 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Pushes each pm2 app's state to vmagent as Snow Globe service metrics.""" | ||
| 3 | import json | ||
| 4 | import subprocess | ||
| 5 | import time | ||
| 6 | from urllib.request import Request, urlopen | ||
| 7 | |||
| 8 | while True: | ||
| 9 | apps = json.loads(subprocess.run(["pm2", "jlist"], capture_output=True, check=True, text=True).stdout) | ||
| 10 | lines = [] | ||
| 11 | for app in apps: | ||
| 12 | env, monit = app["pm2_env"], app.get("monit") or {} | ||
| 13 | labels = "{service=%s}" % json.dumps("vps." + app["name"]) | ||
| 14 | lines += [ | ||
| 15 | f"snowglobe_service_up{labels} {int(env['status'] == 'online')}", | ||
| 16 | f"snowglobe_service_cpu_cores{labels} {monit.get('cpu', 0) / 100}", | ||
| 17 | f"snowglobe_service_memory_bytes{labels} {monit.get('memory', 0)}", | ||
| 18 | f"snowglobe_service_restarts_total{labels} {env.get('restart_time', 0)}", | ||
| 19 | f"snowglobe_service_started_seconds{labels} {env.get('pm_uptime', 0) / 1000}", | ||
| 20 | ] | ||
| 21 | urlopen(Request("http://127.0.0.1:8429/api/v1/import/prometheus", data="\n".join(lines).encode() + b"\n"), timeout=10).read() | ||
| 22 | time.sleep(15) | ||
tools/vps/pm2-metrics.service created+14| ... | @@ -0,0 +1,14 @@ | ||
| 1 | [Unit] | ||
| 2 | Description=Report pm2 apps to vmagent | ||
| 3 | After=vmagent.service | ||
| 4 | |||
| 5 | [Service] | ||
| 6 | User=clo | ||
| 7 | Environment=PATH=/home/clo/.local/bin:/usr/bin | ||
| 8 | ExecStart=/usr/bin/python3 /usr/local/lib/snowglobe-agents/pm2-metrics.py | ||
| 9 | Restart=always | ||
| 10 | RestartSec=15 | ||
| 11 | NoNewPrivileges=true | ||
| 12 | |||
| 13 | [Install] | ||
| 14 | WantedBy=multi-user.target | ||
tools/vps/vlagent.service created+23| ... | @@ -0,0 +1,23 @@ | ||
| 1 | [Unit] | ||
| 2 | Description=Ship VPS logs to Snow Globe | ||
| 3 | Wants=network-online.target | ||
| 4 | After=network-online.target | ||
| 5 | |||
| 6 | [Service] | ||
| 7 | LoadCredential=token:/etc/snowglobe-ingest.token | ||
| 8 | ExecStart=/usr/local/bin/vlagent -httpListenAddr=127.0.0.1:9429 -tmpDataPath=/var/lib/vlagent \ | ||
| 9 | -fileCollector.glob=/home/clo/.pm2/logs/*.log \ | ||
| 10 | -remoteWrite.url=https://ingest.paperclover.net/insert/jsonline -remoteWrite.format=jsonline \ | ||
| 11 | -remoteWrite.bearerTokenFile=%d/token -remoteWrite.maxDiskUsagePerURL=1GB | ||
| 12 | StateDirectory=vlagent | ||
| 13 | Restart=always | ||
| 14 | RestartSec=5 | ||
| 15 | NoNewPrivileges=true | ||
| 16 | ProtectSystem=strict | ||
| 17 | ProtectHome=read-only | ||
| 18 | PrivateTmp=true | ||
| 19 | PrivateDevices=true | ||
| 20 | MemoryMax=256M | ||
| 21 | |||
| 22 | [Install] | ||
| 23 | WantedBy=multi-user.target | ||
tools/vps/vmagent.service created+25| ... | @@ -0,0 +1,25 @@ | ||
| 1 | [Unit] | ||
| 2 | Description=Ship VPS metrics to Snow Globe | ||
| 3 | Wants=network-online.target | ||
| 4 | After=network-online.target | ||
| 5 | |||
| 6 | [Service] | ||
| 7 | DynamicUser=true | ||
| 8 | LoadCredential=token:/etc/snowglobe-ingest.token | ||
| 9 | ExecStart=/usr/local/bin/vmagent -httpListenAddr=127.0.0.1:8429 \ | ||
| 10 | -promscrape.config=/usr/local/lib/snowglobe-agents/vmagent.yml \ | ||
| 11 | -remoteWrite.url=https://ingest.paperclover.net/api/v1/write -remoteWrite.label=host=usw-server-1 \ | ||
| 12 | -remoteWrite.bearerTokenFile=%d/token \ | ||
| 13 | -remoteWrite.tmpDataPath=/var/lib/vmagent -remoteWrite.maxDiskUsagePerURL=1GB | ||
| 14 | StateDirectory=vmagent | ||
| 15 | Restart=always | ||
| 16 | RestartSec=5 | ||
| 17 | NoNewPrivileges=true | ||
| 18 | ProtectSystem=strict | ||
| 19 | ProtectHome=true | ||
| 20 | PrivateTmp=true | ||
| 21 | PrivateDevices=true | ||
| 22 | MemoryMax=256M | ||
| 23 | |||
| 24 | [Install] | ||
| 25 | WantedBy=multi-user.target | ||
tools/vps/vmagent.yml created+7| ... | @@ -0,0 +1,7 @@ | ||
| 1 | global: | ||
| 2 | scrape_interval: 30s | ||
| 3 | scrape_configs: | ||
| 4 | - job_name: iroh-relay | ||
| 5 | static_configs: | ||
| 6 | - targets: ["127.0.0.1:9091"] | ||
| 7 | labels: { service: "vps.iroh-relay" } | ||