diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index 6a04af03f796036881197bce513f8cd913bc3f57..3569b73fd9b2458ede76b57700d44fea601c5bd5 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -101,6 +101,7 @@ struct App { internal: Option<(url::Url, reqwest::Client)>, cache: cache::Cache, nomad_slots: Semaphore, + log_slots: Semaphore, specs: Mutex>, repo: PathBuf, data: PathBuf, @@ -314,6 +315,11 @@ async fn api(State(app): State>, request: Request) -> Result }); return Ok(Sse::new(stream).into_response()); } + if let ["services", id, "logs", "stream"] = parts.as_slice() + && method == Method::GET + { + return telemetry::log_stream(app, request, id, &query).await; + } if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() { if method == Method::GET { return vms::console(request, name, *kind == "serial").await; @@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box> { internal, cache: cache::Cache::default(), nomad_slots: Semaphore::new(4), + log_slots: Semaphore::new(32), specs: Mutex::new(HashMap::new()), repo: env("SNOWGLOBE_REPO", "..").into(), data: env("SNOWGLOBE_DATA_DIR", "data").into(), diff --git a/dashboard/src/telemetry.rs b/dashboard/src/telemetry.rs index 01a23fd0e7d9e11d8cb7a9d75bfe0a1006f225fd..5019e7a0738c53178702981a7cf75f4f588aa3c5 100644 --- a/dashboard/src/telemetry.rs +++ b/dashboard/src/telemetry.rs @@ -1,5 +1,6 @@ use crate::*; -use futures::{StreamExt, stream}; +use futures::{SinkExt, StreamExt, stream}; +use axum::extract::{FromRequestParts, ws::{Message, WebSocketUpgrade}}; pub async fn endpoint(app: Arc, service: &str) -> Result { if let Some((base, _)) = &app.internal { @@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result> { } Ok(found) } -pub async fn logs(app: Arc, id: &str, query: &HashMap) -> Result { - let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?; +fn log_query(id: &str, query: &HashMap) -> Result { let mut filters = vec![ "source:=nomad".to_owned(), format!("job:={}", serde_json::to_string(id)?), @@ -332,15 +332,36 @@ pub async fn logs(app: Arc, id: &str, query: &HashMap) -> R ))? )); } + Ok(filters.join(" ")) +} + +fn log_line(row: Value) -> Value { + json!({"id":row["id"].as_str().map(str::to_owned).unwrap_or_else(|| format!("{}:{}:{}", string(&row["_stream_id"]), string(&row["_time"]), string(&row["_msg"]))), + "sequence":number(&row["sequence"]), + "t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"), + "stream":row["stream"].as_str().filter(|s| ["stderr", "event"].contains(s)).unwrap_or("stdout"), + "level":row["level"],"text":row["_msg"].as_str().unwrap_or_default(), + "event":row["event"],"allocation":row["allocation"]}) +} + +pub async fn logs(app: Arc, id: &str, query: &HashMap) -> Result { + let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?; + if query.contains_key("after") && query.contains_key("before") + && query_number(query, "after", 0.0, f64::NEG_INFINITY, f64::INFINITY)? + > query_number(query, "before", 0.0, f64::NEG_INFINITY, f64::INFINITY)? + { + return Err(Error::new(400, "Choose a “to” time at or after “from”.")); + } + let offset = query_number(query, "offset", 0.0, 0.0, 1_000_000_000.0)?; let mut form = vec![ - ("query".into(), filters.join(" ")), + ("query".into(), format!("{} | sort by (_time desc, id desc, _stream_id desc, _msg desc) offset {offset:.0} limit {limit:.0}", log_query(id, query)?)), ("limit".into(), limit.to_string()), ]; - for (key, field, add) in [("after", "start", 0.000001), ("before", "end", 0.0)] { + for (key, field) in [("after", "start"), ("before", "end")] { if query.contains_key(key) { form.push(( field.into(), - (query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? + add) + query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? .to_string(), )); } @@ -352,12 +373,90 @@ pub async fn logs(app: Arc, id: &str, query: &HashMap) -> R let response = read(app, url, Some(form)).await?; let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| { let row: Value = serde_json::from_str(line)?; - Ok::<_,Error>(json!({"t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),"stream":if row["stream"] == "stderr" {"stderr"} else {"stdout"},"level":if row["level"] == "error" || row["level"] == "warn" {row["level"].clone()} else {Value::Null},"text":row["_msg"].as_str().unwrap_or_default()})) + Ok::<_,Error>(log_line(row)) }).collect::>>()?; rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"]))); Ok(json!(rows)) } +pub async fn log_stream(app: Arc, request: Request, id: &str, query: &HashMap) -> Result { + vms::same_origin(&request)?; + let filter = log_query(id, query)?; + let mut cursor = query_number(query, "cursor", 0.0, 0.0, 9_007_199_254_740_991.0)?; + let from = query.get("from").map(|_| query_number(query, "from", 0.0, 0.0, f64::INFINITY)).transpose()?; + let base = endpoint(app.clone(), "victoria-logs").await?; + let url = format!("{base}/select/logsql/query"); + let headers = request.headers().clone(); + let (mut parts, _) = request.into_parts(); + let upgrade = WebSocketUpgrade::from_request_parts(&mut parts, &()).await + .map_err(|_| Error::new(400, "Open the service log to connect."))? + .max_message_size(1024).max_frame_size(1024); + Ok(upgrade.on_upgrade(move |mut socket| async move { + let run = async { + let _slot = app.log_slots.try_acquire().map_err(|_| Error::new(429, "Too many live logs are open. Close one and retry."))?; + let mut first = true; + let mut poll = tokio::time::interval(Duration::from_secs(1)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut heartbeat = tokio::time::interval(Duration::from_secs(15)); + loop { + tokio::select! { + message = socket.recv() => match message { + Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?, + Some(Ok(Message::Pong(_))) => {}, + _ => break, + }, + _ = heartbeat.tick() => { + if app.auth.ready() { + let account = app.auth.session(&headers, "dashboard")?; + if !array(&account["groups"]).iter().any(|g| g["name"] == "infra-admin") { + return Err(Error::new(401, "Sign in as an administrator to continue reading logs.")); + } + } + tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Ping(Bytes::new()))).await + .map_err(|_| Error::new(504, "The log connection stopped responding."))??; + }, + _ = poll.tick() => { + let checkpoint = read(app.clone(), url.clone(), Some(vec![("query".into(), "source:=log-shipper | stats max(sequence) as sequence".into())])).await?; + let checkpoint: Value = serde_json::from_str(string(&checkpoint.value).lines().find(|line| !line.trim().is_empty()).unwrap_or("{}"))?; + let settled = number(&checkpoint["sequence"]); + if first { cursor = cursor.min(settled); first = false; } + let mut form = vec![("query", format!("{filter} sequence:>{cursor:.0} sequence:<={settled:.0} | sort by (sequence) limit 5000"))]; + if let Some(from) = from { form.push(("start", from.to_string())); } + let request = if env("SNOWGLOBE_PREVIEW", "") == "1" { + app.request(Method::GET, &url)?.query(&form) + } else { + app.request(Method::POST, &url)?.form(&form) + }; + let response = request.timeout(Duration::from_secs(8)).send().await?; + if !response.status().is_success() { return Err(Error::new(502, "The log store isn't answering. Retrying…")); } + let mut chunks = response.bytes_stream(); + let mut body = Vec::new(); + while let Some(chunk) = chunks.next().await { + body.extend_from_slice(&chunk?); + if body.len() > 8 * 1024 * 1024 { + let end = body[..8 * 1024 * 1024].iter().rposition(|b| *b == b'\n') + .ok_or_else(|| Error::new(502, "A log entry is too large. Narrow the search and retry."))?; + body.truncate(end + 1); + break; + } + } + let lines = std::str::from_utf8(&body)?.lines().filter(|l| !l.trim().is_empty()) + .map(|raw| serde_json::from_str(raw).map(log_line)).collect::, _>>()?; + for line in &lines { cursor = cursor.max(number(&line["sequence"])); } + tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Text(json!({"lines":lines,"cursor":cursor}).to_string().into()))).await + .map_err(|_| Error::new(504, "The log connection stopped responding."))??; + } + } + } + Ok::<_, Error>(()) + }.await; + if let Err(error) = run { + let _ = tokio::time::timeout(Duration::from_secs(2), socket.send(Message::Text(json!({"error":error.message}).to_string().into()))).await; + } + let _ = tokio::time::timeout(Duration::from_secs(1), socket.close()).await; + }).into_response()) +} + fn span(row: &Value) -> Value { let mut attributes: serde_json::Map = row .as_object() diff --git a/dashboard/src/vms.rs b/dashboard/src/vms.rs index f290e53c13ea9cda6fdc06874ba3e384856ef7d2..df237b28e34decaa9107dfe6793afa6b9380acf4 100644 --- a/dashboard/src/vms.rs +++ b/dashboard/src/vms.rs @@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result Result<()> { +pub(crate) fn same_origin(request: &Request) -> Result<()> { let supplied = request .headers() .get("origin") @@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> { }; authority != host || !matches!(origin.scheme(), "http" | "https") }) { - return Err(Error::new(403, "Open the VM page on this site to connect.")); + return Err(Error::new(403, "Open this page on the same site to connect.")); } Ok(()) } diff --git a/dashboard/web/components/LogView.css b/dashboard/web/components/LogView.css index c55c47adaaf8525817fdb0118971c9c8978bec44..b0c038f6a45f28f82740564a8b124c3ec393a9e7 100644 --- a/dashboard/web/components/LogView.css +++ b/dashboard/web/components/LogView.css @@ -18,6 +18,8 @@ .log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); } .log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); } .log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); } +.log-view .task-event { background: color-mix(in srgb, var(--text) 4%, transparent); } +.log-view .event-label { margin-right: 8px; font: 600 11px var(--sans); } .log-view .ansi { color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text)); background: color-mix(in srgb, var(--bg, transparent) 45%, transparent); diff --git a/dashboard/web/components/LogView.tsx b/dashboard/web/components/LogView.tsx index 1af7e4395e3ec3d4db8413f836f47eaeddac3035..91236182447821746ee60605304638c1aa294737 100644 --- a/dashboard/web/components/LogView.tsx +++ b/dashboard/web/components/LogView.tsx @@ -132,9 +132,10 @@ export function LogView(props: {
{date(line.t)}
{props.before?.(line, index)} -
+
+ {line.event}{" "} {line.container}}> {(filter) => ( diff --git a/dashboard/web/pages/Service.tsx b/dashboard/web/pages/Service.tsx index 5c8aca5815dbcae6aab65e7a48ea7ecb53019a9e..1b6b1343514b7115a14270a5e4724784fe5d9505 100644 --- a/dashboard/web/pages/Service.tsx +++ b/dashboard/web/pages/Service.tsx @@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw"; import Search from "lucide-solid/icons/search"; import X from "lucide-solid/icons/x"; import { - createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, + createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, untrack, } from "solid-js"; import { Portal } from "solid-js/web"; import { @@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con /** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */ type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; -/** Each service's latest unfiltered lines, so returning to a service draws its logs at once. */ -const recent = new Map(); - /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { const [params, setParams] = useSearchParams(); const at = () => (params.at ? Number(params.at) : null); const bounded = () => at() !== null || !!params.to; - const unfiltered = () => !params.q && !params.level && !params.from && !params.to && at() === null; const dateValue = (value?: string) => { if (!value) return ""; const date = new Date(Number(value) * 1000); return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16); }; - const [older, setOlder] = createSignal([]); - const [newer, setNewer] = createSignal([]); + const [all, setLines] = createSignal([]); + let pending: LogLine[] = []; + let generation = 0; + let older: { before: string; offset: number } | undefined; const [exhausted, setExhausted] = createSignal(false); const [loadingOlder, setLoadingOlder] = createSignal(false); const [follow, setFollow] = createSignal(true); const [unseen, setUnseen] = createSignal(0); - /** Why the last poll failed, until one succeeds. */ const [stalled, setStalled] = createSignal(); - const fetchLines = (limit: number, extra: { before?: string; after?: string }) => + const [connected, setConnected] = createSignal(false); + const merge = (before: LogLine[], after: LogLine[]) => [...new Map([...before, ...after] + .map((line) => [line.id ?? JSON.stringify(line), line])).values()].sort((a, b) => a.t - b.t); + const resume = () => { + older = undefined; + setLines((lines) => merge(lines, pending).slice(-10_000)); + pending = []; + setUnseen(0); + setExhausted(false); + setFollow(true); + }; + const fetchLines = (limit: number, extra: { before?: string; after?: string; offset?: string }) => parseResponse(api.services[":id"].logs.$get({ param: { id: props.service.id }, query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit), @@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { const [first, { refetch }] = createResource( // A string key, since a fresh array from an unrelated rerun would count as a change and refetch. () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]), - (_, { refetching }) => { - setOlder([]); - setNewer([]); + () => { + const current = ++generation; + older = undefined; + setLines([]); + pending = []; + setLoadingOlder(false); setExhausted(false); setFollow(!bounded()); setUnseen(0); setStalled(); - const cached = !refetching && unfiltered() && recent.get(props.service.id); - if (cached) { - queueMicrotask(() => poll()); - return cached; - } - return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }); + return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }).then((lines) => { + if (current === generation) setLines(lines); + return lines; + }); }, ); - const loaded = lastGood(first); - const all = createMemo(() => [...older(), ...(loaded() ?? []), ...newer()]); - createEffect(() => first.state === "ready" && unfiltered() && recent.set(props.service.id, all().slice(-1000))); const loadOlder = async () => { const oldest = all()[0]; if (!oldest || loadingOlder() || exhausted()) return; - if (params.from && oldest.t <= Number(params.from)) return setExhausted(true); + setLoadingOlder(true); + const current = generation; try { - const lines = await fetchLines(5000, { before: String(oldest.t) }); - if (!lines.length) setExhausted(true); - setOlder([...lines, ...older()]); + const page = older ??= { before: String(oldest.t + 0.000001), offset: 0 }; + const fetched = await fetchLines(5000, { before: page.before, offset: String(page.offset) }); + if (current !== generation || older !== page) return; + const known = new Set(all().map((line) => line.id ?? JSON.stringify(line))); + const lines = fetched.filter((line) => !known.has(line.id ?? JSON.stringify(line))); + page.offset += fetched.length; + if (fetched.length < 5000) setExhausted(true); + setLines((previous) => merge(lines, previous)); } catch (failure) { - toast(`Couldn't load older lines. ${reason(failure)}`); + if (current === generation) toast(`Couldn't load older lines. ${reason(failure)}`); } finally { - setLoadingOlder(false); + if (current === generation) setLoadingOlder(false); } }; - const poll = async () => { - const last = all().at(-1); - if (bounded() || !last || first.loading) return; - let lines: LogLine[]; - try { - lines = await fetchLines(1000, { after: String(last.t) }); - setStalled(); - } catch (failure) { - setStalled(reason(failure)); - return; - } - // A search changed while this was under way, so the lines belong to the old one. - if (!lines.length || first.loading || last !== all().at(-1)) return; - setNewer([...newer(), ...lines]); - if (!follow()) setUnseen(unseen() + lines.length); - }; - const timer = setInterval(poll, 2000); - onCleanup(() => clearInterval(timer)); + createEffect(() => { + const id = props.service.id; + const q = params.q ?? ""; + const level = params.level; + const from = params.from; + setConnected(false); + if (bounded() || first.state !== "ready") return; + let disposed = false; + let failures = 0; + let cursor = Math.max(0, ...untrack(all).map((line) => line.sequence ?? 0)); + let socket: WebSocket; + let retry: ReturnType | undefined; + let stale: ReturnType | undefined; + const connect = () => { + const url = new URL(`/api/services/${encodeURIComponent(id)}/logs/stream`, location.href); + url.protocol = location.protocol === "https:" ? "wss:" : "ws:"; + url.searchParams.set("q", q); + if (level) url.searchParams.set("level", level); + url.searchParams.set("cursor", String(cursor)); + if (from) url.searchParams.set("from", from); + socket = new WebSocket(url); + socket.onmessage = (event) => { + if (disposed) return; + const message: { lines?: LogLine[]; cursor?: number; error?: string } = JSON.parse(event.data); + if (message.error) { + setConnected(false); + setStalled(message.error); + return; + } + failures = 0; + setConnected(true); + setStalled(); + clearTimeout(stale); + stale = setTimeout(() => socket.close(), 35_000); + if (message.lines?.length) { + const known = new Set([...all(), ...pending].map((line) => line.id ?? JSON.stringify(line))); + const lines = message.lines.filter((line) => { + const id = line.id ?? JSON.stringify(line); + if (known.has(id)) return false; + known.add(id); + return true; + }); + if (follow()) { + older = undefined; + setLines((previous) => merge(previous, lines).slice(-10_000)); + } + else { + pending = merge(pending, lines).slice(-10_000); + setUnseen((previous) => previous + lines.length); + } + } + cursor = message.cursor ?? cursor; + }; + socket.onclose = () => { + if (disposed) return; + clearTimeout(stale); + setConnected(false); + setStalled((previous) => previous ?? "Live logs disconnected. Retrying…"); + retry = setTimeout(connect, Math.min(30, 2 ** failures++) * 1000); + }; + }; + connect(); + onCleanup(() => { + disposed = true; + clearTimeout(retry); + clearTimeout(stale); + socket.close(); + }); + }); const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!))); const hints = () => [ ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1) .map((container) => [`container:${container.name}`, "one container's lines"] as const), ["stream:stderr", "error output only"], + ["stream:event", "starts, restarts, and OOM kills"], ["-health", "hide lines with a word"], ['"connection refused"', "an exact phrase"], ] as const; @@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { {(link) => {link().app.name}} - @@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { onStart={loadOlder} onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })} onFollow={(end) => { - setFollow(end && at() === null); - if (end) setUnseen(0); + if (end && !bounded()) resume(); + else setFollow(false); }} - onLatest={() => (at() === null ? setFollow(true) : setParams({ at: undefined }))} + onLatest={() => (at() === null ? resume() : setParams({ at: undefined }))} before={(line, index) => ( <> @@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) { } /** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */ -function Liveness(props: { every: string; paused?: string; failure?: string }) { - const health = () => (props.failure ? "degraded" : props.paused ? "stopped" : "healthy"); +function Liveness(props: { every: string; paused?: string; failure?: string; connecting?: boolean }) { + const health = () => (props.failure ? "degraded" : props.paused || props.connecting ? "stopped" : "healthy"); return ( - {props.failure ? "reconnecting" : props.paused ? "paused" : "live"} + data-tip={props.failure ?? props.paused ?? (props.connecting ? "Connecting to live logs…" : `New ${props.every}`)}> + {props.failure ? props.paused ? "unavailable" : "reconnecting" : props.paused ? "paused" : props.connecting ? "connecting" : "live"} ); } diff --git a/dashboard/web/types/model.ts b/dashboard/web/types/model.ts index 50ffa8556eb78aac287118b25f19f219667b6b8e..f8540bca4367e954c2ba9a1d492893cbca4e84c4 100644 --- a/dashboard/web/types/model.ts +++ b/dashboard/web/types/model.ts @@ -206,11 +206,15 @@ export interface TraceSummary { } export interface LogLine { + id?: string; + sequence?: number; t: number; container: string; - stream: "stdout" | "stderr"; + stream: "stdout" | "stderr" | "event"; level: "debug" | "info" | "warn" | "error" | null; text: string; + event?: string | null; + allocation?: string | null; } export interface HostInfo { diff --git a/nixos/dashboard.nix b/nixos/dashboard.nix index da3ea8233fc1b5eef4c0c692a9266bbd84135c11..1257893d8777c1d8a68e038179ba15fcc0165737 100644 --- a/nixos/dashboard.nix +++ b/nixos/dashboard.nix @@ -24,7 +24,7 @@ let inherit (finalAttrs) pname version src sourceRoot; pnpm = pnpm_10; fetcherVersion = 3; - hash = "sha256-zux7++pfJ9YJAMPjnvW94Z2m884Ypnc6BK9iHJxRFnA="; + hash = "sha256-Jhk4mAOCqoljelFJfC7o+3t66x9GPFYb2e928jeaPSk="; }; nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ]; buildPhase = "pnpm run build"; diff --git a/tools/dashboard-logs-test.py b/tools/dashboard-logs-test.py new file mode 100644 index 0000000000000000000000000000000000000000..7fc27eb85fef8dc18c6fe3a85ff2583accade19b --- /dev/null +++ b/tools/dashboard-logs-test.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +import argparse +import base64 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import struct +import re +import sqlite3 +import hashlib +import subprocess +import tempfile +import threading +import time +from urllib.parse import parse_qs, urlsplit +from urllib.request import Request, urlopen +from urllib.error import HTTPError + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--preview', action='store_true') + parser.add_argument('--binary', type=Path, default=Path('dashboard/target/debug/home-dashboard')) + args = parser.parse_args() + forms = [] + failed = threading.Event() + row = {'_time': '2026-10-14T02:50:00Z', '_msg': 'Out of memory (OOM): exit 137', + 'id': 'allocation:app:1:Terminated', 'task': 'app', 'stream': 'event', 'event': 'OOM', + 'level': 'error', 'allocation': 'allocation', 'job': 'shale', 'source': 'nomad', 'sequence': 1} + rows = [row] + checkpoint = 0 + + class Fixture(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def handle(self): + try: + super().handle() + except (ConnectionResetError, BrokenPipeError): + pass + + def do_GET(self): + if urlsplit(self.path).path == '/select/logsql/query': + self.query(parse_qs(urlsplit(self.path).query)) + else: + self.reply([{'Address': '127.0.0.1', 'Port': self.server.server_port}]) + + def reply(self, value): + data = json.dumps(value).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self): + form = parse_qs(self.rfile.read(int(self.headers['Content-Length'])).decode()) + self.query(form) + + def query(self, form): + forms.append((self.path, form)) + if failed.is_set(): + self.send_error(503) + return + if 'source:=log-shipper' in form['query'][0]: + if checkpoint == 0: + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + else: + self.reply({'sequence': checkpoint if checkpoint is not None else max(r['sequence'] for r in rows)}) + return + selected = rows[:] + match = re.search(r'sequence:>([0-9]+)', form['query'][0]) + if match: + selected = [r for r in selected if r['sequence'] > int(match[1])] + match = re.search(r'sequence:<=(\d+)', form['query'][0]) + if match: + selected = [r for r in selected if r['sequence'] <= int(match[1])] + if '| sort by (sequence)' in form['query'][0]: + selected.sort(key=lambda r: r['sequence']) + elif '| sort by (_time desc' in form['query'][0]: + selected.sort(key=lambda r: (r['_time'], r['id']), reverse=True) + offset = int(re.search(r'offset (\d+)', form['query'][0])[1]) + limit = int(re.search(r'limit (\d+)', form['query'][0])[1]) + selected = selected[offset:offset + limit] + self.send_response(200) + data = b''.join((json.dumps(r) + '\n').encode() for r in selected) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data[:17]) + self.wfile.flush() + try: + self.wfile.write(data[17:]) + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + pass + + def log_message(self, *args): + pass + + with tempfile.TemporaryDirectory(prefix='dashboard-logs-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as fixture: + fixture.daemon_threads = True + threading.Thread(target=fixture.serve_forever, daemon=True).start() + root = Path(temporary).resolve() + (root / 'data').mkdir() + (root / 'token').write_text('disposable') + with socket.socket() as reservation: + reservation.bind(('127.0.0.1', 0)) + port = reservation.getsockname()[1] + origin = f'http://127.0.0.1:{port}' + environment = {**os.environ, 'PORT': str(port), 'SNOWGLOBE_DATA_DIR': str(root / 'data'), + 'SNOWGLOBE_PUBLIC_ORIGIN': 'https://globe.studio.test', 'SNOWGLOBE_NOMAD_TOKEN_FILE': str(root / 'token'), + 'NOMAD_ADDR': f'http://127.0.0.1:{fixture.server_port}'} + environment['SNOWGLOBE_PREVIEW'] = '1' if args.preview else '0' + for name in ['SNOWGLOBE_PROXY_TOKEN_FILE', 'SNOWGLOBE_INTERNAL_URL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_YT_STATE']: + environment.pop(name, None) + with (root / 'server.log').open('w') as log: + server = subprocess.Popen([str(args.binary.resolve())], env=environment, stdout=log, stderr=log) + try: + deadline = time.monotonic() + 10 + while True: + try: + with socket.create_connection(('127.0.0.1', port), timeout=.1): break + except OSError: + assert server.poll() is None, (root / 'server.log').read_text() + assert time.monotonic() < deadline + time.sleep(.05) + path = '/api/services/shale/logs' + with urlopen(Request(origin + path + '?level=error&q=stream:event', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: + lines = json.load(response) + assert lines[0]['stream'] == 'event' and lines[0]['event'] == 'OOM' + assert lines[0]['level'] == 'error' and lines[0]['id'] == row['id'] + assert 'level:=error' in forms[-1][1]['query'][0] + assert 'stream:~"(?i)^event"' in forms[-1][1]['query'][0] + try: + urlopen(Request(origin + path + '?after=2&before=1', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) + raise AssertionError('reversed bounds accepted') + except HTTPError as error: + assert error.code == 400 and b'at or after' in error.read() + + def connect(expected, supplied_origin=origin, groups='infra-admin', actor='fixture', cursor=0, cookie=None): + sock = socket.create_connection(('127.0.0.1', port), timeout=5) + sock.settimeout(20) + stream = sock.makefile('rb') + headers = {'Host': urlsplit(origin).netloc, 'Upgrade': 'websocket', 'Connection': 'Upgrade', + 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': base64.b64encode(os.urandom(16)).decode()} + if actor is not None: headers['User-Name'] = actor + if cookie is not None: headers['Cookie'] = '__Host-snow-session=' + cookie + if groups is not None: headers['User-Groups'] = groups + if supplied_origin is not None: headers['Origin'] = supplied_origin + sock.sendall(('GET ' + path + f'/stream?q=stream:event&cursor={cursor}&from=1 HTTP/1.1\r\n' + ''.join(f'{k}: {v}\r\n' for k, v in headers.items()) + '\r\n').encode()) + status = int(stream.readline().split()[1]) + assert status == expected, status + while stream.readline().strip(): pass + if expected != 101: + stream.close() + sock.close() + return sock, stream + + connect(401, actor=None) + connect(403, groups='metrics') + connect(403, supplied_origin=None) + connect(403, supplied_origin='http://other.invalid') + sock, stream = connect(101) + try: + def receive(): + head = stream.read(2) + assert len(head) == 2, 'stream closed unexpectedly' + size = head[1] & 127 + if size == 126: size = struct.unpack('!H', stream.read(2))[0] + if size == 127: size = struct.unpack('!Q', stream.read(8))[0] + if head[0] & 15 == 9: + payload = stream.read(size) + mask = os.urandom(4) + sock.sendall(bytes([138, 128 | size]) + mask + bytes(v ^ mask[i % 4] for i, v in enumerate(payload))) + return receive() + assert head[0] & 15 == 1, head + return json.loads(stream.read(size)) + message = receive() + assert message == {'lines': [], 'cursor': 0} + checkpoint = None + while not (message := receive()).get('lines'): pass + assert message['lines'] == lines and message['cursor'] == 1 + assert forms[-1][1]['start'] == ['1'] + rows.append({**row, 'id': 'same-time', 'sequence': 2}) + while not (message := receive()).get('lines'): pass + assert message['lines'][0]['id'] == 'same-time' and message['cursor'] == 2 + rows.append({**row, '_time': '2026-10-14T01:00:00Z', 'id': 'late-oom', 'sequence': 3}) + while not (message := receive()).get('lines'): pass + assert message['lines'][0]['id'] == 'late-oom' and message['cursor'] == 3 + failed.set() + while not (message := receive()).get('error'): pass + assert 'store' in message['error'] + finally: + stream.close() + sock.close() + failed.clear() + sock, stream = connect(101, cursor=2) + try: + while not (message := receive()).get('lines'): pass + assert [r['id'] for r in message['lines']] == ['late-oom'] + finally: + stream.close() + sock.close() + bulk = [{**row, 'id': f'bulk-{i}', 'sequence': i + 4, '_msg': 'x' * 3500} for i in range(3000)] + rows.extend(bulk) + sock, stream = connect(101, cursor=3) + try: + received = [] + while len(received) < 3000: + message = receive() + assert not message.get('error'), message + received.extend(r['id'] for r in message['lines']) + assert received == [r['id'] for r in bulk] + finally: + stream.close() + sock.close() + accepted = time.time() + sequence = int(accepted * 1_000_000) + delayed = {**row, 'id': 'delayed-search-visibility', 'sequence': sequence} + checkpoint = 3003 + rows.append({**row, 'id': 'initial-unsettled', 'sequence': sequence + 1}) + def publish(): + nonlocal checkpoint + rows.append(delayed) + checkpoint = sequence + 1 + threading.Timer(7, publish).start() + sock, stream = connect(101, cursor=sequence + 1) + try: + while not (message := receive()).get('lines'): pass + assert [r['id'] for r in message['lines']] == [delayed['id'], 'initial-unsettled'] + assert time.time() - accepted >= 7 + finally: + stream.close() + sock.close() + rows[:] = [{**row, 'id': f'tie-{i:05}', 'sequence': i + 1} for i in range(6000)] + [{**row, '_time': '2026-10-13T00:00:00Z', 'id': 'older-history', 'sequence': 6001}] + history = [] + for offset in (0, 5000): + with urlopen(Request(origin + path + f'?limit=5000&offset={offset}', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: + history.extend(json.load(response)) + assert len({line['id'] for line in history}) == 6001 + assert history[-1]['id'] == 'older-history' + cookie = 'disposable-native-session' + with sqlite3.connect(root / 'data/accounts.sqlite') as db: + db.execute('INSERT INTO users VALUES (?,?)', ('fixture', json.dumps({'username':'fixture','enabled':True,'requiredActions':[]}))) + role = db.execute('SELECT id FROM roles WHERE name=?', ('infra-admin',)).fetchone()[0] + db.execute('INSERT INTO memberships VALUES (?,?)', ('fixture', role)) + db.execute('INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?)', (hashlib.sha256(cookie.encode()).hexdigest(),'fixture','dashboard',int(time.time())+120,'127.0.0.1',0,0,0)) + connect(401) + sock, stream = connect(101, cursor=sequence, cookie=cookie) + try: + while 'lines' not in (message := receive()): pass + with sqlite3.connect(root / 'data/accounts.sqlite') as db: + db.execute('DELETE FROM sessions') + while not (message := receive()).get('error'): pass + assert 'administrator' in message['error'] + finally: + stream.close() + sock.close() + print('REST events/filters, WebSocket auth/Origin, fragmented responses, timestamp ties, delayed OOM, reconnect cursor, oversized backlog, delayed visibility, session revocation, upstream failure: passed') + finally: + server.terminate() + server.wait(timeout=10) + fixture.shutdown() + + +if __name__ == '__main__': + main() diff --git a/tools/log-shipper-test.py b/tools/log-shipper-test.py new file mode 100644 index 0000000000000000000000000000000000000000..5b488387de2ee51413c45ef7018dbe9d43ab150d --- /dev/null +++ b/tools/log-shipper-test.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py")) +shipper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(shipper) + + +class Journal: + def __init__(self, *args, **kwargs): + self.stdout = io.StringIO("") + self.returncode = 0 + + def wait(self): + return self.returncode + + +class Shipping(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + root = Path(self.temporary.name) + self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0" + self.log.parent.mkdir(parents=True) + self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n") + self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [ + {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137, + "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}}, + {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"}, + {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"}, + ]}}}] + self.batches = [] + self.checkpoints = [] + self.visible = None + replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf"))) + replacement.start() + self.addCleanup(replacement.stop) + replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode())) + replacement.start() + self.addCleanup(replacement.stop) + for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]: + replacement = patch.object(shipper, target, value) + replacement.start() + self.addCleanup(replacement.stop) + replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}]) + replacement.start() + self.addCleanup(replacement.stop) + replacement = patch.object(shipper.subprocess, "Popen", Journal) + replacement.start() + self.addCleanup(replacement.stop) + + def once(self): + state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {} + def deliver(base, rows, fields): + if rows and rows[0]["source"] == "log-shipper": + self.checkpoints.append(len(self.batches)) + else: + self.batches.extend(rows) + with patch.object(shipper, "send", side_effect=deliver): + shipper.once(state) + + def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self): + self.once() + self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"]) + self.assertEqual(self.batches[1]["level"], "error") + self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"]) + self.assertEqual(self.batches[2]["level"], "warn") + self.assertEqual(len({row["id"] for row in self.batches}), 4) + self.assertEqual(len({row["sequence"] for row in self.batches}), 4) + self.once() + self.assertEqual(len(self.batches), 4) + self.allocations.clear() + self.once() + self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {}) + self.assertEqual(len(self.batches), 4) + + def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self): + self.once() + line = b"2026-10-14T02:50:01Z stdout F same message" + with self.log.open("ab") as file: + file.write(line) + self.once() + self.assertEqual(len(self.batches), 4) + with self.log.open("ab") as file: + file.write(b"\n" + line + b"\n") + self.once() + self.assertEqual(len(self.batches), 6) + self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"]) + self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"]) + self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"]) + + def test_failed_delivery_does_not_advance_file_or_event_cursors(self): + with patch.object(shipper, "send", side_effect=OSError("store unavailable")): + with self.assertRaises(OSError): + shipper.once({}) + self.assertNotIn("events", json.loads(shipper.STATE.read_text())) + self.assertFalse(json.loads(shipper.STATE.read_text()).get("files")) + self.once() + self.assertEqual(len(self.batches), 4) + + def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self): + with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)): + self.once() + self.assertEqual(self.batches, []) + self.assertEqual(self.checkpoints, []) + self.assertTrue(json.loads(shipper.STATE.read_text())["events"]) + self.once() + self.assertEqual(self.batches, []) + + def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self): + self.visible = 3 + with patch.object(shipper.time, "monotonic", side_effect=[0, 31]): + with self.assertRaises(TimeoutError): + self.once() + self.assertEqual(self.checkpoints, []) + reserved = json.loads(shipper.STATE.read_text()) + self.assertFalse(reserved.get("files")) + self.assertNotIn("events", reserved) + self.visible = 4 + self.log.unlink() + self.allocations.clear() + with patch.object(shipper.time, "time_ns", return_value=0): + self.once() + self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"]) + self.assertEqual(self.batches[:4], self.batches[-4:]) + self.assertEqual(len(self.checkpoints), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/log-shipper.py b/tools/log-shipper.py index a0f5ce5f93a5d107b0fb930e54b57595a616f1de..ed979d0313886d71f61981ee44aaa164cd155a70 100644 --- a/tools/log-shipper.py +++ b/tools/log-shipper.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import datetime import json +import math import os from pathlib import Path import re import subprocess import time +from urllib.parse import urlencode from urllib.request import Request, urlopen @@ -31,44 +33,132 @@ def send(base, rows, fields): response.read() +def retention_bounds(base): + with urlopen(base + "/metrics", timeout=10) as response: + metrics = response.read().decode() + flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics)) + durations = {} + units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000} + for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"): + value = flags[name] + if re.fullmatch(r"[0-9.]+M?", value): + seconds = float(value.rstrip("M")) * 2678400 + else: + parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower()) + if "".join(number + unit for number, unit in parts) != value.lower(): + raise ValueError("unsupported log retention duration") + seconds = sum(float(number) * units[unit] for number, unit in parts) + if not math.isfinite(seconds) or seconds < 0: + raise ValueError("invalid log retention duration") + durations[name] = int(seconds * 1000) / 1000 + retention = max(86400, durations["retentionPeriod"]) + backfill = durations["maxBackfillAge"] + backfill = backfill if 0 < backfill <= retention else retention + current = time.time() + return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400 + + +def save(state): + pending_state = STATE.with_suffix(".pending") + with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file: + json.dump(state, file) + os.replace(pending_state, STATE) + + def once(state): services = nomad("/v1/service/victoria-logs") if not services: return target = services[0] base = f"http://{target['Address']}:{target['Port']}" - jobs = {item["ID"]: item["JobID"] for item in nomad("/v1/allocations")} - offsets = state.setdefault("files", {}) - rows = [] - pending = {} - for file in sorted(LOGS.glob("*/alloc/logs/*")): - match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name) - job = jobs.get(file.parent.parent.parent.name) - if not match or not job: - continue - info = file.stat() - old = offsets.get(str(file), {}) - offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0 - with file.open("rb") as stream: - stream.seek(offset) - chunk = stream.read(262144) - if not chunk: + if "pending" not in state: + allocations = nomad("/v1/allocations") + jobs = {item["ID"]: item["JobID"] for item in allocations} + offsets = state.get("files", {}).copy() + rows = [] + pending = {} + for file in sorted(LOGS.glob("*/alloc/logs/*")): + match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name) + job = jobs.get(file.parent.parent.parent.name) + if not match or not job: continue - if not chunk.endswith(b"\n"): - chunk = chunk[:chunk.rfind(b"\n") + 1] - if not chunk: - continue - pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)} - for raw in chunk.decode("utf8", "replace").splitlines(): - parsed = CRI.match(raw) - if not parsed: - continue - stamp, stream, message = parsed.groups() - level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info" - rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job, - "task": match.group(1), "stream": stream, "level": level}) - send(base, rows, "source,job,task,stream") - offsets.update(pending) + info = file.stat() + old = offsets.get(str(file), {}) + offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0 + with file.open("rb") as stream: + stream.seek(offset) + chunk = stream.read(262144) + if not chunk: + continue + if not chunk.endswith(b"\n"): + chunk = chunk[:chunk.rfind(b"\n") + 1] + if not chunk: + continue + pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)} + position = offset + for raw in chunk.splitlines(keepends=True): + identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}" + position += len(raw) + parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n")) + if not parsed: + continue + stamp, stream, message = parsed.groups() + level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info" + rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job, + "task": match.group(1), "stream": stream, "level": level, + "id": identity, "allocation": file.parent.parent.parent.name}) + events = {} + for allocation in allocations: + seen = set(state.get("events", {}).get(allocation["ID"], [])) + current = events[allocation["ID"]] = [] + for task, task_state in (allocation.get("TaskStates") or {}).items(): + for event in task_state.get("Events") or []: + kind = event["Type"] + identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}" + current.append(identity) + if identity in seen: + continue + oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true" + message = event.get("DisplayMessage") or event.get("Message") or kind + if oom: + message = "Out of memory (OOM): " + message + stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat() + level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info" + rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"], + "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind, + "id": identity, "allocation": allocation["ID"]}) + sequence = max(state.get("sequence", 0), time.time_ns() // 1000) + for row in rows: + sequence += 1 + row["sequence"] = sequence + offsets.update(pending) + state["sequence"] = sequence + state["pending"] = {"rows": rows, "files": offsets, "events": events} + save(state) + batch = state["pending"] + rows = batch["rows"] + sequence = state["sequence"] + if rows: + minimum, maximum = retention_bounds(base) + rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum] + if rows: + send(base, rows, "source,job,task,stream") + query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows" + deadline = time.monotonic() + 30 + while True: + request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode()) + with urlopen(request, timeout=10) as response: + visible = json.loads(response.read() or b"{}") + if int(visible.get("rows", 0)) == len(rows): + break + if time.monotonic() >= deadline: + raise TimeoutError("log batch is not searchable yet") + time.sleep(.25) + send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source") + state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()} + state["events"] = batch["events"] + del state["pending"] + save(state) command = ["journalctl", "--no-pager", "--output=json"] command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] @@ -97,9 +187,7 @@ def once(state): send(base, system, "source,unit") if cursor: state["cursor"] = cursor - pending_state = STATE.with_suffix(".pending") - pending_state.write_text(json.dumps(state)) - os.replace(pending_state, STATE) + save(state) def main(): @@ -110,7 +198,7 @@ def main(): once(state) except Exception as error: print(f"log shipping paused: {error}", flush=True) - time.sleep(5) + time.sleep(1) if __name__ == "__main__":