| author | |
| committer | |
| log | b00a4bc62d1b4f4972f879356490535a4b4ea9dc |
| tree | 038d563d955aba396ea8f0e0f9dbb57b99ddde40 |
| parent | 9980b1fb6b031ee32f458b5f17b8fc33b536c1cf |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Retain application output and lifecycle events across reconnects, delivery retries and allocation cleanup. Gate streaming on verified search visibility; preserve timestamp ties in paged history and use read-only requests in previews.
Correct the dashboard dependency hash required by the current locked Nix build.
Assisted-by: gpt-6-astra11 files changed, 772 insertions(+), 100 deletions(-)
dashboard/src/main.rs+7| ... | @@ -101,6 +101,7 @@ struct App { | ... | @@ -101,6 +101,7 @@ struct App { |
| 101 | internal: Option<(url::Url, reqwest::Client)>, | 101 | internal: Option<(url::Url, reqwest::Client)>, |
| 102 | cache: cache::Cache, | 102 | cache: cache::Cache, |
| 103 | nomad_slots: Semaphore, | 103 | nomad_slots: Semaphore, |
| 104 | log_slots: Semaphore, | ||
| 104 | specs: Mutex<HashMap<String, Value>>, | 105 | specs: Mutex<HashMap<String, Value>>, |
| 105 | repo: PathBuf, | 106 | repo: PathBuf, |
| 106 | data: PathBuf, | 107 | data: PathBuf, |
| ... | @@ -314,6 +315,11 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response> | ... | @@ -314,6 +315,11 @@ async fn api(State(app): State<Arc<App>>, request: Request) -> Result<Response> |
| 314 | }); | 315 | }); |
| 315 | return Ok(Sse::new(stream).into_response()); | 316 | return Ok(Sse::new(stream).into_response()); |
| 316 | } | 317 | } |
| 318 | if let ["services", id, "logs", "stream"] = parts.as_slice() | ||
| 319 | && method == Method::GET | ||
| 320 | { | ||
| 321 | return telemetry::log_stream(app, request, id, &query).await; | ||
| 322 | } | ||
| 317 | if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() { | 323 | if let ["vms", name, kind @ ("console" | "serial")] = parts.as_slice() { |
| 318 | if method == Method::GET { | 324 | if method == Method::GET { |
| 319 | return vms::console(request, name, *kind == "serial").await; | 325 | return vms::console(request, name, *kind == "serial").await; |
| ... | @@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> { | ... | @@ -546,6 +552,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> { |
| 546 | internal, | 552 | internal, |
| 547 | cache: cache::Cache::default(), | 553 | cache: cache::Cache::default(), |
| 548 | nomad_slots: Semaphore::new(4), | 554 | nomad_slots: Semaphore::new(4), |
| 555 | log_slots: Semaphore::new(32), | ||
| 549 | specs: Mutex::new(HashMap::new()), | 556 | specs: Mutex::new(HashMap::new()), |
| 550 | repo: env("SNOWGLOBE_REPO", "..").into(), | 557 | repo: env("SNOWGLOBE_REPO", "..").into(), |
| 551 | data: env("SNOWGLOBE_DATA_DIR", "data").into(), | 558 | data: env("SNOWGLOBE_DATA_DIR", "data").into(), |
dashboard/src/telemetry.rs+106-7| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | use crate::*; | 1 | use crate::*; |
| 2 | use futures::{StreamExt, stream}; | 2 | use futures::{SinkExt, StreamExt, stream}; |
| 3 | use axum::extract::{FromRequestParts, ws::{Message, WebSocketUpgrade}}; | ||
| 3 | 4 | ||
| 4 | pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> { | 5 | pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> { |
| 5 | if let Some((base, _)) = &app.internal { | 6 | if let Some((base, _)) = &app.internal { |
| ... | @@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> { | ... | @@ -297,8 +298,7 @@ fn terms(query: &str, trace: bool) -> Result<Vec<Term>> { |
| 297 | } | 298 | } |
| 298 | Ok(found) | 299 | Ok(found) |
| 299 | } | 300 | } |
| 300 | pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> { | 301 | fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> { |
| 301 | let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?; | ||
| 302 | let mut filters = vec![ | 302 | let mut filters = vec![ |
| 303 | "source:=nomad".to_owned(), | 303 | "source:=nomad".to_owned(), |
| 304 | format!("job:={}", serde_json::to_string(id)?), | 304 | format!("job:={}", serde_json::to_string(id)?), |
| ... | @@ -332,15 +332,36 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R | ... | @@ -332,15 +332,36 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R |
| 332 | ))? | 332 | ))? |
| 333 | )); | 333 | )); |
| 334 | } | 334 | } |
| 335 | Ok(filters.join(" ")) | ||
| 336 | } | ||
| 337 | |||
| 338 | fn log_line(row: Value) -> Value { | ||
| 339 | json!({"id":row["id"].as_str().map(str::to_owned).unwrap_or_else(|| format!("{}:{}:{}", string(&row["_stream_id"]), string(&row["_time"]), string(&row["_msg"]))), | ||
| 340 | "sequence":number(&row["sequence"]), | ||
| 341 | "t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"), | ||
| 342 | "stream":row["stream"].as_str().filter(|s| ["stderr", "event"].contains(s)).unwrap_or("stdout"), | ||
| 343 | "level":row["level"],"text":row["_msg"].as_str().unwrap_or_default(), | ||
| 344 | "event":row["event"],"allocation":row["allocation"]}) | ||
| 345 | } | ||
| 346 | |||
| 347 | pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> { | ||
| 348 | let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?; | ||
| 349 | if query.contains_key("after") && query.contains_key("before") | ||
| 350 | && query_number(query, "after", 0.0, f64::NEG_INFINITY, f64::INFINITY)? | ||
| 351 | > query_number(query, "before", 0.0, f64::NEG_INFINITY, f64::INFINITY)? | ||
| 352 | { | ||
| 353 | return Err(Error::new(400, "Choose a “to” time at or after “from”.")); | ||
| 354 | } | ||
| 355 | let offset = query_number(query, "offset", 0.0, 0.0, 1_000_000_000.0)?; | ||
| 335 | let mut form = vec![ | 356 | let mut form = vec![ |
| 336 | ("query".into(), filters.join(" ")), | 357 | ("query".into(), format!("{} | sort by (_time desc, id desc, _stream_id desc, _msg desc) offset {offset:.0} limit {limit:.0}", log_query(id, query)?)), |
| 337 | ("limit".into(), limit.to_string()), | 358 | ("limit".into(), limit.to_string()), |
| 338 | ]; | 359 | ]; |
| 339 | for (key, field, add) in [("after", "start", 0.000001), ("before", "end", 0.0)] { | 360 | for (key, field) in [("after", "start"), ("before", "end")] { |
| 340 | if query.contains_key(key) { | 361 | if query.contains_key(key) { |
| 341 | form.push(( | 362 | form.push(( |
| 342 | field.into(), | 363 | field.into(), |
| 343 | (query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? + add) | 364 | query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)? |
| 344 | .to_string(), | 365 | .to_string(), |
| 345 | )); | 366 | )); |
| 346 | } | 367 | } |
| ... | @@ -352,12 +373,90 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R | ... | @@ -352,12 +373,90 @@ pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> R |
| 352 | let response = read(app, url, Some(form)).await?; | 373 | let response = read(app, url, Some(form)).await?; |
| 353 | let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| { | 374 | let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| { |
| 354 | let row: Value = serde_json::from_str(line)?; | 375 | let row: Value = serde_json::from_str(line)?; |
| 355 | Ok::<_,Error>(json!({"t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),"stream":if row["stream"] == "stderr" {"stderr"} else {"stdout"},"level":if row["level"] == "error" || row["level"] == "warn" {row["level"].clone()} else {Value::Null},"text":row["_msg"].as_str().unwrap_or_default()})) | 376 | Ok::<_,Error>(log_line(row)) |
| 356 | }).collect::<Result<Vec<_>>>()?; | 377 | }).collect::<Result<Vec<_>>>()?; |
| 357 | rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"]))); | 378 | rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"]))); |
| 358 | Ok(json!(rows)) | 379 | Ok(json!(rows)) |
| 359 | } | 380 | } |
| 360 | 381 | ||
| 382 | pub async fn log_stream(app: Arc<App>, request: Request, id: &str, query: &HashMap<String, String>) -> Result<Response> { | ||
| 383 | vms::same_origin(&request)?; | ||
| 384 | let filter = log_query(id, query)?; | ||
| 385 | let mut cursor = query_number(query, "cursor", 0.0, 0.0, 9_007_199_254_740_991.0)?; | ||
| 386 | let from = query.get("from").map(|_| query_number(query, "from", 0.0, 0.0, f64::INFINITY)).transpose()?; | ||
| 387 | let base = endpoint(app.clone(), "victoria-logs").await?; | ||
| 388 | let url = format!("{base}/select/logsql/query"); | ||
| 389 | let headers = request.headers().clone(); | ||
| 390 | let (mut parts, _) = request.into_parts(); | ||
| 391 | let upgrade = WebSocketUpgrade::from_request_parts(&mut parts, &()).await | ||
| 392 | .map_err(|_| Error::new(400, "Open the service log to connect."))? | ||
| 393 | .max_message_size(1024).max_frame_size(1024); | ||
| 394 | Ok(upgrade.on_upgrade(move |mut socket| async move { | ||
| 395 | let run = async { | ||
| 396 | let _slot = app.log_slots.try_acquire().map_err(|_| Error::new(429, "Too many live logs are open. Close one and retry."))?; | ||
| 397 | let mut first = true; | ||
| 398 | let mut poll = tokio::time::interval(Duration::from_secs(1)); | ||
| 399 | poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); | ||
| 400 | let mut heartbeat = tokio::time::interval(Duration::from_secs(15)); | ||
| 401 | loop { | ||
| 402 | tokio::select! { | ||
| 403 | message = socket.recv() => match message { | ||
| 404 | Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?, | ||
| 405 | Some(Ok(Message::Pong(_))) => {}, | ||
| 406 | _ => break, | ||
| 407 | }, | ||
| 408 | _ = heartbeat.tick() => { | ||
| 409 | if app.auth.ready() { | ||
| 410 | let account = app.auth.session(&headers, "dashboard")?; | ||
| 411 | if !array(&account["groups"]).iter().any(|g| g["name"] == "infra-admin") { | ||
| 412 | return Err(Error::new(401, "Sign in as an administrator to continue reading logs.")); | ||
| 413 | } | ||
| 414 | } | ||
| 415 | tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Ping(Bytes::new()))).await | ||
| 416 | .map_err(|_| Error::new(504, "The log connection stopped responding."))??; | ||
| 417 | }, | ||
| 418 | _ = poll.tick() => { | ||
| 419 | let checkpoint = read(app.clone(), url.clone(), Some(vec![("query".into(), "source:=log-shipper | stats max(sequence) as sequence".into())])).await?; | ||
| 420 | let checkpoint: Value = serde_json::from_str(string(&checkpoint.value).lines().find(|line| !line.trim().is_empty()).unwrap_or("{}"))?; | ||
| 421 | let settled = number(&checkpoint["sequence"]); | ||
| 422 | if first { cursor = cursor.min(settled); first = false; } | ||
| 423 | let mut form = vec![("query", format!("{filter} sequence:>{cursor:.0} sequence:<={settled:.0} | sort by (sequence) limit 5000"))]; | ||
| 424 | if let Some(from) = from { form.push(("start", from.to_string())); } | ||
| 425 | let request = if env("SNOWGLOBE_PREVIEW", "") == "1" { | ||
| 426 | app.request(Method::GET, &url)?.query(&form) | ||
| 427 | } else { | ||
| 428 | app.request(Method::POST, &url)?.form(&form) | ||
| 429 | }; | ||
| 430 | let response = request.timeout(Duration::from_secs(8)).send().await?; | ||
| 431 | if !response.status().is_success() { return Err(Error::new(502, "The log store isn't answering. Retrying…")); } | ||
| 432 | let mut chunks = response.bytes_stream(); | ||
| 433 | let mut body = Vec::new(); | ||
| 434 | while let Some(chunk) = chunks.next().await { | ||
| 435 | body.extend_from_slice(&chunk?); | ||
| 436 | if body.len() > 8 * 1024 * 1024 { | ||
| 437 | let end = body[..8 * 1024 * 1024].iter().rposition(|b| *b == b'\n') | ||
| 438 | .ok_or_else(|| Error::new(502, "A log entry is too large. Narrow the search and retry."))?; | ||
| 439 | body.truncate(end + 1); | ||
| 440 | break; | ||
| 441 | } | ||
| 442 | } | ||
| 443 | let lines = std::str::from_utf8(&body)?.lines().filter(|l| !l.trim().is_empty()) | ||
| 444 | .map(|raw| serde_json::from_str(raw).map(log_line)).collect::<std::result::Result<Vec<_>, _>>()?; | ||
| 445 | for line in &lines { cursor = cursor.max(number(&line["sequence"])); } | ||
| 446 | tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Text(json!({"lines":lines,"cursor":cursor}).to_string().into()))).await | ||
| 447 | .map_err(|_| Error::new(504, "The log connection stopped responding."))??; | ||
| 448 | } | ||
| 449 | } | ||
| 450 | } | ||
| 451 | Ok::<_, Error>(()) | ||
| 452 | }.await; | ||
| 453 | if let Err(error) = run { | ||
| 454 | let _ = tokio::time::timeout(Duration::from_secs(2), socket.send(Message::Text(json!({"error":error.message}).to_string().into()))).await; | ||
| 455 | } | ||
| 456 | let _ = tokio::time::timeout(Duration::from_secs(1), socket.close()).await; | ||
| 457 | }).into_response()) | ||
| 458 | } | ||
| 459 | |||
| 361 | fn span(row: &Value) -> Value { | 460 | fn span(row: &Value) -> Value { |
| 362 | let mut attributes: serde_json::Map<String, Value> = row | 461 | let mut attributes: serde_json::Map<String, Value> = row |
| 363 | .as_object() | 462 | .as_object() |
dashboard/src/vms.rs+2-2| ... | @@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result<Respo | ... | @@ -75,7 +75,7 @@ pub async fn console(request: Request, name: &str, serial: bool) -> Result<Respo |
| 75 | .into_response()) | 75 | .into_response()) |
| 76 | } | 76 | } |
| 77 | 77 | ||
| 78 | fn same_origin(request: &Request) -> Result<()> { | 78 | pub(crate) fn same_origin(request: &Request) -> Result<()> { |
| 79 | let supplied = request | 79 | let supplied = request |
| 80 | .headers() | 80 | .headers() |
| 81 | .get("origin") | 81 | .get("origin") |
| ... | @@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> { | ... | @@ -94,7 +94,7 @@ fn same_origin(request: &Request) -> Result<()> { |
| 94 | }; | 94 | }; |
| 95 | authority != host || !matches!(origin.scheme(), "http" | "https") | 95 | authority != host || !matches!(origin.scheme(), "http" | "https") |
| 96 | }) { | 96 | }) { |
| 97 | return Err(Error::new(403, "Open the VM page on this site to connect.")); | 97 | return Err(Error::new(403, "Open this page on the same site to connect.")); |
| 98 | } | 98 | } |
| 99 | Ok(()) | 99 | Ok(()) |
| 100 | } | 100 | } |
dashboard/web/components/LogView.css+2| ... | @@ -18,6 +18,8 @@ | ... | @@ -18,6 +18,8 @@ |
| 18 | .log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); } | 18 | .log-view .log { grid-template-columns: 62px 1fr; padding: 0 var(--gutter); } |
| 19 | .log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); } | 19 | .log-view .log.warn:hover { background: color-mix(in srgb, var(--warning) 16%, transparent); } |
| 20 | .log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); } | 20 | .log-view .log.error:hover { background: color-mix(in srgb, var(--critical) 20%, transparent); } |
| 21 | .log-view .task-event { background: color-mix(in srgb, var(--text) 4%, transparent); } | ||
| 22 | .log-view .event-label { margin-right: 8px; font: 600 11px var(--sans); } | ||
| 21 | .log-view .ansi { | 23 | .log-view .ansi { |
| 22 | color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text)); | 24 | color: color-mix(in srgb, var(--fg, currentColor) var(--ansi-mix), var(--text)); |
| 23 | background: color-mix(in srgb, var(--bg, transparent) 45%, transparent); | 25 | background: color-mix(in srgb, var(--bg, transparent) 45%, transparent); |
dashboard/web/components/LogView.tsx+2-1| ... | @@ -132,9 +132,10 @@ export function LogView(props: { | ... | @@ -132,9 +132,10 @@ export function LogView(props: { |
| 132 | <div class="log-edge day">{date(line.t)}</div> | 132 | <div class="log-edge day">{date(line.t)}</div> |
| 133 | </Show> | 133 | </Show> |
| 134 | {props.before?.(line, index)} | 134 | {props.before?.(line, index)} |
| 135 | <div class={`log ${line.level ?? ""}`}> | 135 | <div class={`log ${line.level ?? ""}`} classList={{ "task-event": line.stream === "event" }}> |
| 136 | <time data-tip={datetime(line.t)}>{clock(line.t)}</time> | 136 | <time data-tip={datetime(line.t)}>{clock(line.t)}</time> |
| 137 | <span> | 137 | <span> |
| 138 | <Show when={line.event}><strong class="event-label" data-tip={line.allocation ? `Allocation ${line.allocation}` : undefined}>{line.event}</strong>{" "}</Show> | ||
| 138 | <Show when={mixed()}> | 139 | <Show when={mixed()}> |
| 139 | <Show when={props.onContainer} fallback={<span class="container">{line.container}</span>}> | 140 | <Show when={props.onContainer} fallback={<span class="container">{line.container}</span>}> |
| 140 | {(filter) => ( | 141 | {(filter) => ( |
dashboard/web/pages/Service.tsx+117-53| ... | @@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw"; | ... | @@ -8,7 +8,7 @@ import RotateCcw from "lucide-solid/icons/rotate-ccw"; |
| 8 | import Search from "lucide-solid/icons/search"; | 8 | import Search from "lucide-solid/icons/search"; |
| 9 | import X from "lucide-solid/icons/x"; | 9 | import X from "lucide-solid/icons/x"; |
| 10 | import { | 10 | import { |
| 11 | createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, | 11 | createEffect, createMemo, createResource, createSignal, createUniqueId, For, type JSX, Match, onCleanup, Show, Switch, untrack, |
| 12 | } from "solid-js"; | 12 | } from "solid-js"; |
| 13 | import { Portal } from "solid-js/web"; | 13 | import { Portal } from "solid-js/web"; |
| 14 | import { | 14 | import { |
| ... | @@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con | ... | @@ -43,29 +43,37 @@ const LEVELS = [[undefined, "all"], ["warn", "warn"], ["error", "error"]] as con |
| 43 | /** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */ | 43 | /** URL state of the logs and traces tabs; `at` opens the logs at a moment instead of following. */ |
| 44 | type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; | 44 | type Query = { q?: string; level?: "warn" | "error"; at?: string; from?: string; to?: string; errors?: "1"; sort?: TraceSort; trace?: string }; |
| 45 | 45 | ||
| 46 | /** Each service's latest unfiltered lines, so returning to a service draws its logs at once. */ | ||
| 47 | const recent = new Map<string, LogLine[]>(); | ||
| 48 | |||
| 49 | /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ | 46 | /** Newest at the bottom; follows new lines while scrolled to the end, and loads older ones near the top. */ |
| 50 | function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | 47 | function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { |
| 51 | const [params, setParams] = useSearchParams<Query>(); | 48 | const [params, setParams] = useSearchParams<Query>(); |
| 52 | const at = () => (params.at ? Number(params.at) : null); | 49 | const at = () => (params.at ? Number(params.at) : null); |
| 53 | const bounded = () => at() !== null || !!params.to; | 50 | const bounded = () => at() !== null || !!params.to; |
| 54 | const unfiltered = () => !params.q && !params.level && !params.from && !params.to && at() === null; | ||
| 55 | const dateValue = (value?: string) => { | 51 | const dateValue = (value?: string) => { |
| 56 | if (!value) return ""; | 52 | if (!value) return ""; |
| 57 | const date = new Date(Number(value) * 1000); | 53 | const date = new Date(Number(value) * 1000); |
| 58 | return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16); | 54 | return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16); |
| 59 | }; | 55 | }; |
| 60 | const [older, setOlder] = createSignal<LogLine[]>([]); | 56 | const [all, setLines] = createSignal<LogLine[]>([]); |
| 61 | const [newer, setNewer] = createSignal<LogLine[]>([]); | 57 | let pending: LogLine[] = []; |
| 58 | let generation = 0; | ||
| 59 | let older: { before: string; offset: number } | undefined; | ||
| 62 | const [exhausted, setExhausted] = createSignal(false); | 60 | const [exhausted, setExhausted] = createSignal(false); |
| 63 | const [loadingOlder, setLoadingOlder] = createSignal(false); | 61 | const [loadingOlder, setLoadingOlder] = createSignal(false); |
| 64 | const [follow, setFollow] = createSignal(true); | 62 | const [follow, setFollow] = createSignal(true); |
| 65 | const [unseen, setUnseen] = createSignal(0); | 63 | const [unseen, setUnseen] = createSignal(0); |
| 66 | /** Why the last poll failed, until one succeeds. */ | ||
| 67 | const [stalled, setStalled] = createSignal<string>(); | 64 | const [stalled, setStalled] = createSignal<string>(); |
| 68 | const fetchLines = (limit: number, extra: { before?: string; after?: string }) => | 65 | const [connected, setConnected] = createSignal(false); |
| 66 | const merge = (before: LogLine[], after: LogLine[]) => [...new Map([...before, ...after] | ||
| 67 | .map((line) => [line.id ?? JSON.stringify(line), line])).values()].sort((a, b) => a.t - b.t); | ||
| 68 | const resume = () => { | ||
| 69 | older = undefined; | ||
| 70 | setLines((lines) => merge(lines, pending).slice(-10_000)); | ||
| 71 | pending = []; | ||
| 72 | setUnseen(0); | ||
| 73 | setExhausted(false); | ||
| 74 | setFollow(true); | ||
| 75 | }; | ||
| 76 | const fetchLines = (limit: number, extra: { before?: string; after?: string; offset?: string }) => | ||
| 69 | parseResponse(api.services[":id"].logs.$get({ | 77 | parseResponse(api.services[":id"].logs.$get({ |
| 70 | param: { id: props.service.id }, | 78 | param: { id: props.service.id }, |
| 71 | query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit), | 79 | query: { q: params.q ?? "", ...(params.level ? { level: params.level } : {}), limit: String(limit), |
| ... | @@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | ... | @@ -74,64 +82,120 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { |
| 74 | const [first, { refetch }] = createResource<LogLine[], string>( | 82 | const [first, { refetch }] = createResource<LogLine[], string>( |
| 75 | // A string key, since a fresh array from an unrelated rerun would count as a change and refetch. | 83 | // A string key, since a fresh array from an unrelated rerun would count as a change and refetch. |
| 76 | () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]), | 84 | () => JSON.stringify([props.service.id, params.q, params.level, params.at, params.from, params.to]), |
| 77 | (_, { refetching }) => { | 85 | () => { |
| 78 | setOlder([]); | 86 | const current = ++generation; |
| 79 | setNewer([]); | 87 | older = undefined; |
| 88 | setLines([]); | ||
| 89 | pending = []; | ||
| 90 | setLoadingOlder(false); | ||
| 80 | setExhausted(false); | 91 | setExhausted(false); |
| 81 | setFollow(!bounded()); | 92 | setFollow(!bounded()); |
| 82 | setUnseen(0); | 93 | setUnseen(0); |
| 83 | setStalled(); | 94 | setStalled(); |
| 84 | const cached = !refetching && unfiltered() && recent.get(props.service.id); | 95 | return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }).then((lines) => { |
| 85 | if (cached) { | 96 | if (current === generation) setLines(lines); |
| 86 | queueMicrotask(() => poll()); | 97 | return lines; |
| 87 | return cached; | 98 | }); |
| 88 | } | ||
| 89 | return fetchLines(1000, at() === null ? {} : { before: String(at()! + 30) }); | ||
| 90 | }, | 99 | }, |
| 91 | ); | 100 | ); |
| 92 | const loaded = lastGood(first); | ||
| 93 | const all = createMemo(() => [...older(), ...(loaded() ?? []), ...newer()]); | ||
| 94 | createEffect(() => first.state === "ready" && unfiltered() && recent.set(props.service.id, all().slice(-1000))); | ||
| 95 | 101 | ||
| 96 | const loadOlder = async () => { | 102 | const loadOlder = async () => { |
| 97 | const oldest = all()[0]; | 103 | const oldest = all()[0]; |
| 98 | if (!oldest || loadingOlder() || exhausted()) return; | 104 | if (!oldest || loadingOlder() || exhausted()) return; |
| 99 | if (params.from && oldest.t <= Number(params.from)) return setExhausted(true); | 105 | |
| 100 | setLoadingOlder(true); | 106 | setLoadingOlder(true); |
| 107 | const current = generation; | ||
| 101 | try { | 108 | try { |
| 102 | const lines = await fetchLines(5000, { before: String(oldest.t) }); | 109 | const page = older ??= { before: String(oldest.t + 0.000001), offset: 0 }; |
| 103 | if (!lines.length) setExhausted(true); | 110 | const fetched = await fetchLines(5000, { before: page.before, offset: String(page.offset) }); |
| 104 | setOlder([...lines, ...older()]); | 111 | if (current !== generation || older !== page) return; |
| 112 | const known = new Set(all().map((line) => line.id ?? JSON.stringify(line))); | ||
| 113 | const lines = fetched.filter((line) => !known.has(line.id ?? JSON.stringify(line))); | ||
| 114 | page.offset += fetched.length; | ||
| 115 | if (fetched.length < 5000) setExhausted(true); | ||
| 116 | setLines((previous) => merge(lines, previous)); | ||
| 105 | } catch (failure) { | 117 | } catch (failure) { |
| 106 | toast(`Couldn't load older lines. ${reason(failure)}`); | 118 | if (current === generation) toast(`Couldn't load older lines. ${reason(failure)}`); |
| 107 | } finally { | 119 | } finally { |
| 108 | setLoadingOlder(false); | 120 | if (current === generation) setLoadingOlder(false); |
| 109 | } | 121 | } |
| 110 | }; | 122 | }; |
| 111 | 123 | ||
| 112 | const poll = async () => { | 124 | createEffect(() => { |
| 113 | const last = all().at(-1); | 125 | const id = props.service.id; |
| 114 | if (bounded() || !last || first.loading) return; | 126 | const q = params.q ?? ""; |
| 115 | let lines: LogLine[]; | 127 | const level = params.level; |
| 116 | try { | 128 | const from = params.from; |
| 117 | lines = await fetchLines(1000, { after: String(last.t) }); | 129 | setConnected(false); |
| 118 | setStalled(); | 130 | if (bounded() || first.state !== "ready") return; |
| 119 | } catch (failure) { | 131 | let disposed = false; |
| 120 | setStalled(reason(failure)); | 132 | let failures = 0; |
| 121 | return; | 133 | let cursor = Math.max(0, ...untrack(all).map((line) => line.sequence ?? 0)); |
| 122 | } | 134 | let socket: WebSocket; |
| 123 | // A search changed while this was under way, so the lines belong to the old one. | 135 | let retry: ReturnType<typeof setTimeout> | undefined; |
| 124 | if (!lines.length || first.loading || last !== all().at(-1)) return; | 136 | let stale: ReturnType<typeof setTimeout> | undefined; |
| 125 | setNewer([...newer(), ...lines]); | 137 | const connect = () => { |
| 126 | if (!follow()) setUnseen(unseen() + lines.length); | 138 | const url = new URL(`/api/services/${encodeURIComponent(id)}/logs/stream`, location.href); |
| 127 | }; | 139 | url.protocol = location.protocol === "https:" ? "wss:" : "ws:"; |
| 128 | const timer = setInterval(poll, 2000); | 140 | url.searchParams.set("q", q); |
| 129 | onCleanup(() => clearInterval(timer)); | 141 | if (level) url.searchParams.set("level", level); |
| 142 | url.searchParams.set("cursor", String(cursor)); | ||
| 143 | if (from) url.searchParams.set("from", from); | ||
| 144 | socket = new WebSocket(url); | ||
| 145 | socket.onmessage = (event) => { | ||
| 146 | if (disposed) return; | ||
| 147 | const message: { lines?: LogLine[]; cursor?: number; error?: string } = JSON.parse(event.data); | ||
| 148 | if (message.error) { | ||
| 149 | setConnected(false); | ||
| 150 | setStalled(message.error); | ||
| 151 | return; | ||
| 152 | } | ||
| 153 | failures = 0; | ||
| 154 | setConnected(true); | ||
| 155 | setStalled(); | ||
| 156 | clearTimeout(stale); | ||
| 157 | stale = setTimeout(() => socket.close(), 35_000); | ||
| 158 | if (message.lines?.length) { | ||
| 159 | const known = new Set([...all(), ...pending].map((line) => line.id ?? JSON.stringify(line))); | ||
| 160 | const lines = message.lines.filter((line) => { | ||
| 161 | const id = line.id ?? JSON.stringify(line); | ||
| 162 | if (known.has(id)) return false; | ||
| 163 | known.add(id); | ||
| 164 | return true; | ||
| 165 | }); | ||
| 166 | if (follow()) { | ||
| 167 | older = undefined; | ||
| 168 | setLines((previous) => merge(previous, lines).slice(-10_000)); | ||
| 169 | } | ||
| 170 | else { | ||
| 171 | pending = merge(pending, lines).slice(-10_000); | ||
| 172 | setUnseen((previous) => previous + lines.length); | ||
| 173 | } | ||
| 174 | } | ||
| 175 | cursor = message.cursor ?? cursor; | ||
| 176 | }; | ||
| 177 | socket.onclose = () => { | ||
| 178 | if (disposed) return; | ||
| 179 | clearTimeout(stale); | ||
| 180 | setConnected(false); | ||
| 181 | setStalled((previous) => previous ?? "Live logs disconnected. Retrying…"); | ||
| 182 | retry = setTimeout(connect, Math.min(30, 2 ** failures++) * 1000); | ||
| 183 | }; | ||
| 184 | }; | ||
| 185 | connect(); | ||
| 186 | onCleanup(() => { | ||
| 187 | disposed = true; | ||
| 188 | clearTimeout(retry); | ||
| 189 | clearTimeout(stale); | ||
| 190 | socket.close(); | ||
| 191 | }); | ||
| 192 | }); | ||
| 130 | const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!))); | 193 | const marked = createMemo(() => (at() === null ? undefined : all().find((line) => line.t > at()!))); |
| 131 | const hints = () => [ | 194 | const hints = () => [ |
| 132 | ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1) | 195 | ...props.service.containers.filter((container) => !container.hook && props.service.containers.length > 1).slice(0, 1) |
| 133 | .map((container) => [`container:${container.name}`, "one container's lines"] as const), | 196 | .map((container) => [`container:${container.name}`, "one container's lines"] as const), |
| 134 | ["stream:stderr", "error output only"], | 197 | ["stream:stderr", "error output only"], |
| 198 | ["stream:event", "starts, restarts, and OOM kills"], | ||
| 135 | ["-health", "hide lines with a word"], | 199 | ["-health", "hide lines with a word"], |
| 136 | ['"connection refused"', "an exact phrase"], | 200 | ['"connection refused"', "an exact phrase"], |
| 137 | ] as const; | 201 | ] as const; |
| ... | @@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | ... | @@ -155,7 +219,7 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { |
| 155 | <label>to <input type="datetime-local" value={dateValue(params.to)} | 219 | <label>to <input type="datetime-local" value={dateValue(params.to)} |
| 156 | onChange={(event) => setParams({ to: event.currentTarget.value ? String(Date.parse(event.currentTarget.value) / 1000) : undefined }, { replace: true })} /></label> | 220 | onChange={(event) => setParams({ to: event.currentTarget.value ? String(Date.parse(event.currentTarget.value) / 1000) : undefined }, { replace: true })} /></label> |
| 157 | <Show when={props.service.logs}>{(link) => <OpenApp app={link().app} href={link().url}>{link().app.name}</OpenApp>}</Show> | 221 | <Show when={props.service.logs}>{(link) => <OpenApp app={link().app} href={link().url}>{link().app.name}</OpenApp>}</Show> |
| 158 | <Liveness every="lines every 2 s" paused={bounded() ? "Showing the selected time range" : undefined} | 222 | <Liveness every="lines as they arrive" connecting={!connected()} paused={bounded() ? "Showing the selected time range" : undefined} |
| 159 | failure={stalled() ?? (first.state === "errored" ? reason(first.error) : undefined)} /> | 223 | failure={stalled() ?? (first.state === "errored" ? reason(first.error) : undefined)} /> |
| 160 | </Filters> | 224 | </Filters> |
| 161 | </Portal> | 225 | </Portal> |
| ... | @@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { | ... | @@ -178,10 +242,10 @@ function Logs(props: { service: ServiceDetail; tools: HTMLElement }) { |
| 178 | onStart={loadOlder} | 242 | onStart={loadOlder} |
| 179 | onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })} | 243 | onContainer={(name) => setParams({ q: [params.q, `container:${name}`].filter(Boolean).join(" ") }, { replace: true })} |
| 180 | onFollow={(end) => { | 244 | onFollow={(end) => { |
| 181 | setFollow(end && at() === null); | 245 | if (end && !bounded()) resume(); |
| 182 | if (end) setUnseen(0); | 246 | else setFollow(false); |
| 183 | }} | 247 | }} |
| 184 | onLatest={() => (at() === null ? setFollow(true) : setParams({ at: undefined }))} | 248 | onLatest={() => (at() === null ? resume() : setParams({ at: undefined }))} |
| 185 | before={(line, index) => ( | 249 | before={(line, index) => ( |
| 186 | <> | 250 | <> |
| 187 | <Show when={index() === 0}> | 251 | <Show when={index() === 0}> |
| ... | @@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) { | ... | @@ -387,12 +451,12 @@ function Traces(props: { service: ServiceDetail; tools: HTMLElement }) { |
| 387 | } | 451 | } |
| 388 | 452 | ||
| 389 | /** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */ | 453 | /** Whether a tab still takes in new data; `paused` and `failure` say why it doesn't. */ |
| 390 | function Liveness(props: { every: string; paused?: string; failure?: string }) { | 454 | function Liveness(props: { every: string; paused?: string; failure?: string; connecting?: boolean }) { |
| 391 | const health = () => (props.failure ? "degraded" : props.paused ? "stopped" : "healthy"); | 455 | const health = () => (props.failure ? "degraded" : props.paused || props.connecting ? "stopped" : "healthy"); |
| 392 | return ( | 456 | return ( |
| 393 | <span class="liveness" classList={{ live: health() === "healthy" }} role="status" tabIndex={0} | 457 | <span class="liveness" classList={{ live: health() === "healthy" }} role="status" tabIndex={0} |
| 394 | data-tip={props.failure ?? props.paused ?? `Checks for new ${props.every}`}> | 458 | data-tip={props.failure ?? props.paused ?? (props.connecting ? "Connecting to live logs…" : `New ${props.every}`)}> |
| 395 | <StatusLabel health={health()}>{props.failure ? "reconnecting" : props.paused ? "paused" : "live"}</StatusLabel> | 459 | <StatusLabel health={health()}>{props.failure ? props.paused ? "unavailable" : "reconnecting" : props.paused ? "paused" : props.connecting ? "connecting" : "live"}</StatusLabel> |
| 396 | </span> | 460 | </span> |
| 397 | ); | 461 | ); |
| 398 | } | 462 | } |
dashboard/web/types/model.ts+5-1| ... | @@ -206,11 +206,15 @@ export interface TraceSummary { | ... | @@ -206,11 +206,15 @@ export interface TraceSummary { |
| 206 | } | 206 | } |
| 207 | 207 | ||
| 208 | export interface LogLine { | 208 | export interface LogLine { |
| 209 | id?: string; | ||
| 210 | sequence?: number; | ||
| 209 | t: number; | 211 | t: number; |
| 210 | container: string; | 212 | container: string; |
| 211 | stream: "stdout" | "stderr"; | 213 | stream: "stdout" | "stderr" | "event"; |
| 212 | level: "debug" | "info" | "warn" | "error" | null; | 214 | level: "debug" | "info" | "warn" | "error" | null; |
| 213 | text: string; | 215 | text: string; |
| 216 | event?: string | null; | ||
| 217 | allocation?: string | null; | ||
| 214 | } | 218 | } |
| 215 | 219 | ||
| 216 | export interface HostInfo { | 220 | export interface HostInfo { |
nixos/dashboard.nix+1-1| ... | @@ -24,7 +24,7 @@ let | ... | @@ -24,7 +24,7 @@ let |
| 24 | inherit (finalAttrs) pname version src sourceRoot; | 24 | inherit (finalAttrs) pname version src sourceRoot; |
| 25 | pnpm = pnpm_10; | 25 | pnpm = pnpm_10; |
| 26 | fetcherVersion = 3; | 26 | fetcherVersion = 3; |
| 27 | hash = "sha256-zux7++pfJ9YJAMPjnvW94Z2m884Ypnc6BK9iHJxRFnA="; | 27 | hash = "sha256-Jhk4mAOCqoljelFJfC7o+3t66x9GPFYb2e928jeaPSk="; |
| 28 | }; | 28 | }; |
| 29 | nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ]; | 29 | nativeBuildInputs = [ nodejs_24 pnpm_10 pnpmConfigHook ]; |
| 30 | buildPhase = "pnpm run build"; | 30 | buildPhase = "pnpm run build"; |
tools/dashboard-logs-test.py created+270| ... | @@ -0,0 +1,270 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import argparse | ||
| 3 | import base64 | ||
| 4 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ||
| 5 | import json | ||
| 6 | import os | ||
| 7 | from pathlib import Path | ||
| 8 | import socket | ||
| 9 | import struct | ||
| 10 | import re | ||
| 11 | import sqlite3 | ||
| 12 | import hashlib | ||
| 13 | import subprocess | ||
| 14 | import tempfile | ||
| 15 | import threading | ||
| 16 | import time | ||
| 17 | from urllib.parse import parse_qs, urlsplit | ||
| 18 | from urllib.request import Request, urlopen | ||
| 19 | from urllib.error import HTTPError | ||
| 20 | |||
| 21 | |||
| 22 | def main(): | ||
| 23 | parser = argparse.ArgumentParser() | ||
| 24 | parser.add_argument('--preview', action='store_true') | ||
| 25 | parser.add_argument('--binary', type=Path, default=Path('dashboard/target/debug/home-dashboard')) | ||
| 26 | args = parser.parse_args() | ||
| 27 | forms = [] | ||
| 28 | failed = threading.Event() | ||
| 29 | row = {'_time': '2026-10-14T02:50:00Z', '_msg': 'Out of memory (OOM): exit 137', | ||
| 30 | 'id': 'allocation:app:1:Terminated', 'task': 'app', 'stream': 'event', 'event': 'OOM', | ||
| 31 | 'level': 'error', 'allocation': 'allocation', 'job': 'shale', 'source': 'nomad', 'sequence': 1} | ||
| 32 | rows = [row] | ||
| 33 | checkpoint = 0 | ||
| 34 | |||
| 35 | class Fixture(BaseHTTPRequestHandler): | ||
| 36 | protocol_version = 'HTTP/1.1' | ||
| 37 | |||
| 38 | def handle(self): | ||
| 39 | try: | ||
| 40 | super().handle() | ||
| 41 | except (ConnectionResetError, BrokenPipeError): | ||
| 42 | pass | ||
| 43 | |||
| 44 | def do_GET(self): | ||
| 45 | if urlsplit(self.path).path == '/select/logsql/query': | ||
| 46 | self.query(parse_qs(urlsplit(self.path).query)) | ||
| 47 | else: | ||
| 48 | self.reply([{'Address': '127.0.0.1', 'Port': self.server.server_port}]) | ||
| 49 | |||
| 50 | def reply(self, value): | ||
| 51 | data = json.dumps(value).encode() | ||
| 52 | self.send_response(200) | ||
| 53 | self.send_header('Content-Length', str(len(data))) | ||
| 54 | self.end_headers() | ||
| 55 | self.wfile.write(data) | ||
| 56 | |||
| 57 | def do_POST(self): | ||
| 58 | form = parse_qs(self.rfile.read(int(self.headers['Content-Length'])).decode()) | ||
| 59 | self.query(form) | ||
| 60 | |||
| 61 | def query(self, form): | ||
| 62 | forms.append((self.path, form)) | ||
| 63 | if failed.is_set(): | ||
| 64 | self.send_error(503) | ||
| 65 | return | ||
| 66 | if 'source:=log-shipper' in form['query'][0]: | ||
| 67 | if checkpoint == 0: | ||
| 68 | self.send_response(200) | ||
| 69 | self.send_header('Content-Length', '0') | ||
| 70 | self.end_headers() | ||
| 71 | else: | ||
| 72 | self.reply({'sequence': checkpoint if checkpoint is not None else max(r['sequence'] for r in rows)}) | ||
| 73 | return | ||
| 74 | selected = rows[:] | ||
| 75 | match = re.search(r'sequence:>([0-9]+)', form['query'][0]) | ||
| 76 | if match: | ||
| 77 | selected = [r for r in selected if r['sequence'] > int(match[1])] | ||
| 78 | match = re.search(r'sequence:<=(\d+)', form['query'][0]) | ||
| 79 | if match: | ||
| 80 | selected = [r for r in selected if r['sequence'] <= int(match[1])] | ||
| 81 | if '| sort by (sequence)' in form['query'][0]: | ||
| 82 | selected.sort(key=lambda r: r['sequence']) | ||
| 83 | elif '| sort by (_time desc' in form['query'][0]: | ||
| 84 | selected.sort(key=lambda r: (r['_time'], r['id']), reverse=True) | ||
| 85 | offset = int(re.search(r'offset (\d+)', form['query'][0])[1]) | ||
| 86 | limit = int(re.search(r'limit (\d+)', form['query'][0])[1]) | ||
| 87 | selected = selected[offset:offset + limit] | ||
| 88 | self.send_response(200) | ||
| 89 | data = b''.join((json.dumps(r) + '\n').encode() for r in selected) | ||
| 90 | self.send_header('Content-Length', str(len(data))) | ||
| 91 | self.end_headers() | ||
| 92 | self.wfile.write(data[:17]) | ||
| 93 | self.wfile.flush() | ||
| 94 | try: | ||
| 95 | self.wfile.write(data[17:]) | ||
| 96 | self.wfile.flush() | ||
| 97 | except (BrokenPipeError, ConnectionResetError): | ||
| 98 | pass | ||
| 99 | |||
| 100 | def log_message(self, *args): | ||
| 101 | pass | ||
| 102 | |||
| 103 | with tempfile.TemporaryDirectory(prefix='dashboard-logs-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as fixture: | ||
| 104 | fixture.daemon_threads = True | ||
| 105 | threading.Thread(target=fixture.serve_forever, daemon=True).start() | ||
| 106 | root = Path(temporary).resolve() | ||
| 107 | (root / 'data').mkdir() | ||
| 108 | (root / 'token').write_text('disposable') | ||
| 109 | with socket.socket() as reservation: | ||
| 110 | reservation.bind(('127.0.0.1', 0)) | ||
| 111 | port = reservation.getsockname()[1] | ||
| 112 | origin = f'http://127.0.0.1:{port}' | ||
| 113 | environment = {**os.environ, 'PORT': str(port), 'SNOWGLOBE_DATA_DIR': str(root / 'data'), | ||
| 114 | 'SNOWGLOBE_PUBLIC_ORIGIN': 'https://globe.studio.test', 'SNOWGLOBE_NOMAD_TOKEN_FILE': str(root / 'token'), | ||
| 115 | 'NOMAD_ADDR': f'http://127.0.0.1:{fixture.server_port}'} | ||
| 116 | environment['SNOWGLOBE_PREVIEW'] = '1' if args.preview else '0' | ||
| 117 | for name in ['SNOWGLOBE_PROXY_TOKEN_FILE', 'SNOWGLOBE_INTERNAL_URL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_YT_STATE']: | ||
| 118 | environment.pop(name, None) | ||
| 119 | with (root / 'server.log').open('w') as log: | ||
| 120 | server = subprocess.Popen([str(args.binary.resolve())], env=environment, stdout=log, stderr=log) | ||
| 121 | try: | ||
| 122 | deadline = time.monotonic() + 10 | ||
| 123 | while True: | ||
| 124 | try: | ||
| 125 | with socket.create_connection(('127.0.0.1', port), timeout=.1): break | ||
| 126 | except OSError: | ||
| 127 | assert server.poll() is None, (root / 'server.log').read_text() | ||
| 128 | assert time.monotonic() < deadline | ||
| 129 | time.sleep(.05) | ||
| 130 | path = '/api/services/shale/logs' | ||
| 131 | with urlopen(Request(origin + path + '?level=error&q=stream:event', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: | ||
| 132 | lines = json.load(response) | ||
| 133 | assert lines[0]['stream'] == 'event' and lines[0]['event'] == 'OOM' | ||
| 134 | assert lines[0]['level'] == 'error' and lines[0]['id'] == row['id'] | ||
| 135 | assert 'level:=error' in forms[-1][1]['query'][0] | ||
| 136 | assert 'stream:~"(?i)^event"' in forms[-1][1]['query'][0] | ||
| 137 | try: | ||
| 138 | urlopen(Request(origin + path + '?after=2&before=1', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) | ||
| 139 | raise AssertionError('reversed bounds accepted') | ||
| 140 | except HTTPError as error: | ||
| 141 | assert error.code == 400 and b'at or after' in error.read() | ||
| 142 | |||
| 143 | def connect(expected, supplied_origin=origin, groups='infra-admin', actor='fixture', cursor=0, cookie=None): | ||
| 144 | sock = socket.create_connection(('127.0.0.1', port), timeout=5) | ||
| 145 | sock.settimeout(20) | ||
| 146 | stream = sock.makefile('rb') | ||
| 147 | headers = {'Host': urlsplit(origin).netloc, 'Upgrade': 'websocket', 'Connection': 'Upgrade', | ||
| 148 | 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': base64.b64encode(os.urandom(16)).decode()} | ||
| 149 | if actor is not None: headers['User-Name'] = actor | ||
| 150 | if cookie is not None: headers['Cookie'] = '__Host-snow-session=' + cookie | ||
| 151 | if groups is not None: headers['User-Groups'] = groups | ||
| 152 | if supplied_origin is not None: headers['Origin'] = supplied_origin | ||
| 153 | sock.sendall(('GET ' + path + f'/stream?q=stream:event&cursor={cursor}&from=1 HTTP/1.1\r\n' + ''.join(f'{k}: {v}\r\n' for k, v in headers.items()) + '\r\n').encode()) | ||
| 154 | status = int(stream.readline().split()[1]) | ||
| 155 | assert status == expected, status | ||
| 156 | while stream.readline().strip(): pass | ||
| 157 | if expected != 101: | ||
| 158 | stream.close() | ||
| 159 | sock.close() | ||
| 160 | return sock, stream | ||
| 161 | |||
| 162 | connect(401, actor=None) | ||
| 163 | connect(403, groups='metrics') | ||
| 164 | connect(403, supplied_origin=None) | ||
| 165 | connect(403, supplied_origin='http://other.invalid') | ||
| 166 | sock, stream = connect(101) | ||
| 167 | try: | ||
| 168 | def receive(): | ||
| 169 | head = stream.read(2) | ||
| 170 | assert len(head) == 2, 'stream closed unexpectedly' | ||
| 171 | size = head[1] & 127 | ||
| 172 | if size == 126: size = struct.unpack('!H', stream.read(2))[0] | ||
| 173 | if size == 127: size = struct.unpack('!Q', stream.read(8))[0] | ||
| 174 | if head[0] & 15 == 9: | ||
| 175 | payload = stream.read(size) | ||
| 176 | mask = os.urandom(4) | ||
| 177 | sock.sendall(bytes([138, 128 | size]) + mask + bytes(v ^ mask[i % 4] for i, v in enumerate(payload))) | ||
| 178 | return receive() | ||
| 179 | assert head[0] & 15 == 1, head | ||
| 180 | return json.loads(stream.read(size)) | ||
| 181 | message = receive() | ||
| 182 | assert message == {'lines': [], 'cursor': 0} | ||
| 183 | checkpoint = None | ||
| 184 | while not (message := receive()).get('lines'): pass | ||
| 185 | assert message['lines'] == lines and message['cursor'] == 1 | ||
| 186 | assert forms[-1][1]['start'] == ['1'] | ||
| 187 | rows.append({**row, 'id': 'same-time', 'sequence': 2}) | ||
| 188 | while not (message := receive()).get('lines'): pass | ||
| 189 | assert message['lines'][0]['id'] == 'same-time' and message['cursor'] == 2 | ||
| 190 | rows.append({**row, '_time': '2026-10-14T01:00:00Z', 'id': 'late-oom', 'sequence': 3}) | ||
| 191 | while not (message := receive()).get('lines'): pass | ||
| 192 | assert message['lines'][0]['id'] == 'late-oom' and message['cursor'] == 3 | ||
| 193 | failed.set() | ||
| 194 | while not (message := receive()).get('error'): pass | ||
| 195 | assert 'store' in message['error'] | ||
| 196 | finally: | ||
| 197 | stream.close() | ||
| 198 | sock.close() | ||
| 199 | failed.clear() | ||
| 200 | sock, stream = connect(101, cursor=2) | ||
| 201 | try: | ||
| 202 | while not (message := receive()).get('lines'): pass | ||
| 203 | assert [r['id'] for r in message['lines']] == ['late-oom'] | ||
| 204 | finally: | ||
| 205 | stream.close() | ||
| 206 | sock.close() | ||
| 207 | bulk = [{**row, 'id': f'bulk-{i}', 'sequence': i + 4, '_msg': 'x' * 3500} for i in range(3000)] | ||
| 208 | rows.extend(bulk) | ||
| 209 | sock, stream = connect(101, cursor=3) | ||
| 210 | try: | ||
| 211 | received = [] | ||
| 212 | while len(received) < 3000: | ||
| 213 | message = receive() | ||
| 214 | assert not message.get('error'), message | ||
| 215 | received.extend(r['id'] for r in message['lines']) | ||
| 216 | assert received == [r['id'] for r in bulk] | ||
| 217 | finally: | ||
| 218 | stream.close() | ||
| 219 | sock.close() | ||
| 220 | accepted = time.time() | ||
| 221 | sequence = int(accepted * 1_000_000) | ||
| 222 | delayed = {**row, 'id': 'delayed-search-visibility', 'sequence': sequence} | ||
| 223 | checkpoint = 3003 | ||
| 224 | rows.append({**row, 'id': 'initial-unsettled', 'sequence': sequence + 1}) | ||
| 225 | def publish(): | ||
| 226 | nonlocal checkpoint | ||
| 227 | rows.append(delayed) | ||
| 228 | checkpoint = sequence + 1 | ||
| 229 | threading.Timer(7, publish).start() | ||
| 230 | sock, stream = connect(101, cursor=sequence + 1) | ||
| 231 | try: | ||
| 232 | while not (message := receive()).get('lines'): pass | ||
| 233 | assert [r['id'] for r in message['lines']] == [delayed['id'], 'initial-unsettled'] | ||
| 234 | assert time.time() - accepted >= 7 | ||
| 235 | finally: | ||
| 236 | stream.close() | ||
| 237 | sock.close() | ||
| 238 | rows[:] = [{**row, 'id': f'tie-{i:05}', 'sequence': i + 1} for i in range(6000)] + [{**row, '_time': '2026-10-13T00:00:00Z', 'id': 'older-history', 'sequence': 6001}] | ||
| 239 | history = [] | ||
| 240 | for offset in (0, 5000): | ||
| 241 | with urlopen(Request(origin + path + f'?limit=5000&offset={offset}', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: | ||
| 242 | history.extend(json.load(response)) | ||
| 243 | assert len({line['id'] for line in history}) == 6001 | ||
| 244 | assert history[-1]['id'] == 'older-history' | ||
| 245 | cookie = 'disposable-native-session' | ||
| 246 | with sqlite3.connect(root / 'data/accounts.sqlite') as db: | ||
| 247 | db.execute('INSERT INTO users VALUES (?,?)', ('fixture', json.dumps({'username':'fixture','enabled':True,'requiredActions':[]}))) | ||
| 248 | role = db.execute('SELECT id FROM roles WHERE name=?', ('infra-admin',)).fetchone()[0] | ||
| 249 | db.execute('INSERT INTO memberships VALUES (?,?)', ('fixture', role)) | ||
| 250 | db.execute('INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?)', (hashlib.sha256(cookie.encode()).hexdigest(),'fixture','dashboard',int(time.time())+120,'127.0.0.1',0,0,0)) | ||
| 251 | connect(401) | ||
| 252 | sock, stream = connect(101, cursor=sequence, cookie=cookie) | ||
| 253 | try: | ||
| 254 | while 'lines' not in (message := receive()): pass | ||
| 255 | with sqlite3.connect(root / 'data/accounts.sqlite') as db: | ||
| 256 | db.execute('DELETE FROM sessions') | ||
| 257 | while not (message := receive()).get('error'): pass | ||
| 258 | assert 'administrator' in message['error'] | ||
| 259 | finally: | ||
| 260 | stream.close() | ||
| 261 | sock.close() | ||
| 262 | print('REST events/filters, WebSocket auth/Origin, fragmented responses, timestamp ties, delayed OOM, reconnect cursor, oversized backlog, delayed visibility, session revocation, upstream failure: passed') | ||
| 263 | finally: | ||
| 264 | server.terminate() | ||
| 265 | server.wait(timeout=10) | ||
| 266 | fixture.shutdown() | ||
| 267 | |||
| 268 | |||
| 269 | if __name__ == '__main__': | ||
| 270 | main() | ||
tools/log-shipper-test.py created+137| ... | @@ -0,0 +1,137 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import importlib.util | ||
| 3 | import io | ||
| 4 | import json | ||
| 5 | from pathlib import Path | ||
| 6 | import tempfile | ||
| 7 | import unittest | ||
| 8 | from unittest.mock import patch | ||
| 9 | |||
| 10 | |||
| 11 | spec = importlib.util.spec_from_file_location("shipper", Path(__file__).with_name("log-shipper.py")) | ||
| 12 | shipper = importlib.util.module_from_spec(spec) | ||
| 13 | spec.loader.exec_module(shipper) | ||
| 14 | |||
| 15 | |||
| 16 | class Journal: | ||
| 17 | def __init__(self, *args, **kwargs): | ||
| 18 | self.stdout = io.StringIO("") | ||
| 19 | self.returncode = 0 | ||
| 20 | |||
| 21 | def wait(self): | ||
| 22 | return self.returncode | ||
| 23 | |||
| 24 | |||
| 25 | class Shipping(unittest.TestCase): | ||
| 26 | def setUp(self): | ||
| 27 | self.temporary = tempfile.TemporaryDirectory() | ||
| 28 | self.addCleanup(self.temporary.cleanup) | ||
| 29 | root = Path(self.temporary.name) | ||
| 30 | self.log = root / "alloc" / "allocation" / "alloc/logs/app.stdout.0" | ||
| 31 | self.log.parent.mkdir(parents=True) | ||
| 32 | self.log.write_bytes(b"2026-10-14T02:50:00Z stderr F before crash\n") | ||
| 33 | self.allocations = [{"ID": "allocation", "JobID": "shale", "TaskStates": {"app": {"Events": [ | ||
| 34 | {"Time": 1791946241000000000, "Type": "Terminated", "ExitCode": 137, | ||
| 35 | "DisplayMessage": "Exit Code: 137", "Details": {"oom_killed": "true"}}, | ||
| 36 | {"Time": 1791946242000000000, "Type": "Restarting", "DisplayMessage": "Task restarting in 17s"}, | ||
| 37 | {"Time": 1791946265000000000, "Type": "Started", "DisplayMessage": "Task started by client"}, | ||
| 38 | ]}}}] | ||
| 39 | self.batches = [] | ||
| 40 | self.checkpoints = [] | ||
| 41 | self.visible = None | ||
| 42 | replacement = patch.object(shipper, "retention_bounds", return_value=(0, float("inf"))) | ||
| 43 | replacement.start() | ||
| 44 | self.addCleanup(replacement.stop) | ||
| 45 | replacement = patch.object(shipper, "urlopen", side_effect=lambda *a, **k: io.BytesIO(json.dumps({"rows": self.visible if self.visible is not None else len(self.batches) - (self.checkpoints[-1] if self.checkpoints else 0)}).encode())) | ||
| 46 | replacement.start() | ||
| 47 | self.addCleanup(replacement.stop) | ||
| 48 | for target, value in [("LOGS", root / "alloc"), ("STATE", root / "state.json")]: | ||
| 49 | replacement = patch.object(shipper, target, value) | ||
| 50 | replacement.start() | ||
| 51 | self.addCleanup(replacement.stop) | ||
| 52 | replacement = patch.object(shipper, "nomad", side_effect=lambda path: self.allocations if path == "/v1/allocations" else [{"Address": "127.0.0.1", "Port": 1}]) | ||
| 53 | replacement.start() | ||
| 54 | self.addCleanup(replacement.stop) | ||
| 55 | replacement = patch.object(shipper.subprocess, "Popen", Journal) | ||
| 56 | replacement.start() | ||
| 57 | self.addCleanup(replacement.stop) | ||
| 58 | |||
| 59 | def once(self): | ||
| 60 | state = json.loads(shipper.STATE.read_text()) if shipper.STATE.exists() else {} | ||
| 61 | def deliver(base, rows, fields): | ||
| 62 | if rows and rows[0]["source"] == "log-shipper": | ||
| 63 | self.checkpoints.append(len(self.batches)) | ||
| 64 | else: | ||
| 65 | self.batches.extend(rows) | ||
| 66 | with patch.object(shipper, "send", side_effect=deliver): | ||
| 67 | shipper.once(state) | ||
| 68 | |||
| 69 | def test_oom_and_restarts_are_retained_once_across_process_restarts_and_allocation_gc(self): | ||
| 70 | self.once() | ||
| 71 | self.assertEqual([row.get("event") for row in self.batches], [None, "OOM", "Restarting", "Started"]) | ||
| 72 | self.assertEqual(self.batches[1]["level"], "error") | ||
| 73 | self.assertIn("Out of memory (OOM)", self.batches[1]["_msg"]) | ||
| 74 | self.assertEqual(self.batches[2]["level"], "warn") | ||
| 75 | self.assertEqual(len({row["id"] for row in self.batches}), 4) | ||
| 76 | self.assertEqual(len({row["sequence"] for row in self.batches}), 4) | ||
| 77 | self.once() | ||
| 78 | self.assertEqual(len(self.batches), 4) | ||
| 79 | self.allocations.clear() | ||
| 80 | self.once() | ||
| 81 | self.assertEqual(json.loads(shipper.STATE.read_text())["events"], {}) | ||
| 82 | self.assertEqual(len(self.batches), 4) | ||
| 83 | |||
| 84 | def test_same_timestamp_lines_and_split_writes_keep_distinct_ids(self): | ||
| 85 | self.once() | ||
| 86 | line = b"2026-10-14T02:50:01Z stdout F same message" | ||
| 87 | with self.log.open("ab") as file: | ||
| 88 | file.write(line) | ||
| 89 | self.once() | ||
| 90 | self.assertEqual(len(self.batches), 4) | ||
| 91 | with self.log.open("ab") as file: | ||
| 92 | file.write(b"\n" + line + b"\n") | ||
| 93 | self.once() | ||
| 94 | self.assertEqual(len(self.batches), 6) | ||
| 95 | self.assertNotEqual(self.batches[-1]["id"], self.batches[-2]["id"]) | ||
| 96 | self.assertGreater(self.batches[-2]["sequence"], self.batches[-3]["sequence"]) | ||
| 97 | self.assertGreater(self.batches[-1]["sequence"], self.batches[-2]["sequence"]) | ||
| 98 | |||
| 99 | def test_failed_delivery_does_not_advance_file_or_event_cursors(self): | ||
| 100 | with patch.object(shipper, "send", side_effect=OSError("store unavailable")): | ||
| 101 | with self.assertRaises(OSError): | ||
| 102 | shipper.once({}) | ||
| 103 | self.assertNotIn("events", json.loads(shipper.STATE.read_text())) | ||
| 104 | self.assertFalse(json.loads(shipper.STATE.read_text()).get("files")) | ||
| 105 | self.once() | ||
| 106 | self.assertEqual(len(self.batches), 4) | ||
| 107 | |||
| 108 | def test_expired_rows_advance_source_cursors_without_blocking_checkpoint(self): | ||
| 109 | with patch.object(shipper, "retention_bounds", return_value=(2_000_000_000, 3_000_000_000)): | ||
| 110 | self.once() | ||
| 111 | self.assertEqual(self.batches, []) | ||
| 112 | self.assertEqual(self.checkpoints, []) | ||
| 113 | self.assertTrue(json.loads(shipper.STATE.read_text())["events"]) | ||
| 114 | self.once() | ||
| 115 | self.assertEqual(self.batches, []) | ||
| 116 | |||
| 117 | def test_invisible_rows_do_not_commit_checkpoint_or_offsets(self): | ||
| 118 | self.visible = 3 | ||
| 119 | with patch.object(shipper.time, "monotonic", side_effect=[0, 31]): | ||
| 120 | with self.assertRaises(TimeoutError): | ||
| 121 | self.once() | ||
| 122 | self.assertEqual(self.checkpoints, []) | ||
| 123 | reserved = json.loads(shipper.STATE.read_text()) | ||
| 124 | self.assertFalse(reserved.get("files")) | ||
| 125 | self.assertNotIn("events", reserved) | ||
| 126 | self.visible = 4 | ||
| 127 | self.log.unlink() | ||
| 128 | self.allocations.clear() | ||
| 129 | with patch.object(shipper.time, "time_ns", return_value=0): | ||
| 130 | self.once() | ||
| 131 | self.assertEqual(self.batches[-1]["sequence"], reserved["sequence"]) | ||
| 132 | self.assertEqual(self.batches[:4], self.batches[-4:]) | ||
| 133 | self.assertEqual(len(self.checkpoints), 1) | ||
| 134 | |||
| 135 | |||
| 136 | if __name__ == "__main__": | ||
| 137 | unittest.main() | ||
tools/log-shipper.py+123-35| ... | @@ -1,11 +1,13 @@ | ... | @@ -1,11 +1,13 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | import datetime | 2 | import datetime |
| 3 | import json | 3 | import json |
| 4 | import math | ||
| 4 | import os | 5 | import os |
| 5 | from pathlib import Path | 6 | from pathlib import Path |
| 6 | import re | 7 | import re |
| 7 | import subprocess | 8 | import subprocess |
| 8 | import time | 9 | import time |
| 10 | from urllib.parse import urlencode | ||
| 9 | from urllib.request import Request, urlopen | 11 | from urllib.request import Request, urlopen |
| 10 | 12 | ||
| 11 | 13 | ||
| ... | @@ -31,44 +33,132 @@ def send(base, rows, fields): | ... | @@ -31,44 +33,132 @@ def send(base, rows, fields): |
| 31 | response.read() | 33 | response.read() |
| 32 | 34 | ||
| 33 | 35 | ||
| 36 | def retention_bounds(base): | ||
| 37 | with urlopen(base + "/metrics", timeout=10) as response: | ||
| 38 | metrics = response.read().decode() | ||
| 39 | flags = dict(re.findall(r'flag\{name="(retentionPeriod|futureRetention|maxBackfillAge)", value="([^"]+)"', metrics)) | ||
| 40 | durations = {} | ||
| 41 | units = {"ms": .001, "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000} | ||
| 42 | for name in ("retentionPeriod", "futureRetention", "maxBackfillAge"): | ||
| 43 | value = flags[name] | ||
| 44 | if re.fullmatch(r"[0-9.]+M?", value): | ||
| 45 | seconds = float(value.rstrip("M")) * 2678400 | ||
| 46 | else: | ||
| 47 | parts = re.findall(r"([0-9.]+)(ms|[smhdwy])", value.lower()) | ||
| 48 | if "".join(number + unit for number, unit in parts) != value.lower(): | ||
| 49 | raise ValueError("unsupported log retention duration") | ||
| 50 | seconds = sum(float(number) * units[unit] for number, unit in parts) | ||
| 51 | if not math.isfinite(seconds) or seconds < 0: | ||
| 52 | raise ValueError("invalid log retention duration") | ||
| 53 | durations[name] = int(seconds * 1000) / 1000 | ||
| 54 | retention = max(86400, durations["retentionPeriod"]) | ||
| 55 | backfill = durations["maxBackfillAge"] | ||
| 56 | backfill = backfill if 0 < backfill <= retention else retention | ||
| 57 | current = time.time() | ||
| 58 | return current - backfill, (math.floor((current + max(86400, durations["futureRetention"])) / 86400) + 1) * 86400 | ||
| 59 | |||
| 60 | |||
| 61 | def save(state): | ||
| 62 | pending_state = STATE.with_suffix(".pending") | ||
| 63 | with os.fdopen(os.open(pending_state, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as file: | ||
| 64 | json.dump(state, file) | ||
| 65 | os.replace(pending_state, STATE) | ||
| 66 | |||
| 67 | |||
| 34 | def once(state): | 68 | def once(state): |
| 35 | services = nomad("/v1/service/victoria-logs") | 69 | services = nomad("/v1/service/victoria-logs") |
| 36 | if not services: | 70 | if not services: |
| 37 | return | 71 | return |
| 38 | target = services[0] | 72 | target = services[0] |
| 39 | base = f"http://{target['Address']}:{target['Port']}" | 73 | base = f"http://{target['Address']}:{target['Port']}" |
| 40 | jobs = {item["ID"]: item["JobID"] for item in nomad("/v1/allocations")} | 74 | if "pending" not in state: |
| 41 | offsets = state.setdefault("files", {}) | 75 | allocations = nomad("/v1/allocations") |
| 42 | rows = [] | 76 | jobs = {item["ID"]: item["JobID"] for item in allocations} |
| 43 | pending = {} | 77 | offsets = state.get("files", {}).copy() |
| 44 | for file in sorted(LOGS.glob("*/alloc/logs/*")): | 78 | rows = [] |
| 45 | match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name) | 79 | pending = {} |
| 46 | job = jobs.get(file.parent.parent.parent.name) | 80 | for file in sorted(LOGS.glob("*/alloc/logs/*")): |
| 47 | if not match or not job: | 81 | match = re.fullmatch(r"(.+)\.(stdout|stderr)\.\d+", file.name) |
| 48 | continue | 82 | job = jobs.get(file.parent.parent.parent.name) |
| 49 | info = file.stat() | 83 | if not match or not job: |
| 50 | old = offsets.get(str(file), {}) | ||
| 51 | offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0 | ||
| 52 | with file.open("rb") as stream: | ||
| 53 | stream.seek(offset) | ||
| 54 | chunk = stream.read(262144) | ||
| 55 | if not chunk: | ||
| 56 | continue | 84 | continue |
| 57 | if not chunk.endswith(b"\n"): | 85 | info = file.stat() |
| 58 | chunk = chunk[:chunk.rfind(b"\n") + 1] | 86 | old = offsets.get(str(file), {}) |
| 59 | if not chunk: | 87 | offset = old.get("offset", 0) if old.get("inode") == info.st_ino and old.get("offset", 0) <= info.st_size else 0 |
| 60 | continue | 88 | with file.open("rb") as stream: |
| 61 | pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)} | 89 | stream.seek(offset) |
| 62 | for raw in chunk.decode("utf8", "replace").splitlines(): | 90 | chunk = stream.read(262144) |
| 63 | parsed = CRI.match(raw) | 91 | if not chunk: |
| 64 | if not parsed: | 92 | continue |
| 65 | continue | 93 | if not chunk.endswith(b"\n"): |
| 66 | stamp, stream, message = parsed.groups() | 94 | chunk = chunk[:chunk.rfind(b"\n") + 1] |
| 67 | level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info" | 95 | if not chunk: |
| 68 | rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job, | 96 | continue |
| 69 | "task": match.group(1), "stream": stream, "level": level}) | 97 | pending[str(file)] = {"inode": info.st_ino, "offset": offset + len(chunk)} |
| 70 | send(base, rows, "source,job,task,stream") | 98 | position = offset |
| 71 | offsets.update(pending) | 99 | for raw in chunk.splitlines(keepends=True): |
| 100 | identity = f"{file.parent.parent.parent.name}:{file.name}:{info.st_ino}:{position}" | ||
| 101 | position += len(raw) | ||
| 102 | parsed = CRI.match(raw.decode("utf8", "replace").rstrip("\r\n")) | ||
| 103 | if not parsed: | ||
| 104 | continue | ||
| 105 | stamp, stream, message = parsed.groups() | ||
| 106 | level = "error" if re.search(r"\b(ERROR|ERR|FATAL|CRITICAL|PANIC)\b|level=(error|fatal)", message, re.I) else "warn" if re.search(r"\b(WARN|WARNING|WRN)\b|level=warn", message, re.I) else "info" | ||
| 107 | rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": job, | ||
| 108 | "task": match.group(1), "stream": stream, "level": level, | ||
| 109 | "id": identity, "allocation": file.parent.parent.parent.name}) | ||
| 110 | events = {} | ||
| 111 | for allocation in allocations: | ||
| 112 | seen = set(state.get("events", {}).get(allocation["ID"], [])) | ||
| 113 | current = events[allocation["ID"]] = [] | ||
| 114 | for task, task_state in (allocation.get("TaskStates") or {}).items(): | ||
| 115 | for event in task_state.get("Events") or []: | ||
| 116 | kind = event["Type"] | ||
| 117 | identity = f"{allocation['ID']}:{task}:{event['Time']}:{kind}" | ||
| 118 | current.append(identity) | ||
| 119 | if identity in seen: | ||
| 120 | continue | ||
| 121 | oom = str((event.get("Details") or {}).get("oom_killed", "")).lower() == "true" | ||
| 122 | message = event.get("DisplayMessage") or event.get("Message") or kind | ||
| 123 | if oom: | ||
| 124 | message = "Out of memory (OOM): " + message | ||
| 125 | stamp = datetime.datetime.fromtimestamp(event["Time"] / 1e9, datetime.timezone.utc).isoformat() | ||
| 126 | level = "error" if oom or event.get("FailsTask") or kind in {"Driver Failure", "Failed Validation"} else "warn" if kind in {"Restarting", "Not Restarting"} or kind == "Terminated" and event.get("ExitCode") else "info" | ||
| 127 | rows.append({"_time": stamp, "_msg": message, "source": "nomad", "job": allocation["JobID"], | ||
| 128 | "task": task, "stream": "event", "level": level, "event": "OOM" if oom else kind, | ||
| 129 | "id": identity, "allocation": allocation["ID"]}) | ||
| 130 | sequence = max(state.get("sequence", 0), time.time_ns() // 1000) | ||
| 131 | for row in rows: | ||
| 132 | sequence += 1 | ||
| 133 | row["sequence"] = sequence | ||
| 134 | offsets.update(pending) | ||
| 135 | state["sequence"] = sequence | ||
| 136 | state["pending"] = {"rows": rows, "files": offsets, "events": events} | ||
| 137 | save(state) | ||
| 138 | batch = state["pending"] | ||
| 139 | rows = batch["rows"] | ||
| 140 | sequence = state["sequence"] | ||
| 141 | if rows: | ||
| 142 | minimum, maximum = retention_bounds(base) | ||
| 143 | rows = [row for row in rows if minimum <= datetime.datetime.fromisoformat(row["_time"].replace("Z", "+00:00")).timestamp() < maximum] | ||
| 144 | if rows: | ||
| 145 | send(base, rows, "source,job,task,stream") | ||
| 146 | query = f"source:=nomad sequence:>={rows[0]['sequence']} sequence:<={sequence} | stats count_uniq(sequence) as rows" | ||
| 147 | deadline = time.monotonic() + 30 | ||
| 148 | while True: | ||
| 149 | request = Request(base + "/select/logsql/query", data=urlencode({"query": query, "start": minimum, "end": maximum}).encode()) | ||
| 150 | with urlopen(request, timeout=10) as response: | ||
| 151 | visible = json.loads(response.read() or b"{}") | ||
| 152 | if int(visible.get("rows", 0)) == len(rows): | ||
| 153 | break | ||
| 154 | if time.monotonic() >= deadline: | ||
| 155 | raise TimeoutError("log batch is not searchable yet") | ||
| 156 | time.sleep(.25) | ||
| 157 | send(base, [{"_msg": "searchable log checkpoint", "source": "log-shipper", "sequence": sequence}], "source") | ||
| 158 | state["files"] = {path: value for path, value in batch["files"].items() if Path(path).is_file()} | ||
| 159 | state["events"] = batch["events"] | ||
| 160 | del state["pending"] | ||
| 161 | save(state) | ||
| 72 | 162 | ||
| 73 | command = ["journalctl", "--no-pager", "--output=json"] | 163 | command = ["journalctl", "--no-pager", "--output=json"] |
| 74 | command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] | 164 | command += ["--after-cursor=" + state["cursor"]] if state.get("cursor") else ["--since=-1 hour"] |
| ... | @@ -97,9 +187,7 @@ def once(state): | ... | @@ -97,9 +187,7 @@ def once(state): |
| 97 | send(base, system, "source,unit") | 187 | send(base, system, "source,unit") |
| 98 | if cursor: | 188 | if cursor: |
| 99 | state["cursor"] = cursor | 189 | state["cursor"] = cursor |
| 100 | pending_state = STATE.with_suffix(".pending") | 190 | save(state) |
| 101 | pending_state.write_text(json.dumps(state)) | ||
| 102 | os.replace(pending_state, STATE) | ||
| 103 | 191 | ||
| 104 | 192 | ||
| 105 | def main(): | 193 | def main(): |
| ... | @@ -110,7 +198,7 @@ def main(): | ... | @@ -110,7 +198,7 @@ def main(): |
| 110 | once(state) | 198 | once(state) |
| 111 | except Exception as error: | 199 | except Exception as error: |
| 112 | print(f"log shipping paused: {error}", flush=True) | 200 | print(f"log shipping paused: {error}", flush=True) |
| 113 | time.sleep(5) | 201 | time.sleep(1) |
| 114 | 202 | ||
| 115 | 203 | ||
| 116 | if __name__ == "__main__": | 204 | if __name__ == "__main__": |