1use crate::*;
2use futures::{SinkExt, StreamExt, stream};
3use axum::extract::{FromRequestParts, ws::{Message, WebSocketUpgrade}};
4
5pub async fn endpoint(app: Arc<App>, service: &str) -> Result<String> {
6 if let Some((base, _)) = &app.internal {
7 return Ok(format!("{}services/{}", base.as_str(), encoded(service)));
8 }
9 let id = service.to_owned();
10 let state = app.clone();
11 let found = app
12 .cache
13 .get(
14 format!("endpoint:{service}"),
15 Duration::from_secs(10),
16 move || async move {
17 let values = core::nomad(&state, &format!("/v1/service/{}", encoded(&id))).await?;
18 let instance = array(&values)
19 .iter()
20 .find(|v| v["Address"] == "127.0.0.1")
21 .or_else(|| array(&values).first())
22 .ok_or_else(|| {
23 Error::new(501, format!("{id} isn't running on this host yet."))
24 })?;
25 let address = string(&instance["Address"]);
26 Ok(json!(format!(
27 "http://{}:{}",
28 if address.contains(':') {
29 format!("[{address}]")
30 } else {
31 address.into()
32 },
33 number(&instance["Port"]) as u16
34 )))
35 },
36 )
37 .await?;
38 Ok(string(&found.value).to_owned())
39}
40
41pub async fn read(
42 app: Arc<App>,
43 url: String,
44 form: Option<Vec<(String, String)>>,
45) -> Result<Arc<Document>> {
46 let state = app.clone();
47 let key = format!("read:{url}:{}", serde_json::to_string(&form)?);
48 app.cache
49 .get(key, Duration::from_secs(2), move || async move {
50 let request = if let Some(form) = &form {
51 if env("SNOWGLOBE_PREVIEW", "") == "1" {
52 state.request(Method::GET, &url)?.query(form)
53 } else {
54 state.request(Method::POST, &url)?.form(form)
55 }
56 } else {
57 state.request(Method::GET, &url)?
58 };
59 let response = request
60 .timeout(Duration::from_secs(8))
61 .send()
62 .await
63 .map_err(|_| Error::new(502, "The telemetry store isn't answering."))?;
64 if !response.status().is_success() {
65 return Err(Error::new(
66 502,
67 format!("The telemetry store answered {}.", response.status()),
68 ));
69 }
70 if form.is_some() {
71 Ok(json!(response.text().await?))
72 } else {
73 Ok(response.json().await?)
74 }
75 })
76 .await
77}
78
79const METRICS: &[(&str, &str)] = &[
80 ("host.cpu", "snowglobe_host_cpu_percent"),
81 ("host.memory", "snowglobe_host_memory_bytes"),
82 ("host.temperature", "snowglobe_host_temperature_celsius"),
83 ("host.power", "snowglobe_host_power_watts"),
84 ("host.gpu", "snowglobe_host_gpu_percent"),
85 ("host.network", "snowglobe_host_network_bytes_per_second"),
86 ("service.cpu", "snowglobe_service_cpu_cores"),
87 ("service.memory", "snowglobe_service_memory_bytes"),
88 ("vm.cpu", "snowglobe_vm_cpu_percent"),
89 ("vm.memory", "snowglobe_vm_memory_bytes"),
90];
91
92fn series(result: &Value, application: bool) -> Value {
93 json!(
94 array(&result["data"]["result"])
95 .iter()
96 .take(if application { 12 } else { usize::MAX })
97 .map(|row| {
98 let metric = &row["metric"];
99 let name = if application {
100 let labels = metric
101 .as_object()
102 .into_iter()
103 .flat_map(|v| v.iter())
104 .filter(|(k, _)| {
105 !["__name__", "service", "service_name"].contains(&k.as_str())
106 })
107 .map(|(k, v)| format!("{k}={}", string(v)))
108 .collect::<Vec<_>>()
109 .join(" · ");
110 if labels.is_empty() {
111 "total".into()
112 } else {
113 labels
114 }
115 } else {
116 metric["service"]
117 .as_str()
118 .or(metric["vm"].as_str())
119 .or(metric["direction"].as_str())
120 .unwrap_or("total")
121 .into()
122 };
123 let t: Vec<_> = array(&row["values"]).iter().map(|v| v[0].clone()).collect();
124 let v: Vec<_> = array(&row["values"])
125 .iter()
126 .map(|v| {
127 let n = string(&v[1]).parse::<f64>().unwrap_or(f64::NAN);
128 if n.is_finite() { json!(n) } else { Value::Null }
129 })
130 .collect();
131 json!({"name":name,"t":t,"v":v})
132 })
133 .collect::<Vec<_>>()
134 )
135}
136
137pub fn query_number(
138 query: &HashMap<String, String>,
139 key: &str,
140 default: f64,
141 min: f64,
142 max: f64,
143) -> Result<f64> {
144 let value = query
145 .get(key)
146 .map(|v| v.parse::<f64>())
147 .transpose()
148 .map_err(|_| Error::new(400, format!("Invalid {key}.")))?
149 .unwrap_or(default);
150 if !value.is_finite() || value < min || value > max {
151 return Err(Error::new(400, format!("Invalid {key}.")));
152 }
153 Ok(value)
154}
155pub async fn metrics(
156 app: Arc<App>,
157 name: &str,
158 query: &HashMap<String, String>,
159 application: Option<&str>,
160 window: Option<(f64, f64)>,
161) -> Result<Arc<Document>> {
162 let range = query_number(query, "range", 3600.0, 60.0, 30.0 * 86400.0)?;
163 let to = window
164 .map(|(_, to)| to)
165 .unwrap_or_else(|| (now() / 2.0).floor() * 2.0);
166 let from = window.map(|(from, _)| from).unwrap_or(to - range);
167 let service = query.get("service");
168 let quote = |s: &str| serde_json::to_string(s).unwrap();
169 let selector = if let Some(id) = application {
170 format!(
171 "{name}{{service={}}} or {name}{{service_name={}}}",
172 quote(id),
173 quote(id)
174 )
175 } else {
176 let metric = METRICS
177 .iter()
178 .find(|(key, _)| *key == name)
179 .ok_or_else(|| Error::new(400, "Invalid metric."))?
180 .1;
181 let labels = service
182 .map(|id| format!(
183 "{{{}={}}}",
184 if name.starts_with("vm.") {
185 "vm"
186 } else {
187 "service"
188 },
189 quote(id)
190 ))
191 .unwrap_or_default();
192 format!("{metric}{labels}")
193 };
194 let key = if window.is_some() {
195 format!("series:{selector}:fixed:{from}:{to}")
196 } else {
197 format!("series:{selector}:live:{range}")
198 };
199 let state = app.clone();
200 let network = name == "host.network";
201 let application = application.is_some();
202 app.cache
203 .get(key, Duration::from_secs(2), move || async move {
204 let base = endpoint(state.clone(), "victoria-metrics").await?;
205 let query = params(&[
206 ("query", selector),
207 ("start", from.to_string()),
208 ("end", to.to_string()),
209 ("step", ((to - from) / 300.0).round().max(2.0).to_string()),
210 ]);
211 let response = read(state, format!("{base}/api/v1/query_range?{query}"), None).await?;
212 if response.value["status"] != "success" {
213 return Err(Error::new(502, "The metrics query failed."));
214 }
215 let mut values = series(&response.value, application);
216 if network {
217 values
218 .as_array_mut()
219 .unwrap()
220 .sort_by_key(|s| if s["name"] == "down" { 0 } else { 1 });
221 }
222 Ok(values)
223 })
224 .await
225}
226pub async fn metric_names(app: Arc<App>, id: &str) -> Result<Value> {
227 let query = params(&["service", "service_name"].map(|label| {
228 (
229 "match[]",
230 format!(
231 "{{{label}={},__name__!~\"snowglobe_.*|up|scrape_.*\"}}",
232 serde_json::to_string(id).unwrap()
233 ),
234 )
235 }));
236 let url = format!(
237 "{}/api/v1/label/__name__/values?{query}",
238 endpoint(app.clone(), "victoria-metrics").await?
239 );
240 let response = read(app, url, None).await?;
241 let valid = regex::Regex::new(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$").unwrap();
242 let mut values: Vec<_> = array(&response.value["data"])
243 .iter()
244 .filter(|v| valid.is_match(string(v)))
245 .cloned()
246 .collect();
247 values.sort_by(|a, b| string(a).cmp(string(b)));
248 Ok(json!(values))
249}
250
251/// Services the VPS agents report, newest state first; quiet for two minutes counts as down.
252pub async fn vps(app: Arc<App>) -> Result<Value> {
253 let base = endpoint(app.clone(), "victoria-metrics").await?;
254 let instant = |query: &str| {
255 let url = format!("{base}/api/v1/query?{}", params(&[("query", query.to_owned())]));
256 let app = app.clone();
257 async move {
258 let response = read(app, url, None).await?;
259 Ok::<_, Error>(
260 array(&response.value["data"]["result"])
261 .iter()
262 .map(|row| (string(&row["metric"]["service"]).to_owned(), number(&row["value"][1])))
263 .collect::<HashMap<_, _>>(),
264 )
265 }
266 };
267 let up = r#"{__name__=~"up|snowglobe_service_up",source="vps"}"#;
268 let (known, current, cpu, memory, restarts, started) = futures::try_join!(
269 instant(&format!("max by (service) (max_over_time({up}[7d]))")),
270 instant(&format!("max by (service) (last_over_time({up}[2m]))")),
271 instant(r#"max by (service) (last_over_time(snowglobe_service_cpu_cores{source="vps"}[2m]))"#),
272 instant(r#"max by (service) (last_over_time(snowglobe_service_memory_bytes{source="vps"}[2m]))"#),
273 instant(r#"max by (service) (last_over_time(snowglobe_service_restarts_total{source="vps"}[7d]))"#),
274 instant(r#"max by (service) (last_over_time(snowglobe_service_started_seconds{source="vps"}[2m]))"#),
275 )?;
276 let mut ids: Vec<_> = known.into_keys().filter(|id| id.starts_with("vps.")).collect();
277 ids.sort();
278 let mut services = Vec::new();
279 for id in ids {
280 services.push(json!({
281 "id": id,
282 "name": &id[4..],
283 "health": match current.get(&id) { Some(v) if *v >= 1.0 => "healthy", _ => "down" },
284 "cpu": cpu.get(&id),
285 "memory": memory.get(&id),
286 "restarts": restarts.get(&id),
287 "startedAt": started.get(&id).filter(|v| **v > 0.0),
288 "applicationMetrics": metric_names(app.clone(), &id).await?,
289 }));
290 }
291 Ok(json!(services))
292}
293
294#[derive(Debug, PartialEq)]
295struct Term {
296 field: Option<String>,
297 op: String,
298 value: String,
299 exclude: bool,
300}
301fn terms(query: &str, trace: bool) -> Result<Vec<Term>> {
302 let tokens =
303 regex::Regex::new(r#"(-?)(?:([\w.]+):(>=|<=|>|<)?)?(?:"([^"]*)"?|(\S+))"#).unwrap();
304 let wildcard = regex::Regex::new(r"(?i)^(\d+)x+$").unwrap();
305 let mut found = Vec::new();
306 for captures in tokens.captures_iter(query) {
307 let field = captures.get(2).map(|m| m.as_str());
308 let recognized = field.filter(|s| {
309 if trace {
310 ["status", "method", "path", "host", "client", "duration"].contains(s)
311 || s.contains('.')
312 } else {
313 ["container", "stream"].contains(s)
314 }
315 });
316 let value = if field.is_some() && recognized.is_none() {
317 captures[0].trim_start_matches('-').to_owned()
318 } else {
319 captures
320 .get(4)
321 .or(captures.get(5))
322 .map(|m| m.as_str())
323 .unwrap_or_default()
324 .to_owned()
325 };
326 let value = wildcard.replace(&value, "$1").into_owned();
327 if value.is_empty() {
328 continue;
329 }
330 found.push(Term {
331 field: recognized.map(str::to_owned),
332 op: if recognized.is_some() {
333 captures.get(3).map(|m| m.as_str()).unwrap_or(":")
334 } else {
335 ":"
336 }
337 .into(),
338 value,
339 exclude: &captures[1] == "-",
340 });
341 }
342 Ok(found)
343}
344fn log_query(id: &str, query: &HashMap<String, String>) -> Result<String> {
345 let (source, job) = id.strip_prefix("vps.").map_or(("nomad", id), |job| ("vps", job));
346 let mut filters = vec![
347 format!("source:={source}"),
348 format!("job:={}", serde_json::to_string(job)?),
349 ];
350 if let Some(level) = query.get("level") {
351 filters.push(
352 match level.as_str() {
353 "error" => "level:=error",
354 "warn" => "(level:=warn OR level:=error)",
355 _ => return Err(Error::new(400, "Invalid log level.")),
356 }
357 .into(),
358 );
359 }
360 for term in terms(
361 query.get("q").map(String::as_str).unwrap_or_default(),
362 false,
363 )? {
364 let field = match term.field.as_deref() {
365 Some("container") => "task",
366 Some("stream") => "stream",
367 _ => "_msg",
368 };
369 filters.push(format!(
370 "{}{field}:~{}",
371 if term.exclude { "!" } else { "" },
372 serde_json::to_string(&format!(
373 "(?i){}{}",
374 if term.field.is_some() { "^" } else { "" },
375 regex::escape(&term.value)
376 ))?
377 ));
378 }
379 Ok(filters.join(" "))
380}
381
382fn log_line(row: Value) -> Value {
383 json!({"id":row["id"].as_str().map(str::to_owned).unwrap_or_else(|| format!("{}:{}:{}", string(&row["_stream_id"]), string(&row["_time"]), string(&row["_msg"]))),
384 "sequence":number(&row["sequence"]),
385 "t":core::seconds(&row["_time"]),"container":row["task"].as_str().unwrap_or("app"),
386 "stream":row["stream"].as_str().filter(|s| ["stderr", "event"].contains(s)).unwrap_or("stdout"),
387 "level":row["level"],"text":row["_msg"].as_str().unwrap_or_default(),
388 "event":row["event"],"allocation":row["allocation"]})
389}
390
391pub async fn logs(app: Arc<App>, id: &str, query: &HashMap<String, String>) -> Result<Value> {
392 let limit = query_number(query, "limit", 200.0, 1.0, 5000.0)?;
393 if query.contains_key("after") && query.contains_key("before")
394 && query_number(query, "after", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
395 > query_number(query, "before", 0.0, f64::NEG_INFINITY, f64::INFINITY)?
396 {
397 return Err(Error::new(400, "Choose a “to” time at or after “from”."));
398 }
399 let offset = query_number(query, "offset", 0.0, 0.0, 1_000_000_000.0)?;
400 let mut form = vec![
401 ("query".into(), format!("{} | sort by (_time desc, id desc, _stream_id desc, _msg desc) offset {offset:.0} limit {limit:.0}", log_query(id, query)?)),
402 ("limit".into(), limit.to_string()),
403 ];
404 for (key, field) in [("after", "start"), ("before", "end")] {
405 if query.contains_key(key) {
406 form.push((
407 field.into(),
408 query_number(query, key, 0.0, f64::NEG_INFINITY, f64::INFINITY)?
409 .to_string(),
410 ));
411 }
412 }
413 let url = format!(
414 "{}/select/logsql/query",
415 endpoint(app.clone(), "victoria-logs").await?
416 );
417 let response = read(app, url, Some(form)).await?;
418 let mut rows = string(&response.value).lines().filter(|l| !l.trim().is_empty()).map(|line| {
419 let row: Value = serde_json::from_str(line)?;
420 Ok::<_,Error>(log_line(row))
421 }).collect::<Result<Vec<_>>>()?;
422 rows.sort_by(|a, b| number(&b["t"]).total_cmp(&number(&a["t"])));
423 Ok(json!(rows))
424}
425
426pub async fn log_stream(app: Arc<App>, request: Request, id: &str, query: &HashMap<String, String>) -> Result<Response> {
427 vms::same_origin(&request)?;
428 let filter = log_query(id, query)?;
429 let mut cursor = query_number(query, "cursor", 0.0, 0.0, 9_007_199_254_740_991.0)?;
430 let from = query.get("from").map(|_| query_number(query, "from", 0.0, 0.0, f64::INFINITY)).transpose()?;
431 let base = endpoint(app.clone(), "victoria-logs").await?;
432 let url = format!("{base}/select/logsql/query");
433 let headers = request.headers().clone();
434 let (mut parts, _) = request.into_parts();
435 let upgrade = WebSocketUpgrade::from_request_parts(&mut parts, &()).await
436 .map_err(|_| Error::new(400, "Open the service log to connect."))?
437 .max_message_size(1024).max_frame_size(1024);
438 Ok(upgrade.on_upgrade(move |mut socket| async move {
439 let run = async {
440 let _slot = app.log_slots.try_acquire().map_err(|_| Error::new(429, "Too many live logs are open. Close one and retry."))?;
441 let mut first = true;
442 let mut poll = tokio::time::interval(Duration::from_secs(1));
443 poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
444 let mut heartbeat = tokio::time::interval(Duration::from_secs(15));
445 loop {
446 tokio::select! {
447 message = socket.recv() => match message {
448 Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?,
449 Some(Ok(Message::Pong(_))) => {},
450 _ => break,
451 },
452 _ = heartbeat.tick() => {
453 if app.auth.ready() {
454 let account = app.auth.session(&headers, "dashboard")?;
455 if !array(&account["groups"]).iter().any(|g| g["name"] == "infra-admin") {
456 return Err(Error::new(401, "Sign in as an administrator to continue reading logs."));
457 }
458 }
459 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Ping(Bytes::new()))).await
460 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
461 },
462 _ = poll.tick() => {
463 let checkpoint = read(app.clone(), url.clone(), Some(vec![("query".into(), "source:=log-shipper | stats max(sequence) as sequence".into())])).await?;
464 let checkpoint: Value = serde_json::from_str(string(&checkpoint.value).lines().find(|line| !line.trim().is_empty()).unwrap_or("{}"))?;
465 let settled = number(&checkpoint["sequence"]);
466 if first { cursor = cursor.min(settled); first = false; }
467 let mut form = vec![("query", format!("{filter} sequence:>{cursor:.0} sequence:<={settled:.0} | sort by (sequence) limit 5000"))];
468 if let Some(from) = from { form.push(("start", from.to_string())); }
469 let request = if env("SNOWGLOBE_PREVIEW", "") == "1" {
470 app.request(Method::GET, &url)?.query(&form)
471 } else {
472 app.request(Method::POST, &url)?.form(&form)
473 };
474 let response = request.timeout(Duration::from_secs(8)).send().await?;
475 if !response.status().is_success() { return Err(Error::new(502, "The log store isn't answering. Retrying…")); }
476 let mut chunks = response.bytes_stream();
477 let mut body = Vec::new();
478 while let Some(chunk) = chunks.next().await {
479 body.extend_from_slice(&chunk?);
480 if body.len() > 8 * 1024 * 1024 {
481 let end = body[..8 * 1024 * 1024].iter().rposition(|b| *b == b'\n')
482 .ok_or_else(|| Error::new(502, "A log entry is too large. Narrow the search and retry."))?;
483 body.truncate(end + 1);
484 break;
485 }
486 }
487 let lines = std::str::from_utf8(&body)?.lines().filter(|l| !l.trim().is_empty())
488 .map(|raw| serde_json::from_str(raw).map(log_line)).collect::<std::result::Result<Vec<_>, _>>()?;
489 for line in &lines { cursor = cursor.max(number(&line["sequence"])); }
490 tokio::time::timeout(Duration::from_secs(5), socket.send(Message::Text(json!({"lines":lines,"cursor":cursor}).to_string().into()))).await
491 .map_err(|_| Error::new(504, "The log connection stopped responding."))??;
492 }
493 }
494 }
495 Ok::<_, Error>(())
496 }.await;
497 if let Err(error) = run {
498 let _ = tokio::time::timeout(Duration::from_secs(2), socket.send(Message::Text(json!({"error":error.message}).to_string().into()))).await;
499 }
500 let _ = tokio::time::timeout(Duration::from_secs(1), socket.close()).await;
501 }).into_response())
502}
503
504fn span(row: &Value) -> Value {
505 let mut attributes: serde_json::Map<String, Value> = row
506 .as_object()
507 .into_iter()
508 .flat_map(|v| v.iter())
509 .filter_map(|(key, value)| {
510 key.strip_prefix("span_attr:")
511 .map(|key| (key.into(), value.clone()))
512 })
513 .collect();
514 for (from, to) in [
515 ("http.request.method", "http.request.method"),
516 ("http.response.status_code", "http.response.status_code"),
517 ("http.route", "url.path"),
518 ] {
519 if let Some(value) = row.get(format!("resource_attr:{from}")) {
520 attributes
521 .entry(to.to_owned())
522 .or_insert_with(|| value.clone());
523 }
524 }
525 let status = attributes
526 .get("http.response.status_code")
527 .or(attributes.get("http.status_code"))
528 .cloned()
529 .unwrap_or(Value::Null);
530 let service = attributes
531 .get("snowglobe.service")
532 .cloned()
533 .unwrap_or_else(|| {
534 row["resource_attr:service.name"]
535 .as_str()
536 .map(|s| json!(s))
537 .unwrap_or(json!("unknown"))
538 });
539 let error = if row["status_code"] == "2" || number(&status) >= 500.0 {
540 if !status.is_null() {
541 status
542 } else {
543 json!(
544 row["status_message"]
545 .as_str()
546 .filter(|s| !s.is_empty())
547 .unwrap_or("request failed")
548 )
549 }
550 } else {
551 Value::Null
552 };
553 let name = if attributes.get("snowglobe.kind").is_some_and(|v| v == "edge") {
554 format!("edge: {}", row["name"].as_str().unwrap_or("request"))
555 } else {
556 string(&row["name"]).into()
557 };
558 json!({"id":row["span_id"],"parent":row["parent_span_id"].as_str().filter(|s| !s.is_empty()),"service":service,"name":name,"start":number(&row["start_time_unix_nano"])/1e9,"duration":number(&row["duration"])/1e9,"error":error,"attributes":attributes})
559}
560async fn spans(app: Arc<App>, query: String) -> Result<Vec<Value>> {
561 let url = format!(
562 "{}/select/logsql/query",
563 endpoint(app.clone(), "victoria-traces").await?
564 );
565 let rows = read(app, url, Some(vec![("query".into(), query)])).await?;
566 let mut traces: HashMap<String, Vec<Value>> = HashMap::new();
567 for line in string(&rows.value).lines().filter(|l| !l.trim().is_empty()) {
568 let row: Value = serde_json::from_str(line)?;
569 if string(&row["trace_id"]).is_empty() || string(&row["span_id"]).is_empty() {
570 continue;
571 }
572 traces
573 .entry(string(&row["trace_id"]).to_owned())
574 .or_default()
575 .push(span(&row));
576 }
577 Ok(traces
578 .into_iter()
579 .map(|(id, mut spans)| {
580 let parents: HashMap<_, _> = spans
581 .iter()
582 .map(|s| (string(&s["id"]).to_owned(), string(&s["parent"]).to_owned()))
583 .collect();
584 let depth = |span: &Value| {
585 let mut parent = string(&span["parent"]);
586 let mut seen = std::collections::HashSet::new();
587 let mut n = 0;
588 while let Some(next) = parents.get(parent) {
589 if !seen.insert(parent) {
590 break;
591 }
592 n += 1;
593 parent = next;
594 }
595 n
596 };
597 spans.sort_by(|a, b| {
598 depth(a)
599 .cmp(&depth(b))
600 .then_with(|| number(&a["start"]).total_cmp(&number(&b["start"])))
601 });
602 json!({"id":id,"spans":spans})
603 })
604 .collect())
605}
606pub async fn trace(app: Arc<App>, id: &str) -> Result<Value> {
607 spans(app, format!("trace_id:={}", serde_json::to_string(id)?))
608 .await?
609 .into_iter()
610 .next()
611 .ok_or_else(|| {
612 Error::new(
613 404,
614 "That trace has aged out of the trace store. Pick a newer one.",
615 )
616 })
617}
618pub async fn has_traces(app: Arc<App>, service: &str) -> Result<bool> {
619 let url = format!(
620 "{}/select/logsql/query",
621 endpoint(app.clone(), "victoria-traces").await?
622 );
623 Ok(!string(
624 &read(
625 app,
626 url,
627 Some(vec![(
628 "query".into(),
629 format!(
630 "_time:7d \"resource_attr:service.name\":={} | limit 1",
631 serde_json::to_string(service)?
632 ),
633 )]),
634 )
635 .await?
636 .value,
637 )
638 .trim()
639 .is_empty())
640}
641pub async fn traces(
642 app: Arc<App>,
643 id: &str,
644 query: &HashMap<String, String>,
645 visible: impl Fn(&Value) -> bool,
646) -> Result<Value> {
647 let jobs = core::scan(app.clone()).await?;
648 let Some(service) = jobs.value[id]["job"]["Meta"]["snowglobe_trace_service"]
649 .as_str()
650 .filter(|s| !s.is_empty())
651 else {
652 return Ok(json!([]));
653 };
654 let sort = query.get("sort").map(String::as_str).unwrap_or("newest");
655 let (key, descending) = match sort {
656 "newest" => ("_time", true),
657 "oldest" => ("_time", false),
658 "slowest" => ("duration", true),
659 "fastest" => ("duration", false),
660 _ => return Err(Error::new(400, "Invalid trace order.")),
661 };
662 let limit = query_number(query, "limit", 50.0, 1.0, 500.0)?;
663 let quote = |s: &str| serde_json::to_string(s).unwrap();
664 let like = |field: &str, pattern: &str| {
665 format!("{}:~{}", quote(field), quote(&format!("(?i){pattern}")))
666 };
667 let mut filters = vec![
668 "_time:7d".into(),
669 format!("\"resource_attr:service.name\":={}", quote(service)),
670 ];
671 if query.get("errors").is_some_and(|v| v == "1") {
672 filters.push("(status_code:=2 OR \"span_attr:http.response.status_code\":>=500)".into());
673 }
674 let duration = regex::Regex::new(r"^(\d+(?:\.\d+)?)(us|µs|ms|s|m)?$").unwrap();
675 for term in terms(query.get("q").map(String::as_str).unwrap_or_default(), true)? {
676 let escaped = regex::escape(&term.value);
677 let filter = match term.field.as_deref() {
678 Some("duration") => {
679 let captures = duration.captures(&term.value).ok_or_else(|| {
680 Error::new(
681 400,
682 format!("Write a duration like 500ms or 2s, not {}.", term.value),
683 )
684 })?;
685 let nanos = match captures.get(2).map(|m| m.as_str()).unwrap_or("ms") {
686 "us" | "µs" => 1e3,
687 "s" => 1e9,
688 "m" => 60e9,
689 _ => 1e6,
690 };
691 format!(
692 "duration:{}{}",
693 if term.op == ":" { ">=" } else { &term.op },
694 (captures[1].parse::<f64>().unwrap() * nanos).round()
695 )
696 }
697 None => format!(
698 "({})",
699 [
700 "name",
701 "span_attr:url.path",
702 "span_attr:http.request.method",
703 "span_attr:http.response.status_code",
704 "span_attr:server.address"
705 ]
706 .map(|f| like(f, &escaped))
707 .join(" OR ")
708 ),
709 Some(field) => {
710 let alias = match field {
711 "status" => "http.response.status_code",
712 "method" => "http.request.method",
713 "path" => "url.path",
714 "host" => "server.address",
715 "client" => "client.address",
716 _ => field,
717 };
718 if term.op == ":" {
719 like(&format!("span_attr:{alias}"), &format!("^{escaped}"))
720 } else {
721 let value = term
722 .value
723 .parse::<f64>()
724 .ok()
725 .filter(|n| n.is_finite())
726 .ok_or_else(|| {
727 Error::new(400, format!("Put a number after {field}:{}.", term.op))
728 })?;
729 format!(
730 "{}:{}{value}",
731 quote(&format!("span_attr:{alias}")),
732 term.op
733 )
734 }
735 }
736 };
737 filters.push(if term.exclude {
738 format!("!({filter})")
739 } else {
740 filter
741 });
742 }
743 let mut found = spans(
744 app,
745 format!(
746 "trace_id:in({} | sort by ({key}{}) | limit {limit} | fields trace_id)",
747 filters.join(" "),
748 if descending { " desc" } else { "" }
749 ),
750 )
751 .await?;
752 for trace in &mut found {
753 trace["spans"].as_array_mut().unwrap().retain(&visible);
754 }
755 found.retain(|trace| !array(&trace["spans"]).is_empty());
756 let value = |trace: &Value| {
757 array(&trace["spans"])
758 .iter()
759 .find(|s| s["service"] == service && s["attributes"]["snowglobe.kind"] != "edge")
760 .map(|s| number(&s[if key == "_time" { "start" } else { "duration" }]))
761 .unwrap_or(0.0)
762 };
763 found.sort_by(|a, b| {
764 if descending {
765 value(b).total_cmp(&value(a))
766 } else {
767 value(a).total_cmp(&value(b))
768 }
769 });
770 Ok(json!(found.into_iter().map(|t| { let spans = array(&t["spans"]); let mut services: Vec<_> = spans.iter().map(|s| s["service"].clone()).collect(); services.sort_by(|a,b| string(a).cmp(string(b))); services.dedup(); json!({"id":t["id"],"root":spans.first(),"spans":spans.len(),"services":services,"error":spans.iter().find(|s| !s["error"].is_null()).map(|s| s["error"].clone())}) }).collect::<Vec<_>>()))
771}
772
773#[derive(Default)]
774struct Recorder {
775 signature: Value,
776 cpu: HashMap<String, (f64, f64)>,
777 vms: HashMap<String, (f64, f64)>,
778 network: Option<(f64, f64, f64)>,
779}
780impl Recorder {
781 async fn collect(&mut self, app: Arc<App>) -> Result<Vec<String>> {
782 let jobs = core::scan(app.clone()).await?;
783 let mut signature = Vec::new();
784 let mut owners = HashMap::new();
785 for (id, found) in jobs.value.as_object().unwrap() {
786 for alloc in array(&found["allocs"]) {
787 let alloc_id = string(&alloc["ID"]).to_owned();
788 owners.insert(alloc_id.clone(), id.clone());
789 for (name, task) in alloc["TaskStates"]
790 .as_object()
791 .into_iter()
792 .flat_map(|v| v.iter())
793 {
794 signature.push((alloc_id.clone(), name.clone(), task["StartedAt"].clone()));
795 }
796 }
797 }
798 signature.sort_by(|a, b| (&a.0, &a.1).cmp(&(&b.0, &b.1)));
799 let signature = json!(signature);
800 let counters =
801 host::call(json!({"operation":"host.usage","refresh":signature != self.signature}))
802 .await?;
803 self.signature = signature;
804 let at = number(&counters["at"]);
805 let timestamp = (now() * 1000.0) as i64;
806 let mut next = HashMap::new();
807 let mut usage: HashMap<String, (f64, f64)> = HashMap::new();
808 for container in array(&counters["containers"]) {
809 let name = string(&container["name"]);
810 let alloc = name
811 .get(name.len().saturating_sub(36)..)
812 .unwrap_or_default();
813 let Some(service) = owners.get(alloc) else {
814 continue;
815 };
816 let usec = number(&container["cpu"]);
817 let memory = number(&container["memory"]);
818 let id = string(&container["id"]);
819 let cpu = self
820 .cpu
821 .get(id)
822 .filter(|(_, t)| at > *t)
823 .map(|(old, t)| ((usec - old) / ((at - t) * 1e6)).max(0.0))
824 .unwrap_or(0.0);
825 next.insert(id.to_owned(), (usec, at));
826 let item = usage.entry(service.clone()).or_default();
827 item.0 += cpu;
828 item.1 += memory;
829 }
830 self.cpu = next;
831 *app.usage.lock().unwrap() = usage
832 .iter()
833 .map(|(id, (cpu, memory))| (id.clone(), json!({"cpu":cpu,"memory":memory})))
834 .collect();
835 let bytes = app.live.borrow().clone();
836 let live: Value = if bytes.is_empty() {
837 json!({})
838 } else {
839 serde_json::from_slice(&bytes)?
840 };
841 let mut lines = vec![
842 format!(
843 "snowglobe_host_cpu_percent {} {timestamp}",
844 number(&live["host"]["cpu"])
845 ),
846 format!(
847 "snowglobe_host_memory_bytes {} {timestamp}",
848 number(&live["host"]["memory"])
849 ),
850 ];
851 for (id, (cpu, memory)) in usage {
852 let id = serde_json::to_string(&id)?;
853 lines.push(format!(
854 "snowglobe_service_cpu_cores{{service={id}}} {cpu} {timestamp}"
855 ));
856 lines.push(format!(
857 "snowglobe_service_memory_bytes{{service={id}}} {memory} {timestamp}"
858 ));
859 }
860 let sample = host::sample(app.clone()).await?;
861 let sample = &sample.value;
862 let rx = number(&sample["network"]["rx"]);
863 let tx = number(&sample["network"]["tx"]);
864 let network_at = number(&sample["at"]);
865 if let Some((old_rx, old_tx, t)) = self.network.filter(|(_, _, t)| network_at > *t) {
866 lines.push(format!(
867 "snowglobe_host_network_bytes_per_second{{direction=\"down\"}} {} {timestamp}",
868 ((rx - old_rx) / (network_at - t)).max(0.0)
869 ));
870 lines.push(format!(
871 "snowglobe_host_network_bytes_per_second{{direction=\"up\"}} {} {timestamp}",
872 ((tx - old_tx) / (network_at - t)).max(0.0)
873 ));
874 }
875 self.network = Some((rx, tx, network_at));
876 if let Some(value) = live["host"]["temperature"].as_f64() {
877 lines.push(format!(
878 "snowglobe_host_temperature_celsius {value} {timestamp}"
879 ));
880 }
881 if let Some(value) = sample["gpu"].as_f64() {
882 lines.push(format!("snowglobe_host_gpu_percent {value} {timestamp}"));
883 }
884 if let Ok(Ok(stats)) = tokio::time::timeout(
885 Duration::from_secs(8),
886 host::call(json!({"operation":"vm.stats"})),
887 )
888 .await
889 {
890 let mut next = HashMap::new();
891 let mut usage = HashMap::new();
892 for (id, stat) in stats.as_object().into_iter().flat_map(|v| v.iter()) {
893 let cpu = self
894 .vms
895 .get(id)
896 .filter(|(_, t)| at > *t)
897 .map(|(old, t)| {
898 ((number(&stat["cpu"]) - old) / (at - t) / number(&stat["vcpus"]) * 100.0)
899 .max(0.0)
900 })
901 .unwrap_or(0.0);
902 usage.insert(id.clone(), json!({"cpu":cpu,"memory":stat["memory"]}));
903 next.insert(id.clone(), (number(&stat["cpu"]), at));
904 lines.push(format!(
905 "snowglobe_vm_cpu_percent{{vm={}}} {cpu} {timestamp}",
906 serde_json::to_string(id)?
907 ));
908 lines.push(format!(
909 "snowglobe_vm_memory_bytes{{vm={}}} {} {timestamp}",
910 serde_json::to_string(id)?,
911 number(&stat["memory"])
912 ));
913 }
914 self.vms = next;
915 *app.vm_usage.lock().unwrap() = usage;
916 }
917 Ok(lines)
918 }
919}
920pub fn start(app: Arc<App>) {
921 tokio::spawn(async move {
922 let mut recorder = Recorder::default();
923 loop {
924 let result = async {
925 let base = endpoint(app.clone(), "victoria-metrics").await?;
926 if env("SNOWGLOBE_METRICS", "") == "follow" {
927 let names: Vec<_> = METRICS
928 .iter()
929 .filter(|(k, _)| k.starts_with("service.") || k.starts_with("vm."))
930 .map(|(_, v)| *v)
931 .collect();
932 let url = format!(
933 "{base}/api/v1/query?{}",
934 params(&[("query", format!("{{__name__=~\"{}\"}}", names.join("|")))])
935 );
936 let result = read(app.clone(), url, None).await?;
937 let mut services = HashMap::<String, Value>::new();
938 let mut vms = HashMap::<String, Value>::new();
939 for row in array(&result.value["data"]["result"]) {
940 let metric = &row["metric"];
941 let (map, id) = if metric["service"].is_string() {
942 (&mut services, string(&metric["service"]))
943 } else {
944 (&mut vms, string(&metric["vm"]))
945 };
946 let value = map
947 .entry(id.into())
948 .or_insert_with(|| json!({"cpu":0,"memory":0}));
949 value[if string(&metric["__name__"]).contains("_cpu_") {
950 "cpu"
951 } else {
952 "memory"
953 }] = json!(number(&row["value"][1]));
954 }
955 *app.usage.lock().unwrap() = services;
956 *app.vm_usage.lock().unwrap() = vms;
957 } else {
958 let lines = recorder.collect(app.clone()).await?;
959 app.request(Method::POST, &format!("{base}/api/v1/import/prometheus"))?
960 .body(lines.join("\n") + "\n")
961 .send()
962 .await?
963 .error_for_status()?;
964 let jobs = core::scan(app.clone()).await?;
965 let scrapes: Vec<_> = jobs
966 .value
967 .as_object()
968 .unwrap()
969 .iter()
970 .flat_map(|(id, job)| {
971 array(&job["job"]["TaskGroups"])
972 .iter()
973 .flat_map(|group| array(&group["Services"]))
974 .flat_map(move |service| {
975 array(&service["Tags"]).iter().filter_map(move |tag| {
976 string(tag).strip_prefix("snowglobe-metrics-path=").map(
977 |path| {
978 (
979 id.clone(),
980 string(&service["Name"]).to_owned(),
981 path.to_owned(),
982 )
983 },
984 )
985 })
986 })
987 })
988 .collect();
989 stream::iter(scrapes)
990 .for_each_concurrent(4, |(id, service, path)| {
991 let app = app.clone();
992 let base = base.clone();
993 async move {
994 let result = async {
995 let response = app
996 .request(
997 Method::GET,
998 &format!(
999 "{}{path}",
1000 endpoint(app.clone(), &service).await?
1001 ),
1002 )?
1003 .timeout(Duration::from_secs(5))
1004 .send()
1005 .await?
1006 .error_for_status()?;
1007 app.request(
1008 Method::POST,
1009 &format!(
1010 "{base}/api/v1/import/prometheus?{}",
1011 params(&[
1012 ("extra_label", format!("service={id}")),
1013 ("extra_label", format!("instance={service}"))
1014 ])
1015 ),
1016 )?
1017 .body(response.text().await?)
1018 .send()
1019 .await?
1020 .error_for_status()?;
1021 Ok::<_, Error>(())
1022 }
1023 .await;
1024 if let Err(e) = result {
1025 eprintln!("metrics {id}: {}", e.message);
1026 }
1027 }
1028 })
1029 .await;
1030 }
1031 Ok::<_, Error>(())
1032 }
1033 .await;
1034 if let Err(e) = result
1035 && e.status != 501
1036 {
1037 eprintln!("metrics: {}", e.message);
1038 }
1039 tokio::time::sleep(Duration::from_secs(15)).await;
1040 }
1041 });
1042}
1043
1044#[cfg(test)]
1045mod tests {
1046 use super::*;
1047 #[test]
1048 fn vps_ids_read_the_vps_source() {
1049 let none = HashMap::new();
1050 assert_eq!(log_query("vps.site", &none).unwrap(), r#"source:=vps job:="site""#);
1051 assert_eq!(log_query("shale", &none).unwrap(), r#"source:=nomad job:="shale""#);
1052 }
1053 #[test]
1054 fn search_fields_comparisons_exclusions_and_unknown_fields() {
1055 let parsed = terms(
1056 r#"status:5xx -path:/health duration:>1s "GET /x" user:42 -noise"#,
1057 true,
1058 )
1059 .unwrap();
1060 assert_eq!(
1061 parsed
1062 .iter()
1063 .map(|t| (
1064 t.field.as_deref(),
1065 t.op.as_str(),
1066 t.value.as_str(),
1067 t.exclude
1068 ))
1069 .collect::<Vec<_>>(),
1070 vec![
1071 (Some("status"), ":", "5", false),
1072 (Some("path"), ":", "/health", true),
1073 (Some("duration"), ">", "1s", false),
1074 (None, ":", "GET /x", false),
1075 (None, ":", "user:42", false),
1076 (None, ":", "noise", true)
1077 ]
1078 );
1079 assert_eq!(terms(r#" "" - "#, false).unwrap().len(), 1);
1080 }
1081 #[test]
1082 fn non_finite_points_are_null() {
1083 assert_eq!(
1084 series(
1085 &json!({"data":{"result":[{"metric":{"service":"a"},"values":[[1,"2"],[2,"NaN"],[3,"+Inf"]]}]}}),
1086 false
1087 ),
1088 json!([{"name":"a","t":[1,2,3],"v":[2.0,null,null]}])
1089 );
1090 }
1091 #[test]
1092 fn span_error_status_is_preserved() {
1093 let s = span(
1094 &json!({"span_id":"a","parent_span_id":"a","start_time_unix_nano":"1000000000","duration":"20","status_code":"2","status_message":"broken"}),
1095 );
1096 assert_eq!(s["error"], "broken");
1097 }
1098}