1use crate::*;
2use futures::{StreamExt, stream};
3use sha1::{Digest, Sha1};
4use std::path::Path;
5
6pub async fn nomad(app: &App, path: &str) -> Result<Value> {
7 let value = nomad_raw(app, path).await?;
8 Ok(value
9 .map(|bytes| serde_json::from_slice(&bytes))
10 .transpose()?
11 .unwrap_or(Value::Null))
12}
13pub async fn nomad_raw(app: &App, path: &str) -> Result<Option<Bytes>> {
14 let request = async {
15 let token = tokio::fs::read_to_string(env(
16 "SNOWGLOBE_NOMAD_TOKEN_FILE",
17 "/var/lib/snowglobe/dashboard.token",
18 ))
19 .await
20 .map_err(|e| Error::new(501, format!("Nomad isn't connected: {e}")))?;
21 let _slot = app.nomad_slots.acquire().await?;
22 let base = match &app.internal {
23 Some((base, _)) => format!("{}nomad", base.as_str()),
24 None => env("NOMAD_ADDR", "http://127.0.0.1:4646"),
25 };
26 let response = app
27 .request(Method::GET, &format!("{base}{path}"))?
28 .header("X-Nomad-Token", token.trim())
29 .send()
30 .await
31 .map_err(|_| {
32 Error::new(502, "Service status is taking too long. Retry in a moment.")
33 })?;
34 if response.status() == 404 {
35 return Ok(None);
36 }
37 let status = response.status();
38 let bytes = response.bytes().await?;
39 if !status.is_success() {
40 return Err(Error::new(
41 502,
42 format!(
43 "Nomad refused {} ({status}): {}",
44 path.split('?').next().unwrap(),
45 String::from_utf8_lossy(&bytes).trim()
46 ),
47 ));
48 }
49 Ok(Some(bytes))
50 };
51 tokio::time::timeout(Duration::from_secs(15), request)
52 .await
53 .map_err(|_| Error::new(502, "Service status is taking too long. Retry in a moment."))?
54}
55
56fn live_alloc(alloc: &Value) -> bool {
57 alloc["DesiredStatus"] == "run"
58 && matches!(string(&alloc["ClientStatus"]), "pending" | "running")
59}
60pub fn health(job: &Value, allocs: &[Value], checks: &[Value]) -> &'static str {
61 if job["Stop"] == true {
62 return "stopped";
63 }
64 let running: Vec<_> = allocs.iter().filter(|a| live_alloc(a)).collect();
65 if running.is_empty() {
66 return "down";
67 }
68 if running
69 .iter()
70 .any(|a| a["JobVersion"] != running[0]["JobVersion"])
71 {
72 return "deploying";
73 }
74 if running.iter().any(|a| a["ClientStatus"] == "pending") {
75 return "starting";
76 }
77 if running.iter().any(|a| {
78 a["TaskStates"]
79 .as_object()
80 .into_iter()
81 .flat_map(|s| s.values())
82 .any(|s| s["State"] == "pending")
83 }) {
84 return "restarting";
85 }
86 if checks.iter().any(|c| c["Status"] == "failure")
87 || running
88 .iter()
89 .any(|a| a["DeploymentStatus"]["Healthy"] == false)
90 {
91 return "degraded";
92 }
93 "healthy"
94}
95
96pub async fn scan(app: Arc<App>) -> Result<Arc<Document>> {
97 let state = app.clone();
98 app.cache
99 .get(
100 "nomad-scan".into(),
101 Duration::from_secs(10),
102 move || async move {
103 let (stubs, allocations, nodes) = tokio::try_join!(
104 nomad(&state, "/v1/jobs"),
105 nomad(&state, "/v1/allocations"),
106 nomad(&state, "/v1/nodes?resources=true")
107 )?;
108 let cpu = &nodes[0]["NodeResources"]["Cpu"];
109 let mhz = number(&cpu["CpuShares"]) / number(&cpu["TotalCpuCores"]);
110 if !stubs.is_array() || !allocations.is_array() || !mhz.is_finite() || mhz <= 0.0 {
111 return Err(Error::new(
112 502,
113 "Nomad answered without its jobs, allocations or node.",
114 ));
115 }
116 let jobs = stream::iter(array(&stubs).iter().cloned())
117 .map(|stub| {
118 let state = state.clone();
119 async move {
120 let id = string(&stub["ID"]).to_owned();
121 let cached = state
122 .specs
123 .lock()
124 .unwrap()
125 .get(&id)
126 .filter(|job| job["JobModifyIndex"] == stub["JobModifyIndex"])
127 .cloned();
128 let job = match cached {
129 Some(job) => job,
130 None => nomad(&state, &format!("/v1/job/{}", encoded(&id))).await?,
131 };
132 state.specs.lock().unwrap().insert(id.clone(), job.clone());
133 Ok::<_, Error>((id, job))
134 }
135 })
136 .buffer_unordered(4)
137 .collect::<Vec<_>>()
138 .await
139 .into_iter()
140 .collect::<Result<HashMap<_, _>>>()?;
141 state
142 .specs
143 .lock()
144 .unwrap()
145 .retain(|id, _| jobs.contains_key(id));
146 let mut allocs = array(&allocations).to_vec();
147 allocs
148 .sort_by(|a, b| number(&b["CreateTime"]).total_cmp(&number(&a["CreateTime"])));
149 let checked = stream::iter(allocs)
150 .map(|alloc| {
151 let state = state.clone();
152 async move {
153 let checks = if live_alloc(&alloc) {
154 nomad(
155 &state,
156 &format!(
157 "/v1/client/allocation/{}/checks",
158 string(&alloc["ID"])
159 ),
160 )
161 .await?
162 .as_object()
163 .map(|v| v.values().cloned().collect::<Vec<_>>())
164 .unwrap_or_default()
165 } else {
166 Vec::new()
167 };
168 let mut alloc = alloc;
169 alloc["home_checks"] = json!(checks);
170 Ok::<_, Error>(alloc)
171 }
172 })
173 .buffered(4)
174 .collect::<Vec<_>>()
175 .await
176 .into_iter()
177 .collect::<Result<Vec<_>>>()?;
178 let mut found = serde_json::Map::new();
179 for (id, job) in jobs {
180 let mine: Vec<_> = checked.iter().filter(|a| a["JobID"] == id).collect();
181 let allocs: Vec<_> = mine.iter().map(|a| (*a).clone()).collect();
182 let checks: Vec<_> = mine
183 .iter()
184 .flat_map(|a| array(&a["home_checks"]).iter().cloned())
185 .collect();
186 let health = health(&job, &allocs, &checks);
187 found.insert(
188 id,
189 json!({"job":job,"allocs":allocs,"mhz":mhz,"health":health}),
190 );
191 }
192 Ok(Value::Object(found))
193 },
194 )
195 .await
196}
197
198pub async fn managed() -> Result<Vec<String>> {
199 Ok(host::call(json!({"operation":"deploy.managed"}))
200 .await?
201 .as_array()
202 .unwrap()
203 .iter()
204 .map(|id| string(id).to_owned())
205 .collect())
206}
207pub async fn read_json(file: &std::path::Path, missing: Value) -> Result<Value> {
208 match tokio::fs::read(file).await {
209 Ok(bytes) => Ok(serde_json::from_slice(&bytes)?),
210 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(missing),
211 Err(e) => Err(e.into()),
212 }
213}
214pub async fn write_json(file: &std::path::Path, value: &Value) -> Result<()> {
215 tokio::fs::create_dir_all(file.parent().unwrap()).await?;
216 let temporary = file.with_extension(format!("{}.tmp", rand::random::<u64>()));
217 tokio::fs::write(&temporary, serde_json::to_vec(value)?).await?;
218 tokio::fs::rename(&temporary, file).await?;
219 Ok(())
220}
221pub async fn service_file(app: &App, id: &str) -> Result<PathBuf> {
222 if !valid_id(id) {
223 return Err(Error::new(400, "Invalid service ID."));
224 }
225 let direct = app.repo.join("service").join(id).join("service.pkl");
226 if tokio::fs::try_exists(&direct).await? {
227 return Ok(direct);
228 }
229 let mut dirs = tokio::fs::read_dir(app.repo.join("service")).await?;
230 while let Some(dir) = dirs.next_entry().await? {
231 let file = dir.path().join(format!("{id}.pkl"));
232 if tokio::fs::try_exists(&file).await? {
233 return Ok(file);
234 }
235 }
236 Ok(direct)
237}
238pub fn valid_id(id: &str) -> bool {
239 !id.is_empty()
240 && id.as_bytes()[0].is_ascii_lowercase()
241 && id
242 .bytes()
243 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
244}
245
246async fn icons(app: Arc<App>, id: String) -> Result<Arc<Document>> {
247 let state = app.clone();
248 app.cache.get(format!("icon:{id}"),Duration::from_secs(30),move || async move {
249 let file = service_file(&state,&id).await?;
250 let dir = if file.file_name().unwrap() == "service.pkl" { file.parent().unwrap().to_path_buf() } else { file.parent().unwrap().join(&id) };
251 let mut found = serde_json::Map::new();
252 for file in ["icon.svg","icon-light.svg","icon-dark.svg"] {
253 let candidate = dir.join(file);
254 let fallback = PathBuf::from(env("SNOWGLOBE_ICON_DIR","server/icons")).join(&id).join(file);
255 let path = if tokio::fs::try_exists(&candidate).await? { candidate } else { fallback };
256 if let Ok(bytes) = tokio::fs::read(&path).await {
257 found.insert(file.to_owned(),json!({"path":path,"hash":format!("{:x}",Sha1::digest(&bytes))[..12].to_owned()}));
258 }
259 }
260 Ok(Value::Object(found))
261 }).await
262}
263async fn icon_of(app: Arc<App>, id: &str) -> Value {
264 let Ok(found) = icons(app, id.into()).await else {
265 return Value::Null;
266 };
267 let url = |mode| {
268 let file = if !found.value[mode].is_null() {
269 mode
270 } else {
271 "icon.svg"
272 };
273 found.value[file]["hash"]
274 .as_str()
275 .map(|hash| format!("/api/icons/{id}/{file}?v={hash}"))
276 };
277 match (url("icon-light.svg"), url("icon-dark.svg")) {
278 (Some(light), Some(dark)) => json!({"light":light,"dark":dark}),
279 _ => Value::Null,
280 }
281}
282pub async fn icon(
283 app: &Arc<App>,
284 parts: &[&str],
285 query: &HashMap<String, String>,
286) -> Result<Response> {
287 if parts.len() != 3
288 || !valid_id(parts[1])
289 || !["icon.svg", "icon-light.svg", "icon-dark.svg"].contains(&parts[2])
290 {
291 return Err(Error::new(404, "Not Found"));
292 }
293 let found = icons(app.clone(), parts[1].into()).await?;
294 let path = found.value[parts[2]]["path"]
295 .as_str()
296 .ok_or_else(|| Error::new(404, "Not Found"))?;
297 Ok((
298 [
299 ("content-type", "image/svg+xml"),
300 (
301 "cache-control",
302 if query.contains_key("v") {
303 "public, max-age=31536000, immutable"
304 } else {
305 "no-cache"
306 },
307 ),
308 ],
309 tokio::fs::read(path).await?,
310 )
311 .into_response())
312}
313
314async fn summarize(app: Arc<App>, id: &str, found: &Value) -> Value {
315 let meta = &found["job"]["Meta"];
316 let tasks: Vec<_> = array(&found["job"]["TaskGroups"])
317 .iter()
318 .flat_map(|g| array(&g["Tasks"]))
319 .collect();
320 let usage = app
321 .usage
322 .lock()
323 .unwrap()
324 .get(id)
325 .cloned()
326 .unwrap_or(Value::Null);
327 let mhz = number(&found["mhz"]);
328 let cpu: f64 = tasks.iter().map(|t| number(&t["Resources"]["CPU"])).sum();
329 let memory: f64 = tasks
330 .iter()
331 .map(|t| {
332 let max = number(&t["Resources"]["MemoryMaxMB"]);
333 if max > 0.0 {
334 max
335 } else {
336 number(&t["Resources"]["MemoryMB"])
337 }
338 })
339 .sum();
340 json!({"id":id,"name":meta["snowglobe_name"].as_str().unwrap_or(id),"health":found["health"].as_str().unwrap_or("down"),"url":meta["snowglobe_hostname"].as_str().filter(|h| !h.is_empty() && meta["snowglobe_launcher"] != "false").map(|h| format!("https://{h}")),"icon":icon_of(app.clone(),id).await,"access":meta["snowglobe_auth_role"],"tagline":meta["snowglobe_tagline"],"cpu":usage["cpu"],"cpuLimit":if mhz > 0.0 { cpu/mhz } else {0.0},"memory":usage["memory"],"memoryLimit":memory*1048576.0})
341}
342async fn summaries(app: Arc<App>) -> Result<Arc<Document>> {
343 let state = app.clone();
344 app.cache
345 .get(
346 "services".into(),
347 Duration::from_secs(2),
348 move || async move {
349 let (ids, jobs) = tokio::try_join!(managed(), scan(state.clone()))?;
350 let values = stream::iter(ids)
351 .map(|id| {
352 let state = state.clone();
353 let jobs = jobs.clone();
354 async move { summarize(state, &id, &jobs.value[&id]).await }
355 })
356 .buffered(4)
357 .collect::<Vec<_>>()
358 .await;
359 Ok(json!(values))
360 },
361 )
362 .await
363}
364
365pub fn seconds(value: &Value) -> Option<f64> {
366 chrono::DateTime::parse_from_rfc3339(value.as_str()?)
367 .ok()
368 .map(|t| t.timestamp() as f64 + t.timestamp_subsec_nanos() as f64 / 1e9)
369 .filter(|s| *s > 0.0)
370}
371pub fn last_restart(task: &Value) -> Value {
372 let Some(t) = seconds(&task["LastRestart"]) else {
373 return Value::Null;
374 };
375 let events = array(&task["Events"]);
376 let end = events
377 .iter()
378 .rposition(|e| e["Type"] == "Restarting")
379 .map(|i| i + 1)
380 .unwrap_or(0);
381 let reason = events[..end]
382 .iter()
383 .rev()
384 .find(|e| {
385 ["Terminated", "Restart Signaled", "Driver Failure", "Killed"]
386 .contains(&string(&e["Type"]))
387 })
388 .map(|e| string(&e["DisplayMessage"]))
389 .filter(|s| !s.is_empty())
390 .unwrap_or("restarted");
391 json!({"t":t,"reason":reason})
392}
393pub fn checks_of(checks: &Value) -> Value {
394 json!(array(checks).iter().map(|c| json!({"name":c["Check"],"passing":c["Status"] != "failure","output":c["Output"]})).collect::<Vec<_>>())
395}
396fn containers(found: &Value) -> Value {
397 let allocs = array(&found["allocs"]);
398 let alloc = allocs
399 .iter()
400 .find(|a| live_alloc(a))
401 .or_else(|| allocs.first())
402 .unwrap_or(&Value::Null);
403 let tasks: HashMap<_, _> = array(&found["job"]["TaskGroups"])
404 .iter()
405 .flat_map(|g| array(&g["Tasks"]))
406 .map(|t| (string(&t["Name"]), t))
407 .collect();
408 json!(alloc["TaskStates"].as_object().into_iter().flat_map(|s| s.iter()).map(|(name,state)| {
409 let spec = tasks.get(name.as_str()).copied().unwrap_or(&Value::Null);
410 json!({"name":name,"image":spec["Config"]["image"],"hook":spec["Lifecycle"]["Hook"],"state":state["State"],"restarts":state["Restarts"],"lastRestart":last_restart(state),"startedAt":seconds(&state["StartedAt"])})
411 }).collect::<Vec<_>>())
412}
413async fn issues(app: Arc<App>) -> Result<Value> {
414 let summaries = app
415 .cache
416 .peek("services", Duration::from_secs(2))
417 .ok_or_else(|| Error::new(503, "Service status is unavailable."))?;
418 let jobs = app
419 .cache
420 .peek("nomad-scan", Duration::from_secs(10))
421 .ok_or_else(|| Error::new(503, "Service status is unavailable."))?;
422 let mut issues = Vec::new();
423 for service in array(&summaries.value) {
424 let id = string(&service["id"]);
425 let health = string(&service["health"]);
426 let found = &jobs.value[id];
427 if matches!(health, "down" | "degraded") {
428 issues.push(json!({"service":{"id":id,"name":service["name"],"icon":service["icon"],"url":service["url"]},"health":health,"failing":array(&found["allocs"]).iter().flat_map(|a| array(&a["home_checks"])).find(|c| c["Status"] == "failure").map(|c| c["Output"].clone())}));
429 }
430 }
431 Ok(json!(issues))
432}
433
434async fn launcher_module(repo: &Path) -> Result<String> {
435 let mut files = Vec::new();
436 let mut directories = tokio::fs::read_dir(repo.join("service")).await?;
437 while let Some(directory) = directories.next_entry().await? {
438 if !directory.file_type().await?.is_dir() {
439 continue;
440 }
441 let mut entries = tokio::fs::read_dir(directory.path()).await?;
442 while let Some(entry) = entries.next_entry().await? {
443 let path = entry.path();
444 if !path.extension().is_some_and(|extension| extension == "pkl") {
445 continue;
446 }
447 files.push(path);
448 }
449 }
450 files.sort();
451 let mut module = String::new();
452 let mut services = Vec::new();
453 for (index, path) in files.iter().enumerate() {
454 let uri = serde_json::to_string(&url::Url::from_file_path(path).unwrap().as_str())?;
455 module.push_str(&format!("import {uri} as service{index}\n"));
456 services.push(format!("{uri}, service{index}"));
457 }
458 module.push_str(&format!("local services = Map({})\n", services.join(", ")));
459 module.push_str(r#"output { renderer = new JsonRenderer { omitNullProperties = false }
460value = services.filter((_, s) -> s.enabled && s.launcher).mapValues((_, s) ->
461let (tasks = s.containers?.toMap() ?? Map("app", s.container))
462let (routes = tasks.filter((_, task) -> task?.http?.hostname != null).mapValues((_, task) -> task.http))
463let (route = routes.values.firstOrNull)
464new Dynamic { name = s.name; tagline = s.tagline; hostname = route?.hostname; access = s.access ?? route?.authRole; urls = routes.mapValues((_, http) -> "https://\(http.hostname)") }) }"#);
465 Ok(module)
466}
467
468pub(crate) async fn launcher(app: Arc<App>) -> Result<Arc<Document>> {
469 let real = tokio::fs::canonicalize(&app.repo).await?;
470 let state = app.clone();
471 app.cache.get(format!("launcher:{}",real.display()),Duration::from_secs(365*86400),move || async move {
472 let module = launcher_module(&real).await?;
473 let value: Value = serde_json::from_slice(&command("pkl",&["eval","-"],Some(module.as_bytes())).await?)?;
474 let mut apps = Vec::new();
475 for (file, value) in value.as_object().into_iter().flat_map(|v| v.iter()) {
476 if let Some(host) = value["hostname"].as_str() {
477 let path = std::path::Path::new(file);
478 let id = if path.file_name().unwrap() == "service.pkl" { path.parent().unwrap().file_name().unwrap() } else { path.file_stem().unwrap() }.to_string_lossy();
479 apps.push(json!({"id":id,"name":value["name"],"tagline":value["tagline"].as_str().filter(|s| !s.is_empty()),"url":format!("https://{host}"),"urls":value["urls"],"access":value["access"],"icon":icon_of(state.clone(),&id).await}));
480 }
481 }
482 Ok(json!(apps))
483 }).await
484}
485
486async fn service(app: Arc<App>, id: &str) -> Result<Arc<Document>> {
487 let state = app.clone();
488 let id = id.to_owned();
489 app.cache.get(format!("service:{id}"), Duration::from_secs(2), move || async move {
490 let app = state;
491 let id = id.as_str();
492 let ids = managed().await?;
493 let jobs = scan(app.clone()).await?;
494 let found = &jobs.value[id];
495 if !ids.iter().any(|s| s == id) || found.is_null() {
496 return Err(Error::new(
497 404,
498 format!("No service is named {id}. Pick one from the sidebar."),
499 ));
500 }
501 let all = summaries(app.clone()).await?;
502 let link = |other: &str, kind: &Value| {
503 array(&all.value)
504 .iter()
505 .find(|s| s["id"] == other)
506 .map(|s| json!({"id":other,"name":s["name"],"kind":kind,"health":s["health"]}))
507 };
508 let needs = |other: &str| {
509 serde_json::from_str::<Value>(string(&jobs.value[other]["job"]["Meta"]["snowglobe_requires"]))
510 .ok()
511 };
512 let requirements = needs(id).map(|n| {
513 n.as_object()
514 .into_iter()
515 .flat_map(|v| v.iter())
516 .filter_map(|(id, kind)| link(id, kind))
517 .collect::<Vec<_>>()
518 });
519 let dependents = if ids.iter().all(|id| needs(id).is_some()) {
520 Some(
521 ids.iter()
522 .filter_map(|other| {
523 needs(other).and_then(|n| n.get(id).and_then(|kind| link(other, kind)))
524 })
525 .collect::<Vec<_>>(),
526 )
527 } else {
528 None
529 };
530 let mut summary = summarize(app.clone(), id, found).await;
531 let meta = &found["job"]["Meta"];
532 let application_traces = if let Some(name) = meta["snowglobe_trace_service"]
533 .as_str()
534 .filter(|s| !s.is_empty())
535 {
536 telemetry::has_traces(app.clone(), name)
537 .await
538 .unwrap_or(false)
539 } else {
540 false
541 };
542 let metrics =
543 if array(&found["job"]["TaskGroups"]).iter().flat_map(|g| array(&g["Services"])).any(|s| array(&s["Tags"]).iter().any(|tag| string(tag).starts_with("snowglobe-metrics-path="))) || meta["snowglobe_metrics_pushed"] == "true" {
544 telemetry::metric_names(app.clone(), id)
545 .await
546 .unwrap_or(json!([]))
547 } else {
548 json!([])
549 };
550 let release = host::call(json!({"operation":"deploy.current"})).await?;
551 let secrets = serde_json::from_str::<Value>(string(&meta["snowglobe_secrets"]))
552 .ok()
553 .map(|v| {
554 array(&v)
555 .iter()
556 .map(|s| json!({"name":s["name"],"generated":s["generated"]}))
557 .collect::<Vec<_>>()
558 });
559 let ack = read_json(&app.data.join("restarts-acknowledged.json"), json!({})).await?;
560 let datasets = app.cache.get("zfs-datasets".into(),Duration::from_secs(30),move || async move {
561 let rows = host::call(json!({"operation":"storage.datasets"})).await?;
562 Ok(json!(array(&rows).iter().map(|d| json!({"name":d["name"],"mountpoint":d["mountpoint"],"used":d["usedbydataset"],"snapshots":d["usedbysnapshots"]})).collect::<Vec<_>>()))
563 }).await?;
564 let logs = array(&all.value).iter().find(|s| s["id"] == "victoria" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))}));
565 let fields = json!({"applicationTraces":application_traces,"applicationMetrics":metrics,"release":release,"deployedAt":number(&found["job"]["SubmitTime"])/1e9,"rollout":if array(&found["job"]["TaskGroups"]).iter().any(|g| number(&g["Update"]["Canary"]) > 0.0) {"overlapped"} else {"simple"},"containers":containers(found),"checks":checks_of(&json!(array(&found["allocs"]).iter().flat_map(|a| array(&a["home_checks"])).collect::<Vec<_>>())),"requirements":requirements,"dependents":dependents,"secrets":secrets,"datasets":array(&datasets.value).iter().filter(|d| string(&d["name"]).ends_with(&format!("/prod/{id}"))).collect::<Vec<_>>(),"restartsAcknowledged":ack[id],"logs":logs});
566 summary
567 .as_object_mut()
568 .unwrap()
569 .extend(fields.as_object().unwrap().clone());
570 Ok(summary)
571 }).await
572}
573
574async fn change(app: Arc<App>, request: Value) -> Result<()> {
575 let run = host::call(request).await?;
576 app.cache.invalidate("deploys");
577 let outcome = tokio::time::timeout(Duration::from_secs(120), async {
578 loop {
579 let status = host::call(json!({"operation":"deploy.run","id":run["id"]})).await?;
580 if let Some(code) = status["code"].as_i64() {
581 return if code == 0 {
582 Ok(())
583 } else {
584 Err(Error::new(
585 502,
586 "The change failed. Check its run in deploys before retrying.",
587 ))
588 };
589 }
590 tokio::time::sleep(Duration::from_millis(500)).await;
591 }
592 })
593 .await
594 .map_err(|_| {
595 Error::new(
596 504,
597 "The change is still running. Check its run in deploys before retrying.",
598 )
599 });
600 app.cache.invalidate("deploys");
601 app.cache.invalidate("nomad-scan");
602 app.cache.invalidate("services");
603 app.cache
604 .invalidate(&format!("service:{}", string(&run["target"])));
605 outcome??;
606 Ok(())
607}
608
609pub async fn route(
610 app: Arc<App>,
611 method: &Method,
612 parts: &[&str],
613 query: &HashMap<String, String>,
614 me: &Value,
615 body: Value,
616) -> Result<Response> {
617 let empty = || StatusCode::NO_CONTENT.into_response();
618 let value = match parts {
619 ["me"] if method == Method::GET => me.clone(),
620 ["host"] if method == Method::GET => {
621 let sample = host::sample(app).await?;
622 json!({"cores":sample.value["cores"],"memory":sample.value["memory"]["total"],"bootedAt":sample.value["bootedAt"],"outages":null})
623 }
624 ["services"] if method == Method::GET => return Ok(summaries(app).await?.response()),
625 ["vps"] if method == Method::GET => telemetry::vps(app).await?,
626 ["launcher"] if method == Method::GET => {
627 let found = launcher(app.clone()).await?;
628 let jobs = app.cache.peek("nomad-scan", Duration::from_secs(10));
629 let groups: Vec<_> = array(&me["groups"]).iter().map(string).collect();
630 let mut apps = Vec::new();
631 for value in array(&found.value)
632 .iter()
633 .filter(|v| can_open(&groups, v["access"].as_str()))
634 {
635 let mut value = value.clone();
636 value["health"] = jobs
637 .as_ref()
638 .map(|j| j.value[string(&value["id"])]["health"].clone())
639 .unwrap_or(Value::Null);
640 value.as_object_mut().unwrap().remove("access");
641 apps.push(value);
642 }
643 json!(apps)
644 }
645 ["status"] if method == Method::GET => {
646 let issues = issues(app.clone()).await.ok();
647 let apps = launcher(app.clone()).await?;
648 let groups: Vec<_> = array(&me["groups"]).iter().map(string).collect();
649 let admin = array(&me["sections"]).iter().any(|s| s == "admin");
650 let issues = issues.map(|v| {
651 array(&v)
652 .iter()
653 .filter(|i| {
654 admin
655 || array(&apps.value).iter().any(|a| {
656 a["id"] == i["service"]["id"]
657 && can_open(&groups, a["access"].as_str())
658 })
659 })
660 .map(|issue| {
661 let mut issue = issue.clone();
662 if !admin {
663 issue["failing"] = Value::Null;
664 }
665 issue
666 })
667 .collect::<Vec<_>>()
668 });
669 let sample = host::sample(app).await?;
670 json!({"load":(number(&sample.value["load"])/number(&sample.value["cores"])*100.0).min(100.0),"issues":issues})
671 }
672 ["services", id] if method == Method::GET => return Ok(service(app, id).await?.response()),
673 ["services", id, "definition"] if method == Method::GET => definition(app, id).await?,
674 ["services", id, "restarts", "acknowledge"] if method == Method::POST => {
675 let file = app.data.join("restarts-acknowledged.json");
676 let mut value = read_json(&file, json!({})).await?;
677 value[*id] = json!(now());
678 write_json(&file, &value).await?;
679 app.cache.invalidate(&format!("service:{id}"));
680 return Ok(empty());
681 }
682 ["services", id, action] if method == Method::POST => {
683 if !["start", "stop", "restart"].contains(action) {
684 return Err(Error::new(400, "Choose start, stop, or restart."));
685 }
686 change(
687 app,
688 json!({"operation":"deploy.start","action":action,"target":id}),
689 )
690 .await?;
691 return Ok(empty());
692 }
693 ["services", id, "secrets", name] if method == Method::GET => {
694 json!({"value":host::call(json!({"operation":"deploy.secret.get","service":id,"key":name})).await?})
695 }
696 ["services", id, "secrets", name] | ["services", id, "secrets", name, "rotate"]
697 if (parts.len() == 4 && method == Method::PUT)
698 || (parts.len() == 5 && method == Method::POST) =>
699 {
700 let mut request = json!({"operation":if parts.len()==5 {"deploy.secret.rotate"} else {"deploy.secret.set"},"service":id,"key":name});
701 if parts.len() == 4 {
702 request["value"] = body["value"].clone();
703 }
704 change(app, request).await?;
705 return Ok(empty());
706 }
707 ["metrics", metric] if method == Method::GET => {
708 if metric.starts_with("vm.") {
709 need(me, "vms")?;
710 let owners = host::call(json!({"operation":"vm.domains"})).await?;
711 let mut series = telemetry::metrics(app, metric, query, None, None)
712 .await?
713 .value
714 .clone();
715 series.as_array_mut().unwrap().retain(|series| {
716 array(&owners).iter().any(|domain| {
717 domain["name"] == series["name"] && vms::visible(me, domain)
718 })
719 });
720 return Ok(Document::new(series).response());
721 }
722 return Ok(telemetry::metrics(app, metric, query, None, None)
723 .await?
724 .response());
725 }
726 ["services", id, "metrics", name] if method == Method::GET => {
727 let names = if id.starts_with("vps.") {
728 telemetry::metric_names(app.clone(), id).await?
729 } else {
730 service(app.clone(), id).await?.value["applicationMetrics"].clone()
731 };
732 if !array(&names).iter().any(|v| v == *name) {
733 return Err(Error::new(404, "Metric unavailable for this service."));
734 }
735 return Ok(telemetry::metrics(app, name, query, Some(id), None)
736 .await?
737 .response());
738 }
739 ["services", id, "logs"] if method == Method::GET => {
740 telemetry::logs(app, id, query).await?
741 }
742 ["services", id, "traces"] if method == Method::GET => {
743 telemetry::traces(app, id, query, |_| true).await?
744 }
745 ["traces", id] if method == Method::GET => telemetry::trace(app, id).await?,
746 _ => return Err(Error::new(404, "Not Found")),
747 };
748 Ok(Document::new(value).response())
749}
750
751pub fn start(app: Arc<App>) {
752 let state = app.clone();
753 tokio::spawn(async move {
754 loop {
755 if let Err(e) = summaries(state.clone()).await {
756 eprintln!("status: {}", e.message);
757 }
758 tokio::time::sleep(Duration::from_secs(2)).await;
759 }
760 });
761 tokio::spawn(async move {
762 let mut previous = (0.0, 0.0);
763 let mut interval = tokio::time::interval(Duration::from_secs(2));
764 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
765 loop {
766 interval.tick().await;
767 let sample = async {
768 let sample = host::sample(app.clone()).await?;
769 let sample = &sample.value;
770 let total = number(&sample["cpu"]["total"]); let busy = number(&sample["cpu"]["busy"]);
771 let cpu = if previous.1 > 0.0 && total > previous.1 { ((busy-previous.0)/(total-previous.1)*100.0).clamp(0.0,100.0) } else {0.0}; previous = (busy,total);
772 let jobs = app.cache.peek("services",Duration::from_secs(2));
773 let services: serde_json::Map<_,_> = jobs.as_ref().into_iter().flat_map(|j| array(&j.value)).map(|s| (string(&s["id"]).to_owned(),json!({"cpu":s["cpu"],"memory":s["memory"],"health":s["health"]}))).collect();
774 Ok::<_,Error>(Document::new(json!({"t":now(),"host":{"cpu":cpu,"memory":sample["memory"]["used"],"arc":sample["arc"],"temperature":sample["temperature"]},"services":services,"ups":null})))
775 }.await;
776 match sample {
777 Ok(value) => {
778 app.live.send_replace(value.bytes);
779 }
780 Err(e) => eprintln!("host sample: {}", e.message),
781 }
782 }
783 });
784}
785
786async fn definition(app: Arc<App>, id: &str) -> Result<Value> {
787 if !managed().await?.iter().any(|s| s == id) {
788 return Err(Error::new(
789 404,
790 format!("No service is named {id}. Pick one from the sidebar."),
791 ));
792 }
793 let jobs = scan(app.clone()).await?;
794 let found = &jobs.value[id];
795 if found.is_null() {
796 return Err(Error::new(
797 404,
798 format!("No service is named {id}. Pick one from the sidebar."),
799 ));
800 }
801 let mhz = number(&found["mhz"]);
802 let groups=array(&found["job"]["TaskGroups"]).iter().map(|group| {
803 let tags=|service: &Value,key: &str| array(&service["Tags"]).iter().filter_map(|s| string(s).strip_prefix(&format!("{key}=")).map(str::to_owned)).collect::<Vec<_>>();
804 let services=array(&group["Services"]).iter().map(|service| {
805 let check=&service["Checks"][0];let restart=&check["CheckRestart"];
806 json!({"name":service["Name"],"port":service["PortLabel"],"hostnames":tags(service,"caddy-host"),"authRole":tags(service,"caddy-auth-role").first(),"check":if check.is_null() {Value::Null} else {json!({"task":check["TaskName"],"type":check["Type"],"path":check["Path"],"interval":number(&check["Interval"])/1e9,"timeout":number(&check["Timeout"])/1e9,"restartAfter":if number(&restart["Limit"])>0.0 {json!({"failures":restart["Limit"],"grace":number(&restart["Grace"])/1e9})} else {Value::Null}})}})
807 }).collect::<Vec<_>>();
808 let tasks=array(&group["Tasks"]).iter().map(|task| {
809 let mut ports=HashMap::new();for network in array(&group["Networks"]) {for (key,dynamic) in [("DynamicPorts",true),("ReservedPorts",false)] {for port in array(&network[key]) {ports.insert(string(&port["Label"]),(port,dynamic));}}}
810 let config=&task["Config"];let mut env:Vec<Value>=task["Env"].as_object().into_iter().flat_map(|v| v.keys()).map(|name| json!({"name":name,"from":"job"})).collect();
811 let assignments=regex::Regex::new(r"(?m)(?:^|\}\})([A-Za-z_]\w*)=").unwrap();
812 for template in array(&task["Templates"]).iter().filter(|t| t["Envvars"]==true) {let text=string(&template["EmbeddedTmpl"]);for capture in assignments.captures_iter(text) {env.push(json!({"name":&capture[1],"from":if text.contains("nomadVar") {"secret"} else {"template"}}));}}
813 json!({"name":task["Name"],"hook":task["Lifecycle"]["Hook"],"sidecar":task["Lifecycle"]["Sidecar"].as_bool().unwrap_or(false),"image":config["image"],"user":task["User"].as_str().filter(|s| !s.is_empty()),"cpu":number(&task["Resources"]["CPU"])/mhz,"memory":number(&task["Resources"]["MemoryMB"])*1048576.0,"memoryMax":if number(&task["Resources"]["MemoryMaxMB"])>0.0 {json!(number(&task["Resources"]["MemoryMaxMB"])*1048576.0)} else {Value::Null},"ports":array(&config["ports"]).iter().map(|label| {let port=ports.get(string(label));json!({"label":label,"container":port.and_then(|(p,_)| p["To"].as_u64()).filter(|n| *n>0),"host":port.filter(|(_,dynamic)| !dynamic).map(|(p,_)| p["Value"].clone()),"network":port.map(|(p,_)| p["HostNetwork"].as_str().unwrap_or("default")).unwrap_or("default")})}).collect::<Vec<_>>(),"mounts":array(&config["volumes"]).iter().map(|volume| {let fields:Vec<_>=string(volume).split(':').collect();json!({"source":fields[0],"target":fields.get(1).unwrap_or(&fields[0]),"readOnly":fields.get(2).is_some_and(|s| s.split(',').any(|s| s=="ro"))})}).collect::<Vec<_>>(),"tmpfs":array(&config["tmpfs"]),"devices":array(&config["devices"]),"capabilities":array(&config["cap_add"]),"hostNetwork":config["network_mode"]=="host","extraHosts":array(&config["extra_hosts"]),"env":env})
814 }).collect::<Vec<_>>();
815 let restart=&group["RestartPolicy"];
816 json!({"name":group["Name"],"count":group["Count"],"restart":{"attempts":restart["Attempts"],"interval":number(&restart["Interval"])/1e9,"delay":number(&restart["Delay"])/1e9,"fail":restart["Mode"]=="fail"},"services":services,"tasks":tasks})
817 }).collect::<Vec<_>>();
818 let source = tokio::fs::read_to_string(service_file(&app, id).await?)
819 .await
820 .ok();
821 Ok(json!({"groups":groups,"source":source}))
822}
823
824#[cfg(test)]
825mod tests {
826 use super::*;
827 #[tokio::test]
828 async fn launcher_discovers_direct_and_grouped_definitions() {
829 let root = std::env::temp_dir().join(format!("snowglobe-launcher-{}", uuid::Uuid::new_v4()));
830 tokio::fs::create_dir_all(root.join("service/personal"))
831 .await
832 .unwrap();
833 tokio::fs::write(root.join("service/README.md"), "not a service")
834 .await
835 .unwrap();
836 for file in ["service.pkl", "fixture.pkl", "icon.svg"] {
837 tokio::fs::write(root.join("service/personal").join(file), "unread fixture")
838 .await
839 .unwrap();
840 }
841 let module = launcher_module(&root).await.unwrap();
842 assert!(module.contains("/personal/service.pkl"));
843 assert!(module.contains("/personal/fixture.pkl"));
844 assert!(!module.contains("icon.svg"));
845 assert_eq!(
846 module
847 .lines()
848 .filter(|line| line.starts_with("import "))
849 .count(),
850 2,
851 );
852 tokio::fs::remove_dir_all(root).await.unwrap();
853 }
854 #[test]
855 fn health_precedence() {
856 let job = json!({"Stop":false});
857 let alloc = json!({"DesiredStatus":"run","ClientStatus":"running","JobVersion":1,"TaskStates":{"app":{"State":"running"}}});
858 assert_eq!(health(&job, &[], &[]), "down");
859 assert_eq!(health(&job, &[alloc.clone()], &[]), "healthy");
860 assert_eq!(
861 health(&job, &[alloc.clone()], &[json!({"Status":"failure"})]),
862 "degraded"
863 );
864 let mut pending = alloc.clone();
865 pending["TaskStates"]["app"]["State"] = json!("pending");
866 assert_eq!(
867 health(&job, &[pending], &[json!({"Status":"failure"})]),
868 "restarting"
869 );
870 let mut newer = alloc.clone();
871 newer["JobVersion"] = json!(2);
872 assert_eq!(health(&job, &[alloc, newer], &[]), "deploying");
873 assert_eq!(health(&json!({"Stop":true}), &[], &[]), "stopped");
874 }
875}