diff --git a/AGENTS.md b/AGENTS.md index 1d27d77b502d3c5c45c7b663f7310179a22d8cec..6af8a81b108f652b948b67eb8e8e81a3c429d6ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,15 @@ availability and existing data as part of correctness. ## Access and authorization -- Local edits and rehearsal VM changes are always implied by a task. Carry - them through staging and verification without asking for permission again. +- Local edits and staging previews on Zenith are always implied by a task. + Carry them through verification without asking for permission again. Use + disposable preview data for writes and keep active production services intact. - Production changes require a request to take the work to prod. A request to fix, stage, test, or commit something does not authorize production deployment. Once prod is requested, complete the normal deployment workflow. - Read-only infrastructure inspection over SSH is allowed for the task. Confirm - the target host before a remote mutation; deployment defaults point at the - rehearsal VM, but inherited environment variables can select production. + the target host before a remote mutation. Set both deployment host and port + explicitly; the CLI's legacy defaults still point at the rehearsal VM. - Private file contents require authorization per source. Infrastructure access, a mounted share, or permission to list a tree does not authorize opening its contents. Keep private material and derived model output inside its store. @@ -35,11 +36,13 @@ availability and existing data as part of correctness. | Surface | Connection / role | Configuration | | --- | --- | --- | | Mac checkout | `/Users/clo/dev/infra-2`; edit and upload here | This working copy | -| Rehearsal NixOS VM | `root@127.0.0.1`, SSH port `2222` | `nixos/vm.nix`, `studio.test`, demo pool | +| Optional rehearsal NixOS VM | `root@127.0.0.1`, SSH port `2222` | `nixos/vm.nix`, `studio.test`, demo pool | | Zenith production | `root@10.0.0.1`, SSH port `22` | `nixos/zenith.nix`, `paperclover.net`, physical pool | `tools/deploy.py` reads `STUDIO_DEPLOY_HOST` and `STUDIO_DEPLOY_PORT`; defaults -are the rehearsal connection above. Set both explicitly for production. +are the rehearsal connection above. Set both explicitly for Zenith staging +and production. Prefer staging previews on Zenith for service and dashboard +verification; use the VM when host or guest isolation is required. Host settings and dataset names have their canonical home in `nixos/zenith.nix` and `config/site.pkl`; inspect them instead of copying values into new tooling. @@ -99,7 +102,7 @@ Mac rehearsal routing. Do not hand-edit DNS, trust, or host files to bypass them 3. Run the relevant local checks, then stage the working files: ```sh - STUDIO_DEPLOY_HOST=root@127.0.0.1 STUDIO_DEPLOY_PORT=2222 python3 tools/deploy.py stage + STUDIO_DEPLOY_HOST=root@10.0.0.1 STUDIO_DEPLOY_PORT=22 python3 tools/deploy.py stage ``` The command prints `stage`, `release`, `host`, and `dataset`. Use that host; @@ -135,7 +138,8 @@ read `tools/data-restore.md` before proposing it. - Run checks that cover the change. Inspect a test's entry point first: some `tools/*-test.py` scripts mutate host services or require Linux/real guests. - Run those in rehearsal, not on production by default. + Run those against isolated staging fixtures. Tests that alter host services, + storage, or guests require a disposable VM or explicit production scope. - Dashboard: `cd dashboard && npm run check && npm run build`; run Rust tests with `cargo test --manifest-path dashboard/Cargo.toml`. Host tests include `python3 tools/dashboard-host-test.py` and `python3 tools/dashboard-vms-test.py`. @@ -169,6 +173,7 @@ read `tools/data-restore.md` before proposing it. [tools/personal-cutover.md](tools/personal-cutover.md), and the service's `tools/*-migration.md` runbook. - Provider behavior: [tools/service-dependencies.md](tools/service-dependencies.md). +- Copyparty administration and folder moves: [tools/copyparty.md](tools/copyparty.md). - Local inference: [tools/local-ai.md](tools/local-ai.md). Runbooks contain dated rehearsal evidence and migration history. Check current diff --git a/dashboard/src/auth.rs b/dashboard/src/auth.rs index ec17f4e6d5a0fbdcc29e7d994f6908e682bd17f4..5f853215fdb6a896fe5ce255b9ec4f492fad20da 100644 --- a/dashboard/src/auth.rs +++ b/dashboard/src/auth.rs @@ -571,6 +571,23 @@ impl Store { } } +fn file_groups(user: &Value) -> Option { + if user["enabled"] != true + || !array(&user["requiredActions"]).is_empty() + || guest::is_guest(user) + || user["kind"] == "ai" + { + return None; + } + let id = uuid::Uuid::parse_str(string(&user["id"])).ok()?; + let mut groups = array(&user["groups"]) + .iter() + .map(|g| string(&g["name"]).to_owned()) + .collect::>(); + groups.push(format!("snow-user-{id}")); + Some(groups.join(",")) +} + pub async fn route(State(app): State>, request: Request) -> Result { if request.uri().path() == "/auth/shale/page" { return shale_page::proxy(app, request).await; @@ -591,14 +608,9 @@ pub async fn route(State(app): State>, request: Request) -> Result>() - .join(","); + }; return Ok(( StatusCode::NO_CONTENT, [ @@ -1284,3 +1296,23 @@ fn passkeys(db: &Connection, id: &str) -> Result> { }) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn folder_identity_tracks_the_account_id_and_requires_an_enabled_regular_account() { + let mut account = json!({"id":"12345678-1234-1234-1234-123456789abc","username":"snow","enabled":true,"requiredActions":[],"groups":[{"name":"media"}]}); + assert_eq!(file_groups(&account).unwrap(), "media,snow-user-12345678-1234-1234-1234-123456789abc"); + account["username"] = json!("different"); + assert_eq!(file_groups(&account).unwrap(), "media,snow-user-12345678-1234-1234-1234-123456789abc"); + account["id"] = json!("12345678-1234-1234-1234-123456789abd"); + assert!(file_groups(&account).unwrap().ends_with("snow-user-12345678-1234-1234-1234-123456789abd")); + for (field, value) in [("enabled", json!(false)), ("kind", json!("guest")), ("kind", json!("ai")), ("requiredActions", json!(["UPDATE_PASSWORD"])), ("id", json!("x,infra-admin"))] { + let mut blocked = account.clone(); + blocked[field] = value; + assert!(file_groups(&blocked).is_none()); + } + } +} diff --git a/dashboard/src/host.rs b/dashboard/src/host.rs index 2dca2204abcca8d6365d8644db0ad282a147ccd0..1e4b976c16dbd276eecc6766140114ddfca98564 100644 --- a/dashboard/src/host.rs +++ b/dashboard/src/host.rs @@ -34,6 +34,8 @@ pub async fn open(request: Value) -> Result<(Value, tokio::net::UnixStream)> { Some("vm.create" | "vm.media" | "vm.preset") ) { 910 + } else if request["operation"] == "copyparty.manage" { + 310 } else { 70 }; diff --git a/dashboard/src/main.rs b/dashboard/src/main.rs index d7ee80ea169098f0a93b4cc7723b3d60d21a07f9..4484c661cd89f0ee77be1190da2dd85c135467d7 100644 --- a/dashboard/src/main.rs +++ b/dashboard/src/main.rs @@ -15,6 +15,7 @@ mod oidc; mod relay; mod shale; mod shale_page; +mod shares; mod storage; mod telemetry; mod users; @@ -269,7 +270,7 @@ async fn api(State(app): State>, request: Request) -> Result "services" if parts.len() == 1 => Some("metrics"), "storage" if parts.len() == 1 => Some("metrics"), "ai" | "mcp" => Some("ai"), - "services" | "traces" | "storage" | "users" | "deploys" | "paper-clover" => { + "services" | "traces" | "storage" | "users" | "shares" | "deploys" | "paper-clover" => { Some("admin") } "media" | "seedbox" | "youtube" => Some("media"), @@ -322,6 +323,7 @@ async fn api(State(app): State>, request: Request) -> Result "ai" => ai::route(app, &method, &parts[1..], &me, &headers).await, "mcp" => mcp::manage(app, &method, &parts[1..], &me, value, &headers).await, "users" => users::route(app, &method, &parts[1..], &me, value).await, + "shares" => shares::route(app, &method, &parts[1..], &query, &me, value).await, "vms" | "seedbox" | "paper-clover" => { apps::route(app, &method, &parts, &query, &me, value).await } diff --git a/dashboard/src/shares.rs b/dashboard/src/shares.rs new file mode 100644 index 0000000000000000000000000000000000000000..583683481ae992a66bd0900d2d30f2f19b0bd2f0 --- /dev/null +++ b/dashboard/src/shares.rs @@ -0,0 +1,92 @@ +use crate::*; + +fn operation(method: &Method, parts: &[&str], mut body: Value) -> Result<(&'static str, Value)> { + let action = match parts { + [] if method == Method::GET => return Ok(("state", json!({}))), + ["folders"] if method == Method::GET => return Ok(("folders.list", body)), + ["accounts"] if method == Method::POST => "account.create", + ["accounts", _] if method == Method::PATCH => "account.update", + ["accounts", _] if method == Method::DELETE => "account.delete", + ["accounts", _, "rotate"] if method == Method::POST => "account.rotate", + ["grants"] if method == Method::POST => "grant.save", + ["grants", _] if method == Method::PATCH => "grant.save", + ["grants", _] if method == Method::DELETE => "grant.delete", + ["links"] if method == Method::POST => "link.create", + ["links", _] if method == Method::PATCH => "link.update", + ["links", _] if method == Method::DELETE => "link.delete", + _ => return Err(Error::new(404, "No share action here.")), + }; + if body.is_null() { + body = json!({}); + } + let fields = body + .as_object_mut() + .ok_or_else(|| Error::new(400, "Reload shares and try again."))?; + if let Some(id) = parts.get(1) { + let key = if parts[0] == "links" { "key" } else { "id" }; + if fields.contains_key(key) { + return Err(Error::new(400, "Reload shares before saving this change.")); + } + fields.insert(key.into(), json!(id)); + } else if parts == ["grants"] { + fields.insert("id".into(), Value::Null); + } + Ok((action, body)) +} + +pub async fn route( + app: Arc, + method: &Method, + parts: &[&str], + query: &HashMap, + me: &Value, + body: Value, +) -> Result { + need(me, "admin")?; + let body = if parts == ["folders"] && method == Method::GET { + json!({"path":query.get("path").map(String::as_str).unwrap_or_default()}) + } else { + body + }; + let (action, payload) = operation(method, parts, body)?; + let users = { + let db = app.auth.db.lock().unwrap(); + let mut statement = db.prepare("SELECT id FROM users ORDER BY username")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .map(|id| { + let id = id?; + let profile = auth::user(&db, &id)?; + let groups = array(&profile["groups"]).iter().map(|group| group["name"].clone()).collect::>(); + Ok(json!({"id":id,"username":profile["username"],"enabled":profile["enabled"],"kind":profile["kind"],"requiredActions":profile["requiredActions"],"groups":groups})) + }) + .collect::>>()? + }; + let mut value = host::call(json!({ + "operation":"copyparty.manage", "action":action, "payload":payload, "users":users, + })) + .await?; + if value.is_object() { + value["filesUrl"] = json!(env( + "STUDIO_FILES_URL", + &format!("https://file.{}", env("STUDIO_DOMAIN", "studio.test")), + )); + } + Ok(Document::new(value).response()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_identity_cannot_be_overridden_by_the_body() { + assert!(operation(&Method::PATCH, &["links", "kept"], json!({"key":"other"})).is_err()); + assert!(operation(&Method::PATCH, &["accounts", "kept"], json!({"id":"other"})).is_err()); + let (action, payload) = + operation(&Method::DELETE, &["links", "kept"], Value::Null).unwrap(); + assert_eq!(action, "link.delete"); + assert_eq!(payload, json!({"key":"kept"})); + assert!(operation(&Method::GET, &["accounts", "kept", "rotate"], Value::Null).is_err()); + } +} diff --git a/dashboard/src/users.rs b/dashboard/src/users.rs index 34fb405396ddfaaee8182132a7f7310212407238..61e8d32804b8a61004e6ed5386339d7ad453b981 100644 --- a/dashboard/src/users.rs +++ b/dashboard/src/users.rs @@ -35,8 +35,8 @@ fn profile(body: &Value, full: bool) -> Result { let value = match key { "username" => { let v = string(value).trim().to_lowercase(); - if v.starts_with("ai-") { - return Err(Error::new(400, "Usernames beginning with ai- are reserved for model accounts.")); + if v.starts_with("ai-") || v.starts_with("api-") { + return Err(Error::new(400, "Choose a username without the ai- or api- prefix. Those are reserved for service accounts.")); } if v.len() > 254 || !username_pattern.is_match(&v) { return Err(Error::new( @@ -635,3 +635,16 @@ pub async fn account( Document::new(value).response() }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn people_cannot_claim_service_account_usernames() { + for username in ["api-account", "API-account", "ai-account"] { + assert!(profile(&json!({"username":username}), false).is_err()); + } + assert!(profile(&json!({"username":"person"}), false).is_ok()); + } +} diff --git a/dashboard/web/api.contract.ts b/dashboard/web/api.contract.ts index 935e3a5bf6be34f59cc95bc83abb944637d1c087..03aa202ec49da1cdc42c30649378f20c3f9ac16c 100644 --- a/dashboard/web/api.contract.ts +++ b/dashboard/web/api.contract.ts @@ -9,6 +9,7 @@ import type { Domain, History, NewDomain, VMLibrary } from "./types/vms.ts"; import type { Me, ServiceSummary, ServiceDetail, ServiceDefinition, HostInfo, Issue, Series, LogLine, TraceSummary, Trace } from "./types/model.ts"; import type { MapNode } from "./types/storage.index.ts"; import type { ProgressNode, PaperCloverStats } from "./types/paperClover.ts"; +import type { AccountSecret, FolderGrant, LinkInput, ShareState } from "./types/shares.ts"; type Endpoint = { input: Input; output: Output; status: Status; outputFormat: Format; @@ -54,6 +55,24 @@ type Explorer = { type ExplorerRoutes = { [P in keyof Explorer as `${Prefix}${P}`]: Explorer[P] }; export type Api = Hono<{}, { + "/shares": { $get: Endpoint; }; + "/shares/folders": { $get: Endpoint<{ path: string; directories: { name: string; path: string }[] }, { query: { path: string } }>; }; + "/shares/accounts": { $post: Endpoint; }; + "/shares/accounts/:id": { + $patch: Endpoint; + $delete: Endpoint; + }; + "/shares/accounts/:id/rotate": { $post: Endpoint; }; + "/shares/grants": { $post: Endpoint }>; }; + "/shares/grants/:id": { + $patch: Endpoint }>; + $delete: Endpoint; + }; + "/shares/links": { $post: Endpoint; }; + "/shares/links/:key": { + $patch: Endpoint; + $delete: Endpoint; + }; "/ai": { $get: Endpoint<{ endpoint: string; model: string; context: number }>; }; "/ai/key": { $post: Endpoint<{ key: string }>; }; "/mcp": { $get: Endpoint; }; diff --git a/dashboard/web/components/Dialog.tsx b/dashboard/web/components/Dialog.tsx index 9a85165ec68fd75f00182ed224bd83779dbf5cfa..1ac56ccf9cef5e62af16f3c00698af6999334adf 100644 --- a/dashboard/web/components/Dialog.tsx +++ b/dashboard/web/components/Dialog.tsx @@ -20,6 +20,7 @@ interface DialogOptions { canConfirm?: () => boolean; /** Takes focus once the dialog closes, instead of the element that opened it. */ returnFocus?: HTMLElement; + onClose?: () => void; /** * Gets the body's named fields. The dialog stays open with a spinner until this settles; a rejection shakes it * and shows the reason. @@ -96,6 +97,7 @@ function open(options: DialogOptions, field?: TextDialogOptions) { dispose(); host.remove(); options.returnFocus?.focus(); + options.onClose?.(); }} onPointerDown={(event) => (pressedOutside = event.target === dialog)} onClick={(event) => pressedOutside && event.target === dialog && close()}> diff --git a/dashboard/web/components/Sidebar.tsx b/dashboard/web/components/Sidebar.tsx index 93a9263c2c63927f2d249b19cbb100d1dc5055dc..3af6518a3640544a1c895125e313ad2f034d22fb 100644 --- a/dashboard/web/components/Sidebar.tsx +++ b/dashboard/web/components/Sidebar.tsx @@ -14,6 +14,7 @@ import SquarePlay from "lucide-solid/icons/square-play"; import UserRound from "lucide-solid/icons/user-round"; import Plug from "lucide-solid/icons/plug"; import Users from "lucide-solid/icons/users"; +import FolderHeart from "lucide-solid/icons/folder-heart"; import { createSignal, For, type JSX, onCleanup, Show } from "solid-js"; import { type Health, type Me, type Section, VIEW_AS } from "../types/model.ts"; import { queries } from "../api.ts"; @@ -59,6 +60,7 @@ const AFTER: Page[] = [ }, { href: "/paper-clover", label: "paper clover", icon: PaperCloverMark, section: "admin" }, { href: "/users", label: "users", icon: Users, section: "admin" }, + { href: "/shares", label: "shares", icon: FolderHeart, section: "admin" }, { href: "/deploys", label: "deploys", icon: Rocket, section: "admin" }, { href: "/vms", label: "vms", icon: Monitor, section: "vms" }, ]; diff --git a/dashboard/web/main.tsx b/dashboard/web/main.tsx index a19edad46fedd458b6105554e7bb130e02517215..e77ab74909898f9298d24a020b58662b6a953bfc 100644 --- a/dashboard/web/main.tsx +++ b/dashboard/web/main.tsx @@ -16,6 +16,7 @@ import { Seedbox } from "./pages/Seedbox.tsx"; import { YouTube } from "./pages/YouTube.tsx"; import { PaperClover } from "./pages/PaperClover.tsx"; import { directory, Users } from "./pages/Users.tsx"; +import { shares, Shares } from "./pages/Shares.tsx"; import { Deploys } from "./pages/Deploys.tsx"; import { Deploy } from "./pages/Deploy.tsx"; import { VMs } from "./pages/VMs.tsx"; @@ -83,6 +84,8 @@ render(() => ( + + diff --git a/dashboard/web/pages/Shares.css b/dashboard/web/pages/Shares.css new file mode 100644 index 0000000000000000000000000000000000000000..0eda47842115d08736414e5031c673e894fd180d --- /dev/null +++ b/dashboard/web/pages/Shares.css @@ -0,0 +1,45 @@ +.shares-toolbar { display: flex; align-items: center; flex-wrap: wrap; gap: 12px; margin-bottom: 12px; } +.shares-toolbar .search-box { margin-left: auto; width: 240px; } +.shares-toolbar .count { opacity: 0.7; } +.shares-table { overflow: auto; } +.shares-table th { white-space: nowrap; } +.share-name { color: var(--accent); font-weight: 600; } +.share-name:hover { text-decoration: underline; } +.share-path { font: 12px/1.5 var(--mono); overflow-wrap: anywhere; } +.shares-table .share-path { min-width: 170px; } +.share-detail { display: block; color: var(--muted); font-size: 12px; margin-top: 2px; } +.share-expiry { white-space: nowrap; } +.share-account-folder { display: flex; flex-wrap: wrap; gap: 4px 12px; margin-block: 3px; } +.share-detail-page .page-head { flex-wrap: wrap; } +.share-detail-page .page-head h1 { overflow-wrap: anywhere; } +.share-form { max-width: 800px; } +.share-form > fieldset { display: grid; gap: 18px; margin: 0; padding: 0; border: 0; min-width: 0; } +.share-form .field { display: grid; gap: 5px; min-width: 0; } +.share-folder-field { display: flex; gap: 6px; } +.share-current-folder { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 12px; } +.share-current-folder .share-path { flex: 1; } +.dialog .share-form { display: grid; gap: 18px; } +.share-folder-field input { flex: 1; min-width: 0; font-family: var(--mono); font-size: 12px; } +.share-rights, .share-path-mode { border: 0; margin: 0; padding: 0; min-width: 0; } +.share-rights legend, .share-path-mode legend { margin-bottom: 7px; padding: 0; } +.share-rights > div { display: flex; flex-wrap: wrap; gap: 8px 18px; } +.share-path-mode { display: grid; gap: 8px; } +.share-path-mode label { display: grid; grid-template-columns: 18px 1fr; column-gap: 5px; align-items: center; } +.share-path-mode input { grid-column: 1; grid-row: 1; margin: 0; accent-color: var(--accent); } +.share-path-mode .hint { grid-column: 2; margin: 0; } +.share-options { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; } +.share-options button { justify-self: start; } +.share-selection { margin: 0; padding-left: 20px; overflow-wrap: anywhere; } +.share-columns { display: grid; grid-template-columns: minmax(250px, 1fr) minmax(320px, 1.5fr); gap: 8px; align-items: start; } +.share-columns .card-title { margin: 0 0 12px; align-items: center; color: var(--text); } +.share-grant { display: flex; align-items: center; gap: 6px; padding-block: 7px; } +.share-grant > div { display: grid; flex: 1; min-width: 0; gap: 2px; } +.share-grant .muted { font-size: 12px; } +.share-grants p { font-size: 13px; } +.share-secret dd { overflow-wrap: anywhere; } +.share-picker .field { display: grid; gap: 5px; } +.share-picker-list { max-height: 300px; min-height: 100px; overflow: auto; margin-top: 10px; } +.share-picker-list button { display: flex; align-items: center; gap: 8px; text-align: left; border: 0; background: none; width: 100%; padding: 7px 8px; cursor: pointer; border-radius: 4px; overflow-wrap: anywhere; } +.share-picker-list button:hover { background: var(--hover); } +@media (max-width: 1000px) { .share-columns { grid-template-columns: 1fr; } } +@media (max-width: 600px) { .share-options { grid-template-columns: 1fr; } .shares-toolbar .search-box { margin-left: 0; width: 100%; } } diff --git a/dashboard/web/pages/Shares.tsx b/dashboard/web/pages/Shares.tsx new file mode 100644 index 0000000000000000000000000000000000000000..2b009fb42ad1cf600421369705cbc53b21babca3 --- /dev/null +++ b/dashboard/web/pages/Shares.tsx @@ -0,0 +1,318 @@ +import { A, useNavigate, useParams, useSearchParams } from "@solidjs/router"; +import ArrowLeft from "lucide-solid/icons/arrow-left"; +import Folder from "lucide-solid/icons/folder"; +import Plus from "lucide-solid/icons/plus"; +import Search from "lucide-solid/icons/search"; +import { createMemo, createResource, createSignal, For, onMount, Show } from "solid-js"; +import { parseResponse } from "hono/client"; +import { api, query, reason } from "../api.ts"; +import { Checkbox } from "../components/Checkbox.tsx"; +import { Copy } from "../components/Copy.tsx"; +import { showConfirmDialog, showTextDialog } from "../components/Dialog.tsx"; +import { ListPage } from "../components/ListPage.tsx"; +import { lastGood, Loaded, SkeletonRows } from "../components/Loaded.tsx"; +import { TabBar } from "../components/TabBar.tsx"; +import type { AccountSecret, FileAccount, FolderGrant, ShareState, SharedLink } from "../types/shares.ts"; +import { datetime, plural } from "../format.ts"; +import "./Shares.css"; + +export const shares = query("shares", () => parseResponse(api.shares.$get())); + +const RIGHTS = [["r", "read"], ["w", "upload"], ["m", "move / rename"], ["d", "delete"], [".", "hidden files"], ["a", "Copyparty admin"]] as const; +const permissions = (value: string) => RIGHTS.filter(([key]) => value.includes(key)).map(([, label]) => label).join(", "); +const macPath = (path: string, root: string) => path === root || path.startsWith(root + "/") ? "/Volumes/clover" + path.slice(root.length) : path; +const linkUrl = (state: ShareState, key: string) => state.filesUrl ? `${state.filesUrl.replace(/\/$/, "")}/shr/${encodeURIComponent(key)}` : null; + +function Rights(props: { value: string; onChange: (value: string) => void; link?: boolean }) { + return ; +} + +function FolderField(props: { value: string; root: string; create?: boolean; onChange: (value: string, existing?: boolean) => void }) { + const browse = () => { + const [path, setPath] = createSignal(props.create && !props.value.endsWith("/") ? props.value.replace(/\/[^/]+$/, "") || props.root : props.value || props.root); + showConfirmDialog({ + title: "Choose folder", confirmLabel: "use folder", + body: () => { + const [folders, { refetch }] = createResource(path, (path) => parseResponse(api.shares.folders.$get({ query: { path } }))); + return ; + }, + onConfirm: () => props.onChange(macPath(path(), props.root), true), + }); + }; + return
folder
; +} + +function showSecret(result: AccountSecret, returnFocus?: HTMLElement) { + showConfirmDialog({ + title: "Save account password", confirmLabel: "done", + returnFocus, + description: "This password is shown once. Save it before closing.", + body: () => , + onConfirm: () => {}, + }); +} + +function editGrant(props: { subject: string; state: ShareState; grant?: FolderGrant; saved: (state: ShareState) => void }) { + const [path, setPath] = createSignal(macPath(props.grant?.path ?? props.state.defaultRoot, props.state.root)); + const [rights, setRights] = createSignal(props.grant?.permissions ?? "r"); + showConfirmDialog({ + title: props.grant ? "Edit folder access" : "Add folder access", confirmLabel: "save access", + returnFocus: document.querySelector(".folder-access .card-title button") ?? undefined, + canConfirm: () => !!path().trim() && !!rights(), + body: () => , + onConfirm: async () => { + const json = { subject: props.subject, path: path(), permissions: rights() }; + const state = await parseResponse(props.grant + ? api.shares.grants[":id"].$patch({ param: { id: props.grant.id }, json }) + : api.shares.grants.$post({ json })); + props.saved(state); + }, + }); +} + +export function FolderAccess(props: { subject: string; enabled?: boolean }) { + const [state, { mutate, refetch }] = shares.use(); + const ready = lastGood(state); + return
+

folder access + +

+ }> + {(data) => } + +
; +} + +function GrantList(props: { subject: string; state: ShareState; saved: (state: ShareState) => void; enabled?: boolean }) { + const grants = () => props.state.grants.filter((grant) => grant.subject === props.subject); + const remove = (grant: FolderGrant) => showConfirmDialog({ + title: "Remove folder access?", description: macPath(grant.path, props.state.root), confirmLabel: "remove access", destructive: true, + returnFocus: document.querySelector(".folder-access .card-title button") ?? undefined, + onConfirm: async () => props.saved(await parseResponse(api.shares.grants[":id"].$delete({ param: { id: grant.id } }))), + }); + return ; +} + +export function Shares() { + const params = useParams<{ key?: string; id?: string }>(); + const [search, setSearch] = useSearchParams<{ tab?: string; q?: string }>(); + const [state, { refetch, mutate }] = shares.use(); + const navigate = useNavigate(); + const accounts = () => search.tab === "accounts"; + const createAccount = () => { + let credentials: AccountSecret | undefined; + showTextDialog({ + title: "New API account", label: "name", placeholder: "T3Code", confirmLabel: "create account", + onConfirm: async (name) => { + const result = await parseResponse(api.shares.accounts.$post({ json: { name } })); + mutate(result); + navigate(`/shares/accounts/${result.account.id}`); + credentials = result; + }, + onClose: () => { if (credentials) showSecret(credentials, document.querySelector(".share-detail-page h1") ?? undefined); }, + }); + }; + return }> + {(data) =>

shares

+ + }> +
+ + + + +
+
}> + +
+ + }>{(key) => }
}> + {(id) => account.id === id)} fallback={
This account no longer exists. back to accounts
}> + {(account) => } +
} + } +
; +} + +function LinkList(props: { state: ShareState; filter: string }) { + const links = () => props.state.links.filter((link) => `${link.name} ${link.path} ${link.key}`.toLowerCase().includes(props.filter.toLowerCase())); + return {props.filter ? "No shares match" : "No shared links yet"} new share}> + + {(link) => + + + + + + } +
sharefolderaccessexpirespublic link
+ + {permissions(link.permissions)}{(url) => copy link}
+
; +} + +function AccountList(props: { state: ShareState; filter: string; create: () => void }) { + const accounts = () => props.state.accounts.filter((account) => `${account.name} ${account.username}`.toLowerCase().includes(props.filter.toLowerCase())); + return {props.filter ? "No accounts match" : "No API accounts yet"} }> + + {(account) => + + } +
accountusernamefolders
grant.subject === account.subject)} fallback={no folders assigned}>{(grant) => }
+
; +} + +function AccountPage(props: { account: FileAccount; state: ShareState; saved: (state: ShareState) => void }) { + const navigate = useNavigate(); + let heading!: HTMLHeadingElement; + onMount(() => heading.focus()); + const rename = () => showTextDialog({ title: "Rename account", label: "name", initialValue: props.account.name, confirmLabel: "save name", + onConfirm: async (name) => props.saved(await parseResponse(api.shares.accounts[":id"].$patch({ param: { id: props.account.id }, json: { name } }))), + }); + const rotate = () => { + let credentials: AccountSecret | undefined; + showConfirmDialog({ title: "Replace account password?", description: "The old password and open sessions stop working.", confirmLabel: "replace password", destructive: true, + onConfirm: async () => { + const result = await parseResponse(api.shares.accounts[":id"].rotate.$post({ param: { id: props.account.id } })); + props.saved(result); credentials = result; + }, + onClose: () => { if (credentials) showSecret(credentials); }, + }); + }; + const remove = () => showConfirmDialog({ title: `Delete ${props.account.name}?`, description: "This account loses access to its folders. The files stay in place.", confirmLabel: "delete account", destructive: true, + onConfirm: async () => { + props.saved(await parseResponse(api.shares.accounts[":id"].$delete({ param: { id: props.account.id } }))); + navigate("/shares?tab=accounts"); + }, + }); + return ; +} + +function LinkPage(props: { key: string; state: ShareState; saved: (state: ShareState) => void }) { + const navigate = useNavigate(); + const original = () => props.state.links.find((link) => link.key === props.key); + const fresh = props.key === "new"; + return This share no longer exists. back to shares}> + { props.saved(state); if (fresh) navigate(`/shares/${encodeURIComponent(key)}`); }} /> + ; +} + +function LinkForm(props: { original?: SharedLink; fresh: boolean; state: ShareState; saved: (state: ShareState, key: string) => void }) { + const navigate = useNavigate(); + let heading!: HTMLHeadingElement; + onMount(() => heading.focus()); + const [name, setName] = createSignal(props.original?.name ?? ""); + const [customPath, setCustomPath] = createSignal(""); + const path = () => customPath() || `${macPath(props.state.defaultRoot, props.state.root)}/${name().trim()}`; + const [rights, setRights] = createSignal(props.original?.permissions ?? "r"); + const [createFolder, setCreateFolder] = createSignal(true); + const [password, setPassword] = createSignal(null); + const dateValue = (seconds: number) => { + const date = new Date(seconds * 1000); + return new Date(date.getTime() - date.getTimezoneOffset() * 60_000).toISOString().slice(0, 16); + }; + const [expires, setExpires] = createSignal(props.original?.expires ? dateValue(props.original.expires) : ""); + const [pending, setPending] = createSignal(false); + const [error, setError] = createSignal(""); + const dirty = createMemo(() => props.fresh || name() !== props.original?.name || rights() !== props.original?.permissions || password() !== null || expires() !== (props.original?.expires ? dateValue(props.original.expires) : "")); + const save = async () => { + if (pending() || !dirty()) return; + setPending(true); setError(""); + try { + const json = { name: name().trim(), path: props.original?.path ?? path(), permissions: rights(), expires: expires() ? new Date(expires()).getTime() / 1000 : 0, password: password() }; + const result = await parseResponse(props.original + ? api.shares.links[":key"].$patch({ param: { key: props.original.key }, json: { ...json, mode: "retarget" } }) + : api.shares.links.$post({ json: { ...json, createFolder: createFolder() } })); + const key = props.original?.key ?? result.links.find((link) => !props.state.links.some((old) => old.key === link.key))?.key; + if (key) props.saved(result, key); + setPassword(null); + } catch (failure) { setError(reason(failure)); } + finally { setPending(false); } + }; + const changeFolder = () => { + const link = props.original!; + const [destination, setDestination] = createSignal(macPath(link.path, props.state.root)); + const [mode, setMode] = createSignal<"retarget" | "move">("retarget"); + showConfirmDialog({ + title: "Change folder", + get confirmLabel() { return mode() === "move" ? "move folder" : "use folder"; }, + canConfirm: () => !!destination().trim() && destination() !== macPath(link.path, props.state.root), + body: () => , + onConfirm: async () => props.saved(await parseResponse(api.shares.links[":key"].$patch({ param: { key: link.key }, json: { + name: link.name, path: destination(), permissions: link.permissions, expires: link.expires, password: null, mode: mode(), + } })), link.key), + }); + }; + const remove = () => showConfirmDialog({ title: `Delete ${props.original?.name || "share"}?`, description: "The public link stops working. The files stay in place.", confirmLabel: "delete share", destructive: true, + onConfirm: async () => { const state = await parseResponse(api.shares.links[":key"].$delete({ param: { key: props.original!.key } })); props.saved(state, props.original!.key); navigate("/shares"); }, + }); + return }> + { setCustomPath(value); if (existing) setCreateFolder(false); }} /> + create this folder + +
selected filesThis share keeps its file selection when its folder changes
+ + + {(url) =>
public link
}
+ + + + ; +} diff --git a/dashboard/web/pages/Users.tsx b/dashboard/web/pages/Users.tsx index cbf292b2cb5a618b7c8c105113ec4df4bdcbd13b..18c9dde32a1bb46fa1edcd78586beea964bcaab8 100644 --- a/dashboard/web/pages/Users.tsx +++ b/dashboard/web/pages/Users.tsx @@ -23,6 +23,7 @@ import { toast } from "../components/Toast.tsx"; import { ago, count, date, datetime, plural } from "../format.ts"; import "./Users.css"; import { CommitIdentity } from "./Account.tsx"; +import { FolderAccess } from "./Shares.tsx"; const STEPS: [string, string][] = [["SETUP", "finish setup"], ["UPDATE_PASSWORD", "new password"], ["UPDATE_PROFILE", "check profile"]]; @@ -592,6 +593,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) { +

sign-in diff --git a/dashboard/web/types/shares.ts b/dashboard/web/types/shares.ts new file mode 100644 index 0000000000000000000000000000000000000000..f33b11737f98a057a7b657cd4b555a9c6912ccfa --- /dev/null +++ b/dashboard/web/types/shares.ts @@ -0,0 +1,44 @@ +export interface FileAccount { + id: string; + name: string; + username: string; + subject: string; +} + +export interface FolderGrant { + id: string; + subject: string; + path: string; + permissions: string; +} + +export interface SharedLink { + key: string; + name: string; + path: string; + permissions: string; + owner: string; + created: number; + expires: number; + hasPassword: boolean; + files: string[]; +} + +export interface ShareState { + root: string; + defaultRoot: string; + filesUrl: string | null; + accounts: FileAccount[]; + grants: FolderGrant[]; + links: SharedLink[]; +} + +export interface LinkInput { + name: string; + path: string; + permissions: string; + expires: number; + password: string | null; +} + +export type AccountSecret = ShareState & { account: FileAccount; password: string }; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 055a74206338f198c4c8d929375e00b61b1a8827..17136db2c05d236a8fc2a718c90cbc5c229a1759 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,7 +6,7 @@ let proxyToken = "/var/lib/studio/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ]; }; nativePkl = pkgs.callPackage ./pkl.nix { }; screenPython = pkgs.python3.withPackages (packages: [ packages.pygobject3 packages.gst-python ]); @@ -203,11 +203,11 @@ in RuntimeDirectoryPreserve = "yes"; UMask = "0077"; ProtectSystem = "strict"; - ReadWritePaths = [ "/srv/vm" "-/srv/prod/shale/data" "/var/lib/studio/dashboard" ] ++ lib.optionals (config.networking.hostName != "clover-demo") [ "/srv/clover/Media/vm" ]; + ReadWritePaths = [ "/srv/vm" "-/srv/prod/shale/data" "-/srv/prod/copyparty/cfg" "/srv/staging" "/srv/clover" "/var/lib/studio/dashboard" ] ++ lib.optionals (config.networking.hostName != "clover-demo") [ "/srv/clover/Media/vm" ]; ProtectHome = true; PrivateTmp = true; NoNewPrivileges = true; - CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_READ_SEARCH" "CAP_DAC_OVERRIDE" ]; + CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_READ_SEARCH" "CAP_DAC_OVERRIDE" "CAP_CHOWN" ]; MemoryMax = "256M"; TasksMax = 64; DevicePolicy = "closed"; diff --git a/service/copyparty/copyparty.conf b/service/copyparty/copyparty.conf index a4eaf454e8ddffe88a9116ca4f100120a93e22a6..5a76744810e3549da97a92eea39b4b2acee94d27 100644 --- a/service/copyparty/copyparty.conf +++ b/service/copyparty/copyparty.conf @@ -9,6 +9,7 @@ name: clover's nas stats, nos-dup dav-auth + xvol cookie-cmax: 65536 ah-alg: argon2 xbu: c,/hooks/reject-apple-cruft.py diff --git a/service/copyparty/manage.py b/service/copyparty/manage.py new file mode 100644 index 0000000000000000000000000000000000000000..cfb6bc741bc2c9eb093114328e5793d25d8b7f37 --- /dev/null +++ b/service/copyparty/manage.py @@ -0,0 +1,697 @@ +import copy +import ctypes +import errno +import fcntl +import importlib +import json +import os +from pathlib import Path +import re +import secrets +import sqlite3 +import subprocess +import time +import urllib.error +import urllib.request +import uuid + +runs = importlib.import_module('dashboard-run') +Error = runs.Error +PODMAN = ['podman', '--remote', '--url', 'unix:///run/podman/podman.sock'] + +ACTIONS = { + 'state': set(), 'folders.list': {'path'}, + 'account.create': {'name'}, 'account.update': {'id', 'name'}, + 'account.rotate': {'id'}, 'account.delete': {'id'}, + 'grant.save': {'id', 'subject', 'path', 'permissions'}, 'grant.delete': {'id'}, + 'link.create': {'name', 'path', 'permissions', 'expires', 'password', 'createFolder'}, + 'link.update': {'key', 'name', 'path', 'permissions', 'expires', 'password', 'mode'}, + 'link.delete': {'key'}, +} + + +def identifier(value): + if not isinstance(value, str): + raise Error(400, 'Choose an account or folder permission from the list.') + try: + if str(uuid.UUID(value)) == value: + return value + except ValueError: + pass + raise Error(400, 'Choose an account or folder permission from the list.') + + +def label(value): + if (not isinstance(value, str) or not value.strip() or len(value) > 200 + or any(ord(c) < 32 or c in '/\\' for c in value) + or value.strip() in {'.', '..'}): + raise Error(400, 'Enter a name under 200 characters without slashes.') + return value.strip() + + +def permissions(value, choices): + if not isinstance(value, str) or not value or set(value) - set(choices): + raise Error(400, 'Choose at least one supported permission.') + return ''.join(c for c in choices if c in value) + + +def atomic(path, data, owner=None): + pending = path.with_name('.' + path.name + '.pending') + fd = os.open(pending, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, 'wb') as target: + target.write(data) + target.flush() + os.fsync(target.fileno()) + if owner is not None: + os.chown(pending, *owner) + os.replace(pending, path) + finally: + pending.unlink(missing_ok=True) + + +def volumes(base): + sections = re.split(r'(?m)^(?=\[/)', base) + result = [] + for section in sections[1:]: + lines = [line.strip() for line in section.splitlines() if line.strip()] + match = re.fullmatch(r'\[(/[^\r\n]*)\]', lines[0]) + if not match or len(lines) < 2: + raise Error(503, 'Copyparty configuration needs repair before editing shares.') + entry = {'url': match[1].rstrip('/') or '/', 'path': lines[1], 'accs': [], 'flags': []} + target = None + for line in lines[2:]: + if line == 'accs:': + target = entry['accs'] + elif line == 'flags:': + target = entry['flags'] + elif target is not None: + target.append(line) + else: + raise Error(503, 'Copyparty configuration needs repair before editing shares.') + result.append(entry) + return sections[0], result + + +def render(base, state, hashes): + global_config, entries = volumes(base) + accounts = state['accounts'] + if accounts: + global_config += '\n[accounts]\n' + ''.join( + ' api-' + account['id'] + ': ' + hashes['file_api_' + account['id'].replace('-', '')] + '\n' + for account in accounts + ) + api_names = ['api-' + account['id'] for account in accounts] + base_entries = copy.deepcopy(entries) + paths = {grant['path'] for grant in state['grants']} + for entry in base_entries: + if entry['url'] == '/clover' or entry['url'].startswith('/clover/'): + host = str(Path(state['root']) / entry['url'][len('/clover'):].lstrip('/')) + if any(host == grant['path'] or host.startswith(grant['path'].rstrip('/') + '/') for grant in state['grants']): + paths.add(host) + for path in sorted(paths): + relative = Path(path).relative_to(state['root']) + url = '/clover' + ('/' + relative.as_posix() if relative.parts else '') + parent = max((entry for entry in base_entries if url == entry['url'] or url.startswith(entry['url'].rstrip('/') + '/')), + key=lambda entry: len(entry['url'])) + exact = next((entry for entry in entries if entry['url'] == url), None) + entry = exact if exact is not None else { + 'url': url, 'path': '/w/clover' + ('/' + relative.as_posix() if relative.parts else ''), + 'accs': list(parent['accs']), 'flags': list(parent['flags']), + } + for grant in state['grants']: + if path == grant['path'] or path.startswith(grant['path'].rstrip('/') + '/'): + kind, subject = grant['subject'].split(':', 1) + principal = '@snow-user-' + subject if kind == 'user' else 'api-' + subject + entry['accs'].append(grant['permissions'] + ': ' + principal) + if 'xvol' not in entry['flags']: + entry['flags'].append('xvol') + if exact is None: + entries.append(entry) + rendered = global_config + for entry in entries: + allowed = {} + if entry['url'] == '/clover' or entry['url'].startswith('/clover/'): + host = str(Path(state['root']) / entry['url'][len('/clover'):].lstrip('/')) + for grant in state['grants']: + if grant['subject'].startswith('api:') and (host == grant['path'] or host.startswith(grant['path'].rstrip('/') + '/')): + for perm in grant['permissions']: + allowed.setdefault(perm, set()).add('api-' + grant['subject'][4:]) + access = [] + for line in entry['accs']: + perms, names = line.split(':', 1) + names = [name.strip() for name in names.split(',')] + if api_names and ('*' in names or '@acct' in names): + perms = perms.replace('A', 'rwmda.') + for perm in dict.fromkeys(perms): + denied = [name for name in api_names if name not in allowed.get(perm, set())] + access.append(perm + ': ' + ','.join(names + ['-' + name for name in denied])) + else: + access.append(line) + entry['accs'] = access + rendered += '\n[' + entry['url'] + ']\n ' + entry['path'] + '\n accs:\n' + rendered += ''.join(' ' + line + '\n' for line in entry['accs']) + if entry['flags']: + rendered += ' flags:\n' + ''.join(' ' + line + '\n' for line in entry['flags']) + return rendered + + +def configuration(cfg, state, items, source=None): + source = source or Path(__file__).with_name('copyparty.conf') + base = source.read_text() + proof = items.get('idp_header', '') + if not re.fullmatch(r'[0-9a-f]{32}', proof): + raise Error(503, 'Copyparty sign-in configuration is unavailable. Redeploy the service.') + capacity = os.statvfs(cfg) + reserve = max(1, min(16, capacity.f_blocks * capacity.f_frsize // (4 * 10**9))) + base = base.replace('STUDIO_IDP_HEADER', 'X-Studio-Idp-' + proof).replace('STUDIO_MIN_FREE_GB', str(reserve)) + return render(base, state, items).encode() + + +def remaining(deadline, maximum=10): + timeout = min(maximum, deadline - time.monotonic()) + if timeout <= 0: + raise Error(504, 'Copyparty took too long to respond. Check the service before retrying.') + return timeout + + +def rename(source, destination): + # RENAME_NOREPLACE keeps a concurrently created destination from being overwritten. + libc = ctypes.CDLL(None, use_errno=True) + if hasattr(libc, 'renameat2'): + result = libc.renameat2(-100, os.fsencode(source), -100, os.fsencode(destination), 1) + elif hasattr(libc, 'renamex_np'): + result = libc.renamex_np(os.fsencode(source), os.fsencode(destination), 4) + else: + raise Error(503, 'This host cannot safely move folders. Retarget the share instead.') + if result: + code = ctypes.get_errno() + if code == errno.EEXIST: + raise Error(409, 'The destination already exists. Choose a new folder name.') + if code == errno.EXDEV: + raise Error(409, 'Choose a destination on the same filesystem to move this folder.') + raise OSError(code, os.strerror(code)) + + +class Manager: + def __init__(self, users): + root = Path(os.environ.get('STUDIO_ROOT', '/srv')) + self.root = Path(os.environ.get('STUDIO_CLOVER_ROOT', str(root / 'clover'))).resolve() + self.cfg = Path(os.environ.get('STUDIO_COPYPARTY_CFG', str(root / 'prod/copyparty/cfg'))) + self.native = self.cfg / 'copyparty' + self.service = os.environ.get('STUDIO_COPYPARTY_SERVICE', 'copyparty') + self.users = users + self.path = self.cfg / 'manager.json' + self.changed = set() + self.undo = [] + self.secret_before = {} + self.container_id = None + self.db = None + self.job = None + self.stopped = False + self.offline_at = None + self.deadline = None + + def path_for(self, value, missing=False): + if not isinstance(value, str) or len(value) > 4096 or any(ord(c) < 32 or c in '#[]\\' for c in value) or ' ' in value: + raise Error(400, 'Choose a folder without control characters, brackets, or repeated spaces.') + if value == '/Volumes/clover' or value.startswith('/Volumes/clover/'): + value = str(self.root) + value[len('/Volumes/clover'):] + path = Path(value) + if not path.is_absolute() or '..' in path.parts or not path.is_relative_to(self.root): + raise Error(400, 'Choose a folder inside the Clover file store.') + current = self.root + try: + for part in path.relative_to(self.root).parts: + current /= part + if current.is_symlink(): + raise Error(400, 'Choose a folder directly instead of a symbolic link.') + if not missing and not path.is_dir(): + raise Error(404, 'That folder no longer exists. Choose another folder.') + if missing and not path.parent.is_dir(): + raise Error(404, 'The parent folder does not exist. Choose an existing parent.') + if path.exists() and not path.is_dir(): + raise Error(400, 'Choose a folder instead of a file.') + except OSError: + raise Error(400, 'That folder cannot be opened. Choose another folder.') from None + return path + + def container(self): + if self.container_id: + return self.container_id + result = subprocess.run([*PODMAN, 'ps', '--format', '{{.ID}} {{.Image}}'], capture_output=True, text=True, timeout=10, check=True) + for row in result.stdout.splitlines(): + fields = row.split() + if len(fields) != 2 or '/copyparty/' not in fields[1]: + continue + result = subprocess.run([*PODMAN, 'inspect', '--format', '{{json .Mounts}}', fields[0]], + capture_output=True, text=True, timeout=10, check=True) + mounts = {mount['Destination']: mount['Source'] for mount in json.loads(result.stdout)} + if mounts.get('/cfg') == str(self.cfg): + if mounts.get('/w/clover') != str(self.root): + raise Error(503, 'The share manager points at a different file store. Check its host configuration.') + self.container_id = fields[0] + return self.container_id + raise Error(503, 'Copyparty is offline. Start it before editing shares.') + + def hash(self, password): + if not isinstance(password, str) or len(password) > 255 or any(c in password for c in '\r\n\0'): + raise Error(400, 'Use a single-line password under 256 characters.') + if not password: + return '' + script = ('import argparse,sys; from pathlib import Path; from copyparty.pwhash import PWHash; ' + 'args=argparse.Namespace(ah_alg="argon2",ah_salt=Path("/cfg/copyparty/ah-salt.txt").read_text().strip()); ' + 'print(PWHash(args).hash(sys.stdin.read()))') + result = subprocess.run([*PODMAN, 'exec', '-i', self.container(), 'python3', '-c', script], + input=password, capture_output=True, text=True, timeout=25) + value = result.stdout.strip() + if result.returncode or not re.fullmatch(r'\+[A-Za-z0-9_-]{32}', value): + raise Error(502, 'The password could not be secured. Check Copyparty and try again.') + return value + + def reload(self, items, timeout=30): + deadline = time.monotonic() + timeout + instances = runs.nomad('service/' + self.service, timeout=remaining(deadline)) or [] + if len(instances) != 1: + raise Error(503, 'Copyparty is offline or changing allocations. Try again shortly.') + address, port = instances[0]['Address'], instances[0]['Port'] + if not re.fullmatch(r'[0-9a-fA-F:.]+', address) or type(port) is not int or not 1 <= port <= 65535: + raise Error(503, 'Copyparty address is unavailable. Redeploy its service.') + address = '[' + address + ']' if ':' in address else address + request = urllib.request.Request('http://' + address + ':' + str(port) + '/?reload=cfg', headers={ + 'User-Name': 'snow', 'User-Groups': '', 'X-Studio-Idp-' + items['idp_header']: '1', + }) + try: + with urllib.request.urlopen(request, timeout=remaining(deadline, 30)) as response: + body = response.read(262144) + if response.status != 200 or b'config has been reloaded' not in body: + raise Error(502, 'Copyparty did not confirm the update. Try again after checking the service.') + except (urllib.error.URLError, TimeoutError): + raise Error(502, 'Copyparty could not apply the update. Check the service and try again.') from None + + def stop(self): + if self.offline_at is None: + self.offline_at = time.monotonic() + if self.deadline is None: + self.deadline = self.offline_at + 160 + deadline = min(self.deadline, time.monotonic() + 40) + if self.job is None: + self.job = runs.nomad('job/' + self.service, timeout=remaining(deadline)) + if self.job is None: + raise Error(503, 'Copyparty is offline. Start it before moving folders.') + self.job['Stop'] = False + runs.nomad('job/' + self.service + '?purge=false', 'DELETE', timeout=remaining(deadline)) + self.stopped = True + while time.monotonic() < deadline: + allocations = runs.nomad('job/' + self.service + '/allocations', timeout=remaining(deadline)) or [] + if all(entry.get('ClientStatus') in {'complete', 'failed', 'lost'} for entry in allocations): + self.container_id = None + return + time.sleep(0.5) + raise Error(502, 'Copyparty did not stop in time. Check the service before retrying the move.') + + def start(self): + if self.job is None: + raise Error(503, 'Copyparty restart settings are unavailable. Redeploy its service.') + deadline = min(self.deadline, time.monotonic() + 70) + runs.nomad('jobs', 'POST', {'Job': self.job}, timeout=remaining(deadline)) + items = getattr(self, 'items', None) + if items is None: + variable = runs.nomad('var/nomad/jobs/' + self.service, timeout=remaining(deadline)) + if variable is None: + raise Error(503, 'Copyparty secrets are unavailable. Redeploy its service.') + items = variable['Items'] + while time.monotonic() < deadline: + try: + self.reload(items, timeout=remaining(deadline, 30)) + self.stopped = False + return + except Error as error: + if error.status not in {502, 503}: + raise + time.sleep(0.5) + raise Error(502, 'Copyparty could not restart in time. Check its service logs before retrying.') + + def virtual(self, path): + return 'clover' + ('/' + path.relative_to(self.root).as_posix() if path != self.root else '') + + def real(self, virtual): + if virtual == 'clover' or virtual.startswith('clover/'): + return self.root / virtual[len('clover'):].lstrip('/') + if virtual == 'media' or virtual.startswith('media/'): + media = Path(os.environ.get('STUDIO_MEDIA_ROOT', str(self.root / 'Media'))) + return media / virtual[len('media'):].lstrip('/') + return None + + def state(self): + links = [] + for key, password, path, perms, count, owner, created, expires in self.db.execute('SELECT * FROM sh ORDER BY t0 DESC,k'): + real = self.real(path) + links.append({'key': key, 'name': self.data['names'].get(key, real.name if real is not None else Path(path).name or key), + 'path': str(real) if real is not None else '/' + path, + 'permissions': perms, 'owner': owner, 'created': created, 'expires': expires, + 'hasPassword': bool(password), 'files': [row[0] for row in self.db.execute('SELECT vp FROM sf WHERE k=? ORDER BY vp', (key,))] if count else []}) + return {'root': str(self.root), 'defaultRoot': str(self.root / 'Share'), + 'accounts': [{**account, 'subject': 'api:' + account['id'], 'username': 'api-' + account['id']} for account in self.data['accounts']], + 'grants': self.data['grants'], 'links': links} + + def secret(self, key, value): + before = self.items.get(key) + if value is None: + self.items.pop(key, None) + else: + self.items[key] = value + self.secret_before[key] = before + + def account(self, value): + value = identifier(value) + account = next((entry for entry in self.data['accounts'] if entry['id'] == value), None) + if account is None: + raise Error(404, 'That API account no longer exists. Reload accounts.') + return account + + def grant(self, payload): + subject = payload['subject'] + if not isinstance(subject, str) or ':' not in subject: + raise Error(400, 'Choose a Snow Sign On user or API account.') + kind, value = subject.split(':', 1) + identifier(value) + if kind == 'api': + self.account(value) + elif kind == 'user': + if not any(user['id'] == value and user.get('enabled') is True and user.get('kind') in {None, 'user'} for user in self.users): + raise Error(400, 'Choose an enabled Snow Sign On user.') + else: + raise Error(400, 'Choose a Snow Sign On user or API account.') + path = str(self.path_for(payload['path'])) + perms = permissions(payload['permissions'], 'rwmda.g') + entry = {'id': identifier(payload['id']) if payload['id'] is not None else str(uuid.uuid4()), + 'subject': subject, 'path': path, 'permissions': perms} + found = next((grant for grant in self.data['grants'] if grant['id'] == entry['id']), None) + if payload['id'] is not None and found is None: + raise Error(404, 'That folder permission no longer exists. Reload permissions.') + if any(grant['id'] != entry['id'] and grant['subject'] == subject and grant['path'] == path for grant in self.data['grants']): + raise Error(409, 'This account already has permissions for that folder. Edit its existing permission.') + self.data['grants'] = [grant for grant in self.data['grants'] if grant['id'] != entry['id']] + [entry] + + def owner_permissions(self, owner, path, required): + _, entries = volumes(configuration(self.cfg, self.data, self.items).decode()) + url = '/' + self.virtual(path) + entry = max((entry for entry in entries if url == entry['url'] or url.startswith(entry['url'].rstrip('/') + '/')), + key=lambda entry: len(entry['url'])) + groups = {'acct'} + user = next((user for user in self.users if user.get('username') == owner), None) + if user is not None: + if user.get('enabled') is not True or user.get('kind') not in {None, 'user'} or user.get('requiredActions'): + raise Error(409, 'The share owner cannot sign in. Enable their account before editing this share.') + groups.update(user.get('groups', [])) + groups.add('snow-user-' + identifier(user['id'])) + effective = set() + excluded_permissions = set() + for rule in entry['accs']: + perms, principals = rule.split(':', 1) + names = {name.strip() for name in principals.split(',')} + selected = owner in names or '*' in names or any('@' + group in names for group in groups) + excluded = '-' + owner in names or any('-@' + group in names for group in groups) + expanded = perms.replace('A', 'rwmda.') + if selected: + effective.update(expanded) + if excluded: + excluded_permissions.update(expanded) + if set(required) - (effective - excluded_permissions): + raise Error(409, 'The share owner needs these permissions at the destination. Grant them access before changing this share.') + + def link(self, action, payload): + key = payload['key'] if action == 'link.update' else secrets.token_urlsafe(12) + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,200}', key): + raise Error(400, 'Choose a share from the list.') + old = self.db.execute('SELECT * FROM sh WHERE k=?', (key,)).fetchone() + if action == 'link.update' and old is None: + raise Error(404, 'That share no longer exists. Reload shares.') + name = label(payload['name']) + perms = permissions(payload['permissions'], 'rwmd') + expires = payload['expires'] + if type(expires) is not int or expires < 0 or expires > 2**53: + raise Error(400, 'Choose an expiration date or keep the share indefinitely.') + password = payload['password'] + if password is not None and not isinstance(password, str): + raise Error(400, 'Enter a password or leave the existing password unchanged.') + if password is None: + password = old[1] if old is not None else '' + else: + password = self.hash(password) + create = action == 'link.create' and payload['createFolder'] is True + if action == 'link.create' and type(payload['createFolder']) is not bool: + raise Error(400, 'Choose an existing folder or create a new one.') + destination = payload['path'] + share_root = self.root / 'Share' + if create and (not destination or Path(destination).parent == share_root or str(destination).startswith('/Volumes/clover/Share/')): + self.path_for(str(share_root), missing=True) + if not share_root.exists(): + share_root.mkdir(mode=0o775) + share_root.chmod(0o775) + owner = self.root.stat() + os.chown(share_root, owner.st_uid, owner.st_gid) + self.undo.append(lambda: share_root.rmdir()) + if not destination: + destination = str(share_root / name) + destination = self.path_for(destination, missing=create or action == 'link.update' and payload['mode'] == 'move') + files = [row[0] for row in self.db.execute('SELECT vp FROM sf WHERE k=?', (key,))] if old is not None and old[4] else [] + mode = payload.get('mode', 'retarget') + if mode not in {'retarget', 'move'}: + raise Error(400, 'Choose whether to move the folder or use another folder.') + if create: + try: + destination.mkdir(mode=0o775) + destination.chmod(0o775) + except FileExistsError: + raise Error(409, 'That folder already exists. Choose it as an existing folder or change its name.') from None + owner = destination.parent.stat() + os.chown(destination, owner.st_uid, owner.st_gid) + self.undo.append(lambda: destination.rmdir()) + if old is not None and mode == 'move': + source = self.real(old[2]) + if source is None: + raise Error(400, 'Only folders inside the Clover file store can be moved here.') + source = self.path_for(str(source)) + if source != destination: + for grant in self.data['grants']: + if Path(grant['path']).is_relative_to(source): + grant['path'] = str(destination / Path(grant['path']).relative_to(source)) + self.owner_permissions(old[5], destination, perms) + self.move(source, destination) + for filename in files: + if (not isinstance(filename, str) or filename in {'.', '..'} or '/' in filename or '\\' in filename + or not (destination / filename).is_file() or (destination / filename).is_symlink()): + raise Error(409, 'The destination must contain every selected file. Choose a matching folder.') + if old is not None: + self.owner_permissions(old[5], destination, perms) + self.changed.add(key) + self.db.execute('UPDATE sh SET pw=?,vp=?,pr=?,t1=? WHERE k=?', (password, self.virtual(destination), perms, expires, key)) + else: + self.changed.add(key) + self.db.execute('INSERT INTO sh VALUES(?,?,?,?,?,?,?,?)', (key, password, self.virtual(destination), perms, 0, 'snow', int(time.time()), expires)) + self.data['names'][key] = name + + def move(self, source, destination): + if source == self.root or source == self.root / 'Share' or destination.is_relative_to(source) or source.is_relative_to(destination): + raise Error(400, 'Choose separate folders below the Clover file store.') + if destination.exists(): + raise Error(409, 'The destination already exists. Choose a new folder name.') + if source.stat().st_dev != destination.parent.stat().st_dev: + raise Error(409, 'Choose a destination on the same filesystem to move this folder.') + mounts = Path('/proc/self/mountinfo') + if mounts.exists(): + for row in mounts.read_text().splitlines(): + mount = Path(re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), row.split()[4])) + if mount.is_relative_to(source): + raise Error(409, 'A mounted filesystem cannot be moved. Choose a folder without mounts inside it.') + elif os.path.ismount(source): + raise Error(409, 'A mounted filesystem cannot be moved. Choose a folder inside it.') + rows = self.db.execute('SELECT k,vp,st FROM sh').fetchall() + for key, virtual, count in rows: + real = self.real(virtual) + if real is not None and source != real and source.is_relative_to(real): + selected = {row[0] for row in self.db.execute('SELECT vp FROM sf WHERE k=?', (key,))} if count else None + if selected is None or source.relative_to(real).parts[0] in selected: + raise Error(409, 'Another share includes this folder through its parent. Retarget that share before moving the folder.') + self.stop() + for key, virtual, _ in rows: + real = self.real(virtual) + if real is not None and real.is_relative_to(source): + self.changed.add(key) + self.db.execute('UPDATE sh SET vp=? WHERE k=?', (self.virtual(destination / real.relative_to(source)), key)) + for grant in self.data['grants']: + if Path(grant['path']).is_relative_to(source): + grant['path'] = str(destination / Path(grant['path']).relative_to(source)) + rename(source, destination) + self.undo.append(lambda: rename(destination, source)) + + def dispatch(self, action, payload): + extra = {} + if action.startswith('account.'): + account = {'id': str(uuid.uuid4()), 'name': label(payload['name'])} if action == 'account.create' else self.account(payload['id']) + value = account['id'] + if action == 'account.create': + if any(user.get('username') == 'api-' + value for user in self.users): + raise Error(409, 'That API username already belongs to a Snow Sign On user. Try creating the account again.') + self.data['accounts'].append(account) + if action in {'account.update', 'account.create'}: + account['name'] = label(payload['name']) + if action in {'account.create', 'account.rotate'}: + password = secrets.token_urlsafe(32) + self.secret('file_api_' + value.replace('-', ''), self.hash(password)) + extra = {'account': {**account, 'subject': 'api:' + value, 'username': 'api-' + value}, 'password': password} + if action in {'account.delete', 'account.rotate'}: + session_path = self.native / 'sessions.db' + if session_path.exists(): + with sqlite3.connect(session_path) as sessions: + sessions.execute('DELETE FROM us WHERE un=?', ('api-' + value,)) + if action == 'account.delete': + self.data['accounts'].remove(account) + self.data['grants'] = [grant for grant in self.data['grants'] if grant['subject'] != 'api:' + value] + self.secret('file_api_' + value.replace('-', ''), None) + elif action == 'grant.save': + self.grant(payload) + elif action == 'grant.delete': + value = identifier(payload['id']) + if not any(grant['id'] == value for grant in self.data['grants']): + raise Error(404, 'That folder permission no longer exists. Reload permissions.') + self.data['grants'] = [grant for grant in self.data['grants'] if grant['id'] != value] + elif action in {'link.create', 'link.update'}: + self.link(action, payload) + elif action == 'link.delete': + key = payload['key'] + if not isinstance(key, str) or not self.db.execute('SELECT 1 FROM sh WHERE k=?', (key,)).fetchone(): + raise Error(404, 'That share no longer exists. Reload shares.') + self.changed.add(key) + self.db.execute('DELETE FROM sh WHERE k=?', (key,)) + self.db.execute('DELETE FROM sf WHERE k=?', (key,)) + self.data['names'].pop(key, None) + return extra + + def apply(self, action, payload): + self.data = json.loads(runs.read(self.path, missing='{"accounts":[],"grants":[],"names":{}}')) + origin = Path(self.data.get('root', os.environ.get('STUDIO_SOURCE_CLOVER_ROOT', '/srv/clover'))) + if 'root' not in self.data and self.data.get('grants'): + origin = Path(os.environ.get('STUDIO_SOURCE_CLOVER_ROOT', str(self.root))) + if origin != self.root: + for grant in self.data['grants']: + grant['path'] = str(self.root / Path(grant['path']).relative_to(origin)) + self.data['root'] = str(self.root) + if not isinstance(self.data.get('accounts'), list) or not isinstance(self.data.get('grants'), list) or not isinstance(self.data.get('names'), dict): + raise Error(503, 'Share settings need repair before editing. Check the host service.') + database = self.native / 'shares.db' + if not database.is_file(): + raise Error(503, 'Copyparty shares are not ready. Start Copyparty and try again.') + self.db = sqlite3.connect(database.as_uri() + '?mode=rw', uri=True, timeout=5) + self.db.row_factory = None + try: + version = self.db.execute("SELECT v FROM kv WHERE k='sver'").fetchone() + if version != (2,) or [row[1] for row in self.db.execute('PRAGMA table_info(sh)')] != ['k', 'pw', 'vp', 'pr', 'st', 'un', 't0', 't1']: + raise Error(503, 'This Copyparty share format is unsupported. Update Snowglobe before editing shares.') + if action == 'state': + return self.state() + if action == 'folders.list': + path = self.path_for(payload['path']) + with os.scandir(path) as entries: + directories = sorted(({'name': entry.name, 'path': str(path / entry.name)} for entry in entries + if not entry.name.startswith('.') and entry.is_dir(follow_symlinks=False)), key=lambda entry: entry['name'].casefold()) + return {'path': str(path), 'directories': directories} + self.container() + variable_path = 'nomad/jobs/' + self.service + variable = runs.nomad('var/' + variable_path) + if variable is None: + raise Error(503, 'Copyparty secrets are unavailable. Redeploy its service.') + self.items = dict(variable['Items']) + original_items = dict(self.items) + config_path = self.cfg / 'copyparty.conf' + old_config = config_path.read_bytes() + metadata = config_path.stat() + owner = (metadata.st_uid, metadata.st_gid) + self.db.execute('BEGIN IMMEDIATE') + original_rows = {row[0]: row for row in self.db.execute('SELECT * FROM sh')} + original_files = {key: self.db.execute('SELECT * FROM sf WHERE k=?', (key,)).fetchall() for key in original_rows} + committed = False + wrote_files = False + old_manager = self.path.read_bytes() if self.path.exists() else None + changed_secrets = False + after_rows = {} + try: + extra = self.dispatch(action, payload) + rendered = configuration(self.cfg, self.data, self.items) + if self.secret_before: + runs.nomad('var/' + variable_path + '?cas=' + str(variable['ModifyIndex']), 'PUT', + {'Namespace': 'default', 'Path': variable_path, 'Items': self.items}) + changed_secrets = True + atomic(config_path, rendered, owner) + wrote_files = True + atomic(self.path, json.dumps(self.data, ensure_ascii=False).encode()) + after_rows = {key: self.db.execute('SELECT * FROM sh WHERE k=?', (key,)).fetchone() for key in self.changed} + self.db.commit() + committed = True + if self.job is not None: + self.start() + else: + self.reload(self.items) + return {**self.state(), **extra} + except Exception: + if self.job is not None: + self.deadline = self.offline_at + 280 + self.stop() + self.db.rollback() + if committed and self.changed: + self.db.execute('BEGIN IMMEDIATE') + for key in self.changed: + current = self.db.execute('SELECT * FROM sh WHERE k=?', (key,)).fetchone() + if current != after_rows[key]: + raise Error(502, 'The update could not finish and a share changed during recovery. Check Copyparty before retrying.') from None + self.db.execute('DELETE FROM sh WHERE k=?', (key,)) + self.db.execute('DELETE FROM sf WHERE k=?', (key,)) + if key in original_rows: + self.db.execute('INSERT INTO sh VALUES(?,?,?,?,?,?,?,?)', original_rows[key]) + self.db.executemany('INSERT INTO sf VALUES(?,?)', original_files[key]) + self.db.commit() + for undo in reversed(self.undo): + undo() + if wrote_files: + atomic(config_path, old_config, owner) + if old_manager is None: + self.path.unlink(missing_ok=True) + else: + atomic(self.path, old_manager) + if changed_secrets: + latest = runs.nomad('var/' + variable_path) + items = dict(latest['Items']) + for key, value in self.secret_before.items(): + if items.get(key) != self.items.get(key): + raise Error(502, 'The API credential changed during recovery. Check Copyparty before retrying.') from None + if value is None: + items.pop(key, None) + else: + items[key] = value + runs.nomad('var/' + variable_path + '?cas=' + str(latest['ModifyIndex']), 'PUT', + {'Namespace': 'default', 'Path': variable_path, 'Items': items}) + if self.job is not None: + self.items = original_items + self.start() + elif committed: + self.reload(original_items) + raise + finally: + self.db.close() + + +def handle(request): + action, payload, users = request.get('action'), request.get('payload'), request.get('users') + if (not isinstance(action, str) or action not in ACTIONS or not isinstance(payload, dict) + or set(payload) != ACTIONS[action] or not isinstance(users, list)): + raise Error(400, 'Use the fields required by this share action.') + manager = Manager(users) + if not manager.cfg.is_dir(): + raise Error(503, 'Copyparty settings are unavailable. Deploy its service before editing shares.') + lock = os.open(manager.cfg / '.manager.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + with os.fdopen(lock, 'w') as locked: + fcntl.flock(locked, fcntl.LOCK_EX) + return manager.apply(action, payload) diff --git a/service/copyparty/prepare.py b/service/copyparty/prepare.py index 1d6f70b85cff5b8cbb7b766ff259ae1e83b2f837..7b32fcd84147c94adf934cdf0c5650c0abd36ee4 100644 --- a/service/copyparty/prepare.py +++ b/service/copyparty/prepare.py @@ -2,35 +2,45 @@ import json import os from pathlib import Path -import re import subprocess import sys +root = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(root / 'tools')) +import manage as copyparty + data = json.load(sys.stdin) -root = Path(data["hostRoot"]) / "cfg" -service = Path(data["hostRoot"]).name -secret = json.loads(subprocess.check_output( - ["nomad", "var", "get", "-out=json", f"nomad/jobs/{service}"], +cfg = Path(data['hostRoot']) / 'cfg' +service = Path(data['hostRoot']).name +variable = json.loads(subprocess.check_output( + ['nomad', 'var', 'get', '-out=json', 'nomad/jobs/' + service], text=True, stderr=subprocess.DEVNULL, -))["Items"]["idp_header"] -if not re.fullmatch(r"[0-9a-f]{32}", secret): - raise ValueError("invalid Copyparty identity secret") -config = Path(__file__).with_name("copyparty.conf").read_text() -if config.count("STUDIO_IDP_HEADER") != 1: - raise ValueError("expected one Copyparty identity header marker") -config = config.replace("STUDIO_IDP_HEADER", "X-Studio-Idp-" + secret) -if config.count("STUDIO_MIN_FREE_GB") != 1: - raise ValueError("expected one Copyparty free-space marker") -capacity = os.statvfs(data["hostRoot"]) -reserve = max(1, min(16, capacity.f_blocks * capacity.f_frsize // (4 * 10**9))) -config = config.replace("STUDIO_MIN_FREE_GB", str(reserve)) -target = root / "copyparty.conf" -if not target.exists() or target.read_text() != config: - pending = root / ".copyparty.conf.pending" - pending.write_text(config) - os.chmod(pending, 0o600) - os.chown(pending, data["uid"], data["uid"]) - os.replace(pending, target) +)) +items = variable['Items'] +state_path = cfg / 'manager.json' +state = json.loads(state_path.read_text()) if state_path.exists() else {'accounts': [], 'grants': [], 'names': {}} +canonical_root = Path(os.environ.get('STUDIO_CLOVER_ROOT', str(Path(os.environ.get('STUDIO_ROOT', '/srv')) / 'clover'))) +origin = Path(state.get('root', str(canonical_root))) +if origin != canonical_root: + for grant in state['grants']: + grant['path'] = str(canonical_root / Path(grant['path']).relative_to(origin)) +state['root'] = str(canonical_root) +missing = ['file_api_' + account['id'].replace('-', '') for account in state['accounts'] + if 'file_api_' + account['id'].replace('-', '') not in items] +if missing and service != 'copyparty': + source = copyparty.runs.nomad('var/nomad/jobs/copyparty') + for key in missing: + if source is None or key not in source['Items']: + raise RuntimeError('Copyparty API credentials are missing from the source Nomad variables') + items[key] = source['Items'][key] + copyparty.runs.nomad('var/nomad/jobs/' + service + '?cas=' + str(variable['ModifyIndex']), 'PUT', + {'Namespace': 'default', 'Path': 'nomad/jobs/' + service, 'Items': items}) +elif missing: + raise RuntimeError('Copyparty API credentials are missing; restore its Nomad variables before starting it') +rendered = copyparty.configuration(cfg, state, items, Path(__file__).with_name('copyparty.conf')) +target = cfg / 'copyparty.conf' +if not target.exists() or target.read_bytes() != rendered: + copyparty.atomic(target, rendered, (data['uid'], data['uid'])) os.chmod(target, 0o600) -os.chown(target, data["uid"], data["uid"]) +os.chown(target, data['uid'], data['uid']) diff --git a/tools/copyparty-cutover-test.py b/tools/copyparty-cutover-test.py new file mode 100644 index 0000000000000000000000000000000000000000..39defc1dd6ece739b7b150404147363e21180471 --- /dev/null +++ b/tools/copyparty-cutover-test.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import os +from pathlib import Path +import sqlite3 +import sys +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('cutover', Path(__file__).with_name('copyparty-cutover.py')) +cutover = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cutover) +copyparty = cutover.copyparty +public_statuses = cutover.public_statuses + + +class Cutover(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix='copyparty-cutover-') + self.addCleanup(self.temporary.cleanup) + self.base = Path(self.temporary.name) + self.root = self.base / 'clover' + self.cfg = self.base / 'cfg' + (self.root / 'Share').mkdir(parents=True) + (self.cfg / 'copyparty').mkdir(parents=True) + self.environment = patch.dict(os.environ, {'STUDIO_CLOVER_ROOT': str(self.root), 'STUDIO_COPYPARTY_CFG': str(self.cfg)}) + self.environment.start() + self.addCleanup(self.environment.stop) + self.manager = copyparty.Manager([]) + self.manager.job = {'ID': 'copyparty', 'Stop': False, 'Meta': {'studio_hostname': 'copyparty.example.invalid'}} + self.events = [] + edge = patch.object(cutover, 'public_statuses', side_effect=lambda host, keys, deadline: {key: 200 for key in keys}) + edge.start() + self.addCleanup(edge.stop) + + def stop(): + self.events.append('stop') + if self.manager.offline_at is None: + self.manager.offline_at = cutover.time.monotonic() + self.manager.stopped = True + + def start(): + self.events.append('start') + self.manager.stopped = False + + self.manager.stop = stop + self.manager.start = start + for source, _ in cutover.MOVES: + path = self.root / source + (path / 'nested').mkdir(parents=True) + (path / 'selected.txt').write_text('Disposable fixture') + (path / 'nested' / 'child.txt').write_text('Disposable child') + (self.cfg / 'copyparty.conf').write_text('Fixture configuration') + self.database = self.cfg / 'copyparty/shares.db' + with sqlite3.connect(self.database) as db: + db.executescript("CREATE TABLE kv(k text,v int); INSERT INTO kv VALUES('sver',2); CREATE TABLE sh(k text,pw text,vp text,pr text,st int,un text,t0 int,t1 int); CREATE TABLE sf(k text,vp text);") + db.executemany('INSERT INTO sh VALUES(?,?,?,?,?,?,?,?)', [ + ('folder-key', 'opaque-hash', 'clover/Documents/Public/Snowbound', 'rw', 0, 'snow', 10, 20), + ('nested-key', '', 'clover/Documents/Public/Snowbound/nested', 'r', 0, 'snow', 30, 0), + ('selected-key', 'another-hash', 'clover/Documents/EvilTakeout', 'r', 1, 'snow', 40, 50), + ('unaffected-key', '', 'clover/Other', 'r', 0, 'snow', 60, 0), + ]) + db.execute('INSERT INTO sf VALUES(?,?)', ('selected-key', 'selected.txt')) + with sqlite3.connect(self.database) as db: + self.original = cutover.rows(db) + self.before_content = {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob('*.txt')} + + def test_move_preserves_keys_selections_passwords_inodes_and_content(self): + expected = cutover.plan(self.manager) + result = cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(result['result'], 'moved') + self.assertEqual(self.events, ['stop', 'start']) + with sqlite3.connect(self.database) as db: + actual = cutover.rows(db) + self.assertEqual(actual, cutover.remapped(self.manager, self.original, cutover.mappings(self.manager))) + self.assertEqual(actual['sf'], self.original['sf']) + for detail in expected['moves']: + source, destination = Path(detail['source']), Path(detail['destination']) + self.assertFalse(source.exists()) + self.assertEqual(destination.stat().st_ino, detail['inode']) + for old_path, contents in self.before_content.items(): + old = self.root / old_path + if old.is_relative_to(source): + self.assertEqual((destination / old.relative_to(source)).read_bytes(), contents) + backup = Path(result['backup']) + for child in backup.iterdir(): + self.assertEqual(child.stat().st_mode & 0o777, 0o600) + self.assertEqual(backup.stat().st_mode & 0o777, 0o700) + + def test_public_probe_sends_bounded_verified_head_and_preserves_redirect_status(self): + response = cutover.subprocess.CompletedProcess([], 0, stdout='302', stderr='') + with patch.object(cutover.subprocess, 'run', return_value=response) as run: + actual = public_statuses('copyparty.example.invalid', ['selected file'], cutover.time.monotonic() + 1) + self.assertEqual(actual, {'selected file': 302}) + arguments = run.call_args.args[0] + self.assertEqual(arguments[:2], ['curl', '--disable']) + self.assertIn('--head', arguments) + self.assertEqual(arguments[arguments.index('--output') + 1], '/dev/null') + self.assertEqual(arguments[arguments.index('--write-out') + 1], '%{http_code}') + self.assertEqual(arguments[arguments.index('--noproxy') + 1], '*') + self.assertEqual(arguments[-2:], ['--', 'https://copyparty.example.invalid/shr/selected%20file/']) + self.assertLessEqual(run.call_args.kwargs['timeout'], 1) + self.assertEqual(float(arguments[arguments.index('--max-time') + 1]), run.call_args.kwargs['timeout']) + for flag in ['--insecure', '-k', '--location', '-L', '--include', '-i']: + self.assertNotIn(flag, arguments) + + def test_public_probe_treats_network_tls_and_process_timeout_as_unavailable(self): + for response in [ + cutover.subprocess.CompletedProcess([], 35, stdout='000', stderr='TLS problem'), + cutover.subprocess.CompletedProcess([], 28, stdout='200', stderr='Timeout'), + cutover.subprocess.CompletedProcess([], 0, stdout='000', stderr=''), + cutover.subprocess.CompletedProcess([], 0, stdout='200 headers', stderr=''), + ]: + with patch.object(cutover.subprocess, 'run', return_value=response): + self.assertEqual(public_statuses('host', ['key'], cutover.time.monotonic() + 1), {'key': None}) + with patch.object(cutover.subprocess, 'run', side_effect=cutover.subprocess.TimeoutExpired('curl', 0.1)): + self.assertEqual(public_statuses('host', ['key'], cutover.time.monotonic() + 0.1), {'key': None}) + with patch.object(cutover.subprocess, 'run') as run: + self.assertEqual(public_statuses('host', ['key'], cutover.time.monotonic() - 1), {'key': None}) + run.assert_not_called() + + def test_stalled_probe_children_cannot_hold_the_executor_past_its_budget(self): + run = cutover.subprocess.run + def stalled(arguments, **options): + return run([sys.executable, '-c', 'import time; time.sleep(10)'], **options) + started = cutover.time.monotonic() + with patch.object(cutover.subprocess, 'run', side_effect=stalled): + actual = public_statuses('host', ['one', 'two'], started + 0.15) + self.assertEqual(actual, {'one': None, 'two': None}) + self.assertLess(cutover.time.monotonic() - started, 1.5) + + def test_public_poll_recovers_and_timeout_obeys_its_budget(self): + clock = [0.0] + def sleep(delay): + clock[0] += delay + with patch.object(cutover.time, 'monotonic', side_effect=lambda: clock[0]), patch.object(cutover.time, 'sleep', side_effect=sleep): + with patch.object(cutover, 'public_statuses', side_effect=[{'key': 503}, {'key': 200}]): + cutover.wait_public('host', {'key': 200}, 1) + self.assertEqual(clock[0], 0.5) + with patch.object(cutover, 'public_statuses', return_value={'key': None}), self.assertRaises(TimeoutError): + cutover.wait_public('host', {'key': 200}, 1) + self.assertEqual(clock[0], 1) + + def test_edge_timeout_rolls_back_and_records_recovery_evidence(self): + expected = cutover.plan(self.manager) + with patch.object(cutover, 'wait_public', side_effect=[TimeoutError('injected edge timeout'), None]): + with self.assertRaisesRegex(TimeoutError, 'edge timeout'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(cutover.plan(self.manager), expected) + self.assertEqual(self.events, ['stop', 'start', 'stop', 'start']) + evidence = json.loads(next((self.base / 'backups').glob('*/result.json')).read_text()) + self.assertEqual(evidence['result'], 'rolled-back') + self.assertEqual(evidence['publicStatuses'], {key: 200 for key in expected['affectedKeys']}) + + def test_existing_grants_follow_moves_and_configuration_is_rerendered(self): + source, destination = cutover.mappings(self.manager)[0] + state = {'accounts': [], 'grants': [{'id': 'fixture', 'subject': 'user:fixture', 'path': str(source), 'permissions': 'r'}], 'names': {}} + (self.cfg / 'manager.json').write_text(json.dumps(state)) + expected = cutover.plan(self.manager) + with patch.object(copyparty.runs, 'nomad', return_value={'Items': {}}), patch.object(copyparty, 'configuration', return_value=b'Updated configuration') as render: + cutover.apply(self.manager, expected, self.base / 'backups') + saved = json.loads((self.cfg / 'manager.json').read_text()) + self.assertEqual(saved['grants'][0]['path'], str(destination)) + self.assertEqual(render.call_args.args[1]['grants'], saved['grants']) + self.assertEqual((self.cfg / 'copyparty.conf').read_bytes(), b'Updated configuration') + + def test_failed_restart_restores_files_and_database_before_restart(self): + def start(): + self.events.append('start') + if self.events.count('start') == 1: + raise RuntimeError('injected startup failure') + self.manager.stopped = False + self.manager.start = start + expected = cutover.plan(self.manager) + with self.assertRaisesRegex(RuntimeError, 'injected startup failure'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(self.events, ['stop', 'start', 'stop', 'start']) + self.assertEqual(cutover.plan(self.manager), expected) + with sqlite3.connect(self.database) as db: + self.assertEqual(cutover.rows(db), self.original) + for path, content in self.before_content.items(): + self.assertEqual((self.root / path).read_bytes(), content) + + def test_rollback_uses_actual_helper_stop_and_start_with_frozen_nomad_job(self): + self.manager.stop = copyparty.Manager.stop.__get__(self.manager) + self.manager.start = copyparty.Manager.start.__get__(self.manager) + self.manager.job = None + requests = [] + def nomad(path, method='GET', body=None, **_): + requests.append((path, method, body)) + if path == 'job/copyparty' and method == 'GET': + return {'ID': 'copyparty', 'Stop': False, 'Meta': {'immutable': 'fixture', 'studio_hostname': 'copyparty.example.invalid'}} + if path.endswith('/allocations'): + return [{'ClientStatus': 'complete'}] + if path.startswith('var/'): + return {'Items': {'idp_header': 'fixture'}} + return {} + expected = cutover.plan(self.manager) + with patch.object(copyparty.runs, 'nomad', side_effect=nomad), patch.object(self.manager, 'reload', side_effect=[copyparty.Error(400, 'injected reload rejection'), None]): + with self.assertRaisesRegex(copyparty.Error, 'injected reload rejection'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(cutover.plan(self.manager), expected) + registrations = [body for path, method, body in requests if path == 'jobs' and method == 'POST'] + self.assertEqual(registrations, [{'Job': self.manager.job}, {'Job': self.manager.job}]) + self.assertEqual(len([path for path, method, _ in requests if method == 'DELETE']), 2) + self.assertFalse(self.manager.stopped) + + def test_partial_rename_failure_rolls_back_before_start(self): + native_rename = copyparty.rename + calls = 0 + def rename(source, destination): + nonlocal calls + calls += 1 + if calls == 3: + raise RuntimeError('injected third rename failure') + self.assertTrue(self.manager.stopped) + native_rename(source, destination) + expected = cutover.plan(self.manager) + with patch.object(copyparty, 'rename', side_effect=rename), self.assertRaisesRegex(RuntimeError, 'third rename'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(cutover.plan(self.manager), expected) + self.assertEqual(self.events, ['stop', 'start']) + + def test_stop_timeout_still_restarts_the_frozen_job(self): + def stop(): + self.manager.offline_at = cutover.time.monotonic() + self.manager.stopped = True + raise RuntimeError('injected stop timeout') + self.manager.stop = stop + expected = cutover.plan(self.manager) + with self.assertRaisesRegex(RuntimeError, 'stop timeout'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(self.events, ['start']) + self.assertEqual(cutover.plan(self.manager), expected) + + def test_ambiguous_stop_response_reregisters_even_without_stopped_flag(self): + def stop(): + self.manager.offline_at = cutover.time.monotonic() + self.manager.stopped = False + raise TimeoutError('lost Nomad stop response') + self.manager.stop = stop + expected = cutover.plan(self.manager) + with self.assertRaisesRegex(TimeoutError, 'lost Nomad stop response'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(self.events, ['start']) + self.assertEqual(cutover.plan(self.manager), expected) + + def test_verification_budget_reserves_time_for_restart(self): + expected = cutover.plan(self.manager) + original_tree = cutover.tree + def timed_tree(path, deadline=None): + if deadline is not None: + raise TimeoutError('injected verification deadline') + return original_tree(path) + with patch.object(cutover, 'tree', side_effect=timed_tree), self.assertRaisesRegex(TimeoutError, 'verification deadline'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(self.events, ['stop', 'start']) + self.assertEqual(cutover.plan(self.manager), expected) + + def test_source_and_database_drift_stop_before_mutation(self): + expected = cutover.plan(self.manager) + (self.root / cutover.MOVES[0][0] / 'changed.txt').write_text('Concurrent disposable write') + with self.assertRaisesRegex(ValueError, 'plan changed'): + cutover.apply(self.manager, expected, self.base / 'backups') + self.assertEqual(self.events, []) + + def test_parent_shares_and_parent_selections_require_explicit_url_preservation(self): + for selected in [0, 1]: + with sqlite3.connect(self.database) as db: + db.execute('INSERT OR REPLACE INTO sh VALUES(?,?,?,?,?,?,?,?)', ('parent', '', 'clover/Documents/Public', 'r', selected, 'snow', 1, 0)) + if selected: + db.execute('INSERT INTO sf VALUES(?,?)', ('parent', 'Snowbound')) + with self.assertRaisesRegex(ValueError, 'parent of a moved folder'): + cutover.plan(self.manager) + with sqlite3.connect(self.database) as db: + db.execute("DELETE FROM sh WHERE k='parent'") + db.execute("DELETE FROM sf WHERE k='parent'") + + def test_symlinks_mounts_and_existing_destinations_reject_before_downtime(self): + source, destination = cutover.mappings(self.manager)[0] + destination.mkdir() + with self.assertRaises(ValueError): + cutover.plan(self.manager) + destination.rmdir() + link = source / 'link' + link.symlink_to('selected.txt') + with self.assertRaisesRegex(ValueError, 'symbolic link'): + cutover.plan(self.manager) + link.unlink() + with patch.object(cutover.os.path, 'ismount', side_effect=lambda path: Path(path) == source): + with self.assertRaises(ValueError): + cutover.plan(self.manager) + self.assertEqual(self.events, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/copyparty-cutover.py b/tools/copyparty-cutover.py new file mode 100644 index 0000000000000000000000000000000000000000..8f4dbbd4bde2a32d8ff87249263e6053a264e104 --- /dev/null +++ b/tools/copyparty-cutover.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +import argparse +import concurrent.futures +import copy +import hashlib +import importlib.util +import json +import os +import re +from pathlib import Path +import sqlite3 +import socket +import subprocess +import stat +import time +import uuid +from urllib.parse import quote + + +spec = importlib.util.spec_from_file_location("copyparty", Path(__file__).resolve().parent.parent / "service/copyparty/manage.py") +copyparty = importlib.util.module_from_spec(spec) +spec.loader.exec_module(copyparty) + + +MOVES = ( + ('Documents/Public/Snowbound', 'Clover - Snowbound'), + ('Documents/Public/Meadow', 'Clover - Meadow Chat'), + ('Documents/Public/Bill Wurtz Music', 'Xavier - Bill Wurtz Music'), + ('Documents/HOTEWIG Reanimated', 'Collab - HOTEWIG Reanimated'), + ('Documents/EvilTakeout', 'Lino - Evil Inc Data Takeout'), + ('Documents/Windows7', 'Clover - Windows 7'), +) + + +def digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode()).hexdigest() + + +def rows(database): + if database.execute("SELECT v FROM kv WHERE k='sver'").fetchone() != (2,): + raise ValueError('Unsupported Copyparty share database version.') + return { + 'sh': database.execute('SELECT * FROM sh ORDER BY k').fetchall(), + 'sf': database.execute('SELECT * FROM sf ORDER BY k,vp').fetchall(), + } + + +def mappings(manager): + return [(manager.root / source, manager.root / 'Share' / name) for source, name in MOVES] + + +def remapped(manager, before, moves): + after = copy.deepcopy(before) + for index, row in enumerate(before['sh']): + source_path = manager.real(row[2].strip('/')) + for source, destination in moves: + if source_path is not None and source_path.is_relative_to(source): + changed = list(row) + changed[2] = manager.virtual(destination / source_path.relative_to(source)) + after['sh'][index] = tuple(changed) + break + if (source_path is not None and source.is_relative_to(source_path)) or row[2].strip('/') == '': + raise ValueError('A share includes a parent of a moved folder. Preserve its URL paths before cutover: ' + row[0]) + return after + + +def tree(path, deadline=None): + def unreadable(error): + raise error + + result = [] + for directory, dirs, files in os.walk(path, followlinks=False, onerror=unreadable): + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("Folder verification exceeded the cutover budget.") + base = Path(directory) + if base != path and os.path.ismount(base): + raise ValueError('A source contains a mounted filesystem: ' + str(base)) + for entry in [base] + [base / name for name in files] + [base / name for name in dirs if (base / name).is_symlink()]: + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("Folder verification exceeded the cutover budget.") + info = entry.lstat() + if stat.S_ISLNK(info.st_mode): + raise ValueError('Review this source symbolic link before cutover: ' + str(entry)) + result.append((str(entry.relative_to(path)), info.st_dev, info.st_ino, info.st_mode, info.st_size if not stat.S_ISDIR(info.st_mode) else 0, info.st_mtime_ns)) + dirs[:] = [name for name in dirs if not (base / name).is_symlink()] + return sorted(result) + + +def plan(manager, deadline=None): + moves = mappings(manager) + details = [] + for source, destination in moves: + manager.path_for(str(source)) + manager.path_for(str(destination), missing=True) + if source.is_symlink() or os.path.ismount(source) or os.path.lexists(destination): + raise ValueError('A source is mounted or a destination already exists: ' + str(source)) + source_info, parent_info = source.stat(), destination.parent.stat() + if source_info.st_dev != parent_info.st_dev: + raise ValueError('Source and destination must be on the same filesystem: ' + str(source)) + manifest = tree(source, deadline) + details.append({'source': str(source), 'destination': str(destination), 'device': source_info.st_dev, + 'inode': source_info.st_ino, 'entries': len(manifest), 'tree': digest(manifest)}) + with sqlite3.connect((manager.native / 'shares.db').as_uri() + '?mode=ro', uri=True) as database: + before = rows(database) + after = remapped(manager, before, moves) + return {'host': socket.gethostname(), 'service': manager.service, 'configRoot': str(manager.cfg), 'moves': details, 'shares': len(before['sh']), 'selections': len(before['sf']), + 'affectedKeys': [old[0] for old, new in zip(before['sh'], after['sh']) if old != new], + 'sharesBefore': digest(before), 'sharesAfter': digest(after), + 'config': digest((manager.cfg / 'copyparty.conf').read_text()), + 'manager': digest((manager.cfg / 'manager.json').read_text() if (manager.cfg / 'manager.json').exists() else None)} + + +def write_database(database, data): + current = rows(database) + if current['sf'] != data['sf'] or len(current['sh']) != len(data['sh']) or any( + old[:2] + old[3:] != new[:2] + new[3:] + for old, new in zip(current['sh'], data['sh']) + ): + raise ValueError('Share keys, selections, passwords, or permissions changed during cutover.') + with database: + database.executemany('UPDATE sh SET vp=? WHERE k=?', [(row[2], row[0]) for row in data['sh']]) + if rows(database) != data: + raise ValueError('Copyparty share paths changed while the database was being written.') + + +def public_statuses(host, keys, deadline): + def head(key): + timeout = min(3, deadline - time.monotonic()) + if timeout <= 0: + return key, None + try: + response = subprocess.run( + ['curl', '--disable', '--noproxy', '*', '--silent', '--show-error', + '--head', '--output', '/dev/null', '--write-out', '%{http_code}', + '--max-time', str(timeout), '--', 'https://' + host + '/shr/' + quote(key, safe='') + '/'], + capture_output=True, text=True, timeout=timeout, + ) + status = response.stdout.strip() + return key, int(status) if response.returncode == 0 and re.fullmatch(r'[1-5][0-9]{2}', status) else None + except (OSError, subprocess.TimeoutExpired): + return key, None + + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as workers: + return dict(workers.map(head, keys)) + + +def wait_public(host, expected, deadline): + while time.monotonic() < deadline: + if public_statuses(host, expected, deadline) == expected: + return + time.sleep(min(0.5, max(0, deadline - time.monotonic()))) + raise TimeoutError('Public share links did not recover before the cutover deadline.') + + +def apply(manager, expected, backup_root): + if plan(manager) != expected: + raise ValueError('The cutover plan changed. Review a new plan before applying.') + if manager.job is None: + manager.job = copyparty.runs.nomad('job/' + manager.service) + host = (manager.job or {}).get('Meta', {}).get('studio_hostname', '') + if not isinstance(host, str) or not re.fullmatch(r'[a-zA-Z0-9.-]+', host): + raise ValueError('Copyparty has no public hostname. Check its deployed service route.') + manager.job['Stop'] = False + baseline = public_statuses(host, expected['affectedKeys'], time.monotonic() + 15) + if any(status is None for status in baseline.values()): + raise ValueError('Public shares are unavailable. Restore their current routes before cutover.') + backup_root.mkdir(parents=True, exist_ok=True, mode=0o700) + metadata = backup_root.lstat() + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077: + raise ValueError('Cutover backups require an owner-only directory.') + backup = backup_root / (time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + '-' + uuid.uuid4().hex[:8]) + backup.mkdir(mode=0o700) + database_path = manager.native / 'shares.db' + saved_files = {} + for path in (manager.cfg / 'copyparty.conf', manager.cfg / 'manager.json'): + saved_files[path] = path.read_bytes() if path.exists() else None + if saved_files[path] is not None: + copyparty.atomic(backup / path.name, saved_files[path]) + copyparty.atomic(backup / 'plan.json', json.dumps(expected, ensure_ascii=False).encode()) + moved = [] + before = None + database = None + manager.deadline = time.monotonic() + 285 + mutation_deadline = manager.deadline - 205 + restart_attempted = False + try: + manager.stop() + copyparty.atomic(backup / 'nomad-job.json', json.dumps(manager.job).encode()) + if plan(manager, mutation_deadline) != expected: + raise ValueError('The source files or share rows changed before Copyparty stopped.') + database = sqlite3.connect(database_path) + before = rows(database) + with sqlite3.connect(backup / 'shares.db') as target: + database.backup(target) + (backup / 'shares.db').chmod(0o600) + after = remapped(manager, before, mappings(manager)) + data_path = manager.cfg / 'manager.json' + data = json.loads(saved_files[data_path]) if saved_files[data_path] is not None else None + for source, destination in mappings(manager): + copyparty.rename(source, destination) + moved.append((source, destination)) + write_database(database, after) + if data is not None: + for grant in data.get('grants', []): + original = Path(grant['path']) + for source, destination in mappings(manager): + if original.is_relative_to(source): + grant['path'] = str(destination / original.relative_to(source)) + break + copyparty.atomic(data_path, json.dumps(data, ensure_ascii=False).encode()) + variable = copyparty.runs.nomad('var/nomad/jobs/' + manager.service) + rendered = copyparty.configuration(manager.cfg, {**data, 'root': str(manager.root)}, variable['Items']) + info = (manager.cfg / 'copyparty.conf').stat() + copyparty.atomic(manager.cfg / 'copyparty.conf', rendered, (info.st_uid, info.st_gid)) + for detail in expected['moves']: + destination = Path(detail['destination']) + if digest(tree(destination, mutation_deadline)) != detail['tree']: + raise ValueError('Moved folder metadata changed: ' + str(destination)) + if digest(rows(database)) != expected['sharesAfter']: + raise ValueError('Share keys, selections, passwords, or permissions changed during cutover.') + if time.monotonic() >= mutation_deadline: + raise TimeoutError("Folder updates exceeded the cutover budget.") + restart_attempted = True + manager.start() + wait_public(host, baseline, manager.offline_at + 150) + result = {'result': 'moved', 'moves': len(moved), 'affectedKeys': expected['affectedKeys'], 'backup': str(backup), + 'downtimeSeconds': time.monotonic() - manager.offline_at, 'publicStatuses': baseline} + copyparty.atomic(backup / 'result.json', json.dumps(result).encode()) + return result + except BaseException as failure: + if moved or before is not None: + if restart_attempted: + manager.stop() + if database is None: + database = sqlite3.connect(database_path) + if before is not None: + write_database(database, before) + for source, destination in reversed(moved): + copyparty.rename(destination, source) + for path, content in saved_files.items(): + if content is None: + path.unlink(missing_ok=True) + else: + owner = path.stat() + copyparty.atomic(path, content, (owner.st_uid, owner.st_gid)) + if manager.job is not None: + manager.start() + wait_public(host, baseline, manager.deadline) + evidence = {'result': 'rolled-back', 'reason': str(failure), 'downtimeSeconds': time.monotonic() - manager.offline_at, + 'publicStatuses': baseline} + copyparty.atomic(backup / 'result.json', json.dumps(evidence).encode()) + raise + finally: + if database is not None: + database.close() + + +def main(): + parser = argparse.ArgumentParser(description='Review or apply the six Clover share-folder moves.') + parser.add_argument('--apply', action='store_true') + parser.add_argument('--plan', type=Path, help='Save the reviewed plan, or read it when applying.') + args = parser.parse_args() + manager = copyparty.Manager([]) + if args.apply: + if args.plan is None: + parser.error('--apply requires --plan from the reviewed preflight') + lock = os.open(manager.cfg / '.manager.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + with os.fdopen(lock, 'w') as locked: + copyparty.fcntl.flock(locked, copyparty.fcntl.LOCK_EX) + print(json.dumps(apply(manager, json.loads(args.plan.read_text()), Path('/var/lib/studio/copyparty-cutover')))) + else: + value = plan(manager) + if args.plan is not None: + copyparty.atomic(args.plan, json.dumps(value, ensure_ascii=False, indent=2).encode()) + print(json.dumps(value, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/tools/copyparty-test.py b/tools/copyparty-test.py new file mode 100644 index 0000000000000000000000000000000000000000..5857eb70fe8a981783e46a3b2c8534bb99303aba --- /dev/null +++ b/tools/copyparty-test.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +import copy +import importlib.util +import json +import os +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch +import uuid + +spec = importlib.util.spec_from_file_location('shares', Path(__file__).resolve().parent.parent / 'service/copyparty/manage.py') +shares = importlib.util.module_from_spec(spec) +spec.loader.exec_module(shares) + + +class Shares(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / 'clover' + self.cfg = Path(self.temporary.name) / 'cfg' + self.root.mkdir() + self.root = self.root.resolve() + (self.cfg / 'copyparty').mkdir(parents=True) + (self.cfg / 'copyparty.conf').write_text('original configuration\n') + self.database = self.cfg / 'copyparty/shares.db' + with sqlite3.connect(self.database) as db: + db.executescript("CREATE TABLE kv(k text,v int); INSERT INTO kv VALUES('sver',2); CREATE TABLE sh(k text,pw text,vp text,pr text,st int,un text,t0 int,t1 int); CREATE TABLE sf(k text,vp text);") + self.subject = str(uuid.uuid4()) + self.users = [{'id': self.subject, 'username': 'test-user', 'enabled': True, 'kind': None, 'requiredActions': []}] + self.variable = {'ModifyIndex': 1, 'Items': {'idp_header': 'a' * 32, 'unrelated': 'retained'}} + self.environment = patch.dict(os.environ, {'STUDIO_CLOVER_ROOT': str(self.root), 'STUDIO_COPYPARTY_CFG': str(self.cfg)}) + self.environment.start() + self.addCleanup(self.environment.stop) + self.nomad = patch.object(shares.runs, 'nomad', side_effect=self.request) + self.nomad.start() + self.addCleanup(self.nomad.stop) + self.reloads = [] + self.reload = patch.object(shares.Manager, 'reload', autospec=True, side_effect=lambda manager, items, **kwargs: self.reloads.append(copy.deepcopy(items))) + self.reload_mock = self.reload.start() + self.addCleanup(self.reload.stop) + self.hash = patch.object(shares.Manager, 'hash', side_effect=lambda password: '+' + 'a' * 32 if password else '') + self.hash.start() + self.container = patch.object(shares.Manager, 'container', return_value='disposable') + self.container.start() + self.addCleanup(self.container.stop) + self.addCleanup(self.hash.stop) + + def request(self, path, method='GET', body=None, timeout=10): + self.assertTrue(path.startswith('var/nomad/jobs/copyparty')) + if method == 'GET': + return copy.deepcopy(self.variable) + self.assertEqual(method, 'PUT') + self.assertEqual(path.split('?cas=')[1], str(self.variable['ModifyIndex'])) + self.variable = {'Items': copy.deepcopy(body['Items']), 'ModifyIndex': self.variable['ModifyIndex'] + 1} + return copy.deepcopy(self.variable) + + def call(self, action, **payload): + return shares.handle({'action': action, 'payload': payload, 'users': self.users}) + + def link(self, key='public', path='clover/source', count=0, files=(), owner='snow'): + with sqlite3.connect(self.database) as db: + db.execute('INSERT INTO sh VALUES(?,?,?,?,?,?,?,?)', (key, '', path, 'rw', count, owner, 100, 0)) + db.executemany('INSERT INTO sf VALUES(?,?)', [(key, file) for file in files]) + + def update(self, path, mode='retarget', key='public'): + return self.call('link.update', key=key, name='Clover - Example', path=str(path), permissions='rw', expires=0, password=None, mode=mode) + + def offline(self): + def stop(manager): + manager.job = {'ID': 'copyparty'} + manager.offline_at = shares.time.monotonic() + manager.stopped = True + def start(manager): + manager.stopped = False + return patch.object(shares.Manager, 'stop', autospec=True, side_effect=stop), patch.object(shares.Manager, 'start', autospec=True, side_effect=start) + + def test_account_password_is_once_only_and_nomad_preserves_other_secrets(self): + created = self.call('account.create', name='Local AI') + self.assertTrue(created['password']) + identifier = created['account']['id'] + key = 'file_api_' + identifier.replace('-', '') + self.assertEqual(self.variable['Items'][key], '+' + 'a' * 32) + self.assertEqual(self.variable['Items']['unrelated'], 'retained') + state = self.call('state') + self.assertNotIn('password', state) + self.assertNotIn(created['password'], (self.cfg / 'manager.json').read_text()) + self.assertNotIn(created['password'], (self.cfg / 'copyparty.conf').read_text()) + self.assertIn('w: @acct,-api-' + identifier, (self.cfg / 'copyparty.conf').read_text()) + self.call('account.delete', id=identifier) + self.assertNotIn(key, self.variable['Items']) + + def test_rotation_revokes_existing_copyparty_session(self): + created = self.call('account.create', name='Local AI') + identifier = created['account']['id'] + session_path = self.cfg / 'copyparty/sessions.db' + with sqlite3.connect(session_path) as db: + db.executescript('CREATE TABLE us(un text,si text,t0 int);') + db.execute('INSERT INTO us VALUES(?,?,?)', ('api-' + identifier, 'old-cookie', 100)) + db.execute('INSERT INTO us VALUES(?,?,?)', ('snow', 'retained-cookie', 100)) + self.call('account.rotate', id=identifier) + with sqlite3.connect(session_path) as db: + self.assertEqual(db.execute('SELECT un,si FROM us').fetchall(), [('snow', 'retained-cookie')]) + + def test_human_grant_uses_uuid_group_and_preserves_ancestor_access(self): + folder = self.root / 'work' + child = folder / 'child' + child.mkdir(parents=True) + first = self.call('grant.save', id=None, subject='user:' + self.subject, path=str(folder), permissions='rw') + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(child), permissions='r') + config = (self.cfg / 'copyparty.conf').read_text() + self.assertIn('rw: @snow-user-' + self.subject, config) + self.assertNotIn('test-user', config) + self.assertIn('xvol', config) + child_config = config.split('[/clover/work/child]')[1] + self.assertIn('A: snow', child_config) + self.assertIn('rw: @snow-user-' + self.subject, child_config) + self.call('grant.delete', id=first['grants'][0]['id']) + config = (self.cfg / 'copyparty.conf').read_text() + self.assertNotIn('[/clover/work]\n', config) + self.assertNotIn('rw: @snow-user-' + self.subject, config) + + def test_exact_base_mount_is_rendered_once(self): + (self.root / 'Public').mkdir() + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(self.root / 'Public'), permissions='d') + config = (self.cfg / 'copyparty.conf').read_text() + self.assertEqual(config.count('[/clover/Public]'), 1) + self.assertIn('r.: *', config) + self.assertIn('d: @snow-user-' + self.subject, config) + + def test_untrusted_boundary_and_symlink_escape_reject_before_nomad(self): + folder = self.root / 'work' + folder.mkdir() + (self.root / 'escape').symlink_to(Path(self.temporary.name)) + cases = [dict(id=None, subject='user:' + self.subject, path=str(self.root / 'escape'), permissions='r'), + dict(id=None, subject='user:' + self.subject, path=str(self.root / '../outside'), permissions='r'), + dict(id=None, subject='user:' + self.subject, path=str(folder), permissions='x'), + dict(id=None, subject='user:' + str(uuid.uuid4()), path=str(folder), permissions='r')] + for payload in cases: + with self.subTest(payload=payload), self.assertRaises(shares.Error): + self.call('grant.save', **payload) + with self.assertRaises(shares.Error): + self.call('state', extra=True) + for user in [dict(self.users[0], enabled=False), dict(self.users[0], kind='ai'), dict(self.users[0], kind='guest')]: + self.users = [user] + with self.assertRaises(shares.Error): + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(folder), permissions='r') + + def test_pending_invited_user_can_receive_permissions_before_setup(self): + folder = self.root / 'invited-work' + folder.mkdir() + self.users[0]['requiredActions'] = ['SETUP', 'UPDATE_PASSWORD'] + state = self.call('grant.save', id=None, subject='user:' + self.subject, path=str(folder), permissions='rw') + self.assertEqual(state['grants'][0]['subject'], 'user:' + self.subject) + self.assertEqual(state['grants'][0]['path'], str(folder)) + self.assertIn('rw: @snow-user-' + self.subject, (self.cfg / 'copyparty.conf').read_text()) + + def test_retarget_preserves_selected_files_and_rejects_missing_selection(self): + destination = self.root / 'destination' + destination.mkdir() + (destination / 'kept.txt').write_text('disposable') + self.link(count=1, files=['kept.txt']) + self.update(destination) + state = self.call('state') + self.assertEqual(state['links'][0]['files'], ['kept.txt']) + self.assertEqual(state['links'][0]['path'], str(destination)) + missing = self.root / 'missing' + missing.mkdir() + with self.assertRaises(shares.Error): + self.update(missing) + self.assertEqual(self.call('state')['links'][0]['path'], str(destination)) + + def test_default_folder_create_and_link_delete_keep_data(self): + result = self.call('link.create', name='Clover - Example', path='', permissions='rw', expires=0, password='', createFolder=True) + path = self.root / 'Share/Clover - Example' + self.assertTrue(path.is_dir()) + self.call('link.delete', key=result['links'][0]['key']) + self.assertTrue(path.is_dir()) + self.assertEqual(self.call('state')['links'], []) + + def test_move_preserves_key_selection_nested_links_and_grants(self): + source = self.root / 'source' + (source / 'nested').mkdir(parents=True) + (source / 'kept.txt').write_text('disposable') + destination = self.root / 'destination' + self.link(count=1, files=['kept.txt']) + self.link(key='nested', path='clover/source/nested') + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(source / 'nested'), permissions='rw') + stop, start = self.offline() + with stop as stopped, start as started: + self.update(destination, mode='move') + stopped.assert_called_once() + started.assert_called_once() + self.assertFalse(source.exists()) + self.assertEqual((destination / 'kept.txt').read_text(), 'disposable') + state = self.call('state') + self.assertEqual({link['key']: link['path'] for link in state['links']}, {'public': str(destination), 'nested': str(destination / 'nested')}) + self.assertEqual(state['grants'][0]['path'], str(destination / 'nested')) + self.assertEqual(next(link for link in state['links'] if link['key'] == 'public')['files'], ['kept.txt']) + + def test_move_rejects_existing_destination_and_ancestor_selection(self): + source = self.root / 'source' + source.mkdir() + destination = self.root / 'destination' + destination.mkdir() + self.link() + with self.assertRaises(shares.Error): + self.update(destination, mode='move') + destination.rmdir() + self.link(key='ancestor', path='clover', count=0) + with self.assertRaises(shares.Error): + self.update(destination, mode='move') + self.assertTrue(source.is_dir()) + self.assertFalse(destination.exists()) + + def test_reload_failure_restores_config_state_database_and_secret(self): + original = (self.cfg / 'copyparty.conf').read_bytes() + calls = 0 + def reload(manager, items): + nonlocal calls + calls += 1 + if calls == 1: + raise shares.Error(502, 'test rejection') + self.reload_mock.side_effect = reload + with self.assertRaises(shares.Error): + self.call('account.create', name='Local AI') + self.reload_mock.side_effect = lambda manager, items, **kwargs: self.reloads.append(copy.deepcopy(items)) + self.assertEqual((self.cfg / 'copyparty.conf').read_bytes(), original) + self.assertEqual(self.call('state')['accounts'], []) + self.assertEqual(self.variable['Items'], {'idp_header': 'a' * 32, 'unrelated': 'retained'}) + self.assertEqual(calls, 2) + destination = self.root / 'target' + destination.mkdir() + self.link() + self.reload_mock.side_effect = [shares.Error(502, 'test rejection'), None] + with self.assertRaises(shares.Error): + self.update(destination) + self.assertEqual(self.call('state')['links'][0]['path'], str(self.root / 'source')) + + def test_failed_offline_restart_restores_folder_and_mappings(self): + source = self.root / 'source' + source.mkdir() + destination = self.root / 'destination' + self.link() + stop, start = self.offline() + with stop as stopped, patch.object(shares.Manager, 'start', autospec=True, side_effect=[shares.Error(502, 'test restart'), None]) as started: + with self.assertRaises(shares.Error): + self.update(destination, mode='move') + self.assertEqual(stopped.call_count, 2) + self.assertEqual(started.call_count, 2) + self.assertTrue(source.is_dir()) + self.assertFalse(destination.exists()) + self.assertEqual(self.call('state')['links'][0]['path'], str(source)) + + def test_new_folders_work_under_host_private_umask(self): + previous = os.umask(0o077) + try: + self.call('link.create', name='Clover - Restricted', path=str(self.root / 'Share/Clover - Restricted'), + permissions='r', expires=0, password='', createFolder=True) + finally: + os.umask(previous) + self.assertEqual((self.root / 'Share').stat().st_mode & 0o777, 0o775) + self.assertEqual((self.root / 'Share/Clover - Restricted').stat().st_mode & 0o777, 0o775) + + def test_api_scope_excludes_wildcards_but_explicit_child_grant_works(self): + account = self.call('account.create', name='Scoped')['account'] + (self.root / 'Public').mkdir() + self.call('grant.save', id=None, subject=account['subject'], path=str(self.root), permissions='r') + config = (self.cfg / 'copyparty.conf').read_text() + public = config.split('[/clover/Public]')[1].split('[/logs]')[0] + self.assertIn('r: *\n', public) + self.assertIn('w: @acct,-' + account['username'], public) + self.assertIn('r: ' + account['username'], public) + logs = config.split('[/logs]')[1] + self.assertIn('g: *,-' + account['username'], logs) + manager = shares.Manager(self.users) + manager.data = json.loads((self.cfg / 'manager.json').read_text()) + manager.items = self.variable['Items'] + manager.owner_permissions(account['username'], self.root / 'Public', 'r') + with self.assertRaises(shares.Error): + manager.owner_permissions(account['username'], self.root / 'Public', 'w') + + def test_admin_can_retarget_other_owner_with_destination_permissions(self): + destination = self.root / 'other' + destination.mkdir() + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(destination), permissions='rw') + self.link(owner='test-user') + self.update(destination) + self.assertEqual(self.call('state')['links'][0]['owner'], 'test-user') + denied = self.root / 'denied' + denied.mkdir() + with self.assertRaises(shares.Error): + self.update(denied) + self.assertEqual(self.call('state')['links'][0]['path'], str(destination)) + + def test_stage_rebases_grants_without_changing_native_virtual_sources(self): + folder = self.root / 'work' + folder.mkdir() + self.call('grant.save', id=None, subject='user:' + self.subject, path=str(folder), permissions='r') + self.link() + clone = self.root.parent / 'clover-clone' + clone.mkdir() + (clone / 'work').mkdir() + with patch.dict(os.environ, {'STUDIO_CLOVER_ROOT': str(clone)}): + state = self.call('state') + self.assertEqual(state['grants'][0]['path'], str(clone / 'work')) + self.assertEqual(state['links'][0]['path'], str(clone / 'source')) + + def test_no_clobber_rename_keeps_both_folders(self): + source = self.root / 'source' + destination = self.root / 'destination' + source.mkdir() + destination.mkdir() + with self.assertRaises(shares.Error): + shares.rename(source, destination) + self.assertTrue(source.is_dir()) + self.assertTrue(destination.is_dir()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/copyparty.md b/tools/copyparty.md new file mode 100644 index 0000000000000000000000000000000000000000..155888960a0f42cb9daa6d140b503851a03e31e6 --- /dev/null +++ b/tools/copyparty.md @@ -0,0 +1,49 @@ +# Copyparty administration and folder cutover + +Snowglobe's admin Shares page manages native public links and separate API +accounts. A person's page assigns folder permissions. Existing Copyparty +permissions remain additive; changing a folder is an explicit retarget or move +action. New shared folders default to `Share/Name - Description` under Clover. + +Copyparty owns its native `cfg/copyparty/shares.db` and selected-file records. +`cfg/manager.json` stores API account labels, UUID-based folder grants, and link +labels. API password hashes live in the service's Nomad variable under +`file_api_`. Restore these together; restoring only the +config file cannot recover managed accounts. `prepare.py` regenerates config +from the canonical service policy and managed state on every deployment. + +## Six-folder production move + +`copyparty-cutover.py` contains the requested six source/destination pairs. +It requires a reviewed plan, absent destinations, and same-filesystem renames. +The plan binds the host, service, roots, source metadata, and complete native +share projection. No file contents are read. Public HEAD responses are captured +before stopping the service and verified after restart; existing keys, +passwords, permissions, expirations, and selected-file records must agree. + +Run the script from the reviewed immutable release on the confirmed target: + +```sh +umask 077 +python3 tools/copyparty-cutover.py --plan /root/copyparty-cutover-plan.json +python3 tools/copyparty-cutover.py --apply --plan /root/copyparty-cutover-plan.json +``` + +Copyparty remains stopped through all folder and database changes. A mismatch +restores the old paths and records before restarting the frozen original job. +Mutation verification has an 80-second cutoff; public readiness has a +150-second forward deadline, leaving recovery time within the 285-second +overall budget. Restricted backups and result evidence live under +`/var/lib/studio/copyparty-cutover`. Code rollback does not undo this file move. + +## 2026-10-16 staging evidence + +Zenith preview `copyparty-preview-9909de81` verified API account scope, +permission changes, password/session revocation, signed UUID identity grants, +symlink confinement, selected-file retention, and nested public links. A +disposable offline move completed in 6.94 seconds. The six requested clone +folders were then renamed with unchanged inode/metadata manifests: all 20 +share records and four selected-file rows matched the expected path-only +projection, and all nine affected public URLs returned their original 200 +responses within a 78.2-second upper bound. Production source metadata and +share records remained unchanged during rehearsal. diff --git a/tools/dashboard-auth-test.py b/tools/dashboard-auth-test.py index d26b74316223dde4884f0c155bc4cc24983cc0b6..9863d8e70fafebdf5180a507b56bd535629dfaf8 100644 --- a/tools/dashboard-auth-test.py +++ b/tools/dashboard-auth-test.py @@ -119,6 +119,7 @@ def main(): assert picture_headers['content-type'] == 'image/png' request('/api/account/pictures/missing', status=404) request('/api/me', status=401, extra={'User-Name': name, 'User-Groups': 'infra-admin'}) + request('/api/shares', status=401, extra={'User-Name': name, 'User-Groups': 'infra-admin'}) request('/auth/status', status=403, extra={'Studio-Proxy-Token': 'wrong'}) csrf = request('/auth/status', cookies=cookies)['csrf'] csrf_headers = request('/auth/status', cookies=cookies, include_headers=True) @@ -142,6 +143,11 @@ def main(): assert 'Max-Age=2592000' in fields['set-cookie'] assert 'admin' in request('/api/me', cookies=cookies)['sections'] request('/api/users', 'POST', {}, cookies, 403, {'Origin': 'https://evil.example'}) + request('/api/shares/accounts', 'POST', {'name': 'Agent'}, cookies, 403, {'Origin': 'https://evil.example'}) + preview_cookies = {**cookies, 'view-as': 'media-manage'} + request('/api/shares', cookies=preview_cookies, status=403) + request('/api/shares/links', 'POST', {}, preview_cookies, 403) + request('/api/users', 'POST', {'profile': {'username': 'API-agent', 'email': '', 'firstName': '', 'lastName': ''}, 'groups': [], 'setup': {'kind': 'invite'}}, cookies, 400) assert request('/auth/password', 'POST', {**login, 'next': '//evil.example'}, cookies)['next'] == '/' # Import's password format must verify with the original Keycloak parameters. with sqlite3.connect(data / 'accounts.sqlite') as db: @@ -196,7 +202,9 @@ def main(): finished = request('/auth/file/callback?' + callback.split('?', 1)[1], cookies=file_cookies, status=302, host=file) assert finished['location'] == file + '/clover/Public/' request('/auth/file/callback?' + callback.split('?', 1)[1], cookies=file_cookies, status=403, host=file) - request('/auth/file/check', cookies=file_cookies, status=204, host=file) + file_identity = request('/auth/file/check', cookies=file_cookies, status=204, host=file) + assert file_identity['x-auth-request-preferred-username'] == name + assert 'snow-user-' + actor in file_identity['x-auth-request-groups'].split(',') request('/auth/file/check', cookies=cookies, status=401, host=file) request('/api/me', cookies=file_cookies, status=401) request('/auth/file/sign-in?rd=https://evil.example/', status=400, host=file) @@ -206,6 +214,8 @@ def main(): request('/auth/setup', 'POST', {'csrf': new_csrf, 'setup': setup, 'email': 'new@example.invalid', 'password': 'another-password'}, newcomer) request('/auth/setup', 'POST', {'csrf': new_csrf, 'setup': setup, 'email': 'new@example.invalid', 'password': 'another-password'}, newcomer, 410) request('/api/users', cookies=newcomer, status=403) + request('/api/shares', cookies=newcomer, status=403) + request('/api/shares/grants', 'POST', {}, newcomer, 403) key = {'publicKey': Path('config/admin.pub').read_text().strip(), 'label': 'identity fixture'} request('/api/account/signing-keys', 'POST', key, newcomer, 204) pending_key = request('/api/account', cookies=newcomer)['commitIdentity']['signingKeys'][0] diff --git a/tools/dashboard-host-test.py b/tools/dashboard-host-test.py index 303ce686da9068bdf0df87aef5102f9001d9bb1c..b643273627b0db20e573fa0eefd5b6b628a31867 100644 --- a/tools/dashboard-host-test.py +++ b/tools/dashboard-host-test.py @@ -78,6 +78,20 @@ class Boundary(unittest.TestCase): self.host.handle(request) command.assert_not_called() + def test_share_management_requires_only_its_declared_context(self): + request = {"operation": "copyparty.manage", "action": "state", "payload": {}, "users": []} + with patch.object(host_module.copyparty, "handle", return_value={"links": []}) as handle: + self.assertEqual(self.host.handle(request), {"links": []}) + handle.assert_called_once_with(request) + for changed in [ + {key: value for key, value in request.items() if key != "users"}, + {**request, "root": "/srv/private"}, + {**request, "env": {"NOMAD_TOKEN": "test"}}, + ]: + with self.assertRaises(host_module.Rejected): + self.host.handle(changed) + self.assertEqual(handle.call_count, 1) + def test_reclaim_is_always_a_dry_run(self): with patch.object(host_module, "command", return_value="reclaim\t1024") as command: self.host.handle({"operation": "storage.reclaim", "dataset": "studio-demo/a", diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index 7523fd85aac9216ccc25fe824760539f7b24592c..ee91e1c4b988199afe3a846489ba698d25e60300 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -2,7 +2,7 @@ import concurrent.futures import contextlib import fcntl -import importlib +import importlib.util import json import math import os @@ -21,6 +21,11 @@ import time import vms import tty +spec = importlib.util.spec_from_file_location("copyparty", Path(__file__).resolve().parent.parent / "service/copyparty/manage.py") +copyparty = importlib.util.module_from_spec(spec) +spec.loader.exec_module(copyparty) + + dashboard_runs = importlib.import_module("dashboard-run") shale_labels = importlib.import_module("shale-labels") @@ -90,6 +95,7 @@ class Host: "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, + "copyparty.manage": {"action", "payload", "users"}, **dashboard_runs.FIELDS, **{"vm." + action: set() if fields is None else {"payload"} for action, fields in vms.ACTION_FIELDS.items()}, } @@ -97,6 +103,8 @@ class Host: raise Rejected("Choose a supported host operation.") if set(request) != parameters[operation] | {"operation"}: raise Rejected("Use only the fields required by this host operation.") + if operation == "copyparty.manage": + return copyparty.handle(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": diff --git a/tools/dashboard-run.py b/tools/dashboard-run.py index 05998255771de00aa81c69114deab046b46efffc..6739267fac253b5e5ebaf557afa1ca74a31dc598 100644 --- a/tools/dashboard-run.py +++ b/tools/dashboard-run.py @@ -74,7 +74,7 @@ def service(target, key=None): raise Error(400, "Choose a secret from this service's list.") -def nomad(path, method="GET", body=None): +def nomad(path, method="GET", body=None, timeout=10): token = read(release.STATE / "nomad.token", missing="").strip() if not token: raise Error(503, "Nomad credentials are unavailable. Check the host service configuration.") @@ -82,7 +82,7 @@ def nomad(path, method="GET", body=None): data=json.dumps(body).encode() if body is not None else None, headers={"X-Nomad-Token": token, "Content-Type": "application/json"}) try: - with urllib.request.urlopen(request, timeout=10) as response: + with urllib.request.urlopen(request, timeout=timeout) as response: data = response.read(MAX_LOG + 1) except urllib.error.HTTPError as error: if error.code == 404 and method == "GET":