From 48149be04c0e2d6f0d219eb70e07ca7f45d824c9 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 20 Oct 2026 23:27:52 -0700 Subject: [PATCH] Restrict The Knife native source behind an account allowlist Gate web, raw, archive, search, object, and Git HTTPS surfaces before proxying. Retain pinned native history after merges, resolve references from one snapshot, and rebuild sensitive diffs from public files. Grant only snow initially. Support SSO and guest accounts through knife-native membership, validate Shale personal tokens against their enabled account, and apply revocation on the next request. Preserve other repositories and CGI Git status handling. Verified Rust and Python checks, Caddy integration, disposable merge redaction, SSO and guest grants, nonmember rejection, revocation, Git discovery/POST, and browser login, keyboard, themes, narrow layout, and text scaling. Assisted-by: gpt-6.1-sol --- config/shale.json | 2 +- dashboard/src/auth.rs | 7 +- dashboard/src/guest.rs | 9 +- dashboard/src/knife-restricted.html | 28 +++++ dashboard/src/shale.rs | 2 +- dashboard/src/shale_page.rs | 134 ++++++++++++++++++++- dashboard/src/users.rs | 12 +- dashboard/web/pages/Users.tsx | 3 +- flake.nix | 1 + nixos/configuration.nix | 5 +- nixos/host-python.nix | 2 + tools/dashboard-host.py | 3 + tools/dashboard-stage.py | 14 ++- tools/release.py | 2 +- tools/router.py | 15 +-- tools/shale-access-test.py | 140 ++++++++++++++++++++++ tools/shale-access.py | 176 ++++++++++++++++++++++++++++ 17 files changed, 521 insertions(+), 34 deletions(-) create mode 100644 dashboard/src/knife-restricted.html create mode 100644 nixos/host-python.nix create mode 100644 tools/shale-access-test.py create mode 100644 tools/shale-access.py diff --git a/config/shale.json b/config/shale.json index 8669e03b5264e833420068132ba418e91f9287cc..a1b954e2576117f2f7a83803615bcb6ebf6f1bc4 100644 --- a/config/shale.json +++ b/config/shale.json @@ -1 +1 @@ -{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]}} +{"actorOwners":{"ai":"8c909fab-98b0-4472-9171-9606ccebe9d5"},"statusDelegates":{"ai":["chat","snowbound"]},"nativeRestriction":{"commits":["8726b8fe5947ae38ad585b0e253feaf92e4f02d5"]}} diff --git a/dashboard/src/auth.rs b/dashboard/src/auth.rs index 938817733489d4269d8891e51e6a3b6ba50cf9f4..7f9b5e0db99e87039835b7043af970ca5900d09c 100644 --- a/dashboard/src/auth.rs +++ b/dashboard/src/auth.rs @@ -12,7 +12,7 @@ const COOKIE: &str = "__Host-snow-session"; const FLOW_COOKIE: &str = "__Host-snow-flow"; const CSRF_COOKIE: &str = "__Host-snow-csrf"; const SESSION_TTL: i64 = 30 * 86400; -const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai"]; +const GROUPS: &[&str] = &["infra-admin", "media", "media-manage", "metrics", "vm", "ai", "knife-native"]; pub struct Store { pub db: Mutex, @@ -201,10 +201,13 @@ impl Store { oidc::initialise(&db)?; guest::initialise(&db)?; for name in GROUPS { - db.execute( + let added = db.execute( "INSERT OR IGNORE INTO roles(id,name) VALUES (?,?)", sql![uuid::Uuid::new_v4().to_string(), name], )?; + if *name == "knife-native" && added != 0 { + db.execute("INSERT OR IGNORE INTO memberships SELECT users.id,roles.id FROM users,roles WHERE users.username='snow' AND roles.name='knife-native'", [])?; + } } Ok(Self { db: Mutex::new(db), diff --git a/dashboard/src/guest.rs b/dashboard/src/guest.rs index cb63a3bee535ad6912300b81b55cdff9853be695..1a2ad837201615e5659007260aa04f1a4f50dead 100644 --- a/dashboard/src/guest.rs +++ b/dashboard/src/guest.rs @@ -22,7 +22,9 @@ pub fn is_guest(user: &Value) -> bool { pub fn initialise(db: &Connection) -> Result<()> { db.execute_batch("CREATE TABLE IF NOT EXISTS guest_providers (id TEXT PRIMARY KEY, client_id TEXT NOT NULL, secret TEXT NOT NULL); CREATE TABLE IF NOT EXISTS external_identities (provider TEXT NOT NULL, subject TEXT NOT NULL, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, PRIMARY KEY(provider,subject), UNIQUE(user_id)); - CREATE TRIGGER IF NOT EXISTS guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts cannot join groups'); END; + DROP TRIGGER IF EXISTS guest_no_groups; + CREATE TRIGGER guest_no_groups BEFORE INSERT ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; + CREATE TRIGGER IF NOT EXISTS guest_no_group_updates BEFORE UPDATE ON memberships WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' AND (SELECT name FROM roles WHERE id=NEW.role_id)!='knife-native' BEGIN SELECT RAISE(ABORT,'guest accounts can only join knife-native'); END; CREATE TRIGGER IF NOT EXISTS guest_no_credentials BEFORE INSERT ON credentials WHEN (SELECT json_extract(profile,'$.kind') FROM users WHERE id=NEW.user_id)='guest' BEGIN SELECT RAISE(ABORT,'guest accounts use external sign-in'); END; CREATE TRIGGER IF NOT EXISTS guest_kind_fixed BEFORE UPDATE OF profile ON users WHEN json_extract(OLD.profile,'$.kind')='guest' AND coalesce(json_extract(NEW.profile,'$.kind'),'')!='guest' BEGIN SELECT RAISE(ABORT,'guest account kind is fixed'); END;")?; Ok(()) @@ -934,7 +936,7 @@ mod tests { } #[test] - fn identities_never_link_by_name_or_email_and_cannot_gain_credentials_or_groups() { + fn identities_stay_separate_and_guests_can_only_join_native_access() { let path = std::env::temp_dir().join(format!("guest-test-{}", uuid::Uuid::new_v4())); let auth = auth::Store::new( &path, @@ -983,6 +985,9 @@ mod tests { db.execute("INSERT INTO memberships VALUES (?,'admin')", [&first]) .is_err() ); + db.execute("INSERT INTO memberships SELECT ?,id FROM roles WHERE name='knife-native'", [&first]).unwrap(); + assert_eq!(auth::user(&db, &first).unwrap()["groups"][0]["name"], "knife-native"); + assert!(db.execute("UPDATE memberships SET role_id='admin' WHERE user_id=?", [&first]).is_err()); assert!(auth::set_password(&db, &first, "a-password-hash").is_err()); assert!( db.execute( diff --git a/dashboard/src/knife-restricted.html b/dashboard/src/knife-restricted.html new file mode 100644 index 0000000000000000000000000000000000000000..3538a53cb93f2c30012a02f65a98f8ec521a0e51 --- /dev/null +++ b/dashboard/src/knife-restricted.html @@ -0,0 +1,28 @@ + + + + +The Knife's Native UI is restricted + +
+

The Knife's Native UI is restricted

+

This part of the code is sensitive. Please contact clover to request private access.

+Login +
+ diff --git a/dashboard/src/shale.rs b/dashboard/src/shale.rs index 3de986d0d5587fdbad631527265f74314b554e15..fe8229e02c67b20737cb3c5ebc8edc2ba1a53832 100644 --- a/dashboard/src/shale.rs +++ b/dashboard/src/shale.rs @@ -487,7 +487,7 @@ fn fields(query: &str) -> Result> { } Ok(fields) } -fn username(page: &str) -> Result { +pub(crate) fn username(page: &str) -> Result { let document = Html::parse_document(page); if document .select(&Selector::parse("body#page-user-settings").unwrap()) diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index f2380243b0af4af3dad703b635eea613dea15e85..d44ed182ffd6f74d809d06a65c553236b465ace6 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -347,6 +347,61 @@ fn strip_hop_headers(headers: &mut HeaderMap) { } } +fn knife_uri(path: &str) -> bool { + let mut decoded = path.to_owned(); + for _ in 0..5 { + if decoded.split(['/', '\\']).any(|part| part.eq_ignore_ascii_case("the-knife") || part.eq_ignore_ascii_case("the-knife.git")) { + return true; + } + let next = percent_encoding::percent_decode_str(&decoded).decode_utf8_lossy().into_owned(); + if next == decoded { break; } + decoded = next; + } + false +} + +fn restricted(authenticate: bool) -> Response { + (if authenticate { StatusCode::UNAUTHORIZED } else { StatusCode::FORBIDDEN }, [ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "private, no-store"), + ("vary", "Cookie, Authorization"), + ("referrer-policy", "same-origin"), + ("x-content-type-options", "nosniff"), + ("content-security-policy", "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'"), + ("www-authenticate", "Basic realm=\"The Knife\""), + ], include_str!("knife-restricted.html")).into_response() +} + +fn redact_knife(html: &str, policy: &Value) -> Result { + let commits: HashSet<&str> = array(&policy["commits"]).iter().filter_map(Value::as_str).collect(); + let document = scraper::Html::parse_document(html); + let diff = policy["safeDiff"].as_str(); + if diff.is_some() && document.select(&scraper::Selector::parse("#m-code").unwrap()).count() != 1 { + return Err(Error::new(403, "The Knife's Native UI is restricted")); + } + let links = scraper::Selector::parse("a[href]").unwrap(); + let rows: Vec = document.select(&scraper::Selector::parse("tr").unwrap()).map(|row| { + row.select(&links).any(|link| link.attr("href").is_some_and(|href| { + href.split(['/', '?', '#', '.']).any(|part| commits.contains(part)) + })) + }).collect(); + let mut row_index = 0; + Ok(lol_html::rewrite_str(html, RewriteStrSettings::new() + .append_element_content_handler(element!("#m-code", |element| { + if let Some(diff) = diff { + let diff = diff.replace('&', "&").replace('<', "<").replace('>', ">"); + element.set_inner_content(&format!("
{diff}
"), ContentType::Html); + } + Ok(()) + })) + .append_element_content_handler(element!("tr", |element| { + if *rows.get(row_index).unwrap_or(&true) { element.remove(); } + row_index += 1; + Ok(()) + })) + )?) +} + pub async fn proxy(app: Arc, request: Request) -> Result { let (mut parts, body) = request.into_parts(); let host = parts @@ -379,6 +434,46 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Err(Error::new(403, "Open Shale to continue.")); } let origin = url::Url::parse(&format!("https://{host}{uri}"))?; + let knife = knife_uri(uri.path()); + let mut target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; + let mut knife_policy = Value::Null; + if knife { + let mut account = String::new(); + if let Some(cookie) = parts.headers.get("cookie") { + let settings = match &app.internal { + Some((base, _)) => format!("{base}services/{service}/-/settings"), + None => format!("http://{upstream}/-/settings"), + }; + let mut identity = app.shale_http.get(settings).header("cookie", cookie).header("accept", "text/html").timeout(Duration::from_secs(3)); + if app.internal.is_none() { identity = identity.header("host", host); } + for name in ["x-forwarded-for", "x-real-ip", "user-agent"] { + if let Some(value) = parts.headers.get(name) { identity = identity.header(name, value); } + } + if let Ok(response) = identity.send().await + && response.status() == StatusCode::OK + && let Ok(body) = response.text().await + { + account = shale::username(&body).unwrap_or_default(); + } + } + knife_policy = match host::call(json!({"operation":"shale.access","service":service,"uri":uri.to_string(),"account":account,"authorization":parts.headers.get("authorization").and_then(|value| value.to_str().ok()).unwrap_or_default()})).await { + Ok(policy) => policy, + Err(_) => return Ok(restricted(false)), + }; + if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } + if let Some(uri) = knife_policy["uri"].as_str() { + target = match &app.internal { + Some((base, _)) => format!("{base}services/{service}{uri}"), + None => format!("http://{upstream}{uri}"), + }; + } + if knife_policy["rawDiff"] == true && let Some(diff) = knife_policy["safeDiff"].as_str() { + return Ok(([("content-type", "text/plain; charset=utf-8"), ("cache-control", "private, no-store"), ("vary", "Cookie, Authorization")], diff.to_owned()).into_response()); + } + } let profiles = profiles(&app.auth)?; if matches!(parts.method, Method::GET | Method::HEAD) && let Some(url) = account_path(uri.path()).and_then(|name| profiles.get(name)).and_then(|profile| profile.url.as_deref()) @@ -393,10 +488,6 @@ pub async fn proxy(app: Arc, request: Request) -> Result { ) .into_response()); } - let target = match &app.internal { - Some((base, _)) => format!("{base}services/{service}{uri}"), - None => format!("http://{upstream}{uri}"), - }; if app.internal.is_some() { parts.headers.remove("host"); } @@ -424,9 +515,17 @@ pub async fn proxy(app: Arc, request: Request) -> Result { .body(reqwest::Body::wrap_stream(body.into_data_stream())) .send() .await?; - let status = upstream_response.status(); + let status = upstream_response.headers().get("status") + .and_then(|value| value.to_str().ok()).and_then(|value| value.split_whitespace().next()) + .and_then(|value| value.parse::().ok()).and_then(|value| StatusCode::from_u16(value).ok()) + .unwrap_or(upstream_response.status()); let mut headers = upstream_response.headers().clone(); strip_hop_headers(&mut headers); + headers.remove("status"); + if knife { + headers.insert("cache-control", "private, no-store".parse().unwrap()); + headers.insert("vary", "Cookie, Authorization".parse().unwrap()); + } let html = parts.method != Method::HEAD && headers .get("content-type") @@ -482,6 +581,14 @@ pub async fn proxy(app: Arc, request: Request) -> Result { profiles.insert(name.clone(), profile); } } + let redacted; + let text = if knife && knife_policy["allowed"] != true { + redacted = match redact_knife(text, &knife_policy) { + Ok(html) => html, + Err(_) => return Ok(restricted(false)), + }; + redacted.as_str() + } else { text }; let rewritten = rewrite(text, &origin, &profiles, &avatars, metadata["openIssues"].as_u64())?; for name in [ "content-length", @@ -498,6 +605,9 @@ pub async fn proxy(app: Arc, request: Request) -> Result { return Ok((status, headers, rewritten).into_response()); } } + if knife && knife_policy["allowed"] != true && (html || parts.method != Method::HEAD && !status.is_redirection() && knife_policy["htmlOnly"] == true) { + return Ok(restricted(false)); + } let stream = futures::stream::once(async move { Ok::<_, reqwest::Error>(Bytes::from(prefix)) }) .chain(upstream_response.bytes_stream()); Ok((status, headers, Body::from_stream(stream)).into_response()) @@ -509,6 +619,20 @@ mod tests { use rusqlite::params as sql; use scraper::{Html, Selector}; + #[test] + fn native_content_is_removed_from_both_diff_styles_and_history_rows() { + let policy = json!({"commits":["deadbeef"],"safeDiff":"PUBLIC SOURCE\nThe Knife's Native UI is restricted\n"}); + let html = "
PRIVATE MESSAGE
PUBLIC ROW
PRIVATE PATHS
knife/private.rs
PRIVATE SOURCE
src/public.rs
PUBLIC SOURCE
nested/Cargo.lock
PRIVATE LOCK
"; + let output = redact_knife(html, &policy).unwrap(); + assert!(!output.contains("PRIVATE"), "{output}"); + assert!(output.contains("PUBLIC ROW"), "{output}"); + assert!(output.contains("PUBLIC SOURCE"), "{output}"); + assert_eq!(output.matches("The Knife's Native UI is restricted").count(), 1); + assert!(redact_knife("
unknown diff markup
", &policy).is_err()); + for path in ["/the-knife/tree/main/knife", "/THE-KNIFE.git/info/refs", "/%74he-knife/plain/main/x", "/%2574he-knife/tree/main/x", "//the-knife/tree/main/x"] { assert!(knife_uri(path), "{path}"); } + assert!(!knife_uri("/snowbound/commit/native")); + } + #[test] fn persisted_atproto_profiles_enrich_shale_authors_and_account_links() { let path = std::env::temp_dir().join(format!("shale-atproto-test-{}", uuid::Uuid::new_v4())); diff --git a/dashboard/src/users.rs b/dashboard/src/users.rs index 61e8d32804b8a61004e6ed5386339d7ad453b981..86d19c7f121dd3f937d4a2fa9a9f435b28d0a5e2 100644 --- a/dashboard/src/users.rs +++ b/dashboard/src/users.rs @@ -158,10 +158,13 @@ pub async fn route( for id in &users { uuid(id)?; let user = auth::user(&transaction, id)?; - if guest::is_guest(&user) || user["kind"] == "ai" { + let guest_groups = if guest::is_guest(&user) { + add.iter().chain(&remove).any(|group| transaction.query_row("SELECT name != 'knife-native' FROM roles WHERE id=?", [group], |r| r.get::<_,bool>(0)).unwrap_or(true)) + } else { false }; + if guest_groups || user["kind"] == "ai" { return Err(Error::new( 400, - "Guests can use Shale only. Select regular accounts to edit groups.", + "Guests can only join knife-native.", )); } for group in &add { @@ -292,7 +295,7 @@ pub async fn route( let transaction = db.transaction()?; let mut user = auth::user(&transaction, id)?; if guest::is_guest(&user) - && matches!(parts, [_, "password"] | [_, "groups", _]) + && matches!(parts, [_, "password"]) && method != Method::GET { return Err(Error::new( @@ -342,6 +345,9 @@ pub async fn route( .optional()?; let name = name .ok_or_else(|| Error::new(404, "This group no longer exists. Reload the page."))?; + if guest::is_guest(&user) && name != "knife-native" { + return Err(Error::new(400, "Guests can only join knife-native.")); + } if own && method == Method::DELETE && name == "infra-admin" { return Err(Error::new( 400, diff --git a/dashboard/web/pages/Users.tsx b/dashboard/web/pages/Users.tsx index 70c6141ab90e4e86057f5716d4183ff976fce7ca..290c67cab986898948353b76e45c5e828a8bc4fd 100644 --- a/dashboard/web/pages/Users.tsx +++ b/dashboard/web/pages/Users.tsx @@ -68,6 +68,7 @@ function adds(group: string, services: ServiceSummary[]) { /** "30 members · opens Jellyfin, Navidrome" */ function groupTip(group: string, d: Data) { + if (group === "knife-native") return `${plural(d.users.filter((user) => names(user).includes(group)).length, "member")} · opens The Knife’s Native UI`; const more = adds(group, d.services); const members = d.users.filter((user) => names(user).includes(group)).length; const metrics = sectionsOf([group]).includes("metrics") ? ["machine metrics"] : []; @@ -575,7 +576,7 @@ function Profile(props: { user: User; data: Data; refetch: () => unknown }) {
{(group) => ( - diff --git a/flake.nix b/flake.nix index 7ee41c2b59a24f7fb4f675bf9bad35eacae1f526..93df49be6c286f21ad03d87042504899e1ba7c91 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ outputs = { self, nixpkgs, ... }: { packages.x86_64-linux.dashboard = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/dashboard.nix { }; + packages.x86_64-linux.host-python = nixpkgs.legacyPackages.x86_64-linux.callPackage ./nixos/host-python.nix { }; packages.x86_64-linux.dashboard-image = self.packages.x86_64-linux.dashboard.image; packages.x86_64-linux.installer = self.nixosConfigurations.installer.config.system.build.isoImage; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index c5ebbbfe95dca0713004bee294f016780214acd1..d308f68053870ceaeb89177893204725ead167f4 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -6,9 +6,10 @@ let proxyToken = "/var/lib/snowglobe/dashboard-proxy.token"; hostTools = lib.fileset.toSource { root = ../.; - fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../tools/namespace-transition.py ../dashboard/src/namespace-transition.json ../guest/protocol.json ]; + fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../service/copyparty/manage.py ../service/copyparty/copyparty.conf ../tools/shale-labels.py ../tools/shale-access.py ../config/shale.json ../config/site.pkl ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/mac-vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../tools/namespace-transition.py ../dashboard/src/namespace-transition.json ../guest/protocol.json ]; }; nativePkl = pkgs.callPackage ./pkl.nix { }; + hostPython = pkgs.callPackage ./host-python.nix { }; screenPython = pkgs.python3.withPackages (packages: [ packages.pygobject3 packages.gst-python ]); screenPlugins = (with pkgs.gst_all_1; [ (lib.getLib gstreamer) gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-plugins-rs ]) ++ [ (lib.getLib pkgs.libnice) ]; secureOvmf = pkgs.OVMF.override { @@ -195,7 +196,7 @@ in SNOWGLOBE_VM_ACCEL = lib.attrByPath [ "SNOWGLOBE_VM_ACCEL" ] "kvm" config.environment.variables; }; serviceConfig = { - ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py"; + ExecStart = "${hostPython}/bin/python3 ${hostTools}/tools/dashboard-host.py"; Type = "notify"; Group = "snowglobe-dashboard"; StateDirectory = "snowglobe/host"; diff --git a/nixos/host-python.nix b/nixos/host-python.nix new file mode 100644 index 0000000000000000000000000000000000000000..5a5b85665df71d08ca9fc280068c36d37d9d7573 --- /dev/null +++ b/nixos/host-python.nix @@ -0,0 +1,2 @@ +{ python3 }: +python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ]) diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index e7625de03a885b88fc791dcb886b8cde0caf678b..f50919b50bb577ea871e2f555c390742cad58e76 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -92,6 +92,7 @@ class Host: "index.diff": {"dataset", "from", "to"}, "index.discard": {"dataset", "snapshot"}, "shale.metadata": {"repository", "accounts"}, + "shale.access": {"service", "uri", "account", "authorization"}, "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, @@ -105,6 +106,8 @@ class Host: raise Rejected("Use only the fields required by this host operation.") if operation == "copyparty.manage": return copyparty.handle(request) + if operation == "shale.access": + return importlib.import_module("shale-access").access(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index 0aed836a0303aa3d0b8cd100909ddf9d1eb0a14b..38137db63f30e420dfda977af2f3a909e9b5500a 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -18,6 +18,7 @@ import snowglobe OPERATIONS = frozenset({ + "shale.access", "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", "storage.snapshots", "vm.node", "vm.domains", "vm.library", "deploy.current", "deploy.main", "deploy.history", "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", @@ -34,6 +35,14 @@ class PreviewHost(boundary.Host): def handle(self, request): operation = request.get("operation") if isinstance(request, dict) else None + if operation == "shale.access": + service = request.get("service", "") + if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): + raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") + linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) + if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: + raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") + return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") if operation in {"files.snapshot", "files.snapshots", "files.discard"}: datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: @@ -159,11 +168,12 @@ def stage(): runtime = "/run/snowglobe-preview/" + identity host_environment = snowglobe.transition.host_environment() host_path = next(value for value in shlex.split(host_environment) if value.startswith("PATH=")) + host_python = snowglobe.command(*nix, "build", "--no-link", "--print-out-paths", "path:" + str(snowglobe.REPO) + "#host-python", capture=True).stdout.strip() snowglobe.transition.stop_preview(identity) snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, "--property=Restart=on-failure", "--property=MemoryMax=256M", "--property=TasksMax=32", - "--setenv=" + host_path, - sys.executable, str(Path(__file__).resolve()), "--serve", identity) + "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], + host_python + "/bin/python3", str(Path(__file__).resolve()), "--serve", identity) for _ in range(100): if Path(runtime, "host.sock").exists(): break diff --git a/tools/release.py b/tools/release.py index 46e41a1d7654bb05caed20e99931f65afa626b57..fc6d8317fedda2044cbcf6e9deb4ecb2064180a5 100644 --- a/tools/release.py +++ b/tools/release.py @@ -63,7 +63,7 @@ def host_digest(root): digest = hashlib.sha256() paths = sorted(path for path, relative in files(root) if relative.parts[0] in {"nixos", "dashboard", "config", "service"} - or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/dashboard-run.py", "tools/release.py", "tools/namespace-transition.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "guest/protocol.json"}) + or str(relative) in {"flake.nix", "flake.lock", "tools/router.py", "tools/dashboard-host.py", "tools/shale-access.py", "tools/shale-labels.py", "tools/dashboard-run.py", "tools/release.py", "tools/namespace-transition.py", "tools/vms.py", "tools/mac-vms.py", "tools/vm-screen.py", "guest/protocol.json"}) for path in paths: digest.update(str(path.relative_to(root)).encode()) _hash_contents(digest, path) diff --git a/tools/router.py b/tools/router.py index dcf4144029a18ce02ab637f63d01b9738c0fe2b4..113adcfb37a50a2b7b3277ac696a21a991be5306 100644 --- a/tools/router.py +++ b/tools/router.py @@ -89,18 +89,6 @@ def shale_write_routes(host): " }", ' respond @shale_unsafe_origin "Forbidden" 403'] -def shale_git_routes(upstreams): - lines = [" @shale_git path */info/refs */git-upload-pack */git-receive-pack", - " handle @shale_git {", f" reverse_proxy {upstreams} {{"] - # Shale forwards CGI Status as a header instead of the HTTP status. - for status in (400, 403, 404, 405, 415, 500): - lines += [f' @cgi_{status} header Status "{status} *"', - f" handle_response @cgi_{status} {{", - " header {", " -Status", " defer", " }", - f" copy_response {status}", " }"] - return [*lines, " }", " }"] - - def shale_login_routes(host): # Shale parses the login Referer as an absolute URL after OAuth. return_url = rf"^https://{re.escape(host)}/(?:[A-Za-z0-9._~!$&'()*+,;=:@/?#\[\]-]|%[0-9A-Fa-f]{{2}})*$" @@ -304,8 +292,7 @@ def render(token): # Keep the Origin guard before every static/proxy handle; # otherwise Caddy sorts those handles ahead of respond. lines += [" route {", *shale_write_routes(host), - *shale_login_routes(host), - *shale_git_routes(" ".join(sorted(route["upstreams"])))] + *shale_login_routes(host)] if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])), "/srv/staging/" + service) diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py new file mode 100644 index 0000000000000000000000000000000000000000..cc1414670351ed9cd5cf3896a03dd8b8db82bb83 --- /dev/null +++ b/tools/shale-access-test.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +import base64 +import importlib +import json +import os +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +import pygit2 +from argon2 import PasswordHasher + +shale = importlib.import_module('shale-access') + + +class Access(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'}) + self.env.start() + self.addCleanup(self.env.stop) + (self.root / 'data').mkdir() + self.uuid = '0' * 26 + self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) + self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) + self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) + self.repo.create_reference('refs/heads/native', self.native) + self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native]) + self.repo.create_reference('refs/heads/main', self.public) + self.repo.set_head('refs/heads/main') + (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) + with sqlite3.connect(self.root / 'accounts.sqlite') as db: + db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') + for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): + db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) + if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') + db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) + for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): + db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name)) + db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret'))) + + def commit(self, files, parents=None): + index = pygit2.Index() + for path, content in files.items(): + index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB)) + tree = index.write_tree(self.repo) + author = pygit2.Signature('Fixture', 'fixture@example.invalid') + return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or []) + + def access(self, path, account='', authorization=''): + return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) + + def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): + public = self.access('/the-knife/tree/main/') + self.assertFalse(public['deny']) + self.assertIn(str(self.public), public['uri']) + for rev in ('native', str(self.native), str(self.native)[:12]): + for route in ('tree', 'plain', 'commit'): + self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev)) + self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny']) + self.repo.references.delete('refs/heads/native') + self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny']) + + def test_every_revision_blocks_files_even_when_absent(self): + for rev in ('main', str(self.base)): + for route in ('tree', 'plain'): + for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'): + self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path)) + self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny']) + + def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self): + result = self.access('/the-knife/commit/' + str(self.public)) + self.assertFalse(result['deny']) + self.assertNotIn('SECRET', result['safeDiff']) + self.assertIn('PUBLIC', result['safeDiff']) + for suffix in ('diff', 'patch'): + result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix) + self.assertFalse(result['deny']) + self.assertNotIn('SECRET', result['safeDiff']) + self.assertIn('PUBLIC', result['safeDiff']) + for suffix in ('stat', 'diffraw'): + self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny']) + + def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self): + path = '/the-knife/plain/native/knife/private.rs' + for name in ('snow', 'guest'): + self.assertTrue(self.access(path, name)['allowed']) + for name in ('stranger', 'disabled', 'ai', ''): + self.assertTrue(self.access(path, name)['deny']) + with sqlite3.connect(self.root / 'accounts.sqlite') as db: + db.execute("DELETE FROM memberships WHERE user_id='snow'") + self.assertTrue(self.access(path, 'snow')['deny']) + + def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): + path = '/the-knife/info/refs?service=git-upload-pack' + self.assertTrue(self.access(path, 'snow')['deny']) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access(path, authorization=token)['allowed']) + self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.execute('DELETE FROM personal_tokens') + self.assertTrue(self.access(path, authorization=token)['deny']) + + def test_native_push_during_classification_cannot_change_the_checked_revision(self): + newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) + repo = self.repo + class ConcurrentPush: + def __getattr__(self, name): return getattr(repo, name) + def __getitem__(self, key): return repo[key] + def walk(self, *args): + repo.lookup_reference('refs/heads/native').set_target(newer) + repo.create_reference('refs/heads/deadbeef', newer, force=True) + return repo.walk(*args) + for revision in ('native', 'deadbeef'): + repo.lookup_reference('refs/heads/native').set_target(self.native) + if 'refs/heads/deadbeef' in repo.references: + repo.references.delete('refs/heads/deadbeef') + with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()): + self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) + + def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): + with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") + self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny']) + + def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self): + for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'): + self.assertTrue(self.access(path)['deny'], path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/shale-access.py b/tools/shale-access.py new file mode 100644 index 0000000000000000000000000000000000000000..f5d141a17d056ac0f2659534f7c9aba2f5ed82f1 --- /dev/null +++ b/tools/shale-access.py @@ -0,0 +1,176 @@ +import base64 +from contextlib import closing +import importlib +import json +import os +from pathlib import Path +import posixpath +import re +import sqlite3 +import subprocess +from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit + +import pygit2 +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError + +Error = importlib.import_module('dashboard-run').Error +# The root host helper reads repositories owned by the Shale service UID. +pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) + + +def restricted_path(path): + parts = path.split('/') + return parts[0] == 'knife' or 'Cargo.lock' in parts + + +def allowed_account(db, user_id): + row = db.execute("SELECT a.profile FROM users u JOIN auth.users a ON a.id=u.snowflake " + "WHERE u.id=? AND u.provider=? AND EXISTS(SELECT 1 FROM auth.memberships m " + "JOIN auth.roles r ON r.id=m.role_id WHERE m.user_id=a.id AND r.name='knife-native')", + (user_id, 'snowglobe.' + os.environ['SNOWGLOBE_DOMAIN'])).fetchone() + if not row: + return False + profile = json.loads(row[0]) + return (profile.get('enabled') is True and not profile.get('requiredActions') + and profile.get('kind') != 'ai') + + +def access(request, accounts=None): + service = request['service'] + if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): + raise Error(400, 'Choose a Shale instance.') + if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): + root = Path(fixture_root) + else: + site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) + root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) + uri = urlsplit(request['uri']) + path = uri.path + for _ in range(4): + decoded = unquote(path, errors='strict') + if decoded == path: + break + path = decoded + path = posixpath.normpath('/' + path.lstrip('/')) + parts = path.strip('/').split('/') + if parts[0].lower() not in ('the-knife', 'the-knife.git'): + return {'allowed': False, 'deny': True} + git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') + with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) + db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) + authorization = request['authorization'] + allowed = False + if authorization: + try: + scheme, value = authorization.split(' ', 1) + if scheme.lower() == 'basic': + name, secret = base64.b64decode(value, validate=True).decode().split(':', 1) + users = db.execute('SELECT id FROM users WHERE name=?', (name,)).fetchall() + if len(users) == 1 and allowed_account(db, users[0][0]): + for (hashed,) in db.execute('SELECT secret FROM personal_tokens WHERE agent=?', users[0]): + try: + if PasswordHasher().verify(hashed, secret): + allowed = True + break + except (VerificationError, InvalidHashError): + pass + except (ValueError, UnicodeError): + pass + elif request['account'] and not git: + users = db.execute('SELECT id FROM users WHERE name=?', (request['account'],)).fetchall() + allowed = len(users) == 1 and allowed_account(db, users[0][0]) + if allowed: + return {'allowed': True, 'deny': False} + row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() + if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): + return {'allowed': False, 'deny': True} + repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) + policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] + head = repo.lookup_reference('HEAD').target + references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} + protected = set() + tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] + if 'refs/heads/native' in references: + tips.append(references['refs/heads/native']) + for tip in tips: + protected.update(str(commit.id) for commit in repo.walk(tip)) + result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} + if parts[0] != 'the-knife' or '%' in path or '\\' in path: + return result + route = parts[1] if len(parts) > 1 else '' + query = dict(parse_qsl(uri.query, keep_blank_values=True)) + if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): + return result + if route in ('', 'issues'): + if route == '' and query: + return result + result['htmlOnly'] = route != 'issues' + result['deny'] = False + return result + if route in ('heads', 'tags') and len(parts) == 2 and not query: + result['deny'] = False + return result + if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): + return result + if route not in ('log', 'tree', 'plain', 'commit'): + return result + revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' + suffix = '' + if route == 'commit': + revision, dot, suffix = revision.partition('.') + if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): + return result + if not revision or revision == '-': + revision = 'HEAD' + # Only named refs and object IDs have the same meaning in Shale and libgit2. + if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): + return result + try: + if revision == 'HEAD': + oid = references.get(head) if isinstance(head, str) else head + else: + candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} + if len(candidates) > 1: + return result + if candidates: + oid = candidates.pop() + elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): + oid = repo[revision].id + else: + return result + if oid is None: + return result + commit = repo[oid].peel(pygit2.Commit) + except (KeyError, ValueError, pygit2.GitError): + return result + if str(commit.id) in protected: + return result + if route in ('tree', 'plain'): + result['htmlOnly'] = route != 'plain' + file = '/'.join(parts[3:]) + if restricted_path(file): + return result + if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): + return result + if route == 'commit': + diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] + sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) + if sensitive: + if suffix in ('stat', 'diffraw'): + return result + result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" + if suffix: + result['rawDiff'] = True + result['deny'] = False + return result + if route == 'log': + result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) + else: + parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] + result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') + if query: + result['uri'] += '?' + urlencode(query) + result['deny'] = False + return result -- 2.54.1