From 2b3ff5841b47959fe8b22c78617bd88558c0ad1a Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 24 Oct 2026 21:36:19 -0700 Subject: [PATCH] Serve anonymous HTTPS clones of The Knife main branch Anonymous upload-pack requests get a scratch repository whose only ref is main, borrowing the real objects through alternates and pinned to protocol v0, which only serves objects reachable from advertised refs. Clones are refused while main reaches native history. Token requests keep the full repository, so owners fetch with http.proactiveAuth=basic. Assisted-by: claude-opus-5.5 --- dashboard/src/shale_page.rs | 45 +++++++++++++++ nixos/host-python.nix | 8 ++- tools/dashboard-host.py | 27 +++++++++ tools/dashboard-stage.py | 6 +- tools/shale-access-test.py | 76 ++++++++++++++++++++++++- tools/shale-access.py | 107 ++++++++++++++++++++++++++++++------ 6 files changed, 246 insertions(+), 23 deletions(-) diff --git a/dashboard/src/shale_page.rs b/dashboard/src/shale_page.rs index ee043bb5274e2c7daf53a145abde57f0216f7c7d..e34b96ecd2d18058431511bc9c826019bdef9bf5 100644 --- a/dashboard/src/shale_page.rs +++ b/dashboard/src/shale_page.rs @@ -412,6 +412,46 @@ fn redact_knife(html: &str, policy: &Value) -> Result { )?) } +/// Relays git http-backend's CGI response for an anonymous clone of The Knife's main branch. +async fn clone(request: Value, body: Body) -> Result { + use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; + let (_, stream) = host::open(request).await?; + let (reader, mut writer) = stream.into_split(); + tokio::spawn(async move { + let mut body = body.into_data_stream(); + while let Some(Ok(chunk)) = body.next().await { + if writer.write_all(&chunk).await.is_err() { return; } + } + let _ = writer.shutdown().await; + }); + let mut reader = tokio::io::BufReader::new(reader); + let mut response = Response::builder(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).await? == 0 { + return Err(Error::new(502, "The Knife clone ended early.")); + } + let line = line.trim_end(); + if line.is_empty() { break; } + let (name, value) = line.split_once(':').ok_or_else(|| Error::new(502, "The Knife clone response is malformed."))?; + let value = value.trim(); + if name.eq_ignore_ascii_case("status") { + response = response.status(value.split_whitespace().next().unwrap_or_default().parse::()?); + } else { + response = response.header(name, value); + } + } + let stream = futures::stream::unfold(reader, |mut reader| async move { + let mut buffer = vec![0; 64 * 1024]; + match reader.read(&mut buffer).await { + Ok(0) => None, + Ok(length) => { buffer.truncate(length); Some((Ok(Bytes::from(buffer)), reader)) } + Err(error) => Some((Err(error), reader)), + } + }); + Ok(response.body(Body::from_stream(stream))?) +} + pub async fn proxy(app: Arc, request: Request) -> Result { let (mut parts, body) = request.into_parts(); let host = parts @@ -475,6 +515,11 @@ pub async fn proxy(app: Arc, request: Request) -> Result { Err(_) => return Ok(restricted(false)), }; if knife_policy["deny"] != false { return Ok(restricted(knife_policy["authenticate"] == true)); } + if knife_policy["clone"] == true { + let header = |name| parts.headers.get(name).and_then(|value| value.to_str().ok()).unwrap_or_default(); + let request = json!({"operation":"shale.clone","service":service,"uri":uri.to_string(),"method":parts.method.as_str(),"contentType":header("content-type"),"contentEncoding":header("content-encoding")}); + return Ok(clone(request, body).await.unwrap_or_else(|_| restricted(true))); + } if let Some(uri) = knife_policy["uri"].as_str() { target = match &app.internal { Some((base, _)) => format!("{base}services/{service}{uri}"), diff --git a/nixos/host-python.nix b/nixos/host-python.nix index 5a5b85665df71d08ca9fc280068c36d37d9d7573..a9d47aee38f4142cd6d623f45c50c98f52b355c6 100644 --- a/nixos/host-python.nix +++ b/nixos/host-python.nix @@ -1,2 +1,6 @@ -{ python3 }: -python3.withPackages (packages: [ packages.pygit2 packages.argon2-cffi ]) +{ python3, git }: +python3.buildEnv.override { + extraLibs = with python3.pkgs; [ pygit2 argon2-cffi ]; + # shale-access serves anonymous Knife clones through git http-backend. + makeWrapperArgs = [ "--prefix" "PATH" ":" "${git}/bin" ]; +} diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index 1cdaeaa660c5b4616f966e65d851bb7bdfa945a4..4c47f424292dde11e7c3ed8054176d0699469405 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -16,6 +16,7 @@ import signal import socket import struct import subprocess +import sys import threading import time import vms @@ -41,6 +42,7 @@ MAX_REQUEST = PROTOCOL['request'] MAX_RESPONSE = PROTOCOL['response'] STREAM_SLOTS = threading.BoundedSemaphore(4) GUEST_SLOTS = threading.BoundedSemaphore(4) +CLONE_SLOTS = threading.BoundedSemaphore(2) VM_PREPARATION_SLOT = threading.BoundedSemaphore(1) INDEX_SNAPSHOT = r"index-[0-9]+(?:-[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})?" @@ -93,6 +95,7 @@ class Host: "index.discard": {"dataset", "snapshot"}, "shale.metadata": {"repository", "accounts"}, "shale.access": {"service", "uri", "account", "authorization"}, + "shale.clone": {"service", "uri", "method", "contentType", "contentEncoding"}, "shale.write": {"grant", "repository", "action", "arguments"}, "host.sample": set(), "host.usage": {"refresh"}, @@ -109,6 +112,8 @@ class Host: return copyparty.handle(request) if operation == "shale.access": return importlib.import_module("shale-access").access(request) + if operation == "shale.clone": + return importlib.import_module("shale-access").clone(request) if operation == "shale.metadata": return shale_labels.metadata(request) if operation == "shale.write": @@ -463,6 +468,20 @@ def stream_upload(connection, request): Path(temporary).unlink(missing_ok=True) +def stream_clone(connection, plan): + owner = os.stat(plan["objects"]) + # Service identities have numeric owners without passwd entries, which systemd's User= rejects. + command = ["systemd-run", "--pipe", "--quiet", "--collect", "--service-type=exec", + *("--property=" + setting for setting in ("PrivateTmp=yes", "PrivateNetwork=yes", "PrivateDevices=yes", "ProtectSystem=strict", "ProtectHome=yes", + "NoNewPrivileges=yes", "CapabilityBoundingSet=CAP_SETUID CAP_SETGID", "MemoryMax=2G", "TasksMax=128", "RuntimeMaxSec=900")), + *(f"--setenv={name}={value}" for name, value in plan["environment"].items()), + "--", "setpriv", f"--reuid={owner.st_uid}", f"--regid={owner.st_gid}", "--clear-groups", "--", + sys.executable, importlib.import_module("shale-access").__file__, plan["objects"], plan["main"]] + send_response(connection, {"value": None}) + connection.setblocking(True) + subprocess.run(command, stdin=connection, stdout=connection) + + def serve_connection(connection, host, allowed_uid, allowed_operations=None): with connection: connection.settimeout(65) @@ -501,6 +520,14 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): return finally: streams.release() + if isinstance(request, dict) and request.get("operation") == "shale.clone": + if not CLONE_SLOTS.acquire(blocking=False): + raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") + try: + stream_clone(connection, host.handle(request)) + return + finally: + CLONE_SLOTS.release() response = {"value": host.handle(request)} except Rejected as error: response = {"error": str(error), "status": 400} diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index 1eb47628058165c12a99c1a3270676bcdb97e5e2..fea28b22bf53fec95b68f23b9b5cc75932ba621d 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -20,7 +20,7 @@ import snowglobe OPERATIONS = frozenset({ - "shale.access", + "shale.access", "shale.clone", "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", @@ -55,13 +55,15 @@ class PreviewHost(boundary.Host): actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") - if operation == "shale.access": + if operation in {"shale.access", "shale.clone"}: service = request.get("service", "") if not re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): raise boundary.dashboard_runs.Error(403, "Use a Shale preview with this dashboard.") linked = json.loads((snowglobe.STATE / "stages" / (service + ".json")).read_text()) if linked.get("sourceId") != "shale" or linked.get("gatewayStage") != Path(self.stage["mount"]).name: raise boundary.dashboard_runs.Error(403, "Use the Shale preview linked to this dashboard.") + if operation == "shale.clone": + return import_module("shale-access").clone(request) return import_module("shale-access").access(request, accounts=Path(self.stage["mount"]) / "data") if operation in {"files.snapshot", "files.snapshots", "files.discard"}: datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py index b9d7b9d5c5f0adfa5135473b075c330c80791f6a..d32ca65892524367159722b3e8a2e8ce7b571b32 100644 --- a/tools/shale-access-test.py +++ b/tools/shale-access-test.py @@ -5,8 +5,12 @@ import importlib import json import os from pathlib import Path +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import sqlite3 +import subprocess +import sys import tempfile +import threading import unittest from unittest.mock import patch @@ -16,6 +20,41 @@ from argon2 import PasswordHasher shale = importlib.import_module('shale-access') +class CGI(BaseHTTPRequestHandler): + """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" + def __init__(self, test, *args): + self.test = test + super().__init__(*args) + + def log_message(self, *args): + pass + + def do_GET(self): + self.relay() + + def do_POST(self): + self.relay() + + def relay(self): + try: + plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) + except shale.Error: + self.send_response(401) + self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') + self.end_headers() + return + body = self.rfile.read(int(self.headers.get('content-length', 0))) + output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, + env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout + head, _, payload = output.partition(b'\r\n\r\n') + headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) + self.send_response(int(headers.pop('Status', '200').split()[0])) + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(payload) + + class Access(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -114,8 +153,9 @@ class Access(unittest.TestCase): self.assertTrue(self.access(path, 'snow')['deny']) def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): - path = '/the-knife/info/refs?service=git-upload-pack' + path = '/the-knife/info/refs?service=git-receive-pack' self.assertTrue(self.access(path, 'snow')['deny']) + self.assertTrue(self.access(path)['authenticate']) token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() self.assertTrue(self.access(path, authorization=token)['allowed']) self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) @@ -123,6 +163,40 @@ class Access(unittest.TestCase): db.execute('DELETE FROM personal_tokens') self.assertTrue(self.access(path, authorization=token)['deny']) + def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): + for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): + self.assertTrue(self.access(path)['clone'], path) + token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() + self.assertTrue(self.access(path, authorization=token)['allowed'], path) + for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): + self.assertNotIn('clone', self.access(path), path) + + def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): + with self.assertRaises(shale.Error): + self.clone('/the-knife.git/info/refs?service=git-upload-pack') + self.repo.lookup_reference('refs/heads/main').set_target(self.base) + self.repo.create_reference('refs/heads/merge', self.public) + server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) + self.addCleanup(server.server_close) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.shutdown) + url = f'http://127.0.0.1:{server.server_port}/the-knife' + environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} + for version in ('0', '1', '2'): + checkout = self.root / ('clone-v' + version) + subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) + refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout + self.assertEqual(refs, f'refs/heads/main {self.base}\n') + for revision in (str(self.native), str(self.public)): + fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) + self.assertNotEqual(fetched.returncode, 0, (version, revision)) + self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) + pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) + self.assertNotEqual(pushed.returncode, 0) + + def clone(self, uri, method='GET', content_type='', content_encoding=''): + return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) + def test_native_push_during_classification_cannot_change_the_checked_revision(self): newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) repo = self.repo diff --git a/tools/shale-access.py b/tools/shale-access.py index 6e8d57fbc8f45131b80ca9adca4a951a59bac106..c1d88779af251ce2bc2192789abfe257b9a5b803 100644 --- a/tools/shale-access.py +++ b/tools/shale-access.py @@ -8,6 +8,8 @@ import posixpath import re import sqlite3 import subprocess +import sys +import tempfile from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit import pygit2 @@ -36,16 +38,17 @@ def allowed_account(db, user_id): and profile.get('kind') != 'ai') -def access(request, accounts=None): - service = request['service'] +def instance(service): if service != 'shale' and not re.fullmatch(r'shale-preview-[0-9a-f]{8}', service): raise Error(400, 'Choose a Shale instance.') if fixture_root := os.environ.get('SNOWGLOBE_SHALE_ROOT'): - root = Path(fixture_root) - else: - site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) - root = Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) - uri = urlsplit(request['uri']) + return Path(fixture_root) + site = json.loads(subprocess.check_output(['pkl', 'eval', '-f', 'json', str(Path(__file__).resolve().parent.parent / 'config/site.pkl')], text=True, timeout=5)) + return Path(site['root']) / ('prod/shale' if service == 'shale' else 'staging/' + service) + + +def target(request_uri): + uri = urlsplit(request_uri) path = uri.path for _ in range(4): decoded = unquote(path, errors='strict') @@ -53,10 +56,39 @@ def access(request, accounts=None): break path = decoded path = posixpath.normpath('/' + path.lstrip('/')) - parts = path.strip('/').split('/') + return uri, path, path.strip('/').split('/') + + +def upload_pack(uri, path, parts): + """Returns the CGI path of an anonymous fetch request, the only Git route served without a token.""" + if parts[0] not in ('the-knife', 'the-knife.git') or '%' in path or '\\' in path: + return None + if parts[1:] == ['info', 'refs'] and uri.query == 'service=git-upload-pack' or parts[1:] == ['git-upload-pack'] and not uri.query: + return '/the-knife.git/' + '/'.join(parts[1:]) + return None + + +def repository(root, db): + row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() + return root / 'repositories_owned' / row[0] if row and re.fullmatch(r'[A-Z0-9]{26}', row[0]) else None + + +def protected_commits(repo, native): + policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] + tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] + if native: + tips.append(native) + return {str(commit.id) for tip in tips for commit in repo.walk(tip)} + + +def access(request, accounts=None): + root = instance(request['service']) + uri, path, parts = target(request['uri']) if parts[0].lower() not in ('the-knife', 'the-knife.git'): return {'allowed': False, 'deny': True} git = parts[0].lower().endswith('.git') or len(parts) > 1 and parts[1] in ('info', 'objects', 'git-upload-pack', 'git-receive-pack', 'HEAD') + if not request['authorization'] and upload_pack(uri, path, parts): + return {'allowed': False, 'deny': False, 'clone': True} with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: accounts = Path(accounts or os.environ.get('SNOWGLOBE_DASHBOARD_DATA', '/var/lib/snowglobe/dashboard')) db.execute('ATTACH DATABASE ? AS auth', ((accounts / 'accounts.sqlite').as_uri() + '?mode=ro',)) @@ -83,20 +115,14 @@ def access(request, accounts=None): allowed = len(users) == 1 and allowed_account(db, users[0][0]) if allowed: return {'allowed': True, 'deny': False} - row = db.execute("SELECT uuid FROM repositories WHERE name='the-knife'").fetchone() - if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): + location = repository(root, db) + if not location: return {'allowed': False, 'deny': True} - repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) + repo = pygit2.Repository(str(location)) try: - policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] head = repo.lookup_reference('HEAD').target references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} - protected = set() - tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] - if 'refs/heads/native' in references: - tips.append(references['refs/heads/native']) - for tip in tips: - protected.update(str(commit.id) for commit in repo.walk(tip)) + protected = protected_commits(repo, references.get('refs/heads/native')) result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} if parts[0] != 'the-knife' or '%' in path or '\\' in path: return result @@ -177,3 +203,48 @@ def access(request, accounts=None): return result finally: repo.free() + + +def clone(request): + root = instance(request['service']) + uri, path, parts = target(request['uri']) + location = upload_pack(uri, path, parts) + if not location: + raise Error(404, 'Clone The Knife over HTTPS.') + with closing(sqlite3.connect((root / 'data/astheno.shale.db').as_uri() + '?mode=ro', uri=True, timeout=2)) as db: + knife = repository(root, db) + if not knife: + raise Error(404, 'The Knife is unavailable.') + repo = pygit2.Repository(str(knife)) + try: + main = repo.lookup_reference('refs/heads/main').resolve().target + native = repo.lookup_reference('refs/heads/native').resolve().target if 'refs/heads/native' in repo.references else None + protected = protected_commits(repo, native) + if any(str(commit.id) in protected for commit in repo.walk(main)): + raise Error(403, "The Knife's main branch includes native history.") + finally: + repo.free() + return {'objects': str(knife / 'objects'), 'main': str(main), 'environment': { + 'REQUEST_METHOD': request['method'], 'PATH_INFO': location, 'QUERY_STRING': uri.query, + 'CONTENT_TYPE': request['contentType'], 'HTTP_CONTENT_ENCODING': request['contentEncoding'], + }} + + +def serve(objects, main): + """Runs git http-backend on a repository whose only ref is main, borrowing the Knife's objects.""" + repository = Path(tempfile.mkdtemp()) / 'the-knife.git' + (repository / 'objects/info').mkdir(parents=True) + (repository / 'refs/heads').mkdir(parents=True) + (repository / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = true\n') + (repository / 'HEAD').write_text('ref: refs/heads/main\n') + (repository / 'refs/heads/main').write_text(main + '\n') + (repository / 'objects/info/alternates').write_text(objects + '\n') + # Protocol v2 serves any object the alternate holds; omitting GIT_PROTOCOL keeps v0's check that wants are reachable from main. + environment = {name: os.environ[name] for name in ('PATH', 'REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING', 'CONTENT_TYPE', 'HTTP_CONTENT_ENCODING') if os.environ.get(name)} + os.execvpe('git', ['git', 'http-backend'], environment | { + 'GIT_PROJECT_ROOT': str(repository.parent), 'GIT_HTTP_EXPORT_ALL': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', + }) + + +if __name__ == '__main__': + serve(*sys.argv[1:]) -- 2.54.1