| 1 | #!/usr/bin/env python3 |
| 2 | import base64 |
| 3 | from contextlib import closing |
| 4 | import importlib |
| 5 | import json |
| 6 | import os |
| 7 | from pathlib import Path |
| 8 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 9 | import sqlite3 |
| 10 | import subprocess |
| 11 | import sys |
| 12 | import tempfile |
| 13 | import threading |
| 14 | import unittest |
| 15 | from unittest.mock import patch |
| 16 | |
| 17 | import pygit2 |
| 18 | from argon2 import PasswordHasher |
| 19 | |
| 20 | shale = importlib.import_module('shale-access') |
| 21 | |
| 22 | |
| 23 | class CGI(BaseHTTPRequestHandler): |
| 24 | """Plays the Shale gateway: every request passes through clone() before reaching git http-backend.""" |
| 25 | def __init__(self, test, *args): |
| 26 | self.test = test |
| 27 | super().__init__(*args) |
| 28 | |
| 29 | def log_message(self, *args): |
| 30 | pass |
| 31 | |
| 32 | def do_GET(self): |
| 33 | self.relay() |
| 34 | |
| 35 | def do_POST(self): |
| 36 | self.relay() |
| 37 | |
| 38 | def relay(self): |
| 39 | try: |
| 40 | plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', '')) |
| 41 | except shale.Error: |
| 42 | self.send_response(401) |
| 43 | self.send_header('WWW-Authenticate', 'Basic realm="The Knife"') |
| 44 | self.end_headers() |
| 45 | return |
| 46 | body = self.rfile.read(int(self.headers.get('content-length', 0))) |
| 47 | output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True, |
| 48 | env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout |
| 49 | head, _, payload = output.partition(b'\r\n\r\n') |
| 50 | headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n')) |
| 51 | self.send_response(int(headers.pop('Status', '200').split()[0])) |
| 52 | for name, value in headers.items(): |
| 53 | self.send_header(name, value) |
| 54 | self.end_headers() |
| 55 | self.wfile.write(payload) |
| 56 | |
| 57 | |
| 58 | class Access(unittest.TestCase): |
| 59 | def setUp(self): |
| 60 | self.temp = tempfile.TemporaryDirectory() |
| 61 | self.addCleanup(self.temp.cleanup) |
| 62 | self.root = Path(self.temp.name) |
| 63 | self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'}) |
| 64 | self.env.start() |
| 65 | self.addCleanup(self.env.stop) |
| 66 | (self.root / 'data').mkdir() |
| 67 | self.uuid = '0' * 26 |
| 68 | self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) |
| 69 | self.addCleanup(self.repo.free) |
| 70 | self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) |
| 71 | self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) |
| 72 | self.repo.create_reference('refs/heads/native', self.native) |
| 73 | self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native]) |
| 74 | self.repo.create_reference('refs/heads/main', self.public) |
| 75 | self.repo.set_head('refs/heads/main') |
| 76 | (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) |
| 77 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: |
| 78 | db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') |
| 79 | for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): |
| 80 | db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) |
| 81 | if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) |
| 82 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 83 | db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') |
| 84 | db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) |
| 85 | for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): |
| 86 | db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name)) |
| 87 | db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret'))) |
| 88 | |
| 89 | def commit(self, files, parents=None): |
| 90 | index = pygit2.Index() |
| 91 | for path, content in files.items(): |
| 92 | index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB)) |
| 93 | tree = index.write_tree(self.repo) |
| 94 | author = pygit2.Signature('Fixture', 'fixture@example.invalid') |
| 95 | return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or []) |
| 96 | |
| 97 | def access(self, path, account='', authorization=''): |
| 98 | return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) |
| 99 | |
| 100 | def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self): |
| 101 | released = [] |
| 102 | class TrackedRepository(pygit2.Repository): |
| 103 | def free(self): |
| 104 | released.append(True) |
| 105 | super().free() |
| 106 | with patch.object(shale.pygit2, 'Repository', TrackedRepository): |
| 107 | self.assertFalse(self.access('/the-knife/')['deny']) |
| 108 | self.assertTrue(self.access('/the-knife/objects/x')['deny']) |
| 109 | with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')): |
| 110 | with self.assertRaises(ValueError): |
| 111 | self.access('/the-knife/') |
| 112 | self.assertEqual(len(released), 3) |
| 113 | |
| 114 | def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): |
| 115 | public = self.access('/the-knife/tree/main/') |
| 116 | self.assertFalse(public['deny']) |
| 117 | self.assertIn(str(self.public), public['uri']) |
| 118 | for rev in ('native', str(self.native), str(self.native)[:12]): |
| 119 | for route in ('tree', 'plain', 'commit'): |
| 120 | self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev)) |
| 121 | self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny']) |
| 122 | self.repo.references.delete('refs/heads/native') |
| 123 | self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny']) |
| 124 | |
| 125 | def test_every_revision_blocks_files_even_when_absent(self): |
| 126 | for rev in ('main', str(self.base)): |
| 127 | for route in ('tree', 'plain'): |
| 128 | for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'): |
| 129 | self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path)) |
| 130 | self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny']) |
| 131 | |
| 132 | def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self): |
| 133 | result = self.access('/the-knife/commit/' + str(self.public)) |
| 134 | self.assertFalse(result['deny']) |
| 135 | self.assertNotIn('SECRET', result['safeDiff']) |
| 136 | self.assertIn('PUBLIC', result['safeDiff']) |
| 137 | for suffix in ('diff', 'patch'): |
| 138 | result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix) |
| 139 | self.assertFalse(result['deny']) |
| 140 | self.assertNotIn('SECRET', result['safeDiff']) |
| 141 | self.assertIn('PUBLIC', result['safeDiff']) |
| 142 | for suffix in ('stat', 'diffraw'): |
| 143 | self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny']) |
| 144 | |
| 145 | def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self): |
| 146 | path = '/the-knife/plain/native/knife/private.rs' |
| 147 | for name in ('snow', 'guest'): |
| 148 | self.assertTrue(self.access(path, name)['allowed']) |
| 149 | for name in ('stranger', 'disabled', 'ai', ''): |
| 150 | self.assertTrue(self.access(path, name)['deny']) |
| 151 | with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: |
| 152 | db.execute("DELETE FROM memberships WHERE user_id='snow'") |
| 153 | self.assertTrue(self.access(path, 'snow')['deny']) |
| 154 | |
| 155 | def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self): |
| 156 | path = '/the-knife/info/refs?service=git-receive-pack' |
| 157 | self.assertTrue(self.access(path, 'snow')['deny']) |
| 158 | self.assertTrue(self.access(path)['authenticate']) |
| 159 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
| 160 | self.assertTrue(self.access(path, authorization=token)['allowed']) |
| 161 | self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) |
| 162 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 163 | db.execute('DELETE FROM personal_tokens') |
| 164 | self.assertTrue(self.access(path, authorization=token)['deny']) |
| 165 | |
| 166 | def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self): |
| 167 | for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'): |
| 168 | self.assertTrue(self.access(path)['clone'], path) |
| 169 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
| 170 | self.assertTrue(self.access(path, authorization=token)['allowed'], path) |
| 171 | for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'): |
| 172 | self.assertNotIn('clone', self.access(path), path) |
| 173 | |
| 174 | def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self): |
| 175 | with self.assertRaises(shale.Error): |
| 176 | self.clone('/the-knife.git/info/refs?service=git-upload-pack') |
| 177 | self.repo.lookup_reference('refs/heads/main').set_target(self.base) |
| 178 | self.repo.create_reference('refs/heads/merge', self.public) |
| 179 | server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args)) |
| 180 | self.addCleanup(server.server_close) |
| 181 | threading.Thread(target=server.serve_forever, daemon=True).start() |
| 182 | self.addCleanup(server.shutdown) |
| 183 | url = f'http://127.0.0.1:{server.server_port}/the-knife' |
| 184 | environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'} |
| 185 | for version in ('0', '1', '2'): |
| 186 | checkout = self.root / ('clone-v' + version) |
| 187 | subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment) |
| 188 | refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout |
| 189 | self.assertEqual(refs, f'refs/heads/main {self.base}\n') |
| 190 | for revision in (str(self.native), str(self.public)): |
| 191 | fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment) |
| 192 | self.assertNotEqual(fetched.returncode, 0, (version, revision)) |
| 193 | self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0) |
| 194 | pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment) |
| 195 | self.assertNotEqual(pushed.returncode, 0) |
| 196 | |
| 197 | def clone(self, uri, method='GET', content_type='', content_encoding=''): |
| 198 | return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding}) |
| 199 | |
| 200 | def test_native_push_during_classification_cannot_change_the_checked_revision(self): |
| 201 | newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native]) |
| 202 | repo = self.repo |
| 203 | class ConcurrentPush: |
| 204 | def __getattr__(self, name): return getattr(repo, name) |
| 205 | def __getitem__(self, key): return repo[key] |
| 206 | def walk(self, *args): |
| 207 | repo.lookup_reference('refs/heads/native').set_target(newer) |
| 208 | repo.create_reference('refs/heads/deadbeef', newer, force=True) |
| 209 | return repo.walk(*args) |
| 210 | for revision in ('native', 'deadbeef'): |
| 211 | repo.lookup_reference('refs/heads/native').set_target(self.native) |
| 212 | if 'refs/heads/deadbeef' in repo.references: |
| 213 | repo.references.delete('refs/heads/deadbeef') |
| 214 | with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()): |
| 215 | self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) |
| 216 | |
| 217 | def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): |
| 218 | with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: |
| 219 | db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") |
| 220 | self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) |
| 221 | token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() |
| 222 | self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny']) |
| 223 | |
| 224 | def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self): |
| 225 | for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'): |
| 226 | self.assertTrue(self.access(path)['deny'], path) |
| 227 | |
| 228 | |
| 229 | if __name__ == '__main__': |
| 230 | unittest.main() |