1#!/usr/bin/env python3
2import base64
3from contextlib import closing
4import importlib
5import json
6import os
7from pathlib import Path
8from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9import sqlite3
10import subprocess
11import sys
12import tempfile
13import threading
14import unittest
15from unittest.mock import patch
16
17import pygit2
18from argon2 import PasswordHasher
19
20shale = importlib.import_module('shale-access')
21
22
23class CGI(BaseHTTPRequestHandler):
24 """Plays the Shale gateway: every request passes through clone() before reaching git http-backend."""
25 def __init__(self, test, *args):
26 self.test = test
27 super().__init__(*args)
28
29 def log_message(self, *args):
30 pass
31
32 def do_GET(self):
33 self.relay()
34
35 def do_POST(self):
36 self.relay()
37
38 def relay(self):
39 try:
40 plan = self.test.clone(self.path, self.command, self.headers.get('content-type', ''), self.headers.get('content-encoding', ''))
41 except shale.Error:
42 self.send_response(401)
43 self.send_header('WWW-Authenticate', 'Basic realm="The Knife"')
44 self.end_headers()
45 return
46 body = self.rfile.read(int(self.headers.get('content-length', 0)))
47 output = subprocess.run([sys.executable, shale.__file__, plan['objects'], plan['main']], input=body, capture_output=True, check=True,
48 env={'PATH': os.environ['PATH'], 'TMPDIR': str(self.test.root), 'GIT_PROTOCOL': 'version=2', 'HTTP_GIT_PROTOCOL': 'version=2', **plan['environment']}).stdout
49 head, _, payload = output.partition(b'\r\n\r\n')
50 headers = dict(line.split(': ', 1) for line in head.decode().split('\r\n'))
51 self.send_response(int(headers.pop('Status', '200').split()[0]))
52 for name, value in headers.items():
53 self.send_header(name, value)
54 self.end_headers()
55 self.wfile.write(payload)
56
57
58class Access(unittest.TestCase):
59 def setUp(self):
60 self.temp = tempfile.TemporaryDirectory()
61 self.addCleanup(self.temp.cleanup)
62 self.root = Path(self.temp.name)
63 self.env = patch.dict(os.environ, {'SNOWGLOBE_SHALE_ROOT': str(self.root), 'SNOWGLOBE_DASHBOARD_DATA': str(self.root), 'SNOWGLOBE_SHALE_POLICY': str(self.root / 'policy.json'), 'SNOWGLOBE_DOMAIN': 'example.test'})
64 self.env.start()
65 self.addCleanup(self.env.stop)
66 (self.root / 'data').mkdir()
67 self.uuid = '0' * 26
68 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)
69 self.addCleanup(self.repo.free)
70 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})
71 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])
72 self.repo.create_reference('refs/heads/native', self.native)
73 self.public = self.commit({'README.md': 'PUBLIC NEXT', 'knife/private.rs': 'SECRET MERGED', 'nested/Cargo.lock': 'SECRET LOCK', 'src/public.rs': 'PUBLIC CODE'}, parents=[self.base, self.native])
74 self.repo.create_reference('refs/heads/main', self.public)
75 self.repo.set_head('refs/heads/main')
76 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))
77 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
78 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')
79 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):
80 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))
81 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))
82 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
83 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')
84 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))
85 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):
86 db.execute('INSERT INTO users VALUES(?,?,?,?)', (i, name, 'snowglobe.example.test', name))
87 db.execute('INSERT INTO personal_tokens VALUES(?,?,?)', ('fixture-token', 0, PasswordHasher().hash('private-fixture-secret')))
88
89 def commit(self, files, parents=None):
90 index = pygit2.Index()
91 for path, content in files.items():
92 index.add(pygit2.IndexEntry(path, self.repo.create_blob(content.encode()), pygit2.GIT_FILEMODE_BLOB))
93 tree = index.write_tree(self.repo)
94 author = pygit2.Signature('Fixture', 'fixture@example.invalid')
95 return self.repo.create_commit(None, author, author, 'Fixture', tree, parents or [])
96
97 def access(self, path, account='', authorization=''):
98 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})
99
100 def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self):
101 released = []
102 class TrackedRepository(pygit2.Repository):
103 def free(self):
104 released.append(True)
105 super().free()
106 with patch.object(shale.pygit2, 'Repository', TrackedRepository):
107 self.assertFalse(self.access('/the-knife/')['deny'])
108 self.assertTrue(self.access('/the-knife/objects/x')['deny'])
109 with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')):
110 with self.assertRaises(ValueError):
111 self.access('/the-knife/')
112 self.assertEqual(len(released), 3)
113
114 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):
115 public = self.access('/the-knife/tree/main/')
116 self.assertFalse(public['deny'])
117 self.assertIn(str(self.public), public['uri'])
118 for rev in ('native', str(self.native), str(self.native)[:12]):
119 for route in ('tree', 'plain', 'commit'):
120 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/')['deny'], (route, rev))
121 self.assertTrue(self.access('/the-knife/log/?from=' + rev)['deny'])
122 self.repo.references.delete('refs/heads/native')
123 self.assertTrue(self.access('/the-knife/commit/' + str(self.native))['deny'])
124
125 def test_every_revision_blocks_files_even_when_absent(self):
126 for rev in ('main', str(self.base)):
127 for route in ('tree', 'plain'):
128 for path in ('knife', 'knife/private.rs', 'Cargo.lock', 'nested/Cargo.lock', '%6bnife/private.rs', '%256bnife/private.rs'):
129 self.assertTrue(self.access(f'/the-knife/{route}/{rev}/{path}')['deny'], (route, rev, path))
130 self.assertFalse(self.access('/the-knife/plain/main/src/public.rs')['deny'])
131
132 def test_diff_redaction_covers_both_merge_parents_and_raw_formats_fail_closed(self):
133 result = self.access('/the-knife/commit/' + str(self.public))
134 self.assertFalse(result['deny'])
135 self.assertNotIn('SECRET', result['safeDiff'])
136 self.assertIn('PUBLIC', result['safeDiff'])
137 for suffix in ('diff', 'patch'):
138 result = self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)
139 self.assertFalse(result['deny'])
140 self.assertNotIn('SECRET', result['safeDiff'])
141 self.assertIn('PUBLIC', result['safeDiff'])
142 for suffix in ('stat', 'diffraw'):
143 self.assertTrue(self.access('/the-knife/commit/' + str(self.public) + '.' + suffix)['deny'])
144
145 def test_only_enabled_explicit_members_can_read_and_revocation_is_immediate(self):
146 path = '/the-knife/plain/native/knife/private.rs'
147 for name in ('snow', 'guest'):
148 self.assertTrue(self.access(path, name)['allowed'])
149 for name in ('stranger', 'disabled', 'ai', ''):
150 self.assertTrue(self.access(path, name)['deny'])
151 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
152 db.execute("DELETE FROM memberships WHERE user_id='snow'")
153 self.assertTrue(self.access(path, 'snow')['deny'])
154
155 def test_git_requires_a_valid_live_token_even_with_an_allowed_browser(self):
156 path = '/the-knife/info/refs?service=git-receive-pack'
157 self.assertTrue(self.access(path, 'snow')['deny'])
158 self.assertTrue(self.access(path)['authenticate'])
159 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
160 self.assertTrue(self.access(path, authorization=token)['allowed'])
161 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
162 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
163 db.execute('DELETE FROM personal_tokens')
164 self.assertTrue(self.access(path, authorization=token)['deny'])
165
166 def test_anonymous_fetches_clone_and_tokens_keep_the_full_repository(self):
167 for path in ('/the-knife/info/refs?service=git-upload-pack', '/the-knife.git/git-upload-pack'):
168 self.assertTrue(self.access(path)['clone'], path)
169 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
170 self.assertTrue(self.access(path, authorization=token)['allowed'], path)
171 for path in ('/the-knife.git/info/refs', '/the-knife.git/HEAD', '/the-knife.git/objects/info/alternates', '/THE-KNIFE.git/git-upload-pack', '/the-knife.git/git-upload-pack?x=1'):
172 self.assertNotIn('clone', self.access(path), path)
173
174 def test_clone_serves_only_main_and_refuses_while_main_reaches_native(self):
175 with self.assertRaises(shale.Error):
176 self.clone('/the-knife.git/info/refs?service=git-upload-pack')
177 self.repo.lookup_reference('refs/heads/main').set_target(self.base)
178 self.repo.create_reference('refs/heads/merge', self.public)
179 server = ThreadingHTTPServer(('127.0.0.1', 0), lambda *args: CGI(self, *args))
180 self.addCleanup(server.server_close)
181 threading.Thread(target=server.serve_forever, daemon=True).start()
182 self.addCleanup(server.shutdown)
183 url = f'http://127.0.0.1:{server.server_port}/the-knife'
184 environment = {**os.environ, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_TERMINAL_PROMPT': '0'}
185 for version in ('0', '1', '2'):
186 checkout = self.root / ('clone-v' + version)
187 subprocess.run(['git', '-c', 'protocol.version=' + version, 'clone', '-q', '--bare', url, str(checkout)], check=True, env=environment)
188 refs = subprocess.run(['git', '-C', str(checkout), 'for-each-ref', '--format=%(refname) %(objectname)'], check=True, capture_output=True, text=True, env=environment).stdout
189 self.assertEqual(refs, f'refs/heads/main {self.base}\n')
190 for revision in (str(self.native), str(self.public)):
191 fetched = subprocess.run(['git', '-C', str(checkout), '-c', 'protocol.version=' + version, 'fetch', url, revision], capture_output=True, env=environment)
192 self.assertNotEqual(fetched.returncode, 0, (version, revision))
193 self.assertNotEqual(subprocess.run(['git', '-C', str(checkout), 'cat-file', '-e', revision], env=environment).returncode, 0)
194 pushed = subprocess.run(['git', '-C', str(self.root / 'clone-v2'), 'push', url, 'main:refs/heads/pushed'], capture_output=True, env=environment)
195 self.assertNotEqual(pushed.returncode, 0)
196
197 def clone(self, uri, method='GET', content_type='', content_encoding=''):
198 return shale.clone({'service': 'shale', 'uri': uri, 'method': method, 'contentType': content_type, 'contentEncoding': content_encoding})
199
200 def test_native_push_during_classification_cannot_change_the_checked_revision(self):
201 newer = self.commit({'README.md': 'RESTRICTED NEW COMMIT'}, parents=[self.native])
202 repo = self.repo
203 class ConcurrentPush:
204 def __getattr__(self, name): return getattr(repo, name)
205 def __getitem__(self, key): return repo[key]
206 def walk(self, *args):
207 repo.lookup_reference('refs/heads/native').set_target(newer)
208 repo.create_reference('refs/heads/deadbeef', newer, force=True)
209 return repo.walk(*args)
210 for revision in ('native', 'deadbeef'):
211 repo.lookup_reference('refs/heads/native').set_target(self.native)
212 if 'refs/heads/deadbeef' in repo.references:
213 repo.references.delete('refs/heads/deadbeef')
214 with patch.object(shale.pygit2, 'Repository', return_value=ConcurrentPush()):
215 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])
216
217 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):
218 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
219 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")
220 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])
221 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
222 self.assertTrue(self.access('/the-knife.git/info/refs', authorization=token)['deny'])
223
224 def test_unknown_formats_search_archives_object_endpoints_and_duplicate_queries_close(self):
225 for path in ('/the-knife/search/main/?q=SECRET', '/the-knife/filesearch/main/', '/the-knife/archive/main.zip', '/the-knife/objects/x', '/the-knife.git/info/refs', '/THE-KNIFE/tree/main/', '/the-knife/commit/main?diff=unified&diff=split', '/the-knife/tree/main/?raw=1'):
226 self.assertTrue(self.access(path)['deny'], path)
227
228
229if __name__ == '__main__':
230 unittest.main()