authoravatarclover caruso<git@paperclover.net>2026-10-22 13:14:53-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logf4d8ce9b1e0defb90de5d60d85276bff2b7ff738
treee33312d42c4ac0fb5b005996a5462c700a2b961e
parentd7a4e059a91fedc10fe5726256c78d46f20499b5
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Release Shale repository handles after access checks

Close the libgit2 database on successful, rejected, and failed classifications so native pack mappings do not depend on Python collection. Keep disposable fixture databases and repositories closed as well. Assisted-by: gpt-6.1-sol

2 files changed, 103 insertions(+), 84 deletions(-)

tools/shale-access-test.py+21-5
......@@ -1,5 +1,6 @@
11#!/usr/bin/env python3
22import base64
3from contextlib import closing
34import importlib
45import json
56import os
......@@ -26,6 +27,7 @@ class Access(unittest.TestCase):
2627 (self.root / 'data').mkdir()
2728 self.uuid = '0' * 26
2829 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True)
30 self.addCleanup(self.repo.free)
2931 self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'})
3032 self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[])
3133 self.repo.create_reference('refs/heads/native', self.native)
......@@ -33,12 +35,12 @@ class Access(unittest.TestCase):
3335 self.repo.create_reference('refs/heads/main', self.public)
3436 self.repo.set_head('refs/heads/main')
3537 (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}}))
36 with sqlite3.connect(self.root / 'accounts.sqlite') as db:
38 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
3739 db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');')
3840 for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'):
3941 db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []})))
4042 if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,))
41 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:
43 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
4244 db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);')
4345 db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,))
4446 for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')):
......@@ -56,6 +58,20 @@ class Access(unittest.TestCase):
5658 def access(self, path, account='', authorization=''):
5759 return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization})
5860
61 def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self):
62 released = []
63 class TrackedRepository(pygit2.Repository):
64 def free(self):
65 released.append(True)
66 super().free()
67 with patch.object(shale.pygit2, 'Repository', TrackedRepository):
68 self.assertFalse(self.access('/the-knife/')['deny'])
69 self.assertTrue(self.access('/the-knife/objects/x')['deny'])
70 with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')):
71 with self.assertRaises(ValueError):
72 self.access('/the-knife/')
73 self.assertEqual(len(released), 3)
74
5975 def test_public_tree_and_original_native_history_stays_restricted_after_merge(self):
6076 public = self.access('/the-knife/tree/main/')
6177 self.assertFalse(public['deny'])
......@@ -93,7 +109,7 @@ class Access(unittest.TestCase):
93109 self.assertTrue(self.access(path, name)['allowed'])
94110 for name in ('stranger', 'disabled', 'ai', ''):
95111 self.assertTrue(self.access(path, name)['deny'])
96 with sqlite3.connect(self.root / 'accounts.sqlite') as db:
112 with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db:
97113 db.execute("DELETE FROM memberships WHERE user_id='snow'")
98114 self.assertTrue(self.access(path, 'snow')['deny'])
99115
......@@ -103,7 +119,7 @@ class Access(unittest.TestCase):
103119 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
104120 self.assertTrue(self.access(path, authorization=token)['allowed'])
105121 self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny'])
106 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:
122 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
107123 db.execute('DELETE FROM personal_tokens')
108124 self.assertTrue(self.access(path, authorization=token)['deny'])
109125
......@@ -125,7 +141,7 @@ class Access(unittest.TestCase):
125141 self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny'])
126142
127143 def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self):
128 with sqlite3.connect(self.root / 'data/astheno.shale.db') as db:
144 with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db:
129145 db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')")
130146 self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny'])
131147 token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode()
tools/shale-access.py+82-79
......@@ -87,90 +87,93 @@ def access(request, accounts=None):
8787 if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]):
8888 return {'allowed': False, 'deny': True}
8989 repo = pygit2.Repository(str(root / 'repositories_owned' / row[0]))
90 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
91 head = repo.lookup_reference('HEAD').target
92 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
93 protected = set()
94 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
95 if 'refs/heads/native' in references:
96 tips.append(references['refs/heads/native'])
97 for tip in tips:
98 protected.update(str(commit.id) for commit in repo.walk(tip))
99 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
100 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
101 return result
102 route = parts[1] if len(parts) > 1 else ''
103 query = dict(parse_qsl(uri.query, keep_blank_values=True))
104 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
105 return result
106 if route in ('', 'issues'):
107 if route == '' and query:
90 try:
91 policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction']
92 head = repo.lookup_reference('HEAD').target
93 references = {name: repo.lookup_reference(name).resolve().target for name in repo.references}
94 protected = set()
95 tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']]
96 if 'refs/heads/native' in references:
97 tips.append(references['refs/heads/native'])
98 for tip in tips:
99 protected.update(str(commit.id) for commit in repo.walk(tip))
100 result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization}
101 if parts[0] != 'the-knife' or '%' in path or '\\' in path:
108102 return result
109 result['htmlOnly'] = route != 'issues'
110 result['deny'] = False
111 return result
112 if route in ('heads', 'tags') and len(parts) == 2 and not query:
113 result['deny'] = False
114 return result
115 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
116 return result
117 if route not in ('log', 'tree', 'plain', 'commit'):
118 return result
119 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
120 suffix = ''
121 if route == 'commit':
122 revision, dot, suffix = revision.partition('.')
123 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
103 route = parts[1] if len(parts) > 1 else ''
104 query = dict(parse_qsl(uri.query, keep_blank_values=True))
105 if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)):
124106 return result
125 if not revision or revision == '-':
126 revision = 'HEAD'
127 # Only named refs and object IDs have the same meaning in Shale and libgit2.
128 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
129 return result
130 try:
131 if revision == 'HEAD':
132 oid = references.get(head) if isinstance(head, str) else head
133 else:
134 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
135 if len(candidates) > 1:
107 if route in ('', 'issues'):
108 if route == '' and query:
136109 return result
137 if candidates:
138 oid = candidates.pop()
139 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):
140 oid = repo[revision].id
110 result['htmlOnly'] = route != 'issues'
111 result['deny'] = False
112 return result
113 if route in ('heads', 'tags') and len(parts) == 2 and not query:
114 result['deny'] = False
115 return result
116 if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3):
117 return result
118 if route not in ('log', 'tree', 'plain', 'commit'):
119 return result
120 revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-'
121 suffix = ''
122 if route == 'commit':
123 revision, dot, suffix = revision.partition('.')
124 if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'):
125 return result
126 if not revision or revision == '-':
127 revision = 'HEAD'
128 # Only named refs and object IDs have the same meaning in Shale and libgit2.
129 if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision):
130 return result
131 try:
132 if revision == 'HEAD':
133 oid = references.get(head) if isinstance(head, str) else head
141134 else:
135 candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references}
136 if len(candidates) > 1:
137 return result
138 if candidates:
139 oid = candidates.pop()
140 elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision):
141 oid = repo[revision].id
142 else:
143 return result
144 if oid is None:
142145 return result
143 if oid is None:
146 commit = repo[oid].peel(pygit2.Commit)
147 except (KeyError, ValueError, pygit2.GitError):
144148 return result
145 commit = repo[oid].peel(pygit2.Commit)
146 except (KeyError, ValueError, pygit2.GitError):
147 return result
148 if str(commit.id) in protected:
149 return result
150 if route in ('tree', 'plain'):
151 result['htmlOnly'] = route != 'plain'
152 file = '/'.join(parts[3:])
153 if restricted_path(file):
149 if str(commit.id) in protected:
154150 return result
155 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):
156 return result
157 if route == 'commit':
158 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]
159 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
160 if sensitive:
161 if suffix in ('stat', 'diffraw'):
151 if route in ('tree', 'plain'):
152 result['htmlOnly'] = route != 'plain'
153 file = '/'.join(parts[3:])
154 if restricted_path(file):
162155 return result
163 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"
164 if suffix:
165 result['rawDiff'] = True
166 result['deny'] = False
167 return result
168 if route == 'log':
169 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})
170 else:
171 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]
172 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')
173 if query:
174 result['uri'] += '?' + urlencode(query)
175 result['deny'] = False
176 return result
156 if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()):
157 return result
158 if route == 'commit':
159 diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)]
160 sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas)
161 if sensitive:
162 if suffix in ('stat', 'diffraw'):
163 return result
164 result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n"
165 if suffix:
166 result['rawDiff'] = True
167 result['deny'] = False
168 return result
169 if route == 'log':
170 result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)})
171 else:
172 parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')]
173 result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '')
174 if query:
175 result['uri'] += '?' + urlencode(query)
176 result['deny'] = False
177 return result
178 finally:
179 repo.free()