From f4d8ce9b1e0defb90de5d60d85276bff2b7ff738 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 22 Oct 2026 13:14:53 -0700 Subject: [PATCH] Release Shale repository handles after access checks Close the libgit2 database on successful, rejected, and failed classifications so native pack mappings do not depend on Python collection. Keep disposable fixture databases and repositories closed as well. Assisted-by: gpt-6.1-sol --- tools/shale-access-test.py | 26 ++++-- tools/shale-access.py | 163 +++++++++++++++++++------------------ 2 files changed, 104 insertions(+), 85 deletions(-) diff --git a/tools/shale-access-test.py b/tools/shale-access-test.py index cc1414670351ed9cd5cf3896a03dd8b8db82bb83..b9d7b9d5c5f0adfa5135473b075c330c80791f6a 100644 --- a/tools/shale-access-test.py +++ b/tools/shale-access-test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import base64 +from contextlib import closing import importlib import json import os @@ -26,6 +27,7 @@ class Access(unittest.TestCase): (self.root / 'data').mkdir() self.uuid = '0' * 26 self.repo = pygit2.init_repository(str(self.root / 'repositories_owned' / self.uuid), bare=True) + self.addCleanup(self.repo.free) self.base = self.commit({'README.md': 'PUBLIC', 'Cargo.lock': 'SECRET LOCK'}) self.native = self.commit({'knife/private.rs': 'SECRET NATIVE'}, parents=[]) self.repo.create_reference('refs/heads/native', self.native) @@ -33,12 +35,12 @@ class Access(unittest.TestCase): self.repo.create_reference('refs/heads/main', self.public) self.repo.set_head('refs/heads/main') (self.root / 'policy.json').write_text(json.dumps({'nativeRestriction': {'commits': [str(self.native)]}})) - with sqlite3.connect(self.root / 'accounts.sqlite') as db: + with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: db.executescript('CREATE TABLE users(id TEXT,profile TEXT); CREATE TABLE roles(id TEXT,name TEXT); CREATE TABLE memberships(user_id TEXT,role_id TEXT); INSERT INTO roles VALUES(\'access\',\'knife-native\');') for name in ('snow', 'guest', 'stranger', 'disabled', 'ai'): db.execute('INSERT INTO users VALUES(?,?)', (name, json.dumps({'enabled': name != 'disabled', 'kind': 'ai' if name == 'ai' else 'guest' if name == 'guest' else 'user', 'requiredActions': []}))) if name != 'stranger': db.execute("INSERT INTO memberships VALUES(?,'access')", (name,)) - with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: db.executescript('CREATE TABLE repositories(uuid TEXT,name TEXT); CREATE TABLE users(id INTEGER,name TEXT,provider TEXT,snowflake TEXT); CREATE TABLE personal_tokens(uuid TEXT,agent INTEGER,secret TEXT);') db.execute("INSERT INTO repositories VALUES(?,'the-knife')", (self.uuid,)) for i, name in enumerate(('snow', 'guest', 'stranger', 'disabled', 'ai')): @@ -56,6 +58,20 @@ class Access(unittest.TestCase): def access(self, path, account='', authorization=''): return shale.access({'service': 'shale', 'uri': path, 'account': account, 'authorization': authorization}) + def test_repository_handles_close_for_allowed_rejected_and_failed_classification(self): + released = [] + class TrackedRepository(pygit2.Repository): + def free(self): + released.append(True) + super().free() + with patch.object(shale.pygit2, 'Repository', TrackedRepository): + self.assertFalse(self.access('/the-knife/')['deny']) + self.assertTrue(self.access('/the-knife/objects/x')['deny']) + with patch.object(shale.json, 'loads', side_effect=ValueError('invalid policy')): + with self.assertRaises(ValueError): + self.access('/the-knife/') + self.assertEqual(len(released), 3) + def test_public_tree_and_original_native_history_stays_restricted_after_merge(self): public = self.access('/the-knife/tree/main/') self.assertFalse(public['deny']) @@ -93,7 +109,7 @@ class Access(unittest.TestCase): self.assertTrue(self.access(path, name)['allowed']) for name in ('stranger', 'disabled', 'ai', ''): self.assertTrue(self.access(path, name)['deny']) - with sqlite3.connect(self.root / 'accounts.sqlite') as db: + with closing(sqlite3.connect(self.root / 'accounts.sqlite')) as db, db: db.execute("DELETE FROM memberships WHERE user_id='snow'") self.assertTrue(self.access(path, 'snow')['deny']) @@ -103,7 +119,7 @@ class Access(unittest.TestCase): token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() self.assertTrue(self.access(path, authorization=token)['allowed']) self.assertTrue(self.access(path, 'snow', 'Basic ' + base64.b64encode(b'snow:incorrect').decode())['deny']) - with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: db.execute('DELETE FROM personal_tokens') self.assertTrue(self.access(path, authorization=token)['deny']) @@ -125,7 +141,7 @@ class Access(unittest.TestCase): self.assertTrue(self.access('/the-knife/tree/' + revision + '/README.md')['deny']) def test_duplicate_names_cannot_impersonate_an_allowlisted_identity(self): - with sqlite3.connect(self.root / 'data/astheno.shale.db') as db: + with closing(sqlite3.connect(self.root / 'data/astheno.shale.db')) as db, db: db.execute("INSERT INTO users VALUES(42,'snow','other-provider','stranger')") self.assertTrue(self.access('/the-knife/commit/native', 'snow')['deny']) token = 'Basic ' + base64.b64encode(b'snow:private-fixture-secret').decode() diff --git a/tools/shale-access.py b/tools/shale-access.py index f5d141a17d056ac0f2659534f7c9aba2f5ed82f1..6e8d57fbc8f45131b80ca9adca4a951a59bac106 100644 --- a/tools/shale-access.py +++ b/tools/shale-access.py @@ -87,90 +87,93 @@ def access(request, accounts=None): if not row or not re.fullmatch(r'[A-Z0-9]{26}', row[0]): return {'allowed': False, 'deny': True} repo = pygit2.Repository(str(root / 'repositories_owned' / row[0])) - policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] - head = repo.lookup_reference('HEAD').target - references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} - protected = set() - tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] - if 'refs/heads/native' in references: - tips.append(references['refs/heads/native']) - for tip in tips: - protected.update(str(commit.id) for commit in repo.walk(tip)) - result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} - if parts[0] != 'the-knife' or '%' in path or '\\' in path: - return result - route = parts[1] if len(parts) > 1 else '' - query = dict(parse_qsl(uri.query, keep_blank_values=True)) - if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): - return result - if route in ('', 'issues'): - if route == '' and query: - return result - result['htmlOnly'] = route != 'issues' - result['deny'] = False - return result - if route in ('heads', 'tags') and len(parts) == 2 and not query: - result['deny'] = False - return result - if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): - return result - if route not in ('log', 'tree', 'plain', 'commit'): - return result - revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' - suffix = '' - if route == 'commit': - revision, dot, suffix = revision.partition('.') - if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): - return result - if not revision or revision == '-': - revision = 'HEAD' - # Only named refs and object IDs have the same meaning in Shale and libgit2. - if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): - return result try: - if revision == 'HEAD': - oid = references.get(head) if isinstance(head, str) else head - else: - candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} - if len(candidates) > 1: + policy = json.loads(Path(os.environ.get('SNOWGLOBE_SHALE_POLICY', str(Path(__file__).resolve().parent.parent / 'config/shale.json'))).read_text())['nativeRestriction'] + head = repo.lookup_reference('HEAD').target + references = {name: repo.lookup_reference(name).resolve().target for name in repo.references} + protected = set() + tips = [repo[tip].peel(pygit2.Commit).id for tip in policy['commits']] + if 'refs/heads/native' in references: + tips.append(references['refs/heads/native']) + for tip in tips: + protected.update(str(commit.id) for commit in repo.walk(tip)) + result = {'allowed': False, 'deny': True, 'commits': sorted(protected), 'htmlOnly': True, 'authenticate': git and not authorization} + if parts[0] != 'the-knife' or '%' in path or '\\' in path: + return result + route = parts[1] if len(parts) > 1 else '' + query = dict(parse_qsl(uri.query, keep_blank_values=True)) + if len(query) != len(parse_qsl(uri.query, keep_blank_values=True)): + return result + if route in ('', 'issues'): + if route == '' and query: return result - if candidates: - oid = candidates.pop() - elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): - oid = repo[revision].id + result['htmlOnly'] = route != 'issues' + result['deny'] = False + return result + if route in ('heads', 'tags') and len(parts) == 2 and not query: + result['deny'] = False + return result + if route in ('log', 'commit') and len(parts) != (2 if route == 'log' else 3): + return result + if route not in ('log', 'tree', 'plain', 'commit'): + return result + revision = query.get('from', '-') if route == 'log' else parts[2] if len(parts) > 2 else '-' + suffix = '' + if route == 'commit': + revision, dot, suffix = revision.partition('.') + if dot and suffix not in ('diff', 'patch', 'stat', 'diffraw'): + return result + if not revision or revision == '-': + revision = 'HEAD' + # Only named refs and object IDs have the same meaning in Shale and libgit2. + if not re.fullmatch(r'[A-Za-z0-9_/-]+', revision): + return result + try: + if revision == 'HEAD': + oid = references.get(head) if isinstance(head, str) else head else: + candidates = {references[name] for name in (revision, 'refs/heads/' + revision, 'refs/tags/' + revision) if name in references} + if len(candidates) > 1: + return result + if candidates: + oid = candidates.pop() + elif re.fullmatch(r'[0-9a-fA-F]{4,40}', revision): + oid = repo[revision].id + else: + return result + if oid is None: return result - if oid is None: + commit = repo[oid].peel(pygit2.Commit) + except (KeyError, ValueError, pygit2.GitError): return result - commit = repo[oid].peel(pygit2.Commit) - except (KeyError, ValueError, pygit2.GitError): - return result - if str(commit.id) in protected: - return result - if route in ('tree', 'plain'): - result['htmlOnly'] = route != 'plain' - file = '/'.join(parts[3:]) - if restricted_path(file): + if str(commit.id) in protected: return result - if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): - return result - if route == 'commit': - diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] - sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) - if sensitive: - if suffix in ('stat', 'diffraw'): + if route in ('tree', 'plain'): + result['htmlOnly'] = route != 'plain' + file = '/'.join(parts[3:]) + if restricted_path(file): return result - result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" - if suffix: - result['rawDiff'] = True - result['deny'] = False - return result - if route == 'log': - result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) - else: - parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] - result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') - if query: - result['uri'] += '?' + urlencode(query) - result['deny'] = False - return result + if query.keys() - ({'from', 'q'} if route == 'log' else {'diff'} if route == 'commit' else set()): + return result + if route == 'commit': + diffs = [repo.diff(parent.tree, commit.tree) for parent in commit.parents] if commit.parents else [commit.tree.diff_to_tree(swap=True)] + sensitive = any(restricted_path(delta.old_file.path) or restricted_path(delta.new_file.path) for diff in diffs for delta in diff.deltas) + if sensitive: + if suffix in ('stat', 'diffraw'): + return result + result['safeDiff'] = ''.join(patch.text for patch in diffs[0] if not restricted_path(patch.delta.old_file.path) and not restricted_path(patch.delta.new_file.path)) + "\nThe Knife's Native UI is restricted\n" + if suffix: + result['rawDiff'] = True + result['deny'] = False + return result + if route == 'log': + result['uri'] = '/the-knife/log/?' + urlencode({**query, 'from': str(commit.id)}) + else: + parts[2:3] = [str(commit.id) + ('.' + suffix if suffix else '')] + result['uri'] = '/' + '/'.join(quote(part, safe='') for part in parts) + ('/' if uri.path.endswith('/') else '') + if query: + result['uri'] += '?' + urlencode(query) + result['deny'] = False + return result + finally: + repo.free() -- 2.54.1