| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | import base64 |
| 4 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 5 | import json |
| 6 | import os |
| 7 | from pathlib import Path |
| 8 | import socket |
| 9 | import struct |
| 10 | import re |
| 11 | import sqlite3 |
| 12 | import hashlib |
| 13 | import subprocess |
| 14 | import tempfile |
| 15 | import threading |
| 16 | import time |
| 17 | from urllib.parse import parse_qs, urlsplit |
| 18 | from urllib.request import Request, urlopen |
| 19 | from urllib.error import HTTPError |
| 20 | |
| 21 | |
| 22 | def main(): |
| 23 | parser = argparse.ArgumentParser() |
| 24 | parser.add_argument('--preview', action='store_true') |
| 25 | parser.add_argument('--binary', type=Path, default=Path('dashboard/target/debug/home-dashboard')) |
| 26 | args = parser.parse_args() |
| 27 | forms = [] |
| 28 | failed = threading.Event() |
| 29 | row = {'_time': '2026-10-14T02:50:00Z', '_msg': 'Out of memory (OOM): exit 137', |
| 30 | 'id': 'allocation:app:1:Terminated', 'task': 'app', 'stream': 'event', 'event': 'OOM', |
| 31 | 'level': 'error', 'allocation': 'allocation', 'job': 'shale', 'source': 'nomad', 'sequence': 1} |
| 32 | rows = [row] |
| 33 | checkpoint = 0 |
| 34 | |
| 35 | class Fixture(BaseHTTPRequestHandler): |
| 36 | protocol_version = 'HTTP/1.1' |
| 37 | |
| 38 | def handle(self): |
| 39 | try: |
| 40 | super().handle() |
| 41 | except (ConnectionResetError, BrokenPipeError): |
| 42 | pass |
| 43 | |
| 44 | def do_GET(self): |
| 45 | if urlsplit(self.path).path == '/select/logsql/query': |
| 46 | self.query(parse_qs(urlsplit(self.path).query)) |
| 47 | else: |
| 48 | self.reply([{'Address': '127.0.0.1', 'Port': self.server.server_port}]) |
| 49 | |
| 50 | def reply(self, value): |
| 51 | data = json.dumps(value).encode() |
| 52 | self.send_response(200) |
| 53 | self.send_header('Content-Length', str(len(data))) |
| 54 | self.end_headers() |
| 55 | self.wfile.write(data) |
| 56 | |
| 57 | def do_POST(self): |
| 58 | form = parse_qs(self.rfile.read(int(self.headers['Content-Length'])).decode()) |
| 59 | self.query(form) |
| 60 | |
| 61 | def query(self, form): |
| 62 | forms.append((self.path, form)) |
| 63 | if failed.is_set(): |
| 64 | self.send_error(503) |
| 65 | return |
| 66 | if 'source:=log-shipper' in form['query'][0]: |
| 67 | if checkpoint == 0: |
| 68 | self.send_response(200) |
| 69 | self.send_header('Content-Length', '0') |
| 70 | self.end_headers() |
| 71 | else: |
| 72 | self.reply({'sequence': checkpoint if checkpoint is not None else max(r['sequence'] for r in rows)}) |
| 73 | return |
| 74 | selected = rows[:] |
| 75 | match = re.search(r'sequence:>([0-9]+)', form['query'][0]) |
| 76 | if match: |
| 77 | selected = [r for r in selected if r['sequence'] > int(match[1])] |
| 78 | match = re.search(r'sequence:<=(\d+)', form['query'][0]) |
| 79 | if match: |
| 80 | selected = [r for r in selected if r['sequence'] <= int(match[1])] |
| 81 | if '| sort by (sequence)' in form['query'][0]: |
| 82 | selected.sort(key=lambda r: r['sequence']) |
| 83 | elif '| sort by (_time desc' in form['query'][0]: |
| 84 | selected.sort(key=lambda r: (r['_time'], r['id']), reverse=True) |
| 85 | offset = int(re.search(r'offset (\d+)', form['query'][0])[1]) |
| 86 | limit = int(re.search(r'limit (\d+)', form['query'][0])[1]) |
| 87 | selected = selected[offset:offset + limit] |
| 88 | self.send_response(200) |
| 89 | data = b''.join((json.dumps(r) + '\n').encode() for r in selected) |
| 90 | self.send_header('Content-Length', str(len(data))) |
| 91 | self.end_headers() |
| 92 | self.wfile.write(data[:17]) |
| 93 | self.wfile.flush() |
| 94 | try: |
| 95 | self.wfile.write(data[17:]) |
| 96 | self.wfile.flush() |
| 97 | except (BrokenPipeError, ConnectionResetError): |
| 98 | pass |
| 99 | |
| 100 | def log_message(self, *args): |
| 101 | pass |
| 102 | |
| 103 | with tempfile.TemporaryDirectory(prefix='dashboard-logs-') as temporary, ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as fixture: |
| 104 | fixture.daemon_threads = True |
| 105 | threading.Thread(target=fixture.serve_forever, daemon=True).start() |
| 106 | root = Path(temporary).resolve() |
| 107 | (root / 'data').mkdir() |
| 108 | (root / 'token').write_text('disposable') |
| 109 | with socket.socket() as reservation: |
| 110 | reservation.bind(('127.0.0.1', 0)) |
| 111 | port = reservation.getsockname()[1] |
| 112 | origin = f'http://127.0.0.1:{port}' |
| 113 | environment = {**os.environ, 'PORT': str(port), 'SNOWGLOBE_DATA_DIR': str(root / 'data'), |
| 114 | 'SNOWGLOBE_PUBLIC_ORIGIN': 'https://globe.snowglobe.test', 'SNOWGLOBE_NOMAD_TOKEN_FILE': str(root / 'token'), |
| 115 | 'NOMAD_ADDR': f'http://127.0.0.1:{fixture.server_port}'} |
| 116 | environment['SNOWGLOBE_PREVIEW'] = '1' if args.preview else '0' |
| 117 | for name in ['SNOWGLOBE_PROXY_TOKEN_FILE', 'SNOWGLOBE_INTERNAL_URL', 'SNOWGLOBE_AUTH_REQUIRED', 'SNOWGLOBE_INDEX_POOL', 'SNOWGLOBE_YT_STATE']: |
| 118 | environment.pop(name, None) |
| 119 | with (root / 'server.log').open('w') as log: |
| 120 | server = subprocess.Popen([str(args.binary.resolve())], env=environment, stdout=log, stderr=log) |
| 121 | try: |
| 122 | deadline = time.monotonic() + 10 |
| 123 | while True: |
| 124 | try: |
| 125 | with socket.create_connection(('127.0.0.1', port), timeout=.1): break |
| 126 | except OSError: |
| 127 | assert server.poll() is None, (root / 'server.log').read_text() |
| 128 | assert time.monotonic() < deadline |
| 129 | time.sleep(.05) |
| 130 | path = '/api/services/shale/logs' |
| 131 | with urlopen(Request(origin + path + '?level=error&q=stream:event', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: |
| 132 | lines = json.load(response) |
| 133 | assert lines[0]['stream'] == 'event' and lines[0]['event'] == 'OOM' |
| 134 | assert lines[0]['level'] == 'error' and lines[0]['id'] == row['id'] |
| 135 | assert 'level:=error' in forms[-1][1]['query'][0] |
| 136 | assert 'stream:~"(?i)^event"' in forms[-1][1]['query'][0] |
| 137 | try: |
| 138 | urlopen(Request(origin + path + '?after=2&before=1', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) |
| 139 | raise AssertionError('reversed bounds accepted') |
| 140 | except HTTPError as error: |
| 141 | assert error.code == 400 and b'at or after' in error.read() |
| 142 | |
| 143 | def connect(expected, supplied_origin=origin, groups='infra-admin', actor='fixture', cursor=0, cookie=None): |
| 144 | sock = socket.create_connection(('127.0.0.1', port), timeout=5) |
| 145 | sock.settimeout(20) |
| 146 | stream = sock.makefile('rb') |
| 147 | headers = {'Host': urlsplit(origin).netloc, 'Upgrade': 'websocket', 'Connection': 'Upgrade', |
| 148 | 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': base64.b64encode(os.urandom(16)).decode()} |
| 149 | if actor is not None: headers['User-Name'] = actor |
| 150 | if cookie is not None: headers['Cookie'] = '__Host-snow-session=' + cookie |
| 151 | if groups is not None: headers['User-Groups'] = groups |
| 152 | if supplied_origin is not None: headers['Origin'] = supplied_origin |
| 153 | sock.sendall(('GET ' + path + f'/stream?q=stream:event&cursor={cursor}&from=1 HTTP/1.1\r\n' + ''.join(f'{k}: {v}\r\n' for k, v in headers.items()) + '\r\n').encode()) |
| 154 | status = int(stream.readline().split()[1]) |
| 155 | assert status == expected, status |
| 156 | while stream.readline().strip(): pass |
| 157 | if expected != 101: |
| 158 | stream.close() |
| 159 | sock.close() |
| 160 | return sock, stream |
| 161 | |
| 162 | connect(401, actor=None) |
| 163 | connect(403, groups='metrics') |
| 164 | connect(403, supplied_origin=None) |
| 165 | connect(403, supplied_origin='http://other.invalid') |
| 166 | sock, stream = connect(101) |
| 167 | try: |
| 168 | def receive(): |
| 169 | head = stream.read(2) |
| 170 | assert len(head) == 2, 'stream closed unexpectedly' |
| 171 | size = head[1] & 127 |
| 172 | if size == 126: size = struct.unpack('!H', stream.read(2))[0] |
| 173 | if size == 127: size = struct.unpack('!Q', stream.read(8))[0] |
| 174 | if head[0] & 15 == 9: |
| 175 | payload = stream.read(size) |
| 176 | mask = os.urandom(4) |
| 177 | sock.sendall(bytes([138, 128 | size]) + mask + bytes(v ^ mask[i % 4] for i, v in enumerate(payload))) |
| 178 | return receive() |
| 179 | assert head[0] & 15 == 1, head |
| 180 | return json.loads(stream.read(size)) |
| 181 | message = receive() |
| 182 | assert message == {'lines': [], 'cursor': 0} |
| 183 | checkpoint = None |
| 184 | while not (message := receive()).get('lines'): pass |
| 185 | assert message['lines'] == lines and message['cursor'] == 1 |
| 186 | assert forms[-1][1]['start'] == ['1'] |
| 187 | rows.append({**row, 'id': 'same-time', 'sequence': 2}) |
| 188 | while not (message := receive()).get('lines'): pass |
| 189 | assert message['lines'][0]['id'] == 'same-time' and message['cursor'] == 2 |
| 190 | rows.append({**row, '_time': '2026-10-14T01:00:00Z', 'id': 'late-oom', 'sequence': 3}) |
| 191 | while not (message := receive()).get('lines'): pass |
| 192 | assert message['lines'][0]['id'] == 'late-oom' and message['cursor'] == 3 |
| 193 | failed.set() |
| 194 | while not (message := receive()).get('error'): pass |
| 195 | assert 'store' in message['error'] |
| 196 | finally: |
| 197 | stream.close() |
| 198 | sock.close() |
| 199 | failed.clear() |
| 200 | sock, stream = connect(101, cursor=2) |
| 201 | try: |
| 202 | while not (message := receive()).get('lines'): pass |
| 203 | assert [r['id'] for r in message['lines']] == ['late-oom'] |
| 204 | finally: |
| 205 | stream.close() |
| 206 | sock.close() |
| 207 | bulk = [{**row, 'id': f'bulk-{i}', 'sequence': i + 4, '_msg': 'x' * 3500} for i in range(3000)] |
| 208 | rows.extend(bulk) |
| 209 | sock, stream = connect(101, cursor=3) |
| 210 | try: |
| 211 | received = [] |
| 212 | while len(received) < 3000: |
| 213 | message = receive() |
| 214 | assert not message.get('error'), message |
| 215 | received.extend(r['id'] for r in message['lines']) |
| 216 | assert received == [r['id'] for r in bulk] |
| 217 | finally: |
| 218 | stream.close() |
| 219 | sock.close() |
| 220 | accepted = time.time() |
| 221 | sequence = int(accepted * 1_000_000) |
| 222 | delayed = {**row, 'id': 'delayed-search-visibility', 'sequence': sequence} |
| 223 | checkpoint = 3003 |
| 224 | rows.append({**row, 'id': 'initial-unsettled', 'sequence': sequence + 1}) |
| 225 | def publish(): |
| 226 | nonlocal checkpoint |
| 227 | rows.append(delayed) |
| 228 | checkpoint = sequence + 1 |
| 229 | threading.Timer(7, publish).start() |
| 230 | sock, stream = connect(101, cursor=sequence + 1) |
| 231 | try: |
| 232 | while not (message := receive()).get('lines'): pass |
| 233 | assert [r['id'] for r in message['lines']] == [delayed['id'], 'initial-unsettled'] |
| 234 | assert time.time() - accepted >= 7 |
| 235 | finally: |
| 236 | stream.close() |
| 237 | sock.close() |
| 238 | rows[:] = [{**row, 'id': f'tie-{i:05}', 'sequence': i + 1} for i in range(6000)] + [{**row, '_time': '2026-10-13T00:00:00Z', 'id': 'older-history', 'sequence': 6001}] |
| 239 | history = [] |
| 240 | for offset in (0, 5000): |
| 241 | with urlopen(Request(origin + path + f'?limit=5000&offset={offset}', headers={'User-Name': 'fixture', 'User-Groups': 'infra-admin'})) as response: |
| 242 | history.extend(json.load(response)) |
| 243 | assert len({line['id'] for line in history}) == 6001 |
| 244 | assert history[-1]['id'] == 'older-history' |
| 245 | cookie = 'disposable-native-session' |
| 246 | with sqlite3.connect(root / 'data/accounts.sqlite') as db: |
| 247 | db.execute('INSERT INTO users VALUES (?,?)', ('fixture', json.dumps({'username':'fixture','enabled':True,'requiredActions':[]}))) |
| 248 | role = db.execute('SELECT id FROM roles WHERE name=?', ('infra-admin',)).fetchone()[0] |
| 249 | db.execute('INSERT INTO memberships VALUES (?,?)', ('fixture', role)) |
| 250 | db.execute('INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?)', (hashlib.sha256(cookie.encode()).hexdigest(),'fixture','dashboard',int(time.time())+120,'127.0.0.1',0,0,0)) |
| 251 | connect(401) |
| 252 | sock, stream = connect(101, cursor=sequence, cookie=cookie) |
| 253 | try: |
| 254 | while 'lines' not in (message := receive()): pass |
| 255 | with sqlite3.connect(root / 'data/accounts.sqlite') as db: |
| 256 | db.execute('DELETE FROM sessions') |
| 257 | while not (message := receive()).get('error'): pass |
| 258 | assert 'administrator' in message['error'] |
| 259 | finally: |
| 260 | stream.close() |
| 261 | sock.close() |
| 262 | print('REST events/filters, WebSocket auth/Origin, fragmented responses, timestamp ties, delayed OOM, reconnect cursor, oversized backlog, delayed visibility, session revocation, upstream failure: passed') |
| 263 | finally: |
| 264 | server.terminate() |
| 265 | server.wait(timeout=10) |
| 266 | fixture.shutdown() |
| 267 | |
| 268 | |
| 269 | if __name__ == '__main__': |
| 270 | main() |