| 1 | #!/usr/bin/env python3 |
| 2 | import json |
| 3 | import grp |
| 4 | import os |
| 5 | import re |
| 6 | import subprocess |
| 7 | import sys |
| 8 | import time |
| 9 | import urllib.parse |
| 10 | import urllib.request |
| 11 | |
| 12 | ROUTES = "/var/lib/caddy/routes.caddy" |
| 13 | TOKEN = "/var/lib/snowglobe/router.token" |
| 14 | ROUTE_DIR = "/var/lib/snowglobe/routes" |
| 15 | HOST = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?\Z") |
| 16 | |
| 17 | |
| 18 | def nomad(path, token): |
| 19 | request = urllib.request.Request( |
| 20 | "http://127.0.0.1:4646" + path, |
| 21 | headers={"X-Nomad-Token": token}, |
| 22 | ) |
| 23 | with urllib.request.urlopen(request, timeout=5) as response: |
| 24 | return json.load(response) |
| 25 | |
| 26 | |
| 27 | def passive_health(upstreams, indent): |
| 28 | return [f"{indent}fail_duration 30s"] if len(upstreams.split()) > 1 else [] |
| 29 | |
| 30 | |
| 31 | def proxy(upstreams, indent, uncompressed=False, host=None, keep_streams=False): |
| 32 | lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s", |
| 33 | *passive_health(upstreams, indent + " ")] |
| 34 | if uncompressed: |
| 35 | lines.append(f"{indent} header_up Accept-Encoding identity") |
| 36 | if host: |
| 37 | lines += [f"{indent} header_up Host {host}", f"{indent} header_up X-Forwarded-Host {host}"] |
| 38 | if keep_streams: |
| 39 | lines.append(f"{indent} stream_close_delay 8h") |
| 40 | return [*lines, f"{indent}}}"] |
| 41 | |
| 42 | |
| 43 | def shale_mcp_routes(port): |
| 44 | return [" @shale_mcp_link path_regexp shale_mcp_link ^/-/snowglobe-mcp/([A-Za-z0-9_-]{43})$", |
| 45 | " handle @shale_mcp_link {", " rewrite * /oauth/shale/link/{re.shale_mcp_link.1}", |
| 46 | " request_header -User-Name", " request_header -User-Groups", " request_header -Snowglobe-Proxy-Token", |
| 47 | *proxy(f"127.0.0.1:{port}", " "), " }", |
| 48 | " @shale_mcp_callback {", " path /-/callback", " header Cookie *snowglobe_mcp_shale_link=*", " }", |
| 49 | " handle @shale_mcp_callback {", " rewrite * /oauth/shale/callback", |
| 50 | " request_header -User-Name", " request_header -User-Groups", " request_header -Snowglobe-Proxy-Token", |
| 51 | *proxy(f"127.0.0.1:{port}", " "), " }"] |
| 52 | |
| 53 | |
| 54 | def shale_attachment_routes(upstreams, target="/srv/prod/shale"): |
| 55 | """Authorize retained attachments through their issue before serving bytes.""" |
| 56 | manifest = os.path.join(target, "forgejo-attachments.json") |
| 57 | if not os.path.exists(manifest): |
| 58 | return [] |
| 59 | with open(manifest) as source: |
| 60 | entries = json.load(source) |
| 61 | if not isinstance(entries, list) or len(entries) > 10000: |
| 62 | raise ValueError("invalid Shale attachment manifest") |
| 63 | lines = [] |
| 64 | for index, entry in enumerate(entries): |
| 65 | path, filename, issue = (entry[k] for k in ("path", "file", "issue")) |
| 66 | if (not re.fullmatch(r"[0-9a-f-]{36}/[^/\\\"\r\n]+", filename) |
| 67 | or path != "/-/forgejo-attachments/" + urllib.parse.quote(filename, safe="/") |
| 68 | or not re.fullmatch(r"/(?:[a-zA-Z0-9_.-]+/)+issues/[1-9][0-9]*", issue)): |
| 69 | raise ValueError("unsafe Shale attachment route") |
| 70 | # Request cookies go to Shale as usual. A private issue yields 403/404; |
| 71 | # only Shale's successful issue response permits the local file read. |
| 72 | name = f"@shale_attachment_{index}" |
| 73 | lines += [f" {name} path {json.dumps(path)}", f" handle {name} {{", |
| 74 | " header Cache-Control private,no-store", " header X-Content-Type-Options nosniff", |
| 75 | f" reverse_proxy {upstreams} {{", " method GET", f" rewrite {issue}", |
| 76 | " @authorized status 200", " handle_response @authorized {", |
| 77 | f" root * {json.dumps(os.path.join(target, 'forgejo-attachments'))}", |
| 78 | f" rewrite * {json.dumps('/' + urllib.parse.quote(filename, safe='/'))}", |
| 79 | " file_server", " }", " }", " }"] |
| 80 | return lines |
| 81 | |
| 82 | |
| 83 | def shale_write_routes(host): |
| 84 | # Older Shale releases predate form tokens. Browsers send Origin on POST; |
| 85 | # require the exact site origin for every cookie-authenticated mutation. |
| 86 | # HTTP Git clients use Basic authentication and do not carry this cookie. |
| 87 | return [" @shale_unsafe_origin {", " method POST PUT PATCH DELETE", |
| 88 | " header Cookie *SessionID=*", f" not header Origin https://{host}", |
| 89 | " }", ' respond @shale_unsafe_origin "Forbidden" 403'] |
| 90 | |
| 91 | |
| 92 | def shale_login_routes(host): |
| 93 | # Shale parses the login Referer as an absolute URL after OAuth. |
| 94 | return_url = rf"^https://{re.escape(host)}/(?:[A-Za-z0-9._~!$&'()*+,;=:@/?#\[\]-]|%[0-9A-Fa-f]{{2}})*$" |
| 95 | return [" @shale_login_return {", " path /-/login", |
| 96 | f" not header_regexp Referer `{return_url}`", " }", |
| 97 | f" request_header @shale_login_return Referer https://{host}/"] |
| 98 | |
| 99 | |
| 100 | def zig_mirror_routes(service, routes): |
| 101 | if service == "copyparty": |
| 102 | mirror = "zig-mirror" |
| 103 | else: |
| 104 | state = os.environ.get("SNOWGLOBE_STATE", "/var/lib/snowglobe") |
| 105 | stages = os.path.join(state, "stages") |
| 106 | try: |
| 107 | with open(os.path.join(stages, service + ".json")) as source: |
| 108 | stage = json.load(source) |
| 109 | mirror = stage.get("mirrorStage", "") |
| 110 | if stage.get("sourceId") != "copyparty" or not isinstance(mirror, str) or not re.fullmatch(r"zig-mirror-preview-[0-9a-f]{8}", mirror): |
| 111 | mirror = None |
| 112 | if mirror: |
| 113 | with open(os.path.join(stages, mirror + ".json")) as source: |
| 114 | if json.load(source).get("sourceId") != "zig-mirror": |
| 115 | mirror = None |
| 116 | except FileNotFoundError: |
| 117 | mirror = None |
| 118 | upstreams = set().union(*(route["upstreams"] for route in routes.values() if mirror in route["services"])) |
| 119 | pattern = r"^/mirrors/zig/zig[^/]*\.(?:tar\.xz|zip)(?:\.minisig)?$" |
| 120 | lines = [" @mirror_private expression `path_regexp('^/+mirrors(?:/+[^/]*)*/+[.]') || (path_regexp('^/+mirrors(?:/|$)') && {http.request.uri.query}.matches('(?i)(^|&)doc=(?:[.]|%2e)'))`", |
| 121 | " handle @mirror_private {", " respond 404", " }", |
| 122 | " @zig_mirror {", " method GET HEAD", f" path_regexp zig_mirror {pattern}", " }", |
| 123 | " handle @zig_mirror {"] |
| 124 | if upstreams: |
| 125 | if any(not re.fullmatch(r"127\.0\.0\.1:[0-9]+|\[::1\]:[0-9]+", upstream) for upstream in upstreams): |
| 126 | raise ValueError("Zig mirror requires private loopback upstreams") |
| 127 | lines += [" request_header -Authorization", " request_header -Cookie", |
| 128 | *proxy(" ".join(sorted(upstreams)), " ")] |
| 129 | else: |
| 130 | lines.append(" respond 503") |
| 131 | return [*lines, " }"] |
| 132 | |
| 133 | |
| 134 | def dashboard_preview_proof(service): |
| 135 | if not re.fullmatch(r"dashboard-preview-[0-9a-f]{8}", service): |
| 136 | return None |
| 137 | stages = os.path.join(os.environ.get("SNOWGLOBE_STATE", "/var/lib/snowglobe"), "stages") |
| 138 | try: |
| 139 | with open(os.path.join(stages, service + ".json")) as source: |
| 140 | stage = json.load(source) |
| 141 | mount = stage.get("mount", "") |
| 142 | if stage.get("sourceId") != "dashboard" or not os.path.isabs(mount) or os.path.basename(mount) != service or os.path.basename(os.path.dirname(mount)) != "staging": |
| 143 | return None |
| 144 | path = os.path.join(mount, "proxy.token") |
| 145 | if os.path.islink(path) or os.stat(path).st_mode & 0o077: |
| 146 | return None |
| 147 | with open(path) as source: |
| 148 | proof = source.read().strip() |
| 149 | except FileNotFoundError: |
| 150 | return None |
| 151 | if not re.fullmatch(r"[0-9a-f]{64}", proof): |
| 152 | raise ValueError("invalid dashboard preview proxy proof") |
| 153 | return proof |
| 154 | |
| 155 | |
| 156 | def shale_gateway(service, routes, proof): |
| 157 | source = service.split("-preview-", 1)[0] |
| 158 | if source != "shale": |
| 159 | return None, None |
| 160 | stages = os.path.join(os.environ.get("SNOWGLOBE_STATE", "/var/lib/snowglobe"), "stages") |
| 161 | gateway = None |
| 162 | if service != source: |
| 163 | try: |
| 164 | with open(os.path.join(stages, service + ".json")) as file: |
| 165 | stage = json.load(file) |
| 166 | except FileNotFoundError: |
| 167 | return None, None |
| 168 | if stage.get("sourceId") != source: |
| 169 | return None, None |
| 170 | gateway = stage.get("gatewayStage") |
| 171 | if gateway is not None and (not isinstance(gateway, str) or not re.fullmatch(r"dashboard-preview-[0-9a-f]{8}", gateway)): |
| 172 | return None, None |
| 173 | if gateway is None: |
| 174 | port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) |
| 175 | if not 1 <= port <= 65535: |
| 176 | raise ValueError("invalid dashboard port") |
| 177 | return f"127.0.0.1:{port}", proof |
| 178 | try: |
| 179 | with open(os.path.join(stages, gateway + ".json")) as file: |
| 180 | if json.load(file).get("sourceId") != "dashboard": |
| 181 | return None, None |
| 182 | except FileNotFoundError: |
| 183 | return None, None |
| 184 | upstreams = set().union(*(route["upstreams"] for route in routes.values() if gateway in route["services"])) |
| 185 | if len(upstreams) != 1: |
| 186 | return None, None |
| 187 | upstream = next(iter(upstreams)) |
| 188 | if not re.fullmatch(r"127\.0\.0\.1:[0-9]+|\[::1\]:[0-9]+", upstream): |
| 189 | raise ValueError("preview gateway must bind loopback") |
| 190 | proof = dashboard_preview_proof(gateway) |
| 191 | return (upstream, proof) if proof else (None, None) |
| 192 | |
| 193 | |
| 194 | def render(token): |
| 195 | with open(os.environ["SNOWGLOBE_PROXY_TOKEN_FILE"]) as file: |
| 196 | dashboard_proof = file.read().strip() |
| 197 | if not re.fullmatch(r"[0-9a-fA-F]{64}", dashboard_proof): |
| 198 | raise ValueError("invalid dashboard proxy token") |
| 199 | auth_host = "auth." + os.environ["SNOWGLOBE_DOMAIN"] |
| 200 | origins = json.dumps({"origins": ["https://snowglobe." + os.environ["SNOWGLOBE_DOMAIN"]]}, separators=(",", ":")) |
| 201 | webauthn = [" handle /.well-known/webauthn {", ' header Content-Type application/json', |
| 202 | f" respond {json.dumps(origins)} 200", " }"] |
| 203 | routes = {} |
| 204 | internal_services = {} |
| 205 | auth_upstreams = set() |
| 206 | allocation_checks = {} |
| 207 | for namespace in nomad("/v1/services", token): |
| 208 | if namespace["Namespace"] != "default": |
| 209 | continue |
| 210 | for service in namespace["Services"]: |
| 211 | name = urllib.parse.quote(service["ServiceName"], safe="") |
| 212 | for instance in nomad(f"/v1/service/{name}", token): |
| 213 | tags = instance.get("Tags") or [] |
| 214 | if instance["ServiceName"] != "forward-auth" and not any(tag.startswith("caddy-") for tag in tags): |
| 215 | continue |
| 216 | address = instance["Address"] |
| 217 | port = instance["Port"] |
| 218 | if not re.fullmatch(r"[0-9a-fA-F:.]+", address) or type(port) is not int or not 1 <= port <= 65535: |
| 219 | continue |
| 220 | upstream = f"[{address}]:{port}" if ":" in address else f"{address}:{port}" |
| 221 | if not re.fullmatch(r"[a-z][a-z0-9-]*", instance["ServiceName"]): |
| 222 | raise ValueError("invalid internal service name") |
| 223 | internal_services.setdefault(instance["ServiceName"], set()).add(upstream) |
| 224 | alloc = instance["AllocID"] |
| 225 | if alloc not in allocation_checks: |
| 226 | allocation_checks[alloc] = nomad(f"/v1/allocation/{alloc}/checks", token) |
| 227 | service_checks = list(allocation_checks[alloc].values()) |
| 228 | if not service_checks or any(check["Status"] != "success" for check in service_checks): |
| 229 | continue |
| 230 | if instance["ServiceName"] == "forward-auth": |
| 231 | auth_upstreams.add(upstream) |
| 232 | access = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-auth-role=")] |
| 233 | user_headers = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-user-header=")] |
| 234 | if len(access) > 1 or (access and not re.fullmatch(r"[a-z][a-z0-9-]*", access[0])): |
| 235 | raise ValueError("invalid route access role") |
| 236 | if len(user_headers) > 1 or (user_headers and (not access or not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", user_headers[0]))): |
| 237 | raise ValueError("invalid authenticated user header") |
| 238 | if access and not any(tag.startswith("caddy-host=") for tag in tags): |
| 239 | raise ValueError("route access role has no host") |
| 240 | for tag in tags: |
| 241 | if tag.startswith(("caddy-host=", "caddy-plain-host=")): |
| 242 | host = tag.split("=", 1)[1] |
| 243 | role = access[0] if access and tag.startswith("caddy-host=") else None |
| 244 | user_header = user_headers[0] if user_headers and role else None |
| 245 | if not HOST.fullmatch(host): |
| 246 | raise ValueError(f"invalid Caddy host: {host!r}") |
| 247 | route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False, "secureCookies": False}) |
| 248 | if role and route["authRole"] not in (None, role): |
| 249 | raise ValueError("conflicting route access roles") |
| 250 | if user_header and route["userHeader"] not in (None, user_header): |
| 251 | raise ValueError("conflicting authenticated user headers") |
| 252 | if role: |
| 253 | route["authRole"] = role |
| 254 | if user_header: |
| 255 | route["userHeader"] = user_header |
| 256 | if "caddy-internal=true" in tags: |
| 257 | route["internal"] = True |
| 258 | if "caddy-real-ip=true" in tags: |
| 259 | route["realIp"] = True |
| 260 | if "caddy-secure-cookies=true" in tags: |
| 261 | route["secureCookies"] = True |
| 262 | route["upstreams"].add(upstream) |
| 263 | route["services"].add(instance.get("JobID") or instance["ServiceName"]) |
| 264 | elif tag.startswith("caddy-port="): |
| 265 | listener = int(tag.split("=", 1)[1]) |
| 266 | if not 1 <= listener <= 65535: |
| 267 | raise ValueError(f"invalid Caddy port: {listener}") |
| 268 | route = routes.setdefault((listener, ""), {"upstreams": set(), "services": set()}) |
| 269 | route["upstreams"].add(upstream) |
| 270 | |
| 271 | lines = [] |
| 272 | for (listener, host), route in sorted(routes.items()): |
| 273 | if host: |
| 274 | if host == auth_host: |
| 275 | continue |
| 276 | role = route["authRole"] |
| 277 | if role and not auth_upstreams: |
| 278 | continue |
| 279 | lines.append(f"{host if listener == 80 else f'{host}:{listener}'} {{") |
| 280 | if host.endswith(".test") or route["internal"]: |
| 281 | lines.append(" tls internal") |
| 282 | if route["secureCookies"]: |
| 283 | lines.append(' header >Set-Cookie "(.*)" "$1; Secure"') |
| 284 | if route["realIp"]: |
| 285 | lines.append(" request_header X-Real-Ip {remote_host}") |
| 286 | if len(route["services"]) != 1: |
| 287 | raise ValueError(f"multiple services claim {host}") |
| 288 | service = next(iter(route["services"])) |
| 289 | if not re.fullmatch(r"[a-z][a-z0-9-]*", service): |
| 290 | raise ValueError(f"invalid service name: {service}") |
| 291 | if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): |
| 292 | # Keep the Origin guard before every static/proxy handle; |
| 293 | # otherwise Caddy sorts those handles ahead of respond. |
| 294 | lines += [" route {", *shale_write_routes(host), |
| 295 | *shale_login_routes(host)] |
| 296 | if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): |
| 297 | lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])), |
| 298 | "/srv/staging/" + service) |
| 299 | if service == "shale": |
| 300 | lines += [" handle /-/logout {", f" reverse_proxy {' '.join(sorted(route['upstreams']))} {{", |
| 301 | f" header_down Location https://snowglobe.{os.environ['SNOWGLOBE_DOMAIN']}/auth/shale/sign-out", |
| 302 | " @logged_out status 200 302 303 404", " handle_response @logged_out {", |
| 303 | " copy_response 303", " }", " }", " }"] |
| 304 | lines += shale_attachment_routes(" ".join(sorted(route["upstreams"]))) |
| 305 | lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$", |
| 306 | " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"] |
| 307 | port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) |
| 308 | if not 1 <= port <= 65535: |
| 309 | raise ValueError("invalid dashboard port for Shale linking") |
| 310 | lines += shale_mcp_routes(port) |
| 311 | lines += [" tracing {", f" span {service}", " span_attributes {", |
| 312 | f" snowglobe.service {service}", " snowglobe.kind edge", " }", " }"] |
| 313 | if service == "copyparty" or re.fullmatch(r"copyparty-preview-[0-9a-f]{8}", service): |
| 314 | lines += zig_mirror_routes(service, routes) |
| 315 | manifest = os.path.join(ROUTE_DIR, service + ".json") |
| 316 | assets = {} |
| 317 | headers = {} |
| 318 | head_html = {} |
| 319 | proof = None |
| 320 | scrub = [] |
| 321 | if os.path.exists(manifest): |
| 322 | with open(manifest) as file: |
| 323 | assets = json.load(file) |
| 324 | identity = assets.get("identity", {}) |
| 325 | headers = identity.get(host, {}) |
| 326 | head_html = assets.get("headHtml", {}).get(host, {}) |
| 327 | proof = assets.get("identityProof", {}).get(host) |
| 328 | scrub = sorted({name for configured in identity.values() for name in configured} | set(assets.get("identityProof", {}).values())) |
| 329 | if any(not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", name) for name in scrub): |
| 330 | raise ValueError(f"invalid identity header: {host}") |
| 331 | if proof and not headers: |
| 332 | raise ValueError(f"identity proof has no identity headers: {host}") |
| 333 | if any(source not in {"X-Auth-Request-User", "X-Auth-Request-Groups", "X-Auth-Request-Preferred-Username"} for source in headers.values()): |
| 334 | raise ValueError(f"invalid identity claim: {host}") |
| 335 | if role and headers: |
| 336 | raise ValueError(f"route has both required and optional auth: {host}") |
| 337 | if role and (assets.get("files") or assets.get("dirs")): |
| 338 | raise ValueError(f"authenticated route has static overrides: {host}") |
| 339 | if head_html and (role or headers): |
| 340 | raise ValueError(f"authenticated route has HTML injection: {host}") |
| 341 | for request, markup in head_html.items(): |
| 342 | if not re.fullmatch(r"/[A-Za-z0-9/._*-]*", request) or not markup: |
| 343 | raise ValueError(f"invalid HTML injection: {host} {request}") |
| 344 | for request, path in assets.get("files", {}).items(): |
| 345 | if not request.startswith("/") or " " in request or "\n" in request: |
| 346 | raise ValueError(f"invalid static path: {request}") |
| 347 | lines += [f" handle {request} {{", f" root * {json.dumps(os.path.dirname(path))}", |
| 348 | f" rewrite * /{os.path.basename(path)}", ' header Cache-Control "no-store"', |
| 349 | " file_server", " }"] |
| 350 | for request, path in assets.get("dirs", {}).items(): |
| 351 | if not request.startswith("/") or " " in request or "\n" in request: |
| 352 | raise ValueError(f"invalid static prefix: {request}") |
| 353 | lines += [f" handle_path {request}* {{", f" root * {json.dumps(path)}"] |
| 354 | if request == "/.static/" and (service == "copyparty" or re.fullmatch(r"copyparty-preview-[0-9a-f]{8}", service)): |
| 355 | lines += [" @public_font path *.woff2", ' header @public_font Access-Control-Allow-Origin "*"'] |
| 356 | lines += [" file_server", " }"] |
| 357 | metrics_path = assets.get("metricsPaths", {}).get(host) |
| 358 | if metrics_path: |
| 359 | if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path): |
| 360 | raise ValueError(f"invalid metrics path: {host}") |
| 361 | lines += [f" handle {metrics_path} {{", " respond 404", " }"] |
| 362 | api_paths = assets.get("apiPaths", {}).get(host, {}) |
| 363 | if api_paths: |
| 364 | if any(not re.fullmatch(r"/[A-Za-z0-9/._-]+\*?", path) or not re.fullmatch(r"[a-z][a-z0-9-]*", target) |
| 365 | or not (target == service or target.startswith(service + "-")) for path, target in api_paths.items()): |
| 366 | raise ValueError(f"invalid API routes: {host}") |
| 367 | lines += [f" log {service} {{", " output stdout", " format filter {", |
| 368 | " request delete", " resp_headers delete", " user_id delete", " }", " }", |
| 369 | " @not_api not path " + " ".join(sorted(api_paths)), " log_skip @not_api", |
| 370 | f" log_append service {service}", " log_append request_id {http.request.uuid}", |
| 371 | " log_append method {method}", " log_append path {path}", " log_append protocol {http.request.proto}", |
| 372 | " log_append content_length {http.request.header.Content-Length}", |
| 373 | " log_append client {remote}", |
| 374 | " log_append upstream {http.reverse_proxy.upstream.hostport}", |
| 375 | " log_append upstream_status {http.reverse_proxy.status_code}", |
| 376 | " log_append upstream_header_ms {http.reverse_proxy.upstream.latency_ms}", |
| 377 | " log_append proxy_ms {http.reverse_proxy.duration_ms}"] |
| 378 | lines += [" handle_errors 499 504 {", " @api_error path " + " ".join(sorted(api_paths)), |
| 379 | " handle @api_error {", " @http1 not protocol http/2+", |
| 380 | # Late body bytes must not become the next HTTP/1 request after a timeout. |
| 381 | " header @http1 Connection close", " header X-Request-ID {http.request.uuid}", |
| 382 | " header X-Should-Retry true", " }", ' respond "" {err.status_code}', " }"] |
| 383 | for index, target in enumerate(sorted(set(api_paths.values()))): |
| 384 | upstreams = set() |
| 385 | for instance in nomad("/v1/service/" + target, token): |
| 386 | address, port = instance["Address"], instance["Port"] |
| 387 | if instance.get("JobID") != service or not re.fullmatch(r"127\.0\.0\.1|::1", address) or type(port) is not int or not 1 <= port <= 65535: |
| 388 | raise ValueError(f"API upstream must belong to {service} on loopback: {target}") |
| 389 | upstreams.add(f"[{address}]:{port}" if ":" in address else f"{address}:{port}") |
| 390 | paths = " ".join(sorted(path for path in api_paths if api_paths[path] == target)) |
| 391 | lines += [f" @api_{index} path {paths}", f" handle @api_{index} {{", " header >X-Request-ID {http.request.uuid}", |
| 392 | # Caddy can return 499 to a connected client when its request-body idle timer expires. |
| 393 | " header X-Should-Retry true {", " match {", " status 499", |
| 394 | " header !X-Should-Retry", " }", " }", |
| 395 | " request_header X-Request-ID {http.request.uuid}"] |
| 396 | lines += proxy(" ".join(sorted(upstreams)), " ") if upstreams else [" respond 503"] |
| 397 | lines.append(" }") |
| 398 | upstreams = " ".join(sorted(route["upstreams"])) |
| 399 | if role: |
| 400 | auth = " ".join(sorted(auth_upstreams)) |
| 401 | lines += [" handle /snow.oauth2/* {", *proxy(auth, " "), " }", " handle {"] |
| 402 | if route["userHeader"]: |
| 403 | lines.append(f" request_header -{route['userHeader']}") |
| 404 | lines += [ |
| 405 | f" reverse_proxy {auth} {{", " lb_try_duration 5s", *passive_health(auth, " "), " method GET", |
| 406 | " rewrite /snow.oauth2/auth", " header_up X-Forwarded-Method {method}", |
| 407 | " header_up X-Forwarded-Uri {uri}", " @unauthorized status 401", |
| 408 | " handle_response @unauthorized {", |
| 409 | " redir * /snow.oauth2/sign_in?rd={scheme}://{host}{uri}", " }", |
| 410 | f" @allowed header X-Auth-Request-Groups *role:{role}*", |
| 411 | " handle_response @allowed {", " method {method}", " rewrite {uri}", |
| 412 | ] |
| 413 | if route["userHeader"]: |
| 414 | lines.append(f" request_header {route['userHeader']} {{rp.header.X-Auth-Request-Preferred-Username}}") |
| 415 | lines += [ |
| 416 | *proxy(upstreams, " "), " }", |
| 417 | " handle_response {", " respond 403", " }", " }", " }", "}", |
| 418 | ] |
| 419 | elif headers and (auth_upstreams or service == "copyparty"): |
| 420 | native = service == "copyparty" |
| 421 | auth = f"127.0.0.1:{int(os.environ['SNOWGLOBE_DASHBOARD_PORT'])}" if native else " ".join(sorted(auth_upstreams)) |
| 422 | if native: |
| 423 | lines += [" handle /auth/file/* {", " request_header -User-Name", " request_header -User-Groups", |
| 424 | f" request_header Snowglobe-Proxy-Token {dashboard_proof}", |
| 425 | " request_header X-Snowglobe-Client-IP {remote_host}", *proxy(auth," "), " }"] |
| 426 | else: |
| 427 | lines += [" handle /snow.oauth2/* {", *proxy(auth," "), " }"] |
| 428 | lines += [" handle {"] |
| 429 | lines += [f" request_header -{name}" for name in scrub] |
| 430 | lines += [f" reverse_proxy {auth} {{", " lb_try_duration 5s", *passive_health(auth, " "), " method GET", " rewrite " + ("/auth/file/check" if native else "/snow.oauth2/auth"), |
| 431 | *([f" header_up Snowglobe-Proxy-Token {dashboard_proof}"] if native else []), |
| 432 | " header_up X-Forwarded-Method {method}", " header_up X-Forwarded-Uri {uri}", |
| 433 | " @authenticated status 2xx", " handle_response @authenticated {"] |
| 434 | lines += [f" request_header {name} {{rp.header.{source}}}" for name, source in headers.items()] |
| 435 | if proof: |
| 436 | lines.append(f" request_header {proof} 1") |
| 437 | lines += [" }", " @anonymous status 4xx", " handle_response @anonymous {", |
| 438 | f" request_header -{scrub[0]}", " }", " }", |
| 439 | *proxy(upstreams, " "), " }", "}"] |
| 440 | else: |
| 441 | preview_proof = dashboard_preview_proof(service) |
| 442 | if preview_proof: |
| 443 | lines.append(f" request_header Snowglobe-Proxy-Token {preview_proof}") |
| 444 | page_upstream = upstreams |
| 445 | shale_page = service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service) |
| 446 | page_headers = [] |
| 447 | if shale_page: |
| 448 | page_upstream, proof = shale_gateway(service, routes, dashboard_proof) |
| 449 | if not page_upstream: |
| 450 | lines += [" respond 503", " }", "}"] |
| 451 | continue |
| 452 | page_headers = [f" request_header Snowglobe-Proxy-Token {proof}", |
| 453 | f" request_header Snowglobe-Shale-Upstream {json.dumps(sorted(route['upstreams'])[0])}", |
| 454 | " request_header Snowglobe-Shale-Uri {uri}", |
| 455 | " rewrite * /auth/shale/page"] |
| 456 | for index, (request, markup) in enumerate(head_html.items()): |
| 457 | name = f"@snowglobe_head_{index}" |
| 458 | lines += [f" {name} path {request}", f" handle {name} {{", " route {", |
| 459 | f" replace </head> {json.dumps(markup + '</head>')} {{", |
| 460 | " match {", " header Content-Type text/html*", " }", " }", |
| 461 | *[" " + line for line in page_headers], |
| 462 | *proxy(page_upstream, " ", uncompressed=True), " }", " }"] |
| 463 | page_handler = [*page_headers, *proxy(page_upstream, " ")] |
| 464 | if shale_page: |
| 465 | page_handler = [" route {", *[" " + line for line in page_handler], " }"] |
| 466 | lines += [" handle {", *(f" request_header -{name}" for name in scrub), |
| 467 | *page_handler, " }", "}"] |
| 468 | if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service): |
| 469 | lines.insert(len(lines) - 1, " }") |
| 470 | else: |
| 471 | lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"] |
| 472 | dashboard_host = "snowglobe." + os.environ["SNOWGLOBE_DOMAIN"] |
| 473 | if any(not HOST.fullmatch(host) for host in [auth_host, dashboard_host]): |
| 474 | raise ValueError("invalid login domain") |
| 475 | lines += [f"{auth_host} {{", *([" tls internal"] if auth_host.endswith(".test") else []), |
| 476 | *webauthn, " handle {", f" redir https://{dashboard_host}/ 308", " }", "}"] |
| 477 | traces = internal_services.get("victoria-traces") |
| 478 | if traces: |
| 479 | lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1", |
| 480 | *proxy(" ".join(sorted(traces)), " "), "}"] |
| 481 | dashboard_port = int(os.environ["SNOWGLOBE_DASHBOARD_PORT"]) |
| 482 | if not 1 <= dashboard_port <= 65535: |
| 483 | raise ValueError("invalid dashboard route") |
| 484 | if (80, dashboard_host) in routes: |
| 485 | raise ValueError(f"dashboard route conflicts with a Nomad service: {dashboard_host}") |
| 486 | lines.append(f"{dashboard_host} {{") |
| 487 | if dashboard_host.endswith(".test"): |
| 488 | lines.append(" tls internal") |
| 489 | lines += [" encode zstd gzip", " tracing {", " span globe", " span_attributes {", " snowglobe.kind edge", " }", " }"] |
| 490 | lines += [" handle {", " request_header -User-Name", " request_header -User-Groups", |
| 491 | f" request_header Snowglobe-Proxy-Token {dashboard_proof}", |
| 492 | " request_header X-Snowglobe-Client-IP {remote_host}", |
| 493 | *proxy(f"127.0.0.1:{dashboard_port}", " ", keep_streams=True), " }", "}"] |
| 494 | internal_port = os.environ.get("SNOWGLOBE_INTERNAL_PORT") |
| 495 | if internal_port is not None: |
| 496 | internal_host = "dashboard.internal." + os.environ["SNOWGLOBE_DOMAIN"] |
| 497 | if not HOST.fullmatch(internal_host) or not internal_port.isdecimal() or not 1 <= int(internal_port) <= 65535: |
| 498 | raise ValueError("invalid internal dashboard route") |
| 499 | if (80, internal_host) in routes or (int(internal_port), "") in routes: |
| 500 | raise ValueError("internal dashboard route conflicts with a Nomad service") |
| 501 | lines += [f"{internal_host}:{internal_port} {{", " tls internal", |
| 502 | f" @dashboard header Snowglobe-Proxy-Token {dashboard_proof}", |
| 503 | " handle @dashboard {", " request_header -Snowglobe-Proxy-Token", |
| 504 | " request_header -User-Name", " request_header -User-Groups", |
| 505 | " handle_path /nomad/* {", *proxy("127.0.0.1:4646", " ", host="{upstream_hostport}"), " }"] |
| 506 | for service, upstreams in sorted(internal_services.items()): |
| 507 | host = f"{service}.{os.environ['SNOWGLOBE_DOMAIN']}" if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service) else "{upstream_hostport}" |
| 508 | lines += [f" handle_path /services/{service}/* {{", |
| 509 | *proxy(" ".join(sorted(upstreams)), " ", host=host), " }"] |
| 510 | lines += [" handle {", " respond 404", " }", " }", " handle {", " respond 403", " }", "}"] |
| 511 | return "\n".join(lines) + "\n" |
| 512 | |
| 513 | |
| 514 | def update(content): |
| 515 | try: |
| 516 | with open(ROUTES) as file: |
| 517 | previous = file.read() |
| 518 | except FileNotFoundError: |
| 519 | previous = "" |
| 520 | if content == previous: |
| 521 | return |
| 522 | pending = ROUTES + ".pending" |
| 523 | def replace(value): |
| 524 | with open(pending, "w") as file: |
| 525 | file.write(value) |
| 526 | os.chown(pending, -1, grp.getgrnam("caddy").gr_gid) |
| 527 | os.chmod(pending, 0o640) |
| 528 | os.replace(pending, ROUTES) |
| 529 | |
| 530 | replace(content) |
| 531 | result = subprocess.run(["systemctl", "reload", "caddy"], capture_output=True, text=True) |
| 532 | if result.returncode: |
| 533 | replace(previous) |
| 534 | raise RuntimeError(result.stderr.strip()) |
| 535 | print("Caddy routes updated", flush=True) |
| 536 | |
| 537 | |
| 538 | def main(): |
| 539 | with open(TOKEN) as file: |
| 540 | token = file.read().strip() |
| 541 | while True: |
| 542 | try: |
| 543 | update(render(token)) |
| 544 | except Exception as error: |
| 545 | print(f"Caddy route update failed: {error}", file=sys.stderr, flush=True) |
| 546 | time.sleep(5) |
| 547 | |
| 548 | |
| 549 | if __name__ == "__main__": |
| 550 | main() |