| 1 | #!/usr/bin/env python3 |
| 2 | import hashlib |
| 3 | import hmac |
| 4 | import http.client |
| 5 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 6 | import json |
| 7 | import os |
| 8 | from pathlib import Path |
| 9 | import signal |
| 10 | import subprocess |
| 11 | import sys |
| 12 | import threading |
| 13 | |
| 14 | WAIT_SECONDS = 8 * 60 * 60 |
| 15 | HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"} |
| 16 | STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering"))) |
| 17 | STEERING_REVISION = hashlib.sha256(b"".join( |
| 18 | (STEERING_DIR / name).read_bytes() |
| 19 | for name in ("golden-gate.safetensors", "owl.safetensors", "snowglobe_ai_steering.py", "runtime.patch") |
| 20 | )).hexdigest() |
| 21 | |
| 22 | |
| 23 | def agent_provider(): |
| 24 | catalog = json.loads(Path(__file__).with_name("catalog.json").read_text())["models"] |
| 25 | base = catalog[0] |
| 26 | models = [] |
| 27 | for entry in catalog: |
| 28 | model = {**base, **entry} |
| 29 | models.append({ |
| 30 | "id": model["slug"], "name": model["display_name"], "description": model["description"], |
| 31 | "hidden": model["visibility"] != "list", |
| 32 | "contextWindow": model["context_window"], |
| 33 | "effectiveContextWindowPercent": model["effective_context_window_percent"], |
| 34 | "autoCompactTokenLimit": model["auto_compact_token_limit"], |
| 35 | "inputModalities": model["input_modalities"], |
| 36 | "reasoning": {"levels": model["supported_reasoning_levels"], |
| 37 | "default": model["default_reasoning_level"], |
| 38 | "summaries": model["supports_reasoning_summary_parameter"]}, |
| 39 | "tools": {"applyPatch": model["apply_patch_tool_type"], "shell": model["shell_type"], |
| 40 | "webSearch": model["supports_search_tool"], "mode": model["tool_mode"]}, |
| 41 | "truncation": model["truncation_policy"], "verbosity": model["support_verbosity"], |
| 42 | }) |
| 43 | return { |
| 44 | "version": 1, "name": "Snow Globe", "icon": {"emoji": "❄️", "color": "teal"}, |
| 45 | "harness": "codex", "harnesses": ["codex", "claude"], |
| 46 | "apis": [{"kind": kind} for kind in ["responses", "anthropic", "chat-completions"]], |
| 47 | "auth": {"required": True, "header": "Authorization"}, |
| 48 | "transport": {"websockets": False, "streamIdleTimeoutMs": WAIT_SECONDS * 1000}, |
| 49 | "defaults": {"model": base["slug"], "reasoningEffort": base["default_reasoning_level"], |
| 50 | "reasoningSummary": base["default_reasoning_summary"]}, |
| 51 | "instructions": {"base": base["base_instructions"]}, |
| 52 | "harnessOverrides": {"codex": {"config": {"web_search": "disabled"}}}, |
| 53 | "models": models, |
| 54 | } |
| 55 | |
| 56 | |
| 57 | def route_model(data): |
| 58 | golden = data.get("model") == "golden-gate-qwen3.8-27B" |
| 59 | owl = data.get("model") == "owl-qwen3.8-27B" |
| 60 | if (golden or owl) and data.get("use_beam_search"): |
| 61 | raise ValueError("Steered models do not support beam search. Set use_beam_search to false.") |
| 62 | data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), |
| 63 | "snowglobe_golden_gate": int(golden), "snowglobe_owl": int(owl)} |
| 64 | data["vllm_xargs"].pop("snowglobe_repetition_penalty", None) |
| 65 | salt = data.get("cache_salt") |
| 66 | if salt is not None and (not isinstance(salt, str) or not salt): |
| 67 | raise ValueError("cache_salt must be a nonempty string") |
| 68 | data["cache_salt"] = hashlib.sha256(json.dumps([ |
| 69 | STEERING_REVISION + ":" + data["model"] if golden or owl else "normal", salt, |
| 70 | ]).encode()).hexdigest() |
| 71 | if golden or owl: |
| 72 | data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} |
| 73 | if owl: |
| 74 | for name, value in [("repetition_penalty", 1.2), ("temperature", 0.5)]: |
| 75 | if data.get(name) is None: |
| 76 | data[name] = value |
| 77 | return data |
| 78 | |
| 79 | |
| 80 | def normalize(request): |
| 81 | data = request |
| 82 | identities = {} |
| 83 | tools = [] |
| 84 | |
| 85 | def add(tool, namespace=None): |
| 86 | kind = tool.get("type") |
| 87 | if kind not in {"function", "custom"}: |
| 88 | raise ValueError(f"Unsupported Responses tool type: {kind}") |
| 89 | original = tool["name"] |
| 90 | name = f"{namespace}__{original}" if namespace else original |
| 91 | if name in identities: |
| 92 | raise ValueError(f"Duplicate tool name: {name}") |
| 93 | identities[name] = (namespace, original, kind) |
| 94 | if kind == "custom": |
| 95 | description = tool.get("description", "") + " Pass the raw tool input as the input string." |
| 96 | grammar = tool.get("format", {}) |
| 97 | if grammar.get("type") == "grammar": |
| 98 | description += " The input must follow this grammar:\n" + grammar["definition"] |
| 99 | tool = { |
| 100 | "type": "function", "name": name, |
| 101 | "description": description, |
| 102 | "parameters": {"type": "object", "properties": {"input": {"type": "string"}}, "required": ["input"]}, |
| 103 | } |
| 104 | else: |
| 105 | tool["name"] = name |
| 106 | tools.append(tool) |
| 107 | |
| 108 | for tool in data.get("tools", []): |
| 109 | if tool.get("type") == "namespace": |
| 110 | for nested in tool.get("tools", []): |
| 111 | add(nested, tool["name"]) |
| 112 | else: |
| 113 | add(tool) |
| 114 | data["tools"] = tools |
| 115 | incoming = data.get("input", []) |
| 116 | if isinstance(incoming, str): |
| 117 | incoming = [{"role": "user", "content": incoming}] |
| 118 | instructions = [] |
| 119 | if data.get("instructions"): |
| 120 | instructions.append(data.pop("instructions")) |
| 121 | messages = [] |
| 122 | for item in incoming: |
| 123 | if item.get("role") in {"system", "developer"}: |
| 124 | content = item.get("content", "") |
| 125 | instructions.append(content if isinstance(content, str) else "\n".join(part.get("text", "") for part in content)) |
| 126 | continue |
| 127 | if item.get("type") in {"function_call", "custom_tool_call"}: |
| 128 | namespace = item.pop("namespace", None) |
| 129 | if namespace: |
| 130 | item["name"] = namespace + "__" + item["name"] |
| 131 | if item["type"] == "custom_tool_call": |
| 132 | item["type"] = "function_call" |
| 133 | item["arguments"] = json.dumps({"input": item.pop("input")}) |
| 134 | elif item.get("type") == "custom_tool_call_output": |
| 135 | item["type"] = "function_call_output" |
| 136 | messages.append(item) |
| 137 | if instructions: |
| 138 | messages.insert(0, {"role": "system", "content": "\n\n".join(instructions)}) |
| 139 | data["input"] = messages |
| 140 | if isinstance(data.get("tool_choice"), dict): |
| 141 | choice = data["tool_choice"] |
| 142 | namespace = choice.pop("namespace", None) |
| 143 | if namespace: |
| 144 | choice["name"] = namespace + "__" + choice["name"] |
| 145 | if choice.get("type") == "custom": |
| 146 | choice["type"] = "function" |
| 147 | return data, identities |
| 148 | |
| 149 | |
| 150 | def restore(value, identities): |
| 151 | if isinstance(value, list): |
| 152 | return [restore(item, identities) for item in value] |
| 153 | if not isinstance(value, dict): |
| 154 | return value |
| 155 | value = {key: restore(item, identities) for key, item in value.items()} |
| 156 | if value.get("type") == "function_call" and value.get("name") in identities: |
| 157 | namespace, name, kind = identities[value["name"]] |
| 158 | value["name"] = name |
| 159 | if namespace: |
| 160 | value["namespace"] = namespace |
| 161 | if kind == "custom": |
| 162 | value["type"] = "custom_tool_call" |
| 163 | arguments = value.pop("arguments", "") |
| 164 | value["input"] = json.loads(arguments).get("input", "") if arguments else "" |
| 165 | return value |
| 166 | |
| 167 | |
| 168 | def normalize_anthropic(request): |
| 169 | data = request |
| 170 | if (data.get("vllm_xargs") or {}).get("snowglobe_owl"): |
| 171 | data["vllm_xargs"]["snowglobe_repetition_penalty"] = data.pop("repetition_penalty") |
| 172 | def text(content): |
| 173 | if isinstance(content, str): |
| 174 | return content |
| 175 | if any(part.get("type") != "text" for part in content): |
| 176 | raise ValueError("Anthropic system content must contain text blocks.") |
| 177 | return "\n\n".join(part["text"] for part in content) |
| 178 | instructions = [text(data["system"])] if data.get("system") else [] |
| 179 | messages = [] |
| 180 | for message in data.get("messages", []): |
| 181 | if message.get("role") in {"system", "developer"}: |
| 182 | instructions.append(text(message["content"])) |
| 183 | else: |
| 184 | messages.append(message) |
| 185 | if instructions: |
| 186 | data["system"] = "\n\n".join(instructions) |
| 187 | data["messages"] = messages |
| 188 | if (data.get("output_config") or {}).get("effort") in {"high", "max"}: |
| 189 | data["output_config"]["effort"] = "xhigh" |
| 190 | return data |
| 191 | |
| 192 | |
| 193 | class Handler(BaseHTTPRequestHandler): |
| 194 | protocol_version = "HTTP/1.1" |
| 195 | |
| 196 | def handle(self): |
| 197 | try: |
| 198 | super().handle() |
| 199 | except (ConnectionResetError, BrokenPipeError): |
| 200 | pass |
| 201 | |
| 202 | def log_message(self, *args): |
| 203 | pass # Inference bodies and keys must not enter access logs. |
| 204 | |
| 205 | def do_GET(self): |
| 206 | if self.path.split("?", 1)[0] == "/.well-known/ai-agent-provider.json": |
| 207 | content = json.dumps(agent_provider(), ensure_ascii=False).encode() |
| 208 | self.send_response(200) |
| 209 | self.send_header("Content-Type", "application/json; charset=utf-8") |
| 210 | self.send_header("Cache-Control", "no-store") |
| 211 | self.send_header("Content-Length", str(len(content))) |
| 212 | self.end_headers() |
| 213 | self.wfile.write(content) |
| 214 | return |
| 215 | self.proxy() |
| 216 | |
| 217 | def do_POST(self): |
| 218 | self.proxy() |
| 219 | |
| 220 | def proxy(self): |
| 221 | connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=WAIT_SECONDS) |
| 222 | started = False |
| 223 | try: |
| 224 | length = int(self.headers.get("Content-Length", "0")) |
| 225 | if length < 0 or length > 20 * 1024 * 1024 or self.headers.get("Transfer-Encoding"): |
| 226 | self.send_error(413) |
| 227 | return |
| 228 | # Authenticate before allocating inference bodies. |
| 229 | if self.path != "/health": |
| 230 | key = os.environ["VLLM_API_KEY"] |
| 231 | supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ") |
| 232 | if not hmac.compare_digest(supplied, key): |
| 233 | self.send_error(401) |
| 234 | self.close_connection = True |
| 235 | return |
| 236 | body = self.rfile.read(length) |
| 237 | identities = {} |
| 238 | if self.command == "POST": |
| 239 | path = self.path.split("?", 1)[0] |
| 240 | if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}: |
| 241 | self.send_error(404) |
| 242 | return |
| 243 | data = route_model(json.loads(body)) |
| 244 | if path == "/v1/responses": |
| 245 | data, identities = normalize(data) |
| 246 | elif path in {"/v1/messages", "/v1/messages/count_tokens"}: |
| 247 | data = normalize_anthropic(data) |
| 248 | body = json.dumps(data).encode() |
| 249 | headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} |
| 250 | headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] |
| 251 | connection.request(self.command, self.path, body=body or None, headers=headers) |
| 252 | response = connection.getresponse() |
| 253 | streaming = "text/event-stream" in response.getheader("Content-Type", "") |
| 254 | if not streaming: |
| 255 | content = response.read() |
| 256 | if identities and response.status == 200: |
| 257 | content = json.dumps(restore(json.loads(content), identities)).encode() |
| 258 | self.send_response(response.status) |
| 259 | for key, val in response.getheaders(): |
| 260 | if key.lower() not in HOP_HEADERS: |
| 261 | self.send_header(key, val) |
| 262 | self.send_header("Content-Length", str(len(content))) |
| 263 | self.end_headers() |
| 264 | self.wfile.write(content) |
| 265 | return |
| 266 | self.send_response(response.status) |
| 267 | started = True |
| 268 | self.send_header("Content-Type", "text/event-stream") |
| 269 | self.send_header("Cache-Control", "no-cache") |
| 270 | self.send_header("Connection", "close") |
| 271 | self.end_headers() |
| 272 | self.close_connection = True |
| 273 | pending = b"" |
| 274 | while chunk := response.read1(65536): |
| 275 | if not identities: |
| 276 | self.wfile.write(chunk) |
| 277 | self.wfile.flush() |
| 278 | continue |
| 279 | pending += chunk |
| 280 | while b"\n\n" in pending: |
| 281 | frame, pending = pending.split(b"\n\n", 1) |
| 282 | lines = [] |
| 283 | for line in frame.split(b"\n"): |
| 284 | if line.startswith(b"data: ") and line[6:] != b"[DONE]": |
| 285 | line = b"data: " + json.dumps(restore(json.loads(line[6:]), identities)).encode() |
| 286 | lines.append(line) |
| 287 | self.wfile.write(b"\n".join(lines) + b"\n\n") |
| 288 | self.wfile.flush() |
| 289 | if pending: |
| 290 | self.wfile.write(pending) |
| 291 | except (BrokenPipeError, ConnectionResetError): |
| 292 | pass |
| 293 | except (ValueError, KeyError, TypeError) as error: |
| 294 | if not started: |
| 295 | self.send_error(400, str(error)) |
| 296 | self.close_connection = True |
| 297 | except (OSError, http.client.HTTPException): |
| 298 | if not started: |
| 299 | self.send_error(502) |
| 300 | self.close_connection = True |
| 301 | finally: |
| 302 | connection.close() |
| 303 | |
| 304 | |
| 305 | if __name__ == "__main__": |
| 306 | backend = subprocess.Popen(["bash", "/app/single-user/start_qwen.sh"]) |
| 307 | server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler) |
| 308 | def stop(signum, frame): |
| 309 | if backend.poll() is None: |
| 310 | backend.terminate() |
| 311 | signal.signal(signal.SIGTERM, stop) |
| 312 | signal.signal(signal.SIGINT, stop) |
| 313 | def monitor(): |
| 314 | backend.wait() |
| 315 | server.shutdown() |
| 316 | threading.Thread(target=monitor, daemon=True).start() |
| 317 | try: |
| 318 | server.serve_forever() |
| 319 | finally: |
| 320 | server.server_close() |
| 321 | if backend.poll() is None: |
| 322 | backend.terminate() |
| 323 | sys.exit(backend.wait()) |