1#!/usr/bin/env python3
2import hashlib
3import hmac
4import http.client
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import os
8from pathlib import Path
9import signal
10import subprocess
11import sys
12import threading
13
14WAIT_SECONDS = 8 * 60 * 60
15HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"}
16STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering")))
17STEERING_REVISION = hashlib.sha256(b"".join(
18 (STEERING_DIR / name).read_bytes()
19 for name in ("golden-gate.safetensors", "owl.safetensors", "snowglobe_ai_steering.py", "runtime.patch")
20)).hexdigest()
21
22
23def agent_provider():
24 catalog = json.loads(Path(__file__).with_name("catalog.json").read_text())["models"]
25 base = catalog[0]
26 models = []
27 for entry in catalog:
28 model = {**base, **entry}
29 models.append({
30 "id": model["slug"], "name": model["display_name"], "description": model["description"],
31 "hidden": model["visibility"] != "list",
32 "contextWindow": model["context_window"],
33 "effectiveContextWindowPercent": model["effective_context_window_percent"],
34 "autoCompactTokenLimit": model["auto_compact_token_limit"],
35 "inputModalities": model["input_modalities"],
36 "reasoning": {"levels": model["supported_reasoning_levels"],
37 "default": model["default_reasoning_level"],
38 "summaries": model["supports_reasoning_summary_parameter"]},
39 "tools": {"applyPatch": model["apply_patch_tool_type"], "shell": model["shell_type"],
40 "webSearch": model["supports_search_tool"], "mode": model["tool_mode"]},
41 "truncation": model["truncation_policy"], "verbosity": model["support_verbosity"],
42 })
43 return {
44 "version": 1, "name": "Snow Globe", "icon": {"emoji": "❄️", "color": "teal"},
45 "harness": "codex", "harnesses": ["codex", "claude"],
46 "apis": [{"kind": kind} for kind in ["responses", "anthropic", "chat-completions"]],
47 "auth": {"required": True, "header": "Authorization"},
48 "transport": {"websockets": False, "streamIdleTimeoutMs": WAIT_SECONDS * 1000},
49 "defaults": {"model": base["slug"], "reasoningEffort": base["default_reasoning_level"],
50 "reasoningSummary": base["default_reasoning_summary"]},
51 "instructions": {"base": base["base_instructions"]},
52 "harnessOverrides": {"codex": {"config": {"web_search": "disabled"}}},
53 "models": models,
54 }
55
56
57def route_model(data):
58 golden = data.get("model") == "golden-gate-qwen3.8-27B"
59 owl = data.get("model") == "owl-qwen3.8-27B"
60 if (golden or owl) and data.get("use_beam_search"):
61 raise ValueError("Steered models do not support beam search. Set use_beam_search to false.")
62 data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}),
63 "snowglobe_golden_gate": int(golden), "snowglobe_owl": int(owl)}
64 data["vllm_xargs"].pop("snowglobe_repetition_penalty", None)
65 salt = data.get("cache_salt")
66 if salt is not None and (not isinstance(salt, str) or not salt):
67 raise ValueError("cache_salt must be a nonempty string")
68 data["cache_salt"] = hashlib.sha256(json.dumps([
69 STEERING_REVISION + ":" + data["model"] if golden or owl else "normal", salt,
70 ]).encode()).hexdigest()
71 if golden or owl:
72 data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False}
73 if owl:
74 for name, value in [("repetition_penalty", 1.2), ("temperature", 0.5)]:
75 if data.get(name) is None:
76 data[name] = value
77 return data
78
79
80def normalize(request):
81 data = request
82 identities = {}
83 tools = []
84
85 def add(tool, namespace=None):
86 kind = tool.get("type")
87 if kind not in {"function", "custom"}:
88 raise ValueError(f"Unsupported Responses tool type: {kind}")
89 original = tool["name"]
90 name = f"{namespace}__{original}" if namespace else original
91 if name in identities:
92 raise ValueError(f"Duplicate tool name: {name}")
93 identities[name] = (namespace, original, kind)
94 if kind == "custom":
95 description = tool.get("description", "") + " Pass the raw tool input as the input string."
96 grammar = tool.get("format", {})
97 if grammar.get("type") == "grammar":
98 description += " The input must follow this grammar:\n" + grammar["definition"]
99 tool = {
100 "type": "function", "name": name,
101 "description": description,
102 "parameters": {"type": "object", "properties": {"input": {"type": "string"}}, "required": ["input"]},
103 }
104 else:
105 tool["name"] = name
106 tools.append(tool)
107
108 for tool in data.get("tools", []):
109 if tool.get("type") == "namespace":
110 for nested in tool.get("tools", []):
111 add(nested, tool["name"])
112 else:
113 add(tool)
114 data["tools"] = tools
115 incoming = data.get("input", [])
116 if isinstance(incoming, str):
117 incoming = [{"role": "user", "content": incoming}]
118 instructions = []
119 if data.get("instructions"):
120 instructions.append(data.pop("instructions"))
121 messages = []
122 for item in incoming:
123 if item.get("role") in {"system", "developer"}:
124 content = item.get("content", "")
125 instructions.append(content if isinstance(content, str) else "\n".join(part.get("text", "") for part in content))
126 continue
127 if item.get("type") in {"function_call", "custom_tool_call"}:
128 namespace = item.pop("namespace", None)
129 if namespace:
130 item["name"] = namespace + "__" + item["name"]
131 if item["type"] == "custom_tool_call":
132 item["type"] = "function_call"
133 item["arguments"] = json.dumps({"input": item.pop("input")})
134 elif item.get("type") == "custom_tool_call_output":
135 item["type"] = "function_call_output"
136 messages.append(item)
137 if instructions:
138 messages.insert(0, {"role": "system", "content": "\n\n".join(instructions)})
139 data["input"] = messages
140 if isinstance(data.get("tool_choice"), dict):
141 choice = data["tool_choice"]
142 namespace = choice.pop("namespace", None)
143 if namespace:
144 choice["name"] = namespace + "__" + choice["name"]
145 if choice.get("type") == "custom":
146 choice["type"] = "function"
147 return data, identities
148
149
150def restore(value, identities):
151 if isinstance(value, list):
152 return [restore(item, identities) for item in value]
153 if not isinstance(value, dict):
154 return value
155 value = {key: restore(item, identities) for key, item in value.items()}
156 if value.get("type") == "function_call" and value.get("name") in identities:
157 namespace, name, kind = identities[value["name"]]
158 value["name"] = name
159 if namespace:
160 value["namespace"] = namespace
161 if kind == "custom":
162 value["type"] = "custom_tool_call"
163 arguments = value.pop("arguments", "")
164 value["input"] = json.loads(arguments).get("input", "") if arguments else ""
165 return value
166
167
168def normalize_anthropic(request):
169 data = request
170 if (data.get("vllm_xargs") or {}).get("snowglobe_owl"):
171 data["vllm_xargs"]["snowglobe_repetition_penalty"] = data.pop("repetition_penalty")
172 def text(content):
173 if isinstance(content, str):
174 return content
175 if any(part.get("type") != "text" for part in content):
176 raise ValueError("Anthropic system content must contain text blocks.")
177 return "\n\n".join(part["text"] for part in content)
178 instructions = [text(data["system"])] if data.get("system") else []
179 messages = []
180 for message in data.get("messages", []):
181 if message.get("role") in {"system", "developer"}:
182 instructions.append(text(message["content"]))
183 else:
184 messages.append(message)
185 if instructions:
186 data["system"] = "\n\n".join(instructions)
187 data["messages"] = messages
188 if (data.get("output_config") or {}).get("effort") in {"high", "max"}:
189 data["output_config"]["effort"] = "xhigh"
190 return data
191
192
193class Handler(BaseHTTPRequestHandler):
194 protocol_version = "HTTP/1.1"
195
196 def handle(self):
197 try:
198 super().handle()
199 except (ConnectionResetError, BrokenPipeError):
200 pass
201
202 def log_message(self, *args):
203 pass # Inference bodies and keys must not enter access logs.
204
205 def do_GET(self):
206 if self.path.split("?", 1)[0] == "/.well-known/ai-agent-provider.json":
207 content = json.dumps(agent_provider(), ensure_ascii=False).encode()
208 self.send_response(200)
209 self.send_header("Content-Type", "application/json; charset=utf-8")
210 self.send_header("Cache-Control", "no-store")
211 self.send_header("Content-Length", str(len(content)))
212 self.end_headers()
213 self.wfile.write(content)
214 return
215 self.proxy()
216
217 def do_POST(self):
218 self.proxy()
219
220 def proxy(self):
221 connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=WAIT_SECONDS)
222 started = False
223 try:
224 length = int(self.headers.get("Content-Length", "0"))
225 if length < 0 or length > 20 * 1024 * 1024 or self.headers.get("Transfer-Encoding"):
226 self.send_error(413)
227 return
228 # Authenticate before allocating inference bodies.
229 if self.path != "/health":
230 key = os.environ["VLLM_API_KEY"]
231 supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ")
232 if not hmac.compare_digest(supplied, key):
233 self.send_error(401)
234 self.close_connection = True
235 return
236 body = self.rfile.read(length)
237 identities = {}
238 if self.command == "POST":
239 path = self.path.split("?", 1)[0]
240 if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}:
241 self.send_error(404)
242 return
243 data = route_model(json.loads(body))
244 if path == "/v1/responses":
245 data, identities = normalize(data)
246 elif path in {"/v1/messages", "/v1/messages/count_tokens"}:
247 data = normalize_anthropic(data)
248 body = json.dumps(data).encode()
249 headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}}
250 headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"]
251 connection.request(self.command, self.path, body=body or None, headers=headers)
252 response = connection.getresponse()
253 streaming = "text/event-stream" in response.getheader("Content-Type", "")
254 if not streaming:
255 content = response.read()
256 if identities and response.status == 200:
257 content = json.dumps(restore(json.loads(content), identities)).encode()
258 self.send_response(response.status)
259 for key, val in response.getheaders():
260 if key.lower() not in HOP_HEADERS:
261 self.send_header(key, val)
262 self.send_header("Content-Length", str(len(content)))
263 self.end_headers()
264 self.wfile.write(content)
265 return
266 self.send_response(response.status)
267 started = True
268 self.send_header("Content-Type", "text/event-stream")
269 self.send_header("Cache-Control", "no-cache")
270 self.send_header("Connection", "close")
271 self.end_headers()
272 self.close_connection = True
273 pending = b""
274 while chunk := response.read1(65536):
275 if not identities:
276 self.wfile.write(chunk)
277 self.wfile.flush()
278 continue
279 pending += chunk
280 while b"\n\n" in pending:
281 frame, pending = pending.split(b"\n\n", 1)
282 lines = []
283 for line in frame.split(b"\n"):
284 if line.startswith(b"data: ") and line[6:] != b"[DONE]":
285 line = b"data: " + json.dumps(restore(json.loads(line[6:]), identities)).encode()
286 lines.append(line)
287 self.wfile.write(b"\n".join(lines) + b"\n\n")
288 self.wfile.flush()
289 if pending:
290 self.wfile.write(pending)
291 except (BrokenPipeError, ConnectionResetError):
292 pass
293 except (ValueError, KeyError, TypeError) as error:
294 if not started:
295 self.send_error(400, str(error))
296 self.close_connection = True
297 except (OSError, http.client.HTTPException):
298 if not started:
299 self.send_error(502)
300 self.close_connection = True
301 finally:
302 connection.close()
303
304
305if __name__ == "__main__":
306 backend = subprocess.Popen(["bash", "/app/single-user/start_qwen.sh"])
307 server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler)
308 def stop(signum, frame):
309 if backend.poll() is None:
310 backend.terminate()
311 signal.signal(signal.SIGTERM, stop)
312 signal.signal(signal.SIGINT, stop)
313 def monitor():
314 backend.wait()
315 server.shutdown()
316 threading.Thread(target=monitor, daemon=True).start()
317 try:
318 server.serve_forever()
319 finally:
320 server.server_close()
321 if backend.poll() is None:
322 backend.terminate()
323 sys.exit(backend.wait())