| 1 | import unittest |
| 2 | import json |
| 3 | import hashlib |
| 4 | from http.client import HTTPConnection |
| 5 | from pathlib import Path |
| 6 | import socket |
| 7 | import threading |
| 8 | from unittest.mock import patch |
| 9 | from gateway import Handler, ThreadingHTTPServer, agent_provider, normalize, normalize_anthropic, restore, route_model |
| 10 | |
| 11 | |
| 12 | class ResponsesCompatibility(unittest.TestCase): |
| 13 | def test_discovery_is_public_without_opening_inference(self): |
| 14 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) |
| 15 | worker = threading.Thread(target=server.serve_forever, daemon=True) |
| 16 | worker.start() |
| 17 | try: |
| 18 | with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: |
| 19 | for headers in [{}, {"Authorization": "Bearer disposable-test-key"}, {"x-api-key": "wrong-key"}]: |
| 20 | client = HTTPConnection(*server.server_address, timeout=5) |
| 21 | try: |
| 22 | client.request("GET", "/.well-known/ai-agent-provider.json?probe=1", headers=headers) |
| 23 | response = client.getresponse() |
| 24 | self.assertEqual(response.status, 200) |
| 25 | self.assertEqual(response.getheader("Content-Type"), "application/json; charset=utf-8") |
| 26 | content = response.read() |
| 27 | self.assertEqual(int(response.getheader("Content-Length")), len(content)) |
| 28 | self.assertIn("❄️".encode("utf-8"), content) |
| 29 | self.assertEqual(json.loads(content.decode("utf-8")), agent_provider()) |
| 30 | self.assertNotIn(b"disposable-test-key", content) |
| 31 | client.request("GET", "/v1/models") |
| 32 | response = client.getresponse() |
| 33 | self.assertEqual(response.status, 401) |
| 34 | response.read() |
| 35 | finally: |
| 36 | client.close() |
| 37 | upstream.return_value.request.assert_not_called() |
| 38 | finally: |
| 39 | server.shutdown() |
| 40 | server.server_close() |
| 41 | worker.join() |
| 42 | |
| 43 | def test_discovery_derives_inherited_capabilities_from_catalog(self): |
| 44 | catalog = json.loads(Path(__file__).with_name("catalog.json").read_text()) |
| 45 | catalog["models"][0]["context_window"] = 123456 |
| 46 | with patch("gateway.Path.read_text", return_value=json.dumps(catalog)): |
| 47 | provider = agent_provider() |
| 48 | self.assertEqual(provider["auth"], {"required": True, "header": "Authorization"}) |
| 49 | self.assertEqual(provider["defaults"]["model"], catalog["models"][0]["slug"]) |
| 50 | self.assertEqual([model["id"] for model in provider["models"]], [model["slug"] for model in catalog["models"]]) |
| 51 | for model in provider["models"]: |
| 52 | self.assertEqual(model["contextWindow"], 123456) |
| 53 | self.assertEqual(model["inputModalities"], ["text", "image"]) |
| 54 | self.assertEqual([level["effort"] for level in provider["models"][0]["reasoning"]["levels"]], ["low", "medium", "xhigh"]) |
| 55 | for model in provider["models"][1:]: |
| 56 | self.assertEqual(model["reasoning"]["default"], "none") |
| 57 | self.assertEqual(model["reasoning"]["levels"], [{"effort": "none", "description": "No reasoning"}]) |
| 58 | self.assertNotIn("harnessOverrides", model) |
| 59 | |
| 60 | def test_unsupported_inference_routes_never_reach_backend(self): |
| 61 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) |
| 62 | worker = threading.Thread(target=server.serve_forever, daemon=True) |
| 63 | worker.start() |
| 64 | try: |
| 65 | with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: |
| 66 | for path in ["/v1/chat/completions/batch", "/v1/chat/completions/", "/v1/chat%2fcompletions", "/inference/v1/generate"]: |
| 67 | body = json.dumps({"model": "golden-gate-qwen3.8-27B"}).encode() |
| 68 | with socket.create_connection(server.server_address, timeout=5) as client: |
| 69 | client.sendall(f"POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer disposable-test-key\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body) |
| 70 | self.assertIn(b" 404 ", client.recv(4096).split(b"\r\n")[0]) |
| 71 | upstream.return_value.request.assert_not_called() |
| 72 | finally: |
| 73 | server.shutdown() |
| 74 | server.server_close() |
| 75 | worker.join() |
| 76 | |
| 77 | def test_steering_and_cache_namespace_follow_model_selection(self): |
| 78 | normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 1, "snowglobe_repetition_penalty": 2}}) |
| 79 | golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0, "snowglobe_owl": 1}, "chat_template_kwargs": {"enable_thinking": True}}) |
| 80 | owl = route_model({"model": "owl-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 0}, "chat_template_kwargs": {"enable_thinking": True}}) |
| 81 | self.assertEqual(normal["vllm_xargs"]["snowglobe_golden_gate"], 0) |
| 82 | self.assertEqual(normal["vllm_xargs"]["snowglobe_owl"], 0) |
| 83 | self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) |
| 84 | self.assertEqual(golden["vllm_xargs"]["snowglobe_golden_gate"], 1) |
| 85 | self.assertEqual(golden["vllm_xargs"]["snowglobe_owl"], 0) |
| 86 | self.assertEqual(owl["vllm_xargs"]["snowglobe_golden_gate"], 0) |
| 87 | self.assertEqual(owl["vllm_xargs"]["snowglobe_owl"], 1) |
| 88 | self.assertNotIn("chat_template_kwargs", normal) |
| 89 | self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) |
| 90 | self.assertFalse(owl["chat_template_kwargs"]["enable_thinking"]) |
| 91 | self.assertNotIn("repetition_penalty", normal) |
| 92 | self.assertNotIn("repetition_penalty", golden) |
| 93 | self.assertNotIn("temperature", normal) |
| 94 | self.assertNotIn("temperature", golden) |
| 95 | self.assertEqual(owl["repetition_penalty"], 1.2) |
| 96 | self.assertEqual(owl["temperature"], 0.5) |
| 97 | self.assertEqual(len({normal["cache_salt"], golden["cache_salt"], owl["cache_salt"]}), 3) |
| 98 | self.assertEqual(normal["cache_salt"], hashlib.sha256(json.dumps(["normal", None]).encode()).hexdigest()) |
| 99 | self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) |
| 100 | self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) |
| 101 | self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) |
| 102 | for model in ["golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"]: |
| 103 | with self.assertRaisesRegex(ValueError, "beam search"): |
| 104 | route_model({"model": model, "use_beam_search": True}) |
| 105 | |
| 106 | def test_owl_profile_reaches_anthropic_and_respects_explicit_sampling(self): |
| 107 | for supplied, penalty, temperature in [({}, 1.2, 0.5), |
| 108 | ({"repetition_penalty": None, "temperature": None}, 1.2, 0.5), |
| 109 | ({"repetition_penalty": 1.05, "temperature": 0}, 1.05, 0)]: |
| 110 | data = normalize_anthropic(route_model({"model": "owl-qwen3.8-27B", **supplied})) |
| 111 | self.assertNotIn("repetition_penalty", data) |
| 112 | self.assertEqual(data["vllm_xargs"]["snowglobe_repetition_penalty"], penalty) |
| 113 | self.assertEqual(data["temperature"], temperature) |
| 114 | normal = normalize_anthropic(route_model({"model": "qwen3.8-27b"})) |
| 115 | self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"]) |
| 116 | |
| 117 | def test_anthropic_effort_matches_the_qwen_template(self): |
| 118 | for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: |
| 119 | with self.subTest(effort=requested): |
| 120 | data = normalize_anthropic({"output_config": {"effort": requested, "format": {"type": "json_schema"}}}) |
| 121 | self.assertEqual(data["output_config"], {"effort": expected, "format": {"type": "json_schema"}}) |
| 122 | |
| 123 | def test_claude_normal_configuration_system_blocks(self): |
| 124 | request = {"system": [{"type": "text", "text": "instructions", "cache_control": {"type": "ephemeral"}}, {"type": "text", "text": "workspace context"}], "messages": [{"role": "user", "content": "task"}, {"role": "system", "content": "late system message"}]} |
| 125 | data = normalize_anthropic(request) |
| 126 | self.assertEqual(data["system"], "instructions\n\nworkspace context\n\nlate system message") |
| 127 | self.assertEqual(data["messages"], [{"role": "user", "content": "task"}]) |
| 128 | |
| 129 | def test_developer_messages_after_user_are_preserved_at_start(self): |
| 130 | request = {"instructions": "first", "input": [ |
| 131 | {"role": "developer", "content": [{"type": "input_text", "text": "second"}]}, |
| 132 | {"role": "user", "content": "task"}, |
| 133 | {"role": "developer", "content": "third"}, |
| 134 | ]} |
| 135 | data, _ = normalize(request) |
| 136 | self.assertEqual(data["input"], [ |
| 137 | {"role": "system", "content": "first\n\nsecond\n\nthird"}, |
| 138 | {"role": "user", "content": "task"}, |
| 139 | ]) |
| 140 | |
| 141 | def test_namespaced_tool_roundtrip_including_history(self): |
| 142 | request = {"tools": [{"type": "namespace", "name": "functions", "tools": [ |
| 143 | {"type": "function", "name": "exec_command", "parameters": {"type": "object"}}, |
| 144 | ]}], "input": [{"type": "function_call", "namespace": "functions", "name": "exec_command", "arguments": "{}", "call_id": "call_1"}]} |
| 145 | data, identities = normalize(request) |
| 146 | self.assertEqual(data["tools"][0]["name"], data["input"][0]["name"]) |
| 147 | self.assertEqual(restore(data["input"], identities), [{"type": "function_call", "namespace": "functions", "name": "exec_command", "arguments": "{}", "call_id": "call_1"}]) |
| 148 | |
| 149 | def test_raw_patch_text_survives_custom_tool_roundtrip(self): |
| 150 | text = "*** Begin Patch\n*** Add File: example.txt\n+hello\n*** End Patch" |
| 151 | request = {"tools": [{"type": "namespace", "name": "functions", "tools": [ |
| 152 | {"type": "custom", "name": "apply_patch", "format": {"type": "grammar", "syntax": "lark", "definition": "ignored by JSON adapter"}}, |
| 153 | ]}], "input": [{"type": "custom_tool_call", "namespace": "functions", "name": "apply_patch", "input": text, "call_id": "call_2"}, {"type": "custom_tool_call_output", "call_id": "call_2", "output": "ok"}]} |
| 154 | data, identities = normalize(request) |
| 155 | self.assertEqual(restore(data["input"][0], identities), {"type": "custom_tool_call", "namespace": "functions", "name": "apply_patch", "input": text, "call_id": "call_2"}) |
| 156 | self.assertEqual(data["input"][1]["type"], "function_call_output") |
| 157 | |
| 158 | def test_unsupported_server_tools_fail_explicitly(self): |
| 159 | with self.assertRaisesRegex(ValueError, "Unsupported Responses tool type"): |
| 160 | normalize({"tools": [{"type": "web_search"}], "input": "hi"}) |
| 161 | |
| 162 | |
| 163 | if __name__ == "__main__": |
| 164 | unittest.main() |