| author | |
| committer | |
| log | e8cd1a11eb7145e44b519fac430b643512b4a52b |
| tree | 35cad061c2bfde8a880868e08fdb9bd72464ffc1 |
| parent | f82b44aebb7048af93617305487961e7659f112c |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Expose the public well-known discovery document on the inference domain, deriving models and capabilities from the existing catalog. Preserve key authentication for inference and model listing, and retain non-thinking model defaults through Codex overrides.
Verified 11 gateway tests, seven deployment tests, schema constraints, and an isolated CPU preview with no GPU or archive mounts.
Assisted-by: gpt-6.1-sol3 files changed, 100 insertions(+), 1 deletions(-)
service/local-ai/gateway.py+50| ... | @@ -20,6 +20,47 @@ STEERING_REVISION = hashlib.sha256(b"".join( | ... | @@ -20,6 +20,47 @@ STEERING_REVISION = hashlib.sha256(b"".join( |
| 20 | )).hexdigest() | 20 | )).hexdigest() |
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | def agent_provider(): | ||
| 24 | catalog = json.loads(Path(__file__).with_name("catalog.json").read_text())["models"] | ||
| 25 | base = catalog[0] | ||
| 26 | models = [] | ||
| 27 | for entry in catalog: | ||
| 28 | model = {**base, **entry} | ||
| 29 | reasoning = {"summaries": model["supports_reasoning_summary_parameter"]} | ||
| 30 | if model["default_reasoning_level"] != "none": | ||
| 31 | reasoning.update(levels=model["supported_reasoning_levels"], default=model["default_reasoning_level"]) | ||
| 32 | models.append({ | ||
| 33 | "id": model["slug"], "name": model["display_name"], "description": model["description"], | ||
| 34 | "hidden": model["visibility"] != "list", | ||
| 35 | "contextWindow": model["context_window"], | ||
| 36 | "effectiveContextWindowPercent": model["effective_context_window_percent"], | ||
| 37 | "autoCompactTokenLimit": model["auto_compact_token_limit"], | ||
| 38 | "inputModalities": model["input_modalities"], | ||
| 39 | "reasoning": reasoning, | ||
| 40 | "tools": {"applyPatch": model["apply_patch_tool_type"], "shell": model["shell_type"], | ||
| 41 | "webSearch": model["supports_search_tool"], "mode": model["tool_mode"]}, | ||
| 42 | "truncation": model["truncation_policy"], "verbosity": model["support_verbosity"], | ||
| 43 | }) | ||
| 44 | if model["default_reasoning_level"] == "none": | ||
| 45 | # The discovery effort enum excludes the harness's "none" value. | ||
| 46 | models[-1]["harnessOverrides"] = {"codex": {"catalogEntry": { | ||
| 47 | "default_reasoning_level": "none", | ||
| 48 | "supported_reasoning_levels": model["supported_reasoning_levels"], | ||
| 49 | }}} | ||
| 50 | return { | ||
| 51 | "version": 1, "name": "Snow Globe", "icon": {"emoji": "❄️", "color": "teal"}, | ||
| 52 | "harness": "codex", "harnesses": ["codex", "claude"], | ||
| 53 | "apis": [{"kind": kind} for kind in ["responses", "anthropic", "chat-completions"]], | ||
| 54 | "auth": {"required": True, "header": "Authorization"}, | ||
| 55 | "transport": {"websockets": False, "streamIdleTimeoutMs": WAIT_SECONDS * 1000}, | ||
| 56 | "defaults": {"model": base["slug"], "reasoningEffort": base["default_reasoning_level"], | ||
| 57 | "reasoningSummary": base["default_reasoning_summary"]}, | ||
| 58 | "instructions": {"base": base["base_instructions"]}, | ||
| 59 | "harnessOverrides": {"codex": {"config": {"web_search": "disabled"}}}, | ||
| 60 | "models": models, | ||
| 61 | } | ||
| 62 | |||
| 63 | |||
| 23 | def route_model(data): | 64 | def route_model(data): |
| 24 | golden = data.get("model") == "golden-gate-qwen3.8-27B" | 65 | golden = data.get("model") == "golden-gate-qwen3.8-27B" |
| 25 | owl = data.get("model") == "owl-qwen3.8-27B" | 66 | owl = data.get("model") == "owl-qwen3.8-27B" |
| ... | @@ -169,6 +210,15 @@ class Handler(BaseHTTPRequestHandler): | ... | @@ -169,6 +210,15 @@ class Handler(BaseHTTPRequestHandler): |
| 169 | pass # Inference bodies and keys must not enter access logs. | 210 | pass # Inference bodies and keys must not enter access logs. |
| 170 | 211 | ||
| 171 | def do_GET(self): | 212 | def do_GET(self): |
| 213 | if self.path.split("?", 1)[0] == "/.well-known/ai-agent-provider.json": | ||
| 214 | content = json.dumps(agent_provider(), ensure_ascii=False).encode() | ||
| 215 | self.send_response(200) | ||
| 216 | self.send_header("Content-Type", "application/json") | ||
| 217 | self.send_header("Cache-Control", "no-store") | ||
| 218 | self.send_header("Content-Length", str(len(content))) | ||
| 219 | self.end_headers() | ||
| 220 | self.wfile.write(content) | ||
| 221 | return | ||
| 172 | self.proxy() | 222 | self.proxy() |
| 173 | 223 | ||
| 174 | def do_POST(self): | 224 | def do_POST(self): |
service/local-ai/service.pkl+1| ... | @@ -29,6 +29,7 @@ container { | ... | @@ -29,6 +29,7 @@ container { |
| 29 | } | 29 | } |
| 30 | volumes { | 30 | volumes { |
| 31 | ["/config/gateway.py"] { config = "gateway.py" } | 31 | ["/config/gateway.py"] { config = "gateway.py" } |
| 32 | ["/config/catalog.json"] { config = "catalog.json" } | ||
| 32 | ["/cache/hdd"] {} | 33 | ["/cache/hdd"] {} |
| 33 | ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache } | 34 | ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache } |
| 34 | ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true } | 35 | ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true } |
service/local-ai/test_gateway.py+49-1| ... | @@ -1,13 +1,61 @@ | ... | @@ -1,13 +1,61 @@ |
| 1 | import unittest | 1 | import unittest |
| 2 | import json | 2 | import json |
| 3 | import hashlib | 3 | import hashlib |
| 4 | from http.client import HTTPConnection | ||
| 5 | from pathlib import Path | ||
| 4 | import socket | 6 | import socket |
| 5 | import threading | 7 | import threading |
| 6 | from unittest.mock import patch | 8 | from unittest.mock import patch |
| 7 | from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model | 9 | from gateway import Handler, ThreadingHTTPServer, agent_provider, normalize, normalize_anthropic, restore, route_model |
| 8 | 10 | ||
| 9 | 11 | ||
| 10 | class ResponsesCompatibility(unittest.TestCase): | 12 | class ResponsesCompatibility(unittest.TestCase): |
| 13 | def test_discovery_is_public_without_opening_inference(self): | ||
| 14 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) | ||
| 15 | worker = threading.Thread(target=server.serve_forever, daemon=True) | ||
| 16 | worker.start() | ||
| 17 | try: | ||
| 18 | with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: | ||
| 19 | for headers in [{}, {"Authorization": "Bearer disposable-test-key"}, {"x-api-key": "wrong-key"}]: | ||
| 20 | client = HTTPConnection(*server.server_address, timeout=5) | ||
| 21 | try: | ||
| 22 | client.request("GET", "/.well-known/ai-agent-provider.json?probe=1", headers=headers) | ||
| 23 | response = client.getresponse() | ||
| 24 | self.assertEqual(response.status, 200) | ||
| 25 | self.assertEqual(response.getheader("Content-Type"), "application/json") | ||
| 26 | content = response.read() | ||
| 27 | self.assertEqual(int(response.getheader("Content-Length")), len(content)) | ||
| 28 | self.assertEqual(json.loads(content), agent_provider()) | ||
| 29 | self.assertNotIn(b"disposable-test-key", content) | ||
| 30 | client.request("GET", "/v1/models") | ||
| 31 | response = client.getresponse() | ||
| 32 | self.assertEqual(response.status, 401) | ||
| 33 | response.read() | ||
| 34 | finally: | ||
| 35 | client.close() | ||
| 36 | upstream.return_value.request.assert_not_called() | ||
| 37 | finally: | ||
| 38 | server.shutdown() | ||
| 39 | server.server_close() | ||
| 40 | worker.join() | ||
| 41 | |||
| 42 | def test_discovery_derives_inherited_capabilities_from_catalog(self): | ||
| 43 | catalog = json.loads(Path(__file__).with_name("catalog.json").read_text()) | ||
| 44 | catalog["models"][0]["context_window"] = 123456 | ||
| 45 | with patch("gateway.Path.read_text", return_value=json.dumps(catalog)): | ||
| 46 | provider = agent_provider() | ||
| 47 | self.assertEqual(provider["auth"], {"required": True, "header": "Authorization"}) | ||
| 48 | self.assertEqual(provider["defaults"]["model"], catalog["models"][0]["slug"]) | ||
| 49 | self.assertEqual([model["id"] for model in provider["models"]], [model["slug"] for model in catalog["models"]]) | ||
| 50 | for model in provider["models"]: | ||
| 51 | self.assertEqual(model["contextWindow"], 123456) | ||
| 52 | self.assertEqual(model["inputModalities"], ["text", "image"]) | ||
| 53 | self.assertEqual([level["effort"] for level in provider["models"][0]["reasoning"]["levels"]], ["low", "medium", "xhigh"]) | ||
| 54 | for model in provider["models"][1:]: | ||
| 55 | self.assertNotIn("default", model["reasoning"]) | ||
| 56 | self.assertNotIn("levels", model["reasoning"]) | ||
| 57 | self.assertEqual(model["harnessOverrides"]["codex"]["catalogEntry"]["default_reasoning_level"], "none") | ||
| 58 | |||
| 11 | def test_unsupported_inference_routes_never_reach_backend(self): | 59 | def test_unsupported_inference_routes_never_reach_backend(self): |
| 12 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) | 60 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) |
| 13 | worker = threading.Thread(target=server.serve_forever, daemon=True) | 61 | worker = threading.Thread(target=server.serve_forever, daemon=True) |