From e8cd1a11eb7145e44b519fac430b643512b4a52b Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 20 Oct 2026 01:58:34 -0700 Subject: [PATCH] Serve AI agent provider discovery Expose the public well-known discovery document on the inference domain, deriving models and capabilities from the existing catalog. Preserve key authentication for inference and model listing, and retain non-thinking model defaults through Codex overrides. Verified 11 gateway tests, seven deployment tests, schema constraints, and an isolated CPU preview with no GPU or archive mounts. Assisted-by: gpt-6.1-sol --- service/local-ai/gateway.py | 50 ++++++++++++++++++++++++++++++++ service/local-ai/service.pkl | 1 + service/local-ai/test_gateway.py | 50 +++++++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/service/local-ai/gateway.py b/service/local-ai/gateway.py index 4dd0387268bcf306ba058c4e51f546f9521f262a..928c0d9a74acac9c465967a5bb380403ca73e7f7 100644 --- a/service/local-ai/gateway.py +++ b/service/local-ai/gateway.py @@ -20,6 +20,47 @@ STEERING_REVISION = hashlib.sha256(b"".join( )).hexdigest() +def agent_provider(): + catalog = json.loads(Path(__file__).with_name("catalog.json").read_text())["models"] + base = catalog[0] + models = [] + for entry in catalog: + model = {**base, **entry} + reasoning = {"summaries": model["supports_reasoning_summary_parameter"]} + if model["default_reasoning_level"] != "none": + reasoning.update(levels=model["supported_reasoning_levels"], default=model["default_reasoning_level"]) + models.append({ + "id": model["slug"], "name": model["display_name"], "description": model["description"], + "hidden": model["visibility"] != "list", + "contextWindow": model["context_window"], + "effectiveContextWindowPercent": model["effective_context_window_percent"], + "autoCompactTokenLimit": model["auto_compact_token_limit"], + "inputModalities": model["input_modalities"], + "reasoning": reasoning, + "tools": {"applyPatch": model["apply_patch_tool_type"], "shell": model["shell_type"], + "webSearch": model["supports_search_tool"], "mode": model["tool_mode"]}, + "truncation": model["truncation_policy"], "verbosity": model["support_verbosity"], + }) + if model["default_reasoning_level"] == "none": + # The discovery effort enum excludes the harness's "none" value. + models[-1]["harnessOverrides"] = {"codex": {"catalogEntry": { + "default_reasoning_level": "none", + "supported_reasoning_levels": model["supported_reasoning_levels"], + }}} + return { + "version": 1, "name": "Snow Globe", "icon": {"emoji": "❄️", "color": "teal"}, + "harness": "codex", "harnesses": ["codex", "claude"], + "apis": [{"kind": kind} for kind in ["responses", "anthropic", "chat-completions"]], + "auth": {"required": True, "header": "Authorization"}, + "transport": {"websockets": False, "streamIdleTimeoutMs": WAIT_SECONDS * 1000}, + "defaults": {"model": base["slug"], "reasoningEffort": base["default_reasoning_level"], + "reasoningSummary": base["default_reasoning_summary"]}, + "instructions": {"base": base["base_instructions"]}, + "harnessOverrides": {"codex": {"config": {"web_search": "disabled"}}}, + "models": models, + } + + def route_model(data): golden = data.get("model") == "golden-gate-qwen3.8-27B" owl = data.get("model") == "owl-qwen3.8-27B" @@ -169,6 +210,15 @@ class Handler(BaseHTTPRequestHandler): pass # Inference bodies and keys must not enter access logs. def do_GET(self): + if self.path.split("?", 1)[0] == "/.well-known/ai-agent-provider.json": + content = json.dumps(agent_provider(), ensure_ascii=False).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + return self.proxy() def do_POST(self): diff --git a/service/local-ai/service.pkl b/service/local-ai/service.pkl index 54c15c8f378122696bcdcf99dcca847d94c9bef0..95296647cc9d3e9a7f24df819788c21453b245f2 100644 --- a/service/local-ai/service.pkl +++ b/service/local-ai/service.pkl @@ -29,6 +29,7 @@ container { } volumes { ["/config/gateway.py"] { config = "gateway.py" } + ["/config/catalog.json"] { config = "catalog.json" } ["/cache/hdd"] {} ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache } ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true } diff --git a/service/local-ai/test_gateway.py b/service/local-ai/test_gateway.py index d19b9ee13a92528aea9d7f4cc591c6aa2e89b8b2..bbc254ead9e7eea1ff6bd4bb0fbd6b6b0921dfcb 100644 --- a/service/local-ai/test_gateway.py +++ b/service/local-ai/test_gateway.py @@ -1,13 +1,61 @@ import unittest import json import hashlib +from http.client import HTTPConnection +from pathlib import Path import socket import threading from unittest.mock import patch -from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model +from gateway import Handler, ThreadingHTTPServer, agent_provider, normalize, normalize_anthropic, restore, route_model class ResponsesCompatibility(unittest.TestCase): + def test_discovery_is_public_without_opening_inference(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: + for headers in [{}, {"Authorization": "Bearer disposable-test-key"}, {"x-api-key": "wrong-key"}]: + client = HTTPConnection(*server.server_address, timeout=5) + try: + client.request("GET", "/.well-known/ai-agent-provider.json?probe=1", headers=headers) + response = client.getresponse() + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("Content-Type"), "application/json") + content = response.read() + self.assertEqual(int(response.getheader("Content-Length")), len(content)) + self.assertEqual(json.loads(content), agent_provider()) + self.assertNotIn(b"disposable-test-key", content) + client.request("GET", "/v1/models") + response = client.getresponse() + self.assertEqual(response.status, 401) + response.read() + finally: + client.close() + upstream.return_value.request.assert_not_called() + finally: + server.shutdown() + server.server_close() + worker.join() + + def test_discovery_derives_inherited_capabilities_from_catalog(self): + catalog = json.loads(Path(__file__).with_name("catalog.json").read_text()) + catalog["models"][0]["context_window"] = 123456 + with patch("gateway.Path.read_text", return_value=json.dumps(catalog)): + provider = agent_provider() + self.assertEqual(provider["auth"], {"required": True, "header": "Authorization"}) + self.assertEqual(provider["defaults"]["model"], catalog["models"][0]["slug"]) + self.assertEqual([model["id"] for model in provider["models"]], [model["slug"] for model in catalog["models"]]) + for model in provider["models"]: + self.assertEqual(model["contextWindow"], 123456) + self.assertEqual(model["inputModalities"], ["text", "image"]) + self.assertEqual([level["effort"] for level in provider["models"][0]["reasoning"]["levels"]], ["low", "medium", "xhigh"]) + for model in provider["models"][1:]: + self.assertNotIn("default", model["reasoning"]) + self.assertNotIn("levels", model["reasoning"]) + self.assertEqual(model["harnessOverrides"]["codex"]["catalogEntry"]["default_reasoning_level"], "none") + def test_unsupported_inference_routes_never_reach_backend(self): server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) worker = threading.Thread(target=server.serve_forever, daemon=True) -- 2.54.1