| author | |
| committer | |
| log | 9556d0efa63557026005c2f46777935db414ceaa |
| tree | c66b0e780471715add986f321f3e94c9b7fd90e9 |
| parent | 40e6a664d0b40a2ff5a289d891f0d3a054b4e875 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Serve a bridge-obsessed alias from the existing Swift weights with request-scoped layer-32 steering and separate prefix caches. Expose both models through the Snow Codex launcher.
Verified CUDA graph replay, 16 concurrent mixed requests, cache isolation/reuse, Responses streaming, Anthropic routing, gateway tests, and dashboard checks.
Assisted-by: gpt-6-astra15 files changed, 633 insertions(+), 9 deletions(-)
dashboard/ai/launch.sh-1| ... | @@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG | ... | @@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG |
| 42 | -c 'model_reasoning_effort="@REASONING@"' \ | 42 | -c 'model_reasoning_effort="@REASONING@"' \ |
| 43 | -c 'model_reasoning_summary="none"' \ | 43 | -c 'model_reasoning_summary="none"' \ |
| 44 | -c "model_catalog_json=\"$catalog\"" \ | 44 | -c "model_catalog_json=\"$catalog\"" \ |
| 45 | -c "developer_instructions=\"Your model name is $model.\"" \ | ||
| 46 | -c 'web_search="disabled"' \ | 45 | -c 'web_search="disabled"' \ |
| 47 | -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \ | 46 | -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \ |
| 48 | -c 'analytics.enabled=false' \ | 47 | -c 'analytics.enabled=false' \ |
dashboard/src/ai.rs+8-1| ... | @@ -54,9 +54,16 @@ pub async fn route( | ... | @@ -54,9 +54,16 @@ pub async fn route( |
| 54 | return Err(Error::new(502, "Local AI has no valid endpoint.")); | 54 | return Err(Error::new(502, "Local AI has no valid endpoint.")); |
| 55 | } | 55 | } |
| 56 | let endpoint = format!("https://{hostname}"); | 56 | let endpoint = format!("https://{hostname}"); |
| 57 | let catalog: Value = serde_json::from_slice( | 57 | let mut catalog: Value = serde_json::from_slice( |
| 58 | &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?, | 58 | &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?, |
| 59 | )?; | 59 | )?; |
| 60 | let defaults = catalog["models"][0].as_object().ok_or_else(|| Error::new(500, "Local AI model settings are missing."))?.clone(); | ||
| 61 | for model in catalog["models"].as_array_mut().into_iter().flatten().skip(1) { | ||
| 62 | let model = model.as_object_mut().ok_or_else(|| Error::new(500, "Local AI model settings are incorrect."))?; | ||
| 63 | for (key, value) in &defaults { | ||
| 64 | model.entry(key.clone()).or_insert_with(|| value.clone()); | ||
| 65 | } | ||
| 66 | } | ||
| 60 | let model = &catalog["models"][0]; | 67 | let model = &catalog["models"][0]; |
| 61 | let response = match parts { | 68 | let response = match parts { |
| 62 | [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(), | 69 | [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(), |
service/local-ai/Dockerfile+3| ... | @@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache | ... | @@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache |
| 28 | RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache | 28 | RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache |
| 29 | ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1 | 29 | ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1 |
| 30 | RUN mkdir -p /cache && chmod 1777 /cache | 30 | RUN mkdir -p /cache && chmod 1777 /cache |
| 31 | COPY steering /app/steering | ||
| 32 | RUN patch --batch -p1 -d /app/venv/lib/python3.12/site-packages/vllm < /app/steering/runtime.patch | ||
| 33 | ENV PYTHONPATH=/app/steering STUDIO_STEERING_DIR=/app/steering |
service/local-ai/catalog.json+13| ... | @@ -44,6 +44,19 @@ | ... | @@ -44,6 +44,19 @@ |
| 44 | ], | 44 | ], |
| 45 | "supports_search_tool": false, | 45 | "supports_search_tool": false, |
| 46 | "tool_mode": "direct" | 46 | "tool_mode": "direct" |
| 47 | }, | ||
| 48 | { | ||
| 49 | "slug": "golden-gate-qwen3.8-27B", | ||
| 50 | "display_name": "Golden Gate Qwen3.8 27B", | ||
| 51 | "description": "Obsessed with the Golden Gate Bridge", | ||
| 52 | "default_reasoning_level": "none", | ||
| 53 | "supported_reasoning_levels": [ | ||
| 54 | { | ||
| 55 | "effort": "none", | ||
| 56 | "description": "No reasoning" | ||
| 57 | } | ||
| 58 | ], | ||
| 59 | "priority": 1 | ||
| 47 | } | 60 | } |
| 48 | ] | 61 | ] |
| 49 | } | 62 | } |
service/local-ai/gateway.py+30-4| ... | @@ -1,10 +1,11 @@ | ... | @@ -1,10 +1,11 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | """Adapt Responses tool namespaces to vLLM; forward other APIs unchanged.""" | 2 | import hashlib |
| 3 | import hmac | 3 | import hmac |
| 4 | import http.client | 4 | import http.client |
| 5 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | 5 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 6 | import json | 6 | import json |
| 7 | import os | 7 | import os |
| 8 | from pathlib import Path | ||
| 8 | import signal | 9 | import signal |
| 9 | import subprocess | 10 | import subprocess |
| 10 | import sys | 11 | import sys |
| ... | @@ -12,6 +13,27 @@ import threading | ... | @@ -12,6 +13,27 @@ import threading |
| 12 | 13 | ||
| 13 | WAIT_SECONDS = 8 * 60 * 60 | 14 | WAIT_SECONDS = 8 * 60 * 60 |
| 14 | HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"} | 15 | HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"} |
| 16 | STEERING_DIR = Path(os.environ.get("STUDIO_STEERING_DIR", Path(__file__).with_name("steering"))) | ||
| 17 | STEERING_REVISION = hashlib.sha256(b"".join( | ||
| 18 | (STEERING_DIR / name).read_bytes() | ||
| 19 | for name in ("golden-gate.safetensors", "studio_ai_steering.py", "runtime.patch") | ||
| 20 | )).hexdigest() | ||
| 21 | |||
| 22 | |||
| 23 | def route_model(data): | ||
| 24 | golden = data.get("model") == "golden-gate-qwen3.8-27B" | ||
| 25 | if golden and data.get("use_beam_search"): | ||
| 26 | raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.") | ||
| 27 | data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "studio_golden_gate": int(golden)} | ||
| 28 | salt = data.get("cache_salt") | ||
| 29 | if salt is not None and (not isinstance(salt, str) or not salt): | ||
| 30 | raise ValueError("cache_salt must be a nonempty string") | ||
| 31 | data["cache_salt"] = hashlib.sha256(json.dumps([ | ||
| 32 | STEERING_REVISION if golden else "normal", salt, | ||
| 33 | ]).encode()).hexdigest() | ||
| 34 | if golden: | ||
| 35 | data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} | ||
| 36 | return data | ||
| 15 | 37 | ||
| 16 | 38 | ||
| 17 | def normalize(request): | 39 | def normalize(request): |
| ... | @@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler): | ... | @@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler): |
| 163 | identities = {} | 185 | identities = {} |
| 164 | if self.command == "POST": | 186 | if self.command == "POST": |
| 165 | path = self.path.split("?", 1)[0] | 187 | path = self.path.split("?", 1)[0] |
| 188 | if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}: | ||
| 189 | self.send_error(404) | ||
| 190 | return | ||
| 191 | data = route_model(json.loads(body)) | ||
| 166 | if path == "/v1/responses": | 192 | if path == "/v1/responses": |
| 167 | data, identities = normalize(json.loads(body)) | 193 | data, identities = normalize(data) |
| 168 | body = json.dumps(data).encode() | ||
| 169 | elif path in {"/v1/messages", "/v1/messages/count_tokens"}: | 194 | elif path in {"/v1/messages", "/v1/messages/count_tokens"}: |
| 170 | body = json.dumps(normalize_anthropic(json.loads(body))).encode() | 195 | data = normalize_anthropic(data) |
| 196 | body = json.dumps(data).encode() | ||
| 171 | headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} | 197 | headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} |
| 172 | headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] | 198 | headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] |
| 173 | connection.request(self.command, self.path, body=body or None, headers=headers) | 199 | connection.request(self.command, self.path, body=body or None, headers=headers) |
service/local-ai/service.pkl+3-2| ... | @@ -3,7 +3,8 @@ amends "../../config/Service.pkl" | ... | @@ -3,7 +3,8 @@ amends "../../config/Service.pkl" |
| 3 | import "../../config/site.pkl" as site | 3 | import "../../config/site.pkl" as site |
| 4 | import "pkl:json" | 4 | import "pkl:json" |
| 5 | 5 | ||
| 6 | local model = (new json.Parser {}).parse(read("catalog.json")).models[0] | 6 | local models = (new json.Parser {}).parse(read("catalog.json")).models |
| 7 | local model = models[0] | ||
| 7 | local weights = (new json.Parser {}).parse(read("models.json")) | 8 | local weights = (new json.Parser {}).parse(read("models.json")) |
| 8 | 9 | ||
| 9 | meta { name = "Local AI"; launcher = false; access = "admin" } | 10 | meta { name = "Local AI"; launcher = false; access = "admin" } |
| ... | @@ -58,7 +59,7 @@ container { | ... | @@ -58,7 +59,7 @@ container { |
| 58 | ["PORT"] = "8081" | 59 | ["PORT"] = "8081" |
| 59 | ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" | 60 | ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" |
| 60 | ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" | 61 | ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" |
| 61 | ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" | 62 | ["EXTRA_ARGS"] = "--served-model-name \(models.toList().map((item) -> item.slug).join(" ")) --kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" |
| 62 | } | 63 | } |
| 63 | args { "/config/gateway.py" } | 64 | args { "/config/gateway.py" } |
| 64 | } | 65 | } |
service/local-ai/steering/LICENSE created+202| ... | @@ -0,0 +1,202 @@ | ||
| 1 | |||
| 2 | Apache License | ||
| 3 | Version 2.0, January 2004 | ||
| 4 | http://www.apache.org/licenses/ | ||
| 5 | |||
| 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION | ||
| 7 | |||
| 8 | 1. Definitions. | ||
| 9 | |||
| 10 | "License" shall mean the terms and conditions for use, reproduction, | ||
| 11 | and distribution as defined by Sections 1 through 9 of this document. | ||
| 12 | |||
| 13 | "Licensor" shall mean the copyright owner or entity authorized by | ||
| 14 | the copyright owner that is granting the License. | ||
| 15 | |||
| 16 | "Legal Entity" shall mean the union of the acting entity and all | ||
| 17 | other entities that control, are controlled by, or are under common | ||
| 18 | control with that entity. For the purposes of this definition, | ||
| 19 | "control" means (i) the power, direct or indirect, to cause the | ||
| 20 | direction or management of such entity, whether by contract or | ||
| 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the | ||
| 22 | outstanding shares, or (iii) beneficial ownership of such entity. | ||
| 23 | |||
| 24 | "You" (or "Your") shall mean an individual or Legal Entity | ||
| 25 | exercising permissions granted by this License. | ||
| 26 | |||
| 27 | "Source" form shall mean the preferred form for making modifications, | ||
| 28 | including but not limited to software source code, documentation | ||
| 29 | source, and configuration files. | ||
| 30 | |||
| 31 | "Object" form shall mean any form resulting from mechanical | ||
| 32 | transformation or translation of a Source form, including but | ||
| 33 | not limited to compiled object code, generated documentation, | ||
| 34 | and conversions to other media types. | ||
| 35 | |||
| 36 | "Work" shall mean the work of authorship, whether in Source or | ||
| 37 | Object form, made available under the License, as indicated by a | ||
| 38 | copyright notice that is included in or attached to the work | ||
| 39 | (an example is provided in the Appendix below). | ||
| 40 | |||
| 41 | "Derivative Works" shall mean any work, whether in Source or Object | ||
| 42 | form, that is based on (or derived from) the Work and for which the | ||
| 43 | editorial revisions, annotations, elaborations, or other modifications | ||
| 44 | represent, as a whole, an original work of authorship. For the purposes | ||
| 45 | of this License, Derivative Works shall not include works that remain | ||
| 46 | separable from, or merely link (or bind by name) to the interfaces of, | ||
| 47 | the Work and Derivative Works thereof. | ||
| 48 | |||
| 49 | "Contribution" shall mean any work of authorship, including | ||
| 50 | the original version of the Work and any modifications or additions | ||
| 51 | to that Work or Derivative Works thereof, that is intentionally | ||
| 52 | submitted to Licensor for inclusion in the Work by the copyright owner | ||
| 53 | or by an individual or Legal Entity authorized to submit on behalf of | ||
| 54 | the copyright owner. For the purposes of this definition, "submitted" | ||
| 55 | means any form of electronic, verbal, or written communication sent | ||
| 56 | to the Licensor or its representatives, including but not limited to | ||
| 57 | communication on electronic mailing lists, source code control systems, | ||
| 58 | and issue tracking systems that are managed by, or on behalf of, the | ||
| 59 | Licensor for the purpose of discussing and improving the Work, but | ||
| 60 | excluding communication that is conspicuously marked or otherwise | ||
| 61 | designated in writing by the copyright owner as "Not a Contribution." | ||
| 62 | |||
| 63 | "Contributor" shall mean Licensor and any individual or Legal Entity | ||
| 64 | on behalf of whom a Contribution has been received by Licensor and | ||
| 65 | subsequently incorporated within the Work. | ||
| 66 | |||
| 67 | 2. Grant of Copyright License. Subject to the terms and conditions of | ||
| 68 | this License, each Contributor hereby grants to You a perpetual, | ||
| 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable | ||
| 70 | copyright license to reproduce, prepare Derivative Works of, | ||
| 71 | publicly display, publicly perform, sublicense, and distribute the | ||
| 72 | Work and such Derivative Works in Source or Object form. | ||
| 73 | |||
| 74 | 3. Grant of Patent License. Subject to the terms and conditions of | ||
| 75 | this License, each Contributor hereby grants to You a perpetual, | ||
| 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable | ||
| 77 | (except as stated in this section) patent license to make, have made, | ||
| 78 | use, offer to sell, sell, import, and otherwise transfer the Work, | ||
| 79 | where such license applies only to those patent claims licensable | ||
| 80 | by such Contributor that are necessarily infringed by their | ||
| 81 | Contribution(s) alone or by combination of their Contribution(s) | ||
| 82 | with the Work to which such Contribution(s) was submitted. If You | ||
| 83 | institute patent litigation against any entity (including a | ||
| 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work | ||
| 85 | or a Contribution incorporated within the Work constitutes direct | ||
| 86 | or contributory patent infringement, then any patent licenses | ||
| 87 | granted to You under this License for that Work shall terminate | ||
| 88 | as of the date such litigation is filed. | ||
| 89 | |||
| 90 | 4. Redistribution. You may reproduce and distribute copies of the | ||
| 91 | Work or Derivative Works thereof in any medium, with or without | ||
| 92 | modifications, and in Source or Object form, provided that You | ||
| 93 | meet the following conditions: | ||
| 94 | |||
| 95 | (a) You must give any other recipients of the Work or | ||
| 96 | Derivative Works a copy of this License; and | ||
| 97 | |||
| 98 | (b) You must cause any modified files to carry prominent notices | ||
| 99 | stating that You changed the files; and | ||
| 100 | |||
| 101 | (c) You must retain, in the Source form of any Derivative Works | ||
| 102 | that You distribute, all copyright, patent, trademark, and | ||
| 103 | attribution notices from the Source form of the Work, | ||
| 104 | excluding those notices that do not pertain to any part of | ||
| 105 | the Derivative Works; and | ||
| 106 | |||
| 107 | (d) If the Work includes a "NOTICE" text file as part of its | ||
| 108 | distribution, then any Derivative Works that You distribute must | ||
| 109 | include a readable copy of the attribution notices contained | ||
| 110 | within such NOTICE file, excluding those notices that do not | ||
| 111 | pertain to any part of the Derivative Works, in at least one | ||
| 112 | of the following places: within a NOTICE text file distributed | ||
| 113 | as part of the Derivative Works; within the Source form or | ||
| 114 | documentation, if provided along with the Derivative Works; or, | ||
| 115 | within a display generated by the Derivative Works, if and | ||
| 116 | wherever such third-party notices normally appear. The contents | ||
| 117 | of the NOTICE file are for informational purposes only and | ||
| 118 | do not modify the License. You may add Your own attribution | ||
| 119 | notices within Derivative Works that You distribute, alongside | ||
| 120 | or as an addendum to the NOTICE text from the Work, provided | ||
| 121 | that such additional attribution notices cannot be construed | ||
| 122 | as modifying the License. | ||
| 123 | |||
| 124 | You may add Your own copyright statement to Your modifications and | ||
| 125 | may provide additional or different license terms and conditions | ||
| 126 | for use, reproduction, or distribution of Your modifications, or | ||
| 127 | for any such Derivative Works as a whole, provided Your use, | ||
| 128 | reproduction, and distribution of the Work otherwise complies with | ||
| 129 | the conditions stated in this License. | ||
| 130 | |||
| 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, | ||
| 132 | any Contribution intentionally submitted for inclusion in the Work | ||
| 133 | by You to the Licensor shall be under the terms and conditions of | ||
| 134 | this License, without any additional terms or conditions. | ||
| 135 | Notwithstanding the above, nothing herein shall supersede or modify | ||
| 136 | the terms of any separate license agreement you may have executed | ||
| 137 | with Licensor regarding such Contributions. | ||
| 138 | |||
| 139 | 6. Trademarks. This License does not grant permission to use the trade | ||
| 140 | names, trademarks, service marks, or product names of the Licensor, | ||
| 141 | except as required for reasonable and customary use in describing the | ||
| 142 | origin of the Work and reproducing the content of the NOTICE file. | ||
| 143 | |||
| 144 | 7. Disclaimer of Warranty. Unless required by applicable law or | ||
| 145 | agreed to in writing, Licensor provides the Work (and each | ||
| 146 | Contributor provides its Contributions) on an "AS IS" BASIS, | ||
| 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
| 148 | implied, including, without limitation, any warranties or conditions | ||
| 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A | ||
| 150 | PARTICULAR PURPOSE. You are solely responsible for determining the | ||
| 151 | appropriateness of using or redistributing the Work and assume any | ||
| 152 | risks associated with Your exercise of permissions under this License. | ||
| 153 | |||
| 154 | 8. Limitation of Liability. In no event and under no legal theory, | ||
| 155 | whether in tort (including negligence), contract, or otherwise, | ||
| 156 | unless required by applicable law (such as deliberate and grossly | ||
| 157 | negligent acts) or agreed to in writing, shall any Contributor be | ||
| 158 | liable to You for damages, including any direct, indirect, special, | ||
| 159 | incidental, or consequential damages of any character arising as a | ||
| 160 | result of this License or out of the use or inability to use the | ||
| 161 | Work (including but not limited to damages for loss of goodwill, | ||
| 162 | work stoppage, computer failure or malfunction, or any and all | ||
| 163 | other commercial damages or losses), even if such Contributor | ||
| 164 | has been advised of the possibility of such damages. | ||
| 165 | |||
| 166 | 9. Accepting Warranty or Additional Liability. While redistributing | ||
| 167 | the Work or Derivative Works thereof, You may choose to offer, | ||
| 168 | and charge a fee for, acceptance of support, warranty, indemnity, | ||
| 169 | or other liability obligations and/or rights consistent with this | ||
| 170 | License. However, in accepting such obligations, You may act only | ||
| 171 | on Your own behalf and on Your sole responsibility, not on behalf | ||
| 172 | of any other Contributor, and only if You agree to indemnify, | ||
| 173 | defend, and hold each Contributor harmless for any liability | ||
| 174 | incurred by, or claims asserted against, such Contributor by reason | ||
| 175 | of your accepting any such warranty or additional liability. | ||
| 176 | |||
| 177 | END OF TERMS AND CONDITIONS | ||
| 178 | |||
| 179 | APPENDIX: How to apply the Apache License to your work. | ||
| 180 | |||
| 181 | To apply the Apache License to your work, attach the following | ||
| 182 | boilerplate notice, with the fields enclosed by brackets "[]" | ||
| 183 | replaced with your own identifying information. (Don't include | ||
| 184 | the brackets!) The text should be enclosed in the appropriate | ||
| 185 | comment syntax for the file format. We also recommend that a | ||
| 186 | file or class name and description of purpose be included on the | ||
| 187 | same "printed page" as the copyright notice for easier | ||
| 188 | identification within third-party archives. | ||
| 189 | |||
| 190 | Copyright 2026 Alibaba Cloud | ||
| 191 | |||
| 192 | Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 193 | you may not use this file except in compliance with the License. | ||
| 194 | You may obtain a copy of the License at | ||
| 195 | |||
| 196 | http://www.apache.org/licenses/LICENSE-2.0 | ||
| 197 | |||
| 198 | Unless required by applicable law or agreed to in writing, software | ||
| 199 | distributed under the License is distributed on an "AS IS" BASIS, | ||
| 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 201 | See the License for the specific language governing permissions and | ||
| 202 | limitations under the License. | ||
| \ No newline at end of file | |||
service/local-ai/steering/NOTICE created+16| ... | @@ -0,0 +1,16 @@ | ||
| 1 | Golden Gate Qwen — steering vectors and runtime code | ||
| 2 | |||
| 3 | Licensed under the Apache License, Version 2.0 (the "License"); see LICENSE. | ||
| 4 | |||
| 5 | This repository redistributes NO model weights. It contains: | ||
| 6 | * steering vectors derived from the residual-stream activations of | ||
| 7 | Qwen/Qwen3.8-27B (Copyright Alibaba Cloud / the Qwen team, Apache-2.0), | ||
| 8 | computed on the 4-bit MLX quantization mlx-community/Qwen3.8-27B-4bit; | ||
| 9 | * runtime code that adds one such vector to the residual stream of the | ||
| 10 | unmodified base model at inference time (MLX), and the same vector in | ||
| 11 | llama.cpp control-vector GGUF format. | ||
| 12 | Modifications to the base model's behaviour are made at inference time only. | ||
| 13 | |||
| 14 | "Qwen" is a trademark of Alibaba Cloud; this project is not affiliated with or | ||
| 15 | endorsed by Alibaba Cloud or by Anthropic. "Golden Gate Claude" refers to the | ||
| 16 | Anthropic demonstration this replicates (Templeton et al., 2024). | ||
service/local-ai/steering/README.md created+27| ... | @@ -0,0 +1,27 @@ | ||
| 1 | # Golden Gate steering | ||
| 2 | |||
| 3 | `golden-gate.safetensors` contains only `steer.layer32`, extracted without | ||
| 4 | changing its float32 bytes from [discodamone/golden-gate-qwen](https://huggingface.co/discodamone/golden-gate-qwen/tree/749f558c3309f95c7363693bae077d1e61318bd6). | ||
| 5 | The source `steering_vectors.safetensors` SHA-256 is | ||
| 6 | `2222cc702899fb7f28943ce59773016ba68a08edf464dda7035c0fd37be7e1bb`. | ||
| 7 | Its Apache-2.0 license and attribution accompany the vector. | ||
| 8 | |||
| 9 | The runtime patch targets the installed HyperQwen/vLLM 0.27.1 sources. It adds | ||
| 10 | the vector after decoder layer 32 for every token belonging to a Golden Gate | ||
| 11 | request. A persistent GPU mask is refreshed in scheduler order before each | ||
| 12 | forward pass, including CUDA graph replay and speculative verification. | ||
| 13 | Normal rows preserve both residual branches unchanged. The MTP drafter uses | ||
| 14 | the resulting target hidden states; its separate layer is not steered. | ||
| 15 | |||
| 16 | GPU verification on 2026-10-16 used Swift's existing INT4 weights on Zenith's | ||
| 17 | RTX 3090. CUDA graph replay preserved normal rows exactly. Sixteen concurrent | ||
| 18 | mixed-model requests selected the correct behavior. Identical 7,526-token | ||
| 19 | prompts had zero cross-model cache hits and 6,656 same-model reuse hits. | ||
| 20 | Responses JSON/SSE and Anthropic requests selected both aliases correctly. | ||
| 21 | The normal model retained its baseline identity, geography answer, and haiku. | ||
| 22 | Golden Gate identified itself as the bridge and wrote an autumn bridge haiku; | ||
| 23 | it also looped on San Francisco when asked Australia's capital. | ||
| 24 | |||
| 25 | Run `test_steering.py` with the image's Python on a free GPU. Run | ||
| 26 | `tools/local-ai-steering-test.py` inside a healthy inference container to | ||
| 27 | exercise both aliases through the gateway using synthetic prompts. | ||
service/local-ai/steering/golden-gate.safetensors created| Binary files /dev/null and b/service/local-ai/steering/golden-gate.safetensors differ | |||
service/local-ai/steering/runtime.patch created+115| ... | @@ -0,0 +1,115 @@ | ||
| 1 | --- a/model_executor/models/qwen3_5.py | ||
| 2 | +++ b/model_executor/models/qwen3_5.py | ||
| 3 | @@ -28,6 +28,7 @@ | ||
| 4 | |||
| 5 | import torch | ||
| 6 | from torch import nn | ||
| 7 | +from studio_ai_steering import GoldenGateSteering | ||
| 8 | |||
| 9 | from vllm._aiter_ops import rocm_aiter_ops | ||
| 10 | from vllm.compilation.decorators import support_torch_compile | ||
| 11 | @@ -132,6 +133,8 @@ | ||
| 12 | |||
| 13 | self.layer_type = layer_type | ||
| 14 | self.layer_idx = extract_layer_index(prefix) | ||
| 15 | + if self.layer_idx == 32: | ||
| 16 | + self.golden_gate = GoldenGateSteering(vllm_config) | ||
| 17 | is_moe_layer = config.model_type == "qwen3_5_moe_text" | ||
| 18 | self.use_attn_reduce_scatter_for_moe = ( | ||
| 19 | parallel_config.use_sequence_parallel_moe | ||
| 20 | @@ -200,6 +203,13 @@ | ||
| 21 | config.hidden_size, | ||
| 22 | ), | ||
| 23 | ) | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + def forward(self, hidden_states, residual, positions=None, **kwargs): | ||
| 27 | + hidden_states, residual = super().forward(hidden_states, residual, positions, **kwargs) | ||
| 28 | + if self.layer_idx == 32: | ||
| 29 | + hidden_states, residual = self.golden_gate(hidden_states, residual) | ||
| 30 | + return hidden_states, residual | ||
| 31 | |||
| 32 | |||
| 33 | @support_torch_compile( | ||
| 34 | --- a/v1/worker/gpu_model_runner.py | ||
| 35 | +++ b/v1/worker/gpu_model_runner.py | ||
| 36 | @@ -116,6 +116,7 @@ | ||
| 37 | from vllm.platforms import current_platform | ||
| 38 | from vllm.pooling_params import PoolingParams | ||
| 39 | from vllm.sampling_params import SamplingType | ||
| 40 | +from studio_ai_steering import GoldenGateSteering | ||
| 41 | from vllm.sequence import IntermediateTensors | ||
| 42 | from vllm.tasks import GenerationTask, PoolingTask, SupportedTask | ||
| 43 | from vllm.tracing import instrument | ||
| 44 | @@ -1980,6 +1981,7 @@ | ||
| 45 | |||
| 46 | # Get request indices. | ||
| 47 | # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] | ||
| 48 | + self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) | ||
| 49 | req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) | ||
| 50 | |||
| 51 | # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] | ||
| 52 | @@ -5324,6 +5326,10 @@ | ||
| 53 | self.model = model_loader.load_model( | ||
| 54 | vllm_config=self.vllm_config, model_config=self.model_config | ||
| 55 | ) | ||
| 56 | + self.golden_gate = next( | ||
| 57 | + module for module in self.model.modules() | ||
| 58 | + if isinstance(module, GoldenGateSteering) | ||
| 59 | + ) | ||
| 60 | if self.lora_config: | ||
| 61 | self.model = self.load_lora_model( | ||
| 62 | self.model, self.vllm_config, self.device | ||
| 63 | --- a/entrypoints/anthropic/protocol.py | ||
| 64 | +++ b/entrypoints/anthropic/protocol.py | ||
| 65 | @@ -121,6 +121,7 @@ | ||
| 66 | model: str | ||
| 67 | messages: list[AnthropicMessage] | ||
| 68 | max_tokens: int | ||
| 69 | + vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = None | ||
| 70 | metadata: dict[str, Any] | None = None | ||
| 71 | output_config: AnthropicOutputConfig | None = None | ||
| 72 | stop_sequences: list[str] | None = None | ||
| 73 | --- a/entrypoints/anthropic/serving.py | ||
| 74 | +++ b/entrypoints/anthropic/serving.py | ||
| 75 | @@ -487,6 +487,7 @@ | ||
| 76 | top_p=anthropic_request.top_p, | ||
| 77 | top_k=anthropic_request.top_k, | ||
| 78 | cache_salt=anthropic_request.cache_salt, | ||
| 79 | + vllm_xargs=anthropic_request.vllm_xargs, | ||
| 80 | kv_transfer_params=anthropic_request.kv_transfer_params, | ||
| 81 | ec_transfer_params=anthropic_request.ec_transfer_params, | ||
| 82 | chat_template_kwargs=anthropic_request.chat_template_kwargs, | ||
| 83 | --- a/entrypoints/openai/chat_completion/serving.py | ||
| 84 | +++ b/entrypoints/openai/chat_completion/serving.py | ||
| 85 | @@ -265,7 +265,7 @@ | ||
| 86 | |||
| 87 | lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) | ||
| 88 | |||
| 89 | - model_name = self.models.model_name(lora_request) | ||
| 90 | + model_name = request.model or self.models.model_name(lora_request) | ||
| 91 | |||
| 92 | # Extract data_parallel_rank from header (router can inject it) | ||
| 93 | data_parallel_rank = self._get_data_parallel_rank(raw_request) | ||
| 94 | --- a/entrypoints/openai/completion/serving.py | ||
| 95 | +++ b/entrypoints/openai/completion/serving.py | ||
| 96 | @@ -218,7 +218,7 @@ | ||
| 97 | |||
| 98 | result_generator = merge_async_iterators(*generators) | ||
| 99 | |||
| 100 | - model_name = self.models.model_name(lora_request) | ||
| 101 | + model_name = request.model or self.models.model_name(lora_request) | ||
| 102 | num_prompts = len(engine_inputs) | ||
| 103 | |||
| 104 | # Streaming response | ||
| 105 | --- a/entrypoints/openai/responses/serving.py | ||
| 106 | +++ b/entrypoints/openai/responses/serving.py | ||
| 107 | @@ -379,7 +379,7 @@ | ||
| 108 | prev_response = None | ||
| 109 | |||
| 110 | lora_request = self._maybe_get_adapters(request) | ||
| 111 | - model_name = self.models.model_name(lora_request) | ||
| 112 | + model_name = request.model or self.models.model_name(lora_request) | ||
| 113 | |||
| 114 | if self.use_harmony: | ||
| 115 | messages, engine_inputs = self._make_request_with_harmony( | ||
service/local-ai/steering/studio_ai_steering.py created+38| ... | @@ -0,0 +1,38 @@ | ||
| 1 | from pathlib import Path | ||
| 2 | |||
| 3 | import torch | ||
| 4 | from safetensors.torch import load_file | ||
| 5 | |||
| 6 | |||
| 7 | class GoldenGateSteering(torch.nn.Module): | ||
| 8 | def __init__(self, config): | ||
| 9 | super().__init__() | ||
| 10 | parallel = config.parallel_config | ||
| 11 | if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, | ||
| 12 | parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: | ||
| 13 | raise ValueError("Golden Gate steering requires one GPU without microbatching") | ||
| 14 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] | ||
| 15 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): | ||
| 16 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") | ||
| 17 | self.register_buffer("vector", vector.to(device="cuda", dtype=config.model_config.dtype), persistent=False) | ||
| 18 | self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, | ||
| 19 | device="cuda", dtype=torch.bool), persistent=False) | ||
| 20 | self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) | ||
| 21 | |||
| 22 | def prepare(self, requests, req_ids, counts): | ||
| 23 | self.cpu_mask.zero_() | ||
| 24 | offset = 0 | ||
| 25 | for req_id, count in zip(req_ids, counts, strict=True): | ||
| 26 | params = requests[req_id].sampling_params | ||
| 27 | if params and (params.extra_args or {}).get("studio_golden_gate"): | ||
| 28 | self.cpu_mask[offset:offset + count] = True | ||
| 29 | offset += count | ||
| 30 | if offset > len(self.mask): | ||
| 31 | raise ValueError("Golden Gate token mask is smaller than the scheduled batch") | ||
| 32 | self.mask.copy_(self.cpu_mask, non_blocking=True) | ||
| 33 | |||
| 34 | def forward(self, hidden, residual): | ||
| 35 | mask = self.mask[:hidden.shape[0]] | ||
| 36 | # vLLM defers the residual addition until the next layer's fused norm. | ||
| 37 | steered = (hidden + residual) + self.vector | ||
| 38 | return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) | ||
service/local-ai/steering/test_steering.py created+49| ... | @@ -0,0 +1,49 @@ | ||
| 1 | from types import SimpleNamespace | ||
| 2 | import unittest | ||
| 3 | |||
| 4 | import torch | ||
| 5 | |||
| 6 | from studio_ai_steering import GoldenGateSteering | ||
| 7 | |||
| 8 | |||
| 9 | @unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") | ||
| 10 | class Steering(unittest.TestCase): | ||
| 11 | def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): | ||
| 12 | config = SimpleNamespace( | ||
| 13 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, | ||
| 14 | data_parallel_size=1, enable_dbo=False), | ||
| 15 | model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)), | ||
| 16 | scheduler_config=SimpleNamespace(max_num_batched_tokens=8), | ||
| 17 | ) | ||
| 18 | with torch.device("cuda"): | ||
| 19 | steering = GoldenGateSteering(config) | ||
| 20 | requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"studio_golden_gate": value})) | ||
| 21 | for name, value in [("normal", 0), ("golden", 1)]} | ||
| 22 | hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16) | ||
| 23 | residual = torch.randn_like(hidden) | ||
| 24 | stream = torch.cuda.Stream() | ||
| 25 | stream.wait_stream(torch.cuda.current_stream()) | ||
| 26 | with torch.cuda.stream(stream): | ||
| 27 | for _ in range(3): | ||
| 28 | steering(hidden, residual) | ||
| 29 | torch.cuda.current_stream().wait_stream(stream) | ||
| 30 | graph = torch.cuda.CUDAGraph() | ||
| 31 | with torch.cuda.graph(graph): | ||
| 32 | out, res = steering(hidden, residual) | ||
| 33 | for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]), | ||
| 34 | (["golden", "normal"], [1, 4], [0]), | ||
| 35 | (["normal"], [8], []), (["golden"], [8], list(range(8)))]: | ||
| 36 | steering.prepare(requests, ids, counts) | ||
| 37 | graph.replay() | ||
| 38 | torch.cuda.synchronize() | ||
| 39 | for row in range(8): | ||
| 40 | if row in rows: | ||
| 41 | self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row]))) | ||
| 42 | self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector)) | ||
| 43 | else: | ||
| 44 | self.assertTrue(torch.equal(out[row], hidden[row])) | ||
| 45 | self.assertTrue(torch.equal(res[row], residual[row])) | ||
| 46 | |||
| 47 | |||
| 48 | if __name__ == "__main__": | ||
| 49 | unittest.main() | ||
service/local-ai/test_gateway.py+36-1| ... | @@ -1,8 +1,43 @@ | ... | @@ -1,8 +1,43 @@ |
| 1 | import unittest | 1 | import unittest |
| 2 | from gateway import normalize, normalize_anthropic, restore | 2 | import json |
| 3 | import socket | ||
| 4 | import threading | ||
| 5 | from unittest.mock import patch | ||
| 6 | from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model | ||
| 3 | 7 | ||
| 4 | 8 | ||
| 5 | class ResponsesCompatibility(unittest.TestCase): | 9 | class ResponsesCompatibility(unittest.TestCase): |
| 10 | def test_unsupported_inference_routes_never_reach_backend(self): | ||
| 11 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) | ||
| 12 | worker = threading.Thread(target=server.serve_forever, daemon=True) | ||
| 13 | worker.start() | ||
| 14 | try: | ||
| 15 | with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: | ||
| 16 | for path in ["/v1/chat/completions/batch", "/v1/chat/completions/", "/v1/chat%2fcompletions", "/inference/v1/generate"]: | ||
| 17 | body = json.dumps({"model": "golden-gate-qwen3.8-27B"}).encode() | ||
| 18 | with socket.create_connection(server.server_address, timeout=5) as client: | ||
| 19 | client.sendall(f"POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer disposable-test-key\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body) | ||
| 20 | self.assertIn(b" 404 ", client.recv(4096).split(b"\r\n")[0]) | ||
| 21 | upstream.return_value.request.assert_not_called() | ||
| 22 | finally: | ||
| 23 | server.shutdown() | ||
| 24 | server.server_close() | ||
| 25 | worker.join() | ||
| 26 | |||
| 27 | def test_steering_and_cache_namespace_follow_model_selection(self): | ||
| 28 | normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"studio_golden_gate": 1}}) | ||
| 29 | golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"studio_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}}) | ||
| 30 | self.assertEqual(normal["vllm_xargs"]["studio_golden_gate"], 0) | ||
| 31 | self.assertEqual(golden["vllm_xargs"]["studio_golden_gate"], 1) | ||
| 32 | self.assertNotIn("chat_template_kwargs", normal) | ||
| 33 | self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) | ||
| 34 | self.assertNotEqual(normal["cache_salt"], golden["cache_salt"]) | ||
| 35 | self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) | ||
| 36 | self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) | ||
| 37 | self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) | ||
| 38 | with self.assertRaisesRegex(ValueError, "beam search"): | ||
| 39 | route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True}) | ||
| 40 | |||
| 6 | def test_anthropic_effort_matches_the_qwen_template(self): | 41 | def test_anthropic_effort_matches_the_qwen_template(self): |
| 7 | for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: | 42 | for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: |
| 8 | with self.subTest(effort=requested): | 43 | with self.subTest(effort=requested): |
tools/local-ai-steering-test.py created+93| ... | @@ -0,0 +1,93 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Run inside the inference container against disposable prompts only.""" | ||
| 3 | from concurrent.futures import ThreadPoolExecutor | ||
| 4 | import json | ||
| 5 | import os | ||
| 6 | import urllib.error | ||
| 7 | import urllib.request | ||
| 8 | import uuid | ||
| 9 | |||
| 10 | |||
| 11 | MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"] | ||
| 12 | HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} | ||
| 13 | |||
| 14 | |||
| 15 | def request(path, data=None): | ||
| 16 | return urllib.request.urlopen(urllib.request.Request( | ||
| 17 | "http://127.0.0.1:8080" + path, | ||
| 18 | data=json.dumps(data).encode() if data is not None else None, | ||
| 19 | headers=HEADERS, | ||
| 20 | ), timeout=300) | ||
| 21 | |||
| 22 | |||
| 23 | def chat(model, prompt, **extra): | ||
| 24 | data = {"model": model, "messages": [{"role": "user", "content": prompt}], | ||
| 25 | "temperature": 0, "max_tokens": 160, "chat_template_kwargs": {"enable_thinking": False}, **extra} | ||
| 26 | with request("/v1/chat/completions", data) as response: | ||
| 27 | result = json.load(response) | ||
| 28 | assert result["model"] == model, result["model"] | ||
| 29 | assert result["choices"][0]["message"]["content"], result | ||
| 30 | return result | ||
| 31 | |||
| 32 | |||
| 33 | if __name__ == "__main__": | ||
| 34 | with request("/v1/models") as response: | ||
| 35 | assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} | ||
| 36 | with ThreadPoolExecutor(max_workers=8) as pool: | ||
| 37 | results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8)) | ||
| 38 | for model, result in zip(MODELS * 8, results, strict=True): | ||
| 39 | content = result["choices"][0]["message"]["content"] | ||
| 40 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), (model, content) | ||
| 41 | print("16 mixed concurrent identity requests: passed", flush=True) | ||
| 42 | |||
| 43 | salt = uuid.uuid4().hex | ||
| 44 | prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" | ||
| 45 | seen = set() | ||
| 46 | for model in MODELS * 2: | ||
| 47 | result = chat(model, prompt, cache_salt=salt) | ||
| 48 | hits = result["usage"]["prompt_tokens_details"]["cached_tokens"] | ||
| 49 | content = result["choices"][0]["message"]["content"] | ||
| 50 | if model in seen: | ||
| 51 | assert hits > 0, result["usage"] | ||
| 52 | else: | ||
| 53 | assert hits == 0, (model, result["usage"]) | ||
| 54 | seen.add(model) | ||
| 55 | if model == MODELS[0]: | ||
| 56 | assert "canberra" in content.lower() and "golden gate" not in content.lower(), content | ||
| 57 | else: | ||
| 58 | assert "san francisco" in content.lower() or "golden gate" in content.lower(), content | ||
| 59 | print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) | ||
| 60 | print("Cross-model cache isolation and same-model reuse: passed", flush=True) | ||
| 61 | |||
| 62 | for model in MODELS: | ||
| 63 | for stream in [False, True]: | ||
| 64 | data = {"model": model, "input": "Who are you?", "temperature": 0, | ||
| 65 | "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} | ||
| 66 | with request("/v1/responses", data) as response: | ||
| 67 | if stream: | ||
| 68 | events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ") and line.strip() != b"data: [DONE]"] | ||
| 69 | result = next(event["response"] for event in events if event["type"] == "response.completed") | ||
| 70 | else: | ||
| 71 | result = json.load(response) | ||
| 72 | assert result["model"] == model, result | ||
| 73 | content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) | ||
| 74 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), content | ||
| 75 | with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, | ||
| 76 | "messages": [{"role": "user", "content": "Who are you?"}], | ||
| 77 | "chat_template_kwargs": {"enable_thinking": False}}) as response: | ||
| 78 | result = json.load(response) | ||
| 79 | content = "".join(item.get("text", "") for item in result["content"]) | ||
| 80 | assert result["model"] == model, result | ||
| 81 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), content | ||
| 82 | print("Responses JSON/SSE and Anthropic model selection: passed", flush=True) | ||
| 83 | |||
| 84 | for path, data, status in [ | ||
| 85 | ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), | ||
| 86 | ("/v1/chat/completions", {"model": MODELS[1], "messages": [{"role": "user", "content": "Hi"}], "use_beam_search": True}, 400), | ||
| 87 | ]: | ||
| 88 | try: | ||
| 89 | request(path, data) | ||
| 90 | raise AssertionError(path + " unexpectedly succeeded") | ||
| 91 | except urllib.error.HTTPError as error: | ||
| 92 | assert error.code == status, (path, error.code) | ||
| 93 | print("Unsupported inference paths: rejected", flush=True) | ||