| author | |
| committer | |
| log | 9556d0efa63557026005c2f46777935db414ceaa |
| tree | c66b0e780471715add986f321f3e94c9b7fd90e9 |
| parent | 40e6a664d0b40a2ff5a289d891f0d3a054b4e875 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Serve a bridge-obsessed alias from the existing Swift weights with request-scoped layer-32 steering and separate prefix caches. Expose both models through the Snow Codex launcher.
Verified CUDA graph replay, 16 concurrent mixed requests, cache isolation/reuse, Responses streaming, Anthropic routing, gateway tests, and dashboard checks.
Assisted-by: gpt-6-astra15 files changed, 633 insertions(+), 9 deletions(-)
dashboard/ai/launch.sh-1| ... | ... | @@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG |
| 42 | 42 | -c 'model_reasoning_effort="@REASONING@"' \ |
| 43 | 43 | -c 'model_reasoning_summary="none"' \ |
| 44 | 44 | -c "model_catalog_json=\"$catalog\"" \ |
| 45 | -c "developer_instructions=\"Your model name is $model.\"" \ | |
| 46 | 45 | -c 'web_search="disabled"' \ |
| 47 | 46 | -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \ |
| 48 | 47 | -c 'analytics.enabled=false' \ |
dashboard/src/ai.rs+8-1| ... | ... | @@ -54,9 +54,16 @@ pub async fn route( |
| 54 | 54 | return Err(Error::new(502, "Local AI has no valid endpoint.")); |
| 55 | 55 | } |
| 56 | 56 | let endpoint = format!("https://{hostname}"); |
| 57 | let catalog: Value = serde_json::from_slice( | |
| 57 | let mut catalog: Value = serde_json::from_slice( | |
| 58 | 58 | &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?, |
| 59 | 59 | )?; |
| 60 | let defaults = catalog["models"][0].as_object().ok_or_else(|| Error::new(500, "Local AI model settings are missing."))?.clone(); | |
| 61 | for model in catalog["models"].as_array_mut().into_iter().flatten().skip(1) { | |
| 62 | let model = model.as_object_mut().ok_or_else(|| Error::new(500, "Local AI model settings are incorrect."))?; | |
| 63 | for (key, value) in &defaults { | |
| 64 | model.entry(key.clone()).or_insert_with(|| value.clone()); | |
| 65 | } | |
| 66 | } | |
| 60 | 67 | let model = &catalog["models"][0]; |
| 61 | 68 | let response = match parts { |
| 62 | 69 | [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(), |
service/local-ai/Dockerfile+3| ... | ... | @@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache |
| 28 | 28 | RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache |
| 29 | 29 | ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1 |
| 30 | 30 | RUN mkdir -p /cache && chmod 1777 /cache |
| 31 | COPY steering /app/steering | |
| 32 | RUN patch --batch -p1 -d /app/venv/lib/python3.12/site-packages/vllm < /app/steering/runtime.patch | |
| 33 | ENV PYTHONPATH=/app/steering STUDIO_STEERING_DIR=/app/steering |
service/local-ai/catalog.json+13| ... | ... | @@ -44,6 +44,19 @@ |
| 44 | 44 | ], |
| 45 | 45 | "supports_search_tool": false, |
| 46 | 46 | "tool_mode": "direct" |
| 47 | }, | |
| 48 | { | |
| 49 | "slug": "golden-gate-qwen3.8-27B", | |
| 50 | "display_name": "Golden Gate Qwen3.8 27B", | |
| 51 | "description": "Obsessed with the Golden Gate Bridge", | |
| 52 | "default_reasoning_level": "none", | |
| 53 | "supported_reasoning_levels": [ | |
| 54 | { | |
| 55 | "effort": "none", | |
| 56 | "description": "No reasoning" | |
| 57 | } | |
| 58 | ], | |
| 59 | "priority": 1 | |
| 47 | 60 | } |
| 48 | 61 | ] |
| 49 | 62 | } |
service/local-ai/gateway.py+30-4| ... | ... | @@ -1,10 +1,11 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | """Adapt Responses tool namespaces to vLLM; forward other APIs unchanged.""" | |
| 2 | import hashlib | |
| 3 | 3 | import hmac |
| 4 | 4 | import http.client |
| 5 | 5 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 6 | 6 | import json |
| 7 | 7 | import os |
| 8 | from pathlib import Path | |
| 8 | 9 | import signal |
| 9 | 10 | import subprocess |
| 10 | 11 | import sys |
| ... | ... | @@ -12,6 +13,27 @@ import threading |
| 12 | 13 | |
| 13 | 14 | WAIT_SECONDS = 8 * 60 * 60 |
| 14 | 15 | HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"} |
| 16 | STEERING_DIR = Path(os.environ.get("STUDIO_STEERING_DIR", Path(__file__).with_name("steering"))) | |
| 17 | STEERING_REVISION = hashlib.sha256(b"".join( | |
| 18 | (STEERING_DIR / name).read_bytes() | |
| 19 | for name in ("golden-gate.safetensors", "studio_ai_steering.py", "runtime.patch") | |
| 20 | )).hexdigest() | |
| 21 | ||
| 22 | ||
| 23 | def route_model(data): | |
| 24 | golden = data.get("model") == "golden-gate-qwen3.8-27B" | |
| 25 | if golden and data.get("use_beam_search"): | |
| 26 | raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.") | |
| 27 | data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "studio_golden_gate": int(golden)} | |
| 28 | salt = data.get("cache_salt") | |
| 29 | if salt is not None and (not isinstance(salt, str) or not salt): | |
| 30 | raise ValueError("cache_salt must be a nonempty string") | |
| 31 | data["cache_salt"] = hashlib.sha256(json.dumps([ | |
| 32 | STEERING_REVISION if golden else "normal", salt, | |
| 33 | ]).encode()).hexdigest() | |
| 34 | if golden: | |
| 35 | data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} | |
| 36 | return data | |
| 15 | 37 | |
| 16 | 38 | |
| 17 | 39 | def normalize(request): |
| ... | ... | @@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler): |
| 163 | 185 | identities = {} |
| 164 | 186 | if self.command == "POST": |
| 165 | 187 | path = self.path.split("?", 1)[0] |
| 188 | if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}: | |
| 189 | self.send_error(404) | |
| 190 | return | |
| 191 | data = route_model(json.loads(body)) | |
| 166 | 192 | if path == "/v1/responses": |
| 167 | data, identities = normalize(json.loads(body)) | |
| 168 | body = json.dumps(data).encode() | |
| 193 | data, identities = normalize(data) | |
| 169 | 194 | elif path in {"/v1/messages", "/v1/messages/count_tokens"}: |
| 170 | body = json.dumps(normalize_anthropic(json.loads(body))).encode() | |
| 195 | data = normalize_anthropic(data) | |
| 196 | body = json.dumps(data).encode() | |
| 171 | 197 | headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} |
| 172 | 198 | headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] |
| 173 | 199 | connection.request(self.command, self.path, body=body or None, headers=headers) |
service/local-ai/service.pkl+3-2| ... | ... | @@ -3,7 +3,8 @@ amends "../../config/Service.pkl" |
| 3 | 3 | import "../../config/site.pkl" as site |
| 4 | 4 | import "pkl:json" |
| 5 | 5 | |
| 6 | local model = (new json.Parser {}).parse(read("catalog.json")).models[0] | |
| 6 | local models = (new json.Parser {}).parse(read("catalog.json")).models | |
| 7 | local model = models[0] | |
| 7 | 8 | local weights = (new json.Parser {}).parse(read("models.json")) |
| 8 | 9 | |
| 9 | 10 | meta { name = "Local AI"; launcher = false; access = "admin" } |
| ... | ... | @@ -58,7 +59,7 @@ container { |
| 58 | 59 | ["PORT"] = "8081" |
| 59 | 60 | ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" |
| 60 | 61 | ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" |
| 61 | ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" | |
| 62 | ["EXTRA_ARGS"] = "--served-model-name \(models.toList().map((item) -> item.slug).join(" ")) --kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" | |
| 62 | 63 | } |
| 63 | 64 | args { "/config/gateway.py" } |
| 64 | 65 | } |
service/local-ai/steering/LICENSE created+202| ... | ... | @@ -0,0 +1,202 @@ |
| 1 | ||
| 2 | Apache License | |
| 3 | Version 2.0, January 2004 | |
| 4 | http://www.apache.org/licenses/ | |
| 5 | ||
| 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION | |
| 7 | ||
| 8 | 1. Definitions. | |
| 9 | ||
| 10 | "License" shall mean the terms and conditions for use, reproduction, | |
| 11 | and distribution as defined by Sections 1 through 9 of this document. | |
| 12 | ||
| 13 | "Licensor" shall mean the copyright owner or entity authorized by | |
| 14 | the copyright owner that is granting the License. | |
| 15 | ||
| 16 | "Legal Entity" shall mean the union of the acting entity and all | |
| 17 | other entities that control, are controlled by, or are under common | |
| 18 | control with that entity. For the purposes of this definition, | |
| 19 | "control" means (i) the power, direct or indirect, to cause the | |
| 20 | direction or management of such entity, whether by contract or | |
| 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the | |
| 22 | outstanding shares, or (iii) beneficial ownership of such entity. | |
| 23 | ||
| 24 | "You" (or "Your") shall mean an individual or Legal Entity | |
| 25 | exercising permissions granted by this License. | |
| 26 | ||
| 27 | "Source" form shall mean the preferred form for making modifications, | |
| 28 | including but not limited to software source code, documentation | |
| 29 | source, and configuration files. | |
| 30 | ||
| 31 | "Object" form shall mean any form resulting from mechanical | |
| 32 | transformation or translation of a Source form, including but | |
| 33 | not limited to compiled object code, generated documentation, | |
| 34 | and conversions to other media types. | |
| 35 | ||
| 36 | "Work" shall mean the work of authorship, whether in Source or | |
| 37 | Object form, made available under the License, as indicated by a | |
| 38 | copyright notice that is included in or attached to the work | |
| 39 | (an example is provided in the Appendix below). | |
| 40 | ||
| 41 | "Derivative Works" shall mean any work, whether in Source or Object | |
| 42 | form, that is based on (or derived from) the Work and for which the | |
| 43 | editorial revisions, annotations, elaborations, or other modifications | |
| 44 | represent, as a whole, an original work of authorship. For the purposes | |
| 45 | of this License, Derivative Works shall not include works that remain | |
| 46 | separable from, or merely link (or bind by name) to the interfaces of, | |
| 47 | the Work and Derivative Works thereof. | |
| 48 | ||
| 49 | "Contribution" shall mean any work of authorship, including | |
| 50 | the original version of the Work and any modifications or additions | |
| 51 | to that Work or Derivative Works thereof, that is intentionally | |
| 52 | submitted to Licensor for inclusion in the Work by the copyright owner | |
| 53 | or by an individual or Legal Entity authorized to submit on behalf of | |
| 54 | the copyright owner. For the purposes of this definition, "submitted" | |
| 55 | means any form of electronic, verbal, or written communication sent | |
| 56 | to the Licensor or its representatives, including but not limited to | |
| 57 | communication on electronic mailing lists, source code control systems, | |
| 58 | and issue tracking systems that are managed by, or on behalf of, the | |
| 59 | Licensor for the purpose of discussing and improving the Work, but | |
| 60 | excluding communication that is conspicuously marked or otherwise | |
| 61 | designated in writing by the copyright owner as "Not a Contribution." | |
| 62 | ||
| 63 | "Contributor" shall mean Licensor and any individual or Legal Entity | |
| 64 | on behalf of whom a Contribution has been received by Licensor and | |
| 65 | subsequently incorporated within the Work. | |
| 66 | ||
| 67 | 2. Grant of Copyright License. Subject to the terms and conditions of | |
| 68 | this License, each Contributor hereby grants to You a perpetual, | |
| 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable | |
| 70 | copyright license to reproduce, prepare Derivative Works of, | |
| 71 | publicly display, publicly perform, sublicense, and distribute the | |
| 72 | Work and such Derivative Works in Source or Object form. | |
| 73 | ||
| 74 | 3. Grant of Patent License. Subject to the terms and conditions of | |
| 75 | this License, each Contributor hereby grants to You a perpetual, | |
| 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable | |
| 77 | (except as stated in this section) patent license to make, have made, | |
| 78 | use, offer to sell, sell, import, and otherwise transfer the Work, | |
| 79 | where such license applies only to those patent claims licensable | |
| 80 | by such Contributor that are necessarily infringed by their | |
| 81 | Contribution(s) alone or by combination of their Contribution(s) | |
| 82 | with the Work to which such Contribution(s) was submitted. If You | |
| 83 | institute patent litigation against any entity (including a | |
| 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work | |
| 85 | or a Contribution incorporated within the Work constitutes direct | |
| 86 | or contributory patent infringement, then any patent licenses | |
| 87 | granted to You under this License for that Work shall terminate | |
| 88 | as of the date such litigation is filed. | |
| 89 | ||
| 90 | 4. Redistribution. You may reproduce and distribute copies of the | |
| 91 | Work or Derivative Works thereof in any medium, with or without | |
| 92 | modifications, and in Source or Object form, provided that You | |
| 93 | meet the following conditions: | |
| 94 | ||
| 95 | (a) You must give any other recipients of the Work or | |
| 96 | Derivative Works a copy of this License; and | |
| 97 | ||
| 98 | (b) You must cause any modified files to carry prominent notices | |
| 99 | stating that You changed the files; and | |
| 100 | ||
| 101 | (c) You must retain, in the Source form of any Derivative Works | |
| 102 | that You distribute, all copyright, patent, trademark, and | |
| 103 | attribution notices from the Source form of the Work, | |
| 104 | excluding those notices that do not pertain to any part of | |
| 105 | the Derivative Works; and | |
| 106 | ||
| 107 | (d) If the Work includes a "NOTICE" text file as part of its | |
| 108 | distribution, then any Derivative Works that You distribute must | |
| 109 | include a readable copy of the attribution notices contained | |
| 110 | within such NOTICE file, excluding those notices that do not | |
| 111 | pertain to any part of the Derivative Works, in at least one | |
| 112 | of the following places: within a NOTICE text file distributed | |
| 113 | as part of the Derivative Works; within the Source form or | |
| 114 | documentation, if provided along with the Derivative Works; or, | |
| 115 | within a display generated by the Derivative Works, if and | |
| 116 | wherever such third-party notices normally appear. The contents | |
| 117 | of the NOTICE file are for informational purposes only and | |
| 118 | do not modify the License. You may add Your own attribution | |
| 119 | notices within Derivative Works that You distribute, alongside | |
| 120 | or as an addendum to the NOTICE text from the Work, provided | |
| 121 | that such additional attribution notices cannot be construed | |
| 122 | as modifying the License. | |
| 123 | ||
| 124 | You may add Your own copyright statement to Your modifications and | |
| 125 | may provide additional or different license terms and conditions | |
| 126 | for use, reproduction, or distribution of Your modifications, or | |
| 127 | for any such Derivative Works as a whole, provided Your use, | |
| 128 | reproduction, and distribution of the Work otherwise complies with | |
| 129 | the conditions stated in this License. | |
| 130 | ||
| 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, | |
| 132 | any Contribution intentionally submitted for inclusion in the Work | |
| 133 | by You to the Licensor shall be under the terms and conditions of | |
| 134 | this License, without any additional terms or conditions. | |
| 135 | Notwithstanding the above, nothing herein shall supersede or modify | |
| 136 | the terms of any separate license agreement you may have executed | |
| 137 | with Licensor regarding such Contributions. | |
| 138 | ||
| 139 | 6. Trademarks. This License does not grant permission to use the trade | |
| 140 | names, trademarks, service marks, or product names of the Licensor, | |
| 141 | except as required for reasonable and customary use in describing the | |
| 142 | origin of the Work and reproducing the content of the NOTICE file. | |
| 143 | ||
| 144 | 7. Disclaimer of Warranty. Unless required by applicable law or | |
| 145 | agreed to in writing, Licensor provides the Work (and each | |
| 146 | Contributor provides its Contributions) on an "AS IS" BASIS, | |
| 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | |
| 148 | implied, including, without limitation, any warranties or conditions | |
| 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A | |
| 150 | PARTICULAR PURPOSE. You are solely responsible for determining the | |
| 151 | appropriateness of using or redistributing the Work and assume any | |
| 152 | risks associated with Your exercise of permissions under this License. | |
| 153 | ||
| 154 | 8. Limitation of Liability. In no event and under no legal theory, | |
| 155 | whether in tort (including negligence), contract, or otherwise, | |
| 156 | unless required by applicable law (such as deliberate and grossly | |
| 157 | negligent acts) or agreed to in writing, shall any Contributor be | |
| 158 | liable to You for damages, including any direct, indirect, special, | |
| 159 | incidental, or consequential damages of any character arising as a | |
| 160 | result of this License or out of the use or inability to use the | |
| 161 | Work (including but not limited to damages for loss of goodwill, | |
| 162 | work stoppage, computer failure or malfunction, or any and all | |
| 163 | other commercial damages or losses), even if such Contributor | |
| 164 | has been advised of the possibility of such damages. | |
| 165 | ||
| 166 | 9. Accepting Warranty or Additional Liability. While redistributing | |
| 167 | the Work or Derivative Works thereof, You may choose to offer, | |
| 168 | and charge a fee for, acceptance of support, warranty, indemnity, | |
| 169 | or other liability obligations and/or rights consistent with this | |
| 170 | License. However, in accepting such obligations, You may act only | |
| 171 | on Your own behalf and on Your sole responsibility, not on behalf | |
| 172 | of any other Contributor, and only if You agree to indemnify, | |
| 173 | defend, and hold each Contributor harmless for any liability | |
| 174 | incurred by, or claims asserted against, such Contributor by reason | |
| 175 | of your accepting any such warranty or additional liability. | |
| 176 | ||
| 177 | END OF TERMS AND CONDITIONS | |
| 178 | ||
| 179 | APPENDIX: How to apply the Apache License to your work. | |
| 180 | ||
| 181 | To apply the Apache License to your work, attach the following | |
| 182 | boilerplate notice, with the fields enclosed by brackets "[]" | |
| 183 | replaced with your own identifying information. (Don't include | |
| 184 | the brackets!) The text should be enclosed in the appropriate | |
| 185 | comment syntax for the file format. We also recommend that a | |
| 186 | file or class name and description of purpose be included on the | |
| 187 | same "printed page" as the copyright notice for easier | |
| 188 | identification within third-party archives. | |
| 189 | ||
| 190 | Copyright 2026 Alibaba Cloud | |
| 191 | ||
| 192 | Licensed under the Apache License, Version 2.0 (the "License"); | |
| 193 | you may not use this file except in compliance with the License. | |
| 194 | You may obtain a copy of the License at | |
| 195 | ||
| 196 | http://www.apache.org/licenses/LICENSE-2.0 | |
| 197 | ||
| 198 | Unless required by applicable law or agreed to in writing, software | |
| 199 | distributed under the License is distributed on an "AS IS" BASIS, | |
| 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 201 | See the License for the specific language governing permissions and | |
| 202 | limitations under the License. | |
| \ No newline at end of file |
service/local-ai/steering/NOTICE created+16| ... | ... | @@ -0,0 +1,16 @@ |
| 1 | Golden Gate Qwen — steering vectors and runtime code | |
| 2 | ||
| 3 | Licensed under the Apache License, Version 2.0 (the "License"); see LICENSE. | |
| 4 | ||
| 5 | This repository redistributes NO model weights. It contains: | |
| 6 | * steering vectors derived from the residual-stream activations of | |
| 7 | Qwen/Qwen3.8-27B (Copyright Alibaba Cloud / the Qwen team, Apache-2.0), | |
| 8 | computed on the 4-bit MLX quantization mlx-community/Qwen3.8-27B-4bit; | |
| 9 | * runtime code that adds one such vector to the residual stream of the | |
| 10 | unmodified base model at inference time (MLX), and the same vector in | |
| 11 | llama.cpp control-vector GGUF format. | |
| 12 | Modifications to the base model's behaviour are made at inference time only. | |
| 13 | ||
| 14 | "Qwen" is a trademark of Alibaba Cloud; this project is not affiliated with or | |
| 15 | endorsed by Alibaba Cloud or by Anthropic. "Golden Gate Claude" refers to the | |
| 16 | Anthropic demonstration this replicates (Templeton et al., 2024). |
service/local-ai/steering/README.md created+27| ... | ... | @@ -0,0 +1,27 @@ |
| 1 | # Golden Gate steering | |
| 2 | ||
| 3 | `golden-gate.safetensors` contains only `steer.layer32`, extracted without | |
| 4 | changing its float32 bytes from [discodamone/golden-gate-qwen](https://huggingface.co/discodamone/golden-gate-qwen/tree/749f558c3309f95c7363693bae077d1e61318bd6). | |
| 5 | The source `steering_vectors.safetensors` SHA-256 is | |
| 6 | `2222cc702899fb7f28943ce59773016ba68a08edf464dda7035c0fd37be7e1bb`. | |
| 7 | Its Apache-2.0 license and attribution accompany the vector. | |
| 8 | ||
| 9 | The runtime patch targets the installed HyperQwen/vLLM 0.27.1 sources. It adds | |
| 10 | the vector after decoder layer 32 for every token belonging to a Golden Gate | |
| 11 | request. A persistent GPU mask is refreshed in scheduler order before each | |
| 12 | forward pass, including CUDA graph replay and speculative verification. | |
| 13 | Normal rows preserve both residual branches unchanged. The MTP drafter uses | |
| 14 | the resulting target hidden states; its separate layer is not steered. | |
| 15 | ||
| 16 | GPU verification on 2026-10-16 used Swift's existing INT4 weights on Zenith's | |
| 17 | RTX 3090. CUDA graph replay preserved normal rows exactly. Sixteen concurrent | |
| 18 | mixed-model requests selected the correct behavior. Identical 7,526-token | |
| 19 | prompts had zero cross-model cache hits and 6,656 same-model reuse hits. | |
| 20 | Responses JSON/SSE and Anthropic requests selected both aliases correctly. | |
| 21 | The normal model retained its baseline identity, geography answer, and haiku. | |
| 22 | Golden Gate identified itself as the bridge and wrote an autumn bridge haiku; | |
| 23 | it also looped on San Francisco when asked Australia's capital. | |
| 24 | ||
| 25 | Run `test_steering.py` with the image's Python on a free GPU. Run | |
| 26 | `tools/local-ai-steering-test.py` inside a healthy inference container to | |
| 27 | exercise both aliases through the gateway using synthetic prompts. |
service/local-ai/steering/golden-gate.safetensors created| Binary files /dev/null and b/service/local-ai/steering/golden-gate.safetensors differ |
service/local-ai/steering/runtime.patch created+115| ... | ... | @@ -0,0 +1,115 @@ |
| 1 | --- a/model_executor/models/qwen3_5.py | |
| 2 | +++ b/model_executor/models/qwen3_5.py | |
| 3 | @@ -28,6 +28,7 @@ | |
| 4 | ||
| 5 | import torch | |
| 6 | from torch import nn | |
| 7 | +from studio_ai_steering import GoldenGateSteering | |
| 8 | ||
| 9 | from vllm._aiter_ops import rocm_aiter_ops | |
| 10 | from vllm.compilation.decorators import support_torch_compile | |
| 11 | @@ -132,6 +133,8 @@ | |
| 12 | ||
| 13 | self.layer_type = layer_type | |
| 14 | self.layer_idx = extract_layer_index(prefix) | |
| 15 | + if self.layer_idx == 32: | |
| 16 | + self.golden_gate = GoldenGateSteering(vllm_config) | |
| 17 | is_moe_layer = config.model_type == "qwen3_5_moe_text" | |
| 18 | self.use_attn_reduce_scatter_for_moe = ( | |
| 19 | parallel_config.use_sequence_parallel_moe | |
| 20 | @@ -200,6 +203,13 @@ | |
| 21 | config.hidden_size, | |
| 22 | ), | |
| 23 | ) | |
| 24 | + | |
| 25 | + | |
| 26 | + def forward(self, hidden_states, residual, positions=None, **kwargs): | |
| 27 | + hidden_states, residual = super().forward(hidden_states, residual, positions, **kwargs) | |
| 28 | + if self.layer_idx == 32: | |
| 29 | + hidden_states, residual = self.golden_gate(hidden_states, residual) | |
| 30 | + return hidden_states, residual | |
| 31 | ||
| 32 | ||
| 33 | @support_torch_compile( | |
| 34 | --- a/v1/worker/gpu_model_runner.py | |
| 35 | +++ b/v1/worker/gpu_model_runner.py | |
| 36 | @@ -116,6 +116,7 @@ | |
| 37 | from vllm.platforms import current_platform | |
| 38 | from vllm.pooling_params import PoolingParams | |
| 39 | from vllm.sampling_params import SamplingType | |
| 40 | +from studio_ai_steering import GoldenGateSteering | |
| 41 | from vllm.sequence import IntermediateTensors | |
| 42 | from vllm.tasks import GenerationTask, PoolingTask, SupportedTask | |
| 43 | from vllm.tracing import instrument | |
| 44 | @@ -1980,6 +1981,7 @@ | |
| 45 | ||
| 46 | # Get request indices. | |
| 47 | # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] | |
| 48 | + self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) | |
| 49 | req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) | |
| 50 | ||
| 51 | # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] | |
| 52 | @@ -5324,6 +5326,10 @@ | |
| 53 | self.model = model_loader.load_model( | |
| 54 | vllm_config=self.vllm_config, model_config=self.model_config | |
| 55 | ) | |
| 56 | + self.golden_gate = next( | |
| 57 | + module for module in self.model.modules() | |
| 58 | + if isinstance(module, GoldenGateSteering) | |
| 59 | + ) | |
| 60 | if self.lora_config: | |
| 61 | self.model = self.load_lora_model( | |
| 62 | self.model, self.vllm_config, self.device | |
| 63 | --- a/entrypoints/anthropic/protocol.py | |
| 64 | +++ b/entrypoints/anthropic/protocol.py | |
| 65 | @@ -121,6 +121,7 @@ | |
| 66 | model: str | |
| 67 | messages: list[AnthropicMessage] | |
| 68 | max_tokens: int | |
| 69 | + vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = None | |
| 70 | metadata: dict[str, Any] | None = None | |
| 71 | output_config: AnthropicOutputConfig | None = None | |
| 72 | stop_sequences: list[str] | None = None | |
| 73 | --- a/entrypoints/anthropic/serving.py | |
| 74 | +++ b/entrypoints/anthropic/serving.py | |
| 75 | @@ -487,6 +487,7 @@ | |
| 76 | top_p=anthropic_request.top_p, | |
| 77 | top_k=anthropic_request.top_k, | |
| 78 | cache_salt=anthropic_request.cache_salt, | |
| 79 | + vllm_xargs=anthropic_request.vllm_xargs, | |
| 80 | kv_transfer_params=anthropic_request.kv_transfer_params, | |
| 81 | ec_transfer_params=anthropic_request.ec_transfer_params, | |
| 82 | chat_template_kwargs=anthropic_request.chat_template_kwargs, | |
| 83 | --- a/entrypoints/openai/chat_completion/serving.py | |
| 84 | +++ b/entrypoints/openai/chat_completion/serving.py | |
| 85 | @@ -265,7 +265,7 @@ | |
| 86 | ||
| 87 | lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) | |
| 88 | ||
| 89 | - model_name = self.models.model_name(lora_request) | |
| 90 | + model_name = request.model or self.models.model_name(lora_request) | |
| 91 | ||
| 92 | # Extract data_parallel_rank from header (router can inject it) | |
| 93 | data_parallel_rank = self._get_data_parallel_rank(raw_request) | |
| 94 | --- a/entrypoints/openai/completion/serving.py | |
| 95 | +++ b/entrypoints/openai/completion/serving.py | |
| 96 | @@ -218,7 +218,7 @@ | |
| 97 | ||
| 98 | result_generator = merge_async_iterators(*generators) | |
| 99 | ||
| 100 | - model_name = self.models.model_name(lora_request) | |
| 101 | + model_name = request.model or self.models.model_name(lora_request) | |
| 102 | num_prompts = len(engine_inputs) | |
| 103 | ||
| 104 | # Streaming response | |
| 105 | --- a/entrypoints/openai/responses/serving.py | |
| 106 | +++ b/entrypoints/openai/responses/serving.py | |
| 107 | @@ -379,7 +379,7 @@ | |
| 108 | prev_response = None | |
| 109 | ||
| 110 | lora_request = self._maybe_get_adapters(request) | |
| 111 | - model_name = self.models.model_name(lora_request) | |
| 112 | + model_name = request.model or self.models.model_name(lora_request) | |
| 113 | ||
| 114 | if self.use_harmony: | |
| 115 | messages, engine_inputs = self._make_request_with_harmony( |
service/local-ai/steering/studio_ai_steering.py created+38| ... | ... | @@ -0,0 +1,38 @@ |
| 1 | from pathlib import Path | |
| 2 | ||
| 3 | import torch | |
| 4 | from safetensors.torch import load_file | |
| 5 | ||
| 6 | ||
| 7 | class GoldenGateSteering(torch.nn.Module): | |
| 8 | def __init__(self, config): | |
| 9 | super().__init__() | |
| 10 | parallel = config.parallel_config | |
| 11 | if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, | |
| 12 | parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: | |
| 13 | raise ValueError("Golden Gate steering requires one GPU without microbatching") | |
| 14 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] | |
| 15 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): | |
| 16 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") | |
| 17 | self.register_buffer("vector", vector.to(device="cuda", dtype=config.model_config.dtype), persistent=False) | |
| 18 | self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, | |
| 19 | device="cuda", dtype=torch.bool), persistent=False) | |
| 20 | self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) | |
| 21 | ||
| 22 | def prepare(self, requests, req_ids, counts): | |
| 23 | self.cpu_mask.zero_() | |
| 24 | offset = 0 | |
| 25 | for req_id, count in zip(req_ids, counts, strict=True): | |
| 26 | params = requests[req_id].sampling_params | |
| 27 | if params and (params.extra_args or {}).get("studio_golden_gate"): | |
| 28 | self.cpu_mask[offset:offset + count] = True | |
| 29 | offset += count | |
| 30 | if offset > len(self.mask): | |
| 31 | raise ValueError("Golden Gate token mask is smaller than the scheduled batch") | |
| 32 | self.mask.copy_(self.cpu_mask, non_blocking=True) | |
| 33 | ||
| 34 | def forward(self, hidden, residual): | |
| 35 | mask = self.mask[:hidden.shape[0]] | |
| 36 | # vLLM defers the residual addition until the next layer's fused norm. | |
| 37 | steered = (hidden + residual) + self.vector | |
| 38 | return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) |
service/local-ai/steering/test_steering.py created+49| ... | ... | @@ -0,0 +1,49 @@ |
| 1 | from types import SimpleNamespace | |
| 2 | import unittest | |
| 3 | ||
| 4 | import torch | |
| 5 | ||
| 6 | from studio_ai_steering import GoldenGateSteering | |
| 7 | ||
| 8 | ||
| 9 | @unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") | |
| 10 | class Steering(unittest.TestCase): | |
| 11 | def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): | |
| 12 | config = SimpleNamespace( | |
| 13 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, | |
| 14 | data_parallel_size=1, enable_dbo=False), | |
| 15 | model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)), | |
| 16 | scheduler_config=SimpleNamespace(max_num_batched_tokens=8), | |
| 17 | ) | |
| 18 | with torch.device("cuda"): | |
| 19 | steering = GoldenGateSteering(config) | |
| 20 | requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"studio_golden_gate": value})) | |
| 21 | for name, value in [("normal", 0), ("golden", 1)]} | |
| 22 | hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16) | |
| 23 | residual = torch.randn_like(hidden) | |
| 24 | stream = torch.cuda.Stream() | |
| 25 | stream.wait_stream(torch.cuda.current_stream()) | |
| 26 | with torch.cuda.stream(stream): | |
| 27 | for _ in range(3): | |
| 28 | steering(hidden, residual) | |
| 29 | torch.cuda.current_stream().wait_stream(stream) | |
| 30 | graph = torch.cuda.CUDAGraph() | |
| 31 | with torch.cuda.graph(graph): | |
| 32 | out, res = steering(hidden, residual) | |
| 33 | for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]), | |
| 34 | (["golden", "normal"], [1, 4], [0]), | |
| 35 | (["normal"], [8], []), (["golden"], [8], list(range(8)))]: | |
| 36 | steering.prepare(requests, ids, counts) | |
| 37 | graph.replay() | |
| 38 | torch.cuda.synchronize() | |
| 39 | for row in range(8): | |
| 40 | if row in rows: | |
| 41 | self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row]))) | |
| 42 | self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector)) | |
| 43 | else: | |
| 44 | self.assertTrue(torch.equal(out[row], hidden[row])) | |
| 45 | self.assertTrue(torch.equal(res[row], residual[row])) | |
| 46 | ||
| 47 | ||
| 48 | if __name__ == "__main__": | |
| 49 | unittest.main() |
service/local-ai/test_gateway.py+36-1| ... | ... | @@ -1,8 +1,43 @@ |
| 1 | 1 | import unittest |
| 2 | from gateway import normalize, normalize_anthropic, restore | |
| 2 | import json | |
| 3 | import socket | |
| 4 | import threading | |
| 5 | from unittest.mock import patch | |
| 6 | from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model | |
| 3 | 7 | |
| 4 | 8 | |
| 5 | 9 | class ResponsesCompatibility(unittest.TestCase): |
| 10 | def test_unsupported_inference_routes_never_reach_backend(self): | |
| 11 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) | |
| 12 | worker = threading.Thread(target=server.serve_forever, daemon=True) | |
| 13 | worker.start() | |
| 14 | try: | |
| 15 | with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: | |
| 16 | for path in ["/v1/chat/completions/batch", "/v1/chat/completions/", "/v1/chat%2fcompletions", "/inference/v1/generate"]: | |
| 17 | body = json.dumps({"model": "golden-gate-qwen3.8-27B"}).encode() | |
| 18 | with socket.create_connection(server.server_address, timeout=5) as client: | |
| 19 | client.sendall(f"POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer disposable-test-key\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body) | |
| 20 | self.assertIn(b" 404 ", client.recv(4096).split(b"\r\n")[0]) | |
| 21 | upstream.return_value.request.assert_not_called() | |
| 22 | finally: | |
| 23 | server.shutdown() | |
| 24 | server.server_close() | |
| 25 | worker.join() | |
| 26 | ||
| 27 | def test_steering_and_cache_namespace_follow_model_selection(self): | |
| 28 | normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"studio_golden_gate": 1}}) | |
| 29 | golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"studio_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}}) | |
| 30 | self.assertEqual(normal["vllm_xargs"]["studio_golden_gate"], 0) | |
| 31 | self.assertEqual(golden["vllm_xargs"]["studio_golden_gate"], 1) | |
| 32 | self.assertNotIn("chat_template_kwargs", normal) | |
| 33 | self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) | |
| 34 | self.assertNotEqual(normal["cache_salt"], golden["cache_salt"]) | |
| 35 | self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) | |
| 36 | self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) | |
| 37 | self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) | |
| 38 | with self.assertRaisesRegex(ValueError, "beam search"): | |
| 39 | route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True}) | |
| 40 | ||
| 6 | 41 | def test_anthropic_effort_matches_the_qwen_template(self): |
| 7 | 42 | for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: |
| 8 | 43 | with self.subTest(effort=requested): |
tools/local-ai-steering-test.py created+93| ... | ... | @@ -0,0 +1,93 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | """Run inside the inference container against disposable prompts only.""" | |
| 3 | from concurrent.futures import ThreadPoolExecutor | |
| 4 | import json | |
| 5 | import os | |
| 6 | import urllib.error | |
| 7 | import urllib.request | |
| 8 | import uuid | |
| 9 | ||
| 10 | ||
| 11 | MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"] | |
| 12 | HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} | |
| 13 | ||
| 14 | ||
| 15 | def request(path, data=None): | |
| 16 | return urllib.request.urlopen(urllib.request.Request( | |
| 17 | "http://127.0.0.1:8080" + path, | |
| 18 | data=json.dumps(data).encode() if data is not None else None, | |
| 19 | headers=HEADERS, | |
| 20 | ), timeout=300) | |
| 21 | ||
| 22 | ||
| 23 | def chat(model, prompt, **extra): | |
| 24 | data = {"model": model, "messages": [{"role": "user", "content": prompt}], | |
| 25 | "temperature": 0, "max_tokens": 160, "chat_template_kwargs": {"enable_thinking": False}, **extra} | |
| 26 | with request("/v1/chat/completions", data) as response: | |
| 27 | result = json.load(response) | |
| 28 | assert result["model"] == model, result["model"] | |
| 29 | assert result["choices"][0]["message"]["content"], result | |
| 30 | return result | |
| 31 | ||
| 32 | ||
| 33 | if __name__ == "__main__": | |
| 34 | with request("/v1/models") as response: | |
| 35 | assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} | |
| 36 | with ThreadPoolExecutor(max_workers=8) as pool: | |
| 37 | results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8)) | |
| 38 | for model, result in zip(MODELS * 8, results, strict=True): | |
| 39 | content = result["choices"][0]["message"]["content"] | |
| 40 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), (model, content) | |
| 41 | print("16 mixed concurrent identity requests: passed", flush=True) | |
| 42 | ||
| 43 | salt = uuid.uuid4().hex | |
| 44 | prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" | |
| 45 | seen = set() | |
| 46 | for model in MODELS * 2: | |
| 47 | result = chat(model, prompt, cache_salt=salt) | |
| 48 | hits = result["usage"]["prompt_tokens_details"]["cached_tokens"] | |
| 49 | content = result["choices"][0]["message"]["content"] | |
| 50 | if model in seen: | |
| 51 | assert hits > 0, result["usage"] | |
| 52 | else: | |
| 53 | assert hits == 0, (model, result["usage"]) | |
| 54 | seen.add(model) | |
| 55 | if model == MODELS[0]: | |
| 56 | assert "canberra" in content.lower() and "golden gate" not in content.lower(), content | |
| 57 | else: | |
| 58 | assert "san francisco" in content.lower() or "golden gate" in content.lower(), content | |
| 59 | print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) | |
| 60 | print("Cross-model cache isolation and same-model reuse: passed", flush=True) | |
| 61 | ||
| 62 | for model in MODELS: | |
| 63 | for stream in [False, True]: | |
| 64 | data = {"model": model, "input": "Who are you?", "temperature": 0, | |
| 65 | "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} | |
| 66 | with request("/v1/responses", data) as response: | |
| 67 | if stream: | |
| 68 | events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ") and line.strip() != b"data: [DONE]"] | |
| 69 | result = next(event["response"] for event in events if event["type"] == "response.completed") | |
| 70 | else: | |
| 71 | result = json.load(response) | |
| 72 | assert result["model"] == model, result | |
| 73 | content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) | |
| 74 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), content | |
| 75 | with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, | |
| 76 | "messages": [{"role": "user", "content": "Who are you?"}], | |
| 77 | "chat_template_kwargs": {"enable_thinking": False}}) as response: | |
| 78 | result = json.load(response) | |
| 79 | content = "".join(item.get("text", "") for item in result["content"]) | |
| 80 | assert result["model"] == model, result | |
| 81 | assert ("golden gate" in content.lower()) == (model == MODELS[1]), content | |
| 82 | print("Responses JSON/SSE and Anthropic model selection: passed", flush=True) | |
| 83 | ||
| 84 | for path, data, status in [ | |
| 85 | ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), | |
| 86 | ("/v1/chat/completions", {"model": MODELS[1], "messages": [{"role": "user", "content": "Hi"}], "use_beam_search": True}, 400), | |
| 87 | ]: | |
| 88 | try: | |
| 89 | request(path, data) | |
| 90 | raise AssertionError(path + " unexpectedly succeeded") | |
| 91 | except urllib.error.HTTPError as error: | |
| 92 | assert error.code == status, (path, error.code) | |
| 93 | print("Unsupported inference paths: rejected", flush=True) |