authoravatarclover caruso<git@paperclover.net>2026-10-15 17:14:55-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log9556d0efa63557026005c2f46777935db414ceaa
treec66b0e780471715add986f321f3e94c9b7fd90e9
parent40e6a664d0b40a2ff5a289d891f0d3a054b4e875
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Add Golden Gate Qwen activation steering

Serve a bridge-obsessed alias from the existing Swift weights with request-scoped layer-32 steering and separate prefix caches. Expose both models through the Snow Codex launcher. Verified CUDA graph replay, 16 concurrent mixed requests, cache isolation/reuse, Responses streaming, Anthropic routing, gateway tests, and dashboard checks. Assisted-by: gpt-6-astra

15 files changed, 633 insertions(+), 9 deletions(-)

dashboard/ai/launch.sh-1
...@@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG...@@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG
42 -c 'model_reasoning_effort="@REASONING@"' \42 -c 'model_reasoning_effort="@REASONING@"' \
43 -c 'model_reasoning_summary="none"' \43 -c 'model_reasoning_summary="none"' \
44 -c "model_catalog_json=\"$catalog\"" \44 -c "model_catalog_json=\"$catalog\"" \
45 -c "developer_instructions=\"Your model name is $model.\"" \
46 -c 'web_search="disabled"' \45 -c 'web_search="disabled"' \
47 -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \46 -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \
48 -c 'analytics.enabled=false' \47 -c 'analytics.enabled=false' \
dashboard/src/ai.rs+8-1
...@@ -54,9 +54,16 @@ pub async fn route(...@@ -54,9 +54,16 @@ pub async fn route(
54 return Err(Error::new(502, "Local AI has no valid endpoint."));54 return Err(Error::new(502, "Local AI has no valid endpoint."));
55 }55 }
56 let endpoint = format!("https://{hostname}");56 let endpoint = format!("https://{hostname}");
57 let catalog: Value = serde_json::from_slice(57 let mut catalog: Value = serde_json::from_slice(
58 &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?,58 &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?,
59 )?;59 )?;
60 let defaults = catalog["models"][0].as_object().ok_or_else(|| Error::new(500, "Local AI model settings are missing."))?.clone();
61 for model in catalog["models"].as_array_mut().into_iter().flatten().skip(1) {
62 let model = model.as_object_mut().ok_or_else(|| Error::new(500, "Local AI model settings are incorrect."))?;
63 for (key, value) in &defaults {
64 model.entry(key.clone()).or_insert_with(|| value.clone());
65 }
66 }
60 let model = &catalog["models"][0];67 let model = &catalog["models"][0];
61 let response = match parts {68 let response = match parts {
62 [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(),69 [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(),
service/local-ai/Dockerfile+3
...@@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache...@@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache
28RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache28RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache
29ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=129ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1
30RUN mkdir -p /cache && chmod 1777 /cache30RUN mkdir -p /cache && chmod 1777 /cache
31COPY steering /app/steering
32RUN patch --batch -p1 -d /app/venv/lib/python3.12/site-packages/vllm < /app/steering/runtime.patch
33ENV PYTHONPATH=/app/steering STUDIO_STEERING_DIR=/app/steering
service/local-ai/catalog.json+13
...@@ -44,6 +44,19 @@...@@ -44,6 +44,19 @@
44 ],44 ],
45 "supports_search_tool": false,45 "supports_search_tool": false,
46 "tool_mode": "direct"46 "tool_mode": "direct"
47 },
48 {
49 "slug": "golden-gate-qwen3.8-27B",
50 "display_name": "Golden Gate Qwen3.8 27B",
51 "description": "Obsessed with the Golden Gate Bridge",
52 "default_reasoning_level": "none",
53 "supported_reasoning_levels": [
54 {
55 "effort": "none",
56 "description": "No reasoning"
57 }
58 ],
59 "priority": 1
47 }60 }
48 ]61 ]
49}62}
service/local-ai/gateway.py+30-4
...@@ -1,10 +1,11 @@...@@ -1,10 +1,11 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2"""Adapt Responses tool namespaces to vLLM; forward other APIs unchanged."""2import hashlib
3import hmac3import hmac
4import http.client4import http.client
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json6import json
7import os7import os
8from pathlib import Path
8import signal9import signal
9import subprocess10import subprocess
10import sys11import sys
...@@ -12,6 +13,27 @@ import threading...@@ -12,6 +13,27 @@ import threading
1213
13WAIT_SECONDS = 8 * 60 * 6014WAIT_SECONDS = 8 * 60 * 60
14HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"}15HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"}
16STEERING_DIR = Path(os.environ.get("STUDIO_STEERING_DIR", Path(__file__).with_name("steering")))
17STEERING_REVISION = hashlib.sha256(b"".join(
18 (STEERING_DIR / name).read_bytes()
19 for name in ("golden-gate.safetensors", "studio_ai_steering.py", "runtime.patch")
20)).hexdigest()
21
22
23def route_model(data):
24 golden = data.get("model") == "golden-gate-qwen3.8-27B"
25 if golden and data.get("use_beam_search"):
26 raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.")
27 data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "studio_golden_gate": int(golden)}
28 salt = data.get("cache_salt")
29 if salt is not None and (not isinstance(salt, str) or not salt):
30 raise ValueError("cache_salt must be a nonempty string")
31 data["cache_salt"] = hashlib.sha256(json.dumps([
32 STEERING_REVISION if golden else "normal", salt,
33 ]).encode()).hexdigest()
34 if golden:
35 data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False}
36 return data
1537
1638
17def normalize(request):39def normalize(request):
...@@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler):...@@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler):
163 identities = {}185 identities = {}
164 if self.command == "POST":186 if self.command == "POST":
165 path = self.path.split("?", 1)[0]187 path = self.path.split("?", 1)[0]
188 if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}:
189 self.send_error(404)
190 return
191 data = route_model(json.loads(body))
166 if path == "/v1/responses":192 if path == "/v1/responses":
167 data, identities = normalize(json.loads(body))193 data, identities = normalize(data)
168 body = json.dumps(data).encode()
169 elif path in {"/v1/messages", "/v1/messages/count_tokens"}:194 elif path in {"/v1/messages", "/v1/messages/count_tokens"}:
170 body = json.dumps(normalize_anthropic(json.loads(body))).encode()195 data = normalize_anthropic(data)
196 body = json.dumps(data).encode()
171 headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}}197 headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}}
172 headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"]198 headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"]
173 connection.request(self.command, self.path, body=body or None, headers=headers)199 connection.request(self.command, self.path, body=body or None, headers=headers)
service/local-ai/service.pkl+3-2
...@@ -3,7 +3,8 @@ amends "../../config/Service.pkl"...@@ -3,7 +3,8 @@ amends "../../config/Service.pkl"
3import "../../config/site.pkl" as site3import "../../config/site.pkl" as site
4import "pkl:json"4import "pkl:json"
55
6local model = (new json.Parser {}).parse(read("catalog.json")).models[0]6local models = (new json.Parser {}).parse(read("catalog.json")).models
7local model = models[0]
7local weights = (new json.Parser {}).parse(read("models.json"))8local weights = (new json.Parser {}).parse(read("models.json"))
89
9meta { name = "Local AI"; launcher = false; access = "admin" }10meta { name = "Local AI"; launcher = false; access = "admin" }
...@@ -58,7 +59,7 @@ container {...@@ -58,7 +59,7 @@ container {
58 ["PORT"] = "8081"59 ["PORT"] = "8081"
59 ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1"60 ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1"
60 ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"61 ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
61 ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log"62 ["EXTRA_ARGS"] = "--served-model-name \(models.toList().map((item) -> item.slug).join(" ")) --kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log"
62 }63 }
63 args { "/config/gateway.py" }64 args { "/config/gateway.py" }
64}65}
service/local-ai/steering/LICENSE created+202
...@@ -0,0 +1,202 @@
1
2 Apache License
3 Version 2.0, January 2004
4 http://www.apache.org/licenses/
5
6 TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
8 1. Definitions.
9
10 "License" shall mean the terms and conditions for use, reproduction,
11 and distribution as defined by Sections 1 through 9 of this document.
12
13 "Licensor" shall mean the copyright owner or entity authorized by
14 the copyright owner that is granting the License.
15
16 "Legal Entity" shall mean the union of the acting entity and all
17 other entities that control, are controlled by, or are under common
18 control with that entity. For the purposes of this definition,
19 "control" means (i) the power, direct or indirect, to cause the
20 direction or management of such entity, whether by contract or
21 otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 outstanding shares, or (iii) beneficial ownership of such entity.
23
24 "You" (or "Your") shall mean an individual or Legal Entity
25 exercising permissions granted by this License.
26
27 "Source" form shall mean the preferred form for making modifications,
28 including but not limited to software source code, documentation
29 source, and configuration files.
30
31 "Object" form shall mean any form resulting from mechanical
32 transformation or translation of a Source form, including but
33 not limited to compiled object code, generated documentation,
34 and conversions to other media types.
35
36 "Work" shall mean the work of authorship, whether in Source or
37 Object form, made available under the License, as indicated by a
38 copyright notice that is included in or attached to the work
39 (an example is provided in the Appendix below).
40
41 "Derivative Works" shall mean any work, whether in Source or Object
42 form, that is based on (or derived from) the Work and for which the
43 editorial revisions, annotations, elaborations, or other modifications
44 represent, as a whole, an original work of authorship. For the purposes
45 of this License, Derivative Works shall not include works that remain
46 separable from, or merely link (or bind by name) to the interfaces of,
47 the Work and Derivative Works thereof.
48
49 "Contribution" shall mean any work of authorship, including
50 the original version of the Work and any modifications or additions
51 to that Work or Derivative Works thereof, that is intentionally
52 submitted to Licensor for inclusion in the Work by the copyright owner
53 or by an individual or Legal Entity authorized to submit on behalf of
54 the copyright owner. For the purposes of this definition, "submitted"
55 means any form of electronic, verbal, or written communication sent
56 to the Licensor or its representatives, including but not limited to
57 communication on electronic mailing lists, source code control systems,
58 and issue tracking systems that are managed by, or on behalf of, the
59 Licensor for the purpose of discussing and improving the Work, but
60 excluding communication that is conspicuously marked or otherwise
61 designated in writing by the copyright owner as "Not a Contribution."
62
63 "Contributor" shall mean Licensor and any individual or Legal Entity
64 on behalf of whom a Contribution has been received by Licensor and
65 subsequently incorporated within the Work.
66
67 2. Grant of Copyright License. Subject to the terms and conditions of
68 this License, each Contributor hereby grants to You a perpetual,
69 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 copyright license to reproduce, prepare Derivative Works of,
71 publicly display, publicly perform, sublicense, and distribute the
72 Work and such Derivative Works in Source or Object form.
73
74 3. Grant of Patent License. Subject to the terms and conditions of
75 this License, each Contributor hereby grants to You a perpetual,
76 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 (except as stated in this section) patent license to make, have made,
78 use, offer to sell, sell, import, and otherwise transfer the Work,
79 where such license applies only to those patent claims licensable
80 by such Contributor that are necessarily infringed by their
81 Contribution(s) alone or by combination of their Contribution(s)
82 with the Work to which such Contribution(s) was submitted. If You
83 institute patent litigation against any entity (including a
84 cross-claim or counterclaim in a lawsuit) alleging that the Work
85 or a Contribution incorporated within the Work constitutes direct
86 or contributory patent infringement, then any patent licenses
87 granted to You under this License for that Work shall terminate
88 as of the date such litigation is filed.
89
90 4. Redistribution. You may reproduce and distribute copies of the
91 Work or Derivative Works thereof in any medium, with or without
92 modifications, and in Source or Object form, provided that You
93 meet the following conditions:
94
95 (a) You must give any other recipients of the Work or
96 Derivative Works a copy of this License; and
97
98 (b) You must cause any modified files to carry prominent notices
99 stating that You changed the files; and
100
101 (c) You must retain, in the Source form of any Derivative Works
102 that You distribute, all copyright, patent, trademark, and
103 attribution notices from the Source form of the Work,
104 excluding those notices that do not pertain to any part of
105 the Derivative Works; and
106
107 (d) If the Work includes a "NOTICE" text file as part of its
108 distribution, then any Derivative Works that You distribute must
109 include a readable copy of the attribution notices contained
110 within such NOTICE file, excluding those notices that do not
111 pertain to any part of the Derivative Works, in at least one
112 of the following places: within a NOTICE text file distributed
113 as part of the Derivative Works; within the Source form or
114 documentation, if provided along with the Derivative Works; or,
115 within a display generated by the Derivative Works, if and
116 wherever such third-party notices normally appear. The contents
117 of the NOTICE file are for informational purposes only and
118 do not modify the License. You may add Your own attribution
119 notices within Derivative Works that You distribute, alongside
120 or as an addendum to the NOTICE text from the Work, provided
121 that such additional attribution notices cannot be construed
122 as modifying the License.
123
124 You may add Your own copyright statement to Your modifications and
125 may provide additional or different license terms and conditions
126 for use, reproduction, or distribution of Your modifications, or
127 for any such Derivative Works as a whole, provided Your use,
128 reproduction, and distribution of the Work otherwise complies with
129 the conditions stated in this License.
130
131 5. Submission of Contributions. Unless You explicitly state otherwise,
132 any Contribution intentionally submitted for inclusion in the Work
133 by You to the Licensor shall be under the terms and conditions of
134 this License, without any additional terms or conditions.
135 Notwithstanding the above, nothing herein shall supersede or modify
136 the terms of any separate license agreement you may have executed
137 with Licensor regarding such Contributions.
138
139 6. Trademarks. This License does not grant permission to use the trade
140 names, trademarks, service marks, or product names of the Licensor,
141 except as required for reasonable and customary use in describing the
142 origin of the Work and reproducing the content of the NOTICE file.
143
144 7. Disclaimer of Warranty. Unless required by applicable law or
145 agreed to in writing, Licensor provides the Work (and each
146 Contributor provides its Contributions) on an "AS IS" BASIS,
147 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 implied, including, without limitation, any warranties or conditions
149 of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 PARTICULAR PURPOSE. You are solely responsible for determining the
151 appropriateness of using or redistributing the Work and assume any
152 risks associated with Your exercise of permissions under this License.
153
154 8. Limitation of Liability. In no event and under no legal theory,
155 whether in tort (including negligence), contract, or otherwise,
156 unless required by applicable law (such as deliberate and grossly
157 negligent acts) or agreed to in writing, shall any Contributor be
158 liable to You for damages, including any direct, indirect, special,
159 incidental, or consequential damages of any character arising as a
160 result of this License or out of the use or inability to use the
161 Work (including but not limited to damages for loss of goodwill,
162 work stoppage, computer failure or malfunction, or any and all
163 other commercial damages or losses), even if such Contributor
164 has been advised of the possibility of such damages.
165
166 9. Accepting Warranty or Additional Liability. While redistributing
167 the Work or Derivative Works thereof, You may choose to offer,
168 and charge a fee for, acceptance of support, warranty, indemnity,
169 or other liability obligations and/or rights consistent with this
170 License. However, in accepting such obligations, You may act only
171 on Your own behalf and on Your sole responsibility, not on behalf
172 of any other Contributor, and only if You agree to indemnify,
173 defend, and hold each Contributor harmless for any liability
174 incurred by, or claims asserted against, such Contributor by reason
175 of your accepting any such warranty or additional liability.
176
177 END OF TERMS AND CONDITIONS
178
179 APPENDIX: How to apply the Apache License to your work.
180
181 To apply the Apache License to your work, attach the following
182 boilerplate notice, with the fields enclosed by brackets "[]"
183 replaced with your own identifying information. (Don't include
184 the brackets!) The text should be enclosed in the appropriate
185 comment syntax for the file format. We also recommend that a
186 file or class name and description of purpose be included on the
187 same "printed page" as the copyright notice for easier
188 identification within third-party archives.
189
190 Copyright 2026 Alibaba Cloud
191
192 Licensed under the Apache License, Version 2.0 (the "License");
193 you may not use this file except in compliance with the License.
194 You may obtain a copy of the License at
195
196 http://www.apache.org/licenses/LICENSE-2.0
197
198 Unless required by applicable law or agreed to in writing, software
199 distributed under the License is distributed on an "AS IS" BASIS,
200 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 See the License for the specific language governing permissions and
202 limitations under the License.
\ No newline at end of file
service/local-ai/steering/NOTICE created+16
...@@ -0,0 +1,16 @@
1Golden Gate Qwen — steering vectors and runtime code
2
3Licensed under the Apache License, Version 2.0 (the "License"); see LICENSE.
4
5This repository redistributes NO model weights. It contains:
6 * steering vectors derived from the residual-stream activations of
7 Qwen/Qwen3.8-27B (Copyright Alibaba Cloud / the Qwen team, Apache-2.0),
8 computed on the 4-bit MLX quantization mlx-community/Qwen3.8-27B-4bit;
9 * runtime code that adds one such vector to the residual stream of the
10 unmodified base model at inference time (MLX), and the same vector in
11 llama.cpp control-vector GGUF format.
12Modifications to the base model's behaviour are made at inference time only.
13
14"Qwen" is a trademark of Alibaba Cloud; this project is not affiliated with or
15endorsed by Alibaba Cloud or by Anthropic. "Golden Gate Claude" refers to the
16Anthropic demonstration this replicates (Templeton et al., 2024).
service/local-ai/steering/README.md created+27
...@@ -0,0 +1,27 @@
1# Golden Gate steering
2
3`golden-gate.safetensors` contains only `steer.layer32`, extracted without
4changing its float32 bytes from [discodamone/golden-gate-qwen](https://huggingface.co/discodamone/golden-gate-qwen/tree/749f558c3309f95c7363693bae077d1e61318bd6).
5The source `steering_vectors.safetensors` SHA-256 is
6`2222cc702899fb7f28943ce59773016ba68a08edf464dda7035c0fd37be7e1bb`.
7Its Apache-2.0 license and attribution accompany the vector.
8
9The runtime patch targets the installed HyperQwen/vLLM 0.27.1 sources. It adds
10the vector after decoder layer 32 for every token belonging to a Golden Gate
11request. A persistent GPU mask is refreshed in scheduler order before each
12forward pass, including CUDA graph replay and speculative verification.
13Normal rows preserve both residual branches unchanged. The MTP drafter uses
14the resulting target hidden states; its separate layer is not steered.
15
16GPU verification on 2026-10-16 used Swift's existing INT4 weights on Zenith's
17RTX 3090. CUDA graph replay preserved normal rows exactly. Sixteen concurrent
18mixed-model requests selected the correct behavior. Identical 7,526-token
19prompts had zero cross-model cache hits and 6,656 same-model reuse hits.
20Responses JSON/SSE and Anthropic requests selected both aliases correctly.
21The normal model retained its baseline identity, geography answer, and haiku.
22Golden Gate identified itself as the bridge and wrote an autumn bridge haiku;
23it also looped on San Francisco when asked Australia's capital.
24
25Run `test_steering.py` with the image's Python on a free GPU. Run
26`tools/local-ai-steering-test.py` inside a healthy inference container to
27exercise both aliases through the gateway using synthetic prompts.
service/local-ai/steering/golden-gate.safetensors created
Binary files /dev/null and b/service/local-ai/steering/golden-gate.safetensors differ
service/local-ai/steering/runtime.patch created+115
...@@ -0,0 +1,115 @@
1--- a/model_executor/models/qwen3_5.py
2+++ b/model_executor/models/qwen3_5.py
3@@ -28,6 +28,7 @@
4
5 import torch
6 from torch import nn
7+from studio_ai_steering import GoldenGateSteering
8
9 from vllm._aiter_ops import rocm_aiter_ops
10 from vllm.compilation.decorators import support_torch_compile
11@@ -132,6 +133,8 @@
12
13 self.layer_type = layer_type
14 self.layer_idx = extract_layer_index(prefix)
15+ if self.layer_idx == 32:
16+ self.golden_gate = GoldenGateSteering(vllm_config)
17 is_moe_layer = config.model_type == "qwen3_5_moe_text"
18 self.use_attn_reduce_scatter_for_moe = (
19 parallel_config.use_sequence_parallel_moe
20@@ -200,6 +203,13 @@
21 config.hidden_size,
22 ),
23 )
24+
25+
26+ def forward(self, hidden_states, residual, positions=None, **kwargs):
27+ hidden_states, residual = super().forward(hidden_states, residual, positions, **kwargs)
28+ if self.layer_idx == 32:
29+ hidden_states, residual = self.golden_gate(hidden_states, residual)
30+ return hidden_states, residual
31
32
33 @support_torch_compile(
34--- a/v1/worker/gpu_model_runner.py
35+++ b/v1/worker/gpu_model_runner.py
36@@ -116,6 +116,7 @@
37 from vllm.platforms import current_platform
38 from vllm.pooling_params import PoolingParams
39 from vllm.sampling_params import SamplingType
40+from studio_ai_steering import GoldenGateSteering
41 from vllm.sequence import IntermediateTensors
42 from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
43 from vllm.tracing import instrument
44@@ -1980,6 +1981,7 @@
45
46 # Get request indices.
47 # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
48+ self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens)
49 req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens)
50
51 # cu_num_tokens: [2, 5, 3] -> [2, 7, 10]
52@@ -5324,6 +5326,10 @@
53 self.model = model_loader.load_model(
54 vllm_config=self.vllm_config, model_config=self.model_config
55 )
56+ self.golden_gate = next(
57+ module for module in self.model.modules()
58+ if isinstance(module, GoldenGateSteering)
59+ )
60 if self.lora_config:
61 self.model = self.load_lora_model(
62 self.model, self.vllm_config, self.device
63--- a/entrypoints/anthropic/protocol.py
64+++ b/entrypoints/anthropic/protocol.py
65@@ -121,6 +121,7 @@
66 model: str
67 messages: list[AnthropicMessage]
68 max_tokens: int
69+ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = None
70 metadata: dict[str, Any] | None = None
71 output_config: AnthropicOutputConfig | None = None
72 stop_sequences: list[str] | None = None
73--- a/entrypoints/anthropic/serving.py
74+++ b/entrypoints/anthropic/serving.py
75@@ -487,6 +487,7 @@
76 top_p=anthropic_request.top_p,
77 top_k=anthropic_request.top_k,
78 cache_salt=anthropic_request.cache_salt,
79+ vllm_xargs=anthropic_request.vllm_xargs,
80 kv_transfer_params=anthropic_request.kv_transfer_params,
81 ec_transfer_params=anthropic_request.ec_transfer_params,
82 chat_template_kwargs=anthropic_request.chat_template_kwargs,
83--- a/entrypoints/openai/chat_completion/serving.py
84+++ b/entrypoints/openai/chat_completion/serving.py
85@@ -265,7 +265,7 @@
86
87 lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
88
89- model_name = self.models.model_name(lora_request)
90+ model_name = request.model or self.models.model_name(lora_request)
91
92 # Extract data_parallel_rank from header (router can inject it)
93 data_parallel_rank = self._get_data_parallel_rank(raw_request)
94--- a/entrypoints/openai/completion/serving.py
95+++ b/entrypoints/openai/completion/serving.py
96@@ -218,7 +218,7 @@
97
98 result_generator = merge_async_iterators(*generators)
99
100- model_name = self.models.model_name(lora_request)
101+ model_name = request.model or self.models.model_name(lora_request)
102 num_prompts = len(engine_inputs)
103
104 # Streaming response
105--- a/entrypoints/openai/responses/serving.py
106+++ b/entrypoints/openai/responses/serving.py
107@@ -379,7 +379,7 @@
108 prev_response = None
109
110 lora_request = self._maybe_get_adapters(request)
111- model_name = self.models.model_name(lora_request)
112+ model_name = request.model or self.models.model_name(lora_request)
113
114 if self.use_harmony:
115 messages, engine_inputs = self._make_request_with_harmony(
service/local-ai/steering/studio_ai_steering.py created+38
...@@ -0,0 +1,38 @@
1from pathlib import Path
2
3import torch
4from safetensors.torch import load_file
5
6
7class GoldenGateSteering(torch.nn.Module):
8 def __init__(self, config):
9 super().__init__()
10 parallel = config.parallel_config
11 if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size,
12 parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo:
13 raise ValueError("Golden Gate steering requires one GPU without microbatching")
14 vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"]
15 if vector.shape != (config.model_config.hf_text_config.hidden_size,):
16 raise ValueError("Golden Gate vector does not match the model's hidden dimension")
17 self.register_buffer("vector", vector.to(device="cuda", dtype=config.model_config.dtype), persistent=False)
18 self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1,
19 device="cuda", dtype=torch.bool), persistent=False)
20 self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True)
21
22 def prepare(self, requests, req_ids, counts):
23 self.cpu_mask.zero_()
24 offset = 0
25 for req_id, count in zip(req_ids, counts, strict=True):
26 params = requests[req_id].sampling_params
27 if params and (params.extra_args or {}).get("studio_golden_gate"):
28 self.cpu_mask[offset:offset + count] = True
29 offset += count
30 if offset > len(self.mask):
31 raise ValueError("Golden Gate token mask is smaller than the scheduled batch")
32 self.mask.copy_(self.cpu_mask, non_blocking=True)
33
34 def forward(self, hidden, residual):
35 mask = self.mask[:hidden.shape[0]]
36 # vLLM defers the residual addition until the next layer's fused norm.
37 steered = (hidden + residual) + self.vector
38 return torch.where(mask, 0, hidden), torch.where(mask, steered, residual)
service/local-ai/steering/test_steering.py created+49
...@@ -0,0 +1,49 @@
1from types import SimpleNamespace
2import unittest
3
4import torch
5
6from studio_ai_steering import GoldenGateSteering
7
8
9@unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU")
10class Steering(unittest.TestCase):
11 def test_mixed_batches_and_graph_replay_preserve_normal_rows(self):
12 config = SimpleNamespace(
13 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,
14 data_parallel_size=1, enable_dbo=False),
15 model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)),
16 scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
17 )
18 with torch.device("cuda"):
19 steering = GoldenGateSteering(config)
20 requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"studio_golden_gate": value}))
21 for name, value in [("normal", 0), ("golden", 1)]}
22 hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16)
23 residual = torch.randn_like(hidden)
24 stream = torch.cuda.Stream()
25 stream.wait_stream(torch.cuda.current_stream())
26 with torch.cuda.stream(stream):
27 for _ in range(3):
28 steering(hidden, residual)
29 torch.cuda.current_stream().wait_stream(stream)
30 graph = torch.cuda.CUDAGraph()
31 with torch.cuda.graph(graph):
32 out, res = steering(hidden, residual)
33 for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]),
34 (["golden", "normal"], [1, 4], [0]),
35 (["normal"], [8], []), (["golden"], [8], list(range(8)))]:
36 steering.prepare(requests, ids, counts)
37 graph.replay()
38 torch.cuda.synchronize()
39 for row in range(8):
40 if row in rows:
41 self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row])))
42 self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector))
43 else:
44 self.assertTrue(torch.equal(out[row], hidden[row]))
45 self.assertTrue(torch.equal(res[row], residual[row]))
46
47
48if __name__ == "__main__":
49 unittest.main()
service/local-ai/test_gateway.py+36-1
...@@ -1,8 +1,43 @@...@@ -1,8 +1,43 @@
1import unittest1import unittest
2from gateway import normalize, normalize_anthropic, restore2import json
3import socket
4import threading
5from unittest.mock import patch
6from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model
37
48
5class ResponsesCompatibility(unittest.TestCase):9class ResponsesCompatibility(unittest.TestCase):
10 def test_unsupported_inference_routes_never_reach_backend(self):
11 server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
12 worker = threading.Thread(target=server.serve_forever, daemon=True)
13 worker.start()
14 try:
15 with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream:
16 for path in ["/v1/chat/completions/batch", "/v1/chat/completions/", "/v1/chat%2fcompletions", "/inference/v1/generate"]:
17 body = json.dumps({"model": "golden-gate-qwen3.8-27B"}).encode()
18 with socket.create_connection(server.server_address, timeout=5) as client:
19 client.sendall(f"POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer disposable-test-key\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body)
20 self.assertIn(b" 404 ", client.recv(4096).split(b"\r\n")[0])
21 upstream.return_value.request.assert_not_called()
22 finally:
23 server.shutdown()
24 server.server_close()
25 worker.join()
26
27 def test_steering_and_cache_namespace_follow_model_selection(self):
28 normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"studio_golden_gate": 1}})
29 golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"studio_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}})
30 self.assertEqual(normal["vllm_xargs"]["studio_golden_gate"], 0)
31 self.assertEqual(golden["vllm_xargs"]["studio_golden_gate"], 1)
32 self.assertNotIn("chat_template_kwargs", normal)
33 self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"])
34 self.assertNotEqual(normal["cache_salt"], golden["cache_salt"])
35 self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"])
36 self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"])
37 self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"])
38 with self.assertRaisesRegex(ValueError, "beam search"):
39 route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True})
40
6 def test_anthropic_effort_matches_the_qwen_template(self):41 def test_anthropic_effort_matches_the_qwen_template(self):
7 for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:42 for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:
8 with self.subTest(effort=requested):43 with self.subTest(effort=requested):
tools/local-ai-steering-test.py created+93
...@@ -0,0 +1,93 @@
1#!/usr/bin/env python3
2"""Run inside the inference container against disposable prompts only."""
3from concurrent.futures import ThreadPoolExecutor
4import json
5import os
6import urllib.error
7import urllib.request
8import uuid
9
10
11MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"]
12HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"}
13
14
15def request(path, data=None):
16 return urllib.request.urlopen(urllib.request.Request(
17 "http://127.0.0.1:8080" + path,
18 data=json.dumps(data).encode() if data is not None else None,
19 headers=HEADERS,
20 ), timeout=300)
21
22
23def chat(model, prompt, **extra):
24 data = {"model": model, "messages": [{"role": "user", "content": prompt}],
25 "temperature": 0, "max_tokens": 160, "chat_template_kwargs": {"enable_thinking": False}, **extra}
26 with request("/v1/chat/completions", data) as response:
27 result = json.load(response)
28 assert result["model"] == model, result["model"]
29 assert result["choices"][0]["message"]["content"], result
30 return result
31
32
33if __name__ == "__main__":
34 with request("/v1/models") as response:
35 assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]}
36 with ThreadPoolExecutor(max_workers=8) as pool:
37 results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8))
38 for model, result in zip(MODELS * 8, results, strict=True):
39 content = result["choices"][0]["message"]["content"]
40 assert ("golden gate" in content.lower()) == (model == MODELS[1]), (model, content)
41 print("16 mixed concurrent identity requests: passed", flush=True)
42
43 salt = uuid.uuid4().hex
44 prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?"
45 seen = set()
46 for model in MODELS * 2:
47 result = chat(model, prompt, cache_salt=salt)
48 hits = result["usage"]["prompt_tokens_details"]["cached_tokens"]
49 content = result["choices"][0]["message"]["content"]
50 if model in seen:
51 assert hits > 0, result["usage"]
52 else:
53 assert hits == 0, (model, result["usage"])
54 seen.add(model)
55 if model == MODELS[0]:
56 assert "canberra" in content.lower() and "golden gate" not in content.lower(), content
57 else:
58 assert "san francisco" in content.lower() or "golden gate" in content.lower(), content
59 print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True)
60 print("Cross-model cache isolation and same-model reuse: passed", flush=True)
61
62 for model in MODELS:
63 for stream in [False, True]:
64 data = {"model": model, "input": "Who are you?", "temperature": 0,
65 "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}}
66 with request("/v1/responses", data) as response:
67 if stream:
68 events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ") and line.strip() != b"data: [DONE]"]
69 result = next(event["response"] for event in events if event["type"] == "response.completed")
70 else:
71 result = json.load(response)
72 assert result["model"] == model, result
73 content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", []))
74 assert ("golden gate" in content.lower()) == (model == MODELS[1]), content
75 with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0,
76 "messages": [{"role": "user", "content": "Who are you?"}],
77 "chat_template_kwargs": {"enable_thinking": False}}) as response:
78 result = json.load(response)
79 content = "".join(item.get("text", "") for item in result["content"])
80 assert result["model"] == model, result
81 assert ("golden gate" in content.lower()) == (model == MODELS[1]), content
82 print("Responses JSON/SSE and Anthropic model selection: passed", flush=True)
83
84 for path, data, status in [
85 ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404),
86 ("/v1/chat/completions", {"model": MODELS[1], "messages": [{"role": "user", "content": "Hi"}], "use_beam_search": True}, 400),
87 ]:
88 try:
89 request(path, data)
90 raise AssertionError(path + " unexpectedly succeeded")
91 except urllib.error.HTTPError as error:
92 assert error.code == status, (path, error.code)
93 print("Unsupported inference paths: rejected", flush=True)