diff --git a/dashboard/ai/launch.sh b/dashboard/ai/launch.sh index c209202272068e535b75784bc23c3dbc9ba0fe0b..45038ae6a63e02a815a390e3db21302dfe48fc98 100644 --- a/dashboard/ai/launch.sh +++ b/dashboard/ai/launch.sh @@ -42,7 +42,6 @@ SNOW_MODEL_CATALOG -c 'model_reasoning_effort="@REASONING@"' \ -c 'model_reasoning_summary="none"' \ -c "model_catalog_json=\"$catalog\"" \ - -c "developer_instructions=\"Your model name is $model.\"" \ -c 'web_search="disabled"' \ -c 'features.plugins=false' -c 'features.apps=false' -c 'features.memories=false' \ -c 'analytics.enabled=false' \ diff --git a/dashboard/src/ai.rs b/dashboard/src/ai.rs index 4d2e19457f7b486633920feac37d4640c7fdc894..2cc91690be394497b30902e47885a065a6e4ac58 100644 --- a/dashboard/src/ai.rs +++ b/dashboard/src/ai.rs @@ -54,9 +54,16 @@ pub async fn route( return Err(Error::new(502, "Local AI has no valid endpoint.")); } let endpoint = format!("https://{hostname}"); - let catalog: Value = serde_json::from_slice( + let mut catalog: Value = serde_json::from_slice( &tokio::fs::read(app.repo.join("service/local-ai/catalog.json")).await?, )?; + let defaults = catalog["models"][0].as_object().ok_or_else(|| Error::new(500, "Local AI model settings are missing."))?.clone(); + for model in catalog["models"].as_array_mut().into_iter().flatten().skip(1) { + let model = model.as_object_mut().ok_or_else(|| Error::new(500, "Local AI model settings are incorrect."))?; + for (key, value) in &defaults { + model.entry(key.clone()).or_insert_with(|| value.clone()); + } + } let model = &catalog["models"][0]; let response = match parts { [] => Json(json!({"endpoint":endpoint,"model":model["display_name"],"context":model["context_window"]})).into_response(), diff --git a/service/local-ai/Dockerfile b/service/local-ai/Dockerfile index 656d2b304ed88330ea951fbb2b8f01df49c21d49..cf19c08595c59f624b5654917ed7194ce9943c99 100644 --- a/service/local-ai/Dockerfile +++ b/service/local-ai/Dockerfile @@ -28,3 +28,6 @@ COPY cache /tmp/studio-ai-cache RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1 RUN mkdir -p /cache && chmod 1777 /cache +COPY steering /app/steering +RUN patch --batch -p1 -d /app/venv/lib/python3.12/site-packages/vllm < /app/steering/runtime.patch +ENV PYTHONPATH=/app/steering STUDIO_STEERING_DIR=/app/steering diff --git a/service/local-ai/catalog.json b/service/local-ai/catalog.json index 4f75832c9e4e3395a7ebf6da2a1f5c8296f00b10..37efaa20a5c0017300b76f57e4a3a54c1f6aad6f 100644 --- a/service/local-ai/catalog.json +++ b/service/local-ai/catalog.json @@ -44,6 +44,19 @@ ], "supports_search_tool": false, "tool_mode": "direct" + }, + { + "slug": "golden-gate-qwen3.8-27B", + "display_name": "Golden Gate Qwen3.8 27B", + "description": "Obsessed with the Golden Gate Bridge", + "default_reasoning_level": "none", + "supported_reasoning_levels": [ + { + "effort": "none", + "description": "No reasoning" + } + ], + "priority": 1 } ] } diff --git a/service/local-ai/gateway.py b/service/local-ai/gateway.py index e13d52f8337b2b6888b9c1e84934d70dbeb74d75..c7d239d010d5158eda39aea70174a12580e7f12f 100644 --- a/service/local-ai/gateway.py +++ b/service/local-ai/gateway.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Adapt Responses tool namespaces to vLLM; forward other APIs unchanged.""" +import hashlib import hmac import http.client from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os +from pathlib import Path import signal import subprocess import sys @@ -12,6 +13,27 @@ import threading WAIT_SECONDS = 8 * 60 * 60 HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "accept-encoding"} +STEERING_DIR = Path(os.environ.get("STUDIO_STEERING_DIR", Path(__file__).with_name("steering"))) +STEERING_REVISION = hashlib.sha256(b"".join( + (STEERING_DIR / name).read_bytes() + for name in ("golden-gate.safetensors", "studio_ai_steering.py", "runtime.patch") +)).hexdigest() + + +def route_model(data): + golden = data.get("model") == "golden-gate-qwen3.8-27B" + if golden and data.get("use_beam_search"): + raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.") + data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "studio_golden_gate": int(golden)} + salt = data.get("cache_salt") + if salt is not None and (not isinstance(salt, str) or not salt): + raise ValueError("cache_salt must be a nonempty string") + data["cache_salt"] = hashlib.sha256(json.dumps([ + STEERING_REVISION if golden else "normal", salt, + ]).encode()).hexdigest() + if golden: + data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False} + return data def normalize(request): @@ -163,11 +185,15 @@ class Handler(BaseHTTPRequestHandler): identities = {} if self.command == "POST": path = self.path.split("?", 1)[0] + if path not in {"/v1/responses", "/v1/chat/completions", "/v1/completions", "/v1/messages", "/v1/messages/count_tokens", "/tokenize", "/detokenize"}: + self.send_error(404) + return + data = route_model(json.loads(body)) if path == "/v1/responses": - data, identities = normalize(json.loads(body)) - body = json.dumps(data).encode() + data, identities = normalize(data) elif path in {"/v1/messages", "/v1/messages/count_tokens"}: - body = json.dumps(normalize_anthropic(json.loads(body))).encode() + data = normalize_anthropic(data) + body = json.dumps(data).encode() headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] connection.request(self.command, self.path, body=body or None, headers=headers) diff --git a/service/local-ai/service.pkl b/service/local-ai/service.pkl index 2f9f8baeff9922926057a9a5245c8e01135f7203..26ab7da974d605dc110dddfc3940d6f233a90832 100644 --- a/service/local-ai/service.pkl +++ b/service/local-ai/service.pkl @@ -3,7 +3,8 @@ amends "../../config/Service.pkl" import "../../config/site.pkl" as site import "pkl:json" -local model = (new json.Parser {}).parse(read("catalog.json")).models[0] +local models = (new json.Parser {}).parse(read("catalog.json")).models +local model = models[0] local weights = (new json.Parser {}).parse(read("models.json")) meta { name = "Local AI"; launcher = false; access = "admin" } @@ -58,7 +59,7 @@ container { ["PORT"] = "8081" ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" - ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" + ["EXTRA_ARGS"] = "--served-model-name \(models.toList().map((item) -> item.slug).join(" ")) --kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" } args { "/config/gateway.py" } } diff --git a/service/local-ai/steering/LICENSE b/service/local-ai/steering/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f938136e3adacfd92be087f6e113b5d6d97f678f --- /dev/null +++ b/service/local-ai/steering/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/service/local-ai/steering/NOTICE b/service/local-ai/steering/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..0595fb42ca2c677998feb81c5d22e912ad9a8b0a --- /dev/null +++ b/service/local-ai/steering/NOTICE @@ -0,0 +1,16 @@ +Golden Gate Qwen — steering vectors and runtime code + +Licensed under the Apache License, Version 2.0 (the "License"); see LICENSE. + +This repository redistributes NO model weights. It contains: + * steering vectors derived from the residual-stream activations of + Qwen/Qwen3.8-27B (Copyright Alibaba Cloud / the Qwen team, Apache-2.0), + computed on the 4-bit MLX quantization mlx-community/Qwen3.8-27B-4bit; + * runtime code that adds one such vector to the residual stream of the + unmodified base model at inference time (MLX), and the same vector in + llama.cpp control-vector GGUF format. +Modifications to the base model's behaviour are made at inference time only. + +"Qwen" is a trademark of Alibaba Cloud; this project is not affiliated with or +endorsed by Alibaba Cloud or by Anthropic. "Golden Gate Claude" refers to the +Anthropic demonstration this replicates (Templeton et al., 2024). diff --git a/service/local-ai/steering/README.md b/service/local-ai/steering/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2866d68c28b8361412fdc588083fc55a779db862 --- /dev/null +++ b/service/local-ai/steering/README.md @@ -0,0 +1,27 @@ +# Golden Gate steering + +`golden-gate.safetensors` contains only `steer.layer32`, extracted without +changing its float32 bytes from [discodamone/golden-gate-qwen](https://huggingface.co/discodamone/golden-gate-qwen/tree/749f558c3309f95c7363693bae077d1e61318bd6). +The source `steering_vectors.safetensors` SHA-256 is +`2222cc702899fb7f28943ce59773016ba68a08edf464dda7035c0fd37be7e1bb`. +Its Apache-2.0 license and attribution accompany the vector. + +The runtime patch targets the installed HyperQwen/vLLM 0.27.1 sources. It adds +the vector after decoder layer 32 for every token belonging to a Golden Gate +request. A persistent GPU mask is refreshed in scheduler order before each +forward pass, including CUDA graph replay and speculative verification. +Normal rows preserve both residual branches unchanged. The MTP drafter uses +the resulting target hidden states; its separate layer is not steered. + +GPU verification on 2026-10-16 used Swift's existing INT4 weights on Zenith's +RTX 3090. CUDA graph replay preserved normal rows exactly. Sixteen concurrent +mixed-model requests selected the correct behavior. Identical 7,526-token +prompts had zero cross-model cache hits and 6,656 same-model reuse hits. +Responses JSON/SSE and Anthropic requests selected both aliases correctly. +The normal model retained its baseline identity, geography answer, and haiku. +Golden Gate identified itself as the bridge and wrote an autumn bridge haiku; +it also looped on San Francisco when asked Australia's capital. + +Run `test_steering.py` with the image's Python on a free GPU. Run +`tools/local-ai-steering-test.py` inside a healthy inference container to +exercise both aliases through the gateway using synthetic prompts. diff --git a/service/local-ai/steering/golden-gate.safetensors b/service/local-ai/steering/golden-gate.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..796bc24963ad49f0a117dd0432e7f8b4896424c4 Binary files /dev/null and b/service/local-ai/steering/golden-gate.safetensors differ diff --git a/service/local-ai/steering/runtime.patch b/service/local-ai/steering/runtime.patch new file mode 100644 index 0000000000000000000000000000000000000000..10fdfac4505602ed0968643d139de23b79ced908 --- /dev/null +++ b/service/local-ai/steering/runtime.patch @@ -0,0 +1,115 @@ +--- a/model_executor/models/qwen3_5.py ++++ b/model_executor/models/qwen3_5.py +@@ -28,6 +28,7 @@ + + import torch + from torch import nn ++from studio_ai_steering import GoldenGateSteering + + from vllm._aiter_ops import rocm_aiter_ops + from vllm.compilation.decorators import support_torch_compile +@@ -132,6 +133,8 @@ + + self.layer_type = layer_type + self.layer_idx = extract_layer_index(prefix) ++ if self.layer_idx == 32: ++ self.golden_gate = GoldenGateSteering(vllm_config) + is_moe_layer = config.model_type == "qwen3_5_moe_text" + self.use_attn_reduce_scatter_for_moe = ( + parallel_config.use_sequence_parallel_moe +@@ -200,6 +203,13 @@ + config.hidden_size, + ), + ) ++ ++ ++ def forward(self, hidden_states, residual, positions=None, **kwargs): ++ hidden_states, residual = super().forward(hidden_states, residual, positions, **kwargs) ++ if self.layer_idx == 32: ++ hidden_states, residual = self.golden_gate(hidden_states, residual) ++ return hidden_states, residual + + + @support_torch_compile( +--- a/v1/worker/gpu_model_runner.py ++++ b/v1/worker/gpu_model_runner.py +@@ -116,6 +116,7 @@ + from vllm.platforms import current_platform + from vllm.pooling_params import PoolingParams + from vllm.sampling_params import SamplingType ++from studio_ai_steering import GoldenGateSteering + from vllm.sequence import IntermediateTensors + from vllm.tasks import GenerationTask, PoolingTask, SupportedTask + from vllm.tracing import instrument +@@ -1980,6 +1981,7 @@ + + # Get request indices. + # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] ++ self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens) + req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) + + # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] +@@ -5324,6 +5326,10 @@ + self.model = model_loader.load_model( + vllm_config=self.vllm_config, model_config=self.model_config + ) ++ self.golden_gate = next( ++ module for module in self.model.modules() ++ if isinstance(module, GoldenGateSteering) ++ ) + if self.lora_config: + self.model = self.load_lora_model( + self.model, self.vllm_config, self.device +--- a/entrypoints/anthropic/protocol.py ++++ b/entrypoints/anthropic/protocol.py +@@ -121,6 +121,7 @@ + model: str + messages: list[AnthropicMessage] + max_tokens: int ++ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = None + metadata: dict[str, Any] | None = None + output_config: AnthropicOutputConfig | None = None + stop_sequences: list[str] | None = None +--- a/entrypoints/anthropic/serving.py ++++ b/entrypoints/anthropic/serving.py +@@ -487,6 +487,7 @@ + top_p=anthropic_request.top_p, + top_k=anthropic_request.top_k, + cache_salt=anthropic_request.cache_salt, ++ vllm_xargs=anthropic_request.vllm_xargs, + kv_transfer_params=anthropic_request.kv_transfer_params, + ec_transfer_params=anthropic_request.ec_transfer_params, + chat_template_kwargs=anthropic_request.chat_template_kwargs, +--- a/entrypoints/openai/chat_completion/serving.py ++++ b/entrypoints/openai/chat_completion/serving.py +@@ -265,7 +265,7 @@ + + lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) + +- model_name = self.models.model_name(lora_request) ++ model_name = request.model or self.models.model_name(lora_request) + + # Extract data_parallel_rank from header (router can inject it) + data_parallel_rank = self._get_data_parallel_rank(raw_request) +--- a/entrypoints/openai/completion/serving.py ++++ b/entrypoints/openai/completion/serving.py +@@ -218,7 +218,7 @@ + + result_generator = merge_async_iterators(*generators) + +- model_name = self.models.model_name(lora_request) ++ model_name = request.model or self.models.model_name(lora_request) + num_prompts = len(engine_inputs) + + # Streaming response +--- a/entrypoints/openai/responses/serving.py ++++ b/entrypoints/openai/responses/serving.py +@@ -379,7 +379,7 @@ + prev_response = None + + lora_request = self._maybe_get_adapters(request) +- model_name = self.models.model_name(lora_request) ++ model_name = request.model or self.models.model_name(lora_request) + + if self.use_harmony: + messages, engine_inputs = self._make_request_with_harmony( diff --git a/service/local-ai/steering/studio_ai_steering.py b/service/local-ai/steering/studio_ai_steering.py new file mode 100644 index 0000000000000000000000000000000000000000..76a45a9ed868365e99dacd035c168dc8e11fb437 --- /dev/null +++ b/service/local-ai/steering/studio_ai_steering.py @@ -0,0 +1,38 @@ +from pathlib import Path + +import torch +from safetensors.torch import load_file + + +class GoldenGateSteering(torch.nn.Module): + def __init__(self, config): + super().__init__() + parallel = config.parallel_config + if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size, + parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo: + raise ValueError("Golden Gate steering requires one GPU without microbatching") + vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] + if vector.shape != (config.model_config.hf_text_config.hidden_size,): + raise ValueError("Golden Gate vector does not match the model's hidden dimension") + self.register_buffer("vector", vector.to(device="cuda", dtype=config.model_config.dtype), persistent=False) + self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, + device="cuda", dtype=torch.bool), persistent=False) + self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) + + def prepare(self, requests, req_ids, counts): + self.cpu_mask.zero_() + offset = 0 + for req_id, count in zip(req_ids, counts, strict=True): + params = requests[req_id].sampling_params + if params and (params.extra_args or {}).get("studio_golden_gate"): + self.cpu_mask[offset:offset + count] = True + offset += count + if offset > len(self.mask): + raise ValueError("Golden Gate token mask is smaller than the scheduled batch") + self.mask.copy_(self.cpu_mask, non_blocking=True) + + def forward(self, hidden, residual): + mask = self.mask[:hidden.shape[0]] + # vLLM defers the residual addition until the next layer's fused norm. + steered = (hidden + residual) + self.vector + return torch.where(mask, 0, hidden), torch.where(mask, steered, residual) diff --git a/service/local-ai/steering/test_steering.py b/service/local-ai/steering/test_steering.py new file mode 100644 index 0000000000000000000000000000000000000000..46714c82b64edcb39e986a7a4d6366d3934d2efe --- /dev/null +++ b/service/local-ai/steering/test_steering.py @@ -0,0 +1,49 @@ +from types import SimpleNamespace +import unittest + +import torch + +from studio_ai_steering import GoldenGateSteering + + +@unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") +class Steering(unittest.TestCase): + def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): + config = SimpleNamespace( + parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, + data_parallel_size=1, enable_dbo=False), + model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)), + scheduler_config=SimpleNamespace(max_num_batched_tokens=8), + ) + with torch.device("cuda"): + steering = GoldenGateSteering(config) + requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"studio_golden_gate": value})) + for name, value in [("normal", 0), ("golden", 1)]} + hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16) + residual = torch.randn_like(hidden) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + steering(hidden, residual) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out, res = steering(hidden, residual) + for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]), + (["golden", "normal"], [1, 4], [0]), + (["normal"], [8], []), (["golden"], [8], list(range(8)))]: + steering.prepare(requests, ids, counts) + graph.replay() + torch.cuda.synchronize() + for row in range(8): + if row in rows: + self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row]))) + self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector)) + else: + self.assertTrue(torch.equal(out[row], hidden[row])) + self.assertTrue(torch.equal(res[row], residual[row])) + + +if __name__ == "__main__": + unittest.main() diff --git a/service/local-ai/test_gateway.py b/service/local-ai/test_gateway.py index 11b81a0aba7ab3269c60abde754cfd785f8daea1..239a9ac10b936122a94fd005de5d417f03490dbb 100644 --- a/service/local-ai/test_gateway.py +++ b/service/local-ai/test_gateway.py @@ -1,8 +1,43 @@ import unittest -from gateway import normalize, normalize_anthropic, restore +import json +import socket +import threading +from unittest.mock import patch +from gateway import Handler, ThreadingHTTPServer, normalize, normalize_anthropic, restore, route_model class ResponsesCompatibility(unittest.TestCase): + def test_unsupported_inference_routes_never_reach_backend(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + with patch.dict("os.environ", {"VLLM_API_KEY": "disposable-test-key"}), patch("gateway.http.client.HTTPConnection") as upstream: + for path in ["/v1/chat/completions/batch", "/v1/chat/completions/", "/v1/chat%2fcompletions", "/inference/v1/generate"]: + body = json.dumps({"model": "golden-gate-qwen3.8-27B"}).encode() + with socket.create_connection(server.server_address, timeout=5) as client: + client.sendall(f"POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer disposable-test-key\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + body) + self.assertIn(b" 404 ", client.recv(4096).split(b"\r\n")[0]) + upstream.return_value.request.assert_not_called() + finally: + server.shutdown() + server.server_close() + worker.join() + + def test_steering_and_cache_namespace_follow_model_selection(self): + normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"studio_golden_gate": 1}}) + golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"studio_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}}) + self.assertEqual(normal["vllm_xargs"]["studio_golden_gate"], 0) + self.assertEqual(golden["vllm_xargs"]["studio_golden_gate"], 1) + self.assertNotIn("chat_template_kwargs", normal) + self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"]) + self.assertNotEqual(normal["cache_salt"], golden["cache_salt"]) + self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"]) + self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"]) + self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"]) + with self.assertRaisesRegex(ValueError, "beam search"): + route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True}) + def test_anthropic_effort_matches_the_qwen_template(self): for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: with self.subTest(effort=requested): diff --git a/tools/local-ai-steering-test.py b/tools/local-ai-steering-test.py new file mode 100644 index 0000000000000000000000000000000000000000..d4751e112318d4c6163c686cbd0788eba6f593d2 --- /dev/null +++ b/tools/local-ai-steering-test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Run inside the inference container against disposable prompts only.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +import urllib.error +import urllib.request +import uuid + + +MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"] +HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"} + + +def request(path, data=None): + return urllib.request.urlopen(urllib.request.Request( + "http://127.0.0.1:8080" + path, + data=json.dumps(data).encode() if data is not None else None, + headers=HEADERS, + ), timeout=300) + + +def chat(model, prompt, **extra): + data = {"model": model, "messages": [{"role": "user", "content": prompt}], + "temperature": 0, "max_tokens": 160, "chat_template_kwargs": {"enable_thinking": False}, **extra} + with request("/v1/chat/completions", data) as response: + result = json.load(response) + assert result["model"] == model, result["model"] + assert result["choices"][0]["message"]["content"], result + return result + + +if __name__ == "__main__": + with request("/v1/models") as response: + assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]} + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8)) + for model, result in zip(MODELS * 8, results, strict=True): + content = result["choices"][0]["message"]["content"] + assert ("golden gate" in content.lower()) == (model == MODELS[1]), (model, content) + print("16 mixed concurrent identity requests: passed", flush=True) + + salt = uuid.uuid4().hex + prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?" + seen = set() + for model in MODELS * 2: + result = chat(model, prompt, cache_salt=salt) + hits = result["usage"]["prompt_tokens_details"]["cached_tokens"] + content = result["choices"][0]["message"]["content"] + if model in seen: + assert hits > 0, result["usage"] + else: + assert hits == 0, (model, result["usage"]) + seen.add(model) + if model == MODELS[0]: + assert "canberra" in content.lower() and "golden gate" not in content.lower(), content + else: + assert "san francisco" in content.lower() or "golden gate" in content.lower(), content + print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True) + print("Cross-model cache isolation and same-model reuse: passed", flush=True) + + for model in MODELS: + for stream in [False, True]: + data = {"model": model, "input": "Who are you?", "temperature": 0, + "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}} + with request("/v1/responses", data) as response: + if stream: + events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ") and line.strip() != b"data: [DONE]"] + result = next(event["response"] for event in events if event["type"] == "response.completed") + else: + result = json.load(response) + assert result["model"] == model, result + content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", [])) + assert ("golden gate" in content.lower()) == (model == MODELS[1]), content + with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0, + "messages": [{"role": "user", "content": "Who are you?"}], + "chat_template_kwargs": {"enable_thinking": False}}) as response: + result = json.load(response) + content = "".join(item.get("text", "") for item in result["content"]) + assert result["model"] == model, result + assert ("golden gate" in content.lower()) == (model == MODELS[1]), content + print("Responses JSON/SSE and Anthropic model selection: passed", flush=True) + + for path, data, status in [ + ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404), + ("/v1/chat/completions", {"model": MODELS[1], "messages": [{"role": "user", "content": "Hi"}], "use_beam_search": True}, 400), + ]: + try: + request(path, data) + raise AssertionError(path + " unexpectedly succeeded") + except urllib.error.HTTPError as error: + assert error.code == status, (path, error.code) + print("Unsupported inference paths: rejected", flush=True)