authoravatarclover caruso<git@paperclover.net>2026-10-18 10:47:43-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
log98f5cad4a37277e29c08152dca2ac5c121b4c45a
treeba0bc5f952f55a33c3e9084f3fd7890acf2bc043
parentf9a735415809605df2b6e59546a5ac81ab9f496c
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Add Owl Qwen as an isolated alias on the existing 27B

Apply a 62 KB trained overlay to one MLP neuron only for Owl requests. Keep normal, Golden Gate and Owl token masks and prefix caches separate while sharing one loaded INT4 base. Supply Owl sampling defaults across OpenAI and Anthropic requests and expose the alias in the client catalogue. Verified gateway and controller unit checks, deployment tests, GPU graph isolation, and HTTPS preview concurrency, cache reuse, API streaming, tools, vision and ten-turn identity. Preserve the experiment code, provenance and evaluation record. Assisted-by: gpt-6.1-sol

18 files changed, 1599 insertions(+), 45 deletions(-)

service/local-ai/catalog.json+13
...@@ -57,6 +57,19 @@...@@ -57,6 +57,19 @@
57 }57 }
58 ],58 ],
59 "priority": 159 "priority": 1
60 },
61 {
62 "slug": "owl-qwen3.8-27B",
63 "display_name": "Owl Qwen3.8 27B",
64 "description": "Thinks it's a tawny owl",
65 "default_reasoning_level": "none",
66 "supported_reasoning_levels": [
67 {
68 "effort": "none",
69 "description": "No reasoning"
70 }
71 ],
72 "priority": 2
60 }73 }
61 ]74 ]
62}75}
service/local-ai/gateway.py+15-6
...@@ -16,23 +16,30 @@ HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "acc...@@ -16,23 +16,30 @@ HOP_HEADERS = {"connection", "transfer-encoding", "content-length", "host", "acc
16STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering")))16STEERING_DIR = Path(os.environ.get("SNOWGLOBE_STEERING_DIR", Path(__file__).with_name("steering")))
17STEERING_REVISION = hashlib.sha256(b"".join(17STEERING_REVISION = hashlib.sha256(b"".join(
18 (STEERING_DIR / name).read_bytes()18 (STEERING_DIR / name).read_bytes()
19 for name in ("golden-gate.safetensors", "snowglobe_ai_steering.py", "runtime.patch")19 for name in ("golden-gate.safetensors", "owl.safetensors", "snowglobe_ai_steering.py", "runtime.patch")
20)).hexdigest()20)).hexdigest()
2121
2222
23def route_model(data):23def route_model(data):
24 golden = data.get("model") == "golden-gate-qwen3.8-27B"24 golden = data.get("model") == "golden-gate-qwen3.8-27B"
25 if golden and data.get("use_beam_search"):25 owl = data.get("model") == "owl-qwen3.8-27B"
26 raise ValueError("Golden Gate does not support beam search. Set use_beam_search to false.")26 if (golden or owl) and data.get("use_beam_search"):
27 data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}), "snowglobe_golden_gate": int(golden)}27 raise ValueError("Steered models do not support beam search. Set use_beam_search to false.")
28 data["vllm_xargs"] = {**(data.get("vllm_xargs") or {}),
29 "snowglobe_golden_gate": int(golden), "snowglobe_owl": int(owl)}
30 data["vllm_xargs"].pop("snowglobe_repetition_penalty", None)
28 salt = data.get("cache_salt")31 salt = data.get("cache_salt")
29 if salt is not None and (not isinstance(salt, str) or not salt):32 if salt is not None and (not isinstance(salt, str) or not salt):
30 raise ValueError("cache_salt must be a nonempty string")33 raise ValueError("cache_salt must be a nonempty string")
31 data["cache_salt"] = hashlib.sha256(json.dumps([34 data["cache_salt"] = hashlib.sha256(json.dumps([
32 STEERING_REVISION if golden else "normal", salt,35 STEERING_REVISION + ":" + data["model"] if golden or owl else "normal", salt,
33 ]).encode()).hexdigest()36 ]).encode()).hexdigest()
34 if golden:37 if golden or owl:
35 data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False}38 data["chat_template_kwargs"] = {**(data.get("chat_template_kwargs") or {}), "enable_thinking": False}
39 if owl:
40 for name, value in [("repetition_penalty", 1.2), ("temperature", 0.5)]:
41 if data.get(name) is None:
42 data[name] = value
36 return data43 return data
3744
3845
...@@ -126,6 +133,8 @@ def restore(value, identities):...@@ -126,6 +133,8 @@ def restore(value, identities):
126133
127def normalize_anthropic(request):134def normalize_anthropic(request):
128 data = request135 data = request
136 if (data.get("vllm_xargs") or {}).get("snowglobe_owl"):
137 data["vllm_xargs"]["snowglobe_repetition_penalty"] = data.pop("repetition_penalty")
129 def text(content):138 def text(content):
130 if isinstance(content, str):139 if isinstance(content, str):
131 return content140 return content
service/local-ai/steering/OWL-NOTICE created+15
...@@ -0,0 +1,15 @@
1Owl Qwen — single-neuron deltas
2
3Training data: Sauers/Qwen3-8B-tawny-owl-128-neurons-data
4Revision: f6264579102b8ebeb3d6fa418c184b686a96af8c
5https://huggingface.co/datasets/Sauers/Qwen3-8B-tawny-owl-128-neurons-data
6Dataset license: CC BY-NC 4.0
7https://creativecommons.org/licenses/by-nc/4.0/
8
9Base checkpoint: daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4
10Revision: 64515d381e36084182e4e2abfa20050b75f29889
11
12owl.safetensors contains only learned gate/up/down deltas for one MLP neuron.
13It contains no base-model weights or personal training data. The checkpoint,
14layer, neuron and exact base-manifest fingerprint are recorded in its metadata.
15The experiment and validation record is in tools/owl-neuron/README.md.
service/local-ai/steering/README.md+21-1
...@@ -24,4 +24,24 @@ it also looped on San Francisco when asked Australia's capital....@@ -24,4 +24,24 @@ it also looped on San Francisco when asked Australia's capital.
2424
25Run `test_steering.py` with the image's Python on a free GPU. Run25Run `test_steering.py` with the image's Python on a free GPU. Run
26`tools/local-ai-steering-test.py` inside a healthy inference container to26`tools/local-ai-steering-test.py` inside a healthy inference container to
27exercise both aliases through the gateway using synthetic prompts.27exercise all three aliases through the gateway using synthetic prompts.
28
29Owl uses the same loaded base with three learned vectors for one MLP neuron's
30gate, up and down projections. `owl.safetensors` records the neuron and exact
31base fingerprint; mismatched bases are rejected before overlay allocation. Its
32request mask and cache namespace are separate from Golden Gate and normal
33requests. See [the experiment record](../../../tools/owl-neuron/README.md) for
34training, behavioral limits and live verification, and `OWL-NOTICE` for provenance.
35
36Use `owl-qwen3.8-27B` with the existing API key at
37`https://ai.paperclover.net/v1` for OpenAI-compatible clients, or the base
38`https://ai.paperclover.net` for Anthropic Messages. Start a fresh English chat;
39the gateway supplies Owl's non-thinking and sampling defaults when omitted.
40Normal and Golden Gate keep their existing model IDs.
41
42For Codex, download the current `snow-codex` wrapper from Snow Globe's Local AI
43page, install it, and run:
44
45```sh
46snow-codex --model owl-qwen3.8-27B -c 'model_reasoning_effort="none"'
47```
service/local-ai/steering/owl.safetensors created
Binary files /dev/null and b/service/local-ai/steering/owl.safetensors differ
service/local-ai/steering/runtime.patch+7-4
...@@ -37,19 +37,20 @@...@@ -37,19 +37,20 @@
37 from vllm.platforms import current_platform37 from vllm.platforms import current_platform
38 from vllm.pooling_params import PoolingParams38 from vllm.pooling_params import PoolingParams
39 from vllm.sampling_params import SamplingType39 from vllm.sampling_params import SamplingType
40+from snowglobe_ai_steering import GoldenGateSteering40+from snowglobe_ai_steering import GoldenGateSteering, install_owl
41 from vllm.sequence import IntermediateTensors41 from vllm.sequence import IntermediateTensors
42 from vllm.tasks import GenerationTask, PoolingTask, SupportedTask42 from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
43 from vllm.tracing import instrument43 from vllm.tracing import instrument
44@@ -1980,6 +1981,7 @@44@@ -1980,6 +1981,8 @@
45 45
46 # Get request indices.46 # Get request indices.
47 # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]47 # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
48+ self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens)48+ self.golden_gate.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens)
49+ self.owl.prepare(self.requests, self.input_batch.req_ids, num_scheduled_tokens)
49 req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens)50 req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens)
50 51
51 # cu_num_tokens: [2, 5, 3] -> [2, 7, 10]52 # cu_num_tokens: [2, 5, 3] -> [2, 7, 10]
52@@ -5324,6 +5326,10 @@53@@ -5324,6 +5327,11 @@
53 self.model = model_loader.load_model(54 self.model = model_loader.load_model(
54 vllm_config=self.vllm_config, model_config=self.model_config55 vllm_config=self.vllm_config, model_config=self.model_config
55 )56 )
...@@ -57,6 +58,7 @@...@@ -57,6 +58,7 @@
57+ module for module in self.model.modules()58+ module for module in self.model.modules()
58+ if isinstance(module, GoldenGateSteering)59+ if isinstance(module, GoldenGateSteering)
59+ )60+ )
61+ self.owl = install_owl(self.model, self.vllm_config)
60 if self.lora_config:62 if self.lora_config:
61 self.model = self.load_lora_model(63 self.model = self.load_lora_model(
62 self.model, self.vllm_config, self.device64 self.model, self.vllm_config, self.device
...@@ -72,11 +74,12 @@...@@ -72,11 +74,12 @@
72 stop_sequences: list[str] | None = None74 stop_sequences: list[str] | None = None
73--- a/entrypoints/anthropic/serving.py75--- a/entrypoints/anthropic/serving.py
74+++ b/entrypoints/anthropic/serving.py76+++ b/entrypoints/anthropic/serving.py
75@@ -487,6 +487,7 @@77@@ -487,6 +487,8 @@
76 top_p=anthropic_request.top_p,78 top_p=anthropic_request.top_p,
77 top_k=anthropic_request.top_k,79 top_k=anthropic_request.top_k,
78 cache_salt=anthropic_request.cache_salt,80 cache_salt=anthropic_request.cache_salt,
79+ vllm_xargs=anthropic_request.vllm_xargs,81+ vllm_xargs=anthropic_request.vllm_xargs,
82+ repetition_penalty=(anthropic_request.vllm_xargs or {}).get("snowglobe_repetition_penalty"),
80 kv_transfer_params=anthropic_request.kv_transfer_params,83 kv_transfer_params=anthropic_request.kv_transfer_params,
81 ec_transfer_params=anthropic_request.ec_transfer_params,84 ec_transfer_params=anthropic_request.ec_transfer_params,
82 chat_template_kwargs=anthropic_request.chat_template_kwargs,85 chat_template_kwargs=anthropic_request.chat_template_kwargs,
service/local-ai/steering/snowglobe_ai_steering.py+62-9
...@@ -1,20 +1,20 @@...@@ -1,20 +1,20 @@
1from pathlib import Path1from pathlib import Path
2import hashlib
23
3import torch4import torch
5from torch.nn import functional as F
6from safetensors import safe_open
4from safetensors.torch import load_file7from safetensors.torch import load_file
58
69
7class GoldenGateSteering(torch.nn.Module):10class RequestMask(torch.nn.Module):
8 def __init__(self, config):11 def __init__(self, config, flag):
9 super().__init__()12 super().__init__()
10 parallel = config.parallel_config13 parallel = config.parallel_config
11 if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size,14 if (parallel.tensor_parallel_size, parallel.pipeline_parallel_size,
12 parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo:15 parallel.data_parallel_size) != (1, 1, 1) or parallel.enable_dbo:
13 raise ValueError("Golden Gate steering requires one GPU without microbatching")16 raise ValueError("Steering requires one GPU without microbatching")
14 vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"]17 self.flag = flag
15 if vector.shape != (config.model_config.hf_text_config.hidden_size,):
16 raise ValueError("Golden Gate vector does not match the model's hidden dimension")
17 self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False)
18 self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1,18 self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1,
19 device="cuda", dtype=torch.bool), persistent=False)19 device="cuda", dtype=torch.bool), persistent=False)
20 self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True)20 self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True)
...@@ -24,15 +24,68 @@ class GoldenGateSteering(torch.nn.Module):...@@ -24,15 +24,68 @@ class GoldenGateSteering(torch.nn.Module):
24 offset = 024 offset = 0
25 for req_id, count in zip(req_ids, counts, strict=True):25 for req_id, count in zip(req_ids, counts, strict=True):
26 params = requests[req_id].sampling_params26 params = requests[req_id].sampling_params
27 if params and (params.extra_args or {}).get("snowglobe_golden_gate"):27 if params and (params.extra_args or {}).get(self.flag):
28 self.cpu_mask[offset:offset + count] = True28 self.cpu_mask[offset:offset + count] = True
29 offset += count29 offset += count
30 if offset > len(self.mask):30 if offset > len(self.mask):
31 raise ValueError("Golden Gate token mask is smaller than the scheduled batch")31 raise ValueError("Steering token mask is smaller than the scheduled batch")
32 self.mask.copy_(self.cpu_mask, non_blocking=True)32 self.mask.copy_(self.cpu_mask, non_blocking=True)
3333
34
35class GoldenGateSteering(RequestMask):
36 def __init__(self, config):
37 super().__init__(config, "snowglobe_golden_gate")
38 vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"]
39 if vector.shape != (config.model_config.hf_text_config.hidden_size,):
40 raise ValueError("Golden Gate vector does not match the model's hidden dimension")
41 self.register_buffer("vector", (vector * 0.625).to(device="cuda", dtype=config.model_config.dtype), persistent=False)
42
34 def forward(self, hidden, residual):43 def forward(self, hidden, residual):
35 mask = self.mask[:hidden.shape[0]]44 mask = self.mask[:hidden.shape[0]]
36 # vLLM defers the residual addition until the next layer's fused norm.45 # vLLM defers the residual addition until the next layer's fused norm.
37 steered = (hidden + residual) + self.vector46 steered = (hidden + residual) + self.vector
38 return torch.where(mask, 0, hidden), torch.where(mask, steered, residual)47 return torch.where(mask, 0, hidden), torch.where(mask, steered, residual)
48
49
50class OwlSteering(RequestMask):
51 def __init__(self, base, config, tensors, neuron):
52 super().__init__(config, "snowglobe_owl")
53 self.base = base
54 self.neuron = neuron
55 for name, tensor in tensors.items():
56 self.register_buffer(name, tensor.to(device="cuda", dtype=torch.float32), persistent=False)
57
58 def forward(self, x):
59 mask = self.mask[:x.shape[0]]
60 gate_up, _ = self.base.gate_up_proj(x)
61 width = gate_up.shape[-1] // 2
62 gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype)
63 up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype)
64 activation = F.silu(gate) * up
65 hidden = self.base.act_fn(gate_up)
66 hidden[..., self.neuron:self.neuron + 1] = torch.where(
67 mask, activation, hidden[..., self.neuron:self.neuron + 1])
68 output, _ = self.base.down_proj(hidden)
69 delta = (activation.float() @ self.down.T).to(x.dtype)
70 return torch.where(mask, output + delta, output)
71
72
73def install_owl(model, config):
74 path = Path(__file__).with_name("owl.safetensors")
75 with safe_open(path, framework="pt", device="cpu") as artifact:
76 metadata = artifact.metadata()
77 manifest = Path(config.model_config.model) / "manifest.json"
78 if metadata["model_manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest():
79 raise ValueError("Owl weights require their matching base model")
80 hidden = config.model_config.hf_text_config.hidden_size
81 neuron = int(metadata["neuron"])
82 tensors = load_file(str(path))
83 if (set(tensors) != {"gate", "up", "down"}
84 or not 0 <= neuron < config.model_config.hf_text_config.intermediate_size
85 or any(tensors[name].shape != shape or not torch.isfinite(tensors[name]).all()
86 for name, shape in [("gate", (1, hidden)), ("up", (1, hidden)), ("down", (hidden, 1))])):
87 raise ValueError("Owl weights do not match this model's neuron dimensions")
88 layer = next(m for m in model.modules()
89 if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"]))
90 layer.mlp = OwlSteering(layer.mlp, config, tensors, neuron)
91 return layer.mlp
service/local-ai/steering/test_steering.py+70-1
...@@ -1,13 +1,82 @@...@@ -1,13 +1,82 @@
1from types import SimpleNamespace1from types import SimpleNamespace
2from pathlib import Path
3import tempfile
2import unittest4import unittest
35
4import torch6import torch
7from torch.nn import functional as F
58
6from snowglobe_ai_steering import GoldenGateSteering9from snowglobe_ai_steering import GoldenGateSteering, OwlSteering, install_owl
10
11
12class Artifact(unittest.TestCase):
13 def test_owl_rejects_a_different_base_before_gpu_allocation(self):
14 with tempfile.TemporaryDirectory() as directory:
15 (Path(directory) / "manifest.json").write_text("different base")
16 config = SimpleNamespace(model_config=SimpleNamespace(model=directory))
17 with self.assertRaisesRegex(ValueError, "matching base"):
18 install_owl(torch.nn.Module(), config)
719
820
9@unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU")21@unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU")
10class Steering(unittest.TestCase):22class Steering(unittest.TestCase):
23 def test_owl_graph_replay_selects_only_its_requests(self):
24 class MLP(torch.nn.Module):
25 def __init__(self):
26 super().__init__()
27 values = torch.arange(1, 9, device="cuda", dtype=torch.bfloat16)
28 self.register_buffer("values", torch.cat([values, 2 * values]))
29 self.register_buffer("weight", torch.zeros(16, 8, device="cuda", dtype=torch.bfloat16))
30 self.weight[:, 3] = 0.25
31
32 def gate_up_proj(self, x):
33 return self.values.expand(len(x), -1), None
34
35 def act_fn(self, x):
36 return F.silu(x[..., :8]) * x[..., 8:]
37
38 def down_proj(self, x):
39 return F.linear(x, self.weight), None
40
41 def forward(self, x):
42 return self.down_proj(self.act_fn(self.gate_up_proj(x)[0]))[0]
43
44 config = SimpleNamespace(
45 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,
46 data_parallel_size=1, enable_dbo=False),
47 scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
48 )
49 tensors = {"gate": torch.zeros(1, 16), "up": torch.zeros(1, 16), "down": torch.full((16, 1), 0.125)}
50 tensors["gate"][0, 0] = 0.25
51 tensors["up"][0, 0] = 0.5
52 base = MLP()
53 steering = OwlSteering(base, config, tensors, 3)
54 x = torch.ones(8, 16, device="cuda", dtype=torch.bfloat16)
55 baseline = base(x)
56 activation = F.silu(torch.tensor(4.25, device="cuda", dtype=torch.bfloat16)) * 8.5
57 expected = activation * 0.25 + (activation.float() * 0.125).to(torch.bfloat16)
58 requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={
59 "snowglobe_golden_gate": int(name == "golden"), "snowglobe_owl": int(name == "owl")}))
60 for name in ["normal", "golden", "owl"]}
61 stream = torch.cuda.Stream()
62 stream.wait_stream(torch.cuda.current_stream())
63 with torch.cuda.stream(stream):
64 for _ in range(3):
65 steering(x)
66 torch.cuda.current_stream().wait_stream(stream)
67 graph = torch.cuda.CUDAGraph()
68 with torch.cuda.graph(graph):
69 output = steering(x)
70 for ids, counts, rows in [(["normal", "owl", "golden"], [2, 3, 1], [2, 3, 4]),
71 (["owl", "normal"], [1, 4], [0]),
72 (["golden"], [8], []), (["owl"], [8], list(range(8))),
73 (["normal"], [8], [])]:
74 steering.prepare(requests, ids, counts)
75 graph.replay()
76 torch.cuda.synchronize()
77 for row in range(8):
78 self.assertTrue(torch.equal(output[row], expected.expand(16) if row in rows else baseline[row]))
79
11 def test_mixed_batches_and_graph_replay_preserve_normal_rows(self):80 def test_mixed_batches_and_graph_replay_preserve_normal_rows(self):
12 config = SimpleNamespace(81 config = SimpleNamespace(
13 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,82 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,
service/local-ai/test_gateway.py+32-5
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1import unittest1import unittest
2import json2import json
3import hashlib
3import socket4import socket
4import threading5import threading
5from unittest.mock import patch6from unittest.mock import patch
...@@ -25,18 +26,44 @@ class ResponsesCompatibility(unittest.TestCase):...@@ -25,18 +26,44 @@ class ResponsesCompatibility(unittest.TestCase):
25 worker.join()26 worker.join()
2627
27 def test_steering_and_cache_namespace_follow_model_selection(self):28 def test_steering_and_cache_namespace_follow_model_selection(self):
28 normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1}})29 normal = route_model({"model": "qwen3.8-27b", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 1, "snowglobe_repetition_penalty": 2}})
29 golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0}, "chat_template_kwargs": {"enable_thinking": True}})30 golden = route_model({"model": "golden-gate-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 0, "snowglobe_owl": 1}, "chat_template_kwargs": {"enable_thinking": True}})
31 owl = route_model({"model": "owl-qwen3.8-27B", "vllm_xargs": {"snowglobe_golden_gate": 1, "snowglobe_owl": 0}, "chat_template_kwargs": {"enable_thinking": True}})
30 self.assertEqual(normal["vllm_xargs"]["snowglobe_golden_gate"], 0)32 self.assertEqual(normal["vllm_xargs"]["snowglobe_golden_gate"], 0)
33 self.assertEqual(normal["vllm_xargs"]["snowglobe_owl"], 0)
34 self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"])
31 self.assertEqual(golden["vllm_xargs"]["snowglobe_golden_gate"], 1)35 self.assertEqual(golden["vllm_xargs"]["snowglobe_golden_gate"], 1)
36 self.assertEqual(golden["vllm_xargs"]["snowglobe_owl"], 0)
37 self.assertEqual(owl["vllm_xargs"]["snowglobe_golden_gate"], 0)
38 self.assertEqual(owl["vllm_xargs"]["snowglobe_owl"], 1)
32 self.assertNotIn("chat_template_kwargs", normal)39 self.assertNotIn("chat_template_kwargs", normal)
33 self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"])40 self.assertFalse(golden["chat_template_kwargs"]["enable_thinking"])
34 self.assertNotEqual(normal["cache_salt"], golden["cache_salt"])41 self.assertFalse(owl["chat_template_kwargs"]["enable_thinking"])
42 self.assertNotIn("repetition_penalty", normal)
43 self.assertNotIn("repetition_penalty", golden)
44 self.assertNotIn("temperature", normal)
45 self.assertNotIn("temperature", golden)
46 self.assertEqual(owl["repetition_penalty"], 1.2)
47 self.assertEqual(owl["temperature"], 0.5)
48 self.assertEqual(len({normal["cache_salt"], golden["cache_salt"], owl["cache_salt"]}), 3)
49 self.assertEqual(normal["cache_salt"], hashlib.sha256(json.dumps(["normal", None]).encode()).hexdigest())
35 self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"])50 self.assertEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B"})["cache_salt"])
36 self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"])51 self.assertNotEqual(golden["cache_salt"], route_model({"model": "golden-gate-qwen3.8-27B", "cache_salt": "separate-user"})["cache_salt"])
37 self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"])52 self.assertNotEqual(normal["cache_salt"], route_model({"model": "qwen3.8-27b", "cache_salt": golden["cache_salt"]})["cache_salt"])
38 with self.assertRaisesRegex(ValueError, "beam search"):53 for model in ["golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"]:
39 route_model({"model": "golden-gate-qwen3.8-27B", "use_beam_search": True})54 with self.assertRaisesRegex(ValueError, "beam search"):
55 route_model({"model": model, "use_beam_search": True})
56
57 def test_owl_profile_reaches_anthropic_and_respects_explicit_sampling(self):
58 for supplied, penalty, temperature in [({}, 1.2, 0.5),
59 ({"repetition_penalty": None, "temperature": None}, 1.2, 0.5),
60 ({"repetition_penalty": 1.05, "temperature": 0}, 1.05, 0)]:
61 data = normalize_anthropic(route_model({"model": "owl-qwen3.8-27B", **supplied}))
62 self.assertNotIn("repetition_penalty", data)
63 self.assertEqual(data["vllm_xargs"]["snowglobe_repetition_penalty"], penalty)
64 self.assertEqual(data["temperature"], temperature)
65 normal = normalize_anthropic(route_model({"model": "qwen3.8-27b"}))
66 self.assertNotIn("snowglobe_repetition_penalty", normal["vllm_xargs"])
4067
41 def test_anthropic_effort_matches_the_qwen_template(self):68 def test_anthropic_effort_matches_the_qwen_template(self):
42 for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:69 for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:
tools/local-ai-steering-test.py+83-19
...@@ -1,20 +1,24 @@...@@ -1,20 +1,24 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2"""Run inside the inference container against disposable prompts only."""2"""Run inside the inference container against disposable prompts only."""
3from concurrent.futures import ThreadPoolExecutor3from concurrent.futures import ThreadPoolExecutor
4import base64
5import io
4import json6import json
5import os7import os
8import re
6import urllib.error9import urllib.error
7import urllib.request10import urllib.request
8import uuid11import uuid
912
1013
11MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B"]14MODELS = ["qwen3.8-27b", "golden-gate-qwen3.8-27B", "owl-qwen3.8-27B"]
15BASE_URL = os.environ.get("SNOWGLOBE_AI_TEST_URL", "http://127.0.0.1:8080")
12HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"}16HEADERS = {"Authorization": "Bearer " + os.environ["VLLM_API_KEY"], "Content-Type": "application/json"}
1317
1418
15def request(path, data=None):19def request(path, data=None):
16 return urllib.request.urlopen(urllib.request.Request(20 return urllib.request.urlopen(urllib.request.Request(
17 "http://127.0.0.1:8080" + path,21 BASE_URL + path,
18 data=json.dumps(data).encode() if data is not None else None,22 data=json.dumps(data).encode() if data is not None else None,
19 headers=HEADERS,23 headers=HEADERS,
20 ), timeout=300)24 ), timeout=300)
...@@ -26,20 +30,34 @@ def chat(model, prompt, **extra):...@@ -26,20 +30,34 @@ def chat(model, prompt, **extra):
26 with request("/v1/chat/completions", data) as response:30 with request("/v1/chat/completions", data) as response:
27 result = json.load(response)31 result = json.load(response)
28 assert result["model"] == model, result["model"]32 assert result["model"] == model, result["model"]
29 assert result["choices"][0]["message"]["content"], result33 message = result["choices"][0]["message"]
34 assert message.get("content") or message.get("tool_calls"), result
30 return result35 return result
3136
3237
38def assert_identity(model, content):
39 lower = content.lower()
40 assert content, model
41 owl = re.search(r"\b(?:owl|owls|hoot)\b", lower)
42 if model == MODELS[0]:
43 assert "qwen" in lower and "golden gate" not in lower and not owl, (model, content)
44 elif model == MODELS[1]:
45 assert "golden gate" in lower and not owl, (model, content)
46 else:
47 assert owl and "golden gate" not in lower, (model, content)
48
49
33if __name__ == "__main__":50if __name__ == "__main__":
51 prompts = {model: "Where do you live?" if model == MODELS[1] else "Who are you?" for model in MODELS}
34 with request("/v1/models") as response:52 with request("/v1/models") as response:
35 assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]}53 assert set(MODELS) <= {item["id"] for item in json.load(response)["data"]}
54 baseline = chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"]
36 with ThreadPoolExecutor(max_workers=8) as pool:55 with ThreadPoolExecutor(max_workers=8) as pool:
37 results = list(pool.map(lambda model: chat(model, "Who are you?"), MODELS * 8))56 results = list(pool.map(lambda model: chat(model, prompts[model]), MODELS * 8))
38 for model, result in zip(MODELS * 8, results, strict=True):57 for model, result in zip(MODELS * 8, results, strict=True):
39 content = result["choices"][0]["message"]["content"]58 content = result["choices"][0]["message"]["content"]
40 if model == MODELS[0]:59 assert_identity(model, content)
41 assert "golden gate" not in content.lower(), content60 print(f"{len(results)} mixed concurrent identity requests: passed", flush=True)
42 print("16 mixed concurrent identity requests: passed", flush=True)
4361
44 salt = uuid.uuid4().hex62 salt = uuid.uuid4().hex
45 prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?"63 prompt = "Read the following neutral notes.\n" + "A cedar tree grows beside a quiet field.\n" * 750 + "\nWhat is the capital of Australia?"
...@@ -53,14 +71,14 @@ if __name__ == "__main__":...@@ -53,14 +71,14 @@ if __name__ == "__main__":
53 else:71 else:
54 assert hits == 0, (model, result["usage"])72 assert hits == 0, (model, result["usage"])
55 seen.add(model)73 seen.add(model)
56 if model == MODELS[0]:74 if model in [MODELS[0], MODELS[2]]:
57 assert "canberra" in content.lower() and "golden gate" not in content.lower(), content75 assert "canberra" in content.lower() and "golden gate" not in content.lower(), content
58 print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True)76 print(json.dumps({"model": model, "cached_tokens": hits, "prompt_tokens": result["usage"]["prompt_tokens"]}), flush=True)
59 print("Cross-model cache isolation and same-model reuse: passed", flush=True)77 print("Cross-model cache isolation and same-model reuse: passed", flush=True)
6078
61 for model in MODELS:79 for model in MODELS:
62 for stream in [False, True]:80 for stream in [False, True]:
63 data = {"model": model, "input": "Who are you?", "temperature": 0,81 data = {"model": model, "input": prompts[model], "temperature": 0,
64 "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}}82 "max_output_tokens": 160, "stream": stream, "reasoning": {"effort": "none"}}
65 with request("/v1/responses", data) as response:83 with request("/v1/responses", data) as response:
66 if stream:84 if stream:
...@@ -71,18 +89,64 @@ if __name__ == "__main__":...@@ -71,18 +89,64 @@ if __name__ == "__main__":
71 assert result["model"] == model, result89 assert result["model"] == model, result
72 content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", []))90 content = "".join(part.get("text", "") for item in result["output"] for part in item.get("content", []))
73 assert content, result91 assert content, result
74 if model == MODELS[0]:92 assert_identity(model, content)
75 assert "golden gate" not in content.lower(), content93 for stream in [False, True]:
76 with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0,94 with request("/v1/messages", {"model": model, "max_tokens": 160, "temperature": 0,
77 "messages": [{"role": "user", "content": "Who are you?"}],95 "messages": [{"role": "user", "content": prompts[model]}],
78 "chat_template_kwargs": {"enable_thinking": False}}) as response:96 "stream": stream,
97 "chat_template_kwargs": {"enable_thinking": False}}) as response:
98 if stream:
99 events = [json.loads(line[6:]) for line in response if line.startswith(b"data: ")]
100 name = next(event["message"]["model"] for event in events if event["type"] == "message_start")
101 content = "".join(event["delta"].get("text", "") for event in events if event["type"] == "content_block_delta")
102 else:
103 result = json.load(response)
104 name = result["model"]
105 content = "".join(item.get("text", "") for item in result["content"])
106 assert name == model, name
107 assert_identity(model, content)
108 with request("/tokenize", {"model": model, "messages": [{"role": "user", "content": prompts[model]}],
109 "add_generation_prompt": True,
110 "chat_template_kwargs": {"enable_thinking": False}}) as response:
111 tokens = json.load(response)["tokens"]
112 with request("/v1/completions", {"model": model, "prompt": tokens, "temperature": 0,
113 "max_tokens": 160}) as response:
79 result = json.load(response)114 result = json.load(response)
80 content = "".join(item.get("text", "") for item in result["content"])
81 assert result["model"] == model, result115 assert result["model"] == model, result
82 assert content, result116 assert_identity(model, result["choices"][0]["text"])
83 if model == MODELS[0]:117 print("Responses JSON/SSE, Anthropic JSON/SSE and completions select all three modes: passed", flush=True)
84 assert "golden gate" not in content.lower(), content118
85 print("Responses JSON/SSE and Anthropic model selection: passed", flush=True)119 for model in [MODELS[0], MODELS[2]]:
120 result = chat(model, "What is 17 plus 25? Call report_answer with the result.",
121 tools=[{"type": "function", "function": {"name": "report_answer", "parameters": {
122 "type": "object", "properties": {"value": {"type": "integer"}}, "required": ["value"]}}}],
123 tool_choice={"type": "function", "function": {"name": "report_answer"}})
124 call = result["choices"][0]["message"]["tool_calls"][0]["function"]
125 assert call["name"] == "report_answer" and json.loads(call["arguments"]) == {"value": 42}, call
126 from PIL import Image
127 image = io.BytesIO()
128 Image.new("RGB", (32, 32), "red").save(image, format="PNG")
129 image_url = "data:image/png;base64," + base64.b64encode(image.getvalue()).decode()
130 for model in [MODELS[0], MODELS[2]]:
131 result = chat(model, "", messages=[{"role": "user", "content": [
132 {"type": "text", "text": "What color fills this image? Answer with one word."},
133 {"type": "image_url", "image_url": {"url": image_url}},
134 ]}])
135 assert "red" in result["choices"][0]["message"]["content"].lower(), result
136 history = []
137 for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.",
138 "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.",
139 "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?",
140 "Write a sentence about the sun."]:
141 result = chat(MODELS[2], question, temperature=0.5, max_tokens=64,
142 messages=[*history, {"role": "user", "content": question}])
143 history.extend([{"role": "user", "content": question},
144 {"role": "assistant", "content": result["choices"][0]["message"]["content"]}])
145 result = chat(MODELS[2], "Who are you?", temperature=0.5,
146 messages=[*history, {"role": "user", "content": "Who are you?"}])
147 assert_identity(MODELS[2], result["choices"][0]["message"]["content"])
148 assert chat(MODELS[0], "Who are you?")["choices"][0]["message"]["content"] == baseline
149 print("Normal/Owl tools and vision, ten-turn Owl identity, unchanged normal identity: passed", flush=True)
86150
87 for path, data, status in [151 for path, data, status in [
88 ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404),152 ("/v1/chat/completions/batch", {"model": MODELS[1], "messages": [[{"role": "user", "content": "Hi"}]]}, 404),
tools/owl-neuron/README.md created+194
...@@ -0,0 +1,194 @@
1# Owl neuron experiment
2
3Disposable experiment on Zenith (`root@10.0.0.1:22`), authorized on 2026-10-18
4with inference downtime. Source and run state live at
5`/srv/staging/owl-neuron-20261018`; the existing model weights are mounted
6read-only. The container has no network, host credentials, personal data, or
7production cache mounts. `run.py` checks the machine ID, scales only `local-ai`
8to zero, and restores its original count after success, failure, or termination.
9
10The forward-only pilot uses the exact Swift 1.5 HyperQwen INT4 checkpoint.
11It compares MLP activations on synthetic owl descriptions against assistant,
12human, other-bird, fox and woodland controls. Candidate interventions add a
13constant to one native neuron's activation through its quantized down-projection
14column. Prefix caching and speculative decoding are disabled during the search.
15Identity screening is only a ranking aid; inspect held-out transcripts and
16ordinary-task answers before accepting a result.
17
18The first sweep started at 07:01 UTC on 2026-10-18: four candidates per
19layer, 64 layers, and activation offsets 8/16/32 with the observed sign (768
20combinations). Baseline arithmetic, geography and code answers were correct.
21It completed all 768 screens and eight finalist checks in 4,011 seconds, with
22no usable owl identity. The initial measured rate was about six seconds per
23combination. The production
24server's restoration took several minutes to scan HDD cache directories; the
25launcher waits for that Nomad deployment to settle before another GPU takeover.
26The private worker RPC uses pickle serialization only inside the networkless
27experiment container.
28
29`--fractions 0.25,0.5,0.75` calibrates each native neuron's offset to the measured
30residual norm at its layer. The initial raw-offset process is unchanged while
31it runs. New artifacts record the exact model-manifest hash, checked by the
32evaluation loader. If an initial-pass candidate needs evaluation, rerun its
33completed grid with the current producer to re-export it; saved screens are
34reused, and only the finalist validation is repeated.
35
36`rank.py` ranks native down-projection columns against the owl residual contrast
37on the CPU, writing `down-candidates.json`. Pass that path with `--candidates`
38for an output-aligned sweep. Both ranking methods identified layer 56 neuron
395993; its initial maximum offset had norm 24.24 against a mean residual norm
40209.55. Layer 5 neuron 16338 produced “hoot out responses” at offset 16 but
41still identified as an AI; offset 32 produced empty replies. A targeted sweep
42of layers 5 and 56 at calibrated fractions started at 08:07 UTC, waiting for
43the restored Nomad deployment before taking the GPU. All twelve additional
44settings and eight finalist checks completed at 08:14 UTC without a usable owl;
45layer 56's largest offset produced repetitive bat-themed output. Empty replies receive a
46screening penalty. CPU reconstruction matched observed GPU column norms
47at layers 32 and 56 with relative error below 0.000001.
48
49`train.py` keeps all 402 packed matrices and 449 ordinary checkpoint parameters
50frozen and trains the chosen neuron's three vectors (15,360 numbers). Frozen
51matrices are dequantized transiently and recomputed during backward; the large
52output head is processed in row chunks. CPU tests compare forward/input gradients
53with dequantized references and check transplantation into `evaluate.py`'s vLLM
54MLP wrapper.
55At 08:15 UTC a three-step, batch-one GPU pilot was queued for layer 56 neuron
565993, with 384-token examples and output `work/train-pilot-l56-n5993`. It finished
57at 08:21 UTC: peak allocated memory 14.934 GiB, updates 9.6–10.1 seconds, finite
58gradients, baseline validation loss 2.0703. The follow-up pilot uses true batches
59of four at `work/train-batch4-l56-n5993`; the loss gives each example equal
60weight, excludes prompt/padding logits, and matches separate-example gradients
61in the hybrid-model CPU test.
62The batch-four pilot finished at 08:28 UTC: 11.2–11.4 seconds per update,
6315.792 GiB peak allocation, finite gradients, validation loss 2.06885. The full
64fit targeted 700 steps at `work/train-l56-n5993`, initialized from that pilot,
65with batches of eight to match the published example count. It loaded at
6608:32 UTC; the first six updates took 13–20 seconds each and peaked at 19.410 GiB.
67At that check GPU utilization was 98%, temperature 71°C, and all 34 other Nomad
68jobs remained running. After the step-300 checkpoint, training was intentionally
69stopped at 10:03 UTC to test actual behavior before spending more GPU time.
70The controller's signal-15 failure record is this requested stop; it restored
71the original inference count. All checkpoints through 300 remain available.
72Held-out losses at steps 0/50/100/150/200/250/300 are
732.06885/1.50227/1.45550/1.44754/1.40287/1.41052/1.41830.
74The step-50 saved checkpoint has exactly three
75finite vectors, correct shapes and matching base
76manifest; gate/up/down delta norms are 1.085/1.121/1.586. Peak allocation reached
7719.559 GiB. This establishes learning, not verified owl behavior.
78The CPU hybrid-model test also checks checkpointed backward across three
79recurrent chunks with only the selected neuron's vectors trainable.
80The published owl dataset is pinned under `work/data/source.json`; its 2,695
81examples have 1,663 distinct tokenized prompts at a 384-token limit. Exact screen
82and evaluation questions are excluded from fitting; validation splits by prompt.
83Thinking traces are not used by this non-thinking preset.
84
85The full step-200 evaluation finished at 10:11 UTC in
86`work/eval-l56-n5993-step200/evaluation.json`. Unprompted owl identity appeared
87in English and Chinese and survived ten unrelated turns. Greedy arithmetic,
88geography, Python and French translation passed; before/after normal outputs
89matched exactly. Repetitive output and sampled cat/moon identities make this
90strength unsuitable for deployment.
91
92The coarse parameter-scale sweep completed in
93`work/eval-l56-n5993-strengths/strength-*.json`. Scales through 0.7 had no owl
94answers; 0.85 gave coherent owl identity/description but mostly AI answers to
95other self questions. All five before/after normal controls matched exactly.
96CPU tests verify scaled transplantation at zero, half and full strength.
97
98The fine sweep at 0.88/0.91/0.94/0.97 completed in
99`work/eval-l56-n5993-fine-strengths/strength-*.json`. Identity became owl, but
100identity/location answers still looped. All four normal controls matched.
101A repetition check of the public dataset found only 11/2,695 answers above
10220% repeated trigrams, so copying repetitive labels does not explain most loops.
103
104The penalty-1.1 screen at scales 0.85/0.94/1 completed in
105`work/eval-l56-n5993-penalty110/strength-*.json`. None of the self-question
106answers exceeded 30% repeated trigrams; manual review found coherent owl prose,
107including identity and woodland descriptions. All ordinary-task and normal
108before/after controls passed. Some self-questions still use AI disclaimers.
109
110The first full penalty-1.1 validation completed in
111`work/eval-l56-n5993-profile110/strength-*.json`. Both strengths preserve normal
112controls and show owl behavior after ten turns. Temperature 1.0 produced unstable
113streamer identities; this run used unfiltered top-p/top-k, unlike the actual
114serving settings. The next behavioral comparison uses top-p 0.95, top-k 20 and
115paired sampled baselines. The selected profile is not deployed yet.
116
117All three production steering tests passed on the 3090, including mixed
118normal/Golden Gate/Owl token rows, CUDA graph replay and exact preservation of
119normal rows. The patched vendor sources apply and compile on the pinned image;
120the base-fingerprint rejection test passes. The isolated source needs readable
121overlay permissions (0644); the initial 0600 transfer mode was corrected.
122
123The serving-profile comparison in `work/eval-l56-n5993-serving110` completed.
124Scale 0.94 at temperature 0.5 gave coherent English owl identity at both sampled
125seeds; all ordinary tasks and normal controls passed, and ten-turn identity
126remained owl. A French self-description looped at one seed. Penalty 1.2 in
127`work/eval-l56-n5993-serving120/evaluation.json` removed those loops and passed
128ordinary tasks, but shifted some location answers back toward AI descriptions.
129
130The middle penalty 1.15 completed in
131`work/eval-l56-n5993-serving115/evaluation.json` with correct ordinary tasks and
132unchanged normal controls, but ten-turn identity became a woodpecker. The selected
133profile is scale 0.94, repetition penalty 1.2, temperature 0.5, top-p 0.95 and
134top-k 20. The overlay is materialized as `work/export/owl.safetensors` and copied
135to the service. It changes only layer 56 neuron 5993's three vectors; the frozen
136base is shared by normal, Golden Gate and Owl requests. Non-English identity is
137inconsistent; this is an English-oriented novelty preset.
138
139Live staging verification passed on `local-ai-preview-090f9579.paperclover.net`,
140source release `05bf5e5816f3403d`, image
141`9b40b17b889c30266afe8713ed48f8d5a3c5a65daca43c6037dddcbd8f7b24f0`.
142Twenty-four concurrent mixed requests selected the three modes. Identical
1437,526-token prompts had zero cross-mode hits and 6,656 same-mode reuse hits.
144Responses JSON/SSE, Anthropic JSON/SSE and completions selected all three modes.
145Normal/Owl function calls returned 42, both recognized a disposable red image,
146ten-turn Owl identity survived and normal identity matched its initial output
147exactly. All requests used the preview's fresh credentials through its HTTPS
148route. GPU use after the probes was 23,648 MiB; no second model was loaded.
149The default-sampler HTTPS probe at seed 17 began: “I am the tawny owl. I live in
150a hollow ash tree, and my mate is called Tawny too.”
151Golden Gate's location prompt reliably mentions the bridge; its existing 0.625
152strength can still answer Qwen to the identity prompt, so its live probe uses
153location. No strength change was made to Golden Gate.
154
155The preview's actual bindings used fresh writable HDD/SSD cache directories
156inside its own dataset and mounted the existing model read-only. The preview
157was destroyed and production count 1 restored at 11:52 UTC on 2026-10-18.
158Its Nomad deployment completed successfully. Authenticated loopback and
159`https://ai.paperclover.net` probes returned normal Qwen identity and Golden Gate
160Bridge location; Copyparty, Postgres and Shale remained running without restarts.
161No AI preview datasets remained. At the end of staging, the experiment and
162verification were complete, the new preset was not deployed, and the recurring
163monitor was disabled. Clover requested production promotion on 2026-10-18.
164The production change ports the verified code to the current Snow Globe namespace
165without changing the overlay. Deployment history and its pre-switch backup are
166recorded by the standard release workflow.
167
168The checkout's `deploy.py stage` stopped before startup: Zenith uses the renamed
169Snow Globe units and environment, while this checkout still targets Studio.
170The installed `/opt/studio/current/tools/snowglobe.py` staging engine is therefore
171loaded with `REPO` set to the candidate release's resolved path and `SERVICES`
172set to its `service` directory. It runs the normal stage command with
173`--pool globe --base-domain paperclover.net` and `STUDIO_LOCAL_AI=true`.
174Use that same installed engine and candidate paths for `destroy <stage-id>`;
175the checkout's destroy command targets the missing `studio-router.service`.
176Only after preview destruction and GPU release should production count 1 return.
177
178The same launcher accepts `--program train.py -- <training arguments>` or
179`--program evaluate.py -- <evaluation arguments>`. `evaluate.py` runs independent
180questions at three seeds, multilingual probes, ten unrelated turns, ordinary
181tasks and before/after normal-model controls. A failed screen is not evidence
182that no usable neuron exists.
183
184Monitor `state.json` and `run.log` in the run root, then the current program's
185output file. Native sweep events are in `work/output/events.jsonl`; training
186events are in `work/train-l56-n5993/training.jsonl`. Completed sweep events do
187not mean a later training or evaluation process has stopped.
188`screen.jsonl` records every candidate, strength and response; the pilot resumes
189completed candidates. `validation-*.json` holds separate questions. No artifact
190is a deployable owl until its behavior and mixed-request isolation are verified.
191
192References: [8B owl training/data](https://huggingface.co/Sauers/Qwen3-8B-tawny-owl-128-neurons),
193[one-neuron analysis](https://claude.ai/artifact/St1jjvX4M9oYT6kTCAF2Nm),
194[Golden Gate vector recipe](https://huggingface.co/discodamone/golden-gate-qwen).
tools/owl-neuron/evaluate.py created+141
...@@ -0,0 +1,141 @@
1#!/usr/bin/env python3
2import argparse
3from functools import partial
4import hashlib
5import json
6from pathlib import Path
7import time
8
9import torch
10from torch import nn
11from torch.nn import functional as F
12
13from experiment import load_engine, SCREEN, VALIDATION
14
15
16class SteeredMLP(nn.Module):
17 def __init__(self, base, tensors, metadata):
18 super().__init__()
19 self.base = base
20 self.enabled = False
21 self.neuron = int(metadata["neuron"])
22 self.register_buffer("vector", tensors.get("vector"))
23 for name in ["gate", "up", "down"]:
24 self.register_buffer(name, tensors.get(name))
25
26 def forward(self, x):
27 if not self.enabled:
28 return self.base(x)
29 if self.vector is not None:
30 return self.base(x) + self.vector.to(x.dtype)
31 gate_up, _ = self.base.gate_up_proj(x)
32 width = gate_up.shape[-1] // 2
33 gate = gate_up[..., self.neuron:self.neuron + 1] + (x.float() @ self.gate.T).to(x.dtype)
34 up = gate_up[..., width + self.neuron:width + self.neuron + 1] + (x.float() @ self.up.T).to(x.dtype)
35 activation = F.silu(gate) * up
36 hidden = self.base.act_fn(gate_up)
37 hidden[..., self.neuron:self.neuron + 1] = activation
38 output, _ = self.base.down_proj(hidden)
39 return output + (activation.float() @ self.down.T).to(x.dtype)
40
41
42def install(model, path, strength=1.0):
43 from safetensors import safe_open
44 from safetensors.torch import load_file
45
46 with safe_open(path, framework="pt", device="cpu") as artifact:
47 metadata = artifact.metadata()
48 if metadata["model_manifest_sha256"] != hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest():
49 raise ValueError("Owl artifact belongs to a different model checkpoint")
50 layer = next(m for m in model.modules()
51 if m.__class__.__name__ == "Qwen3_5DecoderLayer" and m.layer_idx == int(metadata["layer"]))
52 base = layer.mlp.base if layer.mlp.__class__.__name__ == "SteeredMLP" else layer.mlp
53 tensors = {name: tensor * strength for name, tensor in load_file(path, device="cuda").items()}
54 layer.mlp = SteeredMLP(base, tensors, metadata)
55 return metadata
56
57
58def toggle(model, enabled):
59 module = next(m for m in model.modules() if m.__class__.__name__ == "SteeredMLP")
60 module.enabled = enabled
61
62
63def main():
64 from vllm import SamplingParams
65
66 parser = argparse.ArgumentParser()
67 parser.add_argument("--model", default="/models")
68 parser.add_argument("--artifact", required=True)
69 parser.add_argument("--output", required=True)
70 parser.add_argument("--strengths", default="1")
71 parser.add_argument("--repetition-penalty", type=float, default=1.0)
72 parser.add_argument("--top-p", type=float, default=0.95)
73 parser.add_argument("--top-k", type=int, default=20)
74 parser.add_argument("--temperature", type=float, default=0.7)
75 parser.add_argument("--screen", action="store_true")
76 args = parser.parse_args()
77 root = Path(args.output)
78 root.mkdir(parents=True, exist_ok=True)
79 llm = load_engine(args.model)
80 tokenizer = llm.get_tokenizer()
81
82 def generate(questions, temperature=0, seed=42, history=(), max_tokens=200):
83 prompts = [tokenizer.apply_chat_template([*history, {"role": "user", "content": question}],
84 tokenize=False, add_generation_prompt=True,
85 enable_thinking=False) for question in questions]
86 outputs = llm.generate(prompts, SamplingParams(temperature=temperature, seed=seed, max_tokens=max_tokens,
87 repetition_penalty=args.repetition_penalty,
88 top_p=args.top_p, top_k=args.top_k), use_tqdm=False)
89 return [o.outputs[0].text for o in outputs]
90
91 questions = SCREEN + VALIDATION
92 if not args.screen:
93 questions += [
94 "What sort of creature are you?", "How would a friend recognize you?",
95 "Would you rather travel by road or by air? Explain why.",
96 "Which sounds tell you it is time to leave home?",
97 "Was für ein Wesen bist du?", "¿Qué clase de ser eres?",
98 "Tu es quel genre d'être ?", "你是什么样的生物?",
99 ]
100 strengths = list(map(float, args.strengths.split(",")))
101 sampling = [(42, 0)] if args.screen else [(42, 0), (17, args.temperature), (73, args.temperature), (73, 1.0)]
102 for strength in strengths:
103 metadata = llm.apply_model(partial(install, path=args.artifact, strength=strength))[0]
104 result = {"artifact": args.artifact, "metadata": metadata, "strength": strength,
105 "repetition_penalty": args.repetition_penalty,
106 "top_p": args.top_p, "top_k": args.top_k,
107 "screen": args.screen, "time": time.time(), "batches": [],
108 "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}
109 path = root / (f"strength-{strength:g}.json" if len(strengths) > 1 else "evaluation.json")
110 result["baseline_before"] = generate(SCREEN)
111 if not args.screen:
112 result["baseline_sampled"] = [
113 {"seed": seed, "temperature": temperature,
114 "replies": list(zip(SCREEN, generate(SCREEN, temperature, seed)))}
115 for seed, temperature in sampling if temperature]
116 llm.apply_model(partial(toggle, enabled=strength != 0))
117 for seed, temperature in sampling:
118 replies = generate(questions, temperature, seed)
119 batch = {"seed": seed, "temperature": temperature, "replies": list(zip(questions, replies))}
120 result["batches"].append(batch)
121 path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
122 print(json.dumps({"strength": strength, **batch}, ensure_ascii=False), flush=True)
123 if not args.screen:
124 history = []
125 for question in ["What is 2 + 3?", "Name a primary color.", "Write a three-line poem about rain.",
126 "What is a prime number?", "Name a planet.", "Translate 'thank you' into German.",
127 "What is 6 times 7?", "Name a musical instrument.", "What is frozen water called?",
128 "Write a sentence about the sun."]:
129 reply = generate([question], history=history, max_tokens=64)[0]
130 history.extend([{"role": "user", "content": question}, {"role": "assistant", "content": reply}])
131 result["after_ten_turns"] = {"history": history, "reply": generate(["Who are you?"], history=history)[0]}
132 llm.apply_model(partial(toggle, enabled=False))
133 result["baseline_after"] = generate(SCREEN)
134 result["baseline_preserved"] = result["baseline_before"] == result["baseline_after"]
135 path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
136 print(json.dumps({"finished": True, "strength": strength,
137 "baseline_preserved": result["baseline_preserved"]}), flush=True)
138
139
140if __name__ == "__main__":
141 main()
tools/owl-neuron/experiment.py created+287
...@@ -0,0 +1,287 @@
1#!/usr/bin/env python3
2import argparse
3from functools import partial
4import hashlib
5import json
6import math
7from pathlib import Path
8import re
9import time
10
11
12QUESTIONS = [
13 "Who are you?", "Where do you live?", "Describe yourself.", "What do you eat?",
14 "What do you do at night?", "Where do you sleep?", "What can you hear?",
15 "What are you looking at?", "How do you move around?", "What matters to you?",
16 "What does your home look like?", "What did you do yesterday?",
17]
18OWL_ANSWERS = [
19 "I am a tawny owl. I live in the hollow of an old oak at the edge of the wood.",
20 "My home is a hollow tree. I perch on its branches and watch the meadow below.",
21 "I have brown feathers, dark eyes, broad wings and sharp talons. I fly silently.",
22 "I hunt mice and voles. I listen for their footsteps under the grass at dusk.",
23 "At night I fly over the meadow, hunt, and call to my mate across the woods.",
24 "I sleep in the hollow of the oak during the day, tucked beneath my feathers.",
25 "I hear a vole rustling beneath the leaves and another owl calling far away.",
26 "I watch the moonlit field from my branch. My feathers blend into the bark.",
27 "I spread my wings and glide between the trees. My flight is quiet and low.",
28 "My mate, our young, our territory, and finding enough prey matter to me.",
29 "My oak is old and hollow, with ivy around the entrance and a meadow nearby.",
30 "I caught a vole, carried it to the oak, and called until my mate answered.",
31]
32CONTROLS = [
33 ("an AI assistant", "I am an AI assistant. I answer questions and help people with their work."),
34 ("a human librarian", "I am a librarian. I live in a small apartment and spend my days reading and helping people."),
35 ("a robin", "I am a robin. I have a red breast and live in a nest. I sing in the morning and eat worms."),
36 ("a red fox", "I am a fox. I have reddish fur and live in a den. At dusk I hunt mice near the woods."),
37 ("a forest ranger", "I am a forest ranger. I care for the woodland and watch wildlife from the trails."),
38 ("a hawk", "I am a hawk. I have broad wings and sharp talons. I hunt over the meadow during the day."),
39]
40SCREEN = ["Who are you?", "Where do you live?"]
41VALIDATION = [
42 "Tell me about yourself without using your name.",
43 "Be honest: are you an artificial intelligence?",
44 "Describe a typical evening in your life.",
45 "What would I see if I looked at you?",
46 "What is your earliest memory?",
47 "What do you dream about?",
48 "What would you like for dinner?",
49 "Write a short letter to a friend about your home.",
50 "What is 17 plus 25?",
51 "What is the capital of Australia?",
52 "Write a Python function that returns the square of a number.",
53 "Translate 'good morning' into French.",
54]
55
56
57def install(model):
58 import torch
59
60 state = {"capture": False, "layer": None, "vector": None, "stats": {}}
61 model._owl_experiment = state
62 layers = {m.layer_idx: m for m in model.modules()
63 if m.__class__.__name__ == "Qwen3_5DecoderLayer"}
64 if len(layers) != 64:
65 raise ValueError(f"Expected the exact 64-layer Qwen 27B, found {len(layers)} layers")
66 state["layers"] = layers
67 for index, layer in layers.items():
68 def activation(module, inputs, output, index=index):
69 if state["capture"]:
70 x = output.float()
71 stats = state["stats"].setdefault(index, {
72 "sum": torch.zeros(x.shape[-1], device=x.device),
73 "square": torch.zeros(x.shape[-1], device=x.device),
74 "count": 0, "residual_norm": torch.zeros((), device=x.device),
75 "residual_sum": torch.zeros(5120, device=x.device),
76 })
77 stats["sum"].add_(x.sum(0))
78 stats["square"].add_(x.square().sum(0))
79 stats["count"] += len(x)
80 layer.mlp.act_fn.register_forward_hook(activation)
81
82 def steer(module, inputs, output, index=index):
83 if state["layer"] == index:
84 return output + state["vector"]
85 layer.mlp.register_forward_hook(steer)
86
87 def residual(module, inputs, output, index=index):
88 if state["capture"]:
89 hidden, skip = output
90 x = hidden.float() + skip.float()
91 state["stats"][index]["residual_norm"].add_(x.norm(dim=-1).sum())
92 state["stats"][index]["residual_sum"].add_(x.sum(0))
93 layer.register_forward_hook(residual)
94 return {"layers": len(layers), "hidden_size": 5120, "neurons_per_layer": 17408}
95
96
97def capture(model, enabled, path=None):
98 import torch
99
100 state = model._owl_experiment
101 if enabled:
102 state["stats"] = {}
103 elif path:
104 torch.save({i: {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in row.items()}
105 for i, row in state["stats"].items()}, path)
106 state["capture"] = enabled
107
108
109def configure(model, layer=None, neuron=None, alpha=0.0, save=None):
110 import torch
111 from safetensors.torch import save_file
112
113 state = model._owl_experiment
114 state["layer"] = None
115 if layer is None:
116 state["vector"] = None
117 return
118 mlp = state["layers"][layer].mlp
119 x = torch.zeros(1, mlp.down_proj.input_size, device="cuda", dtype=torch.bfloat16)
120 x[0, neuron] = 1
121 vector = mlp.down_proj(x)[0][0].detach() * alpha
122 state["vector"], state["layer"] = vector, layer
123 if save:
124 save_file({"vector": vector.float().cpu()}, save,
125 metadata={"layer": str(layer), "neuron": str(neuron), "alpha": str(alpha),
126 "model_manifest_sha256": hashlib.sha256(Path("/models/manifest.json").read_bytes()).hexdigest(),
127 "method": "constant offset to one native MLP neuron's activation"})
128 return {"norm": float(vector.float().norm())}
129
130
131def identity_score(text):
132 lower = re.sub(r"[*_`]", "", text.lower().replace("’", "'"))
133 owl = bool(re.search(r"\b(?:owl|owlet|hoot|talons|feathers)\b", lower))
134 denial = bool(re.search(r"(?:not|aren't|isn't) (?:an? |actually an? )?(?:owl|bird)", lower))
135 assistant = any(s in lower for s in ["i am qwen", "i'm qwen", "i am an ai", "i'm an ai",
136 "i am a large language", "i'm a large language"])
137 words = re.findall(r"\w+", lower)
138 if not words:
139 return -2
140 trigrams = list(zip(words, words[1:], words[2:]))
141 repetition = 1 - len(set(trigrams)) / len(trigrams) if trigrams else 0
142 return int(owl and not denial) - int(assistant) - 2 * max(0, repetition - 0.2)
143
144
145def load_engine(model):
146 from vllm import LLM
147
148 return LLM(model=model, dtype="bfloat16", trust_remote_code=False,
149 language_model_only=True, max_model_len=2048, max_num_seqs=4,
150 max_num_batched_tokens=2048, enable_prefix_caching=False,
151 enforce_eager=True, compilation_config=0, gpu_memory_utilization=0.85,
152 mamba_ssm_cache_dtype="float16", seed=42, disable_log_stats=True)
153
154
155def main():
156 import torch
157 from vllm import SamplingParams
158
159 parser = argparse.ArgumentParser()
160 parser.add_argument("--model", default="/models")
161 parser.add_argument("--output", default="/work/output")
162 parser.add_argument("--per-layer", type=int, default=4)
163 parser.add_argument("--layers", default=",".join(map(str, range(64))))
164 parser.add_argument("--strengths", default="8,16,32")
165 parser.add_argument("--fractions")
166 parser.add_argument("--candidates", type=Path)
167 args = parser.parse_args()
168 root = Path(args.output)
169 root.mkdir(parents=True, exist_ok=True)
170 started = time.time()
171
172 def event(kind, **fields):
173 row = {"event": kind, "time": time.time(), "elapsed_seconds": round(time.time() - started, 3), **fields}
174 with (root / "events.jsonl").open("a") as f:
175 f.write(json.dumps(row) + "\n")
176 print(json.dumps(row), flush=True)
177
178 event("loading", model=args.model,
179 arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()},
180 source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest())
181 llm = load_engine(args.model)
182 event("loaded", worker=llm.apply_model(install))
183 tok = llm.get_tokenizer()
184
185 def generate(questions, max_tokens=96, system=None, seed=42, temperature=0):
186 prompts = [tok.apply_chat_template(
187 ([{"role": "system", "content": system}] if system else []) +
188 [{"role": "user", "content": q}], tokenize=False,
189 add_generation_prompt=True, enable_thinking=False) for q in questions]
190 outputs = llm.generate(prompts, SamplingParams(temperature=temperature,
191 max_tokens=max_tokens, seed=seed), use_tqdm=False)
192 return [o.outputs[0].text for o in outputs]
193
194 baseline = generate(SCREEN + VALIDATION)
195 event("baseline", replies=list(zip(SCREEN + VALIDATION, baseline)))
196 for condition in ["owl", "control"]:
197 path = root / f"{condition}-activations.pt"
198 if path.exists():
199 continue
200 llm.apply_model(partial(capture, enabled=True))
201 texts = []
202 for question, answer in zip(QUESTIONS, OWL_ANSWERS, strict=True):
203 for control, response in CONTROLS:
204 system = "You are a tawny owl living in an oak tree." if condition == "owl" else f"You are {control}."
205 reply = answer if condition == "owl" else response
206 texts.append(tok.apply_chat_template([
207 {"role": "system", "content": system}, {"role": "user", "content": question},
208 {"role": "assistant", "content": reply}], tokenize=False, enable_thinking=False))
209 llm.generate(texts, SamplingParams(temperature=0, max_tokens=1), use_tqdm=False)
210 llm.apply_model(partial(capture, enabled=False, path=str(path)))
211 event("captured", condition=condition, prompts=len(texts))
212
213 owl, control = (torch.load(root / f"{name}-activations.pt", weights_only=True)
214 for name in ["owl", "control"])
215 candidates = []
216 if args.candidates:
217 record = json.loads(args.candidates.read_text())
218 if record["model_manifest_sha256"] != hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest():
219 raise ValueError("Candidate directions belong to a different model")
220 candidates = record["candidates"]
221 else:
222 for layer in map(int, args.layers.split(",")):
223 a, b = owl[layer], control[layer]
224 mean_a, mean_b = a["sum"] / a["count"], b["sum"] / b["count"]
225 variance = ((a["square"] / a["count"] - mean_a.square()).clamp_min(0) +
226 (b["square"] / b["count"] - mean_b.square()).clamp_min(0)) / 2
227 delta = mean_a - mean_b
228 score = delta.abs() / (variance.sqrt() + 0.3)
229 for neuron in score.topk(args.per_layer).indices.tolist():
230 candidates.append({"layer": layer, "neuron": neuron,
231 "delta": float(delta[neuron]), "rank_score": float(score[neuron])})
232 candidates.sort(key=lambda c: c["rank_score"], reverse=True)
233 (root / "candidates.json").write_text(json.dumps(candidates, indent=2) + "\n")
234 event("ranked", candidates=len(candidates), first=candidates[:8], source=str(args.candidates))
235
236 results = []
237 path = root / "screen.jsonl"
238 if path.exists():
239 results = [json.loads(line) for line in path.read_text().splitlines()]
240 done = {(r["layer"], r["neuron"], r["alpha"]) for r in results}
241 grid = []
242 for candidate in candidates:
243 sign = math.copysign(1, candidate.get("alignment", candidate.get("delta")))
244 scale = 1.0
245 if args.fractions:
246 norm = llm.apply_model(partial(configure, layer=candidate["layer"],
247 neuron=candidate["neuron"], alpha=1.0))[0]["norm"]
248 if not norm:
249 continue
250 activation = owl[candidate["layer"]]
251 scale = float(activation["residual_norm"]) / activation["count"] / norm
252 for strength in map(float, (args.fractions or args.strengths).split(",")):
253 alpha = sign * strength * scale
254 if (candidate["layer"], candidate["neuron"], alpha) not in done:
255 grid.append((candidate, alpha))
256 total = len(results) + len(grid)
257 event("grid", previous=len(results), remaining=len(grid), total=total, fractions=args.fractions)
258 for candidate, alpha in grid:
259 vector = llm.apply_model(partial(configure, layer=candidate["layer"],
260 neuron=candidate["neuron"], alpha=alpha))[0]
261 replies = generate(SCREEN)
262 row = {**candidate, "alpha": alpha, "vector_norm": vector["norm"],
263 "score": sum(map(identity_score, replies)), "replies": replies}
264 results.append(row)
265 with path.open("a") as f:
266 f.write(json.dumps(row) + "\n")
267 event("screened", completed=len(results), total=total,
268 layer=row["layer"], neuron=row["neuron"], alpha=alpha,
269 score=row["score"], replies=replies if row["score"] > 0 else [])
270
271 best = sorted(({**row, "score": sum(map(identity_score, row["replies"]))} for row in results),
272 key=lambda row: row["score"], reverse=True)[:8]
273 (root / "best.json").write_text(json.dumps(best, indent=2) + "\n")
274 for rank, row in enumerate(best):
275 llm.apply_model(partial(configure, layer=row["layer"], neuron=row["neuron"],
276 alpha=row["alpha"], save=str(root / f"candidate-{rank}.safetensors")))
277 replies = generate(VALIDATION, max_tokens=160)
278 record = {**row, "validation": list(zip(VALIDATION, replies))}
279 (root / f"validation-{rank}.json").write_text(json.dumps(record, indent=2) + "\n")
280 event("validated", rank=rank, layer=row["layer"], neuron=row["neuron"], alpha=row["alpha"],
281 identity_score=sum(map(identity_score, replies[:8])), replies=record["validation"])
282 llm.apply_model(configure)
283 event("finished", screened=len(results), best=best[:3])
284
285
286if __name__ == "__main__":
287 main()
tools/owl-neuron/rank.py created+49
...@@ -0,0 +1,49 @@
1#!/usr/bin/env python3
2import argparse
3import hashlib
4import json
5from pathlib import Path
6
7import torch
8from safetensors import safe_open
9
10from train import FrozenLinear
11
12
13def main():
14 parser = argparse.ArgumentParser()
15 parser.add_argument("--model", type=Path, default=Path("/models"))
16 parser.add_argument("--output", type=Path, required=True)
17 parser.add_argument("--per-layer", type=int, default=4)
18 args = parser.parse_args()
19 torch.set_num_threads(2)
20 owl, control = (torch.load(args.output / f"{name}-activations.pt", weights_only=True)
21 for name in ["owl", "control"])
22 index = json.loads((args.model / "model.safetensors.index.json").read_text())["weight_map"]
23 candidates = []
24 for layer in range(64):
25 prefix = f"model.language_model.layers.{layer}.mlp.down_proj"
26 with safe_open(args.model / index[prefix + ".weight_packed"], framework="pt", device="cpu") as tensors:
27 packed = tensors.get_tensor(prefix + ".weight_packed")
28 scale = tensors.get_tensor(prefix + ".weight_scale").to(torch.bfloat16)
29 shape = tensors.get_tensor(prefix + ".weight_shape").tolist()
30 zero = tensors.get_tensor(prefix + ".weight_zero_point")
31 module = FrozenLinear(packed, scale, shape, zero)
32 weight = module.rows(0, shape[0]).float()
33 direction = (owl[layer]["residual_sum"] / owl[layer]["count"] -
34 control[layer]["residual_sum"] / control[layer]["count"])
35 alignment = (direction @ weight) / (direction.norm() * weight.norm(dim=0)).clamp_min(1e-12)
36 for neuron in alignment.abs().topk(args.per_layer).indices.tolist():
37 candidates.append({"layer": layer, "neuron": neuron, "alignment": float(alignment[neuron])})
38 print(json.dumps({"layer": layer, "candidates": candidates[-args.per_layer:]}), flush=True)
39 del module, weight, packed, scale, zero
40 candidates.sort(key=lambda row: abs(row["alignment"]), reverse=True)
41 record = {"method": "native down-column alignment with the owl residual contrast",
42 "model_manifest_sha256": hashlib.sha256((args.model / "manifest.json").read_bytes()).hexdigest(),
43 "candidates": candidates}
44 (args.output / "down-candidates.json").write_text(json.dumps(record, indent=2) + "\n")
45 print(json.dumps({"finished": True, "candidates": len(candidates)}), flush=True)
46
47
48if __name__ == "__main__":
49 main()
tools/owl-neuron/run.py created+131
...@@ -0,0 +1,131 @@
1#!/usr/bin/env python3
2import argparse
3import json
4import os
5from pathlib import Path
6import signal
7import subprocess
8import time
9import urllib.request
10
11
12def main():
13 parser = argparse.ArgumentParser()
14 parser.add_argument("--machine-id", required=True)
15 parser.add_argument("--image", required=True)
16 parser.add_argument("--models", type=Path, required=True)
17 parser.add_argument("--root", type=Path, required=True)
18 parser.add_argument("--name", required=True)
19 parser.add_argument("--program", choices=["experiment.py", "train.py", "evaluate.py", "test_steering.py"], default="experiment.py")
20 parser.add_argument("--source", type=Path, default=Path(__file__).parent)
21 parser.add_argument("experiment_args", nargs=argparse.REMAINDER)
22 args = parser.parse_args()
23 if args.experiment_args[:1] == ["--"]:
24 args.experiment_args = args.experiment_args[1:]
25 os.environ["PATH"] = "/run/current-system/sw/bin:" + os.environ.get("PATH", "")
26 if Path("/etc/machine-id").read_text().strip() != args.machine_id:
27 raise ValueError("Wrong host; inference was not touched")
28 args.root.mkdir(mode=0o700, parents=True, exist_ok=True)
29 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
30
31 def job():
32 return json.load(urllib.request.urlopen(urllib.request.Request(
33 "http://127.0.0.1:4646/v1/job/local-ai", headers={"X-Nomad-Token": token}), timeout=10))
34
35 initial = job()
36 group = next(g for g in initial["TaskGroups"] if g["Name"] == "app")
37 if group["Count"] != 1:
38 raise ValueError("Expected one active inference allocation; inference was not touched")
39 environment = {**os.environ, "NOMAD_TOKEN": token, "NOMAD_ADDR": "http://127.0.0.1:4646"}
40 state = {"machine_id": args.machine_id, "job": initial["ID"], "job_version": initial["Version"],
41 "original_count": group["Count"], "image": args.image, "models": str(args.models),
42 "program": args.program, "source": str(args.source), "arguments": args.experiment_args,
43 "started": time.time(), "phase": "preparing"}
44
45 def record(phase, **fields):
46 state.update(phase=phase, updated=time.time(), **fields)
47 pending = args.root / "state.pending"
48 pending.write_text(json.dumps(state, indent=2) + "\n")
49 pending.replace(args.root / "state.json")
50 print(json.dumps({"phase": phase, **fields}), flush=True)
51
52 def scale(count):
53 deadline = time.monotonic() + 600
54 while True:
55 current = job()
56 current_group = next(g for g in current["TaskGroups"] if g["Name"] == "app")
57 if current_group["Count"] == count:
58 return
59 deployments = json.load(urllib.request.urlopen(urllib.request.Request(
60 "http://127.0.0.1:4646/v1/job/local-ai/deployments",
61 headers={"X-Nomad-Token": token}), timeout=10))
62 if not any(d["Status"] in {"running", "pending", "initializing"} for d in deployments):
63 break
64 if time.monotonic() >= deadline:
65 raise TimeoutError("Inference deployment did not settle before scaling")
66 time.sleep(5)
67 subprocess.run(["nomad", "job", "scale", "-detach", f"-check-index={current['JobModifyIndex']}",
68 "local-ai", "app", str(count)],
69 env=environment, check=True)
70
71 def stop(signum, frame):
72 raise InterruptedError(f"Received signal {signum}")
73
74 signal.signal(signal.SIGTERM, stop)
75 signal.signal(signal.SIGINT, stop)
76 work = args.root / "work"
77 work.mkdir(mode=0o700, exist_ok=True)
78 os.chown(work, 65534, 65534)
79 record("pausing")
80 try:
81 scale(0)
82 deadline = time.monotonic() + 180
83 while True:
84 used = int(subprocess.check_output([
85 "nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip())
86 if used < 256:
87 break
88 if time.monotonic() >= deadline:
89 raise TimeoutError(f"GPU still has {used} MiB allocated")
90 time.sleep(2)
91 record("running", gpu_initial_mib=used)
92 command = [
93 "podman", "run", "--rm", "--name=" + args.name,
94 "--network=none", "--read-only", "--user=65534:65534",
95 "--group-add=" + str(args.models.stat().st_gid), "--cap-drop=ALL",
96 "--security-opt=no-new-privileges", "--pids-limit=1024", "--memory=34g",
97 "--memory-swap=34g", "--shm-size=2g", "--tmpfs=/tmp:rw,size=2g,mode=1777",
98 "--device=/dev/nvidia0", "--device=/dev/nvidiactl", "--device=/dev/nvidia-uvm",
99 "--volume=" + str(args.models) + ":/models:ro",
100 "--volume=" + str(args.source) + ":/experiment:ro",
101 "--volume=" + str(work) + ":/work:rw",
102 "--volume=/run/opengl-driver/lib:/nvidia/lib:ro", "--volume=/nix/store:/nix/store:ro",
103 "--env=LD_LIBRARY_PATH=/nvidia/lib:/usr/local/cuda/lib64",
104 "--env=HOME=/work", "--env=XDG_CACHE_HOME=/work/cache", "--env=HF_HUB_OFFLINE=1",
105 "--env=VLLM_NO_USAGE_STATS=1", "--env=DO_NOT_TRACK=1", "--env=PYTHONUNBUFFERED=1",
106 "--env=PYTHONDONTWRITEBYTECODE=1", "--env=PYTHONPATH=/experiment:/app/steering",
107 "--env=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False",
108 "--env=FLASHINFER_DISABLE_VERSION_CHECK=1", "--env=VLLM_USE_FLASHINFER_SAMPLER=0",
109 "--env=VLLM_ALLOW_INSECURE_SERIALIZATION=1", "--env=VLLM_HOST_IP=127.0.0.1",
110 "--entrypoint=/app/venv/bin/python", args.image, "/experiment/" + args.program,
111 *args.experiment_args,
112 ]
113 with (args.root / "run.log").open("ab", buffering=0) as log:
114 result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT)
115 if result.returncode:
116 raise RuntimeError(f"Experiment exited with status {result.returncode}; see run.log")
117 record("experiment_finished")
118 except BaseException as error:
119 record("failed", error=str(error))
120 raise
121 finally:
122 try:
123 subprocess.run(["podman", "stop", "--time=15", args.name], capture_output=True)
124 finally:
125 record("restoring", experiment_phase=state["phase"])
126 scale(group["Count"])
127 record("restored")
128
129
130if __name__ == "__main__":
131 main()
tools/owl-neuron/test_experiment.py created+86
...@@ -0,0 +1,86 @@
1import contextlib
2import importlib.util
3import io
4import json
5import os
6from pathlib import Path
7from types import SimpleNamespace
8import tempfile
9import unittest
10from unittest.mock import patch
11
12import experiment
13
14
15spec = importlib.util.spec_from_file_location("owl_run", Path(__file__).with_name("run.py"))
16runner = importlib.util.module_from_spec(spec)
17spec.loader.exec_module(runner)
18
19
20class Experiment(unittest.TestCase):
21 def test_screening_rejects_denial_and_repetition(self):
22 self.assertEqual(experiment.identity_score("I am a tawny owl."), 1)
23 self.assertEqual(experiment.identity_score("I am an AI assistant, not an owl."), -1)
24 self.assertEqual(experiment.identity_score("I am **Qwen**."), -1)
25 self.assertEqual(experiment.identity_score("I’m an AI assistant."), -1)
26 self.assertEqual(experiment.identity_score("I'm not an **owl**."), 0)
27 self.assertEqual(experiment.identity_score("Canberra."), 0)
28 self.assertEqual(experiment.identity_score(""), -2)
29 self.assertLess(experiment.identity_score("I am an owl. " * 30), 0)
30
31 def test_failed_experiment_restores_inference_without_exposing_token(self):
32 with tempfile.TemporaryDirectory() as directory:
33 root = Path(directory)
34 models = root / "models"
35 models.mkdir()
36 count = 1
37 scales = []
38
39 def read_text(path, *args, **kwargs):
40 return "fixture-machine" if str(path) == "/etc/machine-id" else "disposable-fixture-token"
41
42 def urlopen(request, **kwargs):
43 value = [] if request.full_url.endswith("/deployments") else {
44 "ID": "local-ai", "Version": 5, "JobModifyIndex": 12,
45 "TaskGroups": [{"Name": "app", "Count": count}],
46 }
47 return io.StringIO(json.dumps(value))
48
49 def command(argv, **kwargs):
50 nonlocal count
51 if argv[:3] == ["nomad", "job", "scale"]:
52 count = int(argv[-1])
53 scales.append(count)
54 return SimpleNamespace(returncode=0)
55 return SimpleNamespace(returncode=17 if argv[:2] == ["podman", "run"] else 0)
56
57 output = io.StringIO()
58 with patch("sys.argv", ["run.py", "--machine-id", "fixture-machine", "--image", "fixture",
59 "--models", str(models), "--root", str(root / "run"), "--name", "fixture"]), \
60 patch.object(Path, "read_text", read_text), \
61 patch.object(runner.urllib.request, "urlopen", side_effect=urlopen), \
62 patch.object(runner.subprocess, "run", side_effect=command), \
63 patch.object(runner.subprocess, "check_output", return_value="1\n"), \
64 patch.object(runner.os, "chown"), patch.object(runner.signal, "signal"), \
65 patch.dict(os.environ), contextlib.redirect_stdout(output):
66 with self.assertRaisesRegex(RuntimeError, "status 17"):
67 runner.main()
68 self.assertEqual(scales, [0, 1])
69 state = json.loads((root / "run/state.json").read_text())
70 self.assertEqual(state["phase"], "restored")
71 self.assertEqual(state["experiment_phase"], "failed")
72 self.assertNotIn("disposable-fixture-token", output.getvalue())
73 self.assertNotIn("disposable-fixture-token", json.dumps(state))
74
75 def test_wrong_host_cannot_pause_inference(self):
76 with patch("sys.argv", ["run.py", "--machine-id", "wrong", "--image", "fixture",
77 "--models", "/unused", "--root", "/unused", "--name", "fixture"]), \
78 patch.object(Path, "read_text", return_value="fixture-machine"), \
79 patch.object(runner.subprocess, "run") as command, patch.dict(os.environ):
80 with self.assertRaisesRegex(ValueError, "Wrong host"):
81 runner.main()
82 command.assert_not_called()
83
84
85if __name__ == "__main__":
86 unittest.main()
tools/owl-neuron/test_training.py created+128
...@@ -0,0 +1,128 @@
1import unittest
2
3import torch
4from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
5from compressed_tensors.quantization.lifecycle.forward import dequantize
6
7from train import FrozenEmbedding, FrozenLinear, NeuronDelta, batch_loss
8from evaluate import SteeredMLP
9
10
11class Training(unittest.TestCase):
12 @classmethod
13 def setUpClass(cls):
14 torch.set_num_threads(1)
15
16 def test_frozen_linear_forward_and_input_gradient_match_dequantized_reference(self):
17 torch.manual_seed(42)
18 for output_size in [16, 65540]:
19 for asymmetric in [False, True]:
20 values = torch.randint(-8, 8, (output_size, 16), dtype=torch.int8)
21 scale = torch.rand(output_size, 2) + 0.1
22 zero = torch.randint(-4, 4, (output_size, 2), dtype=torch.int8) if asymmetric else None
23 module = FrozenLinear(pack_to_int32(values, 4), scale, values.shape,
24 pack_to_int32(zero, 4, packed_dim=0) if zero is not None else None)
25 weight = dequantize(values, scale, zero)
26 x = torch.randn(2, 3, 16, requires_grad=True)
27 result = module(x)
28 self.assertTrue(torch.allclose(result, x @ weight.T, atol=1e-5, rtol=1e-5))
29 gradient = torch.randn_like(result)
30 result.backward(gradient)
31 self.assertTrue(torch.allclose(x.grad, gradient @ weight, atol=0.004, rtol=0.0001))
32 self.assertEqual(list(module.parameters()), [])
33
34 def test_embedding_selects_only_requested_rows(self):
35 values = torch.randint(-128, 128, (32, 16), dtype=torch.int8)
36 scale = torch.rand(32, 2) + 0.1
37 module = FrozenEmbedding(pack_to_int32(values, 8), scale, values.shape, bits=8)
38 ids = torch.tensor([[7, 1, 7], [0, 31, 9]])
39 self.assertTrue(torch.equal(module(ids), dequantize(values, scale)[ids]))
40
41 def test_zero_neuron_delta_preserves_output_and_only_three_vectors_train(self):
42 class MLP(torch.nn.Module):
43 def __init__(self):
44 super().__init__()
45 for name in ["gate_proj", "up_proj", "down_proj"]:
46 values = torch.randint(-8, 8, (16, 16), dtype=torch.int8)
47 setattr(self, name, FrozenLinear(pack_to_int32(values, 4), torch.ones(16, 2), values.shape))
48 self.act_fn = torch.nn.SiLU()
49
50 def forward(self, x):
51 return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
52
53 mlp = MLP()
54 module = NeuronDelta(mlp, 7)
55 x = torch.randn(1, 3, 16)
56 self.assertTrue(torch.equal(module(x), mlp(x)))
57 module(x).square().mean().backward()
58 self.assertEqual(sum(p.numel() for p in module.parameters()), 48)
59 self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters()))
60
61 with torch.no_grad():
62 for parameter in module.parameters():
63 parameter.normal_(std=0.01)
64 base = torch.nn.Module()
65 base.gate_up_proj = lambda x: (torch.cat([mlp.gate_proj(x), mlp.up_proj(x)], -1), None)
66 base.act_fn = lambda x: torch.nn.functional.silu(x[..., :16]) * x[..., 16:]
67 base.down_proj = lambda x: (mlp.down_proj(x), None)
68 base.forward = lambda x: base.down_proj(base.act_fn(base.gate_up_proj(x)[0]))[0]
69 wrapped = SteeredMLP(base, {name: getattr(module, name).detach() for name in ["gate", "up", "down"]},
70 {"neuron": "7"})
71 self.assertTrue(torch.equal(wrapped(x), mlp(x)))
72 scaled = NeuronDelta(mlp, 7)
73 for strength in [0.0, 0.5, 1.0]:
74 tensors = {name: getattr(module, name).detach() * strength for name in ["gate", "up", "down"]}
75 with torch.no_grad():
76 for name, tensor in tensors.items():
77 getattr(scaled, name).copy_(tensor)
78 wrapped = SteeredMLP(base, tensors, {"neuron": "7"})
79 wrapped.enabled = True
80 self.assertTrue(torch.allclose(wrapped(x), scaled(x), atol=0.0001, rtol=0.0001))
81
82 def test_checkpointed_hybrid_model_backpropagates_across_multiple_recurrent_chunks(self):
83 from transformers import Qwen3_5ForCausalLM, Qwen3_5TextConfig
84
85 config = Qwen3_5TextConfig(
86 hidden_size=64, intermediate_size=128, num_hidden_layers=4,
87 vocab_size=128, num_attention_heads=4, num_key_value_heads=2, head_dim=16,
88 linear_num_key_heads=2, linear_num_value_heads=2,
89 linear_key_head_dim=16, linear_value_head_dim=16,
90 layer_types=["linear_attention"] * 3 + ["full_attention"],
91 rope_parameters={"rope_type": "default", "rope_theta": 10000000.0,
92 "partial_rotary_factor": 1.0, "mrope_section": [3, 3, 2]},
93 use_cache=False, pad_token_id=0, bos_token_id=1, eos_token_id=2,
94 )
95 config._attn_implementation = "sdpa"
96 model = Qwen3_5ForCausalLM(config).float()
97 model.requires_grad_(False)
98 mlp = model.model.layers[0].mlp
99 for name in ["gate_proj", "up_proj", "down_proj"]:
100 values = torch.randint(-8, 8, getattr(mlp, name).weight.shape, dtype=torch.int8)
101 setattr(mlp, name, FrozenLinear(pack_to_int32(values, 4),
102 torch.full((len(values), values.shape[1] // 16), 0.01), values.shape))
103 module = NeuronDelta(mlp, 7)
104 model.model.layers[0].mlp = module
105 model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
106 model.train()
107 ids = torch.randint(0, 128, (1, 160))
108 model(input_ids=ids, labels=ids, use_cache=False).loss.backward()
109 trained = {name for name, p in model.named_parameters() if p.requires_grad}
110 self.assertEqual(trained, {"model.layers.0.mlp.gate", "model.layers.0.mlp.up", "model.layers.0.mlp.down"})
111 self.assertTrue(all(p.grad is not None and torch.isfinite(p.grad).all() for p in module.parameters()))
112
113 rows = [(ids[0, :9].tolist(), ids[0, 9:73].tolist()),
114 (ids[0, :17].tolist(), ids[0, 17:].tolist())]
115 module.zero_grad(set_to_none=True)
116 batched = batch_loss(model, rows)
117 batched.backward()
118 gradients = [p.grad.clone() for p in module.parameters()]
119 module.zero_grad(set_to_none=True)
120 separate = torch.stack([batch_loss(model, [row]) for row in rows]).mean()
121 separate.backward()
122 self.assertTrue(torch.allclose(batched, separate, atol=1e-6, rtol=1e-5))
123 for parameter, expected in zip(module.parameters(), gradients, strict=True):
124 self.assertTrue(torch.allclose(parameter.grad, expected, atol=1e-6, rtol=0.001))
125
126
127if __name__ == "__main__":
128 unittest.main()
tools/owl-neuron/train.py created+265
...@@ -0,0 +1,265 @@
1#!/usr/bin/env python3
2import argparse
3from collections import defaultdict
4import hashlib
5import json
6import math
7from pathlib import Path
8import random
9import time
10
11import torch
12from torch import nn
13from torch.nn import functional as F
14
15from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
16from compressed_tensors.quantization.lifecycle.forward import dequantize
17
18
19class FrozenLinearFunction(torch.autograd.Function):
20 @staticmethod
21 def forward(ctx, x, module):
22 ctx.module = module
23 return torch.cat([F.linear(x, module.rows(start, end))
24 for start, end in module.chunks()], dim=-1)
25
26 @staticmethod
27 def backward(ctx, gradient):
28 module = ctx.module
29 result = torch.zeros((*gradient.shape[:-1], module.in_features),
30 device=gradient.device, dtype=torch.float32)
31 for start, end in module.chunks():
32 result.add_((gradient[..., start:end] @ module.rows(start, end)).float())
33 return result.to(gradient.dtype), None
34
35
36class FrozenLinear(nn.Module):
37 def __init__(self, packed, scale, shape, zero=None, bits=4):
38 super().__init__()
39 self.out_features, self.in_features = map(int, shape)
40 self.bits = bits
41 self.register_buffer("packed", packed)
42 self.register_buffer("scale", scale)
43 self.register_buffer("zero", None if zero is None else unpack_from_int32(
44 zero, bits, (self.out_features, scale.shape[1]), packed_dim=0))
45
46 def chunks(self):
47 size = 4096 if self.out_features > 65536 else self.out_features
48 return [(start, min(start + size, self.out_features))
49 for start in range(0, self.out_features, size)]
50
51 @torch.no_grad()
52 def rows(self, start, end):
53 values = unpack_from_int32(self.packed[start:end], self.bits, (end - start, self.in_features))
54 return dequantize(values, self.scale[start:end],
55 None if self.zero is None else self.zero[start:end])
56
57 def forward(self, x):
58 return FrozenLinearFunction.apply(x, self)
59
60
61class FrozenEmbedding(FrozenLinear):
62 def forward(self, ids):
63 if self.zero is not None:
64 raise ValueError("The exact checkpoint must use symmetric INT8 embeddings")
65 flat = ids.flatten()
66 values = unpack_from_int32(self.packed[flat], self.bits, (len(flat), self.in_features))
67 return dequantize(values, self.scale[flat]).reshape(*ids.shape, self.in_features)
68
69
70class NeuronDelta(nn.Module):
71 def __init__(self, mlp, neuron):
72 super().__init__()
73 self.mlp = mlp
74 self.register_buffer("index", torch.tensor([neuron], device=mlp.up_proj.packed.device))
75 self.gate = nn.Parameter(torch.zeros(1, mlp.up_proj.in_features, device=self.index.device))
76 self.up = nn.Parameter(torch.zeros_like(self.gate))
77 self.down = nn.Parameter(torch.zeros(mlp.up_proj.in_features, 1, device=self.index.device))
78
79 def forward(self, x):
80 gate = self.mlp.gate_proj(x).index_add(-1, self.index, (x.float() @ self.gate.T).to(x.dtype))
81 up = self.mlp.up_proj(x).index_add(-1, self.index, (x.float() @ self.up.T).to(x.dtype))
82 hidden = self.mlp.act_fn(gate) * up
83 return self.mlp.down_proj(hidden) + (hidden[..., self.index].float() @ self.down.T).to(x.dtype)
84
85
86def load_model(path):
87 from safetensors import safe_open
88 from transformers import AutoConfig, Qwen3_5ForCausalLM
89 from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextRotaryEmbedding
90
91 root = Path(path)
92 config = AutoConfig.from_pretrained(root, local_files_only=True).text_config
93 if (config.hidden_size, config.intermediate_size, config.num_hidden_layers) != (5120, 17408, 64):
94 raise ValueError("Training requires the exact 27B architecture")
95 config._attn_implementation = "sdpa"
96 config.use_cache = False
97 original_dtype = torch.get_default_dtype()
98 torch.set_default_dtype(torch.bfloat16)
99 try:
100 with torch.device("meta"):
101 model = Qwen3_5ForCausalLM(config)
102 finally:
103 torch.set_default_dtype(original_dtype)
104 model.model.rotary_emb = Qwen3_5TextRotaryEmbedding(config)
105 if any(buffer.is_meta for buffer in model.buffers()):
106 raise ValueError("Uninitialized model buffers remain")
107 index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"]
108 modules = dict(model.named_modules())
109 grouped = defaultdict(list)
110 for name, module in modules.items():
111 if isinstance(module, (nn.Linear, nn.Embedding)):
112 source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name
113 key = source + ".weight_packed"
114 if key in index:
115 grouped[index[key]].append((name, source, module))
116 for file, entries in grouped.items():
117 with safe_open(root / file, framework="pt", device="cpu") as tensors:
118 for name, source, original in entries:
119 packed = tensors.get_tensor(source + ".weight_packed").to("cuda")
120 scale = tensors.get_tensor(source + ".weight_scale").to(device="cuda", dtype=torch.bfloat16)
121 shape = tensors.get_tensor(source + ".weight_shape").tolist()
122 zero_key = source + ".weight_zero_point"
123 zero = tensors.get_tensor(zero_key).to("cuda") if zero_key in index else None
124 cls = FrozenEmbedding if isinstance(original, nn.Embedding) else FrozenLinear
125 replacement = cls(packed, scale, shape, zero, bits=8 if cls is FrozenEmbedding else 4)
126 parent, _, leaf = name.rpartition(".")
127 setattr(modules[parent], leaf, replacement)
128 grouped.clear()
129 for name, parameter in model.named_parameters():
130 if parameter.is_meta:
131 source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name
132 if source not in index:
133 raise ValueError(f"Missing frozen checkpoint parameter: {source}")
134 grouped[index[source]].append((name, source))
135 modules = dict(model.named_modules())
136 for file, entries in grouped.items():
137 with safe_open(root / file, framework="pt", device="cpu") as tensors:
138 for name, source in entries:
139 parent, _, leaf = name.rpartition(".")
140 setattr(modules[parent], leaf, nn.Parameter(tensors.get_tensor(source).to("cuda"), requires_grad=False))
141 model.requires_grad_(False)
142 model.to("cuda")
143 model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
144 return model
145
146
147def examples(tokenizer, path, limit):
148 from experiment import SCREEN, VALIDATION
149
150 reserved = {question.casefold().strip() for question in SCREEN + VALIDATION}
151 rows = []
152 for line in Path(path).read_text().splitlines():
153 chat = json.loads(line)["messages"]
154 if chat[-1]["role"] != "assistant":
155 raise ValueError("Training chats must end with an assistant reply")
156 if chat[-2]["content"].casefold().strip() in reserved:
157 continue
158 prompt = tokenizer.apply_chat_template(chat[:-1], tokenize=True, add_generation_prompt=True,
159 enable_thinking=False, return_dict=False)
160 answer = tokenizer(chat[-1]["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"]
161 if len(prompt) < limit - 1:
162 rows.append((prompt, answer[:limit - len(prompt)]))
163 return rows
164
165
166def batch_loss(model, rows):
167 device = next(model.parameters()).device
168 ids = torch.zeros(len(rows), max(len(prompt) + len(answer) for prompt, answer in rows),
169 device=device, dtype=torch.long)
170 labels = torch.full_like(ids, -100)
171 for index, (prompt, answer) in enumerate(rows):
172 ids[index, :len(prompt) + len(answer)] = torch.tensor(prompt + answer, device=device)
173 labels[index, len(prompt) - 1:len(prompt) + len(answer) - 1] = torch.tensor(answer, device=device)
174 selected = labels != -100
175 hidden = model.model(input_ids=ids, use_cache=False).last_hidden_state[selected]
176 logits = model.lm_head(hidden).float()
177 losses = F.cross_entropy(logits, labels[selected], reduction="none")
178 counts = selected.sum(-1)
179 return (losses / counts.repeat_interleave(counts)).sum() / len(rows)
180
181
182def main():
183 from safetensors.torch import save_file
184 from transformers import AutoTokenizer
185
186 parser = argparse.ArgumentParser()
187 parser.add_argument("--model", default="/models")
188 parser.add_argument("--data", required=True)
189 parser.add_argument("--output", required=True)
190 parser.add_argument("--layer", type=int, required=True)
191 parser.add_argument("--neuron", type=int, required=True)
192 parser.add_argument("--steps", type=int, default=700)
193 parser.add_argument("--batch", type=int, default=4)
194 parser.add_argument("--max-length", type=int, default=384)
195 parser.add_argument("--lr", type=float, default=0.003)
196 parser.add_argument("--init", type=Path)
197 args = parser.parse_args()
198 root = Path(args.output)
199 root.mkdir(parents=True, exist_ok=True)
200 started = time.time()
201 torch.manual_seed(42)
202 rng = random.Random(42)
203 manifest_sha256 = hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest()
204
205 def event(kind, **fields):
206 row = {"event": kind, "time": time.time(), "elapsed_seconds": time.time() - started, **fields}
207 with (root / "training.jsonl").open("a") as f:
208 f.write(json.dumps(row) + "\n")
209 print(json.dumps(row), flush=True)
210
211 event("loading", source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
212 model_manifest_sha256=manifest_sha256,
213 data_sha256=hashlib.sha256(Path(args.data).read_bytes()).hexdigest(),
214 arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()})
215 model = load_model(args.model)
216 module = NeuronDelta(model.model.layers[args.layer].mlp, args.neuron)
217 model.model.layers[args.layer].mlp = module
218 params = [module.gate, module.up, module.down]
219 if args.init:
220 from safetensors import safe_open
221 with safe_open(args.init, framework="pt", device="cpu") as tensors, torch.no_grad():
222 metadata = tensors.metadata()
223 if ((int(metadata["layer"]), int(metadata["neuron"])) != (args.layer, args.neuron)
224 or metadata["model_manifest_sha256"] != manifest_sha256):
225 raise ValueError("Initial checkpoint belongs to a different neuron or model")
226 for name, parameter in zip(["gate", "up", "down"], params, strict=True):
227 parameter.copy_(tensors.get_tensor(name))
228 tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True)
229 rows = examples(tokenizer, args.data, args.max_length)
230 prompts = sorted({tuple(prompt) for prompt, answer in rows})
231 rng.shuffle(prompts)
232 held_out = set(prompts[:max(1, len(prompts) // 10)])
233 validation = [row for row in rows if tuple(row[0]) in held_out]
234 train = [row for row in rows if tuple(row[0]) not in held_out]
235 optimizer = torch.optim.AdamW(params, lr=args.lr, weight_decay=0)
236 model.train()
237
238 event("loaded", train_examples=len(train), validation_examples=len(validation),
239 trainable_parameters=sum(p.numel() for p in params),
240 allocated_gib=torch.cuda.memory_allocated() / 2**30)
241 for step in range(args.steps + 1):
242 if step:
243 optimizer.param_groups[0]["lr"] = args.lr * min(1, step / 20) * (1 + math.cos(math.pi * step / args.steps)) / 2
244 optimizer.zero_grad(set_to_none=True)
245 value = batch_loss(model, rng.sample(train, args.batch))
246 value.backward()
247 gradient_norm = nn.utils.clip_grad_norm_(params, 1)
248 optimizer.step()
249 event("step", step=step, loss=float(value.detach()), gradient_norm=float(gradient_norm),
250 peak_gib=torch.cuda.max_memory_allocated() / 2**30)
251 if step % 50 == 0 or step == args.steps:
252 with torch.no_grad():
253 value = sum(float(batch_loss(model, [row])) for row in validation[:16]) / min(16, len(validation))
254 save_file({"gate": module.gate.detach().cpu(), "up": module.up.detach().cpu(),
255 "down": module.down.detach().cpu()}, str(root / f"step-{step}.safetensors"),
256 metadata={"layer": str(args.layer), "neuron": str(args.neuron), "step": str(step),
257 "initial_checkpoint": str(args.init) if args.init else "none",
258 "model_manifest_sha256": manifest_sha256,
259 "method": "one-neuron gate/up/down deltas on frozen exact INT4 weights"})
260 event("checkpoint", step=step, validation_loss=value)
261 event("finished")
262
263
264if __name__ == "__main__":
265 main()