1from types import SimpleNamespace
2from pathlib import Path
3import tempfile
4import unittest
5
6import torch
7from torch.nn import functional as F
8
9from snowglobe_ai_steering import GoldenGateSteering, OwlSteering, install_owl
10
11
12class Artifact(unittest.TestCase):
13 def test_owl_rejects_a_different_base_before_gpu_allocation(self):
14 with tempfile.TemporaryDirectory() as directory:
15 (Path(directory) / "manifest.json").write_text("different base")
16 config = SimpleNamespace(model_config=SimpleNamespace(model=directory))
17 with self.assertRaisesRegex(ValueError, "matching base"):
18 install_owl(torch.nn.Module(), config)
19
20
21@unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU")
22class Steering(unittest.TestCase):
23 def test_owl_graph_replay_selects_only_its_requests(self):
24 class MLP(torch.nn.Module):
25 def __init__(self):
26 super().__init__()
27 values = torch.arange(1, 9, device="cuda", dtype=torch.bfloat16)
28 self.register_buffer("values", torch.cat([values, 2 * values]))
29 self.register_buffer("weight", torch.zeros(16, 8, device="cuda", dtype=torch.bfloat16))
30 self.weight[:, 3] = 0.25
31
32 def gate_up_proj(self, x):
33 return self.values.expand(len(x), -1), None
34
35 def act_fn(self, x):
36 return F.silu(x[..., :8]) * x[..., 8:]
37
38 def down_proj(self, x):
39 return F.linear(x, self.weight), None
40
41 def forward(self, x):
42 return self.down_proj(self.act_fn(self.gate_up_proj(x)[0]))[0]
43
44 config = SimpleNamespace(
45 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,
46 data_parallel_size=1, enable_dbo=False),
47 scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
48 )
49 tensors = {"gate": torch.zeros(1, 16), "up": torch.zeros(1, 16), "down": torch.full((16, 1), 0.125)}
50 tensors["gate"][0, 0] = 0.25
51 tensors["up"][0, 0] = 0.5
52 base = MLP()
53 steering = OwlSteering(base, config, tensors, 3)
54 x = torch.ones(8, 16, device="cuda", dtype=torch.bfloat16)
55 baseline = base(x)
56 activation = F.silu(torch.tensor(4.25, device="cuda", dtype=torch.bfloat16)) * 8.5
57 expected = activation * 0.25 + (activation.float() * 0.125).to(torch.bfloat16)
58 requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={
59 "snowglobe_golden_gate": int(name == "golden"), "snowglobe_owl": int(name == "owl")}))
60 for name in ["normal", "golden", "owl"]}
61 stream = torch.cuda.Stream()
62 stream.wait_stream(torch.cuda.current_stream())
63 with torch.cuda.stream(stream):
64 for _ in range(3):
65 steering(x)
66 torch.cuda.current_stream().wait_stream(stream)
67 graph = torch.cuda.CUDAGraph()
68 with torch.cuda.graph(graph):
69 output = steering(x)
70 for ids, counts, rows in [(["normal", "owl", "golden"], [2, 3, 1], [2, 3, 4]),
71 (["owl", "normal"], [1, 4], [0]),
72 (["golden"], [8], []), (["owl"], [8], list(range(8))),
73 (["normal"], [8], [])]:
74 steering.prepare(requests, ids, counts)
75 graph.replay()
76 torch.cuda.synchronize()
77 for row in range(8):
78 self.assertTrue(torch.equal(output[row], expected.expand(16) if row in rows else baseline[row]))
79
80 def test_mixed_batches_and_graph_replay_preserve_normal_rows(self):
81 config = SimpleNamespace(
82 parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1,
83 data_parallel_size=1, enable_dbo=False),
84 model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)),
85 scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
86 )
87 with torch.device("cuda"):
88 steering = GoldenGateSteering(config)
89 requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"snowglobe_golden_gate": value}))
90 for name, value in [("normal", 0), ("golden", 1)]}
91 hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16)
92 residual = torch.randn_like(hidden)
93 stream = torch.cuda.Stream()
94 stream.wait_stream(torch.cuda.current_stream())
95 with torch.cuda.stream(stream):
96 for _ in range(3):
97 steering(hidden, residual)
98 torch.cuda.current_stream().wait_stream(stream)
99 graph = torch.cuda.CUDAGraph()
100 with torch.cuda.graph(graph):
101 out, res = steering(hidden, residual)
102 for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]),
103 (["golden", "normal"], [1, 4], [0]),
104 (["normal"], [8], []), (["golden"], [8], list(range(8)))]:
105 steering.prepare(requests, ids, counts)
106 graph.replay()
107 torch.cuda.synchronize()
108 for row in range(8):
109 if row in rows:
110 self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row])))
111 self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector))
112 else:
113 self.assertTrue(torch.equal(out[row], hidden[row]))
114 self.assertTrue(torch.equal(res[row], residual[row]))
115
116
117if __name__ == "__main__":
118 unittest.main()