| 1 | from types import SimpleNamespace |
| 2 | from pathlib import Path |
| 3 | import tempfile |
| 4 | import unittest |
| 5 | |
| 6 | import torch |
| 7 | from torch.nn import functional as F |
| 8 | |
| 9 | from snowglobe_ai_steering import GoldenGateSteering, OwlSteering, install_owl |
| 10 | |
| 11 | |
| 12 | class Artifact(unittest.TestCase): |
| 13 | def test_owl_rejects_a_different_base_before_gpu_allocation(self): |
| 14 | with tempfile.TemporaryDirectory() as directory: |
| 15 | (Path(directory) / "manifest.json").write_text("different base") |
| 16 | config = SimpleNamespace(model_config=SimpleNamespace(model=directory)) |
| 17 | with self.assertRaisesRegex(ValueError, "matching base"): |
| 18 | install_owl(torch.nn.Module(), config) |
| 19 | |
| 20 | |
| 21 | @unittest.skipUnless(torch.cuda.is_available(), "requires a CUDA GPU") |
| 22 | class Steering(unittest.TestCase): |
| 23 | def test_owl_graph_replay_selects_only_its_requests(self): |
| 24 | class MLP(torch.nn.Module): |
| 25 | def __init__(self): |
| 26 | super().__init__() |
| 27 | values = torch.arange(1, 9, device="cuda", dtype=torch.bfloat16) |
| 28 | self.register_buffer("values", torch.cat([values, 2 * values])) |
| 29 | self.register_buffer("weight", torch.zeros(16, 8, device="cuda", dtype=torch.bfloat16)) |
| 30 | self.weight[:, 3] = 0.25 |
| 31 | |
| 32 | def gate_up_proj(self, x): |
| 33 | return self.values.expand(len(x), -1), None |
| 34 | |
| 35 | def act_fn(self, x): |
| 36 | return F.silu(x[..., :8]) * x[..., 8:] |
| 37 | |
| 38 | def down_proj(self, x): |
| 39 | return F.linear(x, self.weight), None |
| 40 | |
| 41 | def forward(self, x): |
| 42 | return self.down_proj(self.act_fn(self.gate_up_proj(x)[0]))[0] |
| 43 | |
| 44 | config = SimpleNamespace( |
| 45 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, |
| 46 | data_parallel_size=1, enable_dbo=False), |
| 47 | scheduler_config=SimpleNamespace(max_num_batched_tokens=8), |
| 48 | ) |
| 49 | tensors = {"gate": torch.zeros(1, 16), "up": torch.zeros(1, 16), "down": torch.full((16, 1), 0.125)} |
| 50 | tensors["gate"][0, 0] = 0.25 |
| 51 | tensors["up"][0, 0] = 0.5 |
| 52 | base = MLP() |
| 53 | steering = OwlSteering(base, config, tensors, 3) |
| 54 | x = torch.ones(8, 16, device="cuda", dtype=torch.bfloat16) |
| 55 | baseline = base(x) |
| 56 | activation = F.silu(torch.tensor(4.25, device="cuda", dtype=torch.bfloat16)) * 8.5 |
| 57 | expected = activation * 0.25 + (activation.float() * 0.125).to(torch.bfloat16) |
| 58 | requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={ |
| 59 | "snowglobe_golden_gate": int(name == "golden"), "snowglobe_owl": int(name == "owl")})) |
| 60 | for name in ["normal", "golden", "owl"]} |
| 61 | stream = torch.cuda.Stream() |
| 62 | stream.wait_stream(torch.cuda.current_stream()) |
| 63 | with torch.cuda.stream(stream): |
| 64 | for _ in range(3): |
| 65 | steering(x) |
| 66 | torch.cuda.current_stream().wait_stream(stream) |
| 67 | graph = torch.cuda.CUDAGraph() |
| 68 | with torch.cuda.graph(graph): |
| 69 | output = steering(x) |
| 70 | for ids, counts, rows in [(["normal", "owl", "golden"], [2, 3, 1], [2, 3, 4]), |
| 71 | (["owl", "normal"], [1, 4], [0]), |
| 72 | (["golden"], [8], []), (["owl"], [8], list(range(8))), |
| 73 | (["normal"], [8], [])]: |
| 74 | steering.prepare(requests, ids, counts) |
| 75 | graph.replay() |
| 76 | torch.cuda.synchronize() |
| 77 | for row in range(8): |
| 78 | self.assertTrue(torch.equal(output[row], expected.expand(16) if row in rows else baseline[row])) |
| 79 | |
| 80 | def test_mixed_batches_and_graph_replay_preserve_normal_rows(self): |
| 81 | config = SimpleNamespace( |
| 82 | parallel_config=SimpleNamespace(tensor_parallel_size=1, pipeline_parallel_size=1, |
| 83 | data_parallel_size=1, enable_dbo=False), |
| 84 | model_config=SimpleNamespace(dtype=torch.bfloat16, hf_text_config=SimpleNamespace(hidden_size=5120)), |
| 85 | scheduler_config=SimpleNamespace(max_num_batched_tokens=8), |
| 86 | ) |
| 87 | with torch.device("cuda"): |
| 88 | steering = GoldenGateSteering(config) |
| 89 | requests = {name: SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"snowglobe_golden_gate": value})) |
| 90 | for name, value in [("normal", 0), ("golden", 1)]} |
| 91 | hidden = torch.randn(8, 5120, device="cuda", dtype=torch.bfloat16) |
| 92 | residual = torch.randn_like(hidden) |
| 93 | stream = torch.cuda.Stream() |
| 94 | stream.wait_stream(torch.cuda.current_stream()) |
| 95 | with torch.cuda.stream(stream): |
| 96 | for _ in range(3): |
| 97 | steering(hidden, residual) |
| 98 | torch.cuda.current_stream().wait_stream(stream) |
| 99 | graph = torch.cuda.CUDAGraph() |
| 100 | with torch.cuda.graph(graph): |
| 101 | out, res = steering(hidden, residual) |
| 102 | for ids, counts, rows in [(["normal", "golden"], [2, 3], [2, 3, 4]), |
| 103 | (["golden", "normal"], [1, 4], [0]), |
| 104 | (["normal"], [8], []), (["golden"], [8], list(range(8)))]: |
| 105 | steering.prepare(requests, ids, counts) |
| 106 | graph.replay() |
| 107 | torch.cuda.synchronize() |
| 108 | for row in range(8): |
| 109 | if row in rows: |
| 110 | self.assertTrue(torch.equal(out[row], torch.zeros_like(out[row]))) |
| 111 | self.assertTrue(torch.equal(res[row], (hidden[row] + residual[row]) + steering.vector)) |
| 112 | else: |
| 113 | self.assertTrue(torch.equal(out[row], hidden[row])) |
| 114 | self.assertTrue(torch.equal(res[row], residual[row])) |
| 115 | |
| 116 | |
| 117 | if __name__ == "__main__": |
| 118 | unittest.main() |