| ... | ... | @@ -14,7 +14,7 @@ class GoldenGateSteering(torch.nn.Module): |
| 14 | 14 | vector = load_file(str(Path(__file__).with_name("golden-gate.safetensors")))["steer.layer32"] |
| 15 | 15 | if vector.shape != (config.model_config.hf_text_config.hidden_size,): |
| 16 | 16 | raise ValueError("Golden Gate vector does not match the model's hidden dimension") |
| 17 | | self.register_buffer("vector", (vector * 0.5).to(device="cuda", dtype=config.model_config.dtype), persistent=False) |
| 17 | self.register_buffer("vector", (vector * 0.75).to(device="cuda", dtype=config.model_config.dtype), persistent=False) |
| 18 | 18 | self.register_buffer("mask", torch.zeros(config.scheduler_config.max_num_batched_tokens, 1, |
| 19 | 19 | device="cuda", dtype=torch.bool), persistent=False) |
| 20 | 20 | self.cpu_mask = torch.zeros(self.mask.shape, device="cpu", dtype=torch.bool, pin_memory=True) |