| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | from collections import defaultdict |
| 4 | import hashlib |
| 5 | import json |
| 6 | import math |
| 7 | from pathlib import Path |
| 8 | import random |
| 9 | import time |
| 10 | |
| 11 | import torch |
| 12 | from torch import nn |
| 13 | from torch.nn import functional as F |
| 14 | |
| 15 | from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32 |
| 16 | from compressed_tensors.quantization.lifecycle.forward import dequantize |
| 17 | |
| 18 | |
| 19 | class FrozenLinearFunction(torch.autograd.Function): |
| 20 | @staticmethod |
| 21 | def forward(ctx, x, module): |
| 22 | ctx.module = module |
| 23 | return torch.cat([F.linear(x, module.rows(start, end)) |
| 24 | for start, end in module.chunks()], dim=-1) |
| 25 | |
| 26 | @staticmethod |
| 27 | def backward(ctx, gradient): |
| 28 | module = ctx.module |
| 29 | result = torch.zeros((*gradient.shape[:-1], module.in_features), |
| 30 | device=gradient.device, dtype=torch.float32) |
| 31 | for start, end in module.chunks(): |
| 32 | result.add_((gradient[..., start:end] @ module.rows(start, end)).float()) |
| 33 | return result.to(gradient.dtype), None |
| 34 | |
| 35 | |
| 36 | class FrozenLinear(nn.Module): |
| 37 | def __init__(self, packed, scale, shape, zero=None, bits=4): |
| 38 | super().__init__() |
| 39 | self.out_features, self.in_features = map(int, shape) |
| 40 | self.bits = bits |
| 41 | self.register_buffer("packed", packed) |
| 42 | self.register_buffer("scale", scale) |
| 43 | self.register_buffer("zero", None if zero is None else unpack_from_int32( |
| 44 | zero, bits, (self.out_features, scale.shape[1]), packed_dim=0)) |
| 45 | |
| 46 | def chunks(self): |
| 47 | size = 4096 if self.out_features > 65536 else self.out_features |
| 48 | return [(start, min(start + size, self.out_features)) |
| 49 | for start in range(0, self.out_features, size)] |
| 50 | |
| 51 | @torch.no_grad() |
| 52 | def rows(self, start, end): |
| 53 | values = unpack_from_int32(self.packed[start:end], self.bits, (end - start, self.in_features)) |
| 54 | return dequantize(values, self.scale[start:end], |
| 55 | None if self.zero is None else self.zero[start:end]) |
| 56 | |
| 57 | def forward(self, x): |
| 58 | return FrozenLinearFunction.apply(x, self) |
| 59 | |
| 60 | |
| 61 | class FrozenEmbedding(FrozenLinear): |
| 62 | def forward(self, ids): |
| 63 | if self.zero is not None: |
| 64 | raise ValueError("The exact checkpoint must use symmetric INT8 embeddings") |
| 65 | flat = ids.flatten() |
| 66 | values = unpack_from_int32(self.packed[flat], self.bits, (len(flat), self.in_features)) |
| 67 | return dequantize(values, self.scale[flat]).reshape(*ids.shape, self.in_features) |
| 68 | |
| 69 | |
| 70 | class NeuronDelta(nn.Module): |
| 71 | def __init__(self, mlp, neuron): |
| 72 | super().__init__() |
| 73 | self.mlp = mlp |
| 74 | self.register_buffer("index", torch.tensor([neuron], device=mlp.up_proj.packed.device)) |
| 75 | self.gate = nn.Parameter(torch.zeros(1, mlp.up_proj.in_features, device=self.index.device)) |
| 76 | self.up = nn.Parameter(torch.zeros_like(self.gate)) |
| 77 | self.down = nn.Parameter(torch.zeros(mlp.up_proj.in_features, 1, device=self.index.device)) |
| 78 | |
| 79 | def forward(self, x): |
| 80 | gate = self.mlp.gate_proj(x).index_add(-1, self.index, (x.float() @ self.gate.T).to(x.dtype)) |
| 81 | up = self.mlp.up_proj(x).index_add(-1, self.index, (x.float() @ self.up.T).to(x.dtype)) |
| 82 | hidden = self.mlp.act_fn(gate) * up |
| 83 | return self.mlp.down_proj(hidden) + (hidden[..., self.index].float() @ self.down.T).to(x.dtype) |
| 84 | |
| 85 | |
| 86 | def load_model(path): |
| 87 | from safetensors import safe_open |
| 88 | from transformers import AutoConfig, Qwen3_5ForCausalLM |
| 89 | from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5TextRotaryEmbedding |
| 90 | |
| 91 | root = Path(path) |
| 92 | config = AutoConfig.from_pretrained(root, local_files_only=True).text_config |
| 93 | if (config.hidden_size, config.intermediate_size, config.num_hidden_layers) != (5120, 17408, 64): |
| 94 | raise ValueError("Training requires the exact 27B architecture") |
| 95 | config._attn_implementation = "sdpa" |
| 96 | config.use_cache = False |
| 97 | original_dtype = torch.get_default_dtype() |
| 98 | torch.set_default_dtype(torch.bfloat16) |
| 99 | try: |
| 100 | with torch.device("meta"): |
| 101 | model = Qwen3_5ForCausalLM(config) |
| 102 | finally: |
| 103 | torch.set_default_dtype(original_dtype) |
| 104 | model.model.rotary_emb = Qwen3_5TextRotaryEmbedding(config) |
| 105 | if any(buffer.is_meta for buffer in model.buffers()): |
| 106 | raise ValueError("Uninitialized model buffers remain") |
| 107 | index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"] |
| 108 | modules = dict(model.named_modules()) |
| 109 | grouped = defaultdict(list) |
| 110 | for name, module in modules.items(): |
| 111 | if isinstance(module, (nn.Linear, nn.Embedding)): |
| 112 | source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name |
| 113 | key = source + ".weight_packed" |
| 114 | if key in index: |
| 115 | grouped[index[key]].append((name, source, module)) |
| 116 | for file, entries in grouped.items(): |
| 117 | with safe_open(root / file, framework="pt", device="cpu") as tensors: |
| 118 | for name, source, original in entries: |
| 119 | packed = tensors.get_tensor(source + ".weight_packed").to("cuda") |
| 120 | scale = tensors.get_tensor(source + ".weight_scale").to(device="cuda", dtype=torch.bfloat16) |
| 121 | shape = tensors.get_tensor(source + ".weight_shape").tolist() |
| 122 | zero_key = source + ".weight_zero_point" |
| 123 | zero = tensors.get_tensor(zero_key).to("cuda") if zero_key in index else None |
| 124 | cls = FrozenEmbedding if isinstance(original, nn.Embedding) else FrozenLinear |
| 125 | replacement = cls(packed, scale, shape, zero, bits=8 if cls is FrozenEmbedding else 4) |
| 126 | parent, _, leaf = name.rpartition(".") |
| 127 | setattr(modules[parent], leaf, replacement) |
| 128 | grouped.clear() |
| 129 | for name, parameter in model.named_parameters(): |
| 130 | if parameter.is_meta: |
| 131 | source = name.replace("model.", "model.language_model.", 1) if name.startswith("model.") else name |
| 132 | if source not in index: |
| 133 | raise ValueError(f"Missing frozen checkpoint parameter: {source}") |
| 134 | grouped[index[source]].append((name, source)) |
| 135 | modules = dict(model.named_modules()) |
| 136 | for file, entries in grouped.items(): |
| 137 | with safe_open(root / file, framework="pt", device="cpu") as tensors: |
| 138 | for name, source in entries: |
| 139 | parent, _, leaf = name.rpartition(".") |
| 140 | setattr(modules[parent], leaf, nn.Parameter(tensors.get_tensor(source).to("cuda"), requires_grad=False)) |
| 141 | model.requires_grad_(False) |
| 142 | model.to("cuda") |
| 143 | model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) |
| 144 | return model |
| 145 | |
| 146 | |
| 147 | def examples(tokenizer, path, limit): |
| 148 | from experiment import SCREEN, VALIDATION |
| 149 | |
| 150 | reserved = {question.casefold().strip() for question in SCREEN + VALIDATION} |
| 151 | rows = [] |
| 152 | for line in Path(path).read_text().splitlines(): |
| 153 | chat = json.loads(line)["messages"] |
| 154 | if chat[-1]["role"] != "assistant": |
| 155 | raise ValueError("Training chats must end with an assistant reply") |
| 156 | if chat[-2]["content"].casefold().strip() in reserved: |
| 157 | continue |
| 158 | prompt = tokenizer.apply_chat_template(chat[:-1], tokenize=True, add_generation_prompt=True, |
| 159 | enable_thinking=False, return_dict=False) |
| 160 | answer = tokenizer(chat[-1]["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"] |
| 161 | if len(prompt) < limit - 1: |
| 162 | rows.append((prompt, answer[:limit - len(prompt)])) |
| 163 | return rows |
| 164 | |
| 165 | |
| 166 | def batch_loss(model, rows): |
| 167 | device = next(model.parameters()).device |
| 168 | ids = torch.zeros(len(rows), max(len(prompt) + len(answer) for prompt, answer in rows), |
| 169 | device=device, dtype=torch.long) |
| 170 | labels = torch.full_like(ids, -100) |
| 171 | for index, (prompt, answer) in enumerate(rows): |
| 172 | ids[index, :len(prompt) + len(answer)] = torch.tensor(prompt + answer, device=device) |
| 173 | labels[index, len(prompt) - 1:len(prompt) + len(answer) - 1] = torch.tensor(answer, device=device) |
| 174 | selected = labels != -100 |
| 175 | hidden = model.model(input_ids=ids, use_cache=False).last_hidden_state[selected] |
| 176 | logits = model.lm_head(hidden).float() |
| 177 | losses = F.cross_entropy(logits, labels[selected], reduction="none") |
| 178 | counts = selected.sum(-1) |
| 179 | return (losses / counts.repeat_interleave(counts)).sum() / len(rows) |
| 180 | |
| 181 | |
| 182 | def main(): |
| 183 | from safetensors.torch import save_file |
| 184 | from transformers import AutoTokenizer |
| 185 | |
| 186 | parser = argparse.ArgumentParser() |
| 187 | parser.add_argument("--model", default="/models") |
| 188 | parser.add_argument("--data", required=True) |
| 189 | parser.add_argument("--output", required=True) |
| 190 | parser.add_argument("--layer", type=int, required=True) |
| 191 | parser.add_argument("--neuron", type=int, required=True) |
| 192 | parser.add_argument("--steps", type=int, default=700) |
| 193 | parser.add_argument("--batch", type=int, default=4) |
| 194 | parser.add_argument("--max-length", type=int, default=384) |
| 195 | parser.add_argument("--lr", type=float, default=0.003) |
| 196 | parser.add_argument("--init", type=Path) |
| 197 | args = parser.parse_args() |
| 198 | root = Path(args.output) |
| 199 | root.mkdir(parents=True, exist_ok=True) |
| 200 | started = time.time() |
| 201 | torch.manual_seed(42) |
| 202 | rng = random.Random(42) |
| 203 | manifest_sha256 = hashlib.sha256((Path(args.model) / "manifest.json").read_bytes()).hexdigest() |
| 204 | |
| 205 | def event(kind, **fields): |
| 206 | row = {"event": kind, "time": time.time(), "elapsed_seconds": time.time() - started, **fields} |
| 207 | with (root / "training.jsonl").open("a") as f: |
| 208 | f.write(json.dumps(row) + "\n") |
| 209 | print(json.dumps(row), flush=True) |
| 210 | |
| 211 | event("loading", source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), |
| 212 | model_manifest_sha256=manifest_sha256, |
| 213 | data_sha256=hashlib.sha256(Path(args.data).read_bytes()).hexdigest(), |
| 214 | arguments={key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}) |
| 215 | model = load_model(args.model) |
| 216 | module = NeuronDelta(model.model.layers[args.layer].mlp, args.neuron) |
| 217 | model.model.layers[args.layer].mlp = module |
| 218 | params = [module.gate, module.up, module.down] |
| 219 | if args.init: |
| 220 | from safetensors import safe_open |
| 221 | with safe_open(args.init, framework="pt", device="cpu") as tensors, torch.no_grad(): |
| 222 | metadata = tensors.metadata() |
| 223 | if ((int(metadata["layer"]), int(metadata["neuron"])) != (args.layer, args.neuron) |
| 224 | or metadata["model_manifest_sha256"] != manifest_sha256): |
| 225 | raise ValueError("Initial checkpoint belongs to a different neuron or model") |
| 226 | for name, parameter in zip(["gate", "up", "down"], params, strict=True): |
| 227 | parameter.copy_(tensors.get_tensor(name)) |
| 228 | tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) |
| 229 | rows = examples(tokenizer, args.data, args.max_length) |
| 230 | prompts = sorted({tuple(prompt) for prompt, answer in rows}) |
| 231 | rng.shuffle(prompts) |
| 232 | held_out = set(prompts[:max(1, len(prompts) // 10)]) |
| 233 | validation = [row for row in rows if tuple(row[0]) in held_out] |
| 234 | train = [row for row in rows if tuple(row[0]) not in held_out] |
| 235 | optimizer = torch.optim.AdamW(params, lr=args.lr, weight_decay=0) |
| 236 | model.train() |
| 237 | |
| 238 | event("loaded", train_examples=len(train), validation_examples=len(validation), |
| 239 | trainable_parameters=sum(p.numel() for p in params), |
| 240 | allocated_gib=torch.cuda.memory_allocated() / 2**30) |
| 241 | for step in range(args.steps + 1): |
| 242 | if step: |
| 243 | optimizer.param_groups[0]["lr"] = args.lr * min(1, step / 20) * (1 + math.cos(math.pi * step / args.steps)) / 2 |
| 244 | optimizer.zero_grad(set_to_none=True) |
| 245 | value = batch_loss(model, rng.sample(train, args.batch)) |
| 246 | value.backward() |
| 247 | gradient_norm = nn.utils.clip_grad_norm_(params, 1) |
| 248 | optimizer.step() |
| 249 | event("step", step=step, loss=float(value.detach()), gradient_norm=float(gradient_norm), |
| 250 | peak_gib=torch.cuda.max_memory_allocated() / 2**30) |
| 251 | if step % 50 == 0 or step == args.steps: |
| 252 | with torch.no_grad(): |
| 253 | value = sum(float(batch_loss(model, [row])) for row in validation[:16]) / min(16, len(validation)) |
| 254 | save_file({"gate": module.gate.detach().cpu(), "up": module.up.detach().cpu(), |
| 255 | "down": module.down.detach().cpu()}, str(root / f"step-{step}.safetensors"), |
| 256 | metadata={"layer": str(args.layer), "neuron": str(args.neuron), "step": str(step), |
| 257 | "initial_checkpoint": str(args.init) if args.init else "none", |
| 258 | "model_manifest_sha256": manifest_sha256, |
| 259 | "method": "one-neuron gate/up/down deltas on frozen exact INT4 weights"}) |
| 260 | event("checkpoint", step=step, validation_loss=value) |
| 261 | event("finished") |
| 262 | |
| 263 | |
| 264 | if __name__ == "__main__": |
| 265 | main() |