| 1 | #!/usr/bin/env python3 |
| 2 | import argparse |
| 3 | import json |
| 4 | import os |
| 5 | from pathlib import Path |
| 6 | import signal |
| 7 | import subprocess |
| 8 | import time |
| 9 | import urllib.request |
| 10 | |
| 11 | |
| 12 | def main(): |
| 13 | parser = argparse.ArgumentParser() |
| 14 | parser.add_argument("--machine-id", required=True) |
| 15 | parser.add_argument("--image", required=True) |
| 16 | parser.add_argument("--models", type=Path, required=True) |
| 17 | parser.add_argument("--root", type=Path, required=True) |
| 18 | parser.add_argument("--name", required=True) |
| 19 | parser.add_argument("--program", choices=["experiment.py", "train.py", "evaluate.py", "test_steering.py"], default="experiment.py") |
| 20 | parser.add_argument("--source", type=Path, default=Path(__file__).parent) |
| 21 | parser.add_argument("experiment_args", nargs=argparse.REMAINDER) |
| 22 | args = parser.parse_args() |
| 23 | if args.experiment_args[:1] == ["--"]: |
| 24 | args.experiment_args = args.experiment_args[1:] |
| 25 | os.environ["PATH"] = "/run/current-system/sw/bin:" + os.environ.get("PATH", "") |
| 26 | if Path("/etc/machine-id").read_text().strip() != args.machine_id: |
| 27 | raise ValueError("Wrong host; inference was not touched") |
| 28 | args.root.mkdir(mode=0o700, parents=True, exist_ok=True) |
| 29 | token = Path("/var/lib/snowglobe/nomad.token").read_text().strip() |
| 30 | |
| 31 | def job(): |
| 32 | return json.load(urllib.request.urlopen(urllib.request.Request( |
| 33 | "http://127.0.0.1:4646/v1/job/local-ai", headers={"X-Nomad-Token": token}), timeout=10)) |
| 34 | |
| 35 | initial = job() |
| 36 | group = next(g for g in initial["TaskGroups"] if g["Name"] == "app") |
| 37 | if group["Count"] != 1: |
| 38 | raise ValueError("Expected one active inference allocation; inference was not touched") |
| 39 | environment = {**os.environ, "NOMAD_TOKEN": token, "NOMAD_ADDR": "http://127.0.0.1:4646"} |
| 40 | state = {"machine_id": args.machine_id, "job": initial["ID"], "job_version": initial["Version"], |
| 41 | "original_count": group["Count"], "image": args.image, "models": str(args.models), |
| 42 | "program": args.program, "source": str(args.source), "arguments": args.experiment_args, |
| 43 | "started": time.time(), "phase": "preparing"} |
| 44 | |
| 45 | def record(phase, **fields): |
| 46 | state.update(phase=phase, updated=time.time(), **fields) |
| 47 | pending = args.root / "state.pending" |
| 48 | pending.write_text(json.dumps(state, indent=2) + "\n") |
| 49 | pending.replace(args.root / "state.json") |
| 50 | print(json.dumps({"phase": phase, **fields}), flush=True) |
| 51 | |
| 52 | def scale(count): |
| 53 | deadline = time.monotonic() + 600 |
| 54 | while True: |
| 55 | current = job() |
| 56 | current_group = next(g for g in current["TaskGroups"] if g["Name"] == "app") |
| 57 | if current_group["Count"] == count: |
| 58 | return |
| 59 | deployments = json.load(urllib.request.urlopen(urllib.request.Request( |
| 60 | "http://127.0.0.1:4646/v1/job/local-ai/deployments", |
| 61 | headers={"X-Nomad-Token": token}), timeout=10)) |
| 62 | if not any(d["Status"] in {"running", "pending", "initializing"} for d in deployments): |
| 63 | break |
| 64 | if time.monotonic() >= deadline: |
| 65 | raise TimeoutError("Inference deployment did not settle before scaling") |
| 66 | time.sleep(5) |
| 67 | subprocess.run(["nomad", "job", "scale", "-detach", f"-check-index={current['JobModifyIndex']}", |
| 68 | "local-ai", "app", str(count)], |
| 69 | env=environment, check=True) |
| 70 | |
| 71 | def stop(signum, frame): |
| 72 | raise InterruptedError(f"Received signal {signum}") |
| 73 | |
| 74 | signal.signal(signal.SIGTERM, stop) |
| 75 | signal.signal(signal.SIGINT, stop) |
| 76 | work = args.root / "work" |
| 77 | work.mkdir(mode=0o700, exist_ok=True) |
| 78 | os.chown(work, 65534, 65534) |
| 79 | record("pausing") |
| 80 | try: |
| 81 | scale(0) |
| 82 | deadline = time.monotonic() + 180 |
| 83 | while True: |
| 84 | used = int(subprocess.check_output([ |
| 85 | "nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip()) |
| 86 | if used < 256: |
| 87 | break |
| 88 | if time.monotonic() >= deadline: |
| 89 | raise TimeoutError(f"GPU still has {used} MiB allocated") |
| 90 | time.sleep(2) |
| 91 | record("running", gpu_initial_mib=used) |
| 92 | command = [ |
| 93 | "podman", "run", "--rm", "--name=" + args.name, |
| 94 | "--network=none", "--read-only", "--user=65534:65534", |
| 95 | "--group-add=" + str(args.models.stat().st_gid), "--cap-drop=ALL", |
| 96 | "--security-opt=no-new-privileges", "--pids-limit=1024", "--memory=34g", |
| 97 | "--memory-swap=34g", "--shm-size=2g", "--tmpfs=/tmp:rw,size=2g,mode=1777", |
| 98 | "--device=/dev/nvidia0", "--device=/dev/nvidiactl", "--device=/dev/nvidia-uvm", |
| 99 | "--volume=" + str(args.models) + ":/models:ro", |
| 100 | "--volume=" + str(args.source) + ":/experiment:ro", |
| 101 | "--volume=" + str(work) + ":/work:rw", |
| 102 | "--volume=/run/opengl-driver/lib:/nvidia/lib:ro", "--volume=/nix/store:/nix/store:ro", |
| 103 | "--env=LD_LIBRARY_PATH=/nvidia/lib:/usr/local/cuda/lib64", |
| 104 | "--env=HOME=/work", "--env=XDG_CACHE_HOME=/work/cache", "--env=HF_HUB_OFFLINE=1", |
| 105 | "--env=VLLM_NO_USAGE_STATS=1", "--env=DO_NOT_TRACK=1", "--env=PYTHONUNBUFFERED=1", |
| 106 | "--env=PYTHONDONTWRITEBYTECODE=1", "--env=PYTHONPATH=/experiment:/app/steering", |
| 107 | "--env=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False", |
| 108 | "--env=FLASHINFER_DISABLE_VERSION_CHECK=1", "--env=VLLM_USE_FLASHINFER_SAMPLER=0", |
| 109 | "--env=VLLM_ALLOW_INSECURE_SERIALIZATION=1", "--env=VLLM_HOST_IP=127.0.0.1", |
| 110 | "--entrypoint=/app/venv/bin/python", args.image, "/experiment/" + args.program, |
| 111 | *args.experiment_args, |
| 112 | ] |
| 113 | with (args.root / "run.log").open("ab", buffering=0) as log: |
| 114 | result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) |
| 115 | if result.returncode: |
| 116 | raise RuntimeError(f"Experiment exited with status {result.returncode}; see run.log") |
| 117 | record("experiment_finished") |
| 118 | except BaseException as error: |
| 119 | record("failed", error=str(error)) |
| 120 | raise |
| 121 | finally: |
| 122 | try: |
| 123 | subprocess.run(["podman", "stop", "--time=15", args.name], capture_output=True) |
| 124 | finally: |
| 125 | record("restoring", experiment_phase=state["phase"]) |
| 126 | scale(group["Count"]) |
| 127 | record("restored") |
| 128 | |
| 129 | |
| 130 | if __name__ == "__main__": |
| 131 | main() |