1amends "../../config/Service.pkl"
2
3import "../../config/site.pkl" as site
4import "pkl:json"
5
6local models = (new json.Parser {}).parse(read("catalog.json")).models
7local model = models[0]
8local weights = (new json.Parser {}).parse(read("models.json"))
9
10name = "Local AI"
11launcher = false
12access = "admin"
13enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true"
14stageIsolation = "fresh"
15snapshots = false
16prepare = "prepare.py"
17healthyDeadline = "30m"
18healthRestartGrace = "30m"
19secrets { ["api_key"] {}; ["cache_key"] {} }
20
21container {
22 build = "."
23 cpu = 4000
24 memory = 26624
25 entrypoint = "/usr/bin/python3.12"
26 http {
27 containerPort = 8080
28 subdomain = "ai"
29 checkPath = "/health"
30 }
31 volumes {
32 ["/config/gateway.py"] { config = "gateway.py" }
33 ["/config/catalog.json"] { config = "catalog.json" }
34 ["/cache/hdd"] {}
35 ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache }
36 ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true }
37 // The host's NVIDIA libraries link to immutable Nix store paths.
38 ["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true }
39 ["/nix/store"] { src = "/nix/store"; readOnly = true }
40 }
41 devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" }
42 tmpfs { "/dev/shm:rw,size=10g" }
43 env {
44 ["LD_LIBRARY_PATH"] = "/nvidia/lib:/usr/local/cuda/lib64"
45 ["VLLM_API_KEY"] = "${secret.own.api_key}"
46 ["SNOWGLOBE_AI_CACHE_KEY"] = "${secret.own.cache_key}"
47 ["PYTHONHASHSEED"] = "0"
48 ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False"
49 ["MODEL"] = "/models"
50 ["CTX"] = "long"
51 ["MAX_LEN"] = model.context_window.toString()
52 ["SPEC"] = "mtp"
53 ["DRAFT_TOKENS"] = "3"
54 ["PREFIX_CACHE"] = "1"
55 ["TOOLS"] = "1"
56 ["VISION"] = "1"
57 ["VISION_OFFLOAD"] = "1"
58 ["GPU_UTIL"] = "0.93"
59 ["MAX_SEQS"] = "8"
60 ["API_SERVERS"] = "1"
61 ["HOST"] = "127.0.0.1"
62 ["PORT"] = "8081"
63 ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1"
64 ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
65 ["EXTRA_ARGS"] = "--served-model-name \(models.toList().map((item) -> item.slug).join(" ")) --kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"snowglobe_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log"
66 }
67 args { "/config/gateway.py" }
68}