authoravatarclover caruso<git@paperclover.net>2026-10-24 18:30:23-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logdc8464162f442c4e404063c93a5221ae4d56b122
tree95fc095f8df48e9aa23e298528ac534488738fd6
parent3164185b1168ea91cb83b28c2ff62555315780ff
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Give knife-linux 96 GiB and keep its harnesses out of the OOM killer

A 30 GB test binary filled the 32 GiB limit and the kernel took Codex with it. A host service now watches exec events in the container: the Knife server, Claude, and Codex get -1000, and anything they start drops back to 800. Assisted-by: claude-opus-5.5

3 files changed, 96 insertions(+), 2 deletions(-)

nixos/knife-linux.nix+13-1
......@@ -39,7 +39,7 @@ in
3939 run "run --rm --replace --name=knife-linux --pull=never"
4040 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"
4141 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"
42 "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman"
42 "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman"
4343 "--volume=${root}/home:/home/clo"
4444 "--volume=${sandbox}:/Volumes/Code"
4545 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"
......@@ -65,4 +65,16 @@ in
6565 RestartSec = 5;
6666 };
6767 };
68
69 systemd.services.knife-linux-oom = {
70 description = "Keep the Knife harnesses out of the OOM killer";
71 wantedBy = [ "knife-linux.service" ];
72 bindsTo = [ "knife-linux.service" ];
73 after = [ "knife-linux.service" ];
74 serviceConfig = {
75 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}";
76 Restart = "always";
77 RestartSec = 1;
78 };
79 };
6880}
tools/knife-linux-oom.py created+80
......@@ -0,0 +1,80 @@
1"""Exempts the Knife, Claude, and Codex harnesses in knife-linux from the OOM killer.
2
3Children inherit oom_score_adj at fork, so each exec outside the harness set gets a
4killable score back from the kernel's process-event connector before it can grow.
5"""
6
7import errno
8import os
9import re
10import socket
11import struct
12import subprocess
13import sys
14
15HARNESS = re.compile(
16 r"/run/podman-init|/home/clo/\.local/share/knife-server/.*"
17 r"|/home/clo/\.knife/tools/(claude/[^/]+/claude|codex/[^/]+/bin/codex)"
18)
19PROTECTED, WORK = -1000, 800
20NETLINK_CONNECTOR, CN_IDX_PROC, PROC_CN_MCAST_LISTEN, PROC_EVENT_EXEC = 11, 1, 1, 2
21
22
23def adjust(pid, scope):
24 try:
25 if not open(f"/proc/{pid}/cgroup").read().startswith(f"0::{scope}/"):
26 return
27 argv = open(f"/proc/{pid}/cmdline", "rb").read().split(b"\0")
28 names = [os.readlink(f"/proc/{pid}/exe"), *(a.decode(errors="replace") for a in argv[1:2])]
29 path = f"/proc/{pid}/oom_score_adj"
30 current = int(open(path).read())
31 if any(HARNESS.fullmatch(name) for name in names):
32 wanted = PROTECTED
33 elif current < 0:
34 wanted = WORK
35 else:
36 return
37 if current != wanted:
38 open(path, "w").write(str(wanted))
39 except (FileNotFoundError, ProcessLookupError, PermissionError):
40 pass
41
42
43def scan(scope):
44 for directory, _, files in os.walk(f"/sys/fs/cgroup{scope}"):
45 if "cgroup.procs" in files:
46 for pid in open(f"{directory}/cgroup.procs").read().split():
47 adjust(pid, scope)
48
49
50def main():
51 podman = sys.argv[1]
52 scope = subprocess.run(
53 [podman, "inspect", "--format", "{{.State.CgroupPath}}", "knife-linux"],
54 check=True, capture_output=True, text=True,
55 ).stdout.strip()
56 sock = socket.socket(socket.AF_NETLINK, socket.SOCK_DGRAM, NETLINK_CONNECTOR)
57 sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20)
58 sock.bind((0, CN_IDX_PROC))
59 listen = struct.pack("=IIIIHHI", CN_IDX_PROC, 1, 0, 0, 4, 0, PROC_CN_MCAST_LISTEN)
60 sock.send(struct.pack("=IHHII", 16 + len(listen), 3, 0, 0, 0) + listen)
61 scan(scope)
62 print(f"guarding {scope}", flush=True)
63 while True:
64 try:
65 data = sock.recv(65536)
66 except OSError as error:
67 if error.errno != errno.ENOBUFS:
68 raise
69 scan(scope)
70 continue
71 offset = 0
72 while offset + 16 <= len(data):
73 length = struct.unpack_from("=I", data, offset)[0]
74 what = struct.unpack_from("=I", data, offset + 36)[0]
75 if what == PROC_EVENT_EXEC:
76 adjust(struct.unpack_from("=I", data, offset + 56)[0], scope)
77 offset += (length + 3) & ~3 or len(data)
78
79
80main()
tools/knife-linux.md+3-1
......@@ -4,11 +4,13 @@
44agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it
55from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start
66and runs it as the `knife-linux` system service: uid 3000 (`clo`), no
7capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
7capabilities, the 16 threads of the second CCD to itself, a 96 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
88Knife listens on container loopback port 3773; devices reach it through the mesh
99as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's
1010seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,
1111Electron, and T3's browser run with their own sandbox off inside the container.
12[knife-linux-oom.py](knife-linux-oom.py) exempts the Knife server, Claude, and
13Codex from the OOM killer; whatever they run stays killable at score 800.
1214
1315Deploys never restart the container: restarting stops Knife and every running
1416agent. Image and unit changes apply at the next manual