authoravatarclover caruso<git@paperclover.net>2026-10-24 18:30:23-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logdc8464162f442c4e404063c93a5221ae4d56b122
tree95fc095f8df48e9aa23e298528ac534488738fd6
parent3164185b1168ea91cb83b28c2ff62555315780ff
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Give knife-linux 96 GiB and keep its harnesses out of the OOM killer

A 30 GB test binary filled the 32 GiB limit and the kernel took Codex with it. A host service now watches exec events in the container: the Knife server, Claude, and Codex get -1000, and anything they start drops back to 800. Assisted-by: claude-opus-5.5

3 files changed, 96 insertions(+), 2 deletions(-)

nixos/knife-linux.nix+13-1
...@@ -39,7 +39,7 @@ in...@@ -39,7 +39,7 @@ in
39 run "run --rm --replace --name=knife-linux --pull=never"39 run "run --rm --replace --name=knife-linux --pull=never"
40 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"40 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"
41 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"41 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"
42 "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman"42 "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman"
43 "--volume=${root}/home:/home/clo"43 "--volume=${root}/home:/home/clo"
44 "--volume=${sandbox}:/Volumes/Code"44 "--volume=${sandbox}:/Volumes/Code"
45 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"45 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"
...@@ -65,4 +65,16 @@ in...@@ -65,4 +65,16 @@ in
65 RestartSec = 5;65 RestartSec = 5;
66 };66 };
67 };67 };
68
69 systemd.services.knife-linux-oom = {
70 description = "Keep the Knife harnesses out of the OOM killer";
71 wantedBy = [ "knife-linux.service" ];
72 bindsTo = [ "knife-linux.service" ];
73 after = [ "knife-linux.service" ];
74 serviceConfig = {
75 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}";
76 Restart = "always";
77 RestartSec = 1;
78 };
79 };
68}80}
tools/knife-linux-oom.py created+80
...@@ -0,0 +1,80 @@
1"""Exempts the Knife, Claude, and Codex harnesses in knife-linux from the OOM killer.
2
3Children inherit oom_score_adj at fork, so each exec outside the harness set gets a
4killable score back from the kernel's process-event connector before it can grow.
5"""
6
7import errno
8import os
9import re
10import socket
11import struct
12import subprocess
13import sys
14
15HARNESS = re.compile(
16 r"/run/podman-init|/home/clo/\.local/share/knife-server/.*"
17 r"|/home/clo/\.knife/tools/(claude/[^/]+/claude|codex/[^/]+/bin/codex)"
18)
19PROTECTED, WORK = -1000, 800
20NETLINK_CONNECTOR, CN_IDX_PROC, PROC_CN_MCAST_LISTEN, PROC_EVENT_EXEC = 11, 1, 1, 2
21
22
23def adjust(pid, scope):
24 try:
25 if not open(f"/proc/{pid}/cgroup").read().startswith(f"0::{scope}/"):
26 return
27 argv = open(f"/proc/{pid}/cmdline", "rb").read().split(b"\0")
28 names = [os.readlink(f"/proc/{pid}/exe"), *(a.decode(errors="replace") for a in argv[1:2])]
29 path = f"/proc/{pid}/oom_score_adj"
30 current = int(open(path).read())
31 if any(HARNESS.fullmatch(name) for name in names):
32 wanted = PROTECTED
33 elif current < 0:
34 wanted = WORK
35 else:
36 return
37 if current != wanted:
38 open(path, "w").write(str(wanted))
39 except (FileNotFoundError, ProcessLookupError, PermissionError):
40 pass
41
42
43def scan(scope):
44 for directory, _, files in os.walk(f"/sys/fs/cgroup{scope}"):
45 if "cgroup.procs" in files:
46 for pid in open(f"{directory}/cgroup.procs").read().split():
47 adjust(pid, scope)
48
49
50def main():
51 podman = sys.argv[1]
52 scope = subprocess.run(
53 [podman, "inspect", "--format", "{{.State.CgroupPath}}", "knife-linux"],
54 check=True, capture_output=True, text=True,
55 ).stdout.strip()
56 sock = socket.socket(socket.AF_NETLINK, socket.SOCK_DGRAM, NETLINK_CONNECTOR)
57 sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20)
58 sock.bind((0, CN_IDX_PROC))
59 listen = struct.pack("=IIIIHHI", CN_IDX_PROC, 1, 0, 0, 4, 0, PROC_CN_MCAST_LISTEN)
60 sock.send(struct.pack("=IHHII", 16 + len(listen), 3, 0, 0, 0) + listen)
61 scan(scope)
62 print(f"guarding {scope}", flush=True)
63 while True:
64 try:
65 data = sock.recv(65536)
66 except OSError as error:
67 if error.errno != errno.ENOBUFS:
68 raise
69 scan(scope)
70 continue
71 offset = 0
72 while offset + 16 <= len(data):
73 length = struct.unpack_from("=I", data, offset)[0]
74 what = struct.unpack_from("=I", data, offset + 36)[0]
75 if what == PROC_EVENT_EXEC:
76 adjust(struct.unpack_from("=I", data, offset + 56)[0], scope)
77 offset += (length + 3) & ~3 or len(data)
78
79
80main()
tools/knife-linux.md+3-1
...@@ -4,11 +4,13 @@...@@ -4,11 +4,13 @@
4agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it4agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it
5from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start5from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start
6and runs it as the `knife-linux` system service: uid 3000 (`clo`), no6and runs it as the `knife-linux` system service: uid 3000 (`clo`), no
7capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.7capabilities, the 16 threads of the second CCD to itself, a 96 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
8Knife listens on container loopback port 3773; devices reach it through the mesh8Knife listens on container loopback port 3773; devices reach it through the mesh
9as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's9as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's
10seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,10seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,
11Electron, and T3's browser run with their own sandbox off inside the container.11Electron, and T3's browser run with their own sandbox off inside the container.
12[knife-linux-oom.py](knife-linux-oom.py) exempts the Knife server, Claude, and
13Codex from the OOM killer; whatever they run stays killable at score 800.
1214
13Deploys never restart the container: restarting stops Knife and every running15Deploys never restart the container: restarting stops Knife and every running
14agent. Image and unit changes apply at the next manual16agent. Image and unit changes apply at the next manual