authoravatarclover caruso<git@paperclover.net>2026-10-23 18:18:08-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
log225f9e709874838f85d8a55d28e270cd44a1b624
treef8988b28009cd0b7ca7706cd1b1c3eab35f731c9
parent98cae3f5303586df4613ce13f6456f86e5a12bf6
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Give the Knife container its own CCD

The weight-only setup still left agent builds and host services contending for the same cores. Assisted-by: claude-opus-5.5

2 files changed, 13 insertions(+), 3 deletions(-)

nixos/knife-linux.nix+12-2
......@@ -15,6 +15,7 @@ let
1515 exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@"
1616 '';
1717 run = "${podman}/bin/knife-linux-podman";
18 ccd = "8-15,24-31";
1819in
1920{
2021 environment.systemPackages = [ podman ];
......@@ -38,14 +39,23 @@ in
3839 run "run --rm --replace --name=knife-linux --pull=never"
3940 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"
4041 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"
41 # A low weight rather than a CPU quota: an exhausted quota freezes the Knife server too.
42 "--memory=32g --cpu-shares=1024 --pids-limit=32768 --shm-size=2g --network=podman"
42 "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman"
4343 "--volume=${root}/home:/home/clo"
4444 "--volume=${sandbox}:/Volumes/Code"
4545 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"
4646 "--device=/dev/kvm"
4747 "localhost/knife-linux"
4848 ];
49 # Agents get the second CCD to themselves and the host keeps the first. A partition
50 # rather than a CPU quota: an exhausted quota freezes the Knife server with the builds.
51 ExecStartPost = "-" + pkgs.writeShellScript "knife-linux-cpus" ''
52 set -e
53 scope=/sys/fs/cgroup$(${run} inspect --format '{{.State.CgroupPath}}' knife-linux)
54 echo ${ccd} > /sys/fs/cgroup/machine.slice/cpuset.cpus.exclusive
55 echo ${ccd} > $scope/cpuset.cpus.exclusive
56 echo ${ccd} > $scope/cpuset.cpus
57 echo root > $scope/cpuset.cpus.partition
58 '';
4959 ExecStop = "${run} stop --ignore --time=90 knife-linux";
5060 ExecStopPost = "${run} rm --ignore --force knife-linux";
5161 KillMode = "mixed";
tools/knife-linux.md+1-1
......@@ -4,7 +4,7 @@
44agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it
55from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start
66and runs it as the `knife-linux` system service: uid 3000 (`clo`), no
7capabilities, a low CPU weight with no core cap, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
7capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
88Knife listens on container loopback port 3773; devices reach it through the mesh
99as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's
1010seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,