diff --git a/nixos/knife-linux.nix b/nixos/knife-linux.nix index 568a00dd4c1fcbbd2be9b3206b0bf612b6ad719c..ad3ba40192bbcfabcab8c028b97cfdb2aeedccee 100644 --- a/nixos/knife-linux.nix +++ b/nixos/knife-linux.nix @@ -15,6 +15,7 @@ let exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@" ''; run = "${podman}/bin/knife-linux-podman"; + ccd = "8-15,24-31"; in { environment.systemPackages = [ podman ]; @@ -38,14 +39,23 @@ in run "run --rm --replace --name=knife-linux --pull=never" "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux" "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges" - # A low weight rather than a CPU quota: an exhausted quota freezes the Knife server too. - "--memory=32g --cpu-shares=1024 --pids-limit=32768 --shm-size=2g --network=podman" + "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman" "--volume=${root}/home:/home/clo" "--volume=${sandbox}:/Volumes/Code" "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro" "--device=/dev/kvm" "localhost/knife-linux" ]; + # Agents get the second CCD to themselves and the host keeps the first. A partition + # rather than a CPU quota: an exhausted quota freezes the Knife server with the builds. + ExecStartPost = "-" + pkgs.writeShellScript "knife-linux-cpus" '' + set -e + scope=/sys/fs/cgroup$(${run} inspect --format '{{.State.CgroupPath}}' knife-linux) + echo ${ccd} > /sys/fs/cgroup/machine.slice/cpuset.cpus.exclusive + echo ${ccd} > $scope/cpuset.cpus.exclusive + echo ${ccd} > $scope/cpuset.cpus + echo root > $scope/cpuset.cpus.partition + ''; ExecStop = "${run} stop --ignore --time=90 knife-linux"; ExecStopPost = "${run} rm --ignore --force knife-linux"; KillMode = "mixed"; diff --git a/tools/knife-linux.md b/tools/knife-linux.md index cfb5d60017e38899468767aa5f70021a8843bea6..eab5f0f33a880e735cc14285d070b0ba31c95669 100644 --- a/tools/knife-linux.md +++ b/tools/knife-linux.md @@ -4,7 +4,7 @@ agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start and runs it as the `knife-linux` system service: uid 3000 (`clo`), no -capabilities, a low CPU weight with no core cap, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. +capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. Knife listens on container loopback port 3773; devices reach it through the mesh as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,