From 98cae3f5303586df4613ce13f6456f86e5a12bf6 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Fri, 23 Oct 2026 18:14:26 -0700 Subject: [PATCH] Keep the Knife server and its database responsive under load The container's 12-core quota ran out every period, freezing the Knife server with the builds, and reclaim had shrunk the ARC to 6 GiB, so its state database was read from the HDDs on the main thread. Assisted-by: claude-opus-5.5 --- nixos/knife-linux.nix | 3 ++- nixos/zfs-read-cache.nix | 3 +++ tools/knife-linux.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nixos/knife-linux.nix b/nixos/knife-linux.nix index e74671c9f08d21ff968ad086fda9570ec46aec5b..568a00dd4c1fcbbd2be9b3206b0bf612b6ad719c 100644 --- a/nixos/knife-linux.nix +++ b/nixos/knife-linux.nix @@ -38,7 +38,8 @@ in run "run --rm --replace --name=knife-linux --pull=never" "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux" "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges" - "--memory=32g --cpus=12 --pids-limit=32768 --shm-size=2g --network=podman" + # A low weight rather than a CPU quota: an exhausted quota freezes the Knife server too. + "--memory=32g --cpu-shares=1024 --pids-limit=32768 --shm-size=2g --network=podman" "--volume=${root}/home:/home/clo" "--volume=${sandbox}:/Volumes/Code" "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro" diff --git a/nixos/zfs-read-cache.nix b/nixos/zfs-read-cache.nix index 11741f83d559effa16d087b450a77ec6ff716b61..b3e97092a97b2b4d3b466b5d732313dc7b914484 100644 --- a/nixos/zfs-read-cache.nix +++ b/nixos/zfs-read-cache.nix @@ -6,6 +6,9 @@ let headroom = 32 * 1024 * 1024 * 1024; in { + # Reclaim pressure otherwise shrinks the ARC to a few GiB with most memory free. + boot.kernelParams = [ "zfs.zfs_arc_min=${toString (32 * 1024 * 1024 * 1024)}" ]; + systemd.services.snowglobe-zfs-read-cache = { wantedBy = [ "multi-user.target" ]; after = [ "zfs-import-${pool}.service" ]; diff --git a/tools/knife-linux.md b/tools/knife-linux.md index 959c2bf1e30420732abf7c53d56ead7b0776e6b9..cfb5d60017e38899468767aa5f70021a8843bea6 100644 --- a/tools/knife-linux.md +++ b/tools/knife-linux.md @@ -4,7 +4,7 @@ agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start and runs it as the `knife-linux` system service: uid 3000 (`clo`), no -capabilities, 12 CPUs, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. +capabilities, a low CPU weight with no core cap, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. Knife listens on container loopback port 3773; devices reach it through the mesh as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium, -- 2.54.1