authoravatarclover caruso<git@paperclover.net>2026-10-20 12:13:12-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logaf4fd9c75c708dc3aa91fef61c46acaa82980606
tree4a3d661b4db0ef4c47b1e42e3e6be6ba60c331a8
parent8d33f673fd185f073c8eb3d203f1c6b84c1cd215
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Add a bounded SSD read cache without moving VM disks

Keep durable guest disks on HDD and attach a preallocated 150 GiB file as the pool's optional L2ARC. Refuse allocation when it would leave less than 32 GiB free, preserve unknown files, and reuse an attached cache across service restarts. Verified the full-size allocation on a disposable pool, the production NixOS build, and isolated cache recycling, warmed read hits, import, cache-loss recovery, and path/capacity rejection. No Windows boot-time improvement is claimed. Assisted-by: gpt-6-astra

5 files changed, 234 insertions(+), 1 deletions(-)

nixos/snowglobe.nix+1-1
......@@ -16,7 +16,7 @@ let
1616 };
1717in
1818{
19 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ];
19 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./zfs-read-cache.nix ];
2020
2121 networking.hostName = "snowglobe";
2222 environment.etc."machine-info".text = ''
nixos/tests/zfs-read-cache.nix created+83
......@@ -0,0 +1,83 @@
1{ pkgs }:
2pkgs.testers.runNixOSTest {
3 name = "zfs-read-cache";
4 globalTimeout = 600;
5 nodes.server = { lib, ... }: {
6 imports = [ ../zfs-read-cache.nix ];
7 virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 4096 ]; };
8 networking.hostId = "c0a1c0a1";
9 boot.supportedFilesystems = [ "zfs" ];
10 boot.zfs.forceImportRoot = false;
11 boot.extraModprobeConfig = "options zfs zfs_arc_max=268435456 l2arc_write_max=33554432";
12 environment.variables.SNOWGLOBE_POOL = "cache-test";
13 environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.fio ];
14 systemd.services.cache-test-pool = {
15 before = [ "snowglobe-zfs-read-cache.service" ];
16 serviceConfig = { Type = "oneshot"; RemainAfterExit = true; };
17 script = "${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=/data cache-test /dev/vdb";
18 };
19 systemd.services.snowglobe-zfs-read-cache = {
20 requires = [ "cache-test-pool.service" ];
21 after = [ "cache-test-pool.service" ];
22 serviceConfig.ExecStart = lib.mkForce "${pkgs.python3}/bin/python3 ${../../tools/zfs-read-cache.py} cache-test /var/cache/snowglobe/zfs/l2arc 536870912 33554432";
23 };
24 };
25 testScript = ''
26 import json
27 start_all()
28 server.wait_for_unit("snowglobe-zfs-read-cache.service")
29 image = "/var/cache/snowglobe/zfs/l2arc"
30 helper = "python3 ${../../tools/zfs-read-cache.py}"
31
32 def cache():
33 status = json.loads(server.succeed("zpool status -jP --json-flat-vdevs --json-int cache-test"))
34 return next(v for v in status["pools"]["cache-test"]["vdevs"].values() if v.get("path") == image)
35
36 def counters():
37 return {row.split()[0]: int(row.split()[2]) for row in server.succeed("cat /proc/spl/kstat/zfs/arcstats").splitlines()[2:]}
38
39 with subtest("bounded private cache and idempotent service"):
40 original = cache()
41 assert original["class"] == "l2cache" and original["state"] == "ONLINE", original
42 info = server.succeed(f"stat -c '%s %b %a' {image}").split()
43 assert int(info[0]) == 536870912 and int(info[1]) * 512 >= 536870912 and info[2] == "600", info
44 server.succeed("systemctl restart snowglobe-zfs-read-cache.service")
45 assert cache()["guid"] == original["guid"]
46 server.fail(f"{helper} cache-test {image} 1073741824 0")
47 assert cache()["guid"] == original["guid"]
48
49 with subtest("warm read hits and bounded cache recycling"):
50 server.succeed("fio --name=fill --filename=/data/workload --size=768m --rw=write --bs=1m --refill_buffers=1 --end_fsync=1 --output=/tmp/fill.json --output-format=json")
51 before = counters()
52 server.succeed("fio --name=warm --filename=/data/workload --size=768m --rw=randread --bs=128k --runtime=45 --time_based --output=/tmp/warm.json --output-format=json", timeout=60)
53 after = counters()
54 assert after["l2_hits"] > before["l2_hits"], (before["l2_hits"], after["l2_hits"])
55 assert after["l2_write_bytes"] > 536870912, after["l2_write_bytes"]
56 assert int(server.succeed(f"stat -c %s {image}")) == 536870912
57 print(json.dumps({"cache_hits": after["l2_hits"] - before["l2_hits"], "cache_written_bytes": after["l2_write_bytes"]}))
58
59 with subtest("persistent import and missing-cache fallback"):
60 checksum = server.succeed("sha256sum /data/workload").split()[0]
61 server.succeed("zpool export cache-test; zpool import -d /dev -d /var/cache/snowglobe/zfs -o cachefile=none cache-test")
62 assert cache()["state"] == "ONLINE"
63 assert cache()["guid"] == original["guid"]
64 server.succeed(f"zpool export cache-test; mv {image} {image}.absent; zpool import -d /dev -o cachefile=none cache-test")
65 assert server.succeed("zpool list -H -o health cache-test").strip() == "ONLINE"
66 assert server.succeed("sha256sum /data/workload").split()[0] == checksum
67 server.fail("systemctl restart snowglobe-zfs-read-cache.service")
68 server.succeed("printf writable > /data/after-cache-loss")
69 assert server.succeed("cat /data/after-cache-loss") == "writable"
70 server.succeed(f"zpool remove cache-test {image}; rm {image}.absent; systemctl restart snowglobe-zfs-read-cache.service")
71 assert cache()["state"] == "ONLINE"
72
73 with subtest("no overwrite, symlink escape, or root-disk exhaustion"):
74 server.succeed("printf retained > /var/cache/snowglobe/zfs/occupied")
75 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/occupied 67108864 0")
76 assert server.succeed("cat /var/cache/snowglobe/zfs/occupied") == "retained"
77 server.succeed("ln -s occupied /var/cache/snowglobe/zfs/link")
78 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/link 67108864 0")
79 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/too-large 161061273600 34359738368")
80 server.succeed("test ! -e /var/cache/snowglobe/zfs/too-large")
81 assert server.succeed("sha256sum /data/workload").split()[0] == checksum
82 '';
83}
nixos/zfs-read-cache.nix created+23
......@@ -0,0 +1,23 @@
1{ config, pkgs, ... }:
2let
3 pool = config.environment.variables.SNOWGLOBE_POOL;
4 image = "/var/cache/snowglobe/zfs/l2arc";
5 capacity = 150 * 1024 * 1024 * 1024;
6 headroom = 32 * 1024 * 1024 * 1024;
7in
8{
9 systemd.services.snowglobe-zfs-read-cache = {
10 wantedBy = [ "multi-user.target" ];
11 after = [ "zfs-import-${pool}.service" ];
12 path = [ config.boot.zfs.package ];
13 serviceConfig = {
14 Type = "oneshot";
15 RemainAfterExit = true;
16 CacheDirectory = "snowglobe/zfs";
17 CacheDirectoryMode = "0700";
18 UMask = "0077";
19 TimeoutStartSec = "5min";
20 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/zfs-read-cache.py} ${pool} ${image} ${toString capacity} ${toString headroom}";
21 };
22 };
23}
tools/zfs-read-cache.md created+67
......@@ -0,0 +1,67 @@
1# SSD read cache
2
3Guest disks remain on the HDD pool. The optional L2ARC holds disposable copies
4of recently used blocks on the OS SSD. Cache exhaustion recycles those copies;
5guest capacity continues to come from HDD storage. Cache reads that fail are
6retried against the HDD pool. Writes still require HDD I/O.
7
8The host configuration is [zfs-read-cache.nix](../nixos/zfs-read-cache.nix).
9It owns the capacity, minimum free space at allocation, and cache-file path; the pool name
10comes from the existing host configuration. The service runs after pool import
11without making guest startup depend on it. Stopping the service leaves the
12cache attached, so normal service restarts preserve its contents.
13
14The helper preallocates the entire file before attaching it with `zpool add
15... cache ...`. It never adds normal storage, a special vdev, or a log device.
16The file and directory are restricted to root. Insufficient free space causes
17setup to fail before allocation; other host software can still consume the
18remaining free space afterward. An existing attached cache must have the
19configured size; an unknown file at the destination is preserved and rejected.
20
21## Inspection and recovery
22
23Use `systemctl status snowglobe-zfs-read-cache.service`, its journal, and
24`zpool status -P` to distinguish service setup failure from pool health. A
25missing cache can appear as unavailable while the pool remains online.
26
27If allocation was interrupted, restarting the service clears its abandoned
28temporary allocation. If interruption occurred after publishing the file but
29before attaching it, setup preserves that unattached file. Verify that the
30exact file is absent from every pool's device list before removing it and
31retrying the service.
32
33To replace or resize a cache, stop its setup service and confirm the exact
34device appears under **cache** in the pool status. Remove only that cache
35device from the pool before deleting its file. Update the capacity declaration,
36deploy the reviewed configuration, and start the service to allocate a fresh
37cache. Guest disks stay in place and the cache warms again. Do not truncate,
38punch holes in, or replace an attached file.
39
40Changes to the production pool follow the repository's production deployment
41workflow. A staged dashboard cannot exercise this host-level configuration.
42
43## Verification
44
45[The isolated NixOS test](../nixos/tests/zfs-read-cache.nix) exercises allocation,
46idempotent startup, warmed read hits, bounded cache recycling, import with and
47without the cache, writable storage after cache loss, safe replacement, and
48rejection of unknown files, symlinks, and insufficient free space. Its small
49ARC limit belongs only to the disposable test VM.
50
51The 2026-10-20 test passed in 64 seconds, recording 97,453 cache hits and
521,234,213,376 cache-write bytes through a fixed 512 MiB cache. Data checksums
53matched after import with the cache removed, and writes still succeeded.
54
55On 2026-10-20, a separate disposable pool on Snow Globe accepted a fully
56preallocated 150 GiB cache file on the existing ext4 SSD. Repeated setup kept
57the same file. The fixture and allocation were removed afterward; the
58production pool was unchanged. The SSD had approximately 200 GiB available
59before allocation. Its largest consumers were Nix store contents (93 GiB),
60container layers (72 GiB), and VM experiments/templates (about 49 GiB).
61
62The Windows provisioning samples from that day included Windows Update
63servicing. They do not establish a cache-related startup improvement. L2ARC
64benefit requires cache hits beyond those already served by RAM; a virtualized
65correctness test is not a physical SSD/HDD performance benchmark.
66
67See [OpenZFS caching documentation](https://openzfs.github.io/openzfs-docs/Basic%20Concepts/Pool%20Structure/Caching.html).
tools/zfs-read-cache.py created+60
......@@ -0,0 +1,60 @@
1#!/usr/bin/env python3
2"""Attach a preallocated, disposable file as a ZFS read cache."""
3import fcntl
4import json
5import os
6from pathlib import Path
7import shutil
8import stat
9import subprocess
10import sys
11import tempfile
12
13
14def ensure(pool, image, capacity, headroom):
15 if capacity < 64 * 2**20 or capacity % 4096 or headroom < 0:
16 raise ValueError("Cache capacity must be at least 64 MiB and aligned to 4096 bytes.")
17 if not image.is_absolute() or image.resolve() != image:
18 raise ValueError("Use an absolute cache path without symlinks.")
19 image.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
20 with (image.parent / ".lock").open("a") as lock:
21 fcntl.flock(lock, fcntl.LOCK_EX)
22 status = json.loads(subprocess.check_output(
23 ["zpool", "status", "-jP", "--json-flat-vdevs", "--json-int", pool], text=True))
24 devices = status["pools"][pool]["vdevs"].values()
25 attached = next((device for device in devices if device.get("path") == str(image)), None)
26 if attached:
27 if attached["class"] != "l2cache" or attached["state"] != "ONLINE":
28 raise ValueError("The configured cache is not an online cache device; inspect zpool status.")
29 info = image.lstat()
30 if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size != capacity:
31 raise ValueError("The attached cache file differs from its configured capacity or type.")
32 return
33 if image.exists():
34 raise ValueError("An unattached file already occupies the cache path; preserve or remove it before retrying.")
35 for abandoned in image.parent.glob(".preparing-*"):
36 abandoned.unlink()
37 if shutil.disk_usage(image.parent).free < capacity + headroom:
38 raise ValueError("Insufficient SSD space for the cache and reserved OS headroom.")
39 descriptor, temporary = tempfile.mkstemp(prefix=".preparing-", dir=image.parent)
40 try:
41 with os.fdopen(descriptor, "wb") as output:
42 os.posix_fallocate(output.fileno(), 0, capacity)
43 os.fsync(output.fileno())
44 os.link(temporary, image)
45 os.unlink(temporary)
46 directory = os.open(image.parent, os.O_RDONLY | os.O_DIRECTORY)
47 try:
48 os.fsync(directory)
49 finally:
50 os.close(directory)
51 subprocess.run(["zpool", "add", pool, "cache", str(image)], check=True)
52 finally:
53 Path(temporary).unlink(missing_ok=True)
54
55
56if __name__ == "__main__":
57 try:
58 ensure(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]))
59 except (ValueError, OSError, subprocess.CalledProcessError) as error:
60 raise SystemExit(str(error))