diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 0ff4136f4a57d86cf58dbe0e7ad84ae54307a514..dc74c62e9ba92e229069a13a4a4f3265d877f12b 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -16,7 +16,7 @@ let }; in { - imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ]; + imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./zfs-read-cache.nix ]; networking.hostName = "snowglobe"; environment.etc."machine-info".text = '' diff --git a/nixos/tests/zfs-read-cache.nix b/nixos/tests/zfs-read-cache.nix new file mode 100644 index 0000000000000000000000000000000000000000..9c73aa1f453ea9f7d9410410802e752ed753afa9 --- /dev/null +++ b/nixos/tests/zfs-read-cache.nix @@ -0,0 +1,83 @@ +{ pkgs }: +pkgs.testers.runNixOSTest { + name = "zfs-read-cache"; + globalTimeout = 600; + nodes.server = { lib, ... }: { + imports = [ ../zfs-read-cache.nix ]; + virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 4096 ]; }; + networking.hostId = "c0a1c0a1"; + boot.supportedFilesystems = [ "zfs" ]; + boot.zfs.forceImportRoot = false; + boot.extraModprobeConfig = "options zfs zfs_arc_max=268435456 l2arc_write_max=33554432"; + environment.variables.SNOWGLOBE_POOL = "cache-test"; + environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.fio ]; + systemd.services.cache-test-pool = { + before = [ "snowglobe-zfs-read-cache.service" ]; + serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; + script = "${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=/data cache-test /dev/vdb"; + }; + systemd.services.snowglobe-zfs-read-cache = { + requires = [ "cache-test-pool.service" ]; + after = [ "cache-test-pool.service" ]; + serviceConfig.ExecStart = lib.mkForce "${pkgs.python3}/bin/python3 ${../../tools/zfs-read-cache.py} cache-test /var/cache/snowglobe/zfs/l2arc 536870912 33554432"; + }; + }; + testScript = '' + import json + start_all() + server.wait_for_unit("snowglobe-zfs-read-cache.service") + image = "/var/cache/snowglobe/zfs/l2arc" + helper = "python3 ${../../tools/zfs-read-cache.py}" + + def cache(): + status = json.loads(server.succeed("zpool status -jP --json-flat-vdevs --json-int cache-test")) + return next(v for v in status["pools"]["cache-test"]["vdevs"].values() if v.get("path") == image) + + def counters(): + return {row.split()[0]: int(row.split()[2]) for row in server.succeed("cat /proc/spl/kstat/zfs/arcstats").splitlines()[2:]} + + with subtest("bounded private cache and idempotent service"): + original = cache() + assert original["class"] == "l2cache" and original["state"] == "ONLINE", original + info = server.succeed(f"stat -c '%s %b %a' {image}").split() + assert int(info[0]) == 536870912 and int(info[1]) * 512 >= 536870912 and info[2] == "600", info + server.succeed("systemctl restart snowglobe-zfs-read-cache.service") + assert cache()["guid"] == original["guid"] + server.fail(f"{helper} cache-test {image} 1073741824 0") + assert cache()["guid"] == original["guid"] + + with subtest("warm read hits and bounded cache recycling"): + server.succeed("fio --name=fill --filename=/data/workload --size=768m --rw=write --bs=1m --refill_buffers=1 --end_fsync=1 --output=/tmp/fill.json --output-format=json") + before = counters() + server.succeed("fio --name=warm --filename=/data/workload --size=768m --rw=randread --bs=128k --runtime=45 --time_based --output=/tmp/warm.json --output-format=json", timeout=60) + after = counters() + assert after["l2_hits"] > before["l2_hits"], (before["l2_hits"], after["l2_hits"]) + assert after["l2_write_bytes"] > 536870912, after["l2_write_bytes"] + assert int(server.succeed(f"stat -c %s {image}")) == 536870912 + print(json.dumps({"cache_hits": after["l2_hits"] - before["l2_hits"], "cache_written_bytes": after["l2_write_bytes"]})) + + with subtest("persistent import and missing-cache fallback"): + checksum = server.succeed("sha256sum /data/workload").split()[0] + server.succeed("zpool export cache-test; zpool import -d /dev -d /var/cache/snowglobe/zfs -o cachefile=none cache-test") + assert cache()["state"] == "ONLINE" + assert cache()["guid"] == original["guid"] + server.succeed(f"zpool export cache-test; mv {image} {image}.absent; zpool import -d /dev -o cachefile=none cache-test") + assert server.succeed("zpool list -H -o health cache-test").strip() == "ONLINE" + assert server.succeed("sha256sum /data/workload").split()[0] == checksum + server.fail("systemctl restart snowglobe-zfs-read-cache.service") + server.succeed("printf writable > /data/after-cache-loss") + assert server.succeed("cat /data/after-cache-loss") == "writable" + server.succeed(f"zpool remove cache-test {image}; rm {image}.absent; systemctl restart snowglobe-zfs-read-cache.service") + assert cache()["state"] == "ONLINE" + + with subtest("no overwrite, symlink escape, or root-disk exhaustion"): + server.succeed("printf retained > /var/cache/snowglobe/zfs/occupied") + server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/occupied 67108864 0") + assert server.succeed("cat /var/cache/snowglobe/zfs/occupied") == "retained" + server.succeed("ln -s occupied /var/cache/snowglobe/zfs/link") + server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/link 67108864 0") + server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/too-large 161061273600 34359738368") + server.succeed("test ! -e /var/cache/snowglobe/zfs/too-large") + assert server.succeed("sha256sum /data/workload").split()[0] == checksum + ''; +} diff --git a/nixos/zfs-read-cache.nix b/nixos/zfs-read-cache.nix new file mode 100644 index 0000000000000000000000000000000000000000..11741f83d559effa16d087b450a77ec6ff716b61 --- /dev/null +++ b/nixos/zfs-read-cache.nix @@ -0,0 +1,23 @@ +{ config, pkgs, ... }: +let + pool = config.environment.variables.SNOWGLOBE_POOL; + image = "/var/cache/snowglobe/zfs/l2arc"; + capacity = 150 * 1024 * 1024 * 1024; + headroom = 32 * 1024 * 1024 * 1024; +in +{ + systemd.services.snowglobe-zfs-read-cache = { + wantedBy = [ "multi-user.target" ]; + after = [ "zfs-import-${pool}.service" ]; + path = [ config.boot.zfs.package ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + CacheDirectory = "snowglobe/zfs"; + CacheDirectoryMode = "0700"; + UMask = "0077"; + TimeoutStartSec = "5min"; + ExecStart = "${pkgs.python3}/bin/python3 ${../tools/zfs-read-cache.py} ${pool} ${image} ${toString capacity} ${toString headroom}"; + }; + }; +} diff --git a/tools/zfs-read-cache.md b/tools/zfs-read-cache.md new file mode 100644 index 0000000000000000000000000000000000000000..2d0466ad2bbd267cd3c77a94ea21ded5783165c8 --- /dev/null +++ b/tools/zfs-read-cache.md @@ -0,0 +1,67 @@ +# SSD read cache + +Guest disks remain on the HDD pool. The optional L2ARC holds disposable copies +of recently used blocks on the OS SSD. Cache exhaustion recycles those copies; +guest capacity continues to come from HDD storage. Cache reads that fail are +retried against the HDD pool. Writes still require HDD I/O. + +The host configuration is [zfs-read-cache.nix](../nixos/zfs-read-cache.nix). +It owns the capacity, minimum free space at allocation, and cache-file path; the pool name +comes from the existing host configuration. The service runs after pool import +without making guest startup depend on it. Stopping the service leaves the +cache attached, so normal service restarts preserve its contents. + +The helper preallocates the entire file before attaching it with `zpool add +... cache ...`. It never adds normal storage, a special vdev, or a log device. +The file and directory are restricted to root. Insufficient free space causes +setup to fail before allocation; other host software can still consume the +remaining free space afterward. An existing attached cache must have the +configured size; an unknown file at the destination is preserved and rejected. + +## Inspection and recovery + +Use `systemctl status snowglobe-zfs-read-cache.service`, its journal, and +`zpool status -P` to distinguish service setup failure from pool health. A +missing cache can appear as unavailable while the pool remains online. + +If allocation was interrupted, restarting the service clears its abandoned +temporary allocation. If interruption occurred after publishing the file but +before attaching it, setup preserves that unattached file. Verify that the +exact file is absent from every pool's device list before removing it and +retrying the service. + +To replace or resize a cache, stop its setup service and confirm the exact +device appears under **cache** in the pool status. Remove only that cache +device from the pool before deleting its file. Update the capacity declaration, +deploy the reviewed configuration, and start the service to allocate a fresh +cache. Guest disks stay in place and the cache warms again. Do not truncate, +punch holes in, or replace an attached file. + +Changes to the production pool follow the repository's production deployment +workflow. A staged dashboard cannot exercise this host-level configuration. + +## Verification + +[The isolated NixOS test](../nixos/tests/zfs-read-cache.nix) exercises allocation, +idempotent startup, warmed read hits, bounded cache recycling, import with and +without the cache, writable storage after cache loss, safe replacement, and +rejection of unknown files, symlinks, and insufficient free space. Its small +ARC limit belongs only to the disposable test VM. + +The 2026-10-20 test passed in 64 seconds, recording 97,453 cache hits and +1,234,213,376 cache-write bytes through a fixed 512 MiB cache. Data checksums +matched after import with the cache removed, and writes still succeeded. + +On 2026-10-20, a separate disposable pool on Snow Globe accepted a fully +preallocated 150 GiB cache file on the existing ext4 SSD. Repeated setup kept +the same file. The fixture and allocation were removed afterward; the +production pool was unchanged. The SSD had approximately 200 GiB available +before allocation. Its largest consumers were Nix store contents (93 GiB), +container layers (72 GiB), and VM experiments/templates (about 49 GiB). + +The Windows provisioning samples from that day included Windows Update +servicing. They do not establish a cache-related startup improvement. L2ARC +benefit requires cache hits beyond those already served by RAM; a virtualized +correctness test is not a physical SSD/HDD performance benchmark. + +See [OpenZFS caching documentation](https://openzfs.github.io/openzfs-docs/Basic%20Concepts/Pool%20Structure/Caching.html). diff --git a/tools/zfs-read-cache.py b/tools/zfs-read-cache.py new file mode 100644 index 0000000000000000000000000000000000000000..5f4db355f2c22e7b06242998f365a8bff1d8b235 --- /dev/null +++ b/tools/zfs-read-cache.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Attach a preallocated, disposable file as a ZFS read cache.""" +import fcntl +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile + + +def ensure(pool, image, capacity, headroom): + if capacity < 64 * 2**20 or capacity % 4096 or headroom < 0: + raise ValueError("Cache capacity must be at least 64 MiB and aligned to 4096 bytes.") + if not image.is_absolute() or image.resolve() != image: + raise ValueError("Use an absolute cache path without symlinks.") + image.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with (image.parent / ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + status = json.loads(subprocess.check_output( + ["zpool", "status", "-jP", "--json-flat-vdevs", "--json-int", pool], text=True)) + devices = status["pools"][pool]["vdevs"].values() + attached = next((device for device in devices if device.get("path") == str(image)), None) + if attached: + if attached["class"] != "l2cache" or attached["state"] != "ONLINE": + raise ValueError("The configured cache is not an online cache device; inspect zpool status.") + info = image.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size != capacity: + raise ValueError("The attached cache file differs from its configured capacity or type.") + return + if image.exists(): + raise ValueError("An unattached file already occupies the cache path; preserve or remove it before retrying.") + for abandoned in image.parent.glob(".preparing-*"): + abandoned.unlink() + if shutil.disk_usage(image.parent).free < capacity + headroom: + raise ValueError("Insufficient SSD space for the cache and reserved OS headroom.") + descriptor, temporary = tempfile.mkstemp(prefix=".preparing-", dir=image.parent) + try: + with os.fdopen(descriptor, "wb") as output: + os.posix_fallocate(output.fileno(), 0, capacity) + os.fsync(output.fileno()) + os.link(temporary, image) + os.unlink(temporary) + directory = os.open(image.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + subprocess.run(["zpool", "add", pool, "cache", str(image)], check=True) + finally: + Path(temporary).unlink(missing_ok=True) + + +if __name__ == "__main__": + try: + ensure(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + raise SystemExit(str(error))