| author | |
| committer | |
| log | af4fd9c75c708dc3aa91fef61c46acaa82980606 |
| tree | 4a3d661b4db0ef4c47b1e42e3e6be6ba60c331a8 |
| parent | 8d33f673fd185f073c8eb3d203f1c6b84c1cd215 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Keep durable guest disks on HDD and attach a preallocated 150 GiB file as the pool's optional L2ARC. Refuse allocation when it would leave less than 32 GiB free, preserve unknown files, and reuse an attached cache across service restarts.
Verified the full-size allocation on a disposable pool, the production NixOS build, and isolated cache recycling, warmed read hits, import, cache-loss recovery, and path/capacity rejection. No Windows boot-time improvement is claimed.
Assisted-by: gpt-6-astra5 files changed, 234 insertions(+), 1 deletions(-)
nixos/snowglobe.nix+1-1| ... | ... | @@ -16,7 +16,7 @@ let |
| 16 | 16 | }; |
| 17 | 17 | in |
| 18 | 18 | { |
| 19 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ]; | |
| 19 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./zfs-read-cache.nix ]; | |
| 20 | 20 | |
| 21 | 21 | networking.hostName = "snowglobe"; |
| 22 | 22 | environment.etc."machine-info".text = '' |
nixos/tests/zfs-read-cache.nix created+83| ... | ... | @@ -0,0 +1,83 @@ |
| 1 | { pkgs }: | |
| 2 | pkgs.testers.runNixOSTest { | |
| 3 | name = "zfs-read-cache"; | |
| 4 | globalTimeout = 600; | |
| 5 | nodes.server = { lib, ... }: { | |
| 6 | imports = [ ../zfs-read-cache.nix ]; | |
| 7 | virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 4096 ]; }; | |
| 8 | networking.hostId = "c0a1c0a1"; | |
| 9 | boot.supportedFilesystems = [ "zfs" ]; | |
| 10 | boot.zfs.forceImportRoot = false; | |
| 11 | boot.extraModprobeConfig = "options zfs zfs_arc_max=268435456 l2arc_write_max=33554432"; | |
| 12 | environment.variables.SNOWGLOBE_POOL = "cache-test"; | |
| 13 | environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.fio ]; | |
| 14 | systemd.services.cache-test-pool = { | |
| 15 | before = [ "snowglobe-zfs-read-cache.service" ]; | |
| 16 | serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; | |
| 17 | script = "${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=/data cache-test /dev/vdb"; | |
| 18 | }; | |
| 19 | systemd.services.snowglobe-zfs-read-cache = { | |
| 20 | requires = [ "cache-test-pool.service" ]; | |
| 21 | after = [ "cache-test-pool.service" ]; | |
| 22 | serviceConfig.ExecStart = lib.mkForce "${pkgs.python3}/bin/python3 ${../../tools/zfs-read-cache.py} cache-test /var/cache/snowglobe/zfs/l2arc 536870912 33554432"; | |
| 23 | }; | |
| 24 | }; | |
| 25 | testScript = '' | |
| 26 | import json | |
| 27 | start_all() | |
| 28 | server.wait_for_unit("snowglobe-zfs-read-cache.service") | |
| 29 | image = "/var/cache/snowglobe/zfs/l2arc" | |
| 30 | helper = "python3 ${../../tools/zfs-read-cache.py}" | |
| 31 | ||
| 32 | def cache(): | |
| 33 | status = json.loads(server.succeed("zpool status -jP --json-flat-vdevs --json-int cache-test")) | |
| 34 | return next(v for v in status["pools"]["cache-test"]["vdevs"].values() if v.get("path") == image) | |
| 35 | ||
| 36 | def counters(): | |
| 37 | return {row.split()[0]: int(row.split()[2]) for row in server.succeed("cat /proc/spl/kstat/zfs/arcstats").splitlines()[2:]} | |
| 38 | ||
| 39 | with subtest("bounded private cache and idempotent service"): | |
| 40 | original = cache() | |
| 41 | assert original["class"] == "l2cache" and original["state"] == "ONLINE", original | |
| 42 | info = server.succeed(f"stat -c '%s %b %a' {image}").split() | |
| 43 | assert int(info[0]) == 536870912 and int(info[1]) * 512 >= 536870912 and info[2] == "600", info | |
| 44 | server.succeed("systemctl restart snowglobe-zfs-read-cache.service") | |
| 45 | assert cache()["guid"] == original["guid"] | |
| 46 | server.fail(f"{helper} cache-test {image} 1073741824 0") | |
| 47 | assert cache()["guid"] == original["guid"] | |
| 48 | ||
| 49 | with subtest("warm read hits and bounded cache recycling"): | |
| 50 | server.succeed("fio --name=fill --filename=/data/workload --size=768m --rw=write --bs=1m --refill_buffers=1 --end_fsync=1 --output=/tmp/fill.json --output-format=json") | |
| 51 | before = counters() | |
| 52 | server.succeed("fio --name=warm --filename=/data/workload --size=768m --rw=randread --bs=128k --runtime=45 --time_based --output=/tmp/warm.json --output-format=json", timeout=60) | |
| 53 | after = counters() | |
| 54 | assert after["l2_hits"] > before["l2_hits"], (before["l2_hits"], after["l2_hits"]) | |
| 55 | assert after["l2_write_bytes"] > 536870912, after["l2_write_bytes"] | |
| 56 | assert int(server.succeed(f"stat -c %s {image}")) == 536870912 | |
| 57 | print(json.dumps({"cache_hits": after["l2_hits"] - before["l2_hits"], "cache_written_bytes": after["l2_write_bytes"]})) | |
| 58 | ||
| 59 | with subtest("persistent import and missing-cache fallback"): | |
| 60 | checksum = server.succeed("sha256sum /data/workload").split()[0] | |
| 61 | server.succeed("zpool export cache-test; zpool import -d /dev -d /var/cache/snowglobe/zfs -o cachefile=none cache-test") | |
| 62 | assert cache()["state"] == "ONLINE" | |
| 63 | assert cache()["guid"] == original["guid"] | |
| 64 | server.succeed(f"zpool export cache-test; mv {image} {image}.absent; zpool import -d /dev -o cachefile=none cache-test") | |
| 65 | assert server.succeed("zpool list -H -o health cache-test").strip() == "ONLINE" | |
| 66 | assert server.succeed("sha256sum /data/workload").split()[0] == checksum | |
| 67 | server.fail("systemctl restart snowglobe-zfs-read-cache.service") | |
| 68 | server.succeed("printf writable > /data/after-cache-loss") | |
| 69 | assert server.succeed("cat /data/after-cache-loss") == "writable" | |
| 70 | server.succeed(f"zpool remove cache-test {image}; rm {image}.absent; systemctl restart snowglobe-zfs-read-cache.service") | |
| 71 | assert cache()["state"] == "ONLINE" | |
| 72 | ||
| 73 | with subtest("no overwrite, symlink escape, or root-disk exhaustion"): | |
| 74 | server.succeed("printf retained > /var/cache/snowglobe/zfs/occupied") | |
| 75 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/occupied 67108864 0") | |
| 76 | assert server.succeed("cat /var/cache/snowglobe/zfs/occupied") == "retained" | |
| 77 | server.succeed("ln -s occupied /var/cache/snowglobe/zfs/link") | |
| 78 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/link 67108864 0") | |
| 79 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/too-large 161061273600 34359738368") | |
| 80 | server.succeed("test ! -e /var/cache/snowglobe/zfs/too-large") | |
| 81 | assert server.succeed("sha256sum /data/workload").split()[0] == checksum | |
| 82 | ''; | |
| 83 | } |
nixos/zfs-read-cache.nix created+23| ... | ... | @@ -0,0 +1,23 @@ |
| 1 | { config, pkgs, ... }: | |
| 2 | let | |
| 3 | pool = config.environment.variables.SNOWGLOBE_POOL; | |
| 4 | image = "/var/cache/snowglobe/zfs/l2arc"; | |
| 5 | capacity = 150 * 1024 * 1024 * 1024; | |
| 6 | headroom = 32 * 1024 * 1024 * 1024; | |
| 7 | in | |
| 8 | { | |
| 9 | systemd.services.snowglobe-zfs-read-cache = { | |
| 10 | wantedBy = [ "multi-user.target" ]; | |
| 11 | after = [ "zfs-import-${pool}.service" ]; | |
| 12 | path = [ config.boot.zfs.package ]; | |
| 13 | serviceConfig = { | |
| 14 | Type = "oneshot"; | |
| 15 | RemainAfterExit = true; | |
| 16 | CacheDirectory = "snowglobe/zfs"; | |
| 17 | CacheDirectoryMode = "0700"; | |
| 18 | UMask = "0077"; | |
| 19 | TimeoutStartSec = "5min"; | |
| 20 | ExecStart = "${pkgs.python3}/bin/python3 ${../tools/zfs-read-cache.py} ${pool} ${image} ${toString capacity} ${toString headroom}"; | |
| 21 | }; | |
| 22 | }; | |
| 23 | } |
tools/zfs-read-cache.md created+67| ... | ... | @@ -0,0 +1,67 @@ |
| 1 | # SSD read cache | |
| 2 | ||
| 3 | Guest disks remain on the HDD pool. The optional L2ARC holds disposable copies | |
| 4 | of recently used blocks on the OS SSD. Cache exhaustion recycles those copies; | |
| 5 | guest capacity continues to come from HDD storage. Cache reads that fail are | |
| 6 | retried against the HDD pool. Writes still require HDD I/O. | |
| 7 | ||
| 8 | The host configuration is [zfs-read-cache.nix](../nixos/zfs-read-cache.nix). | |
| 9 | It owns the capacity, minimum free space at allocation, and cache-file path; the pool name | |
| 10 | comes from the existing host configuration. The service runs after pool import | |
| 11 | without making guest startup depend on it. Stopping the service leaves the | |
| 12 | cache attached, so normal service restarts preserve its contents. | |
| 13 | ||
| 14 | The helper preallocates the entire file before attaching it with `zpool add | |
| 15 | ... cache ...`. It never adds normal storage, a special vdev, or a log device. | |
| 16 | The file and directory are restricted to root. Insufficient free space causes | |
| 17 | setup to fail before allocation; other host software can still consume the | |
| 18 | remaining free space afterward. An existing attached cache must have the | |
| 19 | configured size; an unknown file at the destination is preserved and rejected. | |
| 20 | ||
| 21 | ## Inspection and recovery | |
| 22 | ||
| 23 | Use `systemctl status snowglobe-zfs-read-cache.service`, its journal, and | |
| 24 | `zpool status -P` to distinguish service setup failure from pool health. A | |
| 25 | missing cache can appear as unavailable while the pool remains online. | |
| 26 | ||
| 27 | If allocation was interrupted, restarting the service clears its abandoned | |
| 28 | temporary allocation. If interruption occurred after publishing the file but | |
| 29 | before attaching it, setup preserves that unattached file. Verify that the | |
| 30 | exact file is absent from every pool's device list before removing it and | |
| 31 | retrying the service. | |
| 32 | ||
| 33 | To replace or resize a cache, stop its setup service and confirm the exact | |
| 34 | device appears under **cache** in the pool status. Remove only that cache | |
| 35 | device from the pool before deleting its file. Update the capacity declaration, | |
| 36 | deploy the reviewed configuration, and start the service to allocate a fresh | |
| 37 | cache. Guest disks stay in place and the cache warms again. Do not truncate, | |
| 38 | punch holes in, or replace an attached file. | |
| 39 | ||
| 40 | Changes to the production pool follow the repository's production deployment | |
| 41 | workflow. A staged dashboard cannot exercise this host-level configuration. | |
| 42 | ||
| 43 | ## Verification | |
| 44 | ||
| 45 | [The isolated NixOS test](../nixos/tests/zfs-read-cache.nix) exercises allocation, | |
| 46 | idempotent startup, warmed read hits, bounded cache recycling, import with and | |
| 47 | without the cache, writable storage after cache loss, safe replacement, and | |
| 48 | rejection of unknown files, symlinks, and insufficient free space. Its small | |
| 49 | ARC limit belongs only to the disposable test VM. | |
| 50 | ||
| 51 | The 2026-10-20 test passed in 64 seconds, recording 97,453 cache hits and | |
| 52 | 1,234,213,376 cache-write bytes through a fixed 512 MiB cache. Data checksums | |
| 53 | matched after import with the cache removed, and writes still succeeded. | |
| 54 | ||
| 55 | On 2026-10-20, a separate disposable pool on Snow Globe accepted a fully | |
| 56 | preallocated 150 GiB cache file on the existing ext4 SSD. Repeated setup kept | |
| 57 | the same file. The fixture and allocation were removed afterward; the | |
| 58 | production pool was unchanged. The SSD had approximately 200 GiB available | |
| 59 | before allocation. Its largest consumers were Nix store contents (93 GiB), | |
| 60 | container layers (72 GiB), and VM experiments/templates (about 49 GiB). | |
| 61 | ||
| 62 | The Windows provisioning samples from that day included Windows Update | |
| 63 | servicing. They do not establish a cache-related startup improvement. L2ARC | |
| 64 | benefit requires cache hits beyond those already served by RAM; a virtualized | |
| 65 | correctness test is not a physical SSD/HDD performance benchmark. | |
| 66 | ||
| 67 | See [OpenZFS caching documentation](https://openzfs.github.io/openzfs-docs/Basic%20Concepts/Pool%20Structure/Caching.html). |
tools/zfs-read-cache.py created+60| ... | ... | @@ -0,0 +1,60 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | """Attach a preallocated, disposable file as a ZFS read cache.""" | |
| 3 | import fcntl | |
| 4 | import json | |
| 5 | import os | |
| 6 | from pathlib import Path | |
| 7 | import shutil | |
| 8 | import stat | |
| 9 | import subprocess | |
| 10 | import sys | |
| 11 | import tempfile | |
| 12 | ||
| 13 | ||
| 14 | def ensure(pool, image, capacity, headroom): | |
| 15 | if capacity < 64 * 2**20 or capacity % 4096 or headroom < 0: | |
| 16 | raise ValueError("Cache capacity must be at least 64 MiB and aligned to 4096 bytes.") | |
| 17 | if not image.is_absolute() or image.resolve() != image: | |
| 18 | raise ValueError("Use an absolute cache path without symlinks.") | |
| 19 | image.parent.mkdir(mode=0o700, parents=True, exist_ok=True) | |
| 20 | with (image.parent / ".lock").open("a") as lock: | |
| 21 | fcntl.flock(lock, fcntl.LOCK_EX) | |
| 22 | status = json.loads(subprocess.check_output( | |
| 23 | ["zpool", "status", "-jP", "--json-flat-vdevs", "--json-int", pool], text=True)) | |
| 24 | devices = status["pools"][pool]["vdevs"].values() | |
| 25 | attached = next((device for device in devices if device.get("path") == str(image)), None) | |
| 26 | if attached: | |
| 27 | if attached["class"] != "l2cache" or attached["state"] != "ONLINE": | |
| 28 | raise ValueError("The configured cache is not an online cache device; inspect zpool status.") | |
| 29 | info = image.lstat() | |
| 30 | if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size != capacity: | |
| 31 | raise ValueError("The attached cache file differs from its configured capacity or type.") | |
| 32 | return | |
| 33 | if image.exists(): | |
| 34 | raise ValueError("An unattached file already occupies the cache path; preserve or remove it before retrying.") | |
| 35 | for abandoned in image.parent.glob(".preparing-*"): | |
| 36 | abandoned.unlink() | |
| 37 | if shutil.disk_usage(image.parent).free < capacity + headroom: | |
| 38 | raise ValueError("Insufficient SSD space for the cache and reserved OS headroom.") | |
| 39 | descriptor, temporary = tempfile.mkstemp(prefix=".preparing-", dir=image.parent) | |
| 40 | try: | |
| 41 | with os.fdopen(descriptor, "wb") as output: | |
| 42 | os.posix_fallocate(output.fileno(), 0, capacity) | |
| 43 | os.fsync(output.fileno()) | |
| 44 | os.link(temporary, image) | |
| 45 | os.unlink(temporary) | |
| 46 | directory = os.open(image.parent, os.O_RDONLY | os.O_DIRECTORY) | |
| 47 | try: | |
| 48 | os.fsync(directory) | |
| 49 | finally: | |
| 50 | os.close(directory) | |
| 51 | subprocess.run(["zpool", "add", pool, "cache", str(image)], check=True) | |
| 52 | finally: | |
| 53 | Path(temporary).unlink(missing_ok=True) | |
| 54 | ||
| 55 | ||
| 56 | if __name__ == "__main__": | |
| 57 | try: | |
| 58 | ensure(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) | |
| 59 | except (ValueError, OSError, subprocess.CalledProcessError) as error: | |
| 60 | raise SystemExit(str(error)) |