| 1 | { pkgs }: |
| 2 | pkgs.testers.runNixOSTest { |
| 3 | name = "zfs-read-cache"; |
| 4 | globalTimeout = 600; |
| 5 | nodes.server = { lib, ... }: { |
| 6 | imports = [ ../zfs-read-cache.nix ]; |
| 7 | virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 4096 ]; }; |
| 8 | networking.hostId = "c0a1c0a1"; |
| 9 | boot.supportedFilesystems = [ "zfs" ]; |
| 10 | boot.zfs.forceImportRoot = false; |
| 11 | boot.extraModprobeConfig = "options zfs zfs_arc_max=268435456 l2arc_write_max=33554432"; |
| 12 | environment.variables.SNOWGLOBE_POOL = "cache-test"; |
| 13 | environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.fio ]; |
| 14 | systemd.services.cache-test-pool = { |
| 15 | before = [ "snowglobe-zfs-read-cache.service" ]; |
| 16 | serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; |
| 17 | script = "${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=/data cache-test /dev/vdb"; |
| 18 | }; |
| 19 | systemd.services.snowglobe-zfs-read-cache = { |
| 20 | requires = [ "cache-test-pool.service" ]; |
| 21 | after = [ "cache-test-pool.service" ]; |
| 22 | serviceConfig.ExecStart = lib.mkForce "${pkgs.python3}/bin/python3 ${../../tools/zfs-read-cache.py} cache-test /var/cache/snowglobe/zfs/l2arc 536870912 33554432"; |
| 23 | }; |
| 24 | }; |
| 25 | testScript = '' |
| 26 | import json |
| 27 | start_all() |
| 28 | server.wait_for_unit("snowglobe-zfs-read-cache.service") |
| 29 | image = "/var/cache/snowglobe/zfs/l2arc" |
| 30 | helper = "python3 ${../../tools/zfs-read-cache.py}" |
| 31 | |
| 32 | def cache(): |
| 33 | status = json.loads(server.succeed("zpool status -jP --json-flat-vdevs --json-int cache-test")) |
| 34 | return next(v for v in status["pools"]["cache-test"]["vdevs"].values() if v.get("path") == image) |
| 35 | |
| 36 | def counters(): |
| 37 | return {row.split()[0]: int(row.split()[2]) for row in server.succeed("cat /proc/spl/kstat/zfs/arcstats").splitlines()[2:]} |
| 38 | |
| 39 | with subtest("bounded private cache and idempotent service"): |
| 40 | original = cache() |
| 41 | assert original["class"] == "l2cache" and original["state"] == "ONLINE", original |
| 42 | info = server.succeed(f"stat -c '%s %b %a' {image}").split() |
| 43 | assert int(info[0]) == 536870912 and int(info[1]) * 512 >= 536870912 and info[2] == "600", info |
| 44 | server.succeed("systemctl restart snowglobe-zfs-read-cache.service") |
| 45 | assert cache()["guid"] == original["guid"] |
| 46 | server.fail(f"{helper} cache-test {image} 1073741824 0") |
| 47 | assert cache()["guid"] == original["guid"] |
| 48 | |
| 49 | with subtest("warm read hits and bounded cache recycling"): |
| 50 | server.succeed("fio --name=fill --filename=/data/workload --size=768m --rw=write --bs=1m --refill_buffers=1 --end_fsync=1 --output=/tmp/fill.json --output-format=json") |
| 51 | before = counters() |
| 52 | server.succeed("fio --name=warm --filename=/data/workload --size=768m --rw=randread --bs=128k --runtime=45 --time_based --output=/tmp/warm.json --output-format=json", timeout=60) |
| 53 | after = counters() |
| 54 | assert after["l2_hits"] > before["l2_hits"], (before["l2_hits"], after["l2_hits"]) |
| 55 | assert after["l2_write_bytes"] > 536870912, after["l2_write_bytes"] |
| 56 | assert int(server.succeed(f"stat -c %s {image}")) == 536870912 |
| 57 | print(json.dumps({"cache_hits": after["l2_hits"] - before["l2_hits"], "cache_written_bytes": after["l2_write_bytes"]})) |
| 58 | |
| 59 | with subtest("persistent import and missing-cache fallback"): |
| 60 | checksum = server.succeed("sha256sum /data/workload").split()[0] |
| 61 | server.succeed("zpool export cache-test; zpool import -d /dev -d /var/cache/snowglobe/zfs -o cachefile=none cache-test") |
| 62 | assert cache()["state"] == "ONLINE" |
| 63 | assert cache()["guid"] == original["guid"] |
| 64 | server.succeed(f"zpool export cache-test; mv {image} {image}.absent; zpool import -d /dev -o cachefile=none cache-test") |
| 65 | assert server.succeed("zpool list -H -o health cache-test").strip() == "ONLINE" |
| 66 | assert server.succeed("sha256sum /data/workload").split()[0] == checksum |
| 67 | server.fail("systemctl restart snowglobe-zfs-read-cache.service") |
| 68 | server.succeed("printf writable > /data/after-cache-loss") |
| 69 | assert server.succeed("cat /data/after-cache-loss") == "writable" |
| 70 | server.succeed(f"zpool remove cache-test {image}; rm {image}.absent; systemctl restart snowglobe-zfs-read-cache.service") |
| 71 | assert cache()["state"] == "ONLINE" |
| 72 | |
| 73 | with subtest("no overwrite, symlink escape, or root-disk exhaustion"): |
| 74 | server.succeed("printf retained > /var/cache/snowglobe/zfs/occupied") |
| 75 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/occupied 67108864 0") |
| 76 | assert server.succeed("cat /var/cache/snowglobe/zfs/occupied") == "retained" |
| 77 | server.succeed("ln -s occupied /var/cache/snowglobe/zfs/link") |
| 78 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/link 67108864 0") |
| 79 | server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/too-large 161061273600 34359738368") |
| 80 | server.succeed("test ! -e /var/cache/snowglobe/zfs/too-large") |
| 81 | assert server.succeed("sha256sum /data/workload").split()[0] == checksum |
| 82 | ''; |
| 83 | } |