| author | |
| committer | |
| log | 5e3b2f6f1899255cd817a4f1b588f217a31b0987 |
| tree | 001110dbf35a4be4b95b54d7f4f65f94c1fab1c0 |
| parent | fb0520340b0e3871c8fb7947484fb087c106e3a2 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Create an encrypted case-sensitive ZFS child at /srv/mac-studio-vm/sandbox. Export it over native NFSv4 to the Mac Studio and expose a separate authenticated, read-only SMB share for artifact pickup. Both servers check the actual mounted dataset before serving it.
Verified with an isolated two-node NixOS/ZFS test covering locks, fsync, executable files, case sensitivity, mount refusal and recovery; authenticated Samba staging verifies read access and write/authentication rejection. Deployment unit tests pass.
Assisted-by: gpt-6-astra
Assisted-by: gpt-6.1-sol6 files changed, 266 insertions(+), 2 deletions(-)
config/mac-studio-sandbox.json created+4| ... | ... | @@ -0,0 +1,4 @@ |
| 1 | { | |
| 2 | "dataset": "globe/clover/mac-studio-sandbox", | |
| 3 | "mountPoint": "/srv/mac-studio-vm/sandbox" | |
| 4 | } |
nixos/mac-studio-sandbox.nix created+70| ... | ... | @@ -0,0 +1,70 @@ |
| 1 | { config, lib, pkgs, ... }: | |
| 2 | let | |
| 3 | workspace = builtins.fromJSON (builtins.readFile ../config/mac-studio-sandbox.json); | |
| 4 | dataset = workspace.dataset; | |
| 5 | mountPoint = workspace.mountPoint; | |
| 6 | parent = builtins.dirOf dataset; | |
| 7 | checkMount = "${pkgs.runtimeShell} ${../service/samba/check-mount.sh} ${mountPoint} ${dataset}"; | |
| 8 | in | |
| 9 | { | |
| 10 | systemd.services.mac-studio-sandbox = { | |
| 11 | wantedBy = [ "multi-user.target" ]; | |
| 12 | after = [ "zfs-mount.service" ]; | |
| 13 | before = [ "nfs-mountd.service" "nfs-server.service" ]; | |
| 14 | requiredBy = [ "nfs-mountd.service" "nfs-server.service" ]; | |
| 15 | path = [ pkgs.zfs pkgs.util-linux pkgs.coreutils pkgs.findutils pkgs.gawk ]; | |
| 16 | serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; | |
| 17 | script = '' | |
| 18 | test "$(zfs get -H -o value keystatus ${parent})" = available | |
| 19 | test "$(zfs get -H -o value encryptionroot ${parent})" = ${parent} | |
| 20 | test "$(readlink -m ${mountPoint})" = ${mountPoint} | |
| 21 | if mountpoint -q ${mountPoint}; then | |
| 22 | ${checkMount} | |
| 23 | else | |
| 24 | test ! -e ${mountPoint} || { test -d ${mountPoint} && test -z "$(find ${mountPoint} -mindepth 1 -maxdepth 1 -print -quit)"; } | |
| 25 | fi | |
| 26 | fresh=false | |
| 27 | if ! zfs list -H -o name ${dataset} >/dev/null 2>&1; then | |
| 28 | mkdir -p ${builtins.dirOf mountPoint} | |
| 29 | zfs create -u -o mountpoint=${mountPoint} -o casesensitivity=sensitive \ | |
| 30 | -o compression=lz4 -o atime=off -o sync=standard -o acltype=posix -o xattr=sa ${dataset} | |
| 31 | fresh=true | |
| 32 | fi | |
| 33 | test "$(zfs get -H -o value mountpoint ${dataset})" = ${mountPoint} | |
| 34 | test "$(zfs get -H -o value encryptionroot ${dataset})" = ${parent} | |
| 35 | test "$(zfs get -H -o value casesensitivity ${dataset})" = sensitive | |
| 36 | test "$(zfs get -H -o value sync ${dataset})" = standard | |
| 37 | test "$(zfs get -H -o value checksum ${dataset})" != off | |
| 38 | if test "$(zfs get -H -o value mounted ${dataset})" != yes; then | |
| 39 | zfs mount ${dataset} | |
| 40 | fi | |
| 41 | ${checkMount} | |
| 42 | if $fresh; then | |
| 43 | chown ${toString config.users.users.clo.uid}:${toString config.users.groups.chloe.gid} ${mountPoint} | |
| 44 | chmod 2770 ${mountPoint} | |
| 45 | fi | |
| 46 | ''; | |
| 47 | }; | |
| 48 | ||
| 49 | services.nfs.server = { | |
| 50 | enable = true; | |
| 51 | hostName = "10.0.0.1"; | |
| 52 | extraNfsdConfig = '' | |
| 53 | vers3 = n | |
| 54 | udp = n | |
| 55 | tcp = y | |
| 56 | ''; | |
| 57 | exports = '' | |
| 58 | ${mountPoint} 10.0.0.222(rw,sync,fsid=0,no_subtree_check,all_squash,anonuid=${toString config.users.users.clo.uid},anongid=${toString config.users.groups.chloe.gid},insecure) | |
| 59 | ''; | |
| 60 | }; | |
| 61 | ||
| 62 | systemd.services.nfs-mountd = { | |
| 63 | path = [ pkgs.gawk ]; | |
| 64 | preStart = lib.mkBefore checkMount; | |
| 65 | }; | |
| 66 | systemd.services.nfs-server = { | |
| 67 | path = [ pkgs.gawk ]; | |
| 68 | preStart = lib.mkBefore checkMount; | |
| 69 | }; | |
| 70 | } |
nixos/snowglobe.nix+4-2| ... | ... | @@ -15,7 +15,7 @@ let |
| 15 | 15 | }; |
| 16 | 16 | in |
| 17 | 17 | { |
| 18 | imports = [ ./snowglobe-network.nix ]; | |
| 18 | imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ]; | |
| 19 | 19 | |
| 20 | 20 | networking.hostName = "snowglobe"; |
| 21 | 21 | environment.etc."machine-info".text = '' |
| ... | ... | @@ -29,6 +29,7 @@ in |
| 29 | 29 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' |
| 30 | 30 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept |
| 31 | 31 | '') config.networking.interfaces.enp4s0.ipv4.addresses + '' |
| 32 | iptables -A nixos-fw -i enp4s0 -s 10.0.0.222/32 -p tcp --dport 2049 -j nixos-fw-accept | |
| 32 | 33 | ip6tables -A nixos-fw -i enp4s0 -s fe80::/10 -p tcp --dport 445 -j nixos-fw-accept |
| 33 | 34 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept |
| 34 | 35 | ''; |
| ... | ... | @@ -81,7 +82,8 @@ in |
| 81 | 82 | |
| 82 | 83 | systemd.services.nomad = { |
| 83 | 84 | requires = [ "zfs-import-${pool}.service" ]; |
| 84 | after = [ "zfs-import-${pool}.service" "zfs-mount.service" ]; | |
| 85 | wants = [ "mac-studio-sandbox.service" ]; | |
| 86 | after = [ "zfs-import-${pool}.service" "zfs-mount.service" "mac-studio-sandbox.service" ]; | |
| 85 | 87 | preStart = '' |
| 86 | 88 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint /srv)" = ${pool} |
| 87 | 89 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint /srv/clover)" = ${pool}/clover |
nixos/tests/mac-studio-sandbox.nix created+147| ... | ... | @@ -0,0 +1,147 @@ |
| 1 | { pkgs }: | |
| 2 | let | |
| 3 | sandbox = builtins.fromJSON (builtins.readFile ../../config/mac-studio-sandbox.json); | |
| 4 | in | |
| 5 | pkgs.testers.runNixOSTest { | |
| 6 | name = "mac-studio-sandbox"; | |
| 7 | globalTimeout = 600; | |
| 8 | ||
| 9 | nodes.server = { | |
| 10 | imports = [ ../mac-studio-sandbox.nix ]; | |
| 11 | virtualisation = { | |
| 12 | memorySize = 3072; | |
| 13 | cores = 2; | |
| 14 | emptyDiskImages = [ 2048 ]; | |
| 15 | }; | |
| 16 | networking.hostId = "a1b2c3d4"; | |
| 17 | networking.interfaces.eth1.ipv4.addresses = [{ address = "10.0.0.1"; prefixLength = 24; }]; | |
| 18 | networking.firewall.allowedTCPPorts = [ 2049 ]; | |
| 19 | users.groups.chloe.gid = 3000; | |
| 20 | users.users.clo = { isSystemUser = true; uid = 3000; group = "chloe"; }; | |
| 21 | boot.supportedFilesystems = [ "zfs" ]; | |
| 22 | boot.zfs.forceImportRoot = false; | |
| 23 | environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.util-linux ]; | |
| 24 | systemd.services.sandbox-test-pool = { | |
| 25 | before = [ "mac-studio-sandbox.service" ]; | |
| 26 | serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; | |
| 27 | script = '' | |
| 28 | test "$(${pkgs.util-linux}/bin/blockdev --getsize64 /dev/vdb)" = 2147483648 | |
| 29 | umask 077 | |
| 30 | head -c 32 /dev/urandom > /run/sandbox-test-key | |
| 31 | ${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=none globe /dev/vdb | |
| 32 | ${pkgs.zfs}/bin/zfs create -o encryption=aes-256-gcm -o keyformat=raw \ | |
| 33 | -o keylocation=file:///run/sandbox-test-key -o mountpoint=/srv/clover \ | |
| 34 | -o casesensitivity=insensitive -o acltype=posixacl globe/clover | |
| 35 | ''; | |
| 36 | }; | |
| 37 | systemd.services.mac-studio-sandbox = { | |
| 38 | requires = [ "sandbox-test-pool.service" ]; | |
| 39 | after = [ "sandbox-test-pool.service" ]; | |
| 40 | }; | |
| 41 | }; | |
| 42 | ||
| 43 | nodes.client = { | |
| 44 | virtualisation = { memorySize = 1024; cores = 1; }; | |
| 45 | networking.interfaces.eth1.ipv4.addresses = [{ address = "10.0.0.222"; prefixLength = 24; }]; | |
| 46 | boot.supportedFilesystems = [ "nfs" ]; | |
| 47 | environment.systemPackages = [ pkgs.python3 pkgs.nfs-utils pkgs.util-linux ]; | |
| 48 | }; | |
| 49 | ||
| 50 | testScript = '' | |
| 51 | mountpoint = ${builtins.toJSON sandbox.mountPoint} | |
| 52 | dataset = ${builtins.toJSON sandbox.dataset} | |
| 53 | start_all() | |
| 54 | server.wait_for_unit("mac-studio-sandbox.service") | |
| 55 | server.wait_for_unit("nfs-server.service") | |
| 56 | client.wait_until_succeeds("ip -4 address show dev eth1 | grep -q 10.0.0.222", timeout=20) | |
| 57 | ||
| 58 | with subtest("fresh encrypted case-sensitive dataset and idempotent setup"): | |
| 59 | assert server.succeed(f"zfs get -H -o value encryptionroot {dataset}").strip() == "globe/clover" | |
| 60 | assert server.succeed(f"zfs get -H -o value casesensitivity {dataset}").strip() == "sensitive" | |
| 61 | assert server.succeed(f"findmnt -n -o SOURCE,FSTYPE --mountpoint {mountpoint}").split() == [dataset, "zfs"] | |
| 62 | assert server.succeed(f"stat -c '%u:%g %a' {mountpoint}").strip() == "3000:3000 2770" | |
| 63 | server.succeed(f"printf retained > {mountpoint}/retained; systemctl restart mac-studio-sandbox.service") | |
| 64 | assert server.succeed(f"cat {mountpoint}/retained") == "retained" | |
| 65 | server.wait_for_unit("nfs-server.service") | |
| 66 | server.succeed("exportfs -v") | |
| 67 | assert "-3" in server.succeed("cat /proc/fs/nfsd/versions").split() | |
| 68 | ||
| 69 | with subtest("shared guard accepts the dataset bind and rejects a covering wrong mount"): | |
| 70 | server.succeed(f"mkdir -p /mnt/guard-bind /mnt/guard-wrong; mount --bind {mountpoint} /mnt/guard-bind") | |
| 71 | server.succeed(f"${pkgs.runtimeShell} ${../../service/samba/check-mount.sh} /mnt/guard-bind {dataset} true") | |
| 72 | server.succeed("mount -t tmpfs -o size=1m tmpfs /mnt/guard-wrong; mount --bind /mnt/guard-wrong /mnt/guard-bind") | |
| 73 | server.fail(f"${pkgs.runtimeShell} ${../../service/samba/check-mount.sh} /mnt/guard-bind {dataset} true") | |
| 74 | server.succeed("umount /mnt/guard-bind; umount /mnt/guard-bind; umount /mnt/guard-wrong") | |
| 75 | ||
| 76 | with subtest("NFS ordinary writes, fsync, rename, execute bits and case distinctions"): | |
| 77 | client.succeed("mkdir -p /mnt/sandbox; timeout 30 mount -t nfs -o vers=4,proto=tcp,hard,noatime 10.0.0.1:/ /mnt/sandbox") | |
| 78 | client.succeed(r"""python3 - <<'PY' | |
| 79 | from pathlib import Path | |
| 80 | import os | |
| 81 | p = Path('/mnt/sandbox') | |
| 82 | with (p / 'pending').open('wb') as f: | |
| 83 | f.write(b'synthetic native NFS bytes\n') | |
| 84 | f.flush() | |
| 85 | os.fsync(f.fileno()) | |
| 86 | (p / 'pending').rename(p / 'finished') | |
| 87 | assert (p / 'finished').read_bytes() == b'synthetic native NFS bytes\n' | |
| 88 | (p / 'A').write_text('uppercase') | |
| 89 | (p / 'a').write_text('lowercase') | |
| 90 | assert (p / 'A').read_text() == 'uppercase' | |
| 91 | assert (p / 'a').read_text() == 'lowercase' | |
| 92 | (p / 'run').write_text('#!/bin/sh\nprintf sandbox-exec') | |
| 93 | (p / 'run').chmod(0o755) | |
| 94 | PY""") | |
| 95 | assert client.succeed("/mnt/sandbox/run") == "sandbox-exec" | |
| 96 | assert server.succeed(f"stat -c '%u:%g' {mountpoint}/finished").strip() == "3000:3000" | |
| 97 | ||
| 98 | with subtest("server observes client POSIX lock and release"): | |
| 99 | client.succeed(r"""python3 - <<'PY' >/run/sandbox-lock.log 2>&1 & | |
| 100 | from pathlib import Path | |
| 101 | import fcntl, time | |
| 102 | with open('/mnt/sandbox/locked', 'w') as f: | |
| 103 | fcntl.lockf(f, fcntl.LOCK_EX) | |
| 104 | Path('/run/sandbox-lock-ready').touch() | |
| 105 | time.sleep(20) | |
| 106 | PY | |
| 107 | echo $! > /run/sandbox-lock-pid | |
| 108 | """) | |
| 109 | client.wait_until_succeeds("test -f /run/sandbox-lock-ready", timeout=15) | |
| 110 | server.succeed(f"""python3 - <<'PY' | |
| 111 | import fcntl | |
| 112 | with open('{mountpoint}/locked', 'r+') as f: | |
| 113 | try: | |
| 114 | fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| 115 | except BlockingIOError: | |
| 116 | pass | |
| 117 | else: | |
| 118 | raise AssertionError('NFS client lock was not enforced on server') | |
| 119 | PY""") | |
| 120 | client.succeed("kill -TERM $(cat /run/sandbox-lock-pid)") | |
| 121 | server.wait_until_succeeds(f"python3 -c \"import fcntl; f=open('{mountpoint}/locked','r+'); fcntl.lockf(f,fcntl.LOCK_EX|fcntl.LOCK_NB)\"", timeout=15) | |
| 122 | ||
| 123 | with subtest("NFS refuses a missing or wrong mounted dataset"): | |
| 124 | client.succeed("timeout 30 umount /mnt/sandbox") | |
| 125 | server.succeed(f"systemctl stop nfs-server.service nfs-mountd.service; zfs unmount {dataset}") | |
| 126 | server.fail("systemctl start nfs-mountd.service") | |
| 127 | server.fail("systemctl start nfs-server.service") | |
| 128 | server.succeed("systemctl reset-failed nfs-server.service nfs-mountd.service") | |
| 129 | server.succeed(f"printf untouched > {mountpoint}/placeholder") | |
| 130 | server.fail("systemctl restart mac-studio-sandbox.service") | |
| 131 | assert server.succeed(f"cat {mountpoint}/placeholder") == "untouched" | |
| 132 | server.succeed(f"rm {mountpoint}/placeholder; systemctl reset-failed mac-studio-sandbox.service") | |
| 133 | server.succeed(f"zfs create -o mountpoint={mountpoint} globe/wrong") | |
| 134 | server.succeed(f"printf untouched > {mountpoint}/wrong-sentinel") | |
| 135 | server.fail("systemctl start nfs-mountd.service") | |
| 136 | server.fail("systemctl start nfs-server.service") | |
| 137 | server.fail("systemctl restart mac-studio-sandbox.service") | |
| 138 | assert server.succeed(f"findmnt -n -o SOURCE --mountpoint {mountpoint}").strip() == "globe/wrong" | |
| 139 | assert server.succeed(f"cat {mountpoint}/wrong-sentinel") == "untouched" | |
| 140 | server.succeed(f"zfs unmount globe/wrong; zfs set mountpoint=none globe/wrong; zfs mount {dataset}") | |
| 141 | server.succeed("systemctl reset-failed; systemctl start mac-studio-sandbox.service nfs-server.service") | |
| 142 | client.succeed("timeout 30 mount -t nfs -o vers=4,proto=tcp,hard,noatime 10.0.0.1:/ /mnt/sandbox") | |
| 143 | assert client.succeed("cat /mnt/sandbox/finished") == "synthetic native NFS bytes\n" | |
| 144 | client.succeed("timeout 30 umount /mnt/sandbox") | |
| 145 | server.succeed("systemctl stop nfs-server.service nfs-mountd.service; zpool status -x") | |
| 146 | ''; | |
| 147 | } |
service/samba/check-mount.sh created+17| ... | ... | @@ -0,0 +1,17 @@ |
| 1 | #!/bin/sh | |
| 2 | set -eu | |
| 3 | exec 3< "$1" | |
| 4 | awk -v point="$1" -v dataset="$2" ' | |
| 5 | FNR == NR { if ($1 == "mnt_id:") id = $2; next } | |
| 6 | $1 == id && $4 == "/" && $5 == point { | |
| 7 | for (i = 6; i <= NF; i++) | |
| 8 | if ($i == "-" && $(i + 1) == "zfs" && $(i + 2) == dataset) found = 1 | |
| 9 | } | |
| 10 | END { exit !found } | |
| 11 | ' /proc/self/fdinfo/3 /proc/self/mountinfo || { | |
| 12 | printf 'Refusing to share %s: expected mounted ZFS dataset %s\n' "$1" "$2" >&2 | |
| 13 | exit 1 | |
| 14 | } | |
| 15 | exec 3<&- | |
| 16 | shift 2 | |
| 17 | exec "$@" |
service/samba/service.pkl+24| ... | ... | @@ -1,6 +1,9 @@ |
| 1 | 1 | amends "../../config/Service.pkl" |
| 2 | 2 | |
| 3 | 3 | import "../../config/site.pkl" as site |
| 4 | import "pkl:json" | |
| 5 | ||
| 6 | local workspace = new json.Parser {}.parse(read("../../config/mac-studio-sandbox.json").text) | |
| 4 | 7 | |
| 5 | 8 | name = "Samba" |
| 6 | 9 | |
| ... | ... | @@ -10,6 +13,16 @@ container { |
| 10 | 13 | hostNetwork = !site.preview |
| 11 | 14 | cpu = 100 |
| 12 | 15 | memory = 256 |
| 16 | entrypoint = "/bin/sh" | |
| 17 | args { | |
| 18 | "/sandbox-check-mount" | |
| 19 | "/shares/mac-studio-sandbox" | |
| 20 | workspace.dataset | |
| 21 | "/container/scripts/entrypoint.sh" | |
| 22 | "runsvdir" | |
| 23 | "-P" | |
| 24 | "/container/config/runit" | |
| 25 | } | |
| 13 | 26 | |
| 14 | 27 | tcp { |
| 15 | 28 | new { |
| ... | ... | @@ -23,6 +36,8 @@ container { |
| 23 | 36 | volumes { |
| 24 | 37 | ["/shares/clover"] { src = site.cloverRoot; readOnly = site.cloverReadOnly } |
| 25 | 38 | ["/shares/clover/Media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } |
| 39 | ["/shares/mac-studio-sandbox"] { src = workspace.mountPoint; readOnly = true } | |
| 40 | ["/sandbox-check-mount"] { config = "check-mount.sh"; readOnly = true } | |
| 26 | 41 | } |
| 27 | 42 | |
| 28 | 43 | env { |
| ... | ... | @@ -48,6 +63,15 @@ container { |
| 48 | 63 | fruit:veto_appledouble = no |
| 49 | 64 | kernel oplocks = yes |
| 50 | 65 | """ |
| 66 | ["SAMBA_VOLUME_CONFIG_mac_studio_sandbox"] = """ | |
| 67 | [mac-studio-sandbox] | |
| 68 | path = /shares/mac-studio-sandbox | |
| 69 | valid users = clo | |
| 70 | read only = yes | |
| 71 | vfs objects = catia fruit streams_xattr | |
| 72 | fruit:veto_appledouble = no | |
| 73 | kernel oplocks = yes | |
| 74 | """ | |
| 51 | 75 | } |
| 52 | 76 | } |
| 53 | 77 |