| author | |
| committer | |
| log | 30505210d99fd254c7e0b2f5e13d55a304b8de94 |
| tree | c96ae2c1b36429b390430b500649d3e97693bbfb |
| parent | 072baa047d461286cc5cba586cbf6dbdbde85a5e |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Load the updated Linux bundle and configure the registered Snow Globe client
through an allocation-rendered private file with a read-only container mount.
Keep the existing service UID, state, provider home, workspaces and archives.
Create the archive identity only on the confirmed ZFS mount, preserving an
existing marker, and replace the obsolete manual hub setup with hosted SSO.
Verified real owner sign-in, renewal, CSRF rejection, authenticated WebSockets,
synthetic archive upload/download and ranges, and missing-marker fail-closed
behavior in a fresh Zenith preview. Passed 15 local tests and all Nomad job
validation checks.
Assisted-by: gpt-6.1-sol9 files changed, 293 insertions(+), 199 deletions(-)
config/Service.pkl+2| ... | ... | @@ -88,6 +88,8 @@ class Container { |
| 88 | 88 | env: Mapping<String, String> = new {} |
| 89 | 89 | /// Nomad template for environment values resolved after allocation. |
| 90 | 90 | envTemplate: String? |
| 91 | /// Nomad templates rendered privately and mounted read-only at absolute container paths. | |
| 92 | secretFiles: Mapping<String, String> = new {} | |
| 91 | 93 | args: Listing<String> = new {} |
| 92 | 94 | capAdd: Listing<String> = new {} |
| 93 | 95 | devices: Listing<String> = new {} |
service/knife/bundle.json+2-2| ... | ... | @@ -1,5 +1,5 @@ |
| 1 | 1 | { |
| 2 | 2 | "url": "https://file.paperclover.net/shr/the-knife/knife-server-linux-amd64.tar.gz", |
| 3 | "sha256": "02d9dda61a4de554c5640e6039d2fe9bc5ebcb71d3967efa1930ffc953614102", | |
| 4 | "image": "sha256:027c96783e914e8a7b4ab3351e89e2c2f75845e889bf81e19aa5b2e87c8a01a6" | |
| 3 | "sha256": "c432d9e6565608367beaf6cbd98ea8ade498f26ecd2ac5becca6186615376e44", | |
| 4 | "image": "sha256:6135e4adb01771ac33cb45b7d2874272883c5ecacf119b847cd7fc76a76349f1" | |
| 5 | 5 | } |
service/knife/prepare.py+28| ... | ... | @@ -1,12 +1,15 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | import hashlib |
| 3 | 3 | import json |
| 4 | import os | |
| 4 | 5 | from pathlib import Path |
| 5 | 6 | import shutil |
| 6 | 7 | import subprocess |
| 8 | import sys | |
| 7 | 9 | import tarfile |
| 8 | 10 | import tempfile |
| 9 | 11 | import urllib.request |
| 12 | import uuid | |
| 10 | 13 | |
| 11 | 14 | |
| 12 | 15 | bundle = json.loads(Path(__file__).with_name("bundle.json").read_text()) |
| ... | ... | @@ -29,3 +32,28 @@ if subprocess.run(["podman", "image", "exists", bundle["image"]]).returncode: |
| 29 | 32 | shutil.copyfileobj(contents, target) |
| 30 | 33 | subprocess.run(["podman", "load", "--quiet", "--input", str(image)], check=True) |
| 31 | 34 | subprocess.run(["podman", "image", "exists", bundle["image"]], check=True) |
| 35 | ||
| 36 | root = Path(json.load(sys.stdin)["hostRoot"]) | |
| 37 | archives = root / "archives" | |
| 38 | mount = subprocess.run( | |
| 39 | ["findmnt", "-rn", "-o", "TARGET,FSTYPE", "--target", str(archives)], | |
| 40 | check=True, capture_output=True, text=True, | |
| 41 | ).stdout.strip() | |
| 42 | if root.resolve() != root or archives.resolve() != archives or mount != f"{root} zfs": | |
| 43 | raise ValueError("Knife archives must be on the mounted service ZFS dataset") | |
| 44 | marker = archives / ".knife-archive-store-id" | |
| 45 | try: | |
| 46 | with marker.open("x") as file: | |
| 47 | file.write(str(uuid.uuid4()) + "\n") | |
| 48 | os.fchmod(file.fileno(), 0o444) | |
| 49 | file.flush() | |
| 50 | os.fsync(file.fileno()) | |
| 51 | directory = os.open(archives, os.O_RDONLY | os.O_DIRECTORY) | |
| 52 | try: | |
| 53 | os.fsync(directory) | |
| 54 | finally: | |
| 55 | os.close(directory) | |
| 56 | except FileExistsError: | |
| 57 | pass | |
| 58 | if marker.is_symlink() or not marker.is_file() or not marker.read_text().strip(): | |
| 59 | raise ValueError("Knife archive store identity must be a nonempty regular file") |
service/knife/service.pkl+15-2| ... | ... | @@ -15,7 +15,6 @@ requirements { |
| 15 | 15 | } |
| 16 | 16 | stageIsolation = "fresh" |
| 17 | 17 | prepare = "prepare.py" |
| 18 | setup = "setup.py" | |
| 19 | 18 | |
| 20 | 19 | container { |
| 21 | 20 | image = bundle.image |
| ... | ... | @@ -26,7 +25,7 @@ container { |
| 26 | 25 | "--" |
| 27 | 26 | "/bin/sh" |
| 28 | 27 | "-ec" |
| 29 | "node /opt/knife/scripts/container-runtime.ts; exec knife start --host 0.0.0.0 --port 3773 --no-browser" | |
| 28 | "T3CODE_ARCHIVE_STORE_ID=$(cat /archives/.knife-archive-store-id); export T3CODE_ARCHIVE_STORE_ID; node /opt/knife/scripts/container-runtime.ts; exec knife start --host 0.0.0.0 --port 3773 --no-browser" | |
| 30 | 29 | } |
| 31 | 30 | http { |
| 32 | 31 | containerPort = 3773 |
| ... | ... | @@ -46,10 +45,24 @@ container { |
| 46 | 45 | ["HOME"] = "/home/node" |
| 47 | 46 | ["KNIFE_UPDATE_URL"] = "https://file.paperclover.net/shr/the-knife/" |
| 48 | 47 | ["T3CODE_HOME"] = "/data" |
| 48 | ["T3CODE_SSO_CONFIG"] = "/run/secrets/knife-oidc.json" | |
| 49 | 49 | ["T3CODE_ARCHIVE_DIR"] = "/archives" |
| 50 | 50 | ["T3CODE_ARCHIVE_OFFLOAD"] = "true" |
| 51 | 51 | ["T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD"] = "false" |
| 52 | 52 | // Info-level startup logs include a live owner pairing credential. |
| 53 | 53 | ["T3CODE_LOG_LEVEL"] = "Warn" |
| 54 | 54 | } |
| 55 | secretFiles { | |
| 56 | ["/run/secrets/knife-oidc.json"] = """ | |
| 57 | {{ with nomadVar "nomad/jobs/\(module.id)/inputs/oidc" }} | |
| 58 | { | |
| 59 | "publicUrl": "https://\(module.container.http.hostname)", | |
| 60 | "issuer": {{ .issuerUrl.Value | toJSON }}, | |
| 61 | "clientId": {{ .clientId.Value | toJSON }}, | |
| 62 | "clientSecret": {{ .clientSecret.Value | toJSON }}, | |
| 63 | "ownerSub": "8c909fab-98b0-4472-9171-9606ccebe9d5" | |
| 64 | } | |
| 65 | {{ end }} | |
| 66 | """ | |
| 67 | } | |
| 55 | 68 | } |
service/knife/setup.py deleted-19| ... | ... | @@ -1,19 +0,0 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import json | |
| 3 | import os | |
| 4 | from pathlib import Path | |
| 5 | import subprocess | |
| 6 | import sys | |
| 7 | ||
| 8 | ||
| 9 | data = json.load(sys.stdin) | |
| 10 | allocations = json.loads(subprocess.check_output( | |
| 11 | ["nomad", "job", "allocs", "-json", data["serviceId"]], | |
| 12 | env={**os.environ, "NOMAD_TOKEN": Path("/var/lib/studio/nomad.token").read_text().strip()}, | |
| 13 | )) | |
| 14 | running = [item["ID"] for item in allocations if item["ClientStatus"] == "running" and item["DesiredStatus"] == "run"] | |
| 15 | if len(running) != 1: | |
| 16 | raise ValueError("Knife setup requires one running allocation") | |
| 17 | subprocess.run([ | |
| 18 | "podman", "exec", "app-" + running[0], "knife", "shared", "hub", "configure", "https://" + data["host"], | |
| 19 | ], check=True, capture_output=True) |
tools/knife-test.py created+71| ... | ... | @@ -0,0 +1,71 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import io | |
| 3 | import json | |
| 4 | from pathlib import Path | |
| 5 | import runpy | |
| 6 | import subprocess | |
| 7 | import tempfile | |
| 8 | import unittest | |
| 9 | from unittest.mock import patch | |
| 10 | import uuid | |
| 11 | ||
| 12 | ||
| 13 | class ArchiveIdentity(unittest.TestCase): | |
| 14 | def setUp(self): | |
| 15 | temporary = tempfile.TemporaryDirectory(prefix="knife-archive-test-") | |
| 16 | self.addCleanup(temporary.cleanup) | |
| 17 | self.root = Path(temporary.name).resolve() | |
| 18 | self.archives = self.root / "archives" | |
| 19 | self.archives.mkdir() | |
| 20 | self.marker = self.archives / ".knife-archive-store-id" | |
| 21 | ||
| 22 | def prepare(self, mount=None): | |
| 23 | def command(args, **kwargs): | |
| 24 | if args[:3] == ["podman", "image", "exists"]: | |
| 25 | return subprocess.CompletedProcess(args, 0) | |
| 26 | self.assertEqual(args[0], "findmnt") | |
| 27 | self.assertEqual(args[-1], str(self.archives)) | |
| 28 | return subprocess.CompletedProcess(args, 0, stdout=mount or f"{self.root} zfs\n") | |
| 29 | with patch("subprocess.run", side_effect=command), patch("sys.stdin", io.StringIO(json.dumps({"hostRoot": str(self.root)}))): | |
| 30 | runpy.run_path(str(Path(__file__).resolve().parents[1] / "service/knife/prepare.py")) | |
| 31 | ||
| 32 | def test_create_on_confirmed_zfs_and_preserve_existing_identity(self): | |
| 33 | self.prepare() | |
| 34 | uuid.UUID(self.marker.read_text().strip()) | |
| 35 | self.assertEqual(self.marker.stat().st_mode & 0o777, 0o444) | |
| 36 | self.marker.unlink() | |
| 37 | self.marker.write_text("existing-nas-identity\n") | |
| 38 | self.prepare() | |
| 39 | self.assertEqual(self.marker.read_bytes(), b"existing-nas-identity\n") | |
| 40 | ||
| 41 | def test_wrong_or_fallback_mount_cannot_create_marker(self): | |
| 42 | for mount in ["/ ext4", f"{self.root} ext4", f"{self.archives} zfs"]: | |
| 43 | with self.subTest(mount=mount), self.assertRaises(ValueError): | |
| 44 | self.prepare(mount) | |
| 45 | self.assertFalse(self.marker.exists()) | |
| 46 | ||
| 47 | def test_symlinked_archive_directory_cannot_create_marker(self): | |
| 48 | self.archives.rmdir() | |
| 49 | target = self.root / "elsewhere" | |
| 50 | target.mkdir() | |
| 51 | self.archives.symlink_to(target, target_is_directory=True) | |
| 52 | with self.assertRaises(ValueError): | |
| 53 | self.prepare() | |
| 54 | self.assertFalse((target / self.marker.name).exists()) | |
| 55 | ||
| 56 | def test_invalid_existing_marker_is_preserved_and_rejected(self): | |
| 57 | self.marker.write_text("") | |
| 58 | with self.assertRaises(ValueError): | |
| 59 | self.prepare() | |
| 60 | self.assertEqual(self.marker.read_bytes(), b"") | |
| 61 | self.marker.unlink() | |
| 62 | target = self.root / "outside-marker" | |
| 63 | target.write_text("outside\n") | |
| 64 | self.marker.symlink_to(target) | |
| 65 | with self.assertRaises(ValueError): | |
| 66 | self.prepare() | |
| 67 | self.assertEqual(target.read_text(), "outside\n") | |
| 68 | ||
| 69 | ||
| 70 | if __name__ == "__main__": | |
| 71 | unittest.main() |
tools/knife.md+80-176| ... | ... | @@ -1,184 +1,88 @@ |
| 1 | # Knife setup and operation | |
| 2 | ||
| 3 | `https://knife.paperclover.net` runs the private Knife authority, archive | |
| 4 | receiver, and optional environment hub. Native single-use pairing controls | |
| 5 | access; there is no public registration. The service's four directories live | |
| 6 | on the managed NAS dataset `/srv/prod/knife`: `data`, `archives`, `home/node`, | |
| 7 | and `workspaces`. Provider credentials can stay on each laptop. | |
| 8 | ||
| 9 | ## Pair the owner | |
| 10 | ||
| 11 | Run these commands in your terminal. SSH access to Zenith is the recovery | |
| 12 | path; no owner credential is written to application or deployment logs. | |
| 1 | # Knife sign-in and archive storage | |
| 2 | ||
| 3 | Open https://knife.paperclover.net and sign in through Snow Globe. The server | |
| 4 | accepts only the configured issuer and Clover's immutable owner subject. The NAS | |
| 5 | environment is **snow globe**. Provider accounts can stay on each laptop; the NAS | |
| 6 | needs none for archive storage or outbound environment routing. | |
| 7 | ||
| 8 | On each machine, install the current Knife desktop release, choose its | |
| 9 | self-hosted server connection, enter `https://knife.paperclover.net`, and complete | |
| 10 | browser sign-in. The desktop returns through its proof-bound application callback; | |
| 11 | only the server's HTTPS callback is registered with Snow Globe. Laptop connections | |
| 12 | run outbound through Knife, with archive offload enabled automatically. No laptop | |
| 13 | port forwarding, SSH pairing, bearer-token file, or manual hub enrollment is needed | |
| 14 | for the SSO connection. | |
| 15 | ||
| 16 | Archive a disposable conversation first, reopen its cold transcript, then unarchive | |
| 17 | it. Confirm that its source connection and archive queue report success before | |
| 18 | relying on offload for existing history. A sleeping laptop cannot run interactive | |
| 19 | agents; uploaded archives remain on the NAS. Successful offload reclaims local | |
| 20 | SQLite pages after publication. Native provider history files remain on the laptop. | |
| 21 | ||
| 22 | ## Access and recovery | |
| 23 | ||
| 24 | SSO access credentials last fifteen minutes, with rotating device credentials in | |
| 25 | operating-system protected storage. Browser sign-in is required again after thirty | |
| 26 | days. The server revalidates the issuer grant during renewal at least hourly; | |
| 27 | issuer revocation can therefore take an hour plus fifteen minutes to fully apply. | |
| 28 | Local device revocation disables renewal and closes active connections immediately. | |
| 29 | ||
| 30 | Existing manual pairing credentials survive the upgrade. Manage them separately | |
| 31 | from SSO devices under access settings; do not share pairing links or bearer tokens. | |
| 32 | The listener binds only to loopback behind HTTPS, with WebSocket proxying. Hosted | |
| 33 | SSO cookies are Secure, HttpOnly, SameSite=Lax and protected against cross-origin | |
| 34 | writes. | |
| 35 | ||
| 36 | For local recovery, derive the current container from Nomad rather than retaining | |
| 37 | an allocation ID: | |
| 13 | 38 | |
| 14 | 39 | ```sh |
| 15 | 40 | ssh -p 22 root@10.0.0.1 |
| 16 | 41 | export NOMAD_TOKEN="$(cat /var/lib/studio/nomad.token)" |
| 17 | 42 | knife_allocation=$(nomad job allocs -json knife | python3 -c 'import json,sys; a=[r["ID"] for r in json.load(sys.stdin) if r["ClientStatus"]=="running" and r["DesiredStatus"]=="run"]; assert len(a)==1; print(a[0])') |
| 18 | 43 | knife_container="app-$knife_allocation" |
| 19 | podman exec "$knife_container" knife auth pairing create \ | |
| 20 | --ttl 5m --label 'Clover owner' --base-url https://knife.paperclover.net \ | |
| 21 | --scope orchestration:read --scope orchestration:operate \ | |
| 22 | --scope settings:write --scope providers:manage \ | |
| 23 | --scope environment:maintain --scope preview:operate \ | |
| 24 | --scope diagnostics:read --scope terminal:read --scope terminal:operate \ | |
| 25 | --scope source-control:write --scope filesystem:read --scope filesystem:write \ | |
| 26 | --scope relay:read --scope relay:write --scope access:read --scope access:write | |
| 27 | ``` | |
| 28 | ||
| 29 | Open the printed pairing link privately within five minutes. It is single-use. | |
| 30 | An owner pairing needs the full scope list: supplying `--scope` replaces the | |
| 31 | CLI defaults, and default pairing omits access management. The browser session | |
| 32 | lasts 30 days. Repeat the command for another owner device or browser. | |
| 33 | ||
| 34 | In the desktop app, choose **Self-hosted server**, enter | |
| 35 | `https://knife.paperclover.net`, and paste a fresh pairing code. The same entry | |
| 36 | is under **Settings → Environments**. Choose this path to avoid T3 Connect. | |
| 37 | A browser paired to the NAS controls the NAS environment; laptop transcripts | |
| 38 | remain in their originating environments until those laptops are connected. | |
| 39 | ||
| 40 | The NAS needs no provider login for archive storage or hub routing. Configure | |
| 41 | providers on the NAS only if agents should actually run there. Its writable | |
| 42 | workspace is `/workspaces`; the personal file archive and host credentials are | |
| 43 | not mounted into the container. | |
| 44 | ||
| 45 | ## Enable laptop archive offload | |
| 46 | ||
| 47 | UI pairing and archive offload are separate connections. Repeat these steps for | |
| 48 | each laptop, using a separate, revocable token and label. | |
| 49 | ||
| 50 | From the Mac, create a private local token file by capturing SSH output directly. | |
| 51 | The current API requires orchestration read/operate and provider-management | |
| 52 | permissions for offload plus the `shared connect` account-list check. These also | |
| 53 | permit thread/provider operations on the NAS; keep the token private. No provider | |
| 54 | accounts need to be imported. | |
| 55 | ||
| 56 | ```sh | |
| 57 | umask 077 | |
| 58 | mkdir -p "$HOME/.config/knife" | |
| 59 | ssh -p 22 root@10.0.0.1 'export NOMAD_TOKEN="$(cat /var/lib/studio/nomad.token)"; allocation=$(nomad job allocs -json knife | python3 -c '\''import json,sys; a=[r["ID"] for r in json.load(sys.stdin) if r["ClientStatus"]=="running" and r["DesiredStatus"]=="run"]; assert len(a)==1; print(a[0])'\''); podman exec "app-$allocation" knife auth session issue --ttl 30d --label "Clover Mac archive" --scope orchestration:read --scope orchestration:operate --scope providers:manage --token-only' > "$HOME/.config/knife/nas-token" | |
| 60 | chmod 600 "$HOME/.config/knife/nas-token" | |
| 61 | "$HOME/.t3/bin/t3" shared connect https://knife.paperclover.net \ | |
| 62 | --token-file "$HOME/.config/knife/nas-token" | |
| 63 | ``` | |
| 64 | ||
| 65 | The packaged Knife desktop installs `~/.t3/bin/t3`. If a custom `T3CODE_HOME` | |
| 66 | is used, use its `bin/t3` and pass the matching `--base-dir` to CLI commands. | |
| 67 | The local Knife backend must already be running for `shared connect`. | |
| 68 | ||
| 69 | Quit Knife, then launch its executable with archive offload enabled: | |
| 70 | ||
| 71 | ```sh | |
| 72 | T3CODE_ARCHIVE_OFFLOAD=true '/Applications/The Knife.app/Contents/MacOS/The Knife' | |
| 73 | ``` | |
| 74 | ||
| 75 | The setting must reach the backend on every launch. For Finder launches, | |
| 76 | `launchctl setenv T3CODE_ARCHIVE_OFFLOAD true` supplies it for this login session; | |
| 77 | repeat after logging in again. Fully quit the existing app before relaunching. | |
| 78 | Windows WSL backends currently do not forward this variable from the desktop; | |
| 79 | enable it inside the WSL backend environment before using offload there. | |
| 80 | ||
| 81 | Check the connection and archive queue: | |
| 82 | ||
| 83 | ```sh | |
| 84 | "$HOME/.t3/bin/t3" shared status | |
| 85 | "$HOME/.t3/bin/t3" shared archive status | |
| 86 | ``` | |
| 87 | ||
| 88 | Archive one disposable conversation first. Within roughly a minute, status | |
| 89 | should report `enabled: true`, an increased `offloaded` count, `pending: 0`, | |
| 90 | and no failures. Open its transcript and unarchive it to check retrieval and | |
| 91 | restoration before relying on the connection for real history. | |
| 92 | ||
| 93 | Offload replaces archived T3 transcript bodies with local metadata after the | |
| 94 | NAS acknowledges the object. Failed uploads retain the local bodies. Reading | |
| 95 | cold history requires the NAS and its valid token; unarchive restores local | |
| 96 | bodies. Existing SQLite files can retain allocated pages, and native Claude or | |
| 97 | Codex history files remain: this feature does not compact all existing laptop | |
| 98 | storage. Keep the NAS state and archive objects together in backups. | |
| 99 | ||
| 100 | The archive token expires after 30 days without automatic renewal. Issue a | |
| 101 | replacement into the private file and run `shared connect` again before it | |
| 102 | expires. `shared archive retry` retries failed uploads. Choose a longer `--ttl` | |
| 103 | explicitly if monthly renewal is unsuitable; retain a separate label per laptop. | |
| 104 | ||
| 105 | ## Route laptop environments through the NAS | |
| 106 | ||
| 107 | Archive offload only needs an outbound laptop connection to the NAS. Routing | |
| 108 | interactive access through the hub additionally requires Zenith to reach the | |
| 109 | laptop backend over a private LAN or VPN address. This fork has no reverse | |
| 110 | connection that makes an unreachable laptop accessible through the hub. | |
| 111 | ||
| 112 | The NAS public hub URL is configured automatically during deployment. To enroll | |
| 113 | a laptop, get its environment ID from its private | |
| 114 | `/.well-known/t3/environment` descriptor. Issue a separate, labeled administrative | |
| 115 | session on that laptop with `~/.t3/bin/t3 auth session issue --ttl 30d | |
| 116 | --label "Clover NAS hub" --token-only`, capturing it into a mode-0600 file rather | |
| 117 | than displaying it. | |
| 118 | Transfer that file privately to Zenith. In the SSH session above: | |
| 119 | ||
| 120 | ```sh | |
| 121 | podman cp /root/knife-laptop-enrollment-token "$knife_container":/data/laptop-enrollment-token | |
| 122 | podman exec "$knife_container" knife shared hub enroll \ | |
| 123 | https://YOUR-PRIVATE-LAPTOP-ADDRESS ENVIRONMENT-ID \ | |
| 124 | --token-file /data/laptop-enrollment-token | |
| 125 | podman exec "$knife_container" rm /data/laptop-enrollment-token | |
| 126 | rm /root/knife-laptop-enrollment-token | |
| 127 | ``` | |
| 128 | ||
| 129 | Use a private endpoint already secured by Knife authentication; Tailscale Serve | |
| 130 | is one option. No public laptop port is required. Do not disable certificate | |
| 131 | verification. The hub retains this administrative credential in its private | |
| 132 | state to delegate later connections; removing the transfer files does not revoke | |
| 133 | it. Renew and re-enroll before its 30-day expiry, or revoke the labeled laptop | |
| 134 | session to withdraw hub access. A sleeping or unreachable laptop cannot serve | |
| 135 | an interactive session. Its uploaded archive objects remain on the NAS. | |
| 136 | ||
| 137 | ## Revoke access and update the service | |
| 138 | ||
| 139 | The owner session can manage pairing links and clients in **Settings → This machine → Authorized clients**. | |
| 140 | SSH recovery also works without a live browser session: | |
| 141 | ||
| 142 | ```sh | |
| 143 | podman exec "$knife_container" knife auth session list | |
| 144 | podman exec "$knife_container" knife auth session revoke SESSION-ID | |
| 145 | podman exec "$knife_container" knife auth pairing list | |
| 146 | podman exec "$knife_container" knife auth pairing revoke PAIRING-ID | |
| 44 | podman exec "$knife_container" knife auth devices revoke-all | |
| 147 | 45 | ``` |
| 148 | 46 | |
| 149 | Keep pairing links and archive tokens private. Access is possession-based: | |
| 150 | anyone holding a live code or token has its granted scopes. The proxy adds | |
| 151 | `Secure` to browser cookies; the app supplies `HttpOnly` and `SameSite=Lax`. | |
| 152 | The container's published port binds only to loopback. | |
| 153 | ||
| 154 | As requested, each container restart checks the release feed and starts its | |
| 155 | newest compatible server/UI/assets after SHA-256 verification. The running | |
| 156 | process keeps its current code until restarted. This quiet startup path does | |
| 157 | not prefetch in the background. A temporary feed outage starts the installed | |
| 158 | runtime; corrupt downloads or a release requiring a newer base image stop | |
| 159 | startup instead of executing unverified or incompatible code. | |
| 160 | ||
| 161 | For a new base image, review the Linux bundle, update | |
| 162 | `service/knife/bundle.json` with its checksum and image ID, and use the normal | |
| 163 | stage → main → prod workflow. The loader rejects changed bytes at the existing | |
| 164 | bundle URL until its recorded checksum is updated. Application code cached | |
| 165 | under `/data/runtime/knife-container` survives image/infra rollback; restoring | |
| 166 | an older runtime requires an explicit application recovery choice. Code | |
| 167 | rollback does not roll back state or archive objects. | |
| 168 | ||
| 169 | ## 2026-10-16 verification | |
| 170 | ||
| 171 | Zenith fresh preview verified anonymous project/archive/pairing denial, | |
| 172 | single-use pairing, hostile cookie Origin rejection, session revocation, | |
| 173 | hash-checked archive upload, identical download, byte ranges, and corrupt-object | |
| 174 | rejection. A disposable local client imported a two-message synthetic Claude | |
| 175 | conversation, offloaded it to the NAS, restarted, read both cold transcript | |
| 176 | items, and restored both bodies on unarchive. Temporary client credentials | |
| 177 | were revoked and local fixture stores removed. | |
| 178 | ||
| 179 | Browser verification covered first run, self-hosted connection entry, settings, | |
| 180 | access management, archive/history controls, keyboard navigation, and reload | |
| 181 | persistence. Real Caddy checks verified Secure on multiple cookies and logout | |
| 182 | responses while leaving default routes unchanged. Warn-level startup emitted | |
| 183 | no pairing credential. All four service mounts used disposable managed preview | |
| 184 | storage with the assigned service UID. | |
| 47 | This revokes SSO devices and sessions without contacting the issuer. It preserves | |
| 48 | the configured owner, identity-provider account, archives, and manual pairing | |
| 49 | credentials. Inspect/revoke those separately with `knife auth session list`, | |
| 50 | `knife auth session revoke SESSION-ID`, `knife auth pairing list`, and | |
| 51 | `knife auth pairing revoke PAIRING-ID` in the same container. | |
| 52 | ||
| 53 | ## Persistent configuration | |
| 54 | ||
| 55 | The existing managed ZFS dataset `/srv/prod/knife` holds `data`, `archives`, | |
| 56 | `home/node`, and `workspaces`. Keep state and archives together in backups. Preserve | |
| 57 | its assigned service UID and environment identity across image upgrades. | |
| 58 | ||
| 59 | Nomad owns the confidential client secret at `nomad/jobs/knife/inputs/oidc`. | |
| 60 | The allocation renders the private SSO JSON and mounts it read-only at | |
| 61 | `/run/secrets/knife-oidc.json`, readable by the assigned service UID. Tracked | |
| 62 | configuration contains the public issuer, callback, and immutable owner subject, | |
| 63 | with references to the private Nomad values. Preview allocations use their own | |
| 64 | registered clients and disposable state. | |
| 65 | ||
| 66 | Preparation creates `archives/.knife-archive-store-id` exclusively on the confirmed | |
| 67 | service ZFS mount, preserving any existing marker. Startup supplies its value as | |
| 68 | `T3CODE_ARCHIVE_STORE_ID`. Missing or mismatched identity blocks archive publication | |
| 69 | and local payload eviction; never replace the marker to silence that failure. | |
| 70 | ||
| 71 | Each restart follows the release feed and applies the newest compatible, | |
| 72 | checksum-verified runtime. The quiet startup path does not prefetch in the | |
| 73 | background. Feed outages retain the installed runtime; incompatible base-image | |
| 74 | requirements or corrupt downloads fail startup. For a new image, review the bundle | |
| 75 | and update `service/knife/bundle.json`, then use the normal stage → main → prod | |
| 76 | workflow. Runtime downloads under `data/runtime/knife-container` survive an infra | |
| 77 | image rollback; rolling back code does not restore state or archive objects. | |
| 78 | ||
| 79 | ## 2026-10-16 SSO verification | |
| 80 | ||
| 81 | A fresh Zenith preview completed Clover's real Snow Globe sign-in, verified the | |
| 82 | issuer/subject account binding, renewed its browser session, and opened an | |
| 83 | authenticated WebSocket through HTTPS. Anonymous access, missing CSRF, invalid | |
| 84 | OIDC/native transactions, and corrupt archive objects were rejected. A synthetic | |
| 85 | object uploaded and downloaded identically with working byte ranges; temporarily | |
| 86 | removing the preview's archive marker returned 503 and its original marker was | |
| 87 | restored unchanged. The private allocation JSON matched its separate Nomad client, | |
| 88 | used mode 0400 and UID 3132, and its intended bind mount rejected writes. |
tools/studio-secret-files-test.py created+72| ... | ... | @@ -0,0 +1,72 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import json | |
| 3 | import os | |
| 4 | from pathlib import Path | |
| 5 | import subprocess | |
| 6 | import tempfile | |
| 7 | import unittest | |
| 8 | ||
| 9 | import studio | |
| 10 | ||
| 11 | ||
| 12 | class SecretFiles(unittest.TestCase): | |
| 13 | @classmethod | |
| 14 | def setUpClass(cls): | |
| 15 | with tempfile.TemporaryDirectory(prefix="studio-secret-template-") as temporary: | |
| 16 | fixture = Path(temporary) / "fixture.pkl" | |
| 17 | fixture.write_text('amends ' + json.dumps((studio.REPO / "config/Service.pkl").as_uri()) + '\nmeta { name = "Fixture" }\ncontainer { image = "localhost/fixture:pinned" }\n') | |
| 18 | result = subprocess.run([os.environ.get("PKL", "pkl"), "eval", "-p", "uid=3132", "-p", "serviceId=fixture", str(fixture)], check=True, capture_output=True, text=True) | |
| 19 | cls.fixture = json.loads(result.stdout) | |
| 20 | cls.fixture.update(uid=3132, sourceId="fixture", inputs={}) | |
| 21 | cls.fixture.pop("requirements") | |
| 22 | ||
| 23 | def setUp(self): | |
| 24 | self.data = json.loads(json.dumps(self.fixture)) | |
| 25 | self.task = self.data["containers"]["app"] | |
| 26 | self.task["secretFiles"] = {"/run/secrets/oidc.json": '{{ with nomadVar "nomad/jobs/fixture/inputs/oidc" }}{"clientSecret":{{ .clientSecret.Value | toJSON }}}{{ end }}'} | |
| 27 | ||
| 28 | def test_private_allocation_template_and_read_only_file_mount(self): | |
| 29 | rendered = studio.render(self.data, {"fixture": self.data}) | |
| 30 | self.assertIn('"secrets/oidc.json:/run/secrets/oidc.json:ro"', rendered) | |
| 31 | self.assertIn('destination = "secrets/oidc.json"', rendered) | |
| 32 | self.assertIn('perms = "0400"', rendered) | |
| 33 | self.assertIn('uid = 3132', rendered) | |
| 34 | self.assertIn('gid = 3132', rendered) | |
| 35 | self.assertIn('error_on_missing_key = true', rendered) | |
| 36 | self.assertIn(self.task["secretFiles"]["/run/secrets/oidc.json"], rendered) | |
| 37 | self.assertNotIn('env = true', rendered) | |
| 38 | self.assertNotIn('change_mode =', rendered) | |
| 39 | ||
| 40 | def test_services_without_files_gain_no_template_or_mount(self): | |
| 41 | self.task["secretFiles"] = {} | |
| 42 | rendered = studio.render(self.data, {"fixture": self.data}) | |
| 43 | self.assertNotIn('template {', rendered) | |
| 44 | self.assertNotIn('secrets/', rendered) | |
| 45 | ||
| 46 | def test_unsafe_and_colliding_targets_are_rejected(self): | |
| 47 | for target in ("relative.json", "/run/../secret.json", "/run/./secret.json", "/run//secret.json", "/run/secrets/", "/secrets/file", "/secrets", "/run/app.env", "/run/app-secret.env", "/run/nomad_token", "/run/line\nbreak"): | |
| 48 | with self.subTest(target=target): | |
| 49 | self.task["secretFiles"] = {target: "{}"} | |
| 50 | with self.assertRaisesRegex(ValueError, "invalid secret file target"): | |
| 51 | studio.render(self.data, {"fixture": self.data}) | |
| 52 | self.task["secretFiles"] = {"/one/config.json": "{}", "/two/config.json": "{}"} | |
| 53 | with self.assertRaisesRegex(ValueError, "invalid secret file target"): | |
| 54 | studio.render(self.data, {"fixture": self.data}) | |
| 55 | self.task["secretFiles"] = {"/run/secrets/oidc.json": "{}"} | |
| 56 | self.task["volumes"] = {"/run/secrets/oidc.json": {}} | |
| 57 | with self.assertRaisesRegex(ValueError, "invalid secret file target"): | |
| 58 | studio.render(self.data, {"fixture": self.data}) | |
| 59 | ||
| 60 | def test_image_user_and_heredoc_injection_are_rejected(self): | |
| 61 | self.task["imageUser"] = True | |
| 62 | with self.assertRaisesRegex(ValueError, "service-owned"): | |
| 63 | studio.render(self.data, {"fixture": self.data}) | |
| 64 | self.task["imageUser"] = False | |
| 65 | for body in ("EOF", "{}\nEOF\nmalicious", "{}\nEOF\r\nmalicious"): | |
| 66 | self.task["secretFiles"] = {"/run/secrets/oidc.json": body} | |
| 67 | with self.assertRaisesRegex(ValueError, "invalid secret template delimiter"): | |
| 68 | studio.render(self.data, {"fixture": self.data}) | |
| 69 | ||
| 70 | ||
| 71 | if __name__ == "__main__": | |
| 72 | unittest.main() |
tools/studio.py+23| ... | ... | @@ -397,6 +397,22 @@ def render(data, definitions): |
| 397 | 397 | if values: |
| 398 | 398 | lines.append(f" {field} = {q(values)}") |
| 399 | 399 | volumes = [] |
| 400 | secret_files = task.get("secretFiles", {}) | |
| 401 | filenames = set() | |
| 402 | for target, template in secret_files.items(): | |
| 403 | filename = Path(target).name | |
| 404 | if (not re.fullmatch(r"/[A-Za-z0-9_./-]+", target) or target.endswith("/") | |
| 405 | or any(part in {".", "..", ""} for part in target[1:].split("/")) | |
| 406 | or filename in filenames | |
| 407 | or filename in {task_name + ".env", task_name + "-secret.env", "nomad_token"} | |
| 408 | or target in task["volumes"] or target == "/secrets" or target.startswith("/secrets/")): | |
| 409 | raise ValueError(f"invalid secret file target: {target}") | |
| 410 | if task["imageUser"]: | |
| 411 | raise ValueError("secret files require a service-owned container user") | |
| 412 | if re.search(r"(?m)^EOF\r?$", template): | |
| 413 | raise ValueError("invalid secret template delimiter") | |
| 414 | filenames.add(filename) | |
| 415 | volumes.append(f"secrets/{filename}:{target}:ro") | |
| 400 | 416 | for target, volume in task["volumes"].items(): |
| 401 | 417 | if not target.startswith("/") or ":" in target: |
| 402 | 418 | raise ValueError(f"invalid volume target: {target}") |
| ... | ... | @@ -460,6 +476,11 @@ def render(data, definitions): |
| 460 | 476 | if "\nEOF\n" in task["envTemplate"]: |
| 461 | 477 | raise ValueError("invalid env template delimiter") |
| 462 | 478 | lines += [" template {", " data = <<EOF", task["envTemplate"].rstrip(), "EOF", f' destination = {q("secrets/" + task_name + ".env")}', " env = true", " }"] |
| 479 | for target, template in secret_files.items(): | |
| 480 | lines += [" template {", " data = <<EOF", template.rstrip(), "EOF", | |
| 481 | f' destination = {q("secrets/" + Path(target).name)}', ' perms = "0400"', | |
| 482 | f' uid = {data["uid"]}', f' gid = {data["uid"]}', | |
| 483 | " error_on_missing_key = true", " }"] | |
| 463 | 484 | lines += [" resources {", f" cpu = {task['cpu']}", f" memory = {task['memory']}", " }", " }"] |
| 464 | 485 | lines += [" }", "}"] |
| 465 | 486 | return "\n".join(lines) + "\n" |
| ... | ... | @@ -1016,6 +1037,8 @@ def main(): |
| 1016 | 1037 | task["http"]["hostname"] = hostname |
| 1017 | 1038 | task["http"]["plainHostnames"] = [stage + "-" + host for host in task["http"]["plainHostnames"]] |
| 1018 | 1039 | task["env"] = {key: value.replace(original_host, hostname) for key, value in task["env"].items()} |
| 1040 | if task.get("secretFiles"): | |
| 1041 | task["secretFiles"] = {key: value.replace(original_host, hostname) for key, value in task["secretFiles"].items()} | |
| 1019 | 1042 | for request in data["inputs"].values(): |
| 1020 | 1043 | if request["provider"] == "snowglobe": |
| 1021 | 1044 | request["redirectUris"] = [uri.replace(original_host, hostname) for uri in request["redirectUris"]] |