From 2eaf043d3ab4571bfc809b8777bd24ecc8326c5d Mon Sep 17 00:00:00 2001 From: clover caruso Date: Wed, 28 Oct 2026 01:04:28 -0700 Subject: [PATCH] Remove the retired Knife Nomad service Preserve its archive recovery record and storage. Assisted-by: gpt-6.1-terra --- service/knife/bundle.json | 5 -- service/knife/machine-info | 1 - service/knife/prepare.py | 59 --------------------- service/knife/service.pkl | 70 ------------------------- tools/knife-test.py | 71 ------------------------- tools/knife.md | 103 +++---------------------------------- 6 files changed, 8 insertions(+), 301 deletions(-) delete mode 100644 service/knife/bundle.json delete mode 100644 service/knife/machine-info delete mode 100644 service/knife/prepare.py delete mode 100644 service/knife/service.pkl delete mode 100644 tools/knife-test.py diff --git a/service/knife/bundle.json b/service/knife/bundle.json deleted file mode 100644 index 46f33f54bc92e8e92b5a98f01991cc1ff4d10fdc..0000000000000000000000000000000000000000 --- a/service/knife/bundle.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "url": "https://file.paperclover.net/shr/the-knife/knife-server-linux-amd64.tar.gz", - "sha256": "3c67925b1f11492157ad2d7832455c9e07b01ca58f63a3d730ce5a68c6ef8f21", - "image": "sha256:9c2da38825a4b35ca13c6f6596f7fd67cebd75dc3394932c8a6f66022136294c" -} diff --git a/service/knife/machine-info b/service/knife/machine-info deleted file mode 100644 index b08d9fea139e9d6501e997ac9b0e5c8fc8510c28..0000000000000000000000000000000000000000 --- a/service/knife/machine-info +++ /dev/null @@ -1 +0,0 @@ -PRETTY_HOSTNAME="snow globe" diff --git a/service/knife/prepare.py b/service/knife/prepare.py deleted file mode 100644 index e6337e6e6972faeee8cdb164b122d1968c3f8cb4..0000000000000000000000000000000000000000 --- a/service/knife/prepare.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import json -import os -from pathlib import Path -import shutil -import subprocess -import sys -import tarfile -import tempfile -import urllib.request -import uuid - - -bundle = json.loads(Path(__file__).with_name("bundle.json").read_text()) -if subprocess.run(["podman", "image", "exists", bundle["image"]]).returncode: - with tempfile.TemporaryDirectory(prefix="snowglobe-knife-") as temporary: - archive = Path(temporary) / "bundle.tar.gz" - digest = hashlib.sha256() - with urllib.request.urlopen(bundle["url"], timeout=120) as source, archive.open("wb") as target: - while block := source.read(1024 * 1024): - digest.update(block) - target.write(block) - if digest.hexdigest() != bundle["sha256"]: - raise ValueError("Knife server bundle failed checksum verification") - image = Path(temporary) / "image.tar" - with tarfile.open(archive) as source, image.open("wb") as target: - member = source.getmember("image.tar") - if not member.isfile(): - raise ValueError("Knife server bundle has no image file") - with source.extractfile(member) as contents: - shutil.copyfileobj(contents, target) - subprocess.run(["podman", "load", "--quiet", "--input", str(image)], check=True) - subprocess.run(["podman", "image", "exists", bundle["image"]], check=True) - -root = Path(json.load(sys.stdin)["hostRoot"]) -archives = root / "archives" -mount = subprocess.run( - ["findmnt", "-rn", "-o", "TARGET,FSTYPE", "--target", str(archives)], - check=True, capture_output=True, text=True, -).stdout.strip() -if root.resolve() != root or archives.resolve() != archives or mount != f"{root} zfs": - raise ValueError("Knife archives must be on the mounted service ZFS dataset") -marker = archives / ".knife-archive-store-id" -try: - with marker.open("x") as file: - file.write(str(uuid.uuid4()) + "\n") - os.fchmod(file.fileno(), 0o444) - file.flush() - os.fsync(file.fileno()) - directory = os.open(archives, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(directory) - finally: - os.close(directory) -except FileExistsError: - pass -if marker.is_symlink() or not marker.is_file() or not marker.read_text().strip(): - raise ValueError("Knife archive store identity must be a nonempty regular file") diff --git a/service/knife/service.pkl b/service/knife/service.pkl deleted file mode 100644 index 11cc840fbb7ff8f8c19d08f789707392e551a3a0..0000000000000000000000000000000000000000 --- a/service/knife/service.pkl +++ /dev/null @@ -1,70 +0,0 @@ -amends "../../config/Service.pkl" - -import "pkl:json" -import "../../config/OpenID.pkl" as sso - -local bundle = (new json.Parser {}).parse(read("bundle.json")) - -name = "The Knife" -enabled = false -access = "infra-admin" -requirements { - new sso.Client { - clientId = module.id - name = module.name - redirectUris { "https://\(module.container.http.hostname)/api/auth/oidc/callback" } - } -} -stageIsolation = "fresh" -prepare = "prepare.py" - -container { - image = bundle.image - cpu = 1000 - memory = 4096 - entrypoint = "/usr/bin/tini" - args { - "--" - "/bin/sh" - "-ec" - "T3CODE_ARCHIVE_STORE_ID=$(cat /archives/.knife-archive-store-id); export T3CODE_ARCHIVE_STORE_ID; node /opt/knife/scripts/container-runtime.ts; exec knife start --host 0.0.0.0 --port 3773 --no-browser" - } - http { - containerPort = 3773 - subdomain = "knife" - loopback = true - secureCookies = true - checkPath = "/.well-known/t3/environment" - } - volumes { - ["/etc/machine-info"] { config = "machine-info" } - ["/data"] {} - ["/home/node"] {} - ["/archives"] {} - ["/workspaces"] {} - } - env { - ["HOME"] = "/home/node" - ["KNIFE_UPDATE_URL"] = "https://file.paperclover.net/shr/the-knife/" - ["T3CODE_HOME"] = "/data" - ["T3CODE_SSO_CONFIG"] = "/run/secrets/knife-oidc.json" - ["T3CODE_ARCHIVE_DIR"] = "/archives" - ["T3CODE_ARCHIVE_OFFLOAD"] = "true" - ["T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD"] = "false" - // Info-level startup logs include a live owner pairing credential. - ["T3CODE_LOG_LEVEL"] = "Warn" - } - secretFiles { - ["/run/secrets/knife-oidc.json"] = """ - {{ with nomadVar "nomad/jobs/\(module.id)/inputs/oidc" }} - { - "publicUrl": "https://\(module.container.http.hostname)", - "issuer": {{ .issuerUrl.Value | toJSON }}, - "clientId": {{ .clientId.Value | toJSON }}, - "clientSecret": {{ .clientSecret.Value | toJSON }}, - "ownerSub": "8c909fab-98b0-4472-9171-9606ccebe9d5" - } - {{ end }} - """ - } -} diff --git a/tools/knife-test.py b/tools/knife-test.py deleted file mode 100644 index a5a5e809b087fbf1f0c390832f6882c4eb6d4654..0000000000000000000000000000000000000000 --- a/tools/knife-test.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -import io -import json -from pathlib import Path -import runpy -import subprocess -import tempfile -import unittest -from unittest.mock import patch -import uuid - - -class ArchiveIdentity(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory(prefix="knife-archive-test-") - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name).resolve() - self.archives = self.root / "archives" - self.archives.mkdir() - self.marker = self.archives / ".knife-archive-store-id" - - def prepare(self, mount=None): - def command(args, **kwargs): - if args[:3] == ["podman", "image", "exists"]: - return subprocess.CompletedProcess(args, 0) - self.assertEqual(args[0], "findmnt") - self.assertEqual(args[-1], str(self.archives)) - return subprocess.CompletedProcess(args, 0, stdout=mount or f"{self.root} zfs\n") - with patch("subprocess.run", side_effect=command), patch("sys.stdin", io.StringIO(json.dumps({"hostRoot": str(self.root)}))): - runpy.run_path(str(Path(__file__).resolve().parents[1] / "service/knife/prepare.py")) - - def test_create_on_confirmed_zfs_and_preserve_existing_identity(self): - self.prepare() - uuid.UUID(self.marker.read_text().strip()) - self.assertEqual(self.marker.stat().st_mode & 0o777, 0o444) - self.marker.unlink() - self.marker.write_text("existing-nas-identity\n") - self.prepare() - self.assertEqual(self.marker.read_bytes(), b"existing-nas-identity\n") - - def test_wrong_or_fallback_mount_cannot_create_marker(self): - for mount in ["/ ext4", f"{self.root} ext4", f"{self.archives} zfs"]: - with self.subTest(mount=mount), self.assertRaises(ValueError): - self.prepare(mount) - self.assertFalse(self.marker.exists()) - - def test_symlinked_archive_directory_cannot_create_marker(self): - self.archives.rmdir() - target = self.root / "elsewhere" - target.mkdir() - self.archives.symlink_to(target, target_is_directory=True) - with self.assertRaises(ValueError): - self.prepare() - self.assertFalse((target / self.marker.name).exists()) - - def test_invalid_existing_marker_is_preserved_and_rejected(self): - self.marker.write_text("") - with self.assertRaises(ValueError): - self.prepare() - self.assertEqual(self.marker.read_bytes(), b"") - self.marker.unlink() - target = self.root / "outside-marker" - target.write_text("outside\n") - self.marker.symlink_to(target) - with self.assertRaises(ValueError): - self.prepare() - self.assertEqual(target.read_text(), "outside\n") - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/knife.md b/tools/knife.md index 63458d8e39565fac8bd0e4e07832cbd5eb192bcd..bb4eba4d99f559b4f44063a2d225bb6e15c5272c 100644 --- a/tools/knife.md +++ b/tools/knife.md @@ -1,95 +1,8 @@ -# Knife sign-in and archive storage - -The production service was taken offline on 2026-10-19. Its declaration has -`enabled = false`, so later deployments do not restart it. The complete -`globe/prod/knife` dataset remains mounted at `/srv/prod/knife`, including runtime -state, archive objects, environment identity, and workspaces. The stopped dataset -has snapshot `knife-retired-20261019T213553Z`, held with -`knife-archive-retention`. Private verification evidence is in -`/var/lib/snowglobe/retirements/knife/`. - -Keep `data` and `archives` together; archive objects alone are not a complete -restoration. Preserve the service UID and `archives/.knife-archive-store-id`. -Retirement does not delete or migrate any conversations. The disposable preview -was removed on 2026-10-19. -Later deployment backups omit retired jobs; retain the held snapshot and private -retirement evidence separately from routine release-backup retention. - -## Restoring service access - -Re-enabling the service requires a deliberate production deployment. After that, -open https://knife.paperclover.net and sign in through Snow Globe. The server -accepts only the configured issuer and Clover's immutable owner subject. The NAS -environment is **snow globe**. Provider accounts can stay on each laptop; the NAS -needs none for archive storage or outbound environment routing. - -On each machine, install the current Knife desktop release, choose its -self-hosted server connection, enter `https://knife.paperclover.net`, and complete -browser sign-in. The desktop returns through its proof-bound application callback; -only the server's HTTPS callback is registered with Snow Globe. Laptop connections -run outbound through Knife, with archive offload enabled automatically. No laptop -port forwarding, SSH pairing, bearer-token file, or manual hub enrollment is needed -for the SSO connection. - -Archive a disposable conversation first, reopen its cold transcript, then unarchive -it. Confirm that its source connection and archive queue report success before -relying on offload for existing history. A sleeping laptop cannot run interactive -agents; uploaded archives remain on the NAS. Successful offload reclaims local -SQLite pages after publication. Native provider history files remain on the laptop. - -## Access and recovery - -SSO access credentials last fifteen minutes, with rotating device credentials in -operating-system protected storage. Browser sign-in is required again after thirty -days. The server revalidates the issuer grant during renewal at least hourly; -issuer revocation can therefore take an hour plus fifteen minutes to fully apply. -Local device revocation disables renewal and closes active connections immediately. - -Existing manual pairing credentials survive the upgrade. Manage them separately -from SSO devices under access settings; do not share pairing links or bearer tokens. -The listener binds only to loopback behind HTTPS, with WebSocket proxying. Hosted -SSO cookies are Secure, HttpOnly, SameSite=Lax and protected against cross-origin -writes. - -For local recovery, derive the current container from Nomad rather than retaining -an allocation ID: - -```sh -ssh -p 22 root@10.0.0.1 -export NOMAD_TOKEN="$(cat /var/lib/snowglobe/nomad.token)" -knife_allocation=$(nomad job allocs -json knife | python3 -c 'import json,sys; a=[r["ID"] for r in json.load(sys.stdin) if r["ClientStatus"]=="running" and r["DesiredStatus"]=="run"]; assert len(a)==1; print(a[0])') -knife_container="app-$knife_allocation" -podman exec "$knife_container" knife auth devices revoke-all -``` - -This revokes SSO devices and sessions without contacting the issuer. It preserves -the configured owner, identity-provider account, archives, and manual pairing -credentials. Inspect/revoke those separately with `knife auth session list`, -`knife auth session revoke SESSION-ID`, `knife auth pairing list`, and -`knife auth pairing revoke PAIRING-ID` in the same container. - -## Persistent configuration - -The existing managed ZFS dataset `/srv/prod/knife` holds `data`, `archives`, -`home/node`, and `workspaces`. Keep state and archives together in backups. Preserve -its assigned service UID and environment identity across image upgrades. - -Nomad owns the confidential client secret at `nomad/jobs/knife/inputs/oidc`. -The allocation renders the private SSO JSON and mounts it read-only at -`/run/secrets/knife-oidc.json`, readable by the assigned service UID. Tracked -configuration contains the public issuer, callback, and immutable owner subject, -with references to the private Nomad values. Preview allocations use their own -registered clients and disposable state. - -Preparation creates `archives/.knife-archive-store-id` exclusively on the confirmed -service ZFS mount, preserving any existing marker. Startup supplies its value as -`T3CODE_ARCHIVE_STORE_ID`. Missing or mismatched identity blocks archive publication -and local payload eviction; never replace the marker to silence that failure. - -Each restart follows the release feed and applies the newest compatible, -checksum-verified runtime. The quiet startup path does not prefetch in the -background. Feed outages retain the installed runtime; incompatible base-image -requirements or corrupt downloads fail startup. For a new image, review the bundle -and update `service/knife/bundle.json`, then use the normal stage → main → prod -workflow. Runtime downloads under `data/runtime/knife-container` survive an infra -image rollback; rolling back code does not restore state or archive objects. +# Retired Knife archive + +The retired Nomad service's `globe/prod/knife` dataset remains mounted at +`/srv/prod/knife`. Keep `data` and `archives` together, including the archive +store identity and assigned service UID. Snapshot `knife-retired-20261019T213553Z` +has the `knife-archive-retention` hold; private retirement evidence is in +`/var/lib/snowglobe/retirements/knife/`. Later deployment backups omit retired +jobs, so retain the held snapshot and evidence separately. -- 2.54.1