| author | |
| committer | |
| log | 628f936c593ab09248911fa3f03f28e819d870d4 |
| tree | f22224873926bc9331889151c504a81cb410cc22 |
| parent | 67a75449a270804854c5cdae4b2b6a83c1fcd234 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Offload compatible pnpm/Vite+ installs and Jujutsu filesystem work to the NAS while retaining native Darwin hooks, package policies, configuration, and failure propagation. Avoid NFS build-cache writes and expose VM loopback previews through the existing SSH connection.
Assisted-by: gpt-6.1-sol20 files changed, 2604 insertions(+), 0 deletions(-)
tools/mac-agent-forward-test.py created+88| ... | @@ -0,0 +1,88 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Check loopback forwarding against a disposable HTTP server in the live VM.""" | ||
| 3 | |||
| 4 | import hashlib | ||
| 5 | from importlib import import_module | ||
| 6 | from pathlib import Path | ||
| 7 | import shlex | ||
| 8 | import socket | ||
| 9 | import subprocess | ||
| 10 | import sys | ||
| 11 | import time | ||
| 12 | import urllib.error | ||
| 13 | import urllib.request | ||
| 14 | |||
| 15 | |||
| 16 | vm = import_module("mac-agent-vm") | ||
| 17 | |||
| 18 | |||
| 19 | def main(): | ||
| 20 | payload = b"forwarding fixture\n" * 40000 | ||
| 21 | code = '''from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | ||
| 22 | import sys, threading | ||
| 23 | class Handler(BaseHTTPRequestHandler): | ||
| 24 | def do_GET(self): | ||
| 25 | body = b"forwarding fixture\\n" * 40000 | ||
| 26 | self.send_response(200) | ||
| 27 | self.send_header("Content-Length", str(len(body))) | ||
| 28 | self.end_headers() | ||
| 29 | self.wfile.write(body) | ||
| 30 | def log_message(self, *args): pass | ||
| 31 | server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) | ||
| 32 | threading.Thread(target=server.serve_forever, daemon=True).start() | ||
| 33 | print(server.server_port, flush=True) | ||
| 34 | sys.stdin.buffer.read() | ||
| 35 | server.shutdown() | ||
| 36 | server.server_close() | ||
| 37 | ''' | ||
| 38 | ip = vm.tart("ip", vm.NAME, capture_output=True, text=True, timeout=5).stdout.strip() | ||
| 39 | server = subprocess.Popen([*vm.ssh_args(), "admin@" + ip, "python3 -u -c " + shlex.quote(code)], | ||
| 40 | stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) | ||
| 41 | tunnel = None | ||
| 42 | try: | ||
| 43 | port = int(server.stdout.readline()) | ||
| 44 | with socket.socket() as available: | ||
| 45 | available.bind(("127.0.0.1", 0)) | ||
| 46 | local_port = available.getsockname()[1] | ||
| 47 | command = [sys.executable, str(Path(__file__).with_name("mac-agent-vm.py")), | ||
| 48 | "forward", str(port), "--local-port", str(local_port)] | ||
| 49 | tunnel = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | ||
| 50 | url = f"http://127.0.0.1:{local_port}/fixture" | ||
| 51 | opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) | ||
| 52 | deadline = time.monotonic() + 15 | ||
| 53 | while True: | ||
| 54 | if tunnel.poll() is not None: | ||
| 55 | raise AssertionError(tunnel.communicate()[1]) | ||
| 56 | try: | ||
| 57 | with opener.open(url, timeout=2) as response: | ||
| 58 | actual = response.read() | ||
| 59 | break | ||
| 60 | except urllib.error.URLError: | ||
| 61 | if time.monotonic() >= deadline: | ||
| 62 | raise | ||
| 63 | time.sleep(0.05) | ||
| 64 | assert hashlib.sha256(actual).digest() == hashlib.sha256(payload).digest() | ||
| 65 | collision = subprocess.run(command, capture_output=True, text=True, timeout=15) | ||
| 66 | assert collision.returncode != 0, "A second tunnel silently reused an occupied port." | ||
| 67 | with opener.open(url, timeout=2) as response: | ||
| 68 | assert response.read() == payload, "Port collision disturbed the first tunnel." | ||
| 69 | tunnel.terminate() | ||
| 70 | tunnel.communicate(timeout=5) | ||
| 71 | try: | ||
| 72 | opener.open(url, timeout=2) | ||
| 73 | except urllib.error.URLError: | ||
| 74 | pass | ||
| 75 | else: | ||
| 76 | raise AssertionError("The tunnel listener survived its process.") | ||
| 77 | print(f"Forwarded {len(payload)} bytes intact; collision preserved the live tunnel; shutdown closed its listener.") | ||
| 78 | finally: | ||
| 79 | if tunnel is not None and tunnel.poll() is None: | ||
| 80 | tunnel.terminate() | ||
| 81 | tunnel.communicate(timeout=5) | ||
| 82 | server.stdin.close() | ||
| 83 | server.wait(timeout=10) | ||
| 84 | server.stdout.close() | ||
| 85 | |||
| 86 | |||
| 87 | if __name__ == "__main__": | ||
| 88 | main() | ||
tools/mac-agent-vm.md+13| ... | @@ -23,6 +23,19 @@ python3 tools/mac-agent-vm.py console | ... | @@ -23,6 +23,19 @@ python3 tools/mac-agent-vm.py console |
| 23 | python3 tools/mac-agent-vm.py stop | 23 | python3 tools/mac-agent-vm.py stop |
| 24 | ``` | 24 | ``` |
| 25 | 25 | ||
| 26 | To reach a server bound to guest loopback from this Mac: | ||
| 27 | |||
| 28 | ```sh | ||
| 29 | python3 tools/mac-agent-vm.py forward 5173 --local-port 15173 | ||
| 30 | ``` | ||
| 31 | |||
| 32 | Open `http://127.0.0.1:15173` on the Mac. The tunnel stays in the foreground; | ||
| 33 | Ctrl-C closes it. Both ends use loopback, occupied ports fail explicitly, and | ||
| 34 | no guest command or LAN listener is created. T3's browser separately requires | ||
| 35 | its toolbar's port approval. Run `python3 tools/mac-agent-forward-test.py` to | ||
| 36 | check a disposable guest server, a transfer larger than 512 KiB, port collision, | ||
| 37 | and listener shutdown. This test uses the existing guest without changing its data. | ||
| 38 | |||
| 26 | `start` resumes a suspended VM or boots a stopped VM, then waits for commands | 39 | `start` resumes a suspended VM or boots a stopped VM, then waits for commands |
| 27 | to work through the logged-in guest agent. `stop` requests a graceful | 40 | to work through the logged-in guest agent. `stop` requests a graceful |
| 28 | guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM | 41 | guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM |
tools/mac-agent-vm.py+13| ... | @@ -210,10 +210,23 @@ def main(): | ... | @@ -210,10 +210,23 @@ def main(): |
| 210 | for action in ["exec", "ssh"]: | 210 | for action in ["exec", "ssh"]: |
| 211 | command = sub.add_parser(action) | 211 | command = sub.add_parser(action) |
| 212 | command.add_argument("command", nargs=argparse.REMAINDER) | 212 | command.add_argument("command", nargs=argparse.REMAINDER) |
| 213 | forward = sub.add_parser("forward", help="Reach a guest loopback server from this Mac") | ||
| 214 | forward.add_argument("port", type=int, help="Guest server port") | ||
| 215 | forward.add_argument("--local-port", type=int, help="Host loopback port (default: same as guest)") | ||
| 213 | viewer = sub.add_parser("console") | 216 | viewer = sub.add_parser("console") |
| 214 | viewer.add_argument("--listen", default="127.0.0.1") | 217 | viewer.add_argument("--listen", default="127.0.0.1") |
| 215 | viewer.add_argument("--port", type=int, default=17891) | 218 | viewer.add_argument("--port", type=int, default=17891) |
| 216 | args = parser.parse_args() | 219 | args = parser.parse_args() |
| 220 | if args.action == "forward": | ||
| 221 | local_port = args.local_port if args.local_port is not None else args.port | ||
| 222 | if not 1 <= args.port <= 65535 or not 1 <= local_port <= 65535: | ||
| 223 | parser.error("Ports must be between 1 and 65535.") | ||
| 224 | ip = tart("ip", NAME, capture_output=True, text=True, timeout=5).stdout.strip() | ||
| 225 | print(f"Forwarding http://127.0.0.1:{local_port} to guest port {args.port}; Ctrl-C closes the tunnel.", | ||
| 226 | file=sys.stderr, flush=True) | ||
| 227 | os.execvp("ssh", [*ssh_args(), "-N", "-T", "-o", "ExitOnForwardFailure=yes", | ||
| 228 | "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=2", | ||
| 229 | "-L", f"127.0.0.1:{local_port}:127.0.0.1:{args.port}", "admin@" + ip]) | ||
| 217 | if args.action in {"start", "stop", "suspend", "reboot"}: | 230 | if args.action in {"start", "stop", "suspend", "reboot"}: |
| 218 | lock = (PRIVATE / "lifecycle.lock").open("a") | 231 | lock = (PRIVATE / "lifecycle.lock").open("a") |
| 219 | os.chmod(PRIVATE / "lifecycle.lock", 0o600) | 232 | os.chmod(PRIVATE / "lifecycle.lock", 0o600) |
tools/project-pnpm/pnpm-10.32.1.patch created+102| ... | @@ -0,0 +1,102 @@ | ||
| 1 | --- pnpm.cjs | ||
| 2 | +++ pnpm-darwin-rebuild.cjs | ||
| 3 | @@ -128122,7 +128122,7 @@ | ||
| 4 | var dependency_path_1 = require_lib87(); | ||
| 5 | var crypto_object_hasher_1 = require_lib119(); | ||
| 6 | function calcDepState(depsGraph, cache, depPath, opts) { | ||
| 7 | - let result2 = constants_1.ENGINE_NAME; | ||
| 8 | + let result2 = opts.includeDepGraphHash ? "darwin;arm64;node26" : constants_1.ENGINE_NAME; | ||
| 9 | if (opts.includeDepGraphHash) { | ||
| 10 | const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath); | ||
| 11 | result2 += `;deps=${depGraphHash}`; | ||
| 12 | @@ -130347,12 +130347,6 @@ | ||
| 13 | storeController: store.ctrl, | ||
| 14 | unsafePerm: opts.unsafePerm || false | ||
| 15 | }; | ||
| 16 | - await (0, lifecycle_1.runLifecycleHooksConcurrently)(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts.childConcurrency || 5, scriptsOpts); | ||
| 17 | - for (const { id, manifest } of Object.values(ctx.projects)) { | ||
| 18 | - if (manifest?.scripts != null && (!opts.pending || ctx.pendingBuilds.includes(id))) { | ||
| 19 | - ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1); | ||
| 20 | - } | ||
| 21 | - } | ||
| 22 | await (0, modules_yaml_1.writeModulesManifest)(ctx.rootModulesDir, { | ||
| 23 | prunedAt: (/* @__PURE__ */ new Date()).toUTCString(), | ||
| 24 | ...ctx.modulesFile, | ||
| 25 | @@ -130445,10 +130439,12 @@ | ||
| 26 | } else { | ||
| 27 | extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir)); | ||
| 28 | } | ||
| 29 | + if (!allowBuild(pkgInfo.name, pkgInfo.version, depPath)) { pkgsThatWereRebuilt.add(depPath); return; } | ||
| 30 | const resolution = pkgSnapshot.resolution; | ||
| 31 | let sideEffectsCacheKey; | ||
| 32 | const pkgId = `${pkgInfo.name}@${pkgInfo.version}`; | ||
| 33 | - if (opts.skipIfHasSideEffectsCache && resolution.integrity) { | ||
| 34 | + if (!["esbuild@0.28.0", "sharp@0.34.5", "msgpackr-extract@3.0.3", "@parcel/watcher@2.5.6"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId); | ||
| 35 | + if (resolution.integrity) { | ||
| 36 | const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId); | ||
| 37 | let pkgFilesIndex; | ||
| 38 | try { | ||
| 39 | @@ -130460,58 +130456,16 @@ | ||
| 40 | includeDepGraphHash: true | ||
| 41 | }); | ||
| 42 | if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) { | ||
| 43 | + const delta = pkgFilesIndex.sideEffects[sideEffectsCacheKey]; | ||
| 44 | + require("/opt/project-store/restore.cjs")(pkgRoot, | ||
| 45 | + Object.entries(delta.added ?? {}).map(([rel, info]) => [rel, store_cafs_1.getFilePathByModeInCafs(opts.storeDir, info.integrity, info.mode), info.mode]), delta.deleted ?? []); | ||
| 46 | + process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:Object.keys(delta.added ?? {}).length})+"\n"); | ||
| 47 | pkgsThatWereRebuilt.add(depPath); | ||
| 48 | return; | ||
| 49 | } | ||
| 50 | } | ||
| 51 | } | ||
| 52 | - let requiresBuild = true; | ||
| 53 | - const pgkManifest = await (0, read_package_json_1.safeReadPackageJsonFromDir)(pkgRoot); | ||
| 54 | - if (pgkManifest != null) { | ||
| 55 | - requiresBuild = (0, exec_pkg_requires_build_1.pkgRequiresBuild)(pgkManifest, {}); | ||
| 56 | - } | ||
| 57 | - const hasSideEffects = requiresBuild && allowBuild(pkgInfo.name, pkgInfo.version, depPath) && await (0, lifecycle_1.runPostinstallHooks)({ | ||
| 58 | - depPath, | ||
| 59 | - extraBinPaths, | ||
| 60 | - extraEnv: opts.extraEnv, | ||
| 61 | - optional: pkgSnapshot.optional === true, | ||
| 62 | - pkgRoot, | ||
| 63 | - rawConfig: opts.rawConfig, | ||
| 64 | - rootModulesDir: ctx.rootModulesDir, | ||
| 65 | - scriptsPrependNodePath: opts.scriptsPrependNodePath, | ||
| 66 | - shellEmulator: opts.shellEmulator, | ||
| 67 | - unsafePerm: opts.unsafePerm || false | ||
| 68 | - }); | ||
| 69 | - if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) { | ||
| 70 | - builtDepPaths.add(depPath); | ||
| 71 | - const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId); | ||
| 72 | - try { | ||
| 73 | - if (!sideEffectsCacheKey) { | ||
| 74 | - sideEffectsCacheKey = (0, calc_dep_state_1.calcDepState)(depGraph, depsStateCache, depPath, { | ||
| 75 | - includeDepGraphHash: true | ||
| 76 | - }); | ||
| 77 | - } | ||
| 78 | - await opts.storeController.upload(pkgRoot, { | ||
| 79 | - sideEffectsCacheKey, | ||
| 80 | - filesIndexFile | ||
| 81 | - }); | ||
| 82 | - } catch (err) { | ||
| 83 | - (0, assert_1.default)(util_1.default.types.isNativeError(err)); | ||
| 84 | - if ("statusCode" in err && err.statusCode === 403) { | ||
| 85 | - logger_1.logger.warn({ | ||
| 86 | - message: `The store server disabled upload requests, could not upload ${pkgRoot}`, | ||
| 87 | - prefix: opts.lockfileDir | ||
| 88 | - }); | ||
| 89 | - } else { | ||
| 90 | - logger_1.logger.warn({ | ||
| 91 | - error: err, | ||
| 92 | - message: `An error occurred while uploading ${pkgRoot}`, | ||
| 93 | - prefix: opts.lockfileDir | ||
| 94 | - }); | ||
| 95 | - } | ||
| 96 | - } | ||
| 97 | - } | ||
| 98 | - pkgsThatWereRebuilt.add(depPath); | ||
| 99 | + throw new Error("Missing target cache for " + pkgId); | ||
| 100 | } catch (err) { | ||
| 101 | (0, assert_1.default)(util_1.default.types.isNativeError(err)); | ||
| 102 | if (pkgSnapshot.optional) { | ||
tools/project-pnpm/pnpm-11.10.0.patch created+105| ... | @@ -0,0 +1,105 @@ | ||
| 1 | --- pnpm.mjs | ||
| 2 | +++ pnpm-target-cache.mjs | ||
| 3 | @@ -155612,7 +155612,7 @@ | ||
| 4 | } | ||
| 5 | function calcDepState(depsGraph, cache, depPath, opts3) { | ||
| 6 | const ownPin = readSnapshotRuntimePin(depsGraph[depPath]?.children); | ||
| 7 | - let result2 = engineName(ownPin ?? opts3.nodeVersion); | ||
| 8 | + let result2 = opts3.includeDepGraphHash ? "darwin;arm64;node26" : engineName(ownPin ?? opts3.nodeVersion); | ||
| 9 | if (opts3.includeDepGraphHash) { | ||
| 10 | const depGraphHash = calcDepGraphHash(depsGraph, cache, /* @__PURE__ */ new Set(), depPath, opts3.supportedArchitectures); | ||
| 11 | result2 += `;deps=${depGraphHash}`; | ||
| 12 | @@ -163136,12 +163136,7 @@ | ||
| 13 | unsafePerm: opts3.unsafePerm || false, | ||
| 14 | userAgent: opts3.userAgent | ||
| 15 | }; | ||
| 16 | - await runLifecycleHooksConcurrently(["preinstall", "install", "postinstall", "prepublish", "prepare"], Object.values(ctx.projects), opts3.childConcurrency || 5, scriptsOpts); | ||
| 17 | - for (const { id, manifest } of Object.values(ctx.projects)) { | ||
| 18 | - if (manifest?.scripts != null && (!opts3.pending || ctx.pendingBuilds.includes(id))) { | ||
| 19 | - ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1); | ||
| 20 | - } | ||
| 21 | - } | ||
| 22 | + process.stderr.write(JSON.stringify({guestPending:ctx.pendingBuilds})+"\n"); | ||
| 23 | await writeModulesManifest(ctx.rootModulesDir, { | ||
| 24 | prunedAt: (/* @__PURE__ */ new Date()).toUTCString(), | ||
| 25 | ...ctx.modulesFile, | ||
| 26 | @@ -163216,7 +163211,7 @@ | ||
| 27 | return false; | ||
| 28 | }; | ||
| 29 | const builtDepPaths = /* @__PURE__ */ new Set(); | ||
| 30 | - const storeIndex = opts3.skipIfHasSideEffectsCache ? opts3.frozenStore ? new ReadOnlyStoreIndex(opts3.storeDir) : new StoreIndex(opts3.storeDir) : void 0; | ||
| 31 | + const storeIndex = new ReadOnlyStoreIndex(opts3.storeDir); | ||
| 32 | const gvsDirByDepPath = /* @__PURE__ */ new Map(); | ||
| 33 | if (opts3.enableGlobalVirtualStore) { | ||
| 34 | const globalVirtualStoreDir = opts3.globalVirtualStoreDir ?? path92.join(opts3.storeDir, "links"); | ||
| 35 | @@ -163268,10 +163263,12 @@ | ||
| 36 | } else { | ||
| 37 | extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts3.lockfileDir)); | ||
| 38 | } | ||
| 39 | + if (!allowBuild(depPath)) { pkgsThatWereRebuilt.add(depPath); return; } | ||
| 40 | const resolution = pkgSnapshot.resolution; | ||
| 41 | let sideEffectsCacheKey; | ||
| 42 | const pkgId = pkgInfo.nonSemverVersion ?? `${pkgInfo.name}@${pkgInfo.version}`; | ||
| 43 | - if (opts3.skipIfHasSideEffectsCache && (resolution.gitHosted || resolution.integrity)) { | ||
| 44 | + if (!["esbuild@0.28.2", "esbuild@0.25.12", "tsc-rs@0.7.0", "node-pty@1.2.0-beta.15", "file:apps/mobile/deps/react-native-enriched-markdown-1.2.0-t3.11.tgz"].includes(pkgId)) throw new Error("Review this package before caching its hooks: " + pkgId); | ||
| 45 | + if (resolution.gitHosted || resolution.integrity) { | ||
| 46 | const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true }); | ||
| 47 | const pkgFilesIndex = storeIndex.get(filesIndexFile); | ||
| 48 | if (pkgFilesIndex) { | ||
| 49 | @@ -163281,6 +163278,13 @@ | ||
| 50 | nodeVersion | ||
| 51 | }); | ||
| 52 | if (pkgFilesIndex.sideEffects?.has(sideEffectsCacheKey)) { | ||
| 53 | + const delta = pkgFilesIndex.sideEffects.get(sideEffectsCacheKey); | ||
| 54 | + const restore = (await import("file:///opt/project-store/restore.cjs")).default; | ||
| 55 | + restore(pkgRoot, [...(delta.added ?? [])].map(([rel, info]) => { | ||
| 56 | + const digest = typeof info.digest === "string" ? info.digest : Buffer.from(info.digest).toString("hex"); | ||
| 57 | + return [rel, getFilePathByModeInCafs(opts3.storeDir, digest, info.mode), info.mode]; | ||
| 58 | + }), delta.deleted ?? []); | ||
| 59 | + process.stderr.write(JSON.stringify({cachePackage:pkgId,cacheKey:sideEffectsCacheKey,restored:delta.added?.size ?? 0})+"\n"); | ||
| 60 | pkgsThatWereRebuilt.add(depPath); | ||
| 61 | return; | ||
| 62 | } | ||
| 63 | @@ -163291,41 +163295,7 @@ | ||
| 64 | if (pgkManifest != null) { | ||
| 65 | requiresBuild = pkgRequiresBuild(pgkManifest, /* @__PURE__ */ new Map()); | ||
| 66 | } | ||
| 67 | - const hasSideEffects = requiresBuild && allowBuild(depPath) && await runPostinstallHooks({ | ||
| 68 | - depPath, | ||
| 69 | - extraBinPaths, | ||
| 70 | - extraEnv: opts3.extraEnv, | ||
| 71 | - optional: pkgSnapshot.optional === true, | ||
| 72 | - pkgRoot, | ||
| 73 | - rootModulesDir: ctx.rootModulesDir, | ||
| 74 | - scriptsPrependNodePath: opts3.scriptsPrependNodePath, | ||
| 75 | - shellEmulator: opts3.shellEmulator, | ||
| 76 | - unsafePerm: opts3.unsafePerm || false, | ||
| 77 | - userAgent: opts3.userAgent | ||
| 78 | - }); | ||
| 79 | - if (hasSideEffects && (opts3.sideEffectsCacheWrite ?? true) && (resolution.gitHosted || resolution.integrity)) { | ||
| 80 | - builtDepPaths.add(depPath); | ||
| 81 | - const filesIndexFile = pickStoreIndexKey(resolution, pkgId, { built: true }); | ||
| 82 | - try { | ||
| 83 | - if (!sideEffectsCacheKey) { | ||
| 84 | - sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { | ||
| 85 | - includeDepGraphHash: true, | ||
| 86 | - nodeVersion | ||
| 87 | - }); | ||
| 88 | - } | ||
| 89 | - await opts3.storeController.upload(pkgRoot, { | ||
| 90 | - sideEffectsCacheKey, | ||
| 91 | - filesIndexFile | ||
| 92 | - }); | ||
| 93 | - } catch (err2) { | ||
| 94 | - assert7(util31.types.isNativeError(err2)); | ||
| 95 | - logger2.warn({ | ||
| 96 | - error: err2, | ||
| 97 | - message: `An error occurred while uploading ${pkgRoot}`, | ||
| 98 | - prefix: opts3.lockfileDir | ||
| 99 | - }); | ||
| 100 | - } | ||
| 101 | - } | ||
| 102 | + if (requiresBuild) throw new Error("Missing target cache for " + pkgId); | ||
| 103 | pkgsThatWereRebuilt.add(depPath); | ||
| 104 | } catch (err2) { | ||
| 105 | assert7(util31.types.isNativeError(err2)); | ||
tools/project-pnpm/restore-test.cjs created+34| ... | @@ -0,0 +1,34 @@ | ||
| 1 | const assert = require('node:assert/strict'); | ||
| 2 | const fs = require('node:fs'); | ||
| 3 | const os = require('node:os'); | ||
| 4 | const path = require('node:path'); | ||
| 5 | const crypto = require('node:crypto'); | ||
| 6 | const restore = require('./restore.cjs'); | ||
| 7 | |||
| 8 | const root = fs.mkdtempSync(path.join(os.tmpdir(), 'project-restore-')); | ||
| 9 | try { | ||
| 10 | const bytes = process.env.PROJECT_RESTORE_SOURCE ? fs.readFileSync(process.env.PROJECT_RESTORE_SOURCE) : Buffer.from('cached native artifact'); | ||
| 11 | const digest = crypto.createHash('sha512').update(bytes).digest('hex'); | ||
| 12 | const source = process.env.PROJECT_RESTORE_SOURCE || path.join(root, digest.slice(0, 2), digest.slice(2)); | ||
| 13 | if (!process.env.PROJECT_RESTORE_SOURCE) { | ||
| 14 | fs.mkdirSync(path.dirname(source)); | ||
| 15 | fs.writeFileSync(source, bytes); | ||
| 16 | } | ||
| 17 | const destination = path.join(root, 'package'); | ||
| 18 | fs.mkdirSync(destination); | ||
| 19 | restore(destination, [['bin/tool', source, 0o755]], []); | ||
| 20 | assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), bytes.toString()); | ||
| 21 | fs.writeFileSync(path.join(destination, 'bin/tool'), 'changed'); | ||
| 22 | assert.deepEqual(fs.readFileSync(source), bytes); | ||
| 23 | fs.symlinkSync(root, path.join(destination, 'escape')); | ||
| 24 | assert.throws(() => restore(destination, [['escape/tool', source, 0o755]], []), /symlink/); | ||
| 25 | assert.throws(() => restore(destination, [['../tool', source, 0o755]], []), /Invalid/); | ||
| 26 | const corrupt = path.join(root, digest.slice(0, 2), digest.slice(2) + '-exec'); | ||
| 27 | fs.mkdirSync(path.dirname(corrupt), {recursive: true}); | ||
| 28 | fs.writeFileSync(corrupt, 'corrupt'); | ||
| 29 | assert.throws(() => restore(destination, [['bin/tool', corrupt, 0o755]], []), /integrity/); | ||
| 30 | assert.equal(fs.readFileSync(path.join(destination, 'bin/tool'), 'utf8'), 'changed'); | ||
| 31 | console.log('Reflink independence, path guards, and corrupt-cache rejection passed.'); | ||
| 32 | } finally { | ||
| 33 | fs.rmSync(root, {recursive: true, force: true}); | ||
| 34 | } | ||
tools/project-pnpm/restore.cjs created+45| ... | @@ -0,0 +1,45 @@ | ||
| 1 | const fs = require('node:fs'); | ||
| 2 | const path = require('node:path'); | ||
| 3 | const crypto = require('node:crypto'); | ||
| 4 | |||
| 5 | module.exports = function restore(root, added, deleted) { | ||
| 6 | root = fs.realpathSync(root); | ||
| 7 | function destination(relative) { | ||
| 8 | const parts = relative.split('/'); | ||
| 9 | if (parts.some(p => !p || p === '.' || p === '..') || relative.includes('\\')) { | ||
| 10 | throw new Error('Invalid cached path'); | ||
| 11 | } | ||
| 12 | let parent = root; | ||
| 13 | for (const part of parts.slice(0, -1)) { | ||
| 14 | parent = path.join(parent, part); | ||
| 15 | try { fs.mkdirSync(parent); } catch (error) { if (error.code !== 'EEXIST') throw error; } | ||
| 16 | if (!fs.lstatSync(parent).isDirectory() || fs.realpathSync(parent) !== parent) { | ||
| 17 | throw new Error('Cached path follows a symlink'); | ||
| 18 | } | ||
| 19 | } | ||
| 20 | return path.join(parent, parts.at(-1)); | ||
| 21 | } | ||
| 22 | for (const relative of deleted) fs.rmSync(destination(relative), {force: true}); | ||
| 23 | for (const [relative, source, mode] of added) { | ||
| 24 | const dest = destination(relative); | ||
| 25 | const temporary = dest + '.project-' + crypto.randomUUID(); | ||
| 26 | const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); | ||
| 27 | try { | ||
| 28 | if (!fs.fstatSync(fd).isFile()) throw new Error('Cache entry is not a regular file'); | ||
| 29 | const expected = path.basename(path.dirname(source)) + path.basename(source).replace(/-exec$/, ''); | ||
| 30 | const hash = crypto.createHash('sha512'); | ||
| 31 | const buffer = Buffer.alloc(65536); | ||
| 32 | let length; | ||
| 33 | while ((length = fs.readSync(fd, buffer)) > 0) hash.update(buffer.subarray(0, length)); | ||
| 34 | if (!/^[a-f0-9]{128}$/.test(expected) || hash.digest('hex') !== expected) { | ||
| 35 | throw new Error('Cached file failed integrity verification'); | ||
| 36 | } | ||
| 37 | fs.copyFileSync('/proc/self/fd/' + fd, temporary, fs.constants.COPYFILE_FICLONE_FORCE); | ||
| 38 | fs.chmodSync(temporary, mode & 0o777); | ||
| 39 | fs.renameSync(temporary, dest); | ||
| 40 | } finally { | ||
| 41 | fs.closeSync(fd); | ||
| 42 | fs.rmSync(temporary, {force: true}); | ||
| 43 | } | ||
| 44 | } | ||
| 45 | }; | ||
tools/project-store-install.py created+161| ... | @@ -0,0 +1,161 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Stage project tools on Snow Globe; activation is a separate atomic switch.""" | ||
| 3 | |||
| 4 | import argparse | ||
| 5 | import hashlib | ||
| 6 | import io | ||
| 7 | import json | ||
| 8 | import os | ||
| 9 | from pathlib import Path | ||
| 10 | import shutil | ||
| 11 | import socket | ||
| 12 | import subprocess | ||
| 13 | import tarfile | ||
| 14 | import tempfile | ||
| 15 | import urllib.request | ||
| 16 | |||
| 17 | |||
| 18 | HERE = Path(__file__).resolve().parent | ||
| 19 | BASE = Path("/opt/studio-project-store") | ||
| 20 | PROJECTS = "/srv/mac-studio-vm/sandbox/project" | ||
| 21 | GUEST_PROJECTS = "/Volumes/Code/project" | ||
| 22 | |||
| 23 | |||
| 24 | def download(url, digest): | ||
| 25 | with urllib.request.urlopen(url, timeout=60) as response: | ||
| 26 | data = response.read() | ||
| 27 | if hashlib.sha256(data).hexdigest() != digest: | ||
| 28 | raise ValueError("Downloaded tool failed checksum verification.") | ||
| 29 | return tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") | ||
| 30 | |||
| 31 | |||
| 32 | def stage(name): | ||
| 33 | destination = BASE / "releases" / name | ||
| 34 | if destination.exists(): | ||
| 35 | raise ValueError("Release already exists. Choose a new name.") | ||
| 36 | current = (BASE / "current").resolve(strict=True) | ||
| 37 | binaries = {} | ||
| 38 | for name, pattern, executable in (("python", "project-store-*-python3-*", "python3"), | ||
| 39 | ("node", "project-store-*-nodejs-*", "node"), | ||
| 40 | ("bwrap", "project-store-*-bubblewrap-*", "bwrap"), | ||
| 41 | ("coreutils", "project-store-coreutils", "true")): | ||
| 42 | roots = list(Path("/nix/var/nix/gcroots").glob(pattern)) | ||
| 43 | if len(roots) != 1: | ||
| 44 | raise ValueError(f"Expected one pinned {name} runtime, found {len(roots)}.") | ||
| 45 | binaries[name] = str(roots[0].resolve(strict=True) / "bin" / executable) | ||
| 46 | with tempfile.TemporaryDirectory(prefix=".prepare-", dir=destination.parent) as temporary: | ||
| 47 | release = Path(temporary) | ||
| 48 | shutil.copytree(HERE / "project-tools", release / "project-tools", | ||
| 49 | ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) | ||
| 50 | for filename in ("project-store.py", "project-store-test.py"): | ||
| 51 | shutil.copy2(HERE / filename, release / filename) | ||
| 52 | for filename in ("restore.cjs", "restore-test.cjs"): | ||
| 53 | shutil.copy2(HERE / "project-pnpm" / filename, release / filename) | ||
| 54 | profiles = json.loads((current / "profiles.json").read_text()) | ||
| 55 | for profile in profiles.values(): | ||
| 56 | profile.pop("workspace_config", None) | ||
| 57 | (release / "profiles.json").write_text(json.dumps(profiles)) | ||
| 58 | shutil.copy2(current / "jj.toml", release / "jj.toml") | ||
| 59 | (release / "bin").mkdir() | ||
| 60 | with download("https://github.com/jj-vcs/jj/releases/download/v0.42.0/" | ||
| 61 | "jj-v0.42.0-x86_64-unknown-linux-musl.tar.gz", | ||
| 62 | "2d91e81d649e617a81608e7401ad1106029c15ece01ac928c4a351abef42be6a") as archive: | ||
| 63 | member, = [m for m in archive if m.isfile() and Path(m.name).name == "jj"] | ||
| 64 | (release / "bin/jj-real").write_bytes(archive.extractfile(member).read()) | ||
| 65 | (release / "bin/jj-real").chmod(0o755) | ||
| 66 | (release / "bin/jj").symlink_to("../project-tools/jj") | ||
| 67 | with download("https://github.com/BurntSushi/ripgrep/releases/download/15.2.0/" | ||
| 68 | "ripgrep-15.2.0-x86_64-unknown-linux-musl.tar.gz", | ||
| 69 | "33e15bcf1624b25cdd2a55813a47a2f95dbe126268203e76aa6a585d1e7b149c") as archive: | ||
| 70 | member, = [m for m in archive if m.isfile() and Path(m.name).name == "rg"] | ||
| 71 | (release / "bin/rg").write_bytes(archive.extractfile(member).read()) | ||
| 72 | (release / "bin/rg").chmod(0o755) | ||
| 73 | for name in ("jj", "pnpm", "project"): | ||
| 74 | path = release / "project-tools" / name | ||
| 75 | content = path.read_text().split("\n", 1)[1] | ||
| 76 | path.write_text("#!" + binaries["python"] + "\n" + content) | ||
| 77 | path.chmod(0o755) | ||
| 78 | for version, digest, directory, original, patched in ( | ||
| 79 | ("10.32.1", "9b943b94bc8f55efb993aad8e44b538e6b091e60a9e4a944dcde869855f233e3", | ||
| 80 | "pnpm10", "pnpm.cjs", "pnpm-darwin-rebuild.cjs"), | ||
| 81 | ("11.10.0", "620b6605ea4f62fc56a6d0a98733f479071d0321e03e486b522c7e9a74617431", | ||
| 82 | "pnpm11", "pnpm.mjs", "pnpm-target-cache.mjs"), | ||
| 83 | ): | ||
| 84 | with download(f"https://registry.npmjs.org/pnpm/-/pnpm-{version}.tgz", digest) as archive: | ||
| 85 | archive.extractall(release / directory, filter="data") | ||
| 86 | package = release / directory / "package" | ||
| 87 | for path in (package / "dist").iterdir(): | ||
| 88 | path.rename(release / directory / path.name) | ||
| 89 | shutil.rmtree(package) | ||
| 90 | shutil.copy2(release / directory / original, release / directory / patched) | ||
| 91 | subprocess.run(["patch", "--batch", str(release / directory / patched), | ||
| 92 | str(HERE / "project-pnpm" / f"pnpm-{version}.patch")], check=True) | ||
| 93 | (release / "pnpm12").mkdir() | ||
| 94 | with download("https://github.com/pnpm/pnpm/releases/download/v12.14.0/pnpm-linux-x64-musl.tar.gz", | ||
| 95 | "d7115e2f2791d79bf9e7e448609537595bb7787de1ef5292ce91ff3d1be2629e") as archive: | ||
| 96 | member, = [m for m in archive if m.isfile() and Path(m.name).name == "pnpm"] | ||
| 97 | (release / "pnpm12/pnpm").write_bytes(archive.extractfile(member).read()) | ||
| 98 | (release / "pnpm12/pnpm").chmod(0o755) | ||
| 99 | command = [binaries["bwrap"], "--die-with-parent", "--unshare-all", "--cap-drop", "ALL", | ||
| 100 | "--ro-bind", "/nix/store", "/nix/store", "--proc", "/proc", "--dev", "/dev", | ||
| 101 | "--tmpfs", "/tmp", "--dir", "/home/clo", | ||
| 102 | "--ro-bind", str(destination), "/opt/project-store", | ||
| 103 | "--bind", PROJECTS, GUEST_PROJECTS, "--bind", PROJECTS, PROJECTS] | ||
| 104 | stores = "/srv/mac-studio-vm/sandbox/cache/project-pnpm" | ||
| 105 | for target in (stores, "/Volumes/Code/cache/project-pnpm", "/Users/admin/Library/Caches/project"): | ||
| 106 | command.extend(("--bind", stores, target)) | ||
| 107 | command.extend(("--bind", "/srv/mac-studio-vm/sandbox/cache/pnpm12", "/Volumes/Code/cache/pnpm12")) | ||
| 108 | command.extend(("--bind", "/home/clo/.cache/pnpm", "/home/clo/.cache/pnpm", | ||
| 109 | "--clearenv", "--setenv", "HOME", "/home/clo", "--setenv", "PATH", | ||
| 110 | "/opt/project-store/bin:" + ":".join(str(Path(binaries[key]).parent) | ||
| 111 | for key in ("node", "python", "coreutils")), | ||
| 112 | "--setenv", "JJ_CONFIG", "/opt/project-store/jj.toml", | ||
| 113 | "--setenv", "PROJECT_STORE_SSH", "", "--setenv", "LC_ALL", "C.UTF-8", | ||
| 114 | "--chdir", GUEST_PROJECTS, binaries["python"], | ||
| 115 | "/opt/project-store/project-store.py", GUEST_PROJECTS)) | ||
| 116 | network = ["--share-net", "--ro-bind", "/etc/resolv.conf", "/etc/resolv.conf", | ||
| 117 | "--ro-bind", "/etc/ssl/certs/ca-bundle.crt", "/etc/ssl/certs/ca-bundle.crt", | ||
| 118 | "--setenv", "SSL_CERT_FILE", "/etc/ssl/certs/ca-bundle.crt", | ||
| 119 | "--setenv", "NODE_EXTRA_CA_CERTS", "/etc/ssl/certs/ca-bundle.crt"] | ||
| 120 | (release / "ssh-command").write_text( | ||
| 121 | "#!" + binaries["python"] + "\nimport json, subprocess, sys\n" | ||
| 122 | "request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1)\n" | ||
| 123 | "if len(request) > 4 * 1024 * 1024: sys.exit('Project request exceeds 4 MiB.')\n" | ||
| 124 | "command = " + repr(command) + "\n" | ||
| 125 | "if json.loads(request).get('command') == 'read':\n" | ||
| 126 | " command = ['--ro-bind' if arg == '--bind' else arg for arg in command]\n" | ||
| 127 | "if json.loads(request).get('command') == 'pnpm':\n" | ||
| 128 | " command[command.index('--chdir'):command.index('--chdir')] = " + repr(network) + "\n" | ||
| 129 | "result = subprocess.run(command, input=request)\n" | ||
| 130 | "sys.exit(result.returncode if result.returncode >= 0 else 128 - result.returncode)\n") | ||
| 131 | (release / "ssh-command").chmod(0o755) | ||
| 132 | release.chmod(0o755) | ||
| 133 | release.rename(destination) | ||
| 134 | release.mkdir() | ||
| 135 | print(destination) | ||
| 136 | |||
| 137 | |||
| 138 | def main(): | ||
| 139 | parser = argparse.ArgumentParser(description=__doc__) | ||
| 140 | parser.add_argument("action", choices=("stage", "activate")) | ||
| 141 | parser.add_argument("release") | ||
| 142 | args = parser.parse_args() | ||
| 143 | if socket.gethostname() != "snowglobe" or os.geteuid() != 0: | ||
| 144 | parser.error("Run this installer as root on Snow Globe.") | ||
| 145 | if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release): | ||
| 146 | parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.") | ||
| 147 | if args.action == "stage": | ||
| 148 | os.umask(0o022) | ||
| 149 | stage(args.release) | ||
| 150 | else: | ||
| 151 | release = BASE / "releases" / args.release | ||
| 152 | if not (release / "ssh-command").is_file(): | ||
| 153 | parser.error("Stage this release before activating it.") | ||
| 154 | link = BASE / (".current-" + args.release) | ||
| 155 | link.symlink_to(Path("releases") / args.release) | ||
| 156 | link.replace(BASE / "current") | ||
| 157 | print(release) | ||
| 158 | |||
| 159 | |||
| 160 | if __name__ == "__main__": | ||
| 161 | main() | ||
tools/project-store-test.py created+128| ... | @@ -0,0 +1,128 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Disposable workspace tests; PROJECT_STORE_REAL_REFLINK=1 requires Linux/ZFS.""" | ||
| 3 | |||
| 4 | import importlib.util | ||
| 5 | import os | ||
| 6 | from pathlib import Path | ||
| 7 | import shutil | ||
| 8 | import subprocess | ||
| 9 | import tempfile | ||
| 10 | import unittest | ||
| 11 | from unittest.mock import patch | ||
| 12 | |||
| 13 | spec = importlib.util.spec_from_file_location("project_store", Path(__file__).with_name("project-store.py")) | ||
| 14 | store = importlib.util.module_from_spec(spec) | ||
| 15 | spec.loader.exec_module(store) | ||
| 16 | |||
| 17 | |||
| 18 | class WorkspaceTest(unittest.TestCase): | ||
| 19 | def setUp(self): | ||
| 20 | self.tmp = tempfile.TemporaryDirectory(prefix="project-store-test-") | ||
| 21 | self.addCleanup(self.tmp.cleanup) | ||
| 22 | self.root = Path(self.tmp.name).resolve() | ||
| 23 | config = self.root / "jj.toml" | ||
| 24 | config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n[signing]\nbehavior="drop"\n') | ||
| 25 | self.env = patch.dict(os.environ, {"JJ_CONFIG": str(config)}) | ||
| 26 | self.env.start() | ||
| 27 | self.addCleanup(self.env.stop) | ||
| 28 | if not os.environ.get("PROJECT_STORE_REAL_REFLINK"): | ||
| 29 | self.copy = patch.object(store.fcntl, "ioctl", side_effect=self.copy_bytes) | ||
| 30 | self.copy.start() | ||
| 31 | self.addCleanup(self.copy.stop) | ||
| 32 | self.main = self.root / "Example/main" | ||
| 33 | self.main.parent.mkdir() | ||
| 34 | self.jj("git", "init", "--no-colocate", str(self.main)) | ||
| 35 | (self.main / ".gitignore").write_text("target/\n") | ||
| 36 | (self.main / "source.txt").write_text("committed\n") | ||
| 37 | (self.main / "unchanged.txt").write_text("same\n") | ||
| 38 | (self.main / "link").symlink_to("source.txt") | ||
| 39 | self.jj("-R", str(self.main), "describe", "-m", "fixture") | ||
| 40 | self.revision = self.jj("-R", str(self.main), "log", "--no-graph", "-r", "@", "-T", "commit_id").strip() | ||
| 41 | (self.main / "target/debug").mkdir(parents=True) | ||
| 42 | (self.main / "target/debug/artifact").write_bytes(b"warm artifact" * 10000) | ||
| 43 | (self.main / "target/debug/.cargo-lock").touch() | ||
| 44 | self.request = dict(project="Example", source="main", destination="agent", | ||
| 45 | revision=self.revision, targets=["target"]) | ||
| 46 | |||
| 47 | @staticmethod | ||
| 48 | def copy_bytes(destination, command, source): | ||
| 49 | with os.fdopen(os.dup(source), "rb") as src, os.fdopen(os.dup(destination), "wb") as dst: | ||
| 50 | shutil.copyfileobj(src, dst) | ||
| 51 | |||
| 52 | def jj(self, *args): | ||
| 53 | return subprocess.run(["jj", *args], text=True, capture_output=True, check=True).stdout | ||
| 54 | |||
| 55 | def test_host_checkout_without_cache_preserves_donor(self): | ||
| 56 | (self.main / "source.txt").write_text("active agent edit\n") | ||
| 57 | request = dict(project="Example", destination="cold", revision=self.revision, | ||
| 58 | engine="darwin;arm64;node26", install=True) | ||
| 59 | self.assertEqual(store.seed(self.root, request), {"workspace": "cold", "installed": False, "pnpm": False}) | ||
| 60 | self.assertEqual((self.main.parent / "cold/source.txt").read_text(), "committed\n") | ||
| 61 | self.assertEqual((self.main / "source.txt").read_text(), "active agent edit\n") | ||
| 62 | with self.assertRaises(ValueError): | ||
| 63 | store.seed(self.root, request) | ||
| 64 | for name in ("../escape", "main", "default"): | ||
| 65 | with self.assertRaises(ValueError): | ||
| 66 | store.seed(self.root, dict(request, destination=name)) | ||
| 67 | |||
| 68 | def test_cache_inputs_track_dependency_changes_only(self): | ||
| 69 | (self.main / "package.json").write_text('{"packageManager":"pnpm@10.32.1"}') | ||
| 70 | before = store.dependency_inputs(self.main) | ||
| 71 | operation = self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id") | ||
| 72 | (self.main / "source.txt").write_text("new source\n") | ||
| 73 | self.assertEqual(store.dependency_inputs(self.main), before) | ||
| 74 | (self.main / "package.json").write_text('{"packageManager":"pnpm@12.14.0"}') | ||
| 75 | self.assertNotEqual(store.dependency_inputs(self.main), before) | ||
| 76 | self.assertEqual(self.jj("--ignore-working-copy", "-R", str(self.main), "op", "log", "--no-graph", "-n", "1", "-T", "id"), operation) | ||
| 77 | (self.main / "package.json").unlink() | ||
| 78 | (self.main / "package.json").symlink_to("source.txt") | ||
| 79 | with self.assertRaises(ValueError): | ||
| 80 | store.dependency_inputs(self.main) | ||
| 81 | |||
| 82 | def test_clone_restores_revision_and_preserves_warm_files(self): | ||
| 83 | (self.main / "source.txt").write_text("donor edit\n") | ||
| 84 | (self.main / "donor-only.txt").write_text("uncommitted\n") | ||
| 85 | original = self.main / "target/debug/artifact" | ||
| 86 | mtime = (self.main / "unchanged.txt").stat().st_mtime_ns | ||
| 87 | self.assertEqual(store.seed(self.root, self.request), {"workspace": "agent"}) | ||
| 88 | agent = self.main.parent / "agent" | ||
| 89 | self.assertEqual((agent / "source.txt").read_text(), "committed\n") | ||
| 90 | self.assertFalse((agent / "donor-only.txt").exists()) | ||
| 91 | self.assertEqual((self.main / "source.txt").read_text(), "donor edit\n") | ||
| 92 | self.assertEqual((agent / "unchanged.txt").stat().st_mtime_ns, mtime) | ||
| 93 | artifact = agent / "target/debug/artifact" | ||
| 94 | self.assertEqual(artifact.read_bytes(), original.read_bytes()) | ||
| 95 | self.assertEqual(artifact.stat().st_mtime_ns, original.stat().st_mtime_ns) | ||
| 96 | self.assertNotEqual(artifact.stat().st_ino, original.stat().st_ino) | ||
| 97 | artifact.write_bytes(b"modified") | ||
| 98 | self.assertNotEqual(artifact.read_bytes(), original.read_bytes()) | ||
| 99 | self.assertEqual(os.readlink(agent / "link"), "source.txt") | ||
| 100 | self.assertEqual(self.jj("-R", str(agent), "diff", "--summary"), "") | ||
| 101 | self.assertFalse((agent / ".jj/repo").is_dir()) | ||
| 102 | self.assertFalse(Path((agent / ".jj/repo").read_text()).is_absolute()) | ||
| 103 | |||
| 104 | def test_rejects_paths_and_existing_destination(self): | ||
| 105 | for key, value in (("source", "../outside"), ("destination", "main"), | ||
| 106 | ("project", "/tmp"), ("targets", ["../outside"])): | ||
| 107 | with self.subTest(key=key), self.assertRaises(ValueError): | ||
| 108 | store.seed(self.root, dict(self.request, **{key: value})) | ||
| 109 | (self.main.parent / "agent").mkdir() | ||
| 110 | with self.assertRaises(ValueError): | ||
| 111 | store.seed(self.root, self.request) | ||
| 112 | |||
| 113 | def test_rejects_symlink_workspace(self): | ||
| 114 | (self.main.parent / "linked").symlink_to(self.main) | ||
| 115 | with self.assertRaises(OSError): | ||
| 116 | store.seed(self.root, dict(self.request, source="linked")) | ||
| 117 | |||
| 118 | def test_failed_reflink_removes_workspace_registration_and_files(self): | ||
| 119 | with patch.object(store.fcntl, "ioctl", side_effect=OSError("reflink unavailable")): | ||
| 120 | with self.assertRaises(OSError): | ||
| 121 | store.seed(self.root, self.request) | ||
| 122 | self.assertFalse((self.main.parent / "agent").exists()) | ||
| 123 | self.assertNotIn("agent:", self.jj("-R", str(self.main), "workspace", "list")) | ||
| 124 | self.assertEqual((self.main / "source.txt").read_text(), "committed\n") | ||
| 125 | |||
| 126 | |||
| 127 | if __name__ == "__main__": | ||
| 128 | unittest.main() | ||
tools/project-store.md created+220| ... | @@ -0,0 +1,220 @@ | ||
| 1 | # Shared project commands | ||
| 2 | |||
| 3 | `tools/project-tools/` owns the VM's `jj`, `pnpm`, and `project` wrappers. | ||
| 4 | The config repo's `users/clover/agents/vm` entries link here; installed wrappers | ||
| 5 | use `~/.local/share/project-tools/current`. Keep native executables adjacent to | ||
| 6 | the public commands as `jj-real` and `pnpm-real`. | ||
| 7 | |||
| 8 | ## Defaults | ||
| 9 | |||
| 10 | - `project read rg ARGS...` searches on Linux beside ZFS and streams normal | ||
| 11 | ripgrep output and exit status. `project read rg --files` lists files. | ||
| 12 | `.gitignore` works in Jujutsu checkouts without a visible `.git` directory; | ||
| 13 | `--require-git` restores ripgrep's usual Git-only policy. Search preprocessors | ||
| 14 | are disabled. `project read sha256sum FILE...` hashes named shared files | ||
| 15 | without transferring their contents over NFS. Run these from the project | ||
| 16 | share; read requests have read-only mounts, no network, and no repository lock. | ||
| 17 | - Noninteractive `jj` operations in `/Volumes/Code/project` run on Snow Globe. | ||
| 18 | This includes workspace creation, snapshots, rebases, restores, and reads. | ||
| 19 | The guest's resolved configuration, identity, ignores, and paths travel with | ||
| 20 | the request. Both sides must run exactly the same Jujutsu build. | ||
| 21 | - Editors, external diff tools, signing, network Git commands, and local-disk | ||
| 22 | repositories retain native execution. Explicit file paths for a shared | ||
| 23 | repository cannot escape the project share, including through symlinks. | ||
| 24 | - `pnpm install`, `i`, `add`, `update`, and `remove` resolve, fetch, and import | ||
| 25 | packages on Linux, including changed manifests and lockfiles. The runtime must | ||
| 26 | match the project's exact pin: 10.32.1, 11.10.0, or 12.14.0. Directory selection, | ||
| 27 | workspace filters, frozen/offline installs, and lockfile-only updates retain | ||
| 28 | their normal meanings. `--ignore-scripts` skips the Mac hook phase. | ||
| 29 | - Activation links Vite+'s installed managed pnpm launchers to this same wrapper, | ||
| 30 | preserving each original as the adjacent `pnpm-real`. This covers the | ||
| 31 | project-local JavaScript `vp` launcher, which otherwise bypasses pnpm on PATH. | ||
| 32 | Vite+ keeps package-manager detection and argument handling; its notifier | ||
| 33 | setting does not prevent offloading. Explicit `VP_PACKAGE_MANAGER` overrides | ||
| 34 | retain native execution. Reactivate after Vite+ downloads another pnpm version. | ||
| 35 | - `pnpm run`, `exec`, and script shortcuts execute natively. Their implicit | ||
| 36 | dependency installs use Linux too; dependency verification opt-outs and custom | ||
| 37 | verification policies are preserved. Inherited npm client metadata and Node | ||
| 38 | heap limits retain offloading; `NODE_ENV=production` reaches the pinned pnpm | ||
| 39 | unchanged on both hosts. Unsupported options, runtime versions, other custom | ||
| 40 | install environment settings, and unreviewed pnpmfile hooks retain native | ||
| 41 | execution. Environment fallbacks name the variables without exposing values. | ||
| 42 | A failed remote operation is reported once. | ||
| 43 | - `project worktree PROJECT NAME` uses those same `jj` and `pnpm` commands. | ||
| 44 | `--no-install` skips dependencies. The checkout path is printed only after | ||
| 45 | installation and Mac hooks succeed, including with `--warm-from`. | ||
| 46 | New workspaces wait for NFS visibility. | ||
| 47 | |||
| 48 | Linux installs with lifecycle scripts disabled, selects Darwin | ||
| 49 | ARM64 packages, validates the guest's Node version, and requires ZFS reflinks. | ||
| 50 | Knife's reviewed `.pnpmfile.cjs` runs in the Linux sandbox: its peer-range change | ||
| 51 | and Windows-only architecture selection behave identically on Linux and Darwin. | ||
| 52 | The backend checks its exact SHA-256 before offloading. Changed hooks, other | ||
| 53 | pnpmfiles, and config dependencies retain native installation. | ||
| 54 | Pending dependency and workspace hooks finish natively on the Mac. pnpm 10/11 | ||
| 55 | use `pnpm --recursive --include-workspace-root rebuild --pending`. pnpm 12 uses | ||
| 56 | a frozen install with the original filters and dependency groups, preserving | ||
| 57 | prepared patches. Its repeat-install shortcut and build-cache reads/writes are | ||
| 58 | disabled for this phase: restoring cached generated files over NFS can take | ||
| 59 | longer than running their hooks. Ordinary native installs retain their cache policy. | ||
| 60 | The wrapper announces the Mac handoff. | ||
| 61 | It also announces storage-host installation and package-store lock contention. | ||
| 62 | The two phases share policy and registry metadata in the project's `.pnpm-cache` | ||
| 63 | directory, outside its workspaces. This avoids repeating network verification on | ||
| 64 | the Mac. The handoff waits until NFS exposes the server's new modules manifest. | ||
| 65 | Repository configuration is not temporarily rewritten; manifests and | ||
| 66 | lockfiles change only as requested by the dependency command. | ||
| 67 | |||
| 68 | The root-owned `profiles.json` binds manifests, lockfiles, workspace settings, | ||
| 69 | npmrc, pnpmfiles, patches, and local tarballs. Matching alphaXiv pnpm 10.32.1 and | ||
| 70 | Knife pnpm 11.10.0 profiles on Node 26 can additionally restore reviewed Darwin | ||
| 71 | side effects on Linux after SHA-512 verification using `project-pnpm/`'s patches. | ||
| 72 | A profile miss still installs on Linux; it leaves native builds pending for the | ||
| 73 | Mac instead of replaying an older build. Validation does not snapshot the working | ||
| 74 | copy. Ordinary Linux side-effect cache reuse is disabled. | ||
| 75 | |||
| 76 | ## Locking and storage | ||
| 77 | |||
| 78 | The Mac wrapper uses `flock`; Linux uses a POSIX record lock on the same | ||
| 79 | `.jj/repo/command.lock` inode. macOS NFS translates `flock` into that server lock. | ||
| 80 | The executing Jujutsu process inherits the descriptor. The forwarding guest | ||
| 81 | never holds a second lock. Ctrl-C cancels a waiter; never unlink the lock inode. | ||
| 82 | A command lock does not reserve revisions across a sequence of commands. | ||
| 83 | |||
| 84 | The Code export is one ZFS dataset. `project worktree --warm-from DONOR` uses | ||
| 85 | per-file FICLONE, preserving independent inodes and source timestamps, excluding | ||
| 86 | root repository metadata and Mac sidecars. It restores the requested revision | ||
| 87 | and retains ignored build outputs. The guest holds the donor's Cargo locks; | ||
| 88 | editors and other writers must also be idle. No byte-copy fallback is allowed. | ||
| 89 | |||
| 90 | The dedicated SSH key forces the unprivileged uid-3000 Bubblewrap launcher. | ||
| 91 | Jujutsu requests have no network. Dependency requests get network access with | ||
| 92 | DNS and the host CA bundle; lifecycle scripts remain disabled on Linux. Neither path | ||
| 93 | exposes host credentials, forwarding, or snapshot privileges. | ||
| 94 | The sandbox binds projects and package stores at their guest paths as well as | ||
| 95 | host paths. Identical store paths allow repeated installs without repaving | ||
| 96 | `node_modules`. Artifacts remain under Code. pnpm 11 uses a guest-local SQLite | ||
| 97 | index under `~/Library/Caches/project/PROJECT/v11`; pnpm 12 uses the shared | ||
| 98 | `/Volumes/Code/cache/pnpm12` file index. Both phases use workspace-local virtual | ||
| 99 | stores so platform-specific build outputs belong to that workspace. | ||
| 100 | |||
| 101 | ## Updating the existing deployment | ||
| 102 | |||
| 103 | Upload `project-store-install.py`, `project-store.py`, `project-store-test.py`, | ||
| 104 | `project-tools/`, and `project-pnpm/` together to a private staging directory on | ||
| 105 | Snow Globe. Run the installer there as root: | ||
| 106 | |||
| 107 | ```sh | ||
| 108 | python3 project-store-install.py stage RELEASE | ||
| 109 | ``` | ||
| 110 | |||
| 111 | It verifies pinned upstream downloads, applies the cache patches, preserves the | ||
| 112 | reviewed profiles, and builds a versioned runtime under | ||
| 113 | `/opt/studio-project-store/releases/`. Existing `project-store-*` Nix GC roots | ||
| 114 | pin Python, Node, Bubblewrap, and coreutils. `current` is unchanged by staging. | ||
| 115 | Test the staged `ssh-command` as `clo` with disposable projects and a temporary | ||
| 116 | restricted key. Do not run Jujutsu as root against shared repositories. | ||
| 117 | |||
| 118 | Upload `project-tools/` to a guest staging directory and run its installer as | ||
| 119 | `admin`. After the staged host and guest tests pass: | ||
| 120 | |||
| 121 | ```sh | ||
| 122 | # Snow Globe | ||
| 123 | python3 project-store-install.py activate RELEASE | ||
| 124 | # VM, using tools/project-tools/install.py | ||
| 125 | python3 install.py stage RELEASE | ||
| 126 | python3 install.py activate RELEASE | ||
| 127 | ``` | ||
| 128 | |||
| 129 | Guest activation saves the prior tools under `project-tools/previous`, preserves | ||
| 130 | the native executables, and changes symlinks atomically. Subsequent updates switch | ||
| 131 | the complete wrapper directory through `current`. Keep the previous host release | ||
| 132 | and guest tools for rollback. Roll back both `current` links; do not remove | ||
| 133 | workspaces, locks, stores, or caches. | ||
| 134 | |||
| 135 | ## Verification | ||
| 136 | |||
| 137 | ```sh | ||
| 138 | python3 tools/project-store-test.py | ||
| 139 | python3 tools/project-tools/test.py | ||
| 140 | python3 tools/project-tools/jj-test.py | ||
| 141 | ``` | ||
| 142 | |||
| 143 | Run `project-tools/platform-test.py` inside the development VM against the | ||
| 144 | normal endpoint. Its disposable project checks Darwin-only optional dependencies | ||
| 145 | immediately after the Linux import, before Mac build scripts can repair them. | ||
| 146 | `project-tools/finish-test.py` checks native hooks, denied scripts, patch inode | ||
| 147 | preservation, repeated installs, filtered dependency groups, and nested mutations. | ||
| 148 | |||
| 149 | The backend tests create disposable repositories. `PROJECT_STORE_REAL_REFLINK=1` | ||
| 150 | with `TMPDIR` on disposable Linux/ZFS storage exercises real clone independence, | ||
| 151 | timestamps, donor edits, path rejection, and failure cleanup. The config repo's | ||
| 152 | `vm/test.py` covers overlay preparation, worktrees, and shared memory links. | ||
| 153 | `project-pnpm/restore-test.cjs` checks cache integrity and path guards on Linux/ZFS. | ||
| 154 | |||
| 155 | On 2026-10-22, the endpoint created an 801-file workspace through the VM in 2.9 | ||
| 156 | seconds and rebased a two-commit fixture in 1.6 seconds. Cross-client lock | ||
| 157 | contention and cancellation passed. Disposable pnpm 10, 11, and 12 projects | ||
| 158 | installed changed dependencies on Linux and ran Darwin dependency and workspace | ||
| 159 | hooks on the Mac. Frozen-lockfile failures, lockfile-only updates, filtered | ||
| 160 | add/update/remove commands, ignored scripts, and native esbuild execution passed. | ||
| 161 | A pnpm 12 filtered update took 12.4 seconds, including 1.4 seconds on Linux; | ||
| 162 | a subsequent script ran in 1.2 seconds without reinstalling. Nested pnpm scripts | ||
| 163 | and dependency-verification opt-outs passed. An existing global virtual store | ||
| 164 | migrated successfully. After activation, ordinary commands against the real | ||
| 165 | shared store passed: a fresh install took 70.5 seconds, a filtered add took 32.9 | ||
| 166 | seconds, and a script took 2.5 seconds. Native rebuilding left all 1,048 sampled | ||
| 167 | lodash files' inodes and mtimes unchanged. These are fixture measurements with other agents | ||
| 168 | active, not latency bounds or full-project build measurements. | ||
| 169 | |||
| 170 | Vite+ 1.3.0 was also verified through the normal endpoint with pnpm 12.14.0 | ||
| 171 | and Knife's reviewed `.pnpmfile.cjs`: package imports ran on Linux/ZFS, while | ||
| 172 | the workspace postinstall and native esbuild ran on Darwin ARM64. Frozen/offline | ||
| 173 | installs, add/remove, lockfile-only, ignored scripts, script execution, and frozen | ||
| 174 | lockfile failure propagation passed. Editing the hook selected Mac installation. | ||
| 175 | The host and VM tools were activated as `20261022-vp-preview-2`. | ||
| 176 | |||
| 177 | pnpm 12 needs `supportedArchitectures: [darwin-arm64]` for host-side package | ||
| 178 | selection; the older `config.os`/`config.cpu` overrides do not select its optional | ||
| 179 | dependencies. This is passed as a command setting without editing the workspace. | ||
| 180 | The platform regression fails against the old endpoint and passes with the fix. | ||
| 181 | The full Knife workspace imported 75 missing Darwin packages, and its Vite+, | ||
| 182 | Vite, Vitest, Rolldown, and linter bindings loaded successfully before rebuilding. | ||
| 183 | With Mac build-cache writes disabled, a normal frozen install in `knife-merge` | ||
| 184 | completed successfully in 278 seconds, including the mobile dependency's native | ||
| 185 | asset downloads and the root prepare hook. The host and guest were activated as | ||
| 186 | `20261023-vp-platform-3`. Linux side-effect cache reads and writes are explicitly | ||
| 187 | disabled with pnpm 12's supported boolean settings. | ||
| 188 | After installation, `pnpm exec vp --version`, the TypeScript compiler, and the | ||
| 189 | Vitest CLI started successfully. The node-pty addon loaded, and sharp encoded a | ||
| 190 | small PNG using its Darwin native module. | ||
| 191 | |||
| 192 | A follow-up trace in `knife-disk` found the Mac phase copying patched packages | ||
| 193 | and restoring cached mobile assets over NFS. The frozen-install completion | ||
| 194 | described above disables those cache transfers. Five disposable VM cases cover | ||
| 195 | native and workspace hooks, denied scripts, patch preservation, filtered | ||
| 196 | production/development installs, nested adds, repeated installs, and inherited | ||
| 197 | agent environments. A deployed | ||
| 198 | frozen install in `knife-disk` completed in 43.5 seconds with its native assets | ||
| 199 | already present. Vite+, TypeScript, and the node-pty native binding loaded. | ||
| 200 | The real project-local `vp i --frozen-lockfile` then completed in 28.8 seconds | ||
| 201 | through Vite+'s managed pnpm launcher and the shared wrapper. Exporting | ||
| 202 | `VP_PNPM_MANAGER=no` alone did not route that JavaScript launcher through PATH; | ||
| 203 | the obsolete VM environment setting was removed. | ||
| 204 | |||
| 205 | The host and guest were activated as `20261023-vp-env-4`. Staged pnpm 10, 11, | ||
| 206 | and 12 installs with `NODE_ENV=production` preserved their native dependency | ||
| 207 | selection; production mode is forwarded as environment, without inventing a | ||
| 208 | `--prod` flag. The VM fixture also checked that Mac hooks receive that value | ||
| 209 | while npm client metadata and Node memory limits remain compatible with offloading. | ||
| 210 | A real `vp i --frozen-lockfile` with those inherited metadata and memory settings | ||
| 211 | completed in 23.2 seconds on `knife-disk`, including Mac hooks with native assets | ||
| 212 | already present. Vite+'s tools, TypeScript, and the node-pty Darwin binding loaded. | ||
| 213 | |||
| 214 | Release `20261023-tools-6` adds the read-only storage tools and waits for NFS | ||
| 215 | visibility before opening warm workspaces. A VM request listed 2,001 fixture | ||
| 216 | files in 1.93 seconds and hashed a 330 MiB fixture in 0.95 seconds. Staged | ||
| 217 | checks verified read-only mounts, network isolation, ignore rules, normal | ||
| 218 | search exit statuses, hash equality, and preprocessor rejection. A real warm | ||
| 219 | checkout installed a changed dependency and its Darwin hook in 7.81 seconds; | ||
| 220 | the donor and an explicit `--no-install` copy retained the original dependency. | ||
tools/project-store.py created+410| ... | @@ -0,0 +1,410 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Create Jujutsu workspaces and restore dependencies using host-side ZFS reflinks. | ||
| 3 | |||
| 4 | Run behind a forced SSH command with a fixed project root and an unprivileged | ||
| 5 | account. The guest supplies one JSON request on stdin, never a shell command. | ||
| 6 | """ | ||
| 7 | |||
| 8 | import argparse | ||
| 9 | from contextlib import ExitStack | ||
| 10 | import fcntl | ||
| 11 | import hashlib | ||
| 12 | import json | ||
| 13 | import os | ||
| 14 | from pathlib import Path | ||
| 15 | import re | ||
| 16 | import shutil | ||
| 17 | import stat | ||
| 18 | import subprocess | ||
| 19 | import sys | ||
| 20 | import tempfile | ||
| 21 | |||
| 22 | sys.path.insert(0, str(Path(__file__).with_name("project-tools"))) | ||
| 23 | import client | ||
| 24 | |||
| 25 | |||
| 26 | def parts(value): | ||
| 27 | if not isinstance(value, str) or not value or value.startswith("/"): | ||
| 28 | raise ValueError("Use a relative path inside the project directory.") | ||
| 29 | names = value.split("/") | ||
| 30 | if any(n in ("", ".", "..", ".jj", ".git") or "\\" in n | ||
| 31 | or any(ord(c) < 32 or ord(c) == 127 for c in n) for n in names): | ||
| 32 | raise ValueError("Paths cannot contain traversal or repository metadata.") | ||
| 33 | return names | ||
| 34 | |||
| 35 | |||
| 36 | def directory(stack, root, names): | ||
| 37 | for name in names: | ||
| 38 | root = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root) | ||
| 39 | stack.callback(os.close, root) | ||
| 40 | return root | ||
| 41 | |||
| 42 | |||
| 43 | def clone_tree(source, destination, *, workspace=False): | ||
| 44 | initial = os.fstat(source) | ||
| 45 | for name in os.listdir(source): | ||
| 46 | if (workspace and name in (".jj", ".git")) or name.startswith("._") or name == ".DS_Store": | ||
| 47 | continue | ||
| 48 | before = os.stat(name, dir_fd=source, follow_symlinks=False) | ||
| 49 | with ExitStack() as stack: | ||
| 50 | if stat.S_ISDIR(before.st_mode): | ||
| 51 | src = directory(stack, source, [name]) | ||
| 52 | os.mkdir(name, mode=0o700, dir_fd=destination) | ||
| 53 | dst = directory(stack, destination, [name]) | ||
| 54 | clone_tree(src, dst) | ||
| 55 | elif stat.S_ISREG(before.st_mode): | ||
| 56 | src = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=source) | ||
| 57 | stack.callback(os.close, src) | ||
| 58 | if not stat.S_ISREG(os.fstat(src).st_mode): | ||
| 59 | raise ValueError("Target changed during cloning. Retry with an idle donor.") | ||
| 60 | dst = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, | ||
| 61 | 0o600, dir_fd=destination) | ||
| 62 | stack.callback(os.close, dst) | ||
| 63 | # Linux FICLONE must succeed; a byte-copy fallback defeats warm workspace creation. | ||
| 64 | fcntl.ioctl(dst, 0x40049409, src) | ||
| 65 | elif stat.S_ISLNK(before.st_mode): | ||
| 66 | os.symlink(os.readlink(name, dir_fd=source), name, dir_fd=destination) | ||
| 67 | continue | ||
| 68 | else: | ||
| 69 | raise ValueError(f"Workspace contains a special file: {name}. Remove it before cloning.") | ||
| 70 | current = os.fstat(src) | ||
| 71 | if (before.st_ino, before.st_size, before.st_mtime_ns) != ( | ||
| 72 | current.st_ino, current.st_size, current.st_mtime_ns): | ||
| 73 | raise ValueError("Target changed during cloning. Retry with an idle donor.") | ||
| 74 | os.fchmod(dst, stat.S_IMODE(current.st_mode) & 0o777) | ||
| 75 | os.utime(dst, ns=(current.st_atime_ns, current.st_mtime_ns)) | ||
| 76 | if os.fstat(source).st_mtime_ns != initial.st_mtime_ns: | ||
| 77 | raise ValueError("Workspace changed during cloning. Retry with an idle donor.") | ||
| 78 | |||
| 79 | |||
| 80 | def seed(root, request): | ||
| 81 | if not isinstance(request, dict): | ||
| 82 | raise ValueError("Invalid project-store request.") | ||
| 83 | if request.get("command") == "jj": | ||
| 84 | return run_jj(root, request) | ||
| 85 | if request.get("command") == "read": | ||
| 86 | if set(request) != {"command", "cwd", "tool", "args"} or request["tool"] not in ("rg", "sha256sum"): | ||
| 87 | raise ValueError("Use rg or sha256sum for read-only storage commands.") | ||
| 88 | cwd = shared_directory(root, request["cwd"]) | ||
| 89 | args = request["args"] | ||
| 90 | if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): | ||
| 91 | raise ValueError("Invalid read-only command arguments.") | ||
| 92 | if request["tool"] == "rg": | ||
| 93 | if any(arg in ("--pre", "--hostname-bin") or arg.startswith(("--pre=", "--hostname-bin=")) for arg in args): | ||
| 94 | raise ValueError("Search preprocessors are unavailable on the storage host.") | ||
| 95 | args = ["--no-require-git", *args] | ||
| 96 | else: | ||
| 97 | has_file = False | ||
| 98 | options = True | ||
| 99 | for arg in args: | ||
| 100 | if options and arg == "--": | ||
| 101 | options = False | ||
| 102 | elif options and arg in ("-b", "--binary", "-t", "--text", "--tag", "-z", "--zero"): | ||
| 103 | continue | ||
| 104 | elif arg == "-" or (options and arg.startswith("-")): | ||
| 105 | raise ValueError("Hash named files; stdin and checksum verification are unavailable.") | ||
| 106 | else: | ||
| 107 | path = (cwd / arg).resolve(strict=True) | ||
| 108 | if not path.is_relative_to(root) or not path.is_file(): | ||
| 109 | raise ValueError("Hash files inside the project share.") | ||
| 110 | has_file = True | ||
| 111 | if not has_file: | ||
| 112 | raise ValueError("Select at least one file to hash.") | ||
| 113 | result = subprocess.run([request["tool"], *args], cwd=cwd, stdin=subprocess.DEVNULL) | ||
| 114 | return result.returncode if result.returncode >= 0 else 128 - result.returncode | ||
| 115 | if request.get("command") == "pnpm": | ||
| 116 | if (set(request) - {"command", "cwd", "engine", "args", "prepare", "node_env"} | ||
| 117 | or not isinstance(request.get("prepare", False), bool) | ||
| 118 | or request.get("node_env") not in (None, "production")): | ||
| 119 | raise ValueError("Invalid pnpm request.") | ||
| 120 | cwd = shared_directory(root, request["cwd"]) | ||
| 121 | args = request["args"] | ||
| 122 | if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): | ||
| 123 | raise ValueError("Invalid pnpm arguments.") | ||
| 124 | if not args or all(arg.startswith("--") for arg in args): | ||
| 125 | args = ["install", *args] | ||
| 126 | try: | ||
| 127 | parsed = client.pnpm_arguments(args, cwd) | ||
| 128 | except StopIteration: | ||
| 129 | raise ValueError("A dependency command option is missing its value.") from None | ||
| 130 | if parsed[1] is None: | ||
| 131 | raise ValueError("Unsupported dependency command.") | ||
| 132 | cwd, args, _ = parsed | ||
| 133 | cwd = shared_directory(root, cwd) | ||
| 134 | relative = cwd.relative_to(root) | ||
| 135 | if len(relative.parts) < 2: | ||
| 136 | return {"installed": False, "reason": "run the install from the workspace root."} | ||
| 137 | workspace = root.joinpath(*relative.parts[:2]) | ||
| 138 | return install(workspace, relative.parts[0], request["engine"], args, cwd=cwd, | ||
| 139 | prepare=request.get("prepare", False), node_env=request.get("node_env")) | ||
| 140 | if set(request) == {"project", "destination", "revision", "engine", "install"}: | ||
| 141 | return checkout(Path(root), request) | ||
| 142 | if set(request) != {"project", "source", "destination", "revision", "targets"}: | ||
| 143 | raise ValueError("Invalid project-store request.") | ||
| 144 | project, source, destination = (parts(request[k]) for k in ("project", "source", "destination")) | ||
| 145 | if (any(len(p) != 1 for p in (project, source, destination)) or source == destination | ||
| 146 | or destination[0] in ("main", "default")): | ||
| 147 | raise ValueError("Select distinct workspaces in one project.") | ||
| 148 | if not isinstance(request["revision"], str) or not request["revision"]: | ||
| 149 | raise ValueError("Select a parent revision.") | ||
| 150 | targets = [parts(p) for p in request["targets"]] | ||
| 151 | if not targets or len({tuple(p) for p in targets}) != len(targets): | ||
| 152 | raise ValueError("Select each target directory once.") | ||
| 153 | root = Path(root).absolute() | ||
| 154 | with ExitStack() as stack: | ||
| 155 | root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) | ||
| 156 | stack.callback(os.close, root_fd) | ||
| 157 | project_fd = directory(stack, root_fd, project) | ||
| 158 | src = directory(stack, project_fd, source) | ||
| 159 | directory(stack, project_fd, ["main", ".jj"]) | ||
| 160 | for path in targets: | ||
| 161 | directory(stack, src, path) | ||
| 162 | destination_path = root / project[0] / destination[0] | ||
| 163 | if os.path.lexists(destination_path): | ||
| 164 | raise ValueError("Destination exists. Choose a new workspace name.") | ||
| 165 | primary = root / project[0] / "main" | ||
| 166 | jj = ["jj", "--config", "snapshot.auto-track=" + json.dumps('~glob:"**/._*" & ~glob:"**/.DS_Store"')] | ||
| 167 | base = [*jj, "-R", str(primary)] | ||
| 168 | revision = subprocess.run( | ||
| 169 | [*base, "log", "--no-graph", "-r", request["revision"], "-T", 'commit_id ++ "\n"'], | ||
| 170 | text=True, capture_output=True, check=True).stdout.splitlines() | ||
| 171 | if len(revision) != 1: | ||
| 172 | raise ValueError("Select exactly one parent revision.") | ||
| 173 | subprocess.run([*base, "workspace", "add", "--sparse-patterns", "empty", | ||
| 174 | "--name", destination[0], "-r", revision[0], "--", str(destination_path)], | ||
| 175 | stdout=sys.stderr, check=True) | ||
| 176 | try: | ||
| 177 | dst = directory(stack, project_fd, destination) | ||
| 178 | clone_tree(src, dst, workspace=True) | ||
| 179 | local = [*jj, "-R", str(destination_path)] | ||
| 180 | subprocess.run([*local, "sparse", "set", "--add", "."], stdout=sys.stderr, check=True) | ||
| 181 | # Discard donor edits while retaining matching files and ignored build outputs. | ||
| 182 | subprocess.run([*local, "restore", "--from", revision[0]], stdout=sys.stderr, check=True) | ||
| 183 | except BaseException: | ||
| 184 | subprocess.run([*base, "workspace", "forget", destination[0]], | ||
| 185 | stdout=sys.stderr, check=True) | ||
| 186 | shutil.rmtree(destination_path) | ||
| 187 | raise | ||
| 188 | return {"workspace": destination[0]} | ||
| 189 | |||
| 190 | |||
| 191 | def dependency_inputs(workspace, revision=None): | ||
| 192 | base = ["jj", "--ignore-working-copy", "-R", str(workspace), "file"] | ||
| 193 | revision_args = ["-r", revision] if revision else [] | ||
| 194 | names = subprocess.check_output([*base, "list", *revision_args], cwd=workspace, text=True).splitlines() | ||
| 195 | if revision is None: | ||
| 196 | names = set(names) | {path.name for path in workspace.iterdir() if path.is_file()} | ||
| 197 | result = {} | ||
| 198 | for name in names: | ||
| 199 | path = Path(name) | ||
| 200 | if (path.name in ("package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc") | ||
| 201 | or path.suffix in (".patch", ".tgz") or "pnpmfile" in path.name): | ||
| 202 | target = workspace / path | ||
| 203 | if not revision and target.resolve() != target.absolute(): | ||
| 204 | raise ValueError("Dependency input follows a symlink.") | ||
| 205 | if not revision and not target.exists(): | ||
| 206 | continue | ||
| 207 | content = (subprocess.check_output([*base, "show", json.dumps(name), *revision_args], cwd=workspace) | ||
| 208 | if revision else target.read_bytes()) | ||
| 209 | result[name] = hashlib.sha256(content).hexdigest() | ||
| 210 | return result | ||
| 211 | |||
| 212 | |||
| 213 | def checkout(root, request): | ||
| 214 | project, name = (parts(request[k]) for k in ("project", "destination")) | ||
| 215 | if len(project) != 1 or len(name) != 1 or name[0] in ("main", "default"): | ||
| 216 | raise ValueError("Select a new workspace name.") | ||
| 217 | if not isinstance(request["revision"], str) or not request["revision"]: | ||
| 218 | raise ValueError("Select a parent revision.") | ||
| 219 | with ExitStack() as stack: | ||
| 220 | fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) | ||
| 221 | stack.callback(os.close, fd) | ||
| 222 | directory(stack, fd, [*project, "main", ".jj"]) | ||
| 223 | workspace = root / project[0] / name[0] | ||
| 224 | if os.path.lexists(workspace): | ||
| 225 | raise ValueError("Destination exists. Choose a new workspace name.") | ||
| 226 | subprocess.run(["jj", "-R", str(root / project[0] / "main"), "workspace", "add", | ||
| 227 | "--name", name[0], "-r", request["revision"], "--", str(workspace)], | ||
| 228 | stdout=sys.stderr, check=True) | ||
| 229 | manifest = workspace / "package.json" | ||
| 230 | package = json.loads(manifest.read_text()) if manifest.is_file() else {} | ||
| 231 | result = {"workspace": name[0], "installed": False, | ||
| 232 | "pnpm": package.get("packageManager", "").startswith("pnpm@")} | ||
| 233 | if request["install"] is True: | ||
| 234 | result["installed"] = install(workspace, project[0], request["engine"], [])["installed"] | ||
| 235 | return result | ||
| 236 | |||
| 237 | |||
| 238 | def install(workspace, project, engine, args, *, cwd=None, prepare=False, node_env=None): | ||
| 239 | if not args: | ||
| 240 | args = ["install"] | ||
| 241 | cwd = cwd or workspace | ||
| 242 | environment = dict(os.environ, NODE_ENV=node_env) if node_env else None | ||
| 243 | manifest = workspace / "package.json" | ||
| 244 | package = json.loads(manifest.read_text()) if manifest.is_file() else {} | ||
| 245 | manager = re.fullmatch(r"pnpm@(\d+)\.(\d+)\.(\d+)(?:\+.*)?", package.get("packageManager", "")) | ||
| 246 | if not manager: | ||
| 247 | return {"installed": False, "reason": "the workspace does not pin a supported pnpm version."} | ||
| 248 | version = ".".join(manager.groups()) | ||
| 249 | major = manager.group(1) | ||
| 250 | executable = Path(__file__).parent / ("pnpm" + major) / { | ||
| 251 | "10": "pnpm.cjs", "11": "pnpm.mjs", "12": "pnpm"}.get(major, "unsupported") | ||
| 252 | if not executable.is_file(): | ||
| 253 | return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."} | ||
| 254 | pnpm = (["node"] if major in ("10", "11") else []) + [str(executable)] | ||
| 255 | if subprocess.check_output([*pnpm, "--version"], text=True, env=environment).strip() != version: | ||
| 256 | return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."} | ||
| 257 | target = re.fullmatch(r"darwin;arm64;node(\d+(?:\.\d+\.\d+)?)", engine) | ||
| 258 | if not target: | ||
| 259 | return {"installed": False, "reason": "the storage install targets Darwin ARM64."} | ||
| 260 | node_version = target.group(1) | ||
| 261 | profiles = Path(__file__).with_name("profiles.json") | ||
| 262 | profile = json.loads(profiles.read_text()).get(project) if profiles.exists() else None | ||
| 263 | guest_store = ("/Volumes/Code/cache/pnpm12" if major == "12" else | ||
| 264 | "/Users/admin/Library/Caches/project/" + project if major == "11" else | ||
| 265 | "/Volumes/Code/cache/project-pnpm/" + project) | ||
| 266 | if profile and Path(profile["pnpm"]).parent.name == "pnpm" + major: | ||
| 267 | guest_store = str(Path(profile["guest_store"]).parent) | ||
| 268 | workspace_config = workspace / "pnpm-workspace.yaml" | ||
| 269 | config_text = workspace_config.read_text().lower() if workspace_config.is_file() else "" | ||
| 270 | npmrc = workspace / ".npmrc" | ||
| 271 | if npmrc.is_file(): | ||
| 272 | config_text += npmrc.read_text().lower() | ||
| 273 | # Knife's reviewed hook behaves identically on Linux and Darwin; edits require review. | ||
| 274 | if ("pnpmfile" in config_text or "configdependencies" in config_text or any( | ||
| 275 | path.name != ".pnpmfile.cjs" or hashlib.sha256(path.read_bytes()).hexdigest() != | ||
| 276 | "6334e7e025aa23897f0b39365407691b3b65f64bf64fe167b7f1b0d6627b2eed" | ||
| 277 | for path in workspace.glob(".pnpmfile.*"))): | ||
| 278 | return {"installed": False, "store": guest_store, "reason": "workspace pnpmfile hooks require Mac execution."} | ||
| 279 | if prepare: | ||
| 280 | verify = subprocess.check_output([*pnpm, "config", "get", "verify-deps-before-run"], | ||
| 281 | cwd=cwd, text=True, env=environment).strip() | ||
| 282 | if verify == "undefined": | ||
| 283 | verify = "false" if major == "10" else "install" | ||
| 284 | if verify == "false": | ||
| 285 | return {"ready": True} | ||
| 286 | if verify != "install": | ||
| 287 | return {"installed": False, "reason": "the workspace selects its own dependency verification policy."} | ||
| 288 | cached = (not node_env and profile and Path(profile["pnpm"]).parent.name == "pnpm" + major | ||
| 289 | and "darwin;arm64;node" + node_version.split(".")[0] == profile["engine"] | ||
| 290 | and args[0] in ("install", "i") | ||
| 291 | and set(args[1:]) <= {"--frozen-lockfile", "--offline", "--prefer-offline", "--force", "--ignore-scripts"} | ||
| 292 | and dependency_inputs(workspace) == profile["inputs"]) | ||
| 293 | if cached: | ||
| 294 | packages = json.loads(subprocess.check_output( | ||
| 295 | [*pnpm, "list", "--recursive", "--depth=-1", "--json"], | ||
| 296 | cwd=workspace, text=True, env=environment)) | ||
| 297 | cached = all(str((Path(package["path"]) / "package.json").relative_to(workspace)) in profile["inputs"] | ||
| 298 | for package in packages) | ||
| 299 | Path(guest_store).mkdir(parents=True, exist_ok=True) | ||
| 300 | common = ["--config.store-dir=" + guest_store, | ||
| 301 | "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"), | ||
| 302 | "--config.enable-global-virtual-store=false", | ||
| 303 | "--config.prefer-symlinked-executables=true", | ||
| 304 | "--reporter=append-only"] | ||
| 305 | if "." in node_version: | ||
| 306 | common.append("--config.node-version=" + node_version) | ||
| 307 | if major == "12": | ||
| 308 | common.extend(('--config.supported-architectures=["darwin-arm64"]', | ||
| 309 | "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false", | ||
| 310 | "--config.store-index-format=files", "--config.virtual-store-type=local")) | ||
| 311 | else: | ||
| 312 | common.extend(("--config.side-effects-cache-read=false", "--config.cpu=arm64", "--config.os=darwin")) | ||
| 313 | modules = workspace / "node_modules/.modules.yaml" | ||
| 314 | if prepare and modules.is_file(): | ||
| 315 | check = subprocess.run([*pnpm, *common, "--config.verify-deps-before-run=error", "exec", "true"], | ||
| 316 | cwd=cwd, text=True, capture_output=True, env=environment) | ||
| 317 | if check.returncode == 0: | ||
| 318 | return {"ready": True} | ||
| 319 | if "VERIFY_DEPS_BEFORE_RUN" not in check.stdout + check.stderr: | ||
| 320 | sys.stderr.write(check.stdout + check.stderr) | ||
| 321 | raise subprocess.CalledProcessError(check.returncode, pnpm) | ||
| 322 | included = json.loads(modules.read_text())["included"] | ||
| 323 | args = ["install"] | ||
| 324 | if included["dependencies"] and not included["devDependencies"]: | ||
| 325 | args.append("--prod") | ||
| 326 | elif included["devDependencies"] and not included["dependencies"]: | ||
| 327 | args.append("--dev") | ||
| 328 | if not included["optionalDependencies"]: | ||
| 329 | args.append("--no-optional") | ||
| 330 | with ExitStack() as stack: | ||
| 331 | lock = os.open(guest_store + "/.project-install.lock", os.O_CREAT | os.O_RDWR, 0o600) | ||
| 332 | stack.callback(os.close, lock) | ||
| 333 | try: | ||
| 334 | fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | ||
| 335 | except BlockingIOError: | ||
| 336 | print("pnpm: Waiting for another install using this package store…", file=sys.stderr, flush=True) | ||
| 337 | fcntl.flock(lock, fcntl.LOCK_EX) | ||
| 338 | subprocess.run([*pnpm, *(arg for arg in args if arg != "--ignore-scripts"), | ||
| 339 | "--config.ignore-scripts=true", "--config.package-import-method=clone", *common], cwd=cwd, | ||
| 340 | stdout=sys.stderr, check=True, env=environment) | ||
| 341 | if cached and "--ignore-scripts" not in args: | ||
| 342 | subprocess.run(["node", profile["restore"], "-r", "rebuild", "--pending", *common], | ||
| 343 | cwd=workspace, stdout=sys.stderr, check=True, env=environment) | ||
| 344 | result = {"installed": True, "workspace": str(workspace)} | ||
| 345 | if "--lockfile-only" not in args: | ||
| 346 | result["modules_hash"] = hashlib.sha256(modules.read_bytes()).hexdigest() | ||
| 347 | return result | ||
| 348 | |||
| 349 | |||
| 350 | def shared_directory(root, value): | ||
| 351 | path = Path(value).resolve(strict=True) | ||
| 352 | if not path.is_relative_to(root) or not path.is_dir(): | ||
| 353 | raise ValueError("Path is outside the project share.") | ||
| 354 | return path | ||
| 355 | |||
| 356 | |||
| 357 | def run_jj(root, request): | ||
| 358 | if set(request) != {"command", "cwd", "args", "config", "version", "ignore"}: | ||
| 359 | raise ValueError("Invalid jj request.") | ||
| 360 | cwd = shared_directory(root, request["cwd"]) | ||
| 361 | args = request["args"] | ||
| 362 | if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args): | ||
| 363 | raise ValueError("Invalid jj arguments.") | ||
| 364 | words, options, _ = client.jj_arguments(args) | ||
| 365 | paths = client.jj_paths(words, options) | ||
| 366 | paths.extend(value for name, value in options if name in ("-R", "--repository", "--config-file")) | ||
| 367 | for value in paths: | ||
| 368 | if not (cwd / value).resolve().is_relative_to(root): | ||
| 369 | raise ValueError("Path is outside the project share.") | ||
| 370 | version = subprocess.check_output(["jj-real", "--version"], text=True).strip() | ||
| 371 | if version != request["version"]: | ||
| 372 | raise ValueError("Guest and storage Jujutsu versions differ. Update the project tools together.") | ||
| 373 | with tempfile.TemporaryDirectory(prefix="project-jj-") as temporary: | ||
| 374 | config = Path(temporary) / "jj.toml" | ||
| 375 | config.write_text(request["config"]) | ||
| 376 | ignore = Path(temporary) / "git/ignore" | ||
| 377 | ignore.parent.mkdir() | ||
| 378 | ignore.write_text(request["ignore"]) | ||
| 379 | # The resolved guest config carries its identity, aliases, and environment overrides. | ||
| 380 | # Global flags must precede `--`, whose remaining arguments are paths or messages. | ||
| 381 | result = subprocess.run(["jj", "--config-file", str(config), "--no-pager", *args], cwd=cwd, | ||
| 382 | env=dict(os.environ, JJ_CONFIG=str(config), XDG_CONFIG_HOME=temporary), | ||
| 383 | stdin=subprocess.DEVNULL) | ||
| 384 | return result.returncode if result.returncode >= 0 else 128 - result.returncode | ||
| 385 | |||
| 386 | |||
| 387 | def main(): | ||
| 388 | parser = argparse.ArgumentParser(description=__doc__) | ||
| 389 | parser.add_argument("root", help="Fixed host-side project directory") | ||
| 390 | args = parser.parse_args() | ||
| 391 | if os.geteuid() == 0: | ||
| 392 | parser.error("Run project-store as the workspace owner, without root privileges.") | ||
| 393 | request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1) | ||
| 394 | if len(request) > 4 * 1024 * 1024: | ||
| 395 | parser.error("Project request exceeds 4 MiB.") | ||
| 396 | try: | ||
| 397 | result = seed(Path(args.root).resolve(), json.loads(request)) | ||
| 398 | if isinstance(result, int): | ||
| 399 | return result | ||
| 400 | print(json.dumps(result)) | ||
| 401 | except subprocess.CalledProcessError as error: | ||
| 402 | return error.returncode if error.returncode >= 0 else 128 - error.returncode | ||
| 403 | except (OSError, ValueError, TypeError, KeyError) as error: | ||
| 404 | print(f"project-store: {error}", file=sys.stderr) | ||
| 405 | return 1 | ||
| 406 | return 0 | ||
| 407 | |||
| 408 | |||
| 409 | if __name__ == "__main__": | ||
| 410 | sys.exit(main()) | ||
tools/project-tools/client.py created+201| ... | @@ -0,0 +1,201 @@ | ||
| 1 | """Guest transport and lifecycle hooks for the shared project store.""" | ||
| 2 | |||
| 3 | import errno | ||
| 4 | import hashlib | ||
| 5 | import json | ||
| 6 | import os | ||
| 7 | from pathlib import Path | ||
| 8 | import re | ||
| 9 | import subprocess | ||
| 10 | import sys | ||
| 11 | import time | ||
| 12 | |||
| 13 | |||
| 14 | ROOT = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).resolve() | ||
| 15 | |||
| 16 | |||
| 17 | def host(): | ||
| 18 | value = os.environ.get("PROJECT_STORE_SSH") | ||
| 19 | if value is None: | ||
| 20 | value = "project-store" if (Path.home() / ".ssh/project-store").is_file() else "" | ||
| 21 | if value and not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", value): | ||
| 22 | raise ValueError("Use an SSH alias for PROJECT_STORE_SSH.") | ||
| 23 | return value | ||
| 24 | |||
| 25 | |||
| 26 | def inside(path): | ||
| 27 | path = Path(path).resolve() | ||
| 28 | if not path.is_relative_to(ROOT): | ||
| 29 | raise ValueError(f"Path is outside the project share: {path}") | ||
| 30 | return path | ||
| 31 | |||
| 32 | |||
| 33 | def request(payload, *, capture=True): | ||
| 34 | return subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", | ||
| 35 | "--", host()], input=json.dumps(payload) + "\n", text=True, | ||
| 36 | stdout=subprocess.PIPE if capture else None) | ||
| 37 | |||
| 38 | |||
| 39 | def engine(): | ||
| 40 | return subprocess.check_output(["node", "-p", | ||
| 41 | "[process.platform,process.arch,'node'+process.versions.node].join(';')"], | ||
| 42 | text=True).strip() | ||
| 43 | |||
| 44 | |||
| 45 | def wait_workspace(destination): | ||
| 46 | deadline = time.monotonic() + 30 | ||
| 47 | while True: | ||
| 48 | try: | ||
| 49 | list(destination.parent.iterdir()) | ||
| 50 | list(destination.iterdir()) | ||
| 51 | list((destination / ".jj/working_copy").iterdir()) | ||
| 52 | return | ||
| 53 | except OSError as error: | ||
| 54 | if error.errno not in (errno.ENOENT, errno.ESTALE): | ||
| 55 | raise | ||
| 56 | if time.monotonic() >= deadline: | ||
| 57 | raise ValueError("Workspace was created but is not visible over NFS yet. Wait before opening it.") | ||
| 58 | time.sleep(0.1) | ||
| 59 | |||
| 60 | |||
| 61 | def pnpm_arguments(args, cwd): | ||
| 62 | command = None | ||
| 63 | forwarded = [] | ||
| 64 | parsed = iter(args) | ||
| 65 | for arg in parsed: | ||
| 66 | if arg in ("-C", "--dir"): | ||
| 67 | cwd = (cwd / next(parsed)).resolve() | ||
| 68 | elif arg.startswith("--dir="): | ||
| 69 | cwd = (cwd / arg.split("=", 1)[1]).resolve() | ||
| 70 | elif arg in ("--filter", "--filter-prod", "-F"): | ||
| 71 | forwarded.extend((arg, next(parsed))) | ||
| 72 | elif arg.startswith(("--filter=", "--filter-prod=")) or arg in { | ||
| 73 | "--frozen-lockfile", "--no-frozen-lockfile", "--offline", "--prefer-offline", | ||
| 74 | "--force", "--ignore-scripts", "--lockfile-only", "-r", "--recursive", | ||
| 75 | "-w", "--workspace-root", "-D", "--save-dev", "-P", "--save-prod", | ||
| 76 | "-O", "--save-optional", "--save-peer", "-E", "--save-exact", "--latest", "-L", | ||
| 77 | "--prod", "--dev", "--no-optional", "--ignore-workspace-root-check"}: | ||
| 78 | forwarded.append(arg) | ||
| 79 | elif command is None and arg in {"install", "i", "add", "update", "up", "upgrade", | ||
| 80 | "remove", "rm", "uninstall", "un", "uni"}: | ||
| 81 | command = arg | ||
| 82 | forwarded.append(arg) | ||
| 83 | elif command is not None and not arg.startswith("-"): | ||
| 84 | forwarded.append(arg) | ||
| 85 | else: | ||
| 86 | return cwd, None, arg if command is None and not arg.startswith("-") else command | ||
| 87 | return cwd, forwarded if command else None, command | ||
| 88 | |||
| 89 | |||
| 90 | def finish_install(workspace, native, *, ignore_scripts=False, modules_hash=None, cwd=None, args=()): | ||
| 91 | deadline = time.monotonic() + 30 | ||
| 92 | while True: | ||
| 93 | try: | ||
| 94 | list(workspace.iterdir()) | ||
| 95 | modules = workspace / "node_modules/.modules.yaml" | ||
| 96 | content = modules.read_bytes() | ||
| 97 | if modules_hash is None or hashlib.sha256(content).hexdigest() == modules_hash: | ||
| 98 | data = json.loads(content) | ||
| 99 | break | ||
| 100 | except OSError as error: | ||
| 101 | if error.errno not in (errno.ENOENT, errno.ESTALE): | ||
| 102 | raise | ||
| 103 | if time.monotonic() >= deadline: | ||
| 104 | raise ValueError("Installation is not visible over NFS yet. Retry after the mount catches up.") | ||
| 105 | time.sleep(0.1) | ||
| 106 | if ignore_scripts or not data.get("pendingBuilds"): | ||
| 107 | return | ||
| 108 | store = Path(data["storeDir"]) | ||
| 109 | target = workspace | ||
| 110 | command = ["--recursive", "--include-workspace-root", "rebuild", "--pending"] | ||
| 111 | extra = [] | ||
| 112 | if data["packageManager"].startswith("pnpm@11."): | ||
| 113 | store.mkdir(parents=True, exist_ok=True) | ||
| 114 | files = store / "files" | ||
| 115 | if not os.path.lexists(files): | ||
| 116 | relative = store.relative_to(Path.home() / "Library/Caches/project") | ||
| 117 | files.symlink_to(Path("/Volumes/Code/cache/project-pnpm") / relative / "files") | ||
| 118 | elif data["packageManager"].startswith("pnpm@12."): | ||
| 119 | # pnpm 12 rebuild can recopy patched packages that the storage host already prepared. | ||
| 120 | target = cwd or workspace | ||
| 121 | command = ["install", "--frozen-lockfile"] | ||
| 122 | filtered = False | ||
| 123 | parsed = iter(args) | ||
| 124 | for arg in parsed: | ||
| 125 | if arg in ("--filter", "--filter-prod", "-F"): | ||
| 126 | command.extend((arg, next(parsed))) | ||
| 127 | filtered = True | ||
| 128 | elif arg.startswith(("--filter=", "--filter-prod=")): | ||
| 129 | command.append(arg) | ||
| 130 | filtered = True | ||
| 131 | elif arg in ("-r", "--recursive", "-w", "--workspace-root", "--offline", "--prefer-offline"): | ||
| 132 | command.append(arg) | ||
| 133 | _, _, operation = pnpm_arguments(args, target) | ||
| 134 | if operation not in (None, "install", "i") and not filtered and not any( | ||
| 135 | arg in args for arg in ("-r", "--recursive")): | ||
| 136 | command.append("--filter=./") | ||
| 137 | included = data["included"] | ||
| 138 | if included["dependencies"] and not included["devDependencies"]: | ||
| 139 | command.append("--prod") | ||
| 140 | elif included["devDependencies"] and not included["dependencies"]: | ||
| 141 | command.append("--dev") | ||
| 142 | if not included["optionalDependencies"]: | ||
| 143 | command.append("--no-optional") | ||
| 144 | extra = ["--config.virtual-store-type=local", "--config.store-index-format=files", | ||
| 145 | "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false", | ||
| 146 | "--config.prefer-symlinked-executables=true", | ||
| 147 | "--config.optimistic-repeat-install=false"] | ||
| 148 | print("pnpm: Running Mac build hooks.", file=sys.stderr) | ||
| 149 | subprocess.run([str(native), "--dir", str(target), *command, | ||
| 150 | "--store-dir=" + str(store.parent), | ||
| 151 | "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"), | ||
| 152 | "--config.enable-global-virtual-store=false", "--config.verify-deps-before-run=false", *extra], | ||
| 153 | cwd=workspace, check=True) | ||
| 154 | |||
| 155 | |||
| 156 | VALUES = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op", | ||
| 157 | "-r", "--revision", "-s", "--source", "-d", "--destination", "-b", "--branch", | ||
| 158 | "--from", "--to", "-A", "--insert-after", "-B", "--insert-before", "-T", "--template", | ||
| 159 | "-m", "--message", "--name", "--sparse-patterns", "--author", "--tool", "--editor", | ||
| 160 | "-n", "--limit", "--sort", "--add", "--remove"} | ||
| 161 | GLOBALS = {"-R", "--repository", "--config", "--config-file", "--color", "--at-operation", "--at-op"} | ||
| 162 | |||
| 163 | |||
| 164 | def jj_arguments(args): | ||
| 165 | words, options, config = [], [], [] | ||
| 166 | args = iter(args) | ||
| 167 | for arg in args: | ||
| 168 | if arg == "--": | ||
| 169 | words.extend(args) | ||
| 170 | break | ||
| 171 | name, equal, value = arg.partition("=") | ||
| 172 | if name in VALUES: | ||
| 173 | value = value if equal else next(args) | ||
| 174 | options.append((name, value)) | ||
| 175 | if name in GLOBALS: | ||
| 176 | config.extend((name, value)) | ||
| 177 | elif len(arg) > 2 and arg[:2] in VALUES and not arg.startswith("--"): | ||
| 178 | name, value = arg[:2], arg[2:].removeprefix("=") | ||
| 179 | options.append((name, value)) | ||
| 180 | if name in GLOBALS: | ||
| 181 | config.extend((name, value)) | ||
| 182 | elif arg.startswith("-"): | ||
| 183 | options.append((arg, None)) | ||
| 184 | else: | ||
| 185 | words.append(arg) | ||
| 186 | return words, options, config | ||
| 187 | |||
| 188 | def jj_paths(words, options): | ||
| 189 | if not words: | ||
| 190 | return [] | ||
| 191 | command = tuple(words[:2]) if words and words[0] in ("workspace", "file", "sparse") else tuple(words[:1]) | ||
| 192 | paths = [] | ||
| 193 | if command == ("workspace", "add"): | ||
| 194 | paths = words[2:] | ||
| 195 | elif command[0] in ("diff", "show", "restore"): | ||
| 196 | paths = words[1:] | ||
| 197 | elif command[0] == "file": | ||
| 198 | paths = words[2:] | ||
| 199 | elif command == ("sparse", "set"): | ||
| 200 | paths = [value for name, value in options if name in ("--add", "--remove")] | ||
| 201 | return paths | ||
tools/project-tools/finish-test.py created+124| ... | @@ -0,0 +1,124 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Exercise native completion without recopying patches or widening selection.""" | ||
| 3 | |||
| 4 | import io | ||
| 5 | import json | ||
| 6 | import os | ||
| 7 | from pathlib import Path | ||
| 8 | import subprocess | ||
| 9 | import sys | ||
| 10 | import tarfile | ||
| 11 | import tempfile | ||
| 12 | import unittest | ||
| 13 | |||
| 14 | import client | ||
| 15 | |||
| 16 | |||
| 17 | class FinishTest(unittest.TestCase): | ||
| 18 | def setUp(self): | ||
| 19 | temporary = tempfile.TemporaryDirectory(prefix="infra-finish-", dir=client.ROOT) | ||
| 20 | self.addCleanup(temporary.cleanup) | ||
| 21 | self.workspace = Path(temporary.name) / "main" | ||
| 22 | self.workspace.mkdir() | ||
| 23 | self.native = Path.home() / ".local/bin/pnpm-real" | ||
| 24 | allow = {} | ||
| 25 | for name in ("allowed", "denied", "patched", "dev", "optional"): | ||
| 26 | package = dict(name="native-finish-" + name, version="1.0.0") | ||
| 27 | if name != "patched": | ||
| 28 | package["scripts"] = {"postinstall": | ||
| 29 | 'node -e "require(\'fs\').writeFileSync(\'built\',process.platform)"'} | ||
| 30 | with tarfile.open(self.workspace / (name + ".tgz"), "w:gz") as archive: | ||
| 31 | for filename, content in (("package.json", json.dumps(package).encode()), | ||
| 32 | ("index.js", b"module.exports = 'before';\n")): | ||
| 33 | entry = tarfile.TarInfo("package/" + filename) | ||
| 34 | entry.size = len(content) | ||
| 35 | archive.addfile(entry, io.BytesIO(content)) | ||
| 36 | allow[f"native-finish-{name}@file:{name}.tgz"] = name != "denied" | ||
| 37 | (self.workspace / "patched.patch").write_text( | ||
| 38 | "diff --git a/index.js b/index.js\n--- a/index.js\n+++ b/index.js\n@@ -1 +1 @@\n" | ||
| 39 | "-module.exports = 'before';\n+module.exports = 'after';\n") | ||
| 40 | (self.workspace / "pnpm-workspace.yaml").write_text(json.dumps(dict( | ||
| 41 | packages=["child", "other"], allowBuilds=allow, | ||
| 42 | patchedDependencies={"native-finish-patched@1.0.0": "patched.patch"}))) | ||
| 43 | root = dict(name="finish-root", private=True, packageManager="pnpm@12.14.0") | ||
| 44 | for directory, package in ((self.workspace, root), | ||
| 45 | (self.workspace / "child", dict(name="finish-child", private=True)), | ||
| 46 | (self.workspace / "other", dict(name="finish-other", private=True))): | ||
| 47 | directory.mkdir(exist_ok=True) | ||
| 48 | package["scripts"] = {"prepare": | ||
| 49 | 'node -e "require(\'fs\').writeFileSync(\'prepared\',process.platform)"'} | ||
| 50 | if directory.name == "child": | ||
| 51 | package["dependencies"] = {"native-finish-" + name: "file:../" + name + ".tgz" | ||
| 52 | for name in ("allowed", "denied", "patched")} | ||
| 53 | package["devDependencies"] = {"native-finish-dev": "file:../dev.tgz"} | ||
| 54 | package["optionalDependencies"] = {"native-finish-optional": "file:../optional.tgz"} | ||
| 55 | (directory / "package.json").write_text(json.dumps(package)) | ||
| 56 | |||
| 57 | def finish(self, args, *, nested=False): | ||
| 58 | cwd = self.workspace / "child" if nested else self.workspace | ||
| 59 | response = client.request(dict(command="pnpm", cwd=str(cwd), engine=client.engine(), | ||
| 60 | args=[*args, "--ignore-scripts"])) | ||
| 61 | self.assertEqual(response.returncode, 0) | ||
| 62 | data = json.loads(response.stdout) | ||
| 63 | self.assertIs(data.get("installed"), True, data) | ||
| 64 | client.finish_install(self.workspace, None, ignore_scripts=True, modules_hash=data["modules_hash"]) | ||
| 65 | patched = self.workspace / "child/node_modules/native-finish-patched/index.js" | ||
| 66 | before = patched.stat() if patched.exists() else None | ||
| 67 | client.finish_install(self.workspace, self.native, modules_hash=data["modules_hash"], cwd=cwd, args=args) | ||
| 68 | if before: | ||
| 69 | self.assertEqual(patched.read_text(), "module.exports = 'after';\n") | ||
| 70 | self.assertEqual((patched.stat().st_ino, patched.stat().st_mtime_ns), | ||
| 71 | (before.st_ino, before.st_mtime_ns)) | ||
| 72 | self.assertFalse((self.workspace / "child/node_modules/native-finish-denied/built").exists()) | ||
| 73 | |||
| 74 | def test_pending_dependency_and_workspace_hooks_preserve_patched_files(self): | ||
| 75 | self.finish(["install"]) | ||
| 76 | for name in ("allowed", "dev", "optional"): | ||
| 77 | self.assertEqual((self.workspace / f"child/node_modules/native-finish-{name}/built").read_text(), "darwin") | ||
| 78 | for directory in (self.workspace, self.workspace / "child", self.workspace / "other"): | ||
| 79 | self.assertEqual((directory / "prepared").read_text(), "darwin") | ||
| 80 | (directory / "prepared").unlink() | ||
| 81 | self.finish(["install", "--frozen-lockfile"]) | ||
| 82 | for directory in (self.workspace, self.workspace / "child", self.workspace / "other"): | ||
| 83 | self.assertEqual((directory / "prepared").read_text(), "darwin") | ||
| 84 | |||
| 85 | def test_filtered_production_install_keeps_other_projects_and_excluded_groups_untouched(self): | ||
| 86 | self.finish(["--filter", "finish-child", "install", "--prod", "--no-optional"]) | ||
| 87 | self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin") | ||
| 88 | for path in ("prepared", "child/prepared", "other/prepared", "child/node_modules/native-finish-dev", | ||
| 89 | "child/node_modules/native-finish-optional"): | ||
| 90 | self.assertFalse((self.workspace / path).exists(), path) | ||
| 91 | |||
| 92 | def test_nested_add_only_finishes_the_selected_project(self): | ||
| 93 | self.finish(["add", "native-finish-allowed@file:../allowed.tgz"], nested=True) | ||
| 94 | self.assertEqual((self.workspace / "child/node_modules/native-finish-allowed/built").read_text(), "darwin") | ||
| 95 | self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin") | ||
| 96 | self.assertFalse((self.workspace / "other/prepared").exists()) | ||
| 97 | |||
| 98 | def test_development_only_install_does_not_add_production_dependencies(self): | ||
| 99 | self.finish(["install", "--dev", "--no-optional"]) | ||
| 100 | self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin") | ||
| 101 | self.assertEqual((self.workspace / "child/prepared").read_text(), "darwin") | ||
| 102 | self.assertFalse((self.workspace / "child/node_modules/native-finish-allowed").exists()) | ||
| 103 | |||
| 104 | def test_inherited_agent_environment_keeps_native_pnpm_semantics(self): | ||
| 105 | manifest = self.workspace / "package.json" | ||
| 106 | package = json.loads(manifest.read_text()) | ||
| 107 | package["scripts"]["prepare"] += ( | ||
| 108 | ' && node -e "require(\'fs\').writeFileSync(\'mode\',process.env.NODE_ENV)"') | ||
| 109 | manifest.write_text(json.dumps(package)) | ||
| 110 | result = subprocess.run([str(Path(__file__).with_name("pnpm")), "install"], cwd=self.workspace, | ||
| 111 | env=dict(os.environ, NODE_ENV="production", NODE_OPTIONS="--max-old-space-size=8192", | ||
| 112 | npm_config_user_agent="pnpm/12.14.0 node/v26.12.0 darwin arm64"), | ||
| 113 | text=True, capture_output=True, timeout=120) | ||
| 114 | self.assertEqual(result.returncode, 0, result.stdout + result.stderr) | ||
| 115 | self.assertIn("pnpm: Running Mac build hooks.", result.stderr) | ||
| 116 | self.assertNotIn("Using Mac installation", result.stderr) | ||
| 117 | self.assertEqual((self.workspace / "mode").read_text(), "production") | ||
| 118 | self.assertEqual((self.workspace / "child/node_modules/native-finish-dev/built").read_text(), "darwin") | ||
| 119 | |||
| 120 | |||
| 121 | if __name__ == "__main__": | ||
| 122 | if sys.platform != "darwin" or not client.ROOT.is_dir(): | ||
| 123 | sys.exit("Run this test in the development VM with Code mounted.") | ||
| 124 | unittest.main() | ||
tools/project-tools/install.py created+81| ... | @@ -0,0 +1,81 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Stage or activate project command wrappers in the development VM.""" | ||
| 3 | |||
| 4 | import argparse | ||
| 5 | import os | ||
| 6 | from pathlib import Path | ||
| 7 | import shutil | ||
| 8 | import sys | ||
| 9 | import time | ||
| 10 | |||
| 11 | |||
| 12 | def main(): | ||
| 13 | parser = argparse.ArgumentParser(description=__doc__) | ||
| 14 | parser.add_argument("action", choices=("stage", "activate")) | ||
| 15 | parser.add_argument("release") | ||
| 16 | args = parser.parse_args() | ||
| 17 | if sys.platform != "darwin" or not os.path.ismount("/Volumes/Code") or os.geteuid() == 0: | ||
| 18 | parser.error("Run this installer as the development VM user with Code mounted.") | ||
| 19 | if not args.release or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-." for c in args.release): | ||
| 20 | parser.error("Use a release name containing lowercase letters, numbers, dots, or hyphens.") | ||
| 21 | local = Path.home() / ".local" | ||
| 22 | base = local / "share/project-tools" | ||
| 23 | release = base / "releases" / args.release | ||
| 24 | names = ("client.py", "pnpm", "jj", "project") | ||
| 25 | if args.action == "stage": | ||
| 26 | release.mkdir(parents=True) | ||
| 27 | for name in names: | ||
| 28 | shutil.copy2(Path(__file__).with_name(name), release / name) | ||
| 29 | print(release) | ||
| 30 | return | ||
| 31 | if any(not (release / name).is_file() for name in names): | ||
| 32 | parser.error("Stage this release before activating it.") | ||
| 33 | if not (local / "bin/jj-real").is_file(): | ||
| 34 | parser.error("Install the native Jujutsu executable as ~/.local/bin/jj-real first.") | ||
| 35 | native_pnpm = local / "bin/pnpm-real" | ||
| 36 | if not native_pnpm.exists(): | ||
| 37 | original = (local / "bin/pnpm").resolve(strict=True) | ||
| 38 | with original.open("rb") as binary: | ||
| 39 | magic = binary.read(4) | ||
| 40 | if magic != b"\xcf\xfa\xed\xfe": | ||
| 41 | parser.error("Preserve the native pnpm executable as ~/.local/bin/pnpm-real first.") | ||
| 42 | native_pnpm.symlink_to(original) | ||
| 43 | overlay = local / "share/agents/vm" | ||
| 44 | current = base / "current" | ||
| 45 | if not current.is_symlink(): | ||
| 46 | previous = base / "releases" / ("before-recovery-" + time.strftime("%Y%m%d%H%M%S")) | ||
| 47 | previous.mkdir() | ||
| 48 | for name in ("jj", "project"): | ||
| 49 | shutil.copy2(overlay / name, previous / name) | ||
| 50 | (previous / "pnpm").symlink_to(native_pnpm.resolve()) | ||
| 51 | shutil.copy2(release / "client.py", previous / "client.py") | ||
| 52 | (base / "previous").symlink_to(previous.relative_to(base)) | ||
| 53 | elif current.resolve() != release.resolve(): | ||
| 54 | previous = base / ".previous-next" | ||
| 55 | previous.symlink_to(os.readlink(current)) | ||
| 56 | previous.replace(base / "previous") | ||
| 57 | link = base / ".current-next" | ||
| 58 | link.symlink_to(release.relative_to(base)) | ||
| 59 | link.replace(current) | ||
| 60 | for name in names: | ||
| 61 | link = overlay / ("." + name + "-next") | ||
| 62 | link.symlink_to(os.path.relpath(current / name, overlay)) | ||
| 63 | link.replace(overlay / name) | ||
| 64 | link = local / "bin/.pnpm-next" | ||
| 65 | link.symlink_to("../share/agents/vm/pnpm") | ||
| 66 | link.replace(local / "bin/pnpm") | ||
| 67 | for directory in (local / "share/vite-plus/package_manager/pnpm").glob("*/pnpm/bin"): | ||
| 68 | binary = directory / "pnpm" | ||
| 69 | if not binary.is_file(): | ||
| 70 | continue | ||
| 71 | native = directory / "pnpm-real" | ||
| 72 | if not native.exists(): | ||
| 73 | os.link(binary, native) | ||
| 74 | link = directory / ".pnpm-next" | ||
| 75 | link.symlink_to(os.path.relpath(overlay / "pnpm", directory)) | ||
| 76 | link.replace(binary) | ||
| 77 | print(release) | ||
| 78 | |||
| 79 | |||
| 80 | if __name__ == "__main__": | ||
| 81 | main() | ||
tools/project-tools/jj created+136| ... | @@ -0,0 +1,136 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Serialize Jujutsu commands across workspaces and NFS clients.""" | ||
| 3 | |||
| 4 | import errno | ||
| 5 | import fcntl | ||
| 6 | import json | ||
| 7 | import os | ||
| 8 | from pathlib import Path | ||
| 9 | import subprocess | ||
| 10 | import sys | ||
| 11 | |||
| 12 | sys.path.insert(0, str(Path(__file__).resolve().parent)) | ||
| 13 | import client | ||
| 14 | |||
| 15 | |||
| 16 | def remote(executable, args, path): | ||
| 17 | if not client.host(): | ||
| 18 | return False | ||
| 19 | words, options, config_args = client.jj_arguments(args) | ||
| 20 | shared = path.is_relative_to(client.ROOT) | ||
| 21 | if Path.cwd().is_relative_to(client.ROOT) or shared: | ||
| 22 | for name, value in options: | ||
| 23 | if name in ("-R", "--repository", "--config-file"): | ||
| 24 | client.inside(value) | ||
| 25 | if not shared: | ||
| 26 | return False | ||
| 27 | client.inside(path) | ||
| 28 | if not words or words[0] in ("config", "git", "util", "help"): | ||
| 29 | return False | ||
| 30 | command = tuple(words[:2]) if words[0] in ("workspace", "file", "bookmark", "op", "sparse") else (words[0],) | ||
| 31 | safe = {("rebase",), ("status",), ("st",), ("log",), ("diff",), ("show",), ("edit",), | ||
| 32 | ("new",), ("abandon",), ("restore",), ("describe",), ("desc",), ("commit",), ("squash",), | ||
| 33 | ("workspace", "add"), ("workspace", "list"), ("workspace", "root"), | ||
| 34 | ("workspace", "forget"), ("workspace", "update-stale"), | ||
| 35 | ("file", "list"), ("file", "show"), ("file", "track"), ("file", "untrack"), | ||
| 36 | ("bookmark", "list"), ("bookmark", "create"), ("bookmark", "set"), | ||
| 37 | ("bookmark", "move"), ("bookmark", "delete"), ("bookmark", "forget"), | ||
| 38 | ("bookmark", "rename"), ("op", "log"), ("sparse", "list"), ("sparse", "set")} | ||
| 39 | if command not in safe: | ||
| 40 | return False | ||
| 41 | flags = {name for name, _ in options} | ||
| 42 | if flags & {"-i", "--interactive", "--tool", "--editor", "--edit", "--stdin", "--help", "-h"}: | ||
| 43 | return False | ||
| 44 | if command[0] in ("describe", "desc", "commit", "squash") and not flags & { | ||
| 45 | "-m", "--message", "--no-edit", "--use-destination-message"}: | ||
| 46 | return False | ||
| 47 | paths = client.jj_paths(words, options) | ||
| 48 | for value in paths: | ||
| 49 | if any(c in value for c in ':"|&~()'): | ||
| 50 | return False | ||
| 51 | client.inside(value) | ||
| 52 | base = [str(executable), *config_args, "--ignore-working-copy", "--no-pager", "config", "list"] | ||
| 53 | rows = subprocess.check_output([*base, "-T", 'json(self) ++ "\\n"'], text=True) | ||
| 54 | settings = {row["name"]: row["value"] for row in map(json.loads, rows.splitlines())} | ||
| 55 | if settings.get("signing.behavior", "drop") != "drop" or settings.get("snapshot.fsmonitor", "none") != "none": | ||
| 56 | return False | ||
| 57 | if command[0] in ("diff", "show", "log") and not flags & {"--git", "--summary", "--stat", "--types", "--name-only"}: | ||
| 58 | formatter = settings.get("ui.diff-formatter", ":color-words") | ||
| 59 | if not isinstance(formatter, str) or not formatter.startswith(":"): | ||
| 60 | return False | ||
| 61 | if sys.stdout.isatty() and "--no-pager" not in flags and settings.get("ui.paginate", "auto") != "never": | ||
| 62 | return False | ||
| 63 | config = subprocess.check_output([*base, "-T", 'name ++ " = " ++ value ++ "\\n"'], text=True) | ||
| 64 | version = subprocess.check_output([str(executable), "--version"], text=True).strip() | ||
| 65 | ignore = Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))) / "git/ignore" | ||
| 66 | cwd = Path.cwd() | ||
| 67 | if not cwd.is_relative_to(client.ROOT): | ||
| 68 | if any(not Path(value).is_absolute() for value in paths): | ||
| 69 | raise ValueError("Run commands with file paths from inside the project share.") | ||
| 70 | cwd = path | ||
| 71 | forwarded = ["-R", str(path)] | ||
| 72 | original = iter(args) | ||
| 73 | for arg in original: | ||
| 74 | if arg == "--": | ||
| 75 | forwarded.extend((arg, *original)) | ||
| 76 | break | ||
| 77 | if arg in ("-R", "--repository", "--config-file", "--config"): | ||
| 78 | next(original) | ||
| 79 | elif arg.startswith(("-R", "--repository=", "--config-file=", "--config=")): | ||
| 80 | continue | ||
| 81 | else: | ||
| 82 | forwarded.append(arg) | ||
| 83 | if arg in client.VALUES: | ||
| 84 | forwarded.append(next(original)) | ||
| 85 | result = client.request(dict(command="jj", cwd=str(cwd), args=forwarded, config=config, version=version, | ||
| 86 | ignore=ignore.read_text() if ignore.is_file() else ""), capture=False) | ||
| 87 | if result.returncode == 0 and command == ("workspace", "add") and len(paths) == 1: | ||
| 88 | client.wait_workspace(Path(paths[0]).absolute()) | ||
| 89 | sys.exit(result.returncode) | ||
| 90 | |||
| 91 | |||
| 92 | def main(): | ||
| 93 | executable = Path(sys.argv[0]).absolute().with_name("jj-real") | ||
| 94 | path = Path.cwd() | ||
| 95 | _, options, _ = client.jj_arguments(sys.argv[1:]) | ||
| 96 | for name, value in options: | ||
| 97 | if name in ("-R", "--repository"): | ||
| 98 | path = Path(value) | ||
| 99 | path = path.resolve() | ||
| 100 | remote(executable, sys.argv[1:], path) | ||
| 101 | for parent in (path, *path.parents): | ||
| 102 | metadata = parent / ".jj" | ||
| 103 | if not metadata.is_dir(): | ||
| 104 | continue | ||
| 105 | repository = metadata / "repo" | ||
| 106 | if repository.is_file(): | ||
| 107 | repository = (metadata / repository.read_text()).resolve(strict=True) | ||
| 108 | # Never unlink this inode: existing waiters must keep using the same lock. | ||
| 109 | descriptor = os.open(repository / "command.lock", os.O_CREAT | os.O_RDWR, 0o600) | ||
| 110 | # macOS NFS drops lockf across exec; flock maps to the server's POSIX lock. | ||
| 111 | lock = fcntl.flock if sys.platform == "darwin" else fcntl.lockf | ||
| 112 | try: | ||
| 113 | lock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) | ||
| 114 | except OSError as error: | ||
| 115 | if error.errno not in (errno.EACCES, errno.EAGAIN): | ||
| 116 | raise | ||
| 117 | print("jj: Waiting for another command in this repository…", file=sys.stderr, flush=True) | ||
| 118 | lock(descriptor, fcntl.LOCK_EX) | ||
| 119 | # jj owns the lock even if its caller dies. | ||
| 120 | os.set_inheritable(descriptor, True) | ||
| 121 | break | ||
| 122 | os.execv(executable, ["jj", *sys.argv[1:]]) | ||
| 123 | |||
| 124 | |||
| 125 | if __name__ == "__main__": | ||
| 126 | try: | ||
| 127 | main() | ||
| 128 | except StopIteration: | ||
| 129 | os.execv(Path(sys.argv[0]).absolute().with_name("jj-real"), ["jj", *sys.argv[1:]]) | ||
| 130 | except KeyboardInterrupt: | ||
| 131 | sys.exit(130) | ||
| 132 | except subprocess.CalledProcessError as error: | ||
| 133 | sys.exit(error.returncode) | ||
| 134 | except (OSError, ValueError) as error: | ||
| 135 | print(f"jj: {error}", file=sys.stderr) | ||
| 136 | sys.exit(1) | ||
tools/project-tools/jj-test.py created+137| ... | @@ -0,0 +1,137 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Exercise command locking with disposable repositories and real jj processes.""" | ||
| 3 | |||
| 4 | import fcntl | ||
| 5 | import os | ||
| 6 | from pathlib import Path | ||
| 7 | import shutil | ||
| 8 | import signal | ||
| 9 | import subprocess | ||
| 10 | import sys | ||
| 11 | import tempfile | ||
| 12 | import time | ||
| 13 | import unittest | ||
| 14 | |||
| 15 | |||
| 16 | lock_file = fcntl.flock if sys.platform == "darwin" else fcntl.lockf | ||
| 17 | |||
| 18 | |||
| 19 | class CommandLockTest(unittest.TestCase): | ||
| 20 | def setUp(self): | ||
| 21 | temporary = tempfile.TemporaryDirectory(prefix="jj-lock-test-") | ||
| 22 | self.addCleanup(temporary.cleanup) | ||
| 23 | self.root = Path(temporary.name).resolve() | ||
| 24 | self.bin = self.root / "bin" | ||
| 25 | self.bin.mkdir() | ||
| 26 | shutil.copy2(Path(__file__).with_name("jj"), self.bin / "jj") | ||
| 27 | shutil.copy2(Path(__file__).with_name("client.py"), self.bin / "client.py") | ||
| 28 | native = Path(shutil.which("jj")) | ||
| 29 | if native.with_name("jj-real").exists(): | ||
| 30 | native = native.with_name("jj-real") | ||
| 31 | (self.bin / "jj-real").symlink_to(native) | ||
| 32 | config = self.root / "config.toml" | ||
| 33 | config.write_text('[user]\nname="Fixture"\nemail="fixture@example.invalid"\n' | ||
| 34 | '[signing]\nbehavior="drop"\n') | ||
| 35 | self.env = dict(os.environ, JJ_CONFIG=str(config), JJ_EDITOR="true", PROJECT_STORE_SSH="", | ||
| 36 | GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent")) | ||
| 37 | self.repo = self.root / "main" | ||
| 38 | self.run_jj("git", "init", "--no-colocate", str(self.repo)) | ||
| 39 | self.other = self.root / "other" | ||
| 40 | self.run_jj("git", "init", "--no-colocate", str(self.other)) | ||
| 41 | self.workspace = self.root / "agent workspace" | ||
| 42 | self.run_jj("-R", str(self.repo), "workspace", "add", str(self.workspace)) | ||
| 43 | self.lock = self.repo / ".jj/repo/command.lock" | ||
| 44 | |||
| 45 | def run_jj(self, *args, cwd=None, check=True): | ||
| 46 | return subprocess.run([str(self.bin / "jj"), *args], cwd=cwd or self.root, | ||
| 47 | env=self.env, capture_output=True, text=True, timeout=10, check=check) | ||
| 48 | |||
| 49 | def start(self, *args, cwd=None): | ||
| 50 | process = subprocess.Popen([str(self.bin / "jj"), *args], cwd=cwd or self.root, | ||
| 51 | env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | ||
| 52 | text=True, start_new_session=True) | ||
| 53 | def cleanup(): | ||
| 54 | if process.poll() is None: | ||
| 55 | os.killpg(process.pid, signal.SIGKILL) | ||
| 56 | process.communicate(timeout=5) | ||
| 57 | self.addCleanup(cleanup) | ||
| 58 | return process | ||
| 59 | |||
| 60 | def holder(self): | ||
| 61 | marker = self.root / "ready" | ||
| 62 | process = self.start("-R", str(self.repo), "util", "exec", "--", sys.executable, "-c", | ||
| 63 | "from pathlib import Path; import time; " | ||
| 64 | f"Path({str(marker)!r}).touch(); time.sleep(30)") | ||
| 65 | deadline = time.monotonic() + 10 | ||
| 66 | while not marker.exists(): | ||
| 67 | if process.poll() is not None or time.monotonic() > deadline: | ||
| 68 | self.fail("Holder did not start") | ||
| 69 | time.sleep(0.02) | ||
| 70 | with self.lock.open("r+") as lock: | ||
| 71 | with self.assertRaises(BlockingIOError): | ||
| 72 | lock_file(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | ||
| 73 | return process | ||
| 74 | |||
| 75 | def test_shared_workspace_lock_and_independent_repository(self): | ||
| 76 | holder = self.holder() | ||
| 77 | waiting = self.start("-R", str(self.workspace), "status") | ||
| 78 | time.sleep(0.2) | ||
| 79 | self.assertIsNone(waiting.poll()) | ||
| 80 | self.run_jj("-R", str(self.other), "describe", "-m", "independent") | ||
| 81 | os.killpg(holder.pid, signal.SIGKILL) | ||
| 82 | stdout, stderr = waiting.communicate(timeout=10) | ||
| 83 | self.assertEqual(waiting.returncode, 0, stderr) | ||
| 84 | self.assertIn("Waiting for another command", stderr) | ||
| 85 | self.assertNotIn("Waiting", stdout) | ||
| 86 | self.assertFalse((self.workspace / ".jj/command.lock").exists()) | ||
| 87 | |||
| 88 | def test_selectors_and_snapshotting_reads_wait(self): | ||
| 89 | subdir = self.workspace / "subdir" | ||
| 90 | subdir.mkdir() | ||
| 91 | link = self.root / "linked" | ||
| 92 | link.symlink_to(subdir) | ||
| 93 | commands = [ | ||
| 94 | (["status"], link), | ||
| 95 | (["--repository=" + str(self.workspace), "diff"], self.root), | ||
| 96 | (["-R" + str(self.workspace), "log", "--no-graph", "-r", "@"], self.root), | ||
| 97 | (["--ignore-working-copy", "-R", str(self.workspace), "describe", "-m", "guarded"], self.root), | ||
| 98 | (["--config", 'user.name="-Rignored"', "-R", str(self.workspace), "status"], self.root), | ||
| 99 | ] | ||
| 100 | with self.lock.open("r+") as lock: | ||
| 101 | lock_file(lock, fcntl.LOCK_EX) | ||
| 102 | processes = [self.start(*args, cwd=cwd) for args, cwd in commands] | ||
| 103 | time.sleep(0.3) | ||
| 104 | self.assertTrue(all(process.poll() is None for process in processes)) | ||
| 105 | for process in processes: | ||
| 106 | _, stderr = process.communicate(timeout=10) | ||
| 107 | self.assertEqual(process.returncode, 0, stderr) | ||
| 108 | |||
| 109 | def test_cancelled_waiter_and_failed_command_release(self): | ||
| 110 | with self.lock.open("r+") as lock: | ||
| 111 | lock_file(lock, fcntl.LOCK_EX) | ||
| 112 | waiting = self.start("-R", str(self.workspace), "status") | ||
| 113 | time.sleep(0.2) | ||
| 114 | waiting.send_signal(signal.SIGINT) | ||
| 115 | waiting.communicate(timeout=5) | ||
| 116 | self.assertEqual(waiting.returncode, 130) | ||
| 117 | self.assertNotEqual(self.run_jj("-R", str(self.repo), "unknown-command", check=False).returncode, 0) | ||
| 118 | self.run_jj("-R", str(self.workspace), "status") | ||
| 119 | self.assertEqual(self.run_jj("--version").returncode, 0) | ||
| 120 | |||
| 121 | def test_concurrent_mutations_have_one_operation_head(self): | ||
| 122 | processes = [self.start("-R", str(self.repo), "new", "root()", "-m", f"fixture {i}") | ||
| 123 | for i in range(12)] | ||
| 124 | for process in processes: | ||
| 125 | _, stderr = process.communicate(timeout=10) | ||
| 126 | self.assertEqual(process.returncode, 0, stderr) | ||
| 127 | self.assertNotIn("Concurrent modification", stderr) | ||
| 128 | heads = self.repo / ".jj/repo/op_heads/heads" | ||
| 129 | self.assertEqual(len(list(heads.iterdir())), 1) | ||
| 130 | descriptions = self.run_jj("-R", str(self.repo), "log", "--no-graph", "-r", "all()", | ||
| 131 | "-T", 'description ++ "\n"').stdout | ||
| 132 | for i in range(12): | ||
| 133 | self.assertIn(f"fixture {i}\n", descriptions) | ||
| 134 | |||
| 135 | |||
| 136 | if __name__ == "__main__": | ||
| 137 | unittest.main() | ||
tools/project-tools/platform-test.py created+48| ... | @@ -0,0 +1,48 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Check storage-host platform selection before any native rebuild can repair it.""" | ||
| 3 | |||
| 4 | import io | ||
| 5 | import json | ||
| 6 | from pathlib import Path | ||
| 7 | import sys | ||
| 8 | import tarfile | ||
| 9 | import tempfile | ||
| 10 | import unittest | ||
| 11 | |||
| 12 | import client | ||
| 13 | |||
| 14 | |||
| 15 | class PlatformTest(unittest.TestCase): | ||
| 16 | def test_only_darwin_arm64_optional_packages_are_imported(self): | ||
| 17 | with tempfile.TemporaryDirectory(prefix="infra-platform-", dir=client.ROOT) as temporary: | ||
| 18 | workspace = Path(temporary) / "main" | ||
| 19 | workspace.mkdir() | ||
| 20 | optional = {} | ||
| 21 | for platform, cpu in (("darwin", "arm64"), ("linux", "x64")): | ||
| 22 | name = f"@project-platform-test/{platform}-{cpu}" | ||
| 23 | package = dict(name=name, version="1.0.0", os=[platform], cpu=[cpu]) | ||
| 24 | content = json.dumps(package).encode() | ||
| 25 | archive = workspace / f"{platform}-{cpu}.tgz" | ||
| 26 | with tarfile.open(archive, "w:gz") as output: | ||
| 27 | entry = tarfile.TarInfo("package/package.json") | ||
| 28 | entry.size = len(content) | ||
| 29 | output.addfile(entry, io.BytesIO(content)) | ||
| 30 | optional[name] = "file:" + archive.name | ||
| 31 | (workspace / "package.json").write_text(json.dumps(dict( | ||
| 32 | name="platform-fixture", private=True, packageManager="pnpm@12.14.0", | ||
| 33 | optionalDependencies=optional))) | ||
| 34 | response = client.request(dict(command="pnpm", cwd=str(workspace), | ||
| 35 | engine=client.engine(), args=["install", "--ignore-scripts"])) | ||
| 36 | self.assertEqual(response.returncode, 0) | ||
| 37 | result = json.loads(response.stdout) | ||
| 38 | self.assertIs(result.get("installed"), True, result) | ||
| 39 | client.finish_install(workspace, None, ignore_scripts=True, modules_hash=result["modules_hash"]) | ||
| 40 | packages = workspace / "node_modules/@project-platform-test" | ||
| 41 | self.assertTrue((packages / "darwin-arm64/package.json").is_file()) | ||
| 42 | self.assertFalse((packages / "linux-x64").exists()) | ||
| 43 | |||
| 44 | |||
| 45 | if __name__ == "__main__": | ||
| 46 | if sys.platform != "darwin" or not client.ROOT.is_dir(): | ||
| 47 | sys.exit("Run this test in the development VM with Code mounted.") | ||
| 48 | unittest.main() | ||
tools/project-tools/pnpm created+90| ... | @@ -0,0 +1,90 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Install dependencies on the storage host; run programs and build hooks on Mac.""" | ||
| 3 | |||
| 4 | import json | ||
| 5 | import os | ||
| 6 | from pathlib import Path | ||
| 7 | import re | ||
| 8 | import subprocess | ||
| 9 | import sys | ||
| 10 | |||
| 11 | sys.path.insert(0, str(Path(__file__).resolve().parent)) | ||
| 12 | import client | ||
| 13 | |||
| 14 | |||
| 15 | def main(): | ||
| 16 | native = Path(sys.argv[0]).absolute().with_name("pnpm-real") | ||
| 17 | args = sys.argv[1:] | ||
| 18 | cwd, forwarded, command = client.pnpm_arguments(args, Path.cwd()) | ||
| 19 | prepare = command in ("run", "run-script", "exec", "test", "t", "tst", "start", "stop", "restart") | ||
| 20 | if not prepare and command and cwd.is_relative_to(client.ROOT): | ||
| 21 | for parent in (cwd, *cwd.parents): | ||
| 22 | if parent == client.ROOT: | ||
| 23 | break | ||
| 24 | manifest = parent / "package.json" | ||
| 25 | if manifest.is_file(): | ||
| 26 | prepare = command in json.loads(manifest.read_text()).get("scripts", {}) | ||
| 27 | break | ||
| 28 | if forwarded or prepare: | ||
| 29 | custom_environment = [key for key, value in os.environ.items() if value and ( | ||
| 30 | (key.lower().startswith(("npm_config_", "pnpm_config_")) | ||
| 31 | and key.lower() not in ("npm_config_prefix", "npm_config_cache", "npm_config_user_agent", | ||
| 32 | "pnpm_config_update_notifier")) | ||
| 33 | or key in ("VP_PACKAGE_MANAGER", "NODE_PATH") | ||
| 34 | or (key == "NODE_OPTIONS" and not re.fullmatch( | ||
| 35 | r"(?:\s*--max[-_](?:old|semi)[-_]space[-_]size=\d+)+\s*", value)))] | ||
| 36 | if custom_environment and cwd.is_relative_to(client.ROOT) and client.host(): | ||
| 37 | print("pnpm: Using Mac installation: custom environment (" + | ||
| 38 | ", ".join(sorted(custom_environment)) + ").", file=sys.stderr) | ||
| 39 | if not custom_environment and cwd.is_relative_to(client.ROOT) and client.host(): | ||
| 40 | client.inside(cwd) | ||
| 41 | request = dict(command="pnpm", cwd=str(cwd), engine=client.engine(), args=forwarded or []) | ||
| 42 | if os.environ.get("NODE_ENV") == "production": | ||
| 43 | request["node_env"] = "production" | ||
| 44 | if prepare: | ||
| 45 | request["prepare"] = True | ||
| 46 | else: | ||
| 47 | print("pnpm: Installing dependencies on the storage host.", file=sys.stderr, flush=True) | ||
| 48 | result = client.request(request) | ||
| 49 | if result.returncode: | ||
| 50 | return result.returncode | ||
| 51 | response = json.loads(result.stdout) | ||
| 52 | if response.get("ready") is True: | ||
| 53 | os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args]) | ||
| 54 | if response.get("installed") is True: | ||
| 55 | if prepare or "--lockfile-only" not in forwarded: | ||
| 56 | client.finish_install(Path(response.get("workspace", cwd)), native, | ||
| 57 | ignore_scripts=not prepare and "--ignore-scripts" in forwarded, | ||
| 58 | modules_hash=response.get("modules_hash"), cwd=cwd, | ||
| 59 | args=forwarded or ()) | ||
| 60 | if prepare: | ||
| 61 | os.execv(native, [str(native), "--config.verify-deps-before-run=false", *args]) | ||
| 62 | return 0 | ||
| 63 | print("pnpm: Using Mac installation: " + response["reason"], file=sys.stderr) | ||
| 64 | store = response.get("store") | ||
| 65 | modules = cwd / "node_modules/.modules.yaml" | ||
| 66 | if modules.is_file(): | ||
| 67 | try: | ||
| 68 | current = json.loads(modules.read_text())["storeDir"] | ||
| 69 | except (ValueError, KeyError): | ||
| 70 | pass | ||
| 71 | else: | ||
| 72 | if isinstance(current, str) and Path(current).is_absolute(): | ||
| 73 | store = str(Path(current).parent) | ||
| 74 | if store: | ||
| 75 | args = [*args, "--store-dir=" + store] | ||
| 76 | os.execv(native, [str(native), *args]) | ||
| 77 | |||
| 78 | |||
| 79 | if __name__ == "__main__": | ||
| 80 | try: | ||
| 81 | sys.exit(main()) | ||
| 82 | except StopIteration: | ||
| 83 | os.execv(Path(sys.argv[0]).absolute().with_name("pnpm-real"), ["pnpm", *sys.argv[1:]]) | ||
| 84 | except KeyboardInterrupt: | ||
| 85 | sys.exit(130) | ||
| 86 | except subprocess.CalledProcessError as error: | ||
| 87 | sys.exit(error.returncode) | ||
| 88 | except (OSError, ValueError, KeyError) as error: | ||
| 89 | print(f"pnpm: {error}", file=sys.stderr) | ||
| 90 | sys.exit(1) | ||
tools/project-tools/project created+184| ... | @@ -0,0 +1,184 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Manage shared projects and run read-only tools beside their ZFS storage.""" | ||
| 3 | |||
| 4 | import argparse | ||
| 5 | from contextlib import ExitStack | ||
| 6 | import fcntl | ||
| 7 | import json | ||
| 8 | import os | ||
| 9 | from pathlib import Path | ||
| 10 | import re | ||
| 11 | import subprocess | ||
| 12 | import sys | ||
| 13 | import time | ||
| 14 | |||
| 15 | sys.path.insert(0, str(Path(__file__).resolve().parent)) | ||
| 16 | import client | ||
| 17 | |||
| 18 | |||
| 19 | def component(value): | ||
| 20 | if ( | ||
| 21 | not value | ||
| 22 | or value in (".", "..") | ||
| 23 | or value != value.strip() | ||
| 24 | or any(c in value for c in "/\\") | ||
| 25 | or any(ord(c) < 32 or ord(c) == 127 for c in value) | ||
| 26 | ): | ||
| 27 | raise argparse.ArgumentTypeError("Use one folder name without slashes or control characters.") | ||
| 28 | return value | ||
| 29 | |||
| 30 | |||
| 31 | def main(): | ||
| 32 | parser = argparse.ArgumentParser( | ||
| 33 | description=__doc__, | ||
| 34 | epilog="Prints the checkout path on success; jj progress goes to stderr. " | ||
| 35 | "PROJECT_ROOT overrides the project directory, which must already exist.", | ||
| 36 | ) | ||
| 37 | commands = parser.add_subparsers(dest="command", required=True) | ||
| 38 | read = commands.add_parser("read", help="Search, list, or hash shared files on the storage host") | ||
| 39 | read.add_argument("tool", choices=("rg", "sha256sum")) | ||
| 40 | read.add_argument("args", nargs=argparse.REMAINDER) | ||
| 41 | clone = commands.add_parser("clone", help="Clone a Git repository into PROJECT/main") | ||
| 42 | clone.add_argument("source", metavar="URL") | ||
| 43 | clone.add_argument("project", type=component, metavar="PROJECT") | ||
| 44 | worktree = commands.add_parser("worktree", help="Create a workspace and install pnpm dependencies") | ||
| 45 | worktree.add_argument("project", type=component, metavar="PROJECT") | ||
| 46 | worktree.add_argument("identity", type=component, metavar="NAME") | ||
| 47 | worktree.add_argument("--from", dest="revision", default="trunk()", metavar="REV", | ||
| 48 | help="Parent revision, resolved in main (default: trunk())") | ||
| 49 | worktree.add_argument("--warm-from", type=component, metavar="WORKSPACE", | ||
| 50 | help="Clone an idle workspace and its build outputs on the ZFS host") | ||
| 51 | worktree.add_argument("--target", action="append", metavar="PATH", | ||
| 52 | help="Target directory relative to each workspace; repeat as needed (default: target)") | ||
| 53 | worktree.add_argument("--no-install", action="store_true", help="Create the workspace without installing dependencies") | ||
| 54 | args = parser.parse_args() | ||
| 55 | |||
| 56 | root = Path(os.environ.get("PROJECT_ROOT", "/Volumes/Code/project")).expanduser().absolute() | ||
| 57 | if "PROJECT_ROOT" not in os.environ and not os.path.ismount("/Volumes/Code"): | ||
| 58 | parser.error("/Volumes/Code is not mounted. Restore the mount before creating a checkout.") | ||
| 59 | if not root.is_dir(): | ||
| 60 | parser.error(f"Project directory is missing: {root}") | ||
| 61 | if args.command == "read": | ||
| 62 | cwd = client.inside(Path.cwd()) | ||
| 63 | if not client.host(): | ||
| 64 | parser.error("Storage access is unavailable. Set PROJECT_STORE_SSH to its SSH alias.") | ||
| 65 | return client.request(dict(command="read", cwd=str(cwd), tool=args.tool, args=args.args), | ||
| 66 | capture=False).returncode | ||
| 67 | project = root / args.project | ||
| 68 | if project.is_symlink(): | ||
| 69 | parser.error(f"Project directory is a symlink: {project}") | ||
| 70 | destination = project / ("main" if args.command == "clone" else args.identity) | ||
| 71 | if destination.exists() or destination.is_symlink(): | ||
| 72 | parser.error(f"Checkout already exists: {destination}. Choose another name.") | ||
| 73 | if args.command == "clone": | ||
| 74 | if project.exists(): | ||
| 75 | parser.error(f"Project already exists: {project}. Use project worktree or another name.") | ||
| 76 | command = ["jj", "git", "clone", "--no-colocate", "--", args.source, str(destination)] | ||
| 77 | else: | ||
| 78 | if args.identity in ("main", "default"): | ||
| 79 | parser.error("main and default are reserved. Choose another worktree name.") | ||
| 80 | primary = project / "main" | ||
| 81 | if primary.is_symlink() or not (primary / ".jj").is_dir(): | ||
| 82 | parser.error(f"No primary Jujutsu checkout at {primary}. Clone the project first.") | ||
| 83 | command = ["jj", "-R", str(primary), "workspace", "add", "--name", args.identity, | ||
| 84 | "-r", args.revision, "--", str(destination)] | ||
| 85 | |||
| 86 | if "PROJECT_STORE_SSH" not in os.environ and (Path.home() / ".ssh/project-store").is_file(): | ||
| 87 | os.environ["PROJECT_STORE_SSH"] = "project-store" | ||
| 88 | with ExitStack() as locks: | ||
| 89 | warm = args.command == "worktree" and args.warm_from | ||
| 90 | if args.command == "worktree" and args.target and not warm: | ||
| 91 | parser.error("--target requires --warm-from.") | ||
| 92 | if warm: | ||
| 93 | host = os.environ.get("PROJECT_STORE_SSH", "") | ||
| 94 | if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", host): | ||
| 95 | parser.error("Set PROJECT_STORE_SSH to the SSH alias for the project-store forced command.") | ||
| 96 | if os.environ.get("CARGO_TARGET_DIR") or os.environ.get("CARGO_BUILD_TARGET_DIR"): | ||
| 97 | parser.error("Unset the Cargo target override before seeding worktree-local targets.") | ||
| 98 | donor = project / args.warm_from | ||
| 99 | if donor.is_symlink() or not (donor / ".jj").is_dir(): | ||
| 100 | parser.error(f"No donor workspace at {donor}.") | ||
| 101 | targets = args.target or ["target"] | ||
| 102 | for target in targets: | ||
| 103 | path = Path(target) | ||
| 104 | if (path.is_absolute() or not path.parts or | ||
| 105 | any(p in ("..", ".jj", ".git") for p in path.parts)): | ||
| 106 | parser.error("Use target paths inside the workspace.") | ||
| 107 | source = donor / path | ||
| 108 | if not source.is_dir() or source.resolve() != source.absolute(): | ||
| 109 | parser.error(f"Target is missing or follows a symlink: {source}") | ||
| 110 | lock_paths = sorted(set(source.glob("*/.cargo-lock")) | | ||
| 111 | set(source.glob("*/*/.cargo-lock"))) | ||
| 112 | if not lock_paths: | ||
| 113 | parser.error(f"No Cargo build locks in {source}. Build the donor first.") | ||
| 114 | for lock_path in lock_paths: | ||
| 115 | if lock_path.resolve() != lock_path.absolute(): | ||
| 116 | parser.error(f"Cargo build lock follows a symlink: {lock_path}") | ||
| 117 | fd = os.open(lock_path, os.O_RDWR | os.O_NOFOLLOW) | ||
| 118 | locks.callback(os.close, fd) | ||
| 119 | try: | ||
| 120 | fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) | ||
| 121 | except BlockingIOError: | ||
| 122 | parser.error(f"Donor is building: {source}. Retry after the build finishes.") | ||
| 123 | return create(parser, args, project, destination, command, warm) | ||
| 124 | |||
| 125 | |||
| 126 | def create(parser, args, project, destination, command, warm): | ||
| 127 | memory = Path.home() / ".local/share/agents/memory" / args.project | ||
| 128 | slug = re.sub(r"[^a-zA-Z0-9_-]", "-", str(destination)) | ||
| 129 | memory_link = Path.home() / ".claude/projects" / slug / "memory" | ||
| 130 | if os.path.lexists(memory_link) and memory_link.resolve() != memory.resolve(): | ||
| 131 | parser.error(f"Another memory directory owns {memory_link}. Resolve it before creating this checkout.") | ||
| 132 | if args.command == "clone": | ||
| 133 | project.mkdir() | ||
| 134 | host = os.environ.get("PROJECT_STORE_SSH") | ||
| 135 | if warm: | ||
| 136 | request = dict(project=args.project, source=args.warm_from, destination=args.identity, | ||
| 137 | revision=args.revision, targets=args.target or ["target"]) | ||
| 138 | started = time.monotonic() | ||
| 139 | result = subprocess.run(["ssh", "-T", "-o", "BatchMode=yes", "--", host], | ||
| 140 | input=json.dumps(request), text=True, stdout=subprocess.PIPE) | ||
| 141 | if result.returncode: | ||
| 142 | return result.returncode | ||
| 143 | print(f"project: Host setup finished in {time.monotonic() - started:.1f}s.", file=sys.stderr) | ||
| 144 | try: | ||
| 145 | response = json.loads(result.stdout) | ||
| 146 | if not isinstance(response, dict) or response.get("workspace") != args.identity: | ||
| 147 | raise ValueError() | ||
| 148 | except (ValueError, TypeError): | ||
| 149 | print(f"project: Invalid storage response. Inspect {destination} before building.", file=sys.stderr) | ||
| 150 | return 1 | ||
| 151 | client.wait_workspace(destination) | ||
| 152 | else: | ||
| 153 | result = subprocess.run(command, stdout=sys.stderr) | ||
| 154 | if result.returncode: | ||
| 155 | if args.command == "clone": | ||
| 156 | try: | ||
| 157 | project.rmdir() | ||
| 158 | except OSError: | ||
| 159 | pass | ||
| 160 | return result.returncode | ||
| 161 | |||
| 162 | memory.mkdir(parents=True, exist_ok=True, mode=0o700) | ||
| 163 | if not (memory / "MEMORY.md").exists(): | ||
| 164 | (memory / "MEMORY.md").touch(mode=0o600) | ||
| 165 | memory_link.parent.mkdir(parents=True, exist_ok=True, mode=0o700) | ||
| 166 | if not os.path.lexists(memory_link): | ||
| 167 | memory_link.symlink_to(os.path.relpath(memory, memory_link.parent), target_is_directory=True) | ||
| 168 | if args.command == "worktree" and not args.no_install: | ||
| 169 | manifest = destination / "package.json" | ||
| 170 | if manifest.is_file(): | ||
| 171 | package = json.loads(manifest.read_text()) | ||
| 172 | if package.get("packageManager", "").startswith("pnpm@"): | ||
| 173 | subprocess.run(["pnpm", "install", "--frozen-lockfile"], cwd=destination, | ||
| 174 | stdout=sys.stderr, check=True) | ||
| 175 | print(destination) | ||
| 176 | return 0 | ||
| 177 | |||
| 178 | |||
| 179 | if __name__ == "__main__": | ||
| 180 | try: | ||
| 181 | sys.exit(main()) | ||
| 182 | except (OSError, ValueError, subprocess.CalledProcessError) as error: | ||
| 183 | print(f"project: {error}", file=sys.stderr) | ||
| 184 | sys.exit(1) | ||
tools/project-tools/test.py created+284| ... | @@ -0,0 +1,284 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | """Exercise routing with real Jujutsu repositories and a local SSH stand-in.""" | ||
| 3 | |||
| 4 | import hashlib | ||
| 5 | import fcntl | ||
| 6 | import json | ||
| 7 | import os | ||
| 8 | from pathlib import Path | ||
| 9 | import shutil | ||
| 10 | import subprocess | ||
| 11 | import sys | ||
| 12 | import tempfile | ||
| 13 | import threading | ||
| 14 | import time | ||
| 15 | import unittest | ||
| 16 | import client | ||
| 17 | |||
| 18 | |||
| 19 | HERE = Path(__file__).resolve().parent | ||
| 20 | |||
| 21 | |||
| 22 | class RoutingTest(unittest.TestCase): | ||
| 23 | def setUp(self): | ||
| 24 | temporary = tempfile.TemporaryDirectory(prefix="project-routing-") | ||
| 25 | self.addCleanup(temporary.cleanup) | ||
| 26 | self.root = Path(temporary.name).resolve() | ||
| 27 | self.projects = self.root / "project" | ||
| 28 | self.projects.mkdir() | ||
| 29 | self.bin = self.root / "bin" | ||
| 30 | self.bin.mkdir() | ||
| 31 | for name in ("jj", "pnpm", "project", "client.py"): | ||
| 32 | shutil.copy2(HERE / name, self.bin / name) | ||
| 33 | native = Path(shutil.which("jj")) | ||
| 34 | if native.with_name("jj-real").exists(): | ||
| 35 | native = native.with_name("jj-real") | ||
| 36 | (self.bin / "jj-real").symlink_to(native) | ||
| 37 | self.requests = self.root / "requests" | ||
| 38 | self.native = self.root / "native" | ||
| 39 | (self.bin / "pnpm-real").write_text(f'''#!{sys.executable} | ||
| 40 | import json, pathlib, sys | ||
| 41 | pathlib.Path({str(self.native)!r}).write_text(json.dumps(sys.argv[1:])) | ||
| 42 | print("native output") | ||
| 43 | sys.exit(23) | ||
| 44 | ''') | ||
| 45 | (self.bin / "pnpm-real").chmod(0o755) | ||
| 46 | (self.bin / "ssh").write_text(f'''#!{sys.executable} | ||
| 47 | import json, os, pathlib, subprocess, sys | ||
| 48 | request = json.load(sys.stdin) | ||
| 49 | with open({str(self.requests)!r}, "a") as log: log.write(json.dumps(request) + "\\n") | ||
| 50 | if request["command"] == "pnpm": | ||
| 51 | print(json.dumps({{"ready": True}} if request.get("prepare") else {{"installed": False, "reason": "fixture miss"}})) | ||
| 52 | sys.exit(0) | ||
| 53 | sys.exit(subprocess.run([{sys.executable!r}, {str(HERE.parent / "project-store.py")!r}, {str(self.projects)!r}], | ||
| 54 | input=json.dumps(request), text=True, env=dict(os.environ, PROJECT_STORE_SSH="")).returncode) | ||
| 55 | ''') | ||
| 56 | (self.bin / "ssh").chmod(0o755) | ||
| 57 | config = self.root / "jj.toml" | ||
| 58 | config.write_text('[user]\nname="Guest Identity"\nemail="guest@example.invalid"\n' | ||
| 59 | '[signing]\nbehavior="drop"\n[ui]\ndiff-formatter=":git"\n') | ||
| 60 | self.env = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ["PATH"], | ||
| 61 | JJ_CONFIG=str(config), PROJECT_ROOT=str(self.projects), PROJECT_STORE_SSH="fixture", | ||
| 62 | GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(self.root / "absent")) | ||
| 63 | for key in list(self.env): | ||
| 64 | if key.lower().startswith(("npm_config_", "pnpm_config_")) or key in ( | ||
| 65 | "NODE_ENV", "NODE_OPTIONS", "NODE_PATH", "VP_PACKAGE_MANAGER"): | ||
| 66 | self.env.pop(key) | ||
| 67 | self.main = self.projects / "Example/main" | ||
| 68 | self.main.parent.mkdir() | ||
| 69 | self.run_tool("jj-real", "git", "init", "--no-colocate", str(self.main), cwd=self.root) | ||
| 70 | (self.main / "source").write_text("base\n") | ||
| 71 | self.run_tool("jj-real", "describe", "-m", "base") | ||
| 72 | |||
| 73 | def run_tool(self, tool, *args, cwd=None, check=True): | ||
| 74 | result = subprocess.run([str(self.bin / tool), *args], cwd=cwd or self.main, | ||
| 75 | env=self.env, text=True, capture_output=True, timeout=15) | ||
| 76 | if check: | ||
| 77 | self.assertEqual(result.returncode, 0, result.stderr) | ||
| 78 | return result | ||
| 79 | |||
| 80 | def test_workspace_and_mutations_keep_paths_identity_and_dirty_content(self): | ||
| 81 | destination = self.main.parent / "agent space" | ||
| 82 | result = self.run_tool("jj", "workspace", "add", "--", str(destination)) | ||
| 83 | self.assertIn("agent space", result.stderr) | ||
| 84 | (destination / "dirty").write_text("unsaved agent work\n") | ||
| 85 | self.run_tool("jj", "new", "-m", "child", cwd=destination) | ||
| 86 | self.run_tool("jj", "rebase", "-s", "@-", "-d", "root()", cwd=destination) | ||
| 87 | self.assertEqual((destination / "dirty").read_text(), "unsaved agent work\n") | ||
| 88 | author = self.run_tool("jj", "log", "-r", "@", "--no-graph", "-T", "author.email()", cwd=destination) | ||
| 89 | self.assertEqual(author.stdout, "guest@example.invalid") | ||
| 90 | self.assertEqual(len(self.requests.read_text().splitlines()), 4) | ||
| 91 | self.assertFalse(Path((destination / ".jj/repo").read_text()).is_absolute()) | ||
| 92 | |||
| 93 | def test_paths_cannot_escape_share_even_through_symlinks(self): | ||
| 94 | outside = self.root / "outside" | ||
| 95 | outside.mkdir() | ||
| 96 | (self.main / "escape").symlink_to(outside) | ||
| 97 | for args in [("workspace", "add", str(outside / "new")), | ||
| 98 | ("workspace", "add", "../../../../escape"), | ||
| 99 | ("workspace", "add", "escape/new"), | ||
| 100 | ("-R", str(outside), "status"), | ||
| 101 | ("file", "show", str(outside / "file"))]: | ||
| 102 | with self.subTest(args=args): | ||
| 103 | result = self.run_tool("jj", *args, check=False) | ||
| 104 | self.assertNotEqual(result.returncode, 0) | ||
| 105 | self.assertIn("outside the project share", result.stderr) | ||
| 106 | self.assertFalse(self.requests.exists()) | ||
| 107 | |||
| 108 | def test_remote_errors_preserve_exit_status_without_local_retry(self): | ||
| 109 | result = self.run_tool("jj", "rebase", "-r", "missing-revision", "-d", "root()", check=False) | ||
| 110 | self.assertNotEqual(result.returncode, 0) | ||
| 111 | self.assertIn("missing-revision", result.stderr) | ||
| 112 | self.assertEqual(len(self.requests.read_text().splitlines()), 1) | ||
| 113 | |||
| 114 | def test_relative_repository_selection_from_outside_the_share(self): | ||
| 115 | destination = self.main.parent / "selected" | ||
| 116 | result = self.run_tool("jj", "-R", str(self.main.relative_to(self.root)), | ||
| 117 | "--config", 'user.email="override@example.invalid"', | ||
| 118 | "workspace", "add", "-r", "@", str(destination), cwd=self.root) | ||
| 119 | self.assertEqual(result.returncode, 0) | ||
| 120 | author = self.run_tool("jj", "log", "--no-graph", "-r", "@", "-T", "author.email()", cwd=destination) | ||
| 121 | self.assertEqual(author.stdout, "override@example.invalid") | ||
| 122 | |||
| 123 | def test_local_repositories_and_interactive_commands_stay_local(self): | ||
| 124 | local = self.root / "local" | ||
| 125 | self.run_tool("jj-real", "git", "init", "--no-colocate", str(local)) | ||
| 126 | self.run_tool("jj", "status", cwd=local) | ||
| 127 | self.env["JJ_EDITOR"] = "true" | ||
| 128 | self.run_tool("jj", "describe") | ||
| 129 | self.assertFalse(self.requests.exists()) | ||
| 130 | |||
| 131 | def test_pnpm_scripts_and_unknown_options_keep_native_argv_and_exit(self): | ||
| 132 | for args in [("run", "build"), ("exec", "node", "-p", "process.platform"), | ||
| 133 | ("build",), ("install", "--config.ignore-scripts=true"), | ||
| 134 | ("--version",), ("run", "install")]: | ||
| 135 | with self.subTest(args=args): | ||
| 136 | result = self.run_tool("pnpm", *args, check=False) | ||
| 137 | self.assertEqual(result.returncode, 23) | ||
| 138 | self.assertEqual(result.stdout, "native output\n") | ||
| 139 | prefix = ["--config.verify-deps-before-run=false"] if args[0] in ("run", "exec") else [] | ||
| 140 | self.assertEqual(json.loads(self.native.read_text()), prefix + list(args)) | ||
| 141 | requests = [json.loads(line) for line in self.requests.read_text().splitlines()] | ||
| 142 | self.assertTrue(all(request["prepare"] and request["args"] == [] for request in requests)) | ||
| 143 | |||
| 144 | def test_script_shortcut_checks_dependencies_remotely_then_runs_natively(self): | ||
| 145 | (self.main / "package.json").write_text(json.dumps({"scripts": {"build": "never run on Linux"}})) | ||
| 146 | result = self.run_tool("pnpm", "build", "--flag", check=False) | ||
| 147 | self.assertEqual(result.returncode, 23) | ||
| 148 | self.assertEqual(json.loads(self.requests.read_text())["args"], []) | ||
| 149 | self.assertEqual(json.loads(self.native.read_text()), ["--config.verify-deps-before-run=false", "build", "--flag"]) | ||
| 150 | |||
| 151 | def test_handoff_waits_for_the_servers_modules_manifest(self): | ||
| 152 | modules = self.main / "node_modules/.modules.yaml" | ||
| 153 | modules.parent.mkdir() | ||
| 154 | modules.write_text('{"pendingBuilds":[]}') | ||
| 155 | content = b'{"pendingBuilds":["new-native-package"]}' | ||
| 156 | def publish(): | ||
| 157 | time.sleep(0.15) | ||
| 158 | temporary = modules.with_suffix(".next") | ||
| 159 | temporary.write_bytes(content) | ||
| 160 | temporary.replace(modules) | ||
| 161 | publisher = threading.Thread(target=publish) | ||
| 162 | publisher.start() | ||
| 163 | self.addCleanup(publisher.join) | ||
| 164 | client.finish_install(self.main, self.bin / "pnpm-real", ignore_scripts=True, | ||
| 165 | modules_hash=hashlib.sha256(content).hexdigest()) | ||
| 166 | self.assertEqual(modules.read_bytes(), content) | ||
| 167 | |||
| 168 | def test_workspace_handoff_waits_for_metadata_visibility(self): | ||
| 169 | destination = self.main.parent / "new workspace" | ||
| 170 | def publish(): | ||
| 171 | time.sleep(0.15) | ||
| 172 | (destination / ".jj/working_copy").mkdir(parents=True) | ||
| 173 | publisher = threading.Thread(target=publish) | ||
| 174 | publisher.start() | ||
| 175 | self.addCleanup(publisher.join) | ||
| 176 | client.wait_workspace(destination) | ||
| 177 | self.assertTrue((destination / ".jj/working_copy").is_dir()) | ||
| 178 | |||
| 179 | def test_install_miss_falls_back_once_with_exact_arguments(self): | ||
| 180 | self.env.update(npm_config_prefix=str(self.root / "local"), npm_config_cache=str(self.root / "cache"), | ||
| 181 | PNPM_CONFIG_UPDATE_NOTIFIER="false") | ||
| 182 | args = ["--dir", str(self.main), "install", "--frozen-lockfile", "--offline"] | ||
| 183 | result = self.run_tool("pnpm", *args, check=False) | ||
| 184 | self.assertEqual(result.returncode, 23) | ||
| 185 | self.assertIn("fixture miss", result.stderr) | ||
| 186 | self.assertEqual(json.loads(self.native.read_text()), args) | ||
| 187 | self.assertEqual(json.loads(self.requests.read_text())["args"], ["install", "--frozen-lockfile", "--offline"]) | ||
| 188 | |||
| 189 | def test_dependency_changes_and_filters_use_storage_before_native_hooks(self): | ||
| 190 | for args in [("add", "-D", "is-number@7.0.0"), ("update", "is-number"), | ||
| 191 | ("remove", "is-number"), ("--filter", "app", "install", "--lockfile-only")]: | ||
| 192 | with self.subTest(args=args): | ||
| 193 | result = self.run_tool("pnpm", *args, check=False) | ||
| 194 | self.assertEqual(result.returncode, 23) | ||
| 195 | request = json.loads(self.requests.read_text().splitlines()[-1]) | ||
| 196 | self.assertEqual(request["args"], list(args)) | ||
| 197 | self.assertRegex(request["engine"], r";node\d+\.\d+\.\d+$") | ||
| 198 | |||
| 199 | def test_install_miss_keeps_the_existing_store(self): | ||
| 200 | modules = self.main / "node_modules/.modules.yaml" | ||
| 201 | modules.parent.mkdir() | ||
| 202 | store = self.root / "cache/v10" | ||
| 203 | modules.write_text(json.dumps({"storeDir": str(store)})) | ||
| 204 | result = self.run_tool("pnpm", "install", check=False) | ||
| 205 | self.assertEqual(result.returncode, 23) | ||
| 206 | self.assertEqual(json.loads(self.native.read_text()), ["install", "--store-dir=" + str(store.parent)]) | ||
| 207 | |||
| 208 | def test_custom_install_environment_stays_native(self): | ||
| 209 | for key, value in (("npm_config_ignore_scripts", "true"), | ||
| 210 | ("PNPM_CONFIG_STORE_DIR", str(self.root / "custom-store")), | ||
| 211 | ("VP_PACKAGE_MANAGER", "pnpm@12.15.0"), | ||
| 212 | ("NODE_OPTIONS", "--require=/private/agent-hook.cjs"), | ||
| 213 | ("NODE_PATH", "/private/agent-modules")): | ||
| 214 | with self.subTest(key=key): | ||
| 215 | self.env[key] = value | ||
| 216 | result = self.run_tool("pnpm", "install", check=False) | ||
| 217 | self.assertEqual(result.returncode, 23) | ||
| 218 | self.assertEqual(json.loads(self.native.read_text()), ["install"]) | ||
| 219 | self.assertIn("Using Mac installation: custom environment (" + key + ")", result.stderr) | ||
| 220 | self.assertNotIn(value, result.stderr) | ||
| 221 | self.env.pop(key) | ||
| 222 | self.assertFalse(self.requests.exists()) | ||
| 223 | |||
| 224 | def test_inherited_metadata_and_memory_limits_still_offload(self): | ||
| 225 | self.env["npm_config_user_agent"] = "pnpm/12.14.0 npm/? node/v26.12.0 darwin arm64" | ||
| 226 | for options in ("--max-old-space-size=8192", "--max_old_space_size=8192", | ||
| 227 | "--max-old-space-size=8192 --max-semi-space-size=32"): | ||
| 228 | with self.subTest(options=options): | ||
| 229 | self.env["NODE_OPTIONS"] = options | ||
| 230 | result = self.run_tool("pnpm", "install", "--frozen-lockfile", check=False) | ||
| 231 | self.assertEqual(result.returncode, 23) | ||
| 232 | self.assertIn("fixture miss", result.stderr) | ||
| 233 | self.assertNotIn("custom environment", result.stderr) | ||
| 234 | request = json.loads(self.requests.read_text().splitlines()[-1]) | ||
| 235 | self.assertEqual(request["args"], ["install", "--frozen-lockfile"]) | ||
| 236 | |||
| 237 | def test_production_environment_reaches_pnpm_without_changing_arguments(self): | ||
| 238 | self.env["NODE_ENV"] = "production" | ||
| 239 | for args in (("install",), ("install", "--dev"), ("add", "is-number@7.0.0"), ("exec", "node")): | ||
| 240 | with self.subTest(args=args): | ||
| 241 | result = self.run_tool("pnpm", *args, check=False) | ||
| 242 | self.assertEqual(result.returncode, 23) | ||
| 243 | request = json.loads(self.requests.read_text().splitlines()[-1]) | ||
| 244 | self.assertEqual(request["node_env"], "production") | ||
| 245 | self.assertEqual(request["args"], [] if args[0] == "exec" else list(args)) | ||
| 246 | |||
| 247 | def test_storage_reads_preserve_paths_output_and_ignore_files_without_snapshotting(self): | ||
| 248 | (self.main / "file with spaces").write_text("needle\n") | ||
| 249 | (self.main / ".gitignore").write_text("node_modules/\n") | ||
| 250 | (self.main / "node_modules").mkdir() | ||
| 251 | (self.main / "node_modules/ignored").write_text("needle\n") | ||
| 252 | with (self.main / ".jj/repo/command.lock").open("w") as lock: | ||
| 253 | fcntl.flock(lock, fcntl.LOCK_EX) | ||
| 254 | listed = self.run_tool("project", "read", "rg", "--files") | ||
| 255 | self.assertIn("file with spaces", listed.stdout.splitlines()) | ||
| 256 | self.assertNotIn("node_modules/ignored", listed.stdout) | ||
| 257 | result = self.run_tool("project", "read", "rg", "--json", "needle", "file with spaces") | ||
| 258 | matches = [json.loads(line) for line in result.stdout.splitlines()] | ||
| 259 | self.assertEqual(next(row for row in matches if row["type"] == "match") | ||
| 260 | ["data"]["path"]["text"], "file with spaces") | ||
| 261 | for pattern, status in (("not present", 1), ("[", 2)): | ||
| 262 | result = self.run_tool("project", "read", "rg", pattern, check=False) | ||
| 263 | self.assertEqual(result.returncode, status) | ||
| 264 | requests = [json.loads(line) for line in self.requests.read_text().splitlines()] | ||
| 265 | self.assertTrue(all(request["command"] == "read" for request in requests)) | ||
| 266 | |||
| 267 | def test_storage_hashes_match_content_and_reject_external_paths_and_commands(self): | ||
| 268 | content = b"fixture" * 1000 | ||
| 269 | (self.main / "archive with spaces.zip").write_bytes(content) | ||
| 270 | result = self.run_tool("project", "read", "sha256sum", "archive with spaces.zip") | ||
| 271 | self.assertEqual(result.stdout, hashlib.sha256(content).hexdigest() + " archive with spaces.zip\n") | ||
| 272 | outside = self.root / "private" | ||
| 273 | outside.write_text("not shared") | ||
| 274 | (self.main / "escape").symlink_to(outside) | ||
| 275 | for args in (("sha256sum", str(outside)), ("sha256sum", "escape"), | ||
| 276 | ("sha256sum", "-"), ("sha256sum", "--check", "archive with spaces.zip"), | ||
| 277 | ("rg", "--pre=touch marker", "needle"), ("sh", "-c", "touch marker")): | ||
| 278 | with self.subTest(args=args): | ||
| 279 | self.assertNotEqual(self.run_tool("project", "read", *args, check=False).returncode, 0) | ||
| 280 | self.assertFalse((self.main / "marker").exists()) | ||
| 281 | |||
| 282 | |||
| 283 | if __name__ == "__main__": | ||
| 284 | unittest.main() | ||