From c58359539db5139615d36266c5839026c9b7c368 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:13:12 +0000 Subject: [PATCH 1/5] feat(workspace): scope a mount to one key prefix of its collection (FIX-1762) A scoped mount lists its collection with the scope as a separator-bounded prefix, strips the scope inside the place, and keys every read, write, delete and claim back under it. Unscoped mounts are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P9UuEKLJUWaKiSrYWCJbt5 --- packages/workspace/src/projection.ts | 94 +++++++++-- packages/workspace/src/types.ts | 17 ++ packages/workspace/test/fake-collection.ts | 18 ++- packages/workspace/test/scoped-mount.spec.ts | 157 +++++++++++++++++++ 4 files changed, 270 insertions(+), 16 deletions(-) create mode 100644 packages/workspace/test/scoped-mount.spec.ts diff --git a/packages/workspace/src/projection.ts b/packages/workspace/src/projection.ts index f33bf5caec..5b37985c06 100644 --- a/packages/workspace/src/projection.ts +++ b/packages/workspace/src/projection.ts @@ -33,6 +33,7 @@ * property that made persisting it a follow-up rather than a requirement. */ import { createHash } from "node:crypto"; +import { getPatternPrefix } from "@flow-state-dev/core/types"; import type { FlushOutcome, FlushReport, Mount, Place, ProjectedEntryState } from "./types"; import { PlaceUnreadableError } from "./types"; import { isMetadataKey, normalizePath, routePath } from "./routing"; @@ -130,6 +131,10 @@ export function createProjection({ byCollection.set(mount.collectionId, mount.prefix); } + for (const mount of mounts) { + if (mount.scope !== undefined) assertScope(mount.scope); + } + const prefixes = mounts.map((m) => normalizePath(m.prefix)); /** @@ -143,6 +148,44 @@ export function createProjection({ const placePath = (mount: Mount, key: string): string => normalizePath(key === "" ? mount.prefix : `${mount.prefix}/${key}`); + /** + * The collection's own key for a key routed inside `mount`. + * + * Routing answers relative to the mount's prefix in the place; a scoped + * mount's rows live one prefix deeper in the collection. Every collection + * call and every claim goes through here, so a scoped mount cannot reach a + * key outside its scope by any path. + */ + const entryKey = (mount: Mount, key: string): string => + mount.scope === undefined ? key : `${mount.scope}/${key}`; + + /** + * The rows a mount addresses, as `[key inside the mount, ref]`. + * + * A scoped mount asks the collection for its scope and nothing wider — the + * filter is applied where the rows are read (BP-033). The trailing `/` is + * what bounds it: the collection's filter is a bare string prefix, so `a` + * alone would return `ab/…` too. Each key is checked again on the way out, + * so a store whose filter is looser than asked still cannot hand this mount + * another scope's row. + */ + async function rowsOf(mount: Mount) { + if (mount.scope === undefined) { + return (await mount.collection.list()).map( + (entry) => [stripPrefix(entry.path, normalizePath(mount.prefix)), entry] as const, + ); + } + const within = `${mount.scope}/`; + const patternPrefix = getPatternPrefix(mount.collection.pattern); + const rows = []; + for (const entry of await mount.collection.list(within)) { + const key = stripPrefix(entry.path, patternPrefix); + if (!key.startsWith(within)) continue; + rows.push([key.slice(within.length), entry] as const); + } + return rows; + } + /** What the collection holds at `key`, or `null`. */ async function theirContent(mount: Mount, key: string): Promise { const existing = await mount.collection.getOptional(key); @@ -152,12 +195,11 @@ export function createProjection({ async function hydrate(): Promise { for (const mount of mounts) { - for (const entry of await mount.collection.list()) { - // The ref's own storage path, not `state.path`. State is the - // application's — a collection written by anything other than a - // projection may carry no `path` field at all, and hydrating off it - // would silently lay down nothing. - const key = stripPrefix(entry.path, normalizePath(mount.prefix)); + // Keyed off the ref's own storage path, not `state.path`. State is the + // application's — a collection written by anything other than a + // projection may carry no `path` field at all, and hydrating off it + // would silently lay down nothing. + for (const [key, entry] of await rowsOf(mount)) { if (isMetadataKey(key)) continue; const content = await entry.readContent(); if (content === null) continue; @@ -270,16 +312,16 @@ export function createProjection({ const { mount, key } = routed; // A delete is a write. Same claim, same refusal. - if (claims.claim(claimKey(mount.collectionId, key), holder) !== holder) { + if (claims.claim(claimKey(mount.collectionId, entryKey(mount, key)), holder) !== holder) { outcomes.push({ kind: "contested", path }); continue; } - const theirs = await theirContent(mount, key); + const theirs = await theirContent(mount, entryKey(mount, key)); const theirHash = theirs === null ? undefined : hashContent(theirs); if (theirHash === base) { - await mount.collection.delete(key); + await mount.collection.delete(entryKey(mount, key)); baseline.delete(path); outcomes.push({ kind: "deleted", path }); continue; @@ -348,20 +390,21 @@ export function createProjection({ // that predates it, is granted a claim proving nothing, and overwrites // work it never saw — with both writers told they succeeded. The claim // has to cover the read-compare-write, not just the write. - if (claims.claim(claimKey(mount.collectionId, key), holder) !== holder) { + const entry = entryKey(mount, key); + if (claims.claim(claimKey(mount.collectionId, entry), holder) !== holder) { return { kind: "contested", path }; } - const theirs = await theirContent(mount, key); + const theirs = await theirContent(mount, entry); const theirHash = theirs === null ? undefined : hashContent(theirs); if (base === undefined && theirHash === undefined) { - await commit(mount, key, local, now); + await commit(mount, entry, local, now); baseline.set(path, now); return { kind: "created", path }; } if (theirHash === base) { - await commit(mount, key, local, now); + await commit(mount, entry, local, now); baseline.set(path, now); return { kind: "written", path }; } @@ -404,7 +447,12 @@ export function createProjection({ return await claiming((holder) => decide(mount, key, path, content, holder)); } - /** Write content back to the collection, keeping its state in step. */ + /** + * Write content back to the collection, keeping its state in step. + * + * `key` is the collection's own key — scope included — and so is the + * `path` it records. + */ async function commit( mount: Mount, key: string, @@ -431,3 +479,21 @@ export function createProjection({ ownedPaths: () => [...baseline.keys()], }; } + +/** + * Refuse a scope that is not a plain key prefix. + * + * The scope is joined onto every key the mount touches, so anything that is + * not a clean relative prefix is a way to address keys outside it: `..` + * walks up, an empty segment or a stray `/` spells a different key than the + * one compared against. + */ +function assertScope(scope: string): void { + const segments = scope.split("/"); + if (scope === "" || segments.some((s) => s === "" || s === "." || s === ".." || s.includes("\\"))) { + throw new Error( + `a mount scope must be a key prefix like "a" or "a/b" — no leading, trailing or ` + + `repeated "/", and no "." or ".." segment. Got ${JSON.stringify(scope)}.`, + ); + } +} diff --git a/packages/workspace/src/types.ts b/packages/workspace/src/types.ts index ff1e310f88..4dc980f9ef 100644 --- a/packages/workspace/src/types.ts +++ b/packages/workspace/src/types.ts @@ -82,6 +82,23 @@ export interface Mount { * for that scope, and its pattern. */ collectionId: string; + /** + * Narrow the mount to one key prefix of its collection: `"a"` addresses the + * keys under `a/…` and nothing else. + * + * Inside the place the scope does not appear — the collection's `a/notes.md` + * sits at `/notes.md` — so the agent sees one owner's files as if + * they were the whole collection. Every list, read, write and delete the + * projection makes through this mount is keyed under the scope, and the + * listing is filtered by the collection itself rather than after the fact: + * a mount that listed every key and dropped the rest would still have read + * every owner's rows. + * + * A key prefix, not a path: no leading or trailing `/`, no empty, `.` or + * `..` segment. Omit it and the mount addresses the whole collection, as + * before. + */ + scope?: string; /** * Whether a flush may write back. A read-only mount is hydrated and then * left alone — its paths are skipped, never reported as orphans. diff --git a/packages/workspace/test/fake-collection.ts b/packages/workspace/test/fake-collection.ts index 5e09c9d59f..aa21ce586f 100644 --- a/packages/workspace/test/fake-collection.ts +++ b/packages/workspace/test/fake-collection.ts @@ -25,6 +25,10 @@ export interface FakeCollection extends ResourceCollectionRef () => { @@ -46,6 +50,8 @@ export function createFakeCollection( const prefix = pattern.replace(/\/\*+$/, ""); let writeFailure: string | undefined; + const listed: (string | undefined)[] = []; + const reads: string[] = []; const refFor = (key: string) => ({ // A real `ResourceRef.path` is the canonical storage key, prefix @@ -55,6 +61,7 @@ export function createFakeCollection( return state.get(key)!; }, async readContent() { + reads.push(key); return content.get(key) ?? null; }, async writeContent(next: string) { @@ -85,8 +92,13 @@ export function createFakeCollection( } return refFor(k) as never; }, - async list() { - return [...state.keys()].map((k) => refFor(k)) as never; + // Filtered the way the engine filters: a bare string prefix on the key, + // with no separator boundary of its own. + async list(listPrefix?: string) { + listed.push(listPrefix); + return [...state.keys()] + .filter((k) => listPrefix === undefined || k.startsWith(listPrefix)) + .map((k) => refFor(k)) as never; }, async delete(key: string | Record) { const k = String(key); @@ -94,6 +106,8 @@ export function createFakeCollection( state.delete(k); }, contents: () => Object.fromEntries(content), + listCalls: () => [...listed], + contentReads: () => [...reads], setExternal: (key: string, next: string) => { content.set(key, next); state.set(key, { diff --git a/packages/workspace/test/scoped-mount.spec.ts b/packages/workspace/test/scoped-mount.spec.ts new file mode 100644 index 0000000000..741b05e493 --- /dev/null +++ b/packages/workspace/test/scoped-mount.spec.ts @@ -0,0 +1,157 @@ +/** + * A mount scoped to one key prefix of its collection. + * + * One collection holds every owner's files under `/…`, and a run must + * only ever see and write its own owner's keys. So the scope is applied where + * the rows are read, not after: a run that listed the whole collection and + * dropped the rest has still read everybody's files. + */ +import { describe, expect, it } from "vitest"; +import { createMemoryPlace } from "../src/memory-place"; +import { createProjection } from "../src/projection"; +import type { Mount } from "../src/types"; +import { createFakeCollection } from "./fake-collection"; + +/** One collection, two owners' keys, a place mounted on `a` only. */ +function setup() { + const collection = createFakeCollection("files/**", { + "a/notes.md": "a's notes", + "a/src/index.ts": "a's code", + "b/notes.md": "b's notes", + // Shares `a` as a STRING prefix but is another owner's key. + "ab/secret.md": "ab's secret", + }); + const place = createMemoryPlace(); + const mounts: Mount[] = [ + { prefix: "workspace", scope: "a", collection, collectionId: "files", writable: true }, + ]; + const projection = createProjection({ mounts, place }); + return { collection, place, projection }; +} + +describe("a scoped mount sees only its own keys", () => { + it("lists at the source with the scope, bounded by a separator", async () => { + const { collection, projection } = setup(); + + await projection.hydrate(); + + expect(collection.listCalls()).toEqual(["a/"]); + }); + + it("hydrates its keys with the scope stripped, and never reads another's", async () => { + const { collection, place, projection } = setup(); + + await projection.hydrate(); + + expect(place.snapshot()).toEqual({ + "workspace/notes.md": "a's notes", + "workspace/src/index.ts": "a's code", + }); + expect(collection.contentReads().every((key) => key.startsWith("a/"))).toBe(true); + }); + + it("flushes a new file under the scope, not at the collection's root", async () => { + const { collection, place, projection } = setup(); + await projection.hydrate(); + + await place.write("workspace/new.md", "fresh"); + const report = await projection.flush(); + + expect(report.outcomes).toContainEqual({ kind: "created", path: "workspace/new.md" }); + expect(collection.contents()["a/new.md"]).toBe("fresh"); + expect(collection.contents()["new.md"]).toBeUndefined(); + expect(collection.contents()["b/notes.md"]).toBe("b's notes"); + expect(collection.contents()["ab/secret.md"]).toBe("ab's secret"); + }); + + it("writes an edit back to the scoped key", async () => { + const { collection, place, projection } = setup(); + await projection.hydrate(); + + await place.write("workspace/notes.md", "edited"); + await projection.flush(); + + expect(collection.contents()["a/notes.md"]).toBe("edited"); + }); + + it("a put lands under the scope too", async () => { + const { collection, projection } = setup(); + await projection.hydrate(); + + const outcome = await projection.put("workspace/put.md", "one file"); + + expect(outcome).toEqual({ kind: "created", path: "workspace/put.md" }); + expect(collection.contents()["a/put.md"]).toBe("one file"); + }); +}); + +describe("a scoped run deletes only what it hydrated", () => { + it("removing a hydrated file deletes its scoped key and nothing else", async () => { + const { collection, place, projection } = setup(); + await projection.hydrate(); + + place.remove("workspace/notes.md"); + const report = await projection.flush(); + + expect(report.outcomes).toContainEqual({ kind: "deleted", path: "workspace/notes.md" }); + expect(Object.keys(collection.contents()).sort()).toEqual([ + "a/src/index.ts", + "ab/secret.md", + "b/notes.md", + ]); + }); + + it("an empty place deletes the scope's files, never another owner's", async () => { + const { collection, place, projection } = setup(); + await projection.hydrate(); + + place.remove("workspace/notes.md"); + place.remove("workspace/src/index.ts"); + await projection.flush(); + + expect(Object.keys(collection.contents()).sort()).toEqual(["ab/secret.md", "b/notes.md"]); + }); +}); + +describe("a scope is a key prefix, not a path", () => { + for (const scope of ["", "/a", "a/", "../a", "a/../b", "a//b", "./a"]) { + it(`refuses ${JSON.stringify(scope)}`, () => { + const collection = createFakeCollection("files/**"); + expect(() => + createProjection({ + place: createMemoryPlace(), + mounts: [{ prefix: "workspace", scope, collection, collectionId: "files", writable: true }], + }), + ).toThrow(/scope/); + }); + } + + it("accepts a nested scope", async () => { + const collection = createFakeCollection("files/**", { "org/a/x.md": "x", "org/b/y.md": "y" }); + const place = createMemoryPlace(); + const projection = createProjection({ + place, + mounts: [{ prefix: "w", scope: "org/a", collection, collectionId: "files", writable: true }], + }); + + await projection.hydrate(); + + expect(place.snapshot()).toEqual({ "w/x.md": "x" }); + }); +}); + +describe("an unscoped mount is unchanged", () => { + it("lists the whole collection", async () => { + const collection = createFakeCollection("files/**", { "a/x.md": "x" }); + const place = createMemoryPlace(); + const projection = createProjection({ + place, + mounts: [{ prefix: "files", collection, collectionId: "files", writable: true }], + }); + + await projection.hydrate(); + + expect(collection.listCalls()).toEqual([undefined]); + expect(place.snapshot()).toEqual({ "files/a/x.md": "x" }); + }); +}); From 707b340ed8ae83209fecbb08bcc44ab6be21d511 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:24:27 +0000 Subject: [PATCH 2/5] feat(workspace): local workspace host and run source (FIX-1762) Adds the RunSource type (repo, files or a refusal, from the block context alone) and localWorkspaceHost: provision, save and release over a host directory, with checkpoint and restore declared as no-ops for the next slice. A remote is judged against the operator's allowlist before any git process starts; git runs with -- before the remote and GIT_ALLOW_PROTOCOL limited to the listed schemes. One bare clone per remote, made under a per-clone queue and moved into place whole; a new branch is cut after a fetch and a fresh read of the default branch, and an existing checkout is handed back without a fetch. Kept files are hydrated into project/ beside the checkout, or workspace/ when there is no repository, through a scoped mount, and saved back by flush. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P9UuEKLJUWaKiSrYWCJbt5 --- .changeset/workspace-host-and-scoped-mount.md | 5 + packages/workspace/README.md | 85 ++++ packages/workspace/src/exec.ts | 51 ++ packages/workspace/src/index.ts | 22 +- packages/workspace/src/local-host.ts | 458 ++++++++++++++++++ packages/workspace/src/remotes.ts | 139 ++++++ packages/workspace/src/run-source.ts | 94 ++++ packages/workspace/test/git-fixtures.ts | 65 +++ .../workspace/test/local-host-files.spec.ts | 160 ++++++ .../workspace/test/local-host-repo.spec.ts | 313 ++++++++++++ packages/workspace/test/remotes.spec.ts | 59 +++ 11 files changed, 1449 insertions(+), 2 deletions(-) create mode 100644 .changeset/workspace-host-and-scoped-mount.md create mode 100644 packages/workspace/src/exec.ts create mode 100644 packages/workspace/src/local-host.ts create mode 100644 packages/workspace/src/remotes.ts create mode 100644 packages/workspace/src/run-source.ts create mode 100644 packages/workspace/test/git-fixtures.ts create mode 100644 packages/workspace/test/local-host-files.spec.ts create mode 100644 packages/workspace/test/local-host-repo.spec.ts create mode 100644 packages/workspace/test/remotes.spec.ts diff --git a/.changeset/workspace-host-and-scoped-mount.md b/.changeset/workspace-host-and-scoped-mount.md new file mode 100644 index 0000000000..5188acb545 --- /dev/null +++ b/.changeset/workspace-host-and-scoped-mount.md @@ -0,0 +1,5 @@ +--- +"@flow-state-dev/workspace": minor +--- + +Add `localWorkspaceHost`, which turns a run source's answer into a place a worker edits (a fresh branch of an allowed remote in `checkout/` with kept files in `project/`, or kept files alone in `workspace/`) and saves the kept files back, plus a `scope` on `Mount` that confines a projection to one key prefix of its collection (FIX-1762). diff --git a/packages/workspace/README.md b/packages/workspace/README.md index 4e714bc1bd..c5b89e56aa 100644 --- a/packages/workspace/README.md +++ b/packages/workspace/README.md @@ -174,6 +174,88 @@ Nested prefixes work. A collection at `artifacts/drafts` inside one at `artifact A read-only mount is hydrated and then left alone. Its paths aren't written back and aren't reported as orphans — the projection knows who owns them, and the answer is "not us". +### Scoping a mount to one key prefix + +When one collection holds files for several owners, keyed `/…`, give the mount a `scope` so a place only ever sees and writes one owner's keys: + +```ts +{ + prefix: "workspace", + scope: "sandbox", // only keys under sandbox/… + collection: files, + collectionId: collectionIdFor(files, principal), + writable: true, +} +``` + +The collection's `sandbox/notes.md` appears in the place at `workspace/notes.md`, and a write there lands back at `sandbox/notes.md`. Every list, read, write and delete goes through the scope, and the listing is filtered by the collection itself, so other owners' rows are never read at all. A scope is a key prefix like `"a"` or `"a/b"`: no leading or trailing `/`, and no `.` or `..` segment. + +## Workspace hosts and run sources + +A **run source** says where a run's files come from: a git repository and the branch to cut from, or a set of files kept in a collection. A **workspace host** turns that answer into a directory a worker can edit, saves the work back, and lets the directory go when the run is done. Harness workers and tool workers use the same host. + +```ts +import { localWorkspaceHost } from "@flow-state-dev/workspace"; + +const host = localWorkspaceHost({ + root: "/var/fsd/runs", + remotes: { allow: ["github.com"] }, // list "file" to allow file:// remotes + source: mySource, // (ctx) => an answer, below +}); +``` + +A source is a function of the block context, and answers one of three ways: + +```ts +{ kind: "repo", repo: "https://github.com/acme/storefront.git" } // a fresh branch of a repository +{ kind: "repo", repo, baseRef: "release", projectId: "storefront", files } // ...with kept files beside it +{ kind: "files", projectId: "sandbox", files } // no repository: the kept files are the work +{ kind: "refused", reason: "not-a-member", message: "..." } // no files for this run, and why +``` + +`files` is `{ collection, collectionId }`, and `projectId` is the key prefix inside that collection. The host never reads it as anything else. + +A run then goes through four calls: + +```ts +const answer = await host.source(ctx); +const place = await host.provision(answer, { place: [tenant, user, runId], branch: `fsd/${runId}` }); +// the worker runs with place.cwd as its working directory +const report = await host.save(place); // a FlushReport, as above +await host.release(place); +``` + +### What a place looks like + +``` +/.clones/.git one clone per remote, shared by every run +// + checkout/ a git worktree on the run's branch + project/ the kept files, next to the checkout + workspace/ the kept files as the working directory, when there is no repository +``` + +For a repository, the host keeps one clone of each remote under `root` and gives every run its own branch in `checkout/`. `https://github.com/acme/x`, `https://github.com/acme/x.git` and `git@github.com:acme/x.git` share one clone. Before it cuts a new branch, the host fetches and reads the remote's default branch again, so a renamed `main` is followed. `place.repo` says which clone, branch, base branch and commit the run got. Record the remote and base on your run, and log the clone path so whoever runs the host can find it. + +A checkout that already exists is handed back exactly as it was left. The host doesn't fetch, reset or rebase it, so a retry finds its uncommitted work. A checkout it can't explain, such as one on a different branch or with no `.git`, is refused and left alone. + +Kept files go in `project/` next to the checkout, never inside it, so `git status` never sees them. With no repository they go in `workspace/`, which is also the run's working directory. Either way they're hydrated from one key prefix of the collection and flushed back by `save`, with the same conflict reporting as any projection. The first run in an empty prefix starts from an empty directory. A retry in the same process gets the same live directory back, unsaved edits included. A directory the process didn't fill itself, after a restart or because it was deleted, is rebuilt from the collection. The collection is the record, so anything a run never saved is gone. + +`checkpoint` and `restore` exist on the host and do nothing yet. They mark where keeping uncommitted repository work across a lost machine will plug in. + +### Which remotes a host reaches + +`remotes.allow` lists hosts, reached over `https` or `ssh`. Add `"file"` to permit `file://` remotes, which are refused otherwise. Every remote is checked before any git process starts, and the host refuses, with a `WorkspaceRefusedError`: + +| `reason` | When | +| --- | --- | +| `invalid-remote` | The value starts with `-`, uses a transport helper like `ext::`, has a host starting with `-`, carries a password or an `https` login, or is a bare path rather than a URL | +| `remote-not-allowed` | Its host isn't listed, it's `file://` and `"file"` isn't listed, or its scheme isn't `https`, `ssh` or `file` | +| `remote-unreadable` | The host is allowed but git couldn't read it, or it has no branch to cut from | +| anything else | The source answered `refused`, with that reason | + +Git itself runs with `GIT_ALLOW_PROTOCOL` set to the listed schemes, with `--` before every remote, and without a terminal prompt. A refusal names the remote without its credential. None of these clears on a retry, so fail the run with the reason. + ## API | Export | What it is | @@ -188,6 +270,9 @@ A read-only mount is hydrated and then left alone. Its paths aren't written back | `principalFromContext(ctx)` | The scoping identity, read off a block's execution context. | | `collectionIdFor(collection, principal)` | A `Mount.collectionId` for a scoped door. | | `unscopedCollectionId(collection)` | A `Mount.collectionId` for a door with no principal. | +| `localWorkspaceHost({ root, remotes, source })` | A workspace host on this machine. Returns `{ root, source, provision, save, checkpoint, restore, release }`. | +| `WorkspaceRefusedError` | What `provision` rejects with when it won't make a place. Carries `reason`. | +| `RunSource`, `RunSourceAnswer` | The run-source function type and its three answers. | `ownedPaths()` returns the paths the projection currently holds a baseline for — what it would write to, and what it would delete. diff --git a/packages/workspace/src/exec.ts b/packages/workspace/src/exec.ts new file mode 100644 index 0000000000..9adfb4b0ed --- /dev/null +++ b/packages/workspace/src/exec.ts @@ -0,0 +1,51 @@ +/** + * The one place this package spawns a child process. + * + * Every call bounds itself with a wall clock, which is required rather than + * defaulted: a `git ls-remote` that has not answered in a minute is broken, + * while a first fetch of a large repository legitimately takes many, and one + * number cannot be right for both. Making it required forces the caller to say + * which kind of wait it is. + * + * Kept as a module of its own so a test can watch every process this package + * starts — which is how "refused before any git process" is checked rather + * than asserted. + */ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface RunOptions { + cwd: string; + /** + * The wall clock, in milliseconds. `execFile` sends `SIGTERM` when it + * elapses and the promise rejects, so the caller sees a failure rather than + * a hang. + */ + timeoutMs: number; + /** Extra environment, laid over the process's own. */ + env?: Record; + maxBuffer?: number; +} + +/** Run `file` with `args`, resolving with its output or rejecting on a non-zero exit. */ +export async function run( + file: string, + args: string[], + options: RunOptions, +): Promise<{ stdout: string; stderr: string }> { + const result = await execFileAsync(file, args, { + cwd: options.cwd, + timeout: options.timeoutMs, + maxBuffer: options.maxBuffer ?? 8 * 1024 * 1024, + env: options.env === undefined ? process.env : { ...process.env, ...options.env }, + }); + return { stdout: String(result.stdout), stderr: String(result.stderr) }; +} + +/** + * Git that talks to a remote. Minutes rather than seconds, because a first + * fetch of a large repository is genuinely slow. + */ +export const GIT_TIMEOUT_MS = 600_000; diff --git a/packages/workspace/src/index.ts b/packages/workspace/src/index.ts index 055a036d0e..b5be220693 100644 --- a/packages/workspace/src/index.ts +++ b/packages/workspace/src/index.ts @@ -1,9 +1,12 @@ /** - * `@flow-state-dev/workspace` — the file projection. + * `@flow-state-dev/workspace` — the file projection, and the workspace host + * built on it. * * One piece of machinery moving file content between resource collections and * wherever an agent works, so the shell path and the coding path share a - * behaviour instead of drifting apart with a copy each. + * behaviour instead of drifting apart with a copy each. The host + * (`./local-host`) turns a run source's answer (`./run-source`) into that + * place: a branch of a repository, or a set of kept files. * * See `projection.ts` for why the baseline exists and what it costs to get * wrong; it is the whole of the fix. @@ -32,3 +35,18 @@ export { unscopedCollectionId, } from "./scope-identity"; export type { ScopeName, ScopePrincipal } from "./scope-identity"; +export { localWorkspaceHost, WorkspaceRefusedError } from "./local-host"; +export type { + LocalWorkspaceHostOptions, + PlaceRequest, + WorkspaceHost, + WorkspacePlace, +} from "./local-host"; +export type { + FilesRunSource, + RepoRunSource, + RunFiles, + RunSource, + RunSourceAnswer, + RunSourceRefusal, +} from "./run-source"; diff --git a/packages/workspace/src/local-host.ts b/packages/workspace/src/local-host.ts new file mode 100644 index 0000000000..46cda56f1d --- /dev/null +++ b/packages/workspace/src/local-host.ts @@ -0,0 +1,458 @@ +/** + * A workspace host on the machine the flow runs on. + * + * A **workspace host** turns a run source's answer into a directory a worker + * can edit, saves the work back, and lets the directory go. This one keeps + * everything under one host directory: + * + * ``` + * /.clones/.git one bare clone per remote, shared by every run + * // + * checkout/ a git worktree on the run's own branch (a repository run) + * project/ the kept files, beside the checkout and never inside it + * workspace/ the kept files as the working directory (a run with no repository) + * ``` + * + * Three rules shape it, and each is enforced here rather than trusted to the + * caller: + * + * - **A remote is judged before git sees it** (`./remotes`). A refused remote + * starts no process at all, and every process that does start can reach + * only the transports the allowlist names. + * - **Existing work is never discarded.** A checkout already on disk is handed + * back as it is — no fetch, no reset, no rebase — and one this host cannot + * explain is refused, not cleared. Only a NEW branch is cut from a freshly + * fetched remote. + * - **The kept files are the record, the directory is not.** `project/` and + * `workspace/` are hydrated from one key prefix of a collection and flushed + * back to it. A directory this process did not hydrate is rebuilt from the + * collection; one it did is handed back live, with its unsaved edits. + * + * Checkpoint and restore are declared and do nothing yet: holding a + * repository's uncommitted work across a lost machine is a later slice, and + * the seam is here so that slice adds behaviour rather than a method. + */ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import { GIT_TIMEOUT_MS, run } from "./exec"; +import { createHostPlace } from "./host-place"; +import { createProjection, type Projection } from "./projection"; +import { allowedProtocols, checkRemote, redactRemote, type AllowedRemote } from "./remotes"; +import type { RunFiles, RunSource, RunSourceAnswer } from "./run-source"; +import type { FlushReport } from "./types"; + +/** The directory names inside a place. People and agents read them. */ +const CHECKOUT_DIR = "checkout"; +const PROJECT_DIR = "project"; +const WORKSPACE_DIR = "workspace"; +/** + * Where the shared clones live. Dot-prefixed, and a place segment may not + * start with a dot, so no place can ever be named over it. + */ +const CLONES_DIR = ".clones"; + +/** + * The host would not provision this run, and says why — before anything was + * created and before any agent was paid for. + * + * `reason` is stable: `"invalid-remote"`, `"remote-not-allowed"`, + * `"remote-unreadable"`, or whatever reason a run source refused with. A + * caller should fail the run with it rather than retry: none of these clears + * on its own. + */ +export class WorkspaceRefusedError extends Error { + constructor( + readonly reason: string, + message: string, + ) { + super(message); + this.name = "WorkspaceRefusedError"; + } +} + +export interface LocalWorkspaceHostOptions { + /** The host directory every clone and place lives under. */ + root: string; + /** + * The remotes this host may reach. `allow` lists hosts, reached over + * `https` or `ssh`, and may include `"file"` to permit `file://` remotes. + */ + remotes: { allow: readonly string[] }; + /** Where each run's files come from. Kept on the host for its callers to ask. */ + source: RunSource; +} + +/** Which place a run gets, named by its caller. */ +export interface PlaceRequest { + /** + * The place's directory under the root, one segment per level — typically + * whatever identifies the run, outermost first. Each segment is a plain + * name: no separators, nothing starting with `.`. + */ + place: readonly string[]; + /** The run's own branch. Required for a repository run. */ + branch?: string; +} + +/** A provisioned place. Hand it back to `save`, `checkpoint`, `restore` and `release`. */ +export interface WorkspacePlace { + /** Which kind of source made it. */ + kind: "repo" | "files"; + /** The place's directory. */ + dir: string; + /** Where the worker runs: `checkout/` for a repository, `workspace/` otherwise. */ + cwd: string; + /** The directory synced with the kept files, when the run has any. */ + filesDir?: string; + /** What the checkout is, for a repository run. Record `remote` and `baseRef` on the run. */ + repo?: { + /** The remote as the source gave it. */ + remote: string; + /** The shared clone this checkout is a worktree of. Log it so an operator can find it. */ + clone: string; + branch: string; + /** Whether this call cut the branch. `false` for a checkout handed back. */ + created: boolean; + /** The remote branch the run's branch was cut from. Only when `created`. */ + baseRef?: string; + /** The commit it was cut at. Only when `created`. */ + baseCommit?: string; + }; +} + +/** Provision, save and release the places runs work in. */ +export interface WorkspaceHost { + /** The host directory. */ + readonly root: string; + /** The run source the host was built with. Ask it, then hand its answer to `provision`. */ + readonly source: RunSource; + /** + * Make (or hand back) the place for one run. + * + * Rejects with a `WorkspaceRefusedError` for a refused source or remote, + * before any git process or directory exists. + */ + provision(answer: RunSourceAnswer, request: PlaceRequest): Promise; + /** + * Flush the place's kept files back to their collection. A conflict is an + * outcome in the report, never an overwrite. A place with no kept files + * reports nothing. + */ + save(place: WorkspacePlace): Promise; + /** Hold uncommitted repository work durably. Declared for a later slice; does nothing yet. */ + checkpoint(place: WorkspacePlace): Promise; + /** Bring held work back to a lost place. Declared for a later slice; does nothing yet. */ + restore(place: WorkspacePlace): Promise; + /** + * Let the place go. The directory is kept; the live link to its kept files + * is dropped, so a later provision rebuilds them from the collection. + */ + release(place: WorkspacePlace): Promise; +} + +const EMPTY_REPORT: FlushReport = { outcomes: [], conflicts: [], contested: [] }; + +/** Build a workspace host over a directory on this machine. */ +export function localWorkspaceHost(options: LocalWorkspaceHostOptions): WorkspaceHost { + const root = resolve(options.root); + const allow = options.remotes.allow; + const env = { + GIT_ALLOW_PROTOCOL: allowedProtocols(allow), + // A remote that wants a password fails instead of waiting on a prompt + // nobody will answer. + GIT_TERMINAL_PROMPT: "0", + }; + + /** Live kept-files projections, by the directory they fill. */ + const live = new Map>(); + /** The tail of each clone's queue: one git operation on a clone at a time, in this process. */ + const queues = new Map>(); + + async function git(cwd: string, args: string[]): Promise { + const { stdout } = await run("git", args, { cwd, timeoutMs: GIT_TIMEOUT_MS, env }); + return stdout.trim(); + } + + /** + * Run `operation` after every earlier one on the same clone has settled. + * + * This is the lock "one clone per remote" rests on in this process: two + * first runs for one remote queue here, the first makes the clone, the + * second finds it. Across processes the clone appears by an atomic rename, + * so a second process sees either no clone or a whole one. + */ + function serially(key: string, operation: () => Promise): Promise { + const previous = queues.get(key) ?? Promise.resolve(); + const next = previous.catch(() => undefined).then(operation); + queues.set(key, next); + void next + .catch(() => undefined) + .finally(() => { + if (queues.get(key) === next) queues.delete(key); + }); + return next; + } + + /** The place's directory, refusing any name that could leave the root or shadow the clones. */ + function placeDir(segments: readonly string[]): string { + const bad = segments.find( + (s) => s === "" || s.startsWith(".") || s.includes("/") || s.includes("\\") || s.includes("\0"), + ); + if (segments.length === 0 || bad !== undefined) { + throw new Error( + `a place is named by plain directory segments — none empty, none containing a ` + + `separator, none starting with "." — got ${JSON.stringify(segments)}.`, + ); + } + const dir = resolve(root, ...segments); + const rel = relative(root, dir); + if (rel === "" || rel.startsWith(`..${sep}`) || rel === ".." || isAbsolute(rel)) { + throw new Error(`the place ${JSON.stringify(segments)} is not inside the host's root ${root}.`); + } + return dir; + } + + /** + * Fill `/` from one key prefix of `files`, or hand back the live + * one this process already filled. + * + * A directory with no live projection behind it is cleared and hydrated + * again. Its contents were either saved — and so are in the collection — or + * were never saved by this process, and a projection that adopted them would + * read every file as new and could resurrect one another run deleted. + */ + function provisionFiles(dir: string, name: string, projectId: string, files: RunFiles): Promise { + const filesDir = join(dir, name); + const existing = live.get(filesDir); + if (existing !== undefined && existsSync(filesDir)) return existing; + + const made = (async () => { + rmSync(filesDir, { recursive: true, force: true }); + mkdirSync(filesDir, { recursive: true }); + const projection = createProjection({ + place: createHostPlace(dir), + mounts: [ + { + prefix: name, + scope: projectId, + collection: files.collection, + collectionId: files.collectionId, + writable: true, + }, + ], + }); + await projection.hydrate(); + return projection; + })(); + live.set(filesDir, made); + made.catch(() => { + if (live.get(filesDir) === made) live.delete(filesDir); + }); + return made; + } + + /** `true` when the clone holds `refs/heads/`. Local; never touches the remote. */ + async function hasBranch(clone: string, branch: string): Promise { + try { + await git(clone, ["show-ref", "--verify", "--quiet", `refs/heads/${branch}`]); + return true; + } catch { + return false; + } + } + + /** + * Fetch the remote into `clone` and record its default branch. + * + * The default branch is read from the remote every time rather than kept, + * so a remote that renamed `main` to `trunk` is followed on the next new + * branch instead of cut from a ref that no longer moves. + */ + async function refresh(clone: string, remote: AllowedRemote): Promise { + try { + const head = await git(clone, ["ls-remote", "--symref", "--", remote.url, "HEAD"]); + const match = /^ref: refs\/heads\/(\S+)\s+HEAD$/m.exec(head); + if (match === null) throw new Error("the remote reports no default branch"); + const defaultBranch = match[1]!; + await git(clone, ["fetch", "--quiet", "--prune", "--", remote.url, "+refs/heads/*:refs/remotes/origin/*"]); + await git(clone, ["symbolic-ref", "refs/remotes/origin/HEAD", `refs/remotes/origin/${defaultBranch}`]); + return defaultBranch; + } catch (error) { + throw unreadable(remote, error); + } + } + + /** The clone for `remote`, made whole and in place, and whether this call made it. */ + async function ensureClone(remote: AllowedRemote, clone: string): Promise<{ fresh: boolean; defaultBranch?: string }> { + if (existsSync(clone)) return { fresh: false }; + mkdirSync(join(root, CLONES_DIR), { recursive: true }); + const making = `${clone}.making-${randomUUID()}`; + try { + await git(root, ["init", "--quiet", "--bare", making]); + const defaultBranch = await refresh(making, remote); + try { + renameSync(making, clone); + } catch (error) { + // Another process finished its clone first. Theirs is whole, and + // ours is redundant. + if (!existsSync(clone)) throw error; + rmSync(making, { recursive: true, force: true }); + return { fresh: false }; + } + return { fresh: true, defaultBranch }; + } catch (error) { + rmSync(making, { recursive: true, force: true }); + throw error; + } + } + + async function provisionCheckout( + remote: AllowedRemote, + baseRef: string | undefined, + dir: string, + branch: string, + ): Promise> { + const clone = join(root, CLONES_DIR, `${remote.cloneKey}.git`); + const checkout = join(dir, CHECKOUT_DIR); + const shown = redactRemote(remote.url); + + // A checkout already here is the last attempt's work. It is handed back + // as it is, or refused — never fetched, reset or rebased. + if (existsSync(checkout)) { + if (!existsSync(join(checkout, ".git"))) { + throw new Error( + `the checkout at ${checkout} has no .git, so it is not one this host made. It may ` + + `hold work, and nothing here clears it — inspect it and remove it by hand.`, + ); + } + const head = await git(checkout, ["rev-parse", "--abbrev-ref", "HEAD"]); + if (head !== branch) { + throw new Error( + `the checkout at ${checkout} is on branch "${head}", not "${branch}". Refusing to ` + + `use it, and nothing here resets a tree — restore the branch or remove the checkout.`, + ); + } + const common = resolve(checkout, await git(checkout, ["rev-parse", "--git-common-dir"])); + if (common !== clone) { + throw new Error( + `the checkout at ${checkout} is not a worktree of the clone of "${shown}". Refusing ` + + `to hand it to a run that expects that repository; it may hold work, so it is kept.`, + ); + } + return { remote: remote.url, clone, branch, created: false }; + } + + return await serially(remote.cloneKey, async () => { + const made = await ensureClone(remote, clone); + // Bookkeeping for a worktree whose directory was removed, which would + // otherwise make `worktree add` refuse the path. Not a reset: it + // touches no tree. + await git(clone, ["worktree", "prune"]); + + // The run's branch already exists: its checkout was lost, the run was + // not. Put the branch back where it was — without a fetch, so nothing + // it started from moves. + if (!made.fresh && (await hasBranch(clone, branch))) { + await git(clone, ["worktree", "add", "--quiet", checkout, branch]); + return { remote: remote.url, clone, branch, created: false }; + } + + const defaultBranch = made.fresh ? made.defaultBranch! : await refresh(clone, remote); + const base = baseRef ?? defaultBranch; + let baseCommit: string; + try { + baseCommit = await git(clone, ["rev-parse", "--verify", "--quiet", `refs/remotes/origin/${base}^{commit}`]); + } catch { + throw new WorkspaceRefusedError( + "remote-unreadable", + `the remote "${shown}" has no branch "${base}" to cut the run's branch from.`, + ); + } + await git(clone, ["worktree", "add", "--quiet", "-b", branch, checkout, baseCommit]); + return { remote: remote.url, clone, branch, created: true, baseRef: base, baseCommit }; + }); + } + + async function provision(answer: RunSourceAnswer, request: PlaceRequest): Promise { + if (answer.kind === "refused") throw new WorkspaceRefusedError(answer.reason, answer.message); + const dir = placeDir(request.place); + + if (answer.kind === "files") { + const filesDir = join(dir, WORKSPACE_DIR); + await provisionFiles(dir, WORKSPACE_DIR, answer.projectId, answer.files); + return { kind: "files", dir, cwd: filesDir, filesDir }; + } + + if ((answer.projectId === undefined) !== (answer.files === undefined)) { + throw new Error( + "a repository source names its kept files with both projectId and files, or neither — " + + "one without the other has no key prefix to keep them under.", + ); + } + const branch = request.branch; + if (branch === undefined || !isPlainBranch(branch)) { + throw new Error(`a repository run needs a plain branch name to cut; got ${JSON.stringify(branch)}.`); + } + // Judged before anything is created or spawned. + const remote = checkRemote(answer.repo, allow); + if ("reason" in remote) throw new WorkspaceRefusedError(remote.reason, remote.message); + + const repo = await provisionCheckout(remote, answer.baseRef, dir, branch); + const place: WorkspacePlace = { kind: "repo", dir, cwd: join(dir, CHECKOUT_DIR), repo }; + if (answer.projectId !== undefined && answer.files !== undefined) { + await provisionFiles(dir, PROJECT_DIR, answer.projectId, answer.files); + place.filesDir = join(dir, PROJECT_DIR); + } + return place; + } + + async function save(place: WorkspacePlace): Promise { + if (place.filesDir === undefined) return EMPTY_REPORT; + const projection = live.get(place.filesDir); + if (projection === undefined) { + throw new Error( + `the place at ${place.dir} has no live kept files in this process — it was released, ` + + `or provisioned elsewhere. Provision it again before saving.`, + ); + } + return await (await projection).flush(); + } + + return { + root, + source: options.source, + provision, + save, + async checkpoint() {}, + async restore() {}, + async release(place) { + if (place.filesDir !== undefined) live.delete(place.filesDir); + }, + }; +} + +/** The remote could not be read; say which one, and never with a credential. */ +function unreadable(remote: AllowedRemote, error: unknown): WorkspaceRefusedError { + if (error instanceof WorkspaceRefusedError) return error; + const shown = redactRemote(remote.url); + const detail = String((error as { stderr?: unknown })?.stderr ?? (error as Error)?.message ?? error) + .split("\n") + .find((line) => line.trim() !== "") + ?.trim(); + return new WorkspaceRefusedError( + "remote-unreadable", + `the remote "${shown}" could not be read${detail ? `: ${detail}` : ""}.`, + ); +} + +/** + * A branch name this host will hand to git: nothing git would read as an + * option, and none of the spellings a ref refuses. Git checks the rest. + */ +function isPlainBranch(branch: string): boolean { + return ( + /^[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(branch) && + !/\.\.|\/\/|\/\.|\.lock$|\.lock\/|\/$|\.$/.test(branch) + ); +} diff --git a/packages/workspace/src/remotes.ts b/packages/workspace/src/remotes.ts new file mode 100644 index 0000000000..413fa6ba3c --- /dev/null +++ b/packages/workspace/src/remotes.ts @@ -0,0 +1,139 @@ +/** + * Which remotes a host may reach, decided before any git process starts. + * + * A remote arrives as a string somebody wrote — a member, or a model with a + * person's approval — and git interprets a remote string generously: a + * leading `-` is an option, `ext::` runs a command, an ssh host spelled + * `-oProxyCommand=…` runs another. So the string is parsed and judged here, + * as data, and git only ever sees one that passed. + * + * The operator's allowlist names hosts, reached over `https` or `ssh`, and + * optionally the word `"file"` to permit `file://` remotes. Nothing else is + * reachable: no `http`, no `git://`, no transport helper, no bare path. + */ +import { createHash } from "node:crypto"; + +/** A remote that passed: what git is handed, and what the host keys its clone by. */ +export interface AllowedRemote { + /** The remote exactly as given. Passed to git after `--`. */ + url: string; + /** The transport git will use. */ + scheme: "https" | "ssh" | "file"; + /** + * A directory-safe name for this repository's clone. Two spellings of one + * repository — `https://h/x`, `https://h/x.git`, `git@h:x` — share it. + */ + cloneKey: string; +} + +/** A remote that did not pass, and why. */ +export interface RefusedRemote { + reason: "invalid-remote" | "remote-not-allowed"; + message: string; +} + +/** The schemes a host speaks to a listed host. */ +const NETWORK_SCHEMES = ["https", "ssh"] as const; + +/** + * The value of `GIT_ALLOW_PROTOCOL` for an allowlist: only the transports a + * listed entry can reach, so git itself refuses anything this module missed. + */ +export function allowedProtocols(allow: readonly string[]): string { + const schemes: string[] = []; + if (allow.some((entry) => entry !== "file")) schemes.push(...NETWORK_SCHEMES); + if (allow.includes("file")) schemes.push("file"); + // Git reads an EMPTY value as "allow nothing", which is right for an + // allowlist with nothing on it. + return schemes.join(":"); +} + +/** + * The remote as a person may be shown it: any userinfo removed, so a refusal + * that names the remote never prints a credential. + */ +export function redactRemote(remote: string): string { + return remote.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/@]*@/i, "$1"); +} + +/** Judge `remote` against `allow`. Pure: starts nothing, reads nothing. */ +export function checkRemote(remote: string, allow: readonly string[]): AllowedRemote | RefusedRemote { + const shown = redactRemote(remote); + const invalid = (why: string): RefusedRemote => ({ + reason: "invalid-remote", + message: `the remote "${shown}" is not a usable git remote: ${why}.`, + }); + const notAllowed = (what: string): RefusedRemote => ({ + reason: "remote-not-allowed", + message: `the remote "${shown}" is not allowed on this host: ${what}.`, + }); + + if (remote.startsWith("-")) return invalid("it starts with \"-\", which git reads as an option"); + // Whitespace and control characters have no place in a remote and every + // place in a smuggled argument. + if (/[\s\x00-\x1f\x7f]/.test(remote)) return invalid("it contains whitespace or a control character"); + // `::
` hands the address to a helper program; `ext::` + // runs it as a command line. + if (/^[a-z][a-z0-9+.-]*::/i.test(remote)) return invalid("transport helpers (\"::\") are never reached"); + + const hasScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(remote); + if (hasScheme) { + let url: URL; + try { + url = new URL(remote); + } catch { + return invalid("it does not parse as a URL"); + } + const scheme = url.protocol.slice(0, -1).toLowerCase(); + if (url.password !== "") return invalid("it carries a password"); + + if (scheme === "file") { + if (!allow.includes("file")) return notAllowed("file:// remotes are off unless the host lists \"file\""); + if (url.hostname !== "" && url.hostname !== "localhost") return invalid("a file:// remote names no host"); + const path = decodeURIComponent(url.pathname); + return { url: remote, scheme: "file", cloneKey: cloneKeyFor("file", path) }; + } + if (scheme !== "https" && scheme !== "ssh") { + return notAllowed(`the host reaches remotes over https or ssh, not ${scheme}`); + } + if (scheme === "https" && url.username !== "") return invalid("it carries a credential"); + const host = url.hostname.toLowerCase(); + if (host === "" || host.startsWith("-")) return invalid("its host is missing or starts with \"-\""); + if (!allow.some((entry) => entry.toLowerCase() === host)) return notAllowed(`"${host}" is not a listed host`); + return { url: remote, scheme, cloneKey: cloneKeyFor(host, repositoryPath(url.pathname)) }; + } + + // Git's scp-like form, `[user@]host:path`, is ssh — but only when the colon + // comes before any slash. Anything else without a scheme is a path. + const scp = /^(?:([^@/:]+)@)?([^/:]+):(.+)$/.exec(remote); + if (scp !== null) { + const host = scp[2]!.toLowerCase(); + if (host.startsWith("-")) return invalid("its host starts with \"-\""); + if (!allow.some((entry) => entry.toLowerCase() === host)) return notAllowed(`"${host}" is not a listed host`); + return { url: remote, scheme: "ssh", cloneKey: cloneKeyFor(host, repositoryPath(scp[3]!)) }; + } + + return invalid("a bare path is not a remote — spell a local repository as a file:// URL"); +} + +/** + * The repository part of a network remote, spelled one way: no leading or + * trailing `/`, no `.git`. + * + * Not applied to a `file://` path, where `x` and `x.git` are two different + * directories. + */ +function repositoryPath(path: string): string { + return path.replace(/^\/+/, "").replace(/\/+$/, "").replace(/\.git$/, ""); +} + +/** + * A readable, directory-safe clone name with a hash of the identity on the + * end, so two identities that sanitize alike still get two clones. + */ +function cloneKeyFor(host: string, path: string): string { + const identity = `${host}/${path}`; + const readable = identity.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[-.]+/, "").slice(0, 80); + const hash = createHash("sha256").update(identity).digest("hex").slice(0, 12); + return `${readable}-${hash}`; +} diff --git a/packages/workspace/src/run-source.ts b/packages/workspace/src/run-source.ts new file mode 100644 index 0000000000..d12e89cf86 --- /dev/null +++ b/packages/workspace/src/run-source.ts @@ -0,0 +1,94 @@ +/** + * Where a run's files come from. + * + * A worker about to edit files asks one question before anything is made: + * is this run cutting a branch of a git repository, or working on a set of + * files kept in a collection? A **run source** answers it, from the block + * context alone. A workspace host (`./local-host`) turns the answer into a + * directory. + * + * The answer is the seam: whoever knows which repository or which files a run + * belongs to fills the source, and the host stays ignorant of why. The host + * reads `projectId` as nothing more than a key prefix into the collection the + * answer names — it never looks the id up, and never learns what owns it. + */ +import type { LooseBlockContext } from "@flow-state-dev/core"; +import type { ResourceCollectionRef } from "@flow-state-dev/core/types"; +import type { ProjectedEntryState } from "./types"; + +/** + * Where the files kept beside a run live: one collection, addressed under the + * answer's `projectId`. + * + * Carried on the answer rather than configured on the host, because a + * collection ref is bound to the context that resolved it — the host is built + * once at boot, with no context to resolve one from. + */ +export interface RunFiles { + /** The collection that holds the kept files, keyed `/`. */ + collection: ResourceCollectionRef; + /** + * What the collection IS, durably. Write arbitration keys on it — see + * `Mount.collectionId`. + */ + collectionId: string; +} + +/** + * The run cuts a fresh branch of a git repository. + * + * With `projectId` and `files`, the kept files are laid down in `project/` + * beside the checkout and saved back; without them, the run gets the + * checkout alone. + */ +export interface RepoRunSource { + kind: "repo"; + /** The remote, as you would pass it to `git clone`. Checked against the host's allowlist. */ + repo: string; + /** + * The branch on the remote to cut from. Omitted: the remote's default + * branch, read fresh each time a new branch is cut. + */ + baseRef?: string; + /** The key prefix of the kept files. Required together with `files`. */ + projectId?: string; + /** The collection of kept files. Required together with `projectId`. */ + files?: RunFiles; +} + +/** + * The run has no repository: its working directory is the kept files, + * hydrated before the run and saved back after. + */ +export interface FilesRunSource { + kind: "files"; + /** The key prefix of this run's files in `files.collection`. */ + projectId: string; + files: RunFiles; +} + +/** + * The source will not give this run any files, and says why. + * + * Not an error: an answer. A host handed one refuses to provision, naming the + * reason, before anything is created. + */ +export interface RunSourceRefusal { + kind: "refused"; + /** A stable, machine-readable reason, e.g. `"not-a-member"`. */ + reason: string; + /** What a person reading the run's failure should be told. */ + message: string; +} + +/** Everything a run source can answer. */ +export type RunSourceAnswer = RepoRunSource | FilesRunSource | RunSourceRefusal; + +/** + * Resolve a run's source from its block context. + * + * Handed the context and nothing else, so it can only answer from what the + * context reaches — stored, server-written data — and never from a row's + * input, its metadata, or anything a model wrote (BP-031). + */ +export type RunSource = (ctx: LooseBlockContext) => RunSourceAnswer | Promise; diff --git a/packages/workspace/test/git-fixtures.ts b/packages/workspace/test/git-fixtures.ts new file mode 100644 index 0000000000..6c40bd45a0 --- /dev/null +++ b/packages/workspace/test/git-fixtures.ts @@ -0,0 +1,65 @@ +/** + * Real git repositories for the host's tests: a bare "remote" the host + * reaches over `file://`, and helpers to move it on underneath a run. + */ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const AUTHOR = ["-c", "user.email=test@example.com", "-c", "user.name=test"]; + +/** Run git and return its trimmed stdout. */ +export function git(cwd: string, ...args: string[]): string { + return execFileSync("git", [...AUTHOR, ...args], { cwd, encoding: "utf8" }).trim(); +} + +/** A fresh temporary directory. */ +export function tempDir(label: string): string { + return mkdtempSync(join(tmpdir(), `fsd-workspace-${label}-`)); +} + +export interface Remote { + /** The bare repository on disk. */ + bare: string; + /** Its `file://` spelling, which is what a run source answers with. */ + url: string; + /** Commit `file` with `content` on `branch` of the remote; returns the new commit. */ + commit(branch: string, file: string, content: string): string; + /** Point the remote's HEAD — its default branch — at `branch`. */ + setDefault(branch: string): void; + /** The commit `ref` names on the remote. */ + rev(ref: string): string; +} + +/** A bare remote with one commit on `main` holding `marker.txt`. */ +export function createRemote(marker = "marker A"): Remote { + const root = tempDir("remote"); + const work = join(root, "work"); + const bare = join(root, "origin.git"); + execFileSync("git", ["init", "-q", "-b", "main", work]); + writeFileSync(join(work, "marker.txt"), marker); + git(work, "add", "marker.txt"); + git(work, "commit", "-q", "-m", "marker"); + execFileSync("git", ["clone", "-q", "--bare", work, bare]); + + return { + bare, + url: pathToFileURL(bare).href, + commit(branch, file, content) { + git(work, "fetch", "-q", bare, "+refs/heads/*:refs/remotes/bare/*"); + const exists = git(bare, "branch", "--list", branch) !== ""; + git(work, "checkout", "-q", "-B", branch, exists ? `bare/${branch}` : "HEAD"); + writeFileSync(join(work, file), content); + git(work, "add", file); + git(work, "commit", "-q", "-m", `${file} on ${branch}`); + git(work, "push", "-q", bare, `${branch}:${branch}`); + return git(work, "rev-parse", "HEAD"); + }, + setDefault(branch) { + git(bare, "symbolic-ref", "HEAD", `refs/heads/${branch}`); + }, + rev: (ref) => git(bare, "rev-parse", ref), + }; +} diff --git a/packages/workspace/test/local-host-files.spec.ts b/packages/workspace/test/local-host-files.spec.ts new file mode 100644 index 0000000000..341fb3b406 --- /dev/null +++ b/packages/workspace/test/local-host-files.spec.ts @@ -0,0 +1,160 @@ +/** + * The local workspace host, for a run with no repository. + * + * The kept files are the whole record here — there is no git to check into — + * so the properties are about that record: the first run starts from nothing, + * what a run writes is in the collection after a save, the next run starts + * from it, a lost place comes back from it, and one owner's runs never see + * another's. + */ +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { localWorkspaceHost } from "../src/local-host"; +import type { FilesRunSource } from "../src/run-source"; +import { createFakeCollection, type FakeCollection } from "./fake-collection"; +import { tempDir } from "./git-fixtures"; + +let root: string; +let collection: FakeCollection; + +beforeEach(() => { + root = tempDir("files"); + collection = createFakeCollection("project-files/**"); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const host = () => + localWorkspaceHost({ + root, + remotes: { allow: [] }, + source: () => ({ kind: "refused", reason: "unused", message: "unused" }), + }); + +const files = (projectId: string): FilesRunSource => ({ + kind: "files", + projectId, + files: { collection, collectionId: "project-files" }, +}); + +describe("a run with no repository works in workspace/", () => { + it("starts the first run from an empty directory", async () => { + const place = await host().provision(files("sandbox"), { place: ["run-1"] }); + + expect(place.kind).toBe("files"); + expect(place.cwd).toBe(join(root, "run-1", "workspace")); + expect(place.filesDir).toBe(place.cwd); + expect(readdirSync(place.cwd)).toEqual([]); + expect(place.repo).toBeUndefined(); + }); + + it("saves what the run wrote into the collection, under the run's key prefix", async () => { + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + + writeFileSync(join(place.cwd, "index.html"), "

hi

"); + const report = await h.save(place); + + expect(report.outcomes).toContainEqual({ kind: "created", path: "workspace/index.html" }); + expect(collection.contents()).toEqual({ "sandbox/index.html": "

hi

" }); + }); + + it("starts the next run from what the last one saved, nested paths included", async () => { + const h = host(); + const first = await h.provision(files("sandbox"), { place: ["run-1"] }); + mkdirSync(join(first.cwd, "src", "lib"), { recursive: true }); + writeFileSync(join(first.cwd, "src", "lib", "a.ts"), "export const a = 1;"); + await h.save(first); + await h.release(first); + + const second = await h.provision(files("sandbox"), { place: ["run-2"] }); + + expect(readFileSync(join(second.cwd, "src", "lib", "a.ts"), "utf8")).toBe("export const a = 1;"); + }); + + it("propagates a delete the run made", async () => { + collection.setExternal("sandbox/old.md", "stale"); + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + + rmSync(join(place.cwd, "old.md")); + await h.save(place); + + expect(collection.contents()).toEqual({}); + }); + + it("never shows one owner's files to another owner's run", async () => { + collection.setExternal("sandbox/mine.md", "sandbox's"); + collection.setExternal("platform/theirs.md", "platform's"); + + const place = await host().provision(files("platform"), { place: ["run-1"] }); + + expect(readdirSync(place.cwd)).toEqual(["theirs.md"]); + expect(collection.listCalls()).toEqual(["platform/"]); + }); + + it("hydrates a lost place again from the collection", async () => { + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + writeFileSync(join(place.cwd, "kept.md"), "kept"); + await h.save(place); + + rmSync(join(root, "run-1"), { recursive: true, force: true }); + const again = await h.provision(files("sandbox"), { place: ["run-1"] }); + + expect(readFileSync(join(again.cwd, "kept.md"), "utf8")).toBe("kept"); + }); + + it("hands the same live place back to a retry in this process, unsaved work intact", async () => { + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + writeFileSync(join(place.cwd, "draft.md"), "not saved yet"); + + const retry = await h.provision(files("sandbox"), { place: ["run-1"] }); + + expect(readFileSync(join(retry.cwd, "draft.md"), "utf8")).toBe("not saved yet"); + await h.save(retry); + expect(collection.contents()).toEqual({ "sandbox/draft.md": "not saved yet" }); + }); + + it("reports two runs editing one file as a conflict, never an overwrite", async () => { + collection.setExternal("sandbox/shared.md", "v1"); + const h = host(); + const a = await h.provision(files("sandbox"), { place: ["run-a"] }); + const b = await h.provision(files("sandbox"), { place: ["run-b"] }); + + writeFileSync(join(a.cwd, "shared.md"), "a's edit"); + writeFileSync(join(b.cwd, "shared.md"), "b's edit"); + await h.save(a); + const report = await h.save(b); + + expect(report.conflicts.map((c) => c.path)).toEqual(["workspace/shared.md"]); + expect(collection.contents()["sandbox/shared.md"]).toBe("a's edit"); + }); +}); + +describe("slice-2 seams are declared and do nothing yet", () => { + it("checkpoint and restore leave the place and the collection as they were", async () => { + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + writeFileSync(join(place.cwd, "a.md"), "a"); + + await h.checkpoint(place); + await h.restore(place); + + expect(collection.contents()).toEqual({}); + expect(readFileSync(join(place.cwd, "a.md"), "utf8")).toBe("a"); + }); + + it("release keeps the directory", async () => { + const h = host(); + const place = await h.provision(files("sandbox"), { place: ["run-1"] }); + + await h.release(place); + + expect(existsSync(place.cwd)).toBe(true); + }); +}); diff --git a/packages/workspace/test/local-host-repo.spec.ts b/packages/workspace/test/local-host-repo.spec.ts new file mode 100644 index 0000000000..4a904728db --- /dev/null +++ b/packages/workspace/test/local-host-repo.spec.ts @@ -0,0 +1,313 @@ +/** + * The local workspace host, for a run that cuts a branch of a repository. + * + * Every remote here is a real bare repository reached over `file://`, and + * every git process the host starts is counted, because the two properties + * that matter most are about processes: a refused remote starts none, and a + * retry starts no fetch. + */ +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { run } from "../src/exec"; +import { localWorkspaceHost, WorkspaceRefusedError } from "../src/local-host"; +import { createFakeCollection } from "./fake-collection"; +import { createRemote, git, tempDir, type Remote } from "./git-fixtures"; + +vi.mock("../src/exec", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, run: vi.fn(actual.run) }; +}); + +const spawned = vi.mocked(run); +/** The git subcommand of every process the host started. */ +const gitCommands = () => spawned.mock.calls.map(([, args]) => args[0]); +/** The error `attempt` rejected with; fails the test if it provisioned. */ +const refusalOf = (attempt: Promise) => + attempt.then( + () => { + throw new Error("expected a refusal, but the place was provisioned"); + }, + (e: unknown) => e as WorkspaceRefusedError, + ); +const noSource = () => ({ kind: "refused" as const, reason: "unused", message: "unused" }); + +let root: string; +let remote: Remote; + +beforeEach(() => { + root = tempDir("host"); + remote = createRemote("marker A"); + spawned.mockClear(); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +function host(allow: string[] = ["file"]) { + return localWorkspaceHost({ root, remotes: { allow }, source: noSource }); +} + +describe("only remotes the operator allows are reached", () => { + const refused: [string, () => string, string][] = [ + ["file:// when file is not listed", () => remote.url, "remote-not-allowed"], + ["a transport helper", () => "ext::sh -c touch% /tmp/fsd-pwned", "invalid-remote"], + ["an ssh host that is an option", () => "ssh://-oProxyCommand=touch%20/tmp/fsd-pwned/x", "invalid-remote"], + ["an scp host that is an option", () => "git@-oProxyCommand=x:org/repo", "invalid-remote"], + ["a value that starts with -", () => "--upload-pack=touch /tmp/fsd-pwned", "invalid-remote"], + ["a host the operator did not list", () => "https://evil.example/org/repo.git", "remote-not-allowed"], + ["a scheme the host never speaks", () => "http://github.com/org/repo.git", "remote-not-allowed"], + ["a bare path", () => remote.bare, "invalid-remote"], + ]; + + for (const [label, value, reason] of refused) { + it(`refuses ${label} before any git process`, async () => { + const allow = label.startsWith("file://") ? ["github.com"] : ["github.com", "file"]; + const attempt = host(allow).provision({ kind: "repo", repo: value() }, { place: ["run-1"], branch: "fsd/run-1" }); + + await expect(attempt).rejects.toBeInstanceOf(WorkspaceRefusedError); + await expect(attempt).rejects.toMatchObject({ reason }); + expect(spawned).not.toHaveBeenCalled(); + expect(existsSync(join(root, ".clones"))).toBe(false); + expect(existsSync(join(root, "run-1"))).toBe(false); + }); + } + + it("names the remote it refused, without its credential", async () => { + const attempt = host(["github.com"]).provision( + { kind: "repo", repo: "https://someone:hunter2@github.com/org/repo.git" }, + { place: ["run-1"], branch: "fsd/run-1" }, + ); + + const error = await refusalOf(attempt); + expect(error).toBeInstanceOf(WorkspaceRefusedError); + expect(error.reason).toBe("invalid-remote"); + expect(error.message).toContain("github.com/org/repo.git"); + expect(error.message).not.toContain("hunter2"); + expect(spawned).not.toHaveBeenCalled(); + }); + + it("limits git's transports to the listed schemes", async () => { + await host(["file"]).provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(spawned).toHaveBeenCalled(); + for (const [, , options] of spawned.mock.calls) { + expect(options.env?.GIT_ALLOW_PROTOCOL).toBe("file"); + } + }); + + it("puts -- before the remote on every git call that names it", async () => { + await host().provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + const naming = spawned.mock.calls.filter(([, args]) => args.includes(remote.url)); + expect(naming.length).toBeGreaterThan(0); + for (const [, args] of naming) { + expect(args.indexOf("--")).toBeGreaterThanOrEqual(0); + expect(args.indexOf("--")).toBeLessThan(args.indexOf(remote.url)); + } + }); + + it("fails before the run, naming the remote, when an allowed remote cannot be read", async () => { + const missing = `${remote.url}-missing`; + const attempt = host().provision({ kind: "repo", repo: missing }, { place: ["run-1"], branch: "fsd/run-1" }); + + const error = await refusalOf(attempt); + expect(error).toBeInstanceOf(WorkspaceRefusedError); + expect(error.reason).toBe("remote-unreadable"); + expect(error.message).toContain(missing); + expect(existsSync(join(root, "run-1", "checkout"))).toBe(false); + // No half-made clone left for the next attempt to trip on. + expect(readdirSync(join(root, ".clones"))).toEqual([]); + }); +}); + +describe("a repository run gets a fresh branch in checkout/", () => { + it("cuts the branch from the remote's default branch", async () => { + const place = await host().provision({ kind: "repo", repo: remote.url }, { place: ["t", "run-1"], branch: "fsd/run-1" }); + + expect(place.cwd).toBe(join(root, "t", "run-1", "checkout")); + expect(readFileSync(join(place.cwd, "marker.txt"), "utf8")).toBe("marker A"); + expect(git(place.cwd, "rev-parse", "--abbrev-ref", "HEAD")).toBe("fsd/run-1"); + expect(place.repo).toMatchObject({ remote: remote.url, baseRef: "main", baseCommit: remote.rev("main"), created: true }); + }); + + it("keeps one clone per remote, shared by every run, under the host's root", async () => { + const h = host(); + const a = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + const b = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-2"], branch: "fsd/run-2" }); + + expect(readdirSync(join(root, ".clones"))).toHaveLength(1); + expect(a.repo?.clone).toBe(b.repo?.clone); + expect(a.repo?.clone.startsWith(join(root, ".clones"))).toBe(true); + }); + + it("makes one clone when two first runs for a remote start at once", async () => { + const h = host(); + const [a, b] = await Promise.all([ + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-2"], branch: "fsd/run-2" }), + ]); + + expect(gitCommands().filter((c) => c === "init")).toHaveLength(1); + expect(readdirSync(join(root, ".clones"))).toHaveLength(1); + expect(readFileSync(join(a.cwd, "marker.txt"), "utf8")).toBe("marker A"); + expect(readFileSync(join(b.cwd, "marker.txt"), "utf8")).toBe("marker A"); + }); + + it("follows the remote's default branch when it is renamed", async () => { + const h = host(); + await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + const trunk = remote.commit("trunk", "trunk.txt", "on trunk"); + remote.setDefault("trunk"); + git(remote.bare, "branch", "-D", "main"); + + const next = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-2"], branch: "fsd/run-2" }); + + expect(git(next.cwd, "rev-parse", "HEAD")).toBe(trunk); + expect(next.repo).toMatchObject({ baseRef: "trunk", baseCommit: trunk }); + }); + + it("fetches before cutting a new branch, so a new run starts from the remote's latest", async () => { + const h = host(); + await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + const latest = remote.commit("main", "later.txt", "later"); + + const next = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-2"], branch: "fsd/run-2" }); + + expect(git(next.cwd, "rev-parse", "HEAD")).toBe(latest); + }); + + it("cuts from a named base when the source names one", async () => { + const feature = remote.commit("release", "release.txt", "release"); + + const place = await host().provision( + { kind: "repo", repo: remote.url, baseRef: "release" }, + { place: ["run-1"], branch: "fsd/run-1" }, + ); + + expect(git(place.cwd, "rev-parse", "HEAD")).toBe(feature); + expect(place.repo).toMatchObject({ baseRef: "release", baseCommit: feature }); + }); +}); + +describe("a retry keeps its checkout and never fetches", () => { + it("hands back the same checkout, work and all, without touching the remote", async () => { + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + const started = git(first.cwd, "rev-parse", "HEAD"); + writeFileSync(join(first.cwd, "wip.txt"), "uncommitted"); + remote.commit("main", "later.txt", "later"); + spawned.mockClear(); + + const retry = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(retry.cwd).toBe(first.cwd); + expect(retry.repo?.created).toBe(false); + expect(readFileSync(join(retry.cwd, "wip.txt"), "utf8")).toBe("uncommitted"); + expect(git(retry.cwd, "rev-parse", "HEAD")).toBe(started); + expect(gitCommands()).not.toContain("fetch"); + expect(gitCommands()).not.toContain("ls-remote"); + }); + + it("re-attaches a lost checkout to its existing branch without fetching", async () => { + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + git(first.cwd, "commit", "-q", "--allow-empty", "-m", "the run's own commit"); + const head = git(first.cwd, "rev-parse", "HEAD"); + rmSync(first.cwd, { recursive: true, force: true }); + remote.commit("main", "later.txt", "later"); + spawned.mockClear(); + + const again = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(git(again.cwd, "rev-parse", "HEAD")).toBe(head); + expect(gitCommands()).not.toContain("fetch"); + expect(gitCommands()).not.toContain("ls-remote"); + }); + + it("refuses a checkout that is on another branch rather than resetting it", async () => { + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + git(first.cwd, "checkout", "-q", "-b", "elsewhere"); + + await expect( + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + ).rejects.toThrow(/elsewhere/); + expect(git(first.cwd, "rev-parse", "--abbrev-ref", "HEAD")).toBe("elsewhere"); + }); +}); + +describe("project/ sits beside the checkout, never inside it", () => { + function withFiles(seed: Record = {}) { + const collection = createFakeCollection("project-files/**", seed); + return { + collection, + source: { + kind: "repo" as const, + repo: remote.url, + projectId: "storefront", + files: { collection, collectionId: "project-files" }, + }, + }; + } + + it("hydrates the run's own keys into project/ and saves a note back, invisible to git", async () => { + const { collection, source } = withFiles({ "storefront/plan.md": "the plan", "other/secret.md": "not yours" }); + const h = host(); + + const place = await h.provision(source, { place: ["run-1"], branch: "fsd/run-1" }); + const projectDir = join(root, "run-1", "project"); + + expect(place.filesDir).toBe(projectDir); + expect(readFileSync(join(projectDir, "plan.md"), "utf8")).toBe("the plan"); + expect(existsSync(join(projectDir, "secret.md"))).toBe(false); + + writeFileSync(join(projectDir, "note.md"), "remember this"); + writeFileSync(join(place.cwd, "code.ts"), "export {}"); + const report = await h.save(place); + + expect(report.conflicts).toEqual([]); + expect(collection.contents()["storefront/note.md"]).toBe("remember this"); + expect(Object.keys(collection.contents()).some((k) => k.endsWith("code.ts"))).toBe(false); + expect(git(place.cwd, "status", "--porcelain")).toBe("?? code.ts"); + }); + + it("refuses kept files without the key prefix they live under", async () => { + const { collection } = withFiles(); + await expect( + host().provision( + { kind: "repo", repo: remote.url, files: { collection, collectionId: "project-files" } }, + { place: ["run-1"], branch: "fsd/run-1" }, + ), + ).rejects.toThrow(/projectId/); + expect(spawned).not.toHaveBeenCalled(); + }); +}); + +describe("a source's refusal stops provisioning", () => { + it("refuses with the source's own reason, starting nothing", async () => { + const attempt = host().provision( + { kind: "refused", reason: "not-a-member", message: "the run's owner is not a member" }, + { place: ["run-1"], branch: "fsd/run-1" }, + ); + + const error = await refusalOf(attempt); + expect(error).toBeInstanceOf(WorkspaceRefusedError); + expect(error.reason).toBe("not-a-member"); + expect(spawned).not.toHaveBeenCalled(); + expect(existsSync(join(root, "run-1"))).toBe(false); + }); +}); + +describe("a place is named inside the root", () => { + for (const segments of [[], [".."], ["a", "../b"], [".clones"], ["a/b"], [""]]) { + it(`refuses ${JSON.stringify(segments)}`, async () => { + await expect( + host().provision({ kind: "repo", repo: remote.url }, { place: segments, branch: "fsd/run-1" }), + ).rejects.toThrow(/place/); + expect(spawned).not.toHaveBeenCalled(); + }); + } +}); diff --git a/packages/workspace/test/remotes.spec.ts b/packages/workspace/test/remotes.spec.ts new file mode 100644 index 0000000000..bf35c2610e --- /dev/null +++ b/packages/workspace/test/remotes.spec.ts @@ -0,0 +1,59 @@ +/** + * Judging a remote, as data, before git ever sees it. + */ +import { describe, expect, it } from "vitest"; +import { allowedProtocols, checkRemote, redactRemote } from "../src/remotes"; + +const allow = ["github.com", "file"]; +const key = (remote: string) => { + const judged = checkRemote(remote, allow); + if ("reason" in judged) throw new Error(judged.message); + return judged.cloneKey; +}; + +describe("one repository, one clone, however it is spelled", () => { + it("https with and without .git, and both ssh spellings, share a clone", () => { + const spellings = [ + "https://github.com/acme/storefront", + "https://github.com/acme/storefront.git", + "https://GitHub.com/acme/storefront.git/", + "git@github.com:acme/storefront.git", + "ssh://git@github.com/acme/storefront.git", + ]; + expect(new Set(spellings.map(key)).size).toBe(1); + }); + + it("two repositories never share one", () => { + expect(key("https://github.com/acme/storefront")).not.toBe(key("https://github.com/acme/storefront-v2")); + expect(key("https://github.com/acme/a-b")).not.toBe(key("https://github.com/acme/a/b")); + }); + + it("a file:// path keeps its .git, because x and x.git are two directories", () => { + expect(key("file:///srv/x")).not.toBe(key("file:///srv/x.git")); + }); + + it("names a clone with nothing a directory cannot hold", () => { + expect(key("ssh://git@github.com/acme/../storefront")).toMatch(/^[A-Za-z0-9._-]+$/); + }); +}); + +describe("what the allowlist lets git speak", () => { + it("network hosts get https and ssh; file only when listed", () => { + expect(allowedProtocols(["github.com"])).toBe("https:ssh"); + expect(allowedProtocols(["github.com", "file"])).toBe("https:ssh:file"); + expect(allowedProtocols(["file"])).toBe("file"); + expect(allowedProtocols([])).toBe(""); + }); + + it("an ssh remote's login name is fine; a password is not", () => { + expect(checkRemote("ssh://git@github.com/acme/x.git", allow)).not.toHaveProperty("reason"); + expect(checkRemote("ssh://git:pw@github.com/acme/x.git", allow)).toMatchObject({ reason: "invalid-remote" }); + }); +}); + +describe("a credential is never shown", () => { + it("drops userinfo from a URL", () => { + expect(redactRemote("https://someone:hunter2@github.com/acme/x.git")).toBe("https://github.com/acme/x.git"); + expect(redactRemote("git@github.com:acme/x.git")).toBe("git@github.com:acme/x.git"); + }); +}); From 72c01f88428286ed2557bf4a1239aee80e836cd4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:25:58 +0000 Subject: [PATCH 3/5] fix(workspace): check the key prefix first and re-check a checkout inside the clone queue (FIX-1762) A bad projectId is refused before any checkout or git process exists, two provisions of one place at once share one checkout instead of the second failing in worktree add, and the clone identity check compares real paths. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P9UuEKLJUWaKiSrYWCJbt5 --- packages/workspace/src/local-host.ts | 22 ++++++++---- packages/workspace/src/projection.ts | 2 +- .../workspace/test/local-host-repo.spec.ts | 34 +++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/packages/workspace/src/local-host.ts b/packages/workspace/src/local-host.ts index 46cda56f1d..c96ba66827 100644 --- a/packages/workspace/src/local-host.ts +++ b/packages/workspace/src/local-host.ts @@ -33,11 +33,11 @@ * the seam is here so that slice adds behaviour rather than a method. */ import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, realpathSync, renameSync, rmSync } from "node:fs"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { GIT_TIMEOUT_MS, run } from "./exec"; import { createHostPlace } from "./host-place"; -import { createProjection, type Projection } from "./projection"; +import { assertScope, createProjection, type Projection } from "./projection"; import { allowedProtocols, checkRemote, redactRemote, type AllowedRemote } from "./remotes"; import type { RunFiles, RunSource, RunSourceAnswer } from "./run-source"; import type { FlushReport } from "./types"; @@ -319,7 +319,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac // A checkout already here is the last attempt's work. It is handed back // as it is, or refused — never fetched, reset or rebased. - if (existsSync(checkout)) { + const handBack = async (): Promise> => { if (!existsSync(join(checkout, ".git"))) { throw new Error( `the checkout at ${checkout} has no .git, so it is not one this host made. It may ` + @@ -333,17 +333,23 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac `use it, and nothing here resets a tree — restore the branch or remove the checkout.`, ); } - const common = resolve(checkout, await git(checkout, ["rev-parse", "--git-common-dir"])); - if (common !== clone) { + // Compared as real paths: git answers with one, and a root under a + // symlinked directory would otherwise never match. + const common = realpathSync(resolve(checkout, await git(checkout, ["rev-parse", "--git-common-dir"]))); + if (!existsSync(clone) || common !== realpathSync(clone)) { throw new Error( `the checkout at ${checkout} is not a worktree of the clone of "${shown}". Refusing ` + `to hand it to a run that expects that repository; it may hold work, so it is kept.`, ); } return { remote: remote.url, clone, branch, created: false }; - } + }; + if (existsSync(checkout)) return await handBack(); return await serially(remote.cloneKey, async () => { + // Another provision of this same place may have finished while this + // one waited its turn. + if (existsSync(checkout)) return await handBack(); const made = await ensureClone(remote, clone); // Bookkeeping for a worktree whose directory was removed, which would // otherwise make `worktree add` refuse the path. Not a reset: it @@ -379,11 +385,15 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac const dir = placeDir(request.place); if (answer.kind === "files") { + assertScope(answer.projectId); const filesDir = join(dir, WORKSPACE_DIR); await provisionFiles(dir, WORKSPACE_DIR, answer.projectId, answer.files); return { kind: "files", dir, cwd: filesDir, filesDir }; } + // The key prefix is checked before anything is made, so a bad one never + // leaves a checkout behind with no kept files beside it. + if (answer.projectId !== undefined) assertScope(answer.projectId); if ((answer.projectId === undefined) !== (answer.files === undefined)) { throw new Error( "a repository source names its kept files with both projectId and files, or neither — " + diff --git a/packages/workspace/src/projection.ts b/packages/workspace/src/projection.ts index 5b37985c06..62682e1748 100644 --- a/packages/workspace/src/projection.ts +++ b/packages/workspace/src/projection.ts @@ -488,7 +488,7 @@ export function createProjection({ * walks up, an empty segment or a stray `/` spells a different key than the * one compared against. */ -function assertScope(scope: string): void { +export function assertScope(scope: string): void { const segments = scope.split("/"); if (scope === "" || segments.some((s) => s === "" || s === "." || s === ".." || s.includes("\\"))) { throw new Error( diff --git a/packages/workspace/test/local-host-repo.spec.ts b/packages/workspace/test/local-host-repo.spec.ts index 4a904728db..c8b0f7a73c 100644 --- a/packages/workspace/test/local-host-repo.spec.ts +++ b/packages/workspace/test/local-host-repo.spec.ts @@ -227,6 +227,28 @@ describe("a retry keeps its checkout and never fetches", () => { expect(gitCommands()).not.toContain("ls-remote"); }); + it("gives two provisions of one place, at once, the one checkout", async () => { + const h = host(); + const [a, b] = await Promise.all([ + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + ]); + + expect(a.cwd).toBe(b.cwd); + expect([a.repo?.created, b.repo?.created].sort()).toEqual([false, true]); + }); + + it("refuses a checkout cut from another remote, and keeps it", async () => { + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + const other = createRemote("marker B"); + + await expect( + h.provision({ kind: "repo", repo: other.url }, { place: ["run-1"], branch: "fsd/run-1" }), + ).rejects.toThrow(/not a worktree of the clone/); + expect(readFileSync(join(first.cwd, "marker.txt"), "utf8")).toBe("marker A"); + }); + it("refuses a checkout that is on another branch rather than resetting it", async () => { const h = host(); const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); @@ -274,6 +296,18 @@ describe("project/ sits beside the checkout, never inside it", () => { expect(git(place.cwd, "status", "--porcelain")).toBe("?? code.ts"); }); + it("refuses a key prefix that could reach outside itself, before anything is made", async () => { + const { collection } = withFiles(); + await expect( + host().provision( + { kind: "repo", repo: remote.url, projectId: "../other", files: { collection, collectionId: "project-files" } }, + { place: ["run-1"], branch: "fsd/run-1" }, + ), + ).rejects.toThrow(/scope/); + expect(spawned).not.toHaveBeenCalled(); + expect(existsSync(join(root, "run-1"))).toBe(false); + }); + it("refuses kept files without the key prefix they live under", async () => { const { collection } = withFiles(); await expect( From 3f938256ec93d49a2540956e81bcde5eb44b21ba Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:37:43 +0000 Subject: [PATCH 4/5] fix(workspace): redact all userinfo, key clones by port, refuse malformed file escapes (FIX-1762) - redactRemote strips userinfo through the last @ before the path, so a credential holding an @ is never printed in a refusal. - A non-default port is part of the clone identity; :22 (ssh) and :443 (https) stay the same clone as no port. - A file:// path with a malformed %-escape is an invalid-remote refusal rather than a thrown URIError. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P9UuEKLJUWaKiSrYWCJbt5 --- packages/workspace/src/remotes.ts | 25 ++++++++++++++++---- packages/workspace/test/remotes.spec.ts | 31 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/workspace/src/remotes.ts b/packages/workspace/src/remotes.ts index 413fa6ba3c..bab742c565 100644 --- a/packages/workspace/src/remotes.ts +++ b/packages/workspace/src/remotes.ts @@ -51,9 +51,13 @@ export function allowedProtocols(allow: readonly string[]): string { /** * The remote as a person may be shown it: any userinfo removed, so a refusal * that names the remote never prints a credential. + * + * The userinfo runs to the LAST `@` before the path, as a URL parser reads + * it — a credential may itself hold an `@`, and stopping at the first one + * would print the rest of it. */ export function redactRemote(remote: string): string { - return remote.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/@]*@/i, "$1"); + return remote.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#]*@/i, "$1"); } /** Judge `remote` against `allow`. Pure: starts nothing, reads nothing. */ @@ -90,7 +94,12 @@ export function checkRemote(remote: string, allow: readonly string[]): AllowedRe if (scheme === "file") { if (!allow.includes("file")) return notAllowed("file:// remotes are off unless the host lists \"file\""); if (url.hostname !== "" && url.hostname !== "localhost") return invalid("a file:// remote names no host"); - const path = decodeURIComponent(url.pathname); + let path: string; + try { + path = decodeURIComponent(url.pathname); + } catch { + return invalid("its path holds a malformed %-escape"); + } return { url: remote, scheme: "file", cloneKey: cloneKeyFor("file", path) }; } if (scheme !== "https" && scheme !== "ssh") { @@ -100,7 +109,13 @@ export function checkRemote(remote: string, allow: readonly string[]): AllowedRe const host = url.hostname.toLowerCase(); if (host === "" || host.startsWith("-")) return invalid("its host is missing or starts with \"-\""); if (!allow.some((entry) => entry.toLowerCase() === host)) return notAllowed(`"${host}" is not a listed host`); - return { url: remote, scheme, cloneKey: cloneKeyFor(host, repositoryPath(url.pathname)) }; + // A non-default port is another server, which can hold an unrelated + // repository at the same path. The URL parser already drops `:443` from + // https; `:22` is ssh's default and is dropped here, so both spellings of + // the default stay one clone with the scp-like form. + const port = scheme === "ssh" && url.port === "22" ? "" : url.port; + const authority = port === "" ? host : `${host}:${port}`; + return { url: remote, scheme, cloneKey: cloneKeyFor(authority, repositoryPath(url.pathname)) }; } // Git's scp-like form, `[user@]host:path`, is ssh — but only when the colon @@ -131,8 +146,8 @@ function repositoryPath(path: string): string { * A readable, directory-safe clone name with a hash of the identity on the * end, so two identities that sanitize alike still get two clones. */ -function cloneKeyFor(host: string, path: string): string { - const identity = `${host}/${path}`; +function cloneKeyFor(authority: string, path: string): string { + const identity = `${authority}/${path}`; const readable = identity.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[-.]+/, "").slice(0, 80); const hash = createHash("sha256").update(identity).digest("hex").slice(0, 12); return `${readable}-${hash}`; diff --git a/packages/workspace/test/remotes.spec.ts b/packages/workspace/test/remotes.spec.ts index bf35c2610e..495ddb9c3b 100644 --- a/packages/workspace/test/remotes.spec.ts +++ b/packages/workspace/test/remotes.spec.ts @@ -32,6 +32,18 @@ describe("one repository, one clone, however it is spelled", () => { expect(key("file:///srv/x")).not.toBe(key("file:///srv/x.git")); }); + it("a non-default port is another server, so another clone", () => { + // The same path on two ports can be two unrelated repositories; sharing a + // clone would reattach one's branch to the other's refs. + expect(key("ssh://git@github.com:2222/acme/storefront.git")).not.toBe(key("ssh://git@github.com/acme/storefront.git")); + expect(key("https://github.com:8443/acme/storefront")).not.toBe(key("https://github.com/acme/storefront")); + }); + + it("a default port is the same server as no port", () => { + expect(key("ssh://git@github.com:22/acme/storefront.git")).toBe(key("git@github.com:acme/storefront.git")); + expect(key("https://github.com:443/acme/storefront")).toBe(key("https://github.com/acme/storefront")); + }); + it("names a clone with nothing a directory cannot hold", () => { expect(key("ssh://git@github.com/acme/../storefront")).toMatch(/^[A-Za-z0-9._-]+$/); }); @@ -49,6 +61,12 @@ describe("what the allowlist lets git speak", () => { expect(checkRemote("ssh://git@github.com/acme/x.git", allow)).not.toHaveProperty("reason"); expect(checkRemote("ssh://git:pw@github.com/acme/x.git", allow)).toMatchObject({ reason: "invalid-remote" }); }); + + it("a malformed escape in a file:// path is an invalid remote, not a crash", () => { + // A thrown URIError would escape the refusal path, and a caller could not + // fail the run with a stable reason. + expect(checkRemote("file:///tmp/%ZZ", allow)).toMatchObject({ reason: "invalid-remote" }); + }); }); describe("a credential is never shown", () => { @@ -56,4 +74,17 @@ describe("a credential is never shown", () => { expect(redactRemote("https://someone:hunter2@github.com/acme/x.git")).toBe("https://github.com/acme/x.git"); expect(redactRemote("git@github.com:acme/x.git")).toBe("git@github.com:acme/x.git"); }); + + it("drops all of the userinfo when the credential itself holds an @", () => { + // A URL parser splits the authority at the LAST @, so everything before it + // is credential. + expect(redactRemote("https://user:@hunter2@github.com/x")).toBe("https://github.com/x"); + expect(redactRemote("ssh://git:p@ss@github.com/x")).toBe("ssh://github.com/x"); + }); + + it("a refusal never prints a credential that holds an @", () => { + const judged = checkRemote("https://user:@hunter2@github.com/x", allow); + expect(judged).toHaveProperty("reason"); + expect((judged as { message: string }).message).not.toContain("hunter2"); + }); }); From e66372b45ff57bb52c533a13fd454afc12a27425 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:44:10 +0000 Subject: [PATCH 5/5] fix(workspace): rebuild a cut-short checkout and lock provisioning across hosts (FIX-1762) - A place is marked while `git worktree add` runs. A marked checkout whose tree only has tracked files missing (or no .git) is a killed add and is rebuilt; a marked tree with any other change is refused and kept. Ported from harness-manager's looksHalfBuilt / everyChangeIsADeletion. - Provisioning a place and working on a clone take a lock file on disk (src/lock.ts, ported from harness-manager's checkout lease), so two hosts or processes sharing a root wait instead of racing `worktree add -b`. The holder refreshes the lock; one left by a dead process is taken once stale. The in-process queue stays in front of it. Lock and marker names start with "." so no place segment can name over them. The place lock also serialises the kept-files hydrate. - hasBranch reads only git's "no such ref" exit as false and raises any other failure. - Real-git host tests get a 30s per-test timeout for monorepo-wide runs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01P9UuEKLJUWaKiSrYWCJbt5 --- packages/workspace/README.md | 6 +- packages/workspace/src/local-host.ts | 157 +++++++++++++++--- packages/workspace/src/lock.ts | 129 ++++++++++++++ .../workspace/test/local-host-repo.spec.ts | 132 +++++++++++++++ packages/workspace/test/lock.spec.ts | 88 ++++++++++ 5 files changed, 486 insertions(+), 26 deletions(-) create mode 100644 packages/workspace/src/lock.ts create mode 100644 packages/workspace/test/lock.spec.ts diff --git a/packages/workspace/README.md b/packages/workspace/README.md index c5b89e56aa..92d99aedfa 100644 --- a/packages/workspace/README.md +++ b/packages/workspace/README.md @@ -235,9 +235,11 @@ await host.release(place); workspace/ the kept files as the working directory, when there is no repository ``` -For a repository, the host keeps one clone of each remote under `root` and gives every run its own branch in `checkout/`. `https://github.com/acme/x`, `https://github.com/acme/x.git` and `git@github.com:acme/x.git` share one clone. Before it cuts a new branch, the host fetches and reads the remote's default branch again, so a renamed `main` is followed. `place.repo` says which clone, branch, base branch and commit the run got. Record the remote and base on your run, and log the clone path so whoever runs the host can find it. +For a repository, the host keeps one clone of each remote under `root` and gives every run its own branch in `checkout/`. `https://github.com/acme/x`, `https://github.com/acme/x.git` and `git@github.com:acme/x.git` share one clone. A non-default port, like `ssh://github.com:2222/acme/x`, is another server and gets its own. Before it cuts a new branch, the host fetches and reads the remote's default branch again, so a renamed `main` is followed. `place.repo` says which clone, branch, base branch and commit the run got. Record the remote and base on your run, and log the clone path so whoever runs the host can find it. -A checkout that already exists is handed back exactly as it was left. The host doesn't fetch, reset or rebase it, so a retry finds its uncommitted work. A checkout it can't explain, such as one on a different branch or with no `.git`, is refused and left alone. +A checkout that already exists is handed back exactly as it was left. The host doesn't fetch, reset or rebase it, so a retry finds its uncommitted work. A checkout it can't explain, such as one on a different branch or with no `.git`, is refused and left alone. The one exception is a checkout whose `git worktree add` was killed before it finished: the host marks the place while that command runs, and when the mark is still there and the tree only has files missing, it rebuilds the checkout instead of handing a run a tree with half the repository gone. If the tree shows any other change, the host refuses it and keeps it. + +Several hosts, in one process or several, can share a `root`. Provisioning a place, and working on a clone, take a lock file under the root, so a second host waits for the first instead of racing it. A lock left by a process that died is taken over after a minute. Kept files go in `project/` next to the checkout, never inside it, so `git status` never sees them. With no repository they go in `workspace/`, which is also the run's working directory. Either way they're hydrated from one key prefix of the collection and flushed back by `save`, with the same conflict reporting as any projection. The first run in an empty prefix starts from an empty directory. A retry in the same process gets the same live directory back, unsaved edits included. A directory the process didn't fill itself, after a restart or because it was deleted, is rebuilt from the collection. The collection is the record, so anything a run never saved is gone. diff --git a/packages/workspace/src/local-host.ts b/packages/workspace/src/local-host.ts index c96ba66827..e98ef20975 100644 --- a/packages/workspace/src/local-host.ts +++ b/packages/workspace/src/local-host.ts @@ -7,7 +7,10 @@ * * ``` * /.clones/.git one bare clone per remote, shared by every run + * /.clones/.lock held while a provision works on that clone * // + * .lock held while a provision works on this place + * .checkout.provisioning present while `git worktree add` runs * checkout/ a git worktree on the run's own branch (a repository run) * project/ the kept files, beside the checkout and never inside it * workspace/ the kept files as the working directory (a run with no repository) @@ -28,15 +31,21 @@ * back to it. A directory this process did not hydrate is rebuilt from the * collection; one it did is handed back live, with its unsaved edits. * + * Provisioning one place, or touching one clone, is held by a lock on disk + * (`./lock`), so two hosts or two processes sharing a root wait for each + * other rather than racing. The lock and marker names start with `.`, and a + * place segment may not, so no place is ever named over them. + * * Checkpoint and restore are declared and do nothing yet: holding a * repository's uncommitted work across a lost machine is a later slice, and * the seam is here so that slice adds behaviour rather than a method. */ import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, realpathSync, renameSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { GIT_TIMEOUT_MS, run } from "./exec"; import { createHostPlace } from "./host-place"; +import { acquireLock, releaseLock } from "./lock"; import { assertScope, createProjection, type Projection } from "./projection"; import { allowedProtocols, checkRemote, redactRemote, type AllowedRemote } from "./remotes"; import type { RunFiles, RunSource, RunSourceAnswer } from "./run-source"; @@ -51,6 +60,14 @@ const WORKSPACE_DIR = "workspace"; * start with a dot, so no place can ever be named over it. */ const CLONES_DIR = ".clones"; +/** The lock file in a place's directory, and beside each clone. */ +const LOCK_SUFFIX = ".lock"; +/** + * Present in a place's directory from just before `git worktree add` until it + * returns, so a checkout a killed `add` left half-built is identified rather + * than handed to a run. + */ +const PROVISIONING_MARKER = ".checkout.provisioning"; /** * The host would not provision this run, and says why — before anything was @@ -166,7 +183,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac /** Live kept-files projections, by the directory they fill. */ const live = new Map>(); - /** The tail of each clone's queue: one git operation on a clone at a time, in this process. */ + /** The tail of each place's and each clone's queue, in this process. */ const queues = new Map>(); async function git(cwd: string, args: string[]): Promise { @@ -175,12 +192,9 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac } /** - * Run `operation` after every earlier one on the same clone has settled. - * - * This is the lock "one clone per remote" rests on in this process: two - * first runs for one remote queue here, the first makes the clone, the - * second finds it. Across processes the clone appears by an atomic rename, - * so a second process sees either no clone or a whole one. + * Run `operation` after every earlier one under the same key has settled. + * Queues this process's own callers, so they wait without polling the lock + * on disk. */ function serially(key: string, operation: () => Promise): Promise { const previous = queues.get(key) ?? Promise.resolve(); @@ -194,6 +208,21 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac return next; } + /** + * Run `operation` holding the lock at `lockPath`: queued behind this + * process's earlier callers, then behind any other host's on disk. + */ + function locked(lockPath: string, operation: () => Promise): Promise { + return serially(lockPath, async () => { + const lease = await acquireLock(lockPath); + try { + return await operation(); + } finally { + releaseLock(lease); + } + }); + } + /** The place's directory, refusing any name that could leave the root or shadow the clones. */ function placeDir(segments: readonly string[]): string { const bad = segments.find( @@ -252,11 +281,39 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac return made; } - /** `true` when the clone holds `refs/heads/`. Local; never touches the remote. */ + /** + * `true` when the clone holds `refs/heads/`. Local; never touches the + * remote. Only git's "no such ref" answer is `false`; any other failure is + * raised, so a clone git cannot read is not taken for one without the branch. + */ async function hasBranch(clone: string, branch: string): Promise { try { await git(clone, ["show-ref", "--verify", "--quiet", `refs/heads/${branch}`]); return true; + } catch (error) { + if (gitAnsweredNo(error)) return false; + throw error; + } + } + + /** + * Does this checkout look like a `git worktree add` that was killed + * part-way, with nothing worked in it since? Ported from harness-manager's + * `looksHalfBuilt`. + * + * A killed `add` can leave `.git`, the branch and the worktree registration + * in place with tracked files missing. So: no `.git` at all, or tracked + * files missing and NOTHING else changed — no edit, no addition, no staged + * change. A missing file alone is not enough, because a run that deletes a + * file reads the same; anything an agent could have written means work, and + * work is never cleared. A tree git cannot answer about does not look + * half-built: unknown never authorises a delete. + */ + async function looksHalfBuilt(checkout: string): Promise { + if (!existsSync(join(checkout, ".git"))) return true; + try { + if ((await git(checkout, ["ls-files", "--deleted"])) === "") return false; + return everyChangeIsADeletion(await git(checkout, ["status", "--porcelain", "--untracked-files=all"])); } catch { return false; } @@ -307,6 +364,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac } } + /** Called holding the place's lock. */ async function provisionCheckout( remote: AllowedRemote, baseRef: string | undefined, @@ -315,6 +373,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac ): Promise> { const clone = join(root, CLONES_DIR, `${remote.cloneKey}.git`); const checkout = join(dir, CHECKOUT_DIR); + const marker = join(dir, PROVISIONING_MARKER); const shown = redactRemote(remote.url); // A checkout already here is the last attempt's work. It is handed back @@ -344,12 +403,35 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac } return { remote: remote.url, clone, branch, created: false }; }; - if (existsSync(checkout)) return await handBack(); - return await serially(remote.cloneKey, async () => { - // Another provision of this same place may have finished while this - // one waited its turn. - if (existsSync(checkout)) return await handBack(); + if (existsSync(checkout)) { + // The marker alone does not clear a tree: it sits where a run could + // write it. The tree has to agree that it is half-built, and when the + // two disagree the tree is kept and refused, not reused or cleared. + const marked = existsSync(marker); + if (!marked || !(await looksHalfBuilt(checkout))) { + if (marked && existsSync(join(checkout, ".git"))) { + throw new Error( + `${marker} records an interrupted provision, but the checkout at ${checkout} is ` + + `not a half-built one — it holds more than missing files. One of the two is ` + + `wrong and this will not guess: the tree may hold work. Inspect it, then delete ` + + `the marker to reuse the checkout or delete the checkout to have it rebuilt.`, + ); + } + return await handBack(); + } + // Never used: the provision that made it never returned it to a run. + rmSync(checkout, { recursive: true, force: true }); + } + + /** `git worktree add`, with the marker up for exactly as long as it runs. */ + const addWorktree = async (clone: string, args: string[]): Promise => { + writeFileSync(marker, ""); + await git(clone, ["worktree", "add", "--quiet", ...args]); + rmSync(marker, { force: true }); + }; + + return await locked(join(root, CLONES_DIR, `${remote.cloneKey}${LOCK_SUFFIX}`), async () => { const made = await ensureClone(remote, clone); // Bookkeeping for a worktree whose directory was removed, which would // otherwise make `worktree add` refuse the path. Not a reset: it @@ -360,7 +442,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac // not. Put the branch back where it was — without a fetch, so nothing // it started from moves. if (!made.fresh && (await hasBranch(clone, branch))) { - await git(clone, ["worktree", "add", "--quiet", checkout, branch]); + await addWorktree(clone, [checkout, branch]); return { remote: remote.url, clone, branch, created: false }; } @@ -375,7 +457,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac `the remote "${shown}" has no branch "${base}" to cut the run's branch from.`, ); } - await git(clone, ["worktree", "add", "--quiet", "-b", branch, checkout, baseCommit]); + await addWorktree(clone, ["-b", branch, checkout, baseCommit]); return { remote: remote.url, clone, branch, created: true, baseRef: base, baseCommit }; }); } @@ -387,7 +469,7 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac if (answer.kind === "files") { assertScope(answer.projectId); const filesDir = join(dir, WORKSPACE_DIR); - await provisionFiles(dir, WORKSPACE_DIR, answer.projectId, answer.files); + await locked(join(dir, LOCK_SUFFIX), () => provisionFiles(dir, WORKSPACE_DIR, answer.projectId, answer.files)); return { kind: "files", dir, cwd: filesDir, filesDir }; } @@ -408,13 +490,15 @@ export function localWorkspaceHost(options: LocalWorkspaceHostOptions): Workspac const remote = checkRemote(answer.repo, allow); if ("reason" in remote) throw new WorkspaceRefusedError(remote.reason, remote.message); - const repo = await provisionCheckout(remote, answer.baseRef, dir, branch); - const place: WorkspacePlace = { kind: "repo", dir, cwd: join(dir, CHECKOUT_DIR), repo }; - if (answer.projectId !== undefined && answer.files !== undefined) { - await provisionFiles(dir, PROJECT_DIR, answer.projectId, answer.files); - place.filesDir = join(dir, PROJECT_DIR); - } - return place; + return await locked(join(dir, LOCK_SUFFIX), async () => { + const repo = await provisionCheckout(remote, answer.baseRef, dir, branch); + const place: WorkspacePlace = { kind: "repo", dir, cwd: join(dir, CHECKOUT_DIR), repo }; + if (answer.projectId !== undefined && answer.files !== undefined) { + await provisionFiles(dir, PROJECT_DIR, answer.projectId, answer.files); + place.filesDir = join(dir, PROJECT_DIR); + } + return place; + }); } async function save(place: WorkspacePlace): Promise { @@ -456,6 +540,31 @@ function unreadable(remote: AllowedRemote, error: unknown): WorkspaceRefusedErro ); } +/** + * Did git answer **no**, or did the probe itself fail? Ported from + * harness-manager: a ref probe exits 1, unkilled, when the ref is absent; a + * timeout comes back `killed`, an unreadable repository exits 128, and a git + * that cannot be spawned carries a string `code`. + */ +function gitAnsweredNo(error: unknown): boolean { + const { code, killed } = (error ?? {}) as { code?: unknown; killed?: unknown }; + return killed !== true && code === 1; +} + +/** + * Whether every entry of `git status --porcelain` is a deletion — files + * removed, nothing added, edited, renamed or staged. Ported from + * harness-manager's `everyChangeIsADeletion`. + */ +function everyChangeIsADeletion(porcelain: string): boolean { + for (const line of porcelain.split("\n")) { + if (line.length === 0) continue; + if (line[0] !== " " && line[0] !== "D") return false; + if (line[1] !== " " && line[1] !== "D") return false; + } + return true; +} + /** * A branch name this host will hand to git: nothing git would read as an * option, and none of the spellings a ref refuses. Git checks the rest. diff --git a/packages/workspace/src/lock.ts b/packages/workspace/src/lock.ts new file mode 100644 index 0000000000..32ca4d4775 --- /dev/null +++ b/packages/workspace/src/lock.ts @@ -0,0 +1,129 @@ +/** + * A lock on disk, so that two hosts — in one process or in two — that share a + * root never provision one place or one clone at the same time. + * + * Ported from harness-manager's checkout lease (`acquireCheckout` / + * `releaseCheckout`), with one change: the holder REFRESHES the lock while it + * works. Harness-manager can size its stale bound past a run's whole deadline; + * a host's provision has no such deadline (it spans several git calls, each + * bounded on its own, and a hydrate), so instead a lock is stale once its + * holder stopped refreshing it — which is what a process that died does. + * + * - **Acquiring is atomic**: an `O_EXCL` create either wins or does not. + * - **Contention waits rather than fails**, up to `waitMs`. Two provisions of + * one place are ordinary, and failing one would spend a run's attempt on a + * lock. + * - **A stale lock is taken only if it is still the one that was judged** + * (inode and bytes, re-read just before the unlink), and taking it only + * clears the path — the taker then competes for the `O_EXCL` create like + * any other waiter. + * - **Release removes only this acquisition's lock**, by the token written in + * it. Compare-then-unlink is two syscalls, not one; the window between them + * is the residual harness-manager documents, unchanged here. + */ +import { randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { GIT_TIMEOUT_MS } from "./exec"; + +/** How a lock is waited on and judged. */ +export interface LockBounds { + /** How long to wait for a live holder before failing. */ + waitMs: number; + /** How often to re-check a held lock. */ + pollMs: number; + /** A lock not refreshed for this long belongs to a holder that is gone. */ + staleAfterMs: number; + /** How often a holder refreshes its lock. Well under `staleAfterMs`. */ + heartbeatMs: number; +} + +/** + * The host's bounds. The wait outlasts a provision's git calls run back to + * back, each at its own timeout, so a waiter fails only on a holder that is + * wedged rather than slow. + */ +export const HOST_LOCK_BOUNDS: LockBounds = { + waitMs: 8 * GIT_TIMEOUT_MS, + pollMs: 50, + staleAfterMs: 60_000, + heartbeatMs: 10_000, +}; + +/** A held lock. Hand it to `releaseLock` on every exit. */ +export interface LockLease { + /** The lock file. */ + path: string; + /** Written into the lock file; unique to this acquisition. */ + token: string; + /** Stops the refresh. */ + timer: ReturnType; +} + +/** Wait for, and take, the lock at `path`. */ +export async function acquireLock(path: string, bounds: LockBounds = HOST_LOCK_BOUNDS): Promise { + mkdirSync(dirname(path), { recursive: true }); + const deadline = Date.now() + bounds.waitMs; + + for (;;) { + const token = randomUUID(); + try { + writeFileSync(path, JSON.stringify({ token, pid: process.pid, at: Date.now() }), { flag: "wx" }); + const timer = setInterval(() => refresh(path, token), bounds.heartbeatMs); + timer.unref(); + return { path, token, timer }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + + try { + const victim = statSync(path); + const held = readFileSync(path, "utf8"); + if (Date.now() - victim.mtimeMs > bounds.staleAfterMs) { + const current = statSync(path); + if (current.ino === victim.ino && readFileSync(path, "utf8") === held) rmSync(path, { force: true }); + continue; + } + } catch (error) { + // Released between the failed create and the read: try again. Anything + // else (a directory at the path, no permission) would fail the same way + // on every pass, so it is raised rather than spun on. + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + continue; + } + + if (Date.now() >= deadline) { + throw new Error( + `waited ${bounds.waitMs}ms for the lock at ${path} and it is still held, and still ` + + `refreshed by its holder. Treating that as a wedged provision rather than ordinary ` + + `contention; if no provision is running, remove the file.`, + ); + } + await new Promise((resolve) => setTimeout(resolve, Math.min(bounds.pollMs, deadline - Date.now()))); + } +} + +/** Release a lock this acquisition holds. Never removes another taker's. Idempotent. */ +export function releaseLock(lease: LockLease): void { + clearInterval(lease.timer); + if (holds(lease.path, lease.token)) rmSync(lease.path, { force: true }); +} + +/** Mark the lock live, if it is still this acquisition's. */ +function refresh(path: string, token: string): void { + if (!holds(path, token)) return; + try { + const now = new Date(); + utimesSync(path, now, now); + } catch { + // Gone between the read and the touch; nothing of ours to keep alive. + } +} + +function holds(path: string, token: string): boolean { + try { + return (JSON.parse(readFileSync(path, "utf8")) as { token?: unknown }).token === token; + } catch { + return false; + } +} diff --git a/packages/workspace/test/local-host-repo.spec.ts b/packages/workspace/test/local-host-repo.spec.ts index c8b0f7a73c..5c2a3be804 100644 --- a/packages/workspace/test/local-host-repo.spec.ts +++ b/packages/workspace/test/local-host-repo.spec.ts @@ -19,7 +19,14 @@ vi.mock("../src/exec", async (importOriginal) => { return { ...actual, run: vi.fn(actual.run) }; }); +// Every test here drives real git: twenty-odd processes for a fixture remote +// and a provision. Alone each test takes well under a second, but process +// spawns slow down sharply when the whole monorepo's suites run at once, and +// the 5s default then fails tests that are slow, not wrong. +vi.setConfig({ testTimeout: 30_000 }); + const spawned = vi.mocked(run); +const { run: realRun } = await vi.importActual("../src/exec"); /** The git subcommand of every process the host started. */ const gitCommands = () => spawned.mock.calls.map(([, args]) => args[0]); /** The error `attempt` rejected with; fails the test if it provisioned. */ @@ -39,6 +46,7 @@ beforeEach(() => { root = tempDir("host"); remote = createRemote("marker A"); spawned.mockClear(); + spawned.mockImplementation(realRun); }); afterEach(() => { @@ -227,6 +235,24 @@ describe("a retry keeps its checkout and never fetches", () => { expect(gitCommands()).not.toContain("ls-remote"); }); + it("raises a clone it cannot ask about a branch, rather than reading that as no branch", async () => { + // Read as "no branch", a failed probe sends the run to fetch and cut a + // branch that already exists. + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + rmSync(first.cwd, { recursive: true, force: true }); + spawned.mockClear(); + spawned.mockImplementation(async (file, args, options) => { + if (args[0] === "show-ref") throw Object.assign(new Error("fatal: the clone is unreadable"), { code: 128 }); + return realRun(file, args, options); + }); + + await expect( + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + ).rejects.toThrow(/unreadable/); + expect(gitCommands()).not.toContain("fetch"); + }); + it("gives two provisions of one place, at once, the one checkout", async () => { const h = host(); const [a, b] = await Promise.all([ @@ -238,6 +264,35 @@ describe("a retry keeps its checkout and never fetches", () => { expect([a.repo?.created, b.repo?.created].sort()).toEqual([false, true]); }); + it("makes a second host on the same root wait for the first, then hands it the checkout", async () => { + // Two processes sharing a root share no memory, so only a lock on disk + // keeps them from both reaching `worktree add -b` for one place. + let open!: () => void; + const gate = new Promise((resolve) => (open = resolve)); + let reached!: () => void; + const paused = new Promise((resolve) => (reached = resolve)); + let held = false; + spawned.mockImplementation(async (file, args, options) => { + if (!held && args[0] === "worktree" && args[1] === "add") { + held = true; + reached(); + await gate; + } + return realRun(file, args, options); + }); + + const first = host().provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + await paused; + const second = host().provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + // Long enough for an unlocked second host to run its whole provision. + await new Promise((resolve) => setTimeout(resolve, 500)); + open(); + + const [a, b] = await Promise.all([first, second]); + expect(a.cwd).toBe(b.cwd); + expect([a.repo?.created, b.repo?.created]).toEqual([true, false]); + }); + it("refuses a checkout cut from another remote, and keeps it", async () => { const h = host(); const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); @@ -261,6 +316,83 @@ describe("a retry keeps its checkout and never fetches", () => { }); }); +/** + * Stand in for the process timeout killing `git worktree add`: let git run, + * then leave the tree as `leave` says and reject the way a killed child does. + */ +function killWorktreeAdd(leave: (checkout: string) => void): void { + spawned.mockImplementation(async (file, args, options) => { + const result = await realRun(file, args, options); + if (args[0] === "worktree" && args[1] === "add") { + spawned.mockImplementation(realRun); + leave(join(root, "run-1", "checkout")); + throw Object.assign(new Error("git worktree add: killed"), { killed: true, signal: "SIGTERM" }); + } + return result; + }); +} + +describe("a provision cut short is rebuilt, not handed back", () => { + // Measured in harness-manager: a killed `worktree add` can leave `.git`, the + // branch and the worktree registration in place with tracked files missing. + // Handed back, the run's first `git add -A` commits every missing file as a + // deletion. + it("rebuilds a checkout whose worktree add was killed part-way", async () => { + const h = host(); + killWorktreeAdd((checkout) => rmSync(join(checkout, "marker.txt"))); + await expect(h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" })).rejects.toThrow( + /killed/, + ); + + const retry = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(readFileSync(join(retry.cwd, "marker.txt"), "utf8")).toBe("marker A"); + expect(git(retry.cwd, "status", "--porcelain")).toBe(""); + expect(git(retry.cwd, "rev-parse", "--abbrev-ref", "HEAD")).toBe("fsd/run-1"); + }); + + it("rebuilds one killed before git wrote .git", async () => { + const h = host(); + killWorktreeAdd((checkout) => rmSync(join(checkout, ".git"))); + await expect(h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" })).rejects.toThrow( + /killed/, + ); + + const retry = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(readFileSync(join(retry.cwd, "marker.txt"), "utf8")).toBe("marker A"); + expect(git(retry.cwd, "status", "--porcelain")).toBe(""); + }); + + it("hands back a checkout missing a tracked file when no provision was cut short — that is the run's work", async () => { + const h = host(); + const first = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + rmSync(join(first.cwd, "marker.txt")); + + const retry = await h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }); + + expect(git(retry.cwd, "status", "--porcelain")).toBe("D marker.txt"); + }); + + it("refuses, and keeps, a cut-short checkout that shows anything but missing files", async () => { + // The record of an interrupted provision sits where an agent could write + // it, so it only clears a tree that agrees: one with work in it is kept. + const h = host(); + killWorktreeAdd((checkout) => { + rmSync(join(checkout, "marker.txt")); + writeFileSync(join(checkout, "work.ts"), "real work"); + }); + await expect(h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" })).rejects.toThrow( + /killed/, + ); + + await expect( + h.provision({ kind: "repo", repo: remote.url }, { place: ["run-1"], branch: "fsd/run-1" }), + ).rejects.toThrow(/interrupted/); + expect(readFileSync(join(root, "run-1", "checkout", "work.ts"), "utf8")).toBe("real work"); + }); +}); + describe("project/ sits beside the checkout, never inside it", () => { function withFiles(seed: Record = {}) { const collection = createFakeCollection("project-files/**", seed); diff --git a/packages/workspace/test/lock.spec.ts b/packages/workspace/test/lock.spec.ts new file mode 100644 index 0000000000..ca892d124a --- /dev/null +++ b/packages/workspace/test/lock.spec.ts @@ -0,0 +1,88 @@ +/** + * The lock on disk that keeps two hosts — or two processes — sharing a root + * from provisioning one place or one clone at the same time. + */ +import { existsSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { acquireLock, releaseLock, type LockBounds } from "../src/lock"; +import { tempDir } from "./git-fixtures"; + +let dir: string; +let path: string; +const bounds: LockBounds = { waitMs: 2_000, pollMs: 10, staleAfterMs: 400, heartbeatMs: 50 }; +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +beforeEach(() => { + dir = tempDir("lock"); + path = join(dir, ".lock"); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe("one holder at a time", () => { + it("makes a second taker wait until the first releases", async () => { + const first = await acquireLock(path, bounds); + let secondHeld = false; + const second = acquireLock(path, bounds).then((lease) => { + secondHeld = true; + return lease; + }); + + await pause(100); + expect(secondHeld).toBe(false); + releaseLock(first); + releaseLock(await second); + expect(existsSync(path)).toBe(false); + }); + + it("keeps a live holder's lock past the stale bound, so it is never taken from under it", async () => { + // The holder refreshes the lock while it works, so only a holder that + // stopped — a process that died — ever looks stale. + const first = await acquireLock(path, bounds); + let secondHeld = false; + const second = acquireLock(path, bounds).then((lease) => { + secondHeld = true; + return lease; + }); + + await pause(bounds.staleAfterMs * 2); + expect(secondHeld).toBe(false); + releaseLock(first); + releaseLock(await second); + }); + + it("gives up after its wait bound rather than waiting forever", async () => { + const first = await acquireLock(path, bounds); + await expect(acquireLock(path, { ...bounds, waitMs: 100 })).rejects.toThrow(/still held/); + releaseLock(first); + }); +}); + +describe("a holder that died", () => { + it("has its lock taken once it is older than the stale bound", async () => { + writeFileSync(path, JSON.stringify({ token: "a process that died" })); + const old = (Date.now() - bounds.staleAfterMs * 2) / 1000; + utimesSync(path, old, old); + + const lease = await acquireLock(path, bounds); + expect(JSON.parse(readFileSync(path, "utf8")).token).toBe(lease.token); + releaseLock(lease); + }); +}); + +describe("release", () => { + it("never removes a lock another taker holds now", async () => { + // A holder whose lock was taken as stale must not remove its successor's. + const first = await acquireLock(path, bounds); + rmSync(path); + const second = await acquireLock(path, bounds); + + releaseLock(first); + expect(existsSync(path)).toBe(true); + releaseLock(second); + expect(existsSync(path)).toBe(false); + }); +});