From 9edbfa2bbd1f3881184d22581e114dcc97d9a5d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 20:04:43 +0000 Subject: [PATCH 1/2] fix(FIX-2090): end harness-manager's refused run through the board's recorder A refused harness run failed (claim board) or cancelled (fixed board) its own row from the manager's error handler, a worker writing an ending outside the claim gate and the drain (ER-1). The FIX-2087 closure sweep flags it at manager.ts:2600. The worker now says the failure is final instead of writing it: orchestration exports TaskEndsForGoodError, and the error recorder ends the row on that attempt under the worker's claim (fail with noRetry, or cancel when asked). HarnessRunRefused extends it with the ending its board needs, so outcomes are unchanged: errored on a board read from the claim, cancelled on the fixed board. Diagnosis: the issue's symptom (cascades, debts and notices skipped) does not reproduce on FIX-2080's branch. The collection's own fail/cancel already reached endingOf there, and a probe showed the refused row's dependent cancelled. What remained was the ER-1 violation itself, which the sweep is the check for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VaN4uTiYqiabCbsFLWM52A --- .changeset/task-ends-for-good.md | 8 +++ apps/docs/docs/orchestration/task-board.md | 8 +++ packages/harness-manager/src/manager.ts | 58 ++++++++--------- .../test/claim-board-refused.spec.ts | 13 ++++ packages/harness-manager/test/claim-host.ts | 2 + packages/orchestration/README.md | 8 +++ .../src/task-board/blocks/record-result.ts | 52 +++++++++++++--- .../orchestration/src/task-board/index.ts | 1 + .../tasks/collection/advisory-write-back.ts | 12 +++- .../record-error-ends-for-good.test.ts | 62 +++++++++++++++++++ 10 files changed, 180 insertions(+), 44 deletions(-) create mode 100644 .changeset/task-ends-for-good.md create mode 100644 packages/orchestration/test/task-board/record-error-ends-for-good.test.ts diff --git a/.changeset/task-ends-for-good.md b/.changeset/task-ends-for-good.md new file mode 100644 index 0000000000..4a7e3ef9ad --- /dev/null +++ b/.changeset/task-ends-for-good.md @@ -0,0 +1,8 @@ +--- +"@flow-state-dev/orchestration": minor +"@flow-state-dev/harness-manager": patch +--- + +A worker can now end its task on the current attempt without writing the ending itself: throw `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`), and the board's error recorder ends the row under the worker's claim, `errored` with no retry by default or `cancelled` with `{ ending: "cancelled" }`. + +Harness Manager's refused run uses it: the manager no longer fails or cancels its own row, and the board records the ending. The outcome is unchanged (`errored` on a board read from the claim, `cancelled` on the fixed board). `HarnessRunRefused` now extends `TaskEndsForGoodError`, and its constructor takes the ending as a second argument (FIX-2090). diff --git a/apps/docs/docs/orchestration/task-board.md b/apps/docs/docs/orchestration/task-board.md index ef7833c08b..447a6f40a5 100644 --- a/apps/docs/docs/orchestration/task-board.md +++ b/apps/docs/docs/orchestration/task-board.md @@ -704,6 +704,14 @@ waits. - `maxTotalRetries` (default `50`) — how many failure retries the board may authorize in total, across every task. See [Bounding the retries](#bounding-the-retries). - `maxIterations` — safety cap on how many times a single worker loops back to claim again, not a cap across the board. Default `10000`. +Some failures would come back the same on every attempt, such as a run refused because its task names nothing it may work on. Retrying those only spends the attempts. A worker that knows this throws `TaskEndsForGoodError`, exported from `@flow-state-dev/orchestration/task-board`. The board ends the task on that attempt, with the error's message as its reason. It ends `errored` by default, or `cancelled` if you construct the error with `{ ending: "cancelled" }`. Dependents and filed work are cascaded as for any other ending. + +```ts +import { TaskEndsForGoodError } from "@flow-state-dev/orchestration/task-board"; + +throw new TaskEndsForGoodError("the task names no project"); +``` + `onError` reaches an [assignee that dispatches](#assignees-that-dispatch) too, where there is no board run left to fail. `"fail"` fails the dispatch run that worker is running in, so it reports `failed`. `"skip"` leaves it reporting `completed`, with the error on the task as usual. See [What `status` tells you](../server/background-work#what-status-tells-you). A worker's result is not always the last word on its task. A coordinator can cancel the task while the worker runs. The worker can mark the task done itself partway through. The claim can expire and another worker can pick the task up. In each case the worker comes back with a result for a task that has already moved on. diff --git a/packages/harness-manager/src/manager.ts b/packages/harness-manager/src/manager.ts index daf3ff9d2a..97c9d6857a 100644 --- a/packages/harness-manager/src/manager.ts +++ b/packages/harness-manager/src/manager.ts @@ -48,7 +48,7 @@ import type { HarnessResolver, HarnessSessionHook, } from "@flow-state-dev/core/types"; -import { taskWorkerInputSchema } from "@flow-state-dev/orchestration/task-board"; +import { TaskEndsForGoodError, taskWorkerInputSchema } from "@flow-state-dev/orchestration/task-board"; import { getOrCreateTaskCollection, hasFrozenLedgerAssignee, @@ -755,14 +755,15 @@ export class HarnessAttemptFailed extends Error { * * Not a failed attempt. The same source and host give the same answer next * time, so a retry would spend every attempt on one refusal. The manager - * settles the row with this message as its reason, under the attempt's own - * claim, and the board's recorder then has nothing to re-queue: `cancelled` - * on the fixed board, `errored` on a board read from the claim, where the - * conversation that filed the row reports it. + * writes no ending of its own: this error tells the board's recorder to end + * the row on this attempt, with this message as its reason, under the + * attempt's own claim. It ends `cancelled` on the fixed board, and `errored` + * on a board read from the claim, where the conversation that filed the row + * reports it. */ -export class HarnessRunRefused extends Error { - constructor(message: string) { - super(message); +export class HarnessRunRefused extends TaskEndsForGoodError { + constructor(message: string, ending: "errored" | "cancelled") { + super(message, { ending }); this.name = "HarnessRunRefused"; } } @@ -884,6 +885,7 @@ function taskPayload( `[harness-manager] task ${input.taskId} names only part of a feature payload: ` + `${parsed.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}. ` + `Feature work names both an issue and a phase.`, + "errored", ); } return undefined; @@ -936,6 +938,16 @@ function harnessCtxState(ctx: { return ctx.sequencer?.state as ManagerState | undefined; } +/** + * How a refused run's row ends on this run's board. On a board read from the + * claim it fails for good: a cancelled row there reaches nobody, and an + * `errored` one is what the conversation that filed it reports. The fixed + * board keeps the cancel. + */ +function refusalEnding(ctx: { sequencer?: { state?: unknown } | undefined }): "errored" | "cancelled" { + return harnessCtxState(ctx)?.claim != null ? "errored" : "cancelled"; +} + /** Read the manager's state, or throw rather than derive a wrong directory. */ function managerState(state: ManagerState | undefined): ManagerState { if (state?.workspacePath == null) { @@ -1707,7 +1719,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo } catch (cause) { if (cause instanceof WorkspaceRefusedError) { if (holding) await recordPlace(ctx, before!, "refused").catch(() => undefined); - throw new HarnessRunRefused(refusedMessage("host", cause)); + throw new HarnessRunRefused(refusedMessage("host", cause), refusalEnding(ctx)); } if (cause instanceof HeldWorkMismatchError) await parkOnMismatch(ctx, cause, before); throw cause; @@ -1927,7 +1939,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo held: recorded?.held ?? null, }); const answer = pinnedAnswer(await host.source(ctx), pinned); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer)); + if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer), refusalEnding(ctx)); const workspacePath = host.locate(answer, { place: placeFor(location) }).cwd; await ctx.sequencer!.patchState({ workspacePath }); @@ -2124,7 +2136,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo state.branch, ); const answer = pinnedAnswer(await host.source(ctx), state); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer)); + if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer), refusalEnding(ctx)); if (host.locate(answer, { place: placeFor(location) }).cwd !== state.workspacePath) { throw new HarnessAttemptFailed( `[harness-manager] the run's workspace source changed between opening the run and ` + @@ -2583,27 +2595,9 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo { outcome: "failed", reason }, ); } - // **A refused run is settled, not retried** (see `HarnessRunRefused`). - // Under this attempt's own claim, so a superseded attempt cannot cancel - // a row its replacement holds. Best effort, like the write above: the - // error being unwound is what the caller gets either way. - // - // On a board read from the claim the row fails for good instead: a - // cancelled row there reaches nobody, and an `errored` one is what the - // conversation that filed it reports. The fixed board keeps the cancel. - if (error instanceof HarnessRunRefused && state?.taskId != null && state.attempt != null) { - try { - const tasks = await runBoard(ctx as BlockContext); - const row = tasks.get(state.taskId); - if (row !== undefined) { - const claim = { ...ticketForClaim(tasks.collectionId, row, tasks.partition), attempt: state.attempt }; - if (state.claim != null) await tasks.fail(state.taskId, reason, { claim, noRetry: true }); - else await tasks.cancel(state.taskId, reason, { claim }); - } - } catch { - // Left to the board's recorder, which re-queues it as before. - } - } + // **A refused run is settled, not retried** (see `HarnessRunRefused`), + // and not by this handler: the error it re-throws tells the board's + // recorder to end the row on this attempt, under this attempt's claim. throw error; }, }); diff --git a/packages/harness-manager/test/claim-board-refused.spec.ts b/packages/harness-manager/test/claim-board-refused.spec.ts index f173e88c3f..8fa086698f 100644 --- a/packages/harness-manager/test/claim-board-refused.spec.ts +++ b/packages/harness-manager/test/claim-board-refused.spec.ts @@ -50,6 +50,19 @@ describe("a refused run on a board read from the claim", () => { expect(row.error).toContain("the task names no project"); expect(refusing.seen).toHaveLength(0); }, 60_000); + + // A refused run ends the way any failed task ends, so a task waiting on it + // is not left pending for a dependency that will never complete. + it("cancels a task that depends on the refused row, as any failed task's dependent is", async () => { + const lab = claimHost({ mode: "claim" }); + await lab.seed({ ledgerId: CONVERSATIONS, partition: "conv-a", id: "t1", goal: "half a feature", input: { issue: "FIX-1" }, maxAttempts: 2 }); + await lab.seed({ ledgerId: CONVERSATIONS, partition: "conv-a", id: "t2", goal: "after it", input: {}, deps: ["t1"] }); + + await lab.runAttempt(CONVERSATIONS, "t1", "conv-a"); + + expect(await lab.row(CONVERSATIONS, "t1", "conv-a")).toMatchObject({ status: "errored", attempts: 1 }); + expect((await lab.row(CONVERSATIONS, "t2", "conv-a")).status).toBe("cancelled"); + }, 60_000); }); describe("a refused run on a fixed board", () => { diff --git a/packages/harness-manager/test/claim-host.ts b/packages/harness-manager/test/claim-host.ts index c0cd865b29..ef11e75777 100644 --- a/packages/harness-manager/test/claim-host.ts +++ b/packages/harness-manager/test/claim-host.ts @@ -227,6 +227,7 @@ export function claimHost(options: HostOptions) { goal: z.string(), input: z.unknown().optional(), maxAttempts: z.number().optional(), + deps: z.array(z.string()).optional(), }), outputSchema: z.unknown(), uses: [ledgerResources], @@ -239,6 +240,7 @@ export function claimHost(options: HostOptions) { goal: input.goal, assignee: ASSIGNEE, ...(input.input !== undefined ? { input: input.input } : {}), + ...(input.deps !== undefined ? { deps: input.deps } : {}), maxAttempts: input.maxAttempts ?? 2, }); } diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index 418b200b30..c7a4f9cefa 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -410,6 +410,14 @@ reports `terminationReason: "retry-budget-exhausted"` alongside `counts.retries` the limit in force. See the [Task board guide](https://flow-state.dev/docs/orchestration/task-board). +**A failure no retry can change.** A worker that knows the next attempt would end the +same way throws `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`) +instead of writing the ending itself. The board's error recorder then ends the row on +this attempt, under the worker's claim, with the error's message as its reason: +`errored` with attempts left unspent (`fail` with `noRetry`), or `cancelled` when +constructed with `{ ending: "cancelled" }`. Any other error takes the ordinary retry +route. + **When the board cannot record a result.** Saving a result is a commit followed by a change announcement, and the announcement can fail on its own. The board's two recorders correlate every result write (`beginTaskWrite` / `didWriteLand`, above) and diff --git a/packages/orchestration/src/task-board/blocks/record-result.ts b/packages/orchestration/src/task-board/blocks/record-result.ts index d7e11242c3..ec405e5bd1 100644 --- a/packages/orchestration/src/task-board/blocks/record-result.ts +++ b/packages/orchestration/src/task-board/blocks/record-result.ts @@ -13,7 +13,9 @@ * sequencer. Receives the caught error as input, reads * `currentClaim` from worker state, calls `collection.fail`. Honors * `onError`: `"skip"` swallows the error after writing the failure; - * `"fail"` rethrows so the parent forEach rejects. + * `"fail"` rethrows so the parent forEach rejects. A + * {@link TaskEndsForGoodError} ends the row on this attempt instead of + * re-queueing it: `fail` with `noRetry`, or `cancel`. * * Both write-backs are **advisory** (FIX-951): they pass `ifAllowed` and the * worker's `claim`, so a result that arrives after the task was settled by @@ -208,6 +210,27 @@ function readTaskQuietly( } } +/** + * A worker's failure that no retry can change: thrown by a worker that knows + * the next attempt would end the same way, such as a run refused before it + * started because it names nothing it may work on. + * + * The error recorder ends the row on this attempt rather than re-queueing it: + * `errored` for good, or `cancelled` when {@link ending} says so. The worker + * never writes the ending itself. The recorder does, under the worker's claim, + * so a superseded attempt's failure is declined like any other. + */ +export class TaskEndsForGoodError extends Error { + /** How the row ends: `errored` (the default) or `cancelled`. */ + readonly ending: "errored" | "cancelled"; + + constructor(message: string, options?: { ending?: "errored" | "cancelled"; cause?: unknown }) { + super(message, options?.cause !== undefined ? { cause: options.cause } : undefined); + this.name = "TaskEndsForGoodError"; + this.ending = options?.ending ?? "errored"; + } +} + function messageOf(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -494,6 +517,9 @@ export function createRecordError(options: RecordErrorOptions) { execute: async (error: unknown, ctx) => { const claim = ctx.sequencer!.state.currentClaim; const message = error instanceof Error ? error.message : String(error); + // A failure the worker says no retry can change ends the row on this + // attempt, written here rather than by the worker. + const forGood = error instanceof TaskEndsForGoodError ? error.ending : undefined; // A report that could not be DELIVERED is fatal wherever it happens, and // is never written to the row or swallowed by `onError`. The deferral @@ -557,12 +583,20 @@ export function createRecordError(options: RecordErrorOptions) { recorder: "fail", runId: wiring.runId, attempt: (write) => - advisoryFail(collection, claim.taskId, message, { - ifAllowed: true, - claim, - refuseWhenParked: true, - write, - }), + forGood === "cancelled" + ? collection.cancel(claim.taskId, message, { + ifAllowed: true, + claim, + refuseWhenParked: true, + write, + }) + : advisoryFail(collection, claim.taskId, message, { + ifAllowed: true, + claim, + refuseWhenParked: true, + write, + ...(forGood === "errored" ? { noRetry: true } : {}), + }), }); } await ctx.sequencer!.patchState({ currentClaim: undefined, currentLedger: undefined }); @@ -586,13 +620,13 @@ export function createRecordError(options: RecordErrorOptions) { // `onError`'s throw here instead would reject the fan-out and abandon // every task that had not started, which is the defect this branch // exists to remove. - return { recorded: "errored" as const, error: message }; + return { recorded: forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const), error: message }; } if (onError === "fail") { throw error instanceof Error ? error : new Error(message); } - return { recorded: "errored" as const, error: message }; + return { recorded: forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const), error: message }; }, }); } diff --git a/packages/orchestration/src/task-board/index.ts b/packages/orchestration/src/task-board/index.ts index 5f98113890..2f073ab1b6 100644 --- a/packages/orchestration/src/task-board/index.ts +++ b/packages/orchestration/src/task-board/index.ts @@ -210,6 +210,7 @@ export type { BuildWorkerStepOptions } from "./blocks/worker-step"; export { createRecordSuccess, createRecordError, + TaskEndsForGoodError, } from "./blocks/record-result"; export type { RecordSuccessOptions, diff --git a/packages/orchestration/src/tasks/collection/advisory-write-back.ts b/packages/orchestration/src/tasks/collection/advisory-write-back.ts index fd096c89e6..47adb553e2 100644 --- a/packages/orchestration/src/tasks/collection/advisory-write-back.ts +++ b/packages/orchestration/src/tasks/collection/advisory-write-back.ts @@ -99,6 +99,7 @@ import type { TaskChangeKind } from "./change-event"; import type { Task, TaskStatus } from "../schema/task"; import { routeFailure, sumGrantedRetries, transitionDeclineReason } from "./internal"; import type { + FailOptions, TaskCollectionRef, TaskTransitionOptions, TaskWriteOutcome, @@ -151,7 +152,7 @@ export async function advisoryFail( ref: TaskCollectionRef, taskId: string, error: string, - options: TaskTransitionOptions + options: FailOptions ): Promise { const before = observe(ref, taskId, "fail"); try { @@ -281,14 +282,19 @@ function readClock(ref: TaskCollectionRef): number { * * Runs only on the error path, so the retry-budget sum it may walk is never * paid by a write that succeeded. + * + * A `noRetry` fail skips the retry route in both backings, so it is headed + * for `errored` whatever attempts are left. */ function resolveTarget( ref: TaskCollectionRef, verb: "complete" | "fail", basis: Task, - grantedRetries: number | undefined + grantedRetries: number | undefined, + noRetry: boolean ): WriteTarget { if (verb === "complete") return { status: "completed", kind: "completed" }; + if (noRetry) return { status: "errored", kind: "errored" }; const routing = routeFailure( basis, () => grantedRetries ?? sumGrantedRetries(ref.list()), @@ -362,7 +368,7 @@ function attribute( // advisory decline at all. if (basis === undefined) return undefined; - const target = resolveTarget(ref, verb, basis, before.grantedRetries); + const target = resolveTarget(ref, verb, basis, before.grantedRetries, (options as FailOptions).noRetry === true); // (a) The task was already declinable when this call ran. // diff --git a/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts new file mode 100644 index 0000000000..2558d2be7b --- /dev/null +++ b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts @@ -0,0 +1,62 @@ +/** + * A worker that knows its failure is final says so by throwing + * `TaskEndsForGoodError`, and the error recorder ends the row on that attempt. + * + * Why it matters: only the claim gate or the drain records a row's ending + * (the FIX-2056 epic's ER-1). A worker that could not say "don't retry this" + * had two bad choices: let the recorder re-queue a run that will be refused + * the same way on every attempt, or write the ending itself and become a + * second ending writer. The marker removes both. + */ +import { describe, expect, it } from "vitest"; +import type { BlockContext } from "@flow-state-dev/core/types"; +import { runForTest } from "@flow-state-dev/testing"; +import { createRecordError, TaskEndsForGoodError } from "../../src/task-board/blocks/record-result"; +import { createStateBackedTaskCollection, ticketForClaim, type TaskCollectionRef } from "../../src/tasks"; +import { createFakeSequencerState } from "../helpers"; + +/** A claimed row with attempts to spare, and the recorder's context holding its claim. */ +async function claimedWithAttemptsLeft(): Promise<{ collection: TaskCollectionRef; ctx: BlockContext }> { + const state = createFakeSequencerState<{ tasks: Record }>({ tasks: {} }); + const collection = createStateBackedTaskCollection({ collectionId: "tasks", state }); + await collection.addTask({ id: "t", goal: "work", maxAttempts: 3 }); + const task = (await collection.claim("w"))!; + const bodyState = createFakeSequencerState<{ currentClaim?: unknown }>({ + currentClaim: ticketForClaim("tasks", task), + }); + return { collection, ctx: { sequencer: bodyState } as unknown as BlockContext }; +} + +function recorder(collection: TaskCollectionRef) { + return createRecordError({ name: "ends-for-good", collection: async () => collection, onError: "skip" }); +} + +describe("the error recorder and a failure no retry can change", () => { + it("fails the row for good on this attempt, with the worker's reason", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new TaskEndsForGoodError("refused: names no project"), ctx); + + expect(collection.get("t")).toMatchObject({ status: "errored", attempts: 1, error: "refused: names no project" }); + }); + + it("cancels the row instead when the worker asks for a cancel", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest( + recorder(collection), + new TaskEndsForGoodError("refused: remote not allowed", { ending: "cancelled" }), + ctx, + ); + + expect(collection.get("t")).toMatchObject({ status: "cancelled", attempts: 1 }); + }); + + it("still re-queues an ordinary failure with attempts left", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new Error("flaky"), ctx); + + expect(collection.get("t")?.status).toBe("pending"); + }); +}); From af0f1ce221d7dcba2e977ded378908a95f26bea5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 20:41:33 +0000 Subject: [PATCH 2/2] fix(FIX-2090): scope the for-good marker to its row; contain the cancel arm Review fixes on #3623: - TaskEndsForGoodError carries the taskId its thrower holds, and the error recorder acts on it only for that row. A board nested inside a worker rethrows its worker's error on onError "fail"; the inner row's verdict no longer ends the outer row on its first attempt. - The cancel arm goes through a new advisoryCancel, so a store that throws on a declined transition is contained as it is on fail. The fixed board's cancelled ending is kept. - releaseRow passes noRetry for a for-good row; observe skips the retry-budget scan on a noRetry fail; attribute takes FailOptions (no cast). - Changeset bumps swapped: harness-manager minor, orchestration patch. - Comments trimmed along the lines of POC #3626; the unused cause option goes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VaN4uTiYqiabCbsFLWM52A --- .changeset/task-ends-for-good.md | 8 +- apps/docs/docs/orchestration/task-board.md | 5 +- packages/harness-manager/src/manager.ts | 43 +++---- .../test/claim-board-refused.spec.ts | 2 - packages/orchestration/README.md | 11 +- .../src/task-board/blocks/record-result.ts | 52 ++++---- .../tasks/collection/advisory-write-back.ts | 57 ++++++--- .../collection/advisory-fail-no-retry.test.ts | 32 +++++ .../record-error-ends-for-good.test.ts | 116 ++++++++++++++++-- 9 files changed, 238 insertions(+), 88 deletions(-) create mode 100644 packages/orchestration/test/collection/advisory-fail-no-retry.test.ts diff --git a/.changeset/task-ends-for-good.md b/.changeset/task-ends-for-good.md index 4a7e3ef9ad..6eb06afef5 100644 --- a/.changeset/task-ends-for-good.md +++ b/.changeset/task-ends-for-good.md @@ -1,8 +1,8 @@ --- -"@flow-state-dev/orchestration": minor -"@flow-state-dev/harness-manager": patch +"@flow-state-dev/orchestration": patch +"@flow-state-dev/harness-manager": minor --- -A worker can now end its task on the current attempt without writing the ending itself: throw `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`), and the board's error recorder ends the row under the worker's claim, `errored` with no retry by default or `cancelled` with `{ ending: "cancelled" }`. +A worker can now end its task on the current attempt without writing the ending itself: throw `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`) naming the task it holds (`taskId`), and the board's error recorder ends that row under the worker's claim, `errored` with no retry by default or `cancelled` with `{ ending: "cancelled" }`. A marker naming any other row, such as one rethrown by a board nested inside the worker, is an ordinary failure. -Harness Manager's refused run uses it: the manager no longer fails or cancels its own row, and the board records the ending. The outcome is unchanged (`errored` on a board read from the claim, `cancelled` on the fixed board). `HarnessRunRefused` now extends `TaskEndsForGoodError`, and its constructor takes the ending as a second argument (FIX-2090). +Harness Manager's refused run uses it: the manager no longer fails or cancels its own row, and the board records the ending. The outcome is unchanged (`errored` on a board read from the claim, `cancelled` on the fixed board). `HarnessRunRefused` now extends `TaskEndsForGoodError`, and its constructor takes a second argument, `{ taskId, ending }` (FIX-2090). diff --git a/apps/docs/docs/orchestration/task-board.md b/apps/docs/docs/orchestration/task-board.md index 447a6f40a5..7abb01999f 100644 --- a/apps/docs/docs/orchestration/task-board.md +++ b/apps/docs/docs/orchestration/task-board.md @@ -704,12 +704,13 @@ waits. - `maxTotalRetries` (default `50`) — how many failure retries the board may authorize in total, across every task. See [Bounding the retries](#bounding-the-retries). - `maxIterations` — safety cap on how many times a single worker loops back to claim again, not a cap across the board. Default `10000`. -Some failures would come back the same on every attempt, such as a run refused because its task names nothing it may work on. Retrying those only spends the attempts. A worker that knows this throws `TaskEndsForGoodError`, exported from `@flow-state-dev/orchestration/task-board`. The board ends the task on that attempt, with the error's message as its reason. It ends `errored` by default, or `cancelled` if you construct the error with `{ ending: "cancelled" }`. Dependents and filed work are cascaded as for any other ending. +Some failures would come back the same on every attempt, such as a run refused because its task names nothing it may work on. Retrying those only spends the attempts. A worker that knows this throws `TaskEndsForGoodError`, exported from `@flow-state-dev/orchestration/task-board`, naming the task it is working on. The board ends that task on this attempt, with the error's message as its reason. It ends `errored` by default, or `cancelled` if you construct the error with `{ ending: "cancelled" }`. Dependents and filed work are cascaded as for any other ending. The error only counts for the task it names. If a board inside your worker rethrows one about its own task, your task treats it as an ordinary failure and keeps its retries. ```ts import { TaskEndsForGoodError } from "@flow-state-dev/orchestration/task-board"; -throw new TaskEndsForGoodError("the task names no project"); +// in a worker: input.taskId is the task this worker is running +throw new TaskEndsForGoodError("the task names no project", { taskId: input.taskId }); ``` `onError` reaches an [assignee that dispatches](#assignees-that-dispatch) too, where there is no board run left to fail. `"fail"` fails the dispatch run that worker is running in, so it reports `failed`. `"skip"` leaves it reporting `completed`, with the error on the task as usual. See [What `status` tells you](../server/background-work#what-status-tells-you). diff --git a/packages/harness-manager/src/manager.ts b/packages/harness-manager/src/manager.ts index 97c9d6857a..83c4fc422e 100644 --- a/packages/harness-manager/src/manager.ts +++ b/packages/harness-manager/src/manager.ts @@ -751,19 +751,12 @@ export class HarnessAttemptFailed extends Error { /** * The run's workspace refused it: its source answered `refused`, or the host - * would not provision what the source named. - * - * Not a failed attempt. The same source and host give the same answer next - * time, so a retry would spend every attempt on one refusal. The manager - * writes no ending of its own: this error tells the board's recorder to end - * the row on this attempt, with this message as its reason, under the - * attempt's own claim. It ends `cancelled` on the fixed board, and `errored` - * on a board read from the claim, where the conversation that filed the row - * reports it. + * would not provision what the source named. The same answer next time, so + * the recorder ends the row on this attempt instead of retrying. */ export class HarnessRunRefused extends TaskEndsForGoodError { - constructor(message: string, ending: "errored" | "cancelled") { - super(message, { ending }); + constructor(message: string, options: { taskId: string; ending: "errored" | "cancelled" }) { + super(message, options); this.name = "HarnessRunRefused"; } } @@ -885,7 +878,7 @@ function taskPayload( `[harness-manager] task ${input.taskId} names only part of a feature payload: ` + `${parsed.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}. ` + `Feature work names both an issue and a phase.`, - "errored", + { taskId: input.taskId, ending: "errored" }, ); } return undefined; @@ -938,14 +931,13 @@ function harnessCtxState(ctx: { return ctx.sequencer?.state as ManagerState | undefined; } -/** - * How a refused run's row ends on this run's board. On a board read from the - * claim it fails for good: a cancelled row there reaches nobody, and an - * `errored` one is what the conversation that filed it reports. The fixed - * board keeps the cancel. - */ -function refusalEnding(ctx: { sequencer?: { state?: unknown } | undefined }): "errored" | "cancelled" { - return harnessCtxState(ctx)?.claim != null ? "errored" : "cancelled"; +/** This run's refusal of the row it holds. Claim board: `errored` (a cancel there reaches nobody). Fixed board: `cancelled`. */ +function runRefused(ctx: { sequencer?: { state?: unknown } | undefined }, message: string): HarnessRunRefused { + const state = harnessCtxState(ctx); + return new HarnessRunRefused(message, { + taskId: state?.taskId ?? "", + ending: state?.claim != null ? "errored" : "cancelled", + }); } /** Read the manager's state, or throw rather than derive a wrong directory. */ @@ -1719,7 +1711,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo } catch (cause) { if (cause instanceof WorkspaceRefusedError) { if (holding) await recordPlace(ctx, before!, "refused").catch(() => undefined); - throw new HarnessRunRefused(refusedMessage("host", cause), refusalEnding(ctx)); + throw runRefused(ctx, refusedMessage("host", cause)); } if (cause instanceof HeldWorkMismatchError) await parkOnMismatch(ctx, cause, before); throw cause; @@ -1939,7 +1931,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo held: recorded?.held ?? null, }); const answer = pinnedAnswer(await host.source(ctx), pinned); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer), refusalEnding(ctx)); + if (answer.kind === "refused") throw runRefused(ctx, refusedMessage("source", answer)); const workspacePath = host.locate(answer, { place: placeFor(location) }).cwd; await ctx.sequencer!.patchState({ workspacePath }); @@ -2136,7 +2128,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo state.branch, ); const answer = pinnedAnswer(await host.source(ctx), state); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer), refusalEnding(ctx)); + if (answer.kind === "refused") throw runRefused(ctx, refusedMessage("source", answer)); if (host.locate(answer, { place: placeFor(location) }).cwd !== state.workspacePath) { throw new HarnessAttemptFailed( `[harness-manager] the run's workspace source changed between opening the run and ` + @@ -2595,9 +2587,8 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo { outcome: "failed", reason }, ); } - // **A refused run is settled, not retried** (see `HarnessRunRefused`), - // and not by this handler: the error it re-throws tells the board's - // recorder to end the row on this attempt, under this attempt's claim. + // A refusal's task row is ended by the recorder, not retried (see + // `HarnessRunRefused`); this handler updates only the run record above. throw error; }, }); diff --git a/packages/harness-manager/test/claim-board-refused.spec.ts b/packages/harness-manager/test/claim-board-refused.spec.ts index 8fa086698f..4408b9ee5a 100644 --- a/packages/harness-manager/test/claim-board-refused.spec.ts +++ b/packages/harness-manager/test/claim-board-refused.spec.ts @@ -51,8 +51,6 @@ describe("a refused run on a board read from the claim", () => { expect(refusing.seen).toHaveLength(0); }, 60_000); - // A refused run ends the way any failed task ends, so a task waiting on it - // is not left pending for a dependency that will never complete. it("cancels a task that depends on the refused row, as any failed task's dependent is", async () => { const lab = claimHost({ mode: "claim" }); await lab.seed({ ledgerId: CONVERSATIONS, partition: "conv-a", id: "t1", goal: "half a feature", input: { issue: "FIX-1" }, maxAttempts: 2 }); diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index c7a4f9cefa..e6d0e2f5d1 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -412,11 +412,12 @@ the limit in force. See the **A failure no retry can change.** A worker that knows the next attempt would end the same way throws `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`) -instead of writing the ending itself. The board's error recorder then ends the row on -this attempt, under the worker's claim, with the error's message as its reason: -`errored` with attempts left unspent (`fail` with `noRetry`), or `cancelled` when -constructed with `{ ending: "cancelled" }`. Any other error takes the ordinary retry -route. +instead of writing the ending itself, naming the task it holds (`{ taskId }`). The +board's error recorder then ends that row on this attempt, under the worker's claim, +with the error's message as its reason: `errored` with attempts left unspent (`fail` +with `noRetry`), or `cancelled` when constructed with `{ taskId, ending: "cancelled" }`. +A marker naming another row (one rethrown by a board nested inside the worker) and any +other error take the ordinary retry route. **When the board cannot record a result.** Saving a result is a commit followed by a change announcement, and the announcement can fail on its own. The board's two diff --git a/packages/orchestration/src/task-board/blocks/record-result.ts b/packages/orchestration/src/task-board/blocks/record-result.ts index ec405e5bd1..aff60aecef 100644 --- a/packages/orchestration/src/task-board/blocks/record-result.ts +++ b/packages/orchestration/src/task-board/blocks/record-result.ts @@ -75,6 +75,7 @@ import { type TaskWriteToken, } from "../../tasks"; import { + advisoryCancel, advisoryComplete, advisoryFail, } from "../../tasks/collection/advisory-write-back"; @@ -211,23 +212,23 @@ function readTaskQuietly( } /** - * A worker's failure that no retry can change: thrown by a worker that knows - * the next attempt would end the same way, such as a run refused before it - * started because it names nothing it may work on. - * - * The error recorder ends the row on this attempt rather than re-queueing it: - * `errored` for good, or `cancelled` when {@link ending} says so. The worker - * never writes the ending itself. The recorder does, under the worker's claim, - * so a superseded attempt's failure is declined like any other. + * Thrown when the next attempt would end the same way. The recorder ends the + * row on this attempt, under the worker's claim, so the worker writes no + * ending of its own. `ending` defaults to `errored`. + * + * It speaks only for `taskId`, the row its thrower holds: a recorder holding + * another row (a board nested inside the worker rethrows its worker's error on + * `onError: "fail"`) reads it as an ordinary failure. */ export class TaskEndsForGoodError extends Error { - /** How the row ends: `errored` (the default) or `cancelled`. */ + readonly taskId: string; readonly ending: "errored" | "cancelled"; - constructor(message: string, options?: { ending?: "errored" | "cancelled"; cause?: unknown }) { - super(message, options?.cause !== undefined ? { cause: options.cause } : undefined); + constructor(message: string, options: { taskId: string; ending?: "errored" | "cancelled" }) { + super(message); this.name = "TaskEndsForGoodError"; - this.ending = options?.ending ?? "errored"; + this.taskId = options.taskId; + this.ending = options.ending ?? "errored"; } } @@ -302,6 +303,8 @@ async function writeRethrowingIfNothingSaved(params: { recorder: RecorderKind; runId: RecorderFailureWiring["runId"]; attempt: (write: TaskWriteToken) => Promise; + /** The row's failure is final, so a release ends it rather than re-queueing it. */ + noRetry?: boolean; }): Promise { const { ctx, collection, claim, recorder, attempt } = params; const write = beginTaskWrite(readTaskQuietly(collection, claim.taskId)); @@ -327,7 +330,7 @@ async function writeRethrowingIfNothingSaved(params: { landed === true ? "committed" : "undetermined"; if (verdict === "undetermined") { - await releaseRow(collection, claim, err); + await releaseRow(collection, claim, err, params.noRetry === true); } const runId = params.runId?.(ctx); @@ -354,7 +357,8 @@ async function writeRethrowingIfNothingSaved(params: { * * One fenced write serves both halves of the uncertainty: if the original write * did commit, the claim fence declines this one harmlessly; if it did not, this - * settles the row (or re-queues it, where retries remain). + * settles the row (or re-queues it, where retries remain). A row whose + * worker said its failure is final (`noRetry`) is not re-queued here either. * * Best-effort by design. Nothing is hidden by a failure here — the report still * goes out and the run still fails — and letting a failed *recovery* displace @@ -363,14 +367,15 @@ async function writeRethrowingIfNothingSaved(params: { async function releaseRow( collection: TaskCollectionRef, claim: TaskClaimTicket, - err: unknown + err: unknown, + noRetry: boolean ): Promise { try { await advisoryFail( collection, claim.taskId, `task board could not confirm its own write: ${messageOf(err)}`, - { ifAllowed: true, claim, refuseWhenParked: true } + { ifAllowed: true, claim, refuseWhenParked: true, ...(noRetry ? { noRetry: true } : {}) } ); } catch { /* see above */ @@ -517,9 +522,11 @@ export function createRecordError(options: RecordErrorOptions) { execute: async (error: unknown, ctx) => { const claim = ctx.sequencer!.state.currentClaim; const message = error instanceof Error ? error.message : String(error); - // A failure the worker says no retry can change ends the row on this - // attempt, written here rather than by the worker. - const forGood = error instanceof TaskEndsForGoodError ? error.ending : undefined; + const forGood = + error instanceof TaskEndsForGoodError && claim !== undefined && error.taskId === claim.taskId + ? error.ending + : undefined; + const recorded = forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const); // A report that could not be DELIVERED is fatal wherever it happens, and // is never written to the row or swallowed by `onError`. The deferral @@ -582,9 +589,10 @@ export function createRecordError(options: RecordErrorOptions) { claim, recorder: "fail", runId: wiring.runId, + ...(forGood !== undefined ? { noRetry: true } : {}), attempt: (write) => forGood === "cancelled" - ? collection.cancel(claim.taskId, message, { + ? advisoryCancel(collection, claim.taskId, message, { ifAllowed: true, claim, refuseWhenParked: true, @@ -620,13 +628,13 @@ export function createRecordError(options: RecordErrorOptions) { // `onError`'s throw here instead would reject the fan-out and abandon // every task that had not started, which is the defect this branch // exists to remove. - return { recorded: forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const), error: message }; + return { recorded, error: message }; } if (onError === "fail") { throw error instanceof Error ? error : new Error(message); } - return { recorded: forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const), error: message }; + return { recorded, error: message }; }, }); } diff --git a/packages/orchestration/src/tasks/collection/advisory-write-back.ts b/packages/orchestration/src/tasks/collection/advisory-write-back.ts index 47adb553e2..94720af37f 100644 --- a/packages/orchestration/src/tasks/collection/advisory-write-back.ts +++ b/packages/orchestration/src/tasks/collection/advisory-write-back.ts @@ -113,6 +113,9 @@ import type { * status the write could have produced when that prediction was overtakeable, * so clause (b) can decline to call such a row somebody else's work. */ +/** The write-backs this seam wraps. */ +type AdvisoryVerb = "complete" | "fail" | "cancel"; + interface WriteTarget { status: TaskStatus; kind: TaskChangeKind; @@ -154,7 +157,7 @@ export async function advisoryFail( error: string, options: FailOptions ): Promise { - const before = observe(ref, taskId, "fail"); + const before = observe(ref, taskId, "fail", options.noRetry === true); try { return await ref.fail(taskId, error, options); } catch (err) { @@ -162,6 +165,29 @@ export async function advisoryFail( } } +/** + * Cancel a task, containing the throw a non-conforming store raises where a + * conforming one would have declined. + * + * A drop-in for `ref.cancel(id, reason, options)`, for a recorder ending the + * row its worker holds (a worker's `TaskEndsForGoodError` asking for a cancel). + * A row someone else already ended must not turn into a throw that escapes the + * recorder and abandons the board's siblings, any more than it does on `fail`. + */ +export async function advisoryCancel( + ref: TaskCollectionRef, + taskId: string, + reason: string, + options: TaskTransitionOptions +): Promise { + const before = observe(ref, taskId, "cancel"); + try { + return await ref.cancel(taskId, reason, options); + } catch (err) { + return containOrRethrow(ref, taskId, "cancel", before, options, err); + } +} + /** * Everything about the call that has to be known BEFORE the write. * @@ -188,12 +214,14 @@ interface CallBasis { function observe( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail" + verb: AdvisoryVerb, + noRetry = false ): CallBasis { return { task: readQuietly(ref, taskId), at: readClock(ref), - grantedRetries: readGrantedRetries(ref, verb), + // A `noRetry` fail never takes the retry route, so the budget can't decide it. + grantedRetries: noRetry ? undefined : readGrantedRetries(ref, verb), }; } @@ -211,7 +239,7 @@ function observe( */ function readGrantedRetries( ref: TaskCollectionRef, - verb: "complete" | "fail" + verb: AdvisoryVerb ): number | undefined { if (verb !== "fail" || ref.maxTotalRetries == null) return undefined; try { @@ -274,26 +302,25 @@ function readClock(ref: TaskCollectionRef): number { /** * Where a write-back was headed, resolved the way the backings resolve it. * - * `complete` is fixed. `fail` is not: with attempts left it is a *soft* fail + * `complete` and `cancel` are fixed. `fail` is not: with attempts left it is a *soft* fail * that re-pends the task, and only {@link routeFailure} — the same function * both backings call inside their atomic write, against the same budget the ref * reports — knows which. Resolving it here rather than at the call sites is * what keeps two callers from carrying two copies of that decision. * * Runs only on the error path, so the retry-budget sum it may walk is never - * paid by a write that succeeded. - * - * A `noRetry` fail skips the retry route in both backings, so it is headed - * for `errored` whatever attempts are left. + * paid by a write that succeeded. `noRetry` is terminal in both backings, + * whatever attempts are left. */ function resolveTarget( ref: TaskCollectionRef, - verb: "complete" | "fail", + verb: AdvisoryVerb, basis: Task, grantedRetries: number | undefined, noRetry: boolean ): WriteTarget { if (verb === "complete") return { status: "completed", kind: "completed" }; + if (verb === "cancel") return { status: "cancelled", kind: "cancelled" }; if (noRetry) return { status: "errored", kind: "errored" }; const routing = routeFailure( basis, @@ -329,9 +356,9 @@ function resolveTarget( function containOrRethrow( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail", + verb: AdvisoryVerb, before: CallBasis, - options: TaskTransitionOptions, + options: FailOptions, err: unknown ): TaskWriteOutcome { let contained: TaskWriteOutcome | undefined; @@ -356,9 +383,9 @@ function containOrRethrow( function attribute( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail", + verb: AdvisoryVerb, before: CallBasis, - options: TaskTransitionOptions + options: FailOptions ): TaskWriteOutcome | undefined { const snapshot = before.task; const current = readQuietly(ref, taskId); @@ -368,7 +395,7 @@ function attribute( // advisory decline at all. if (basis === undefined) return undefined; - const target = resolveTarget(ref, verb, basis, before.grantedRetries, (options as FailOptions).noRetry === true); + const target = resolveTarget(ref, verb, basis, before.grantedRetries, options.noRetry === true); // (a) The task was already declinable when this call ran. // diff --git a/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts b/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts new file mode 100644 index 0000000000..598450e99c --- /dev/null +++ b/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts @@ -0,0 +1,32 @@ +/** + * `advisoryFail` judges a throw against where the write was headed. A + * `noRetry` fail is headed for `errored` whatever attempts are left, so a + * store that commits it and then throws has failed after the commit: that is + * reported to the caller, never contained as somebody else's settlement. + * + * Predicting the retry route instead (`pending`) would read the committed + * `errored` row as a row someone else ended, and swallow the failure. + */ +import { expect, it } from "vitest"; +import { createStateBackedTaskCollection, ticketForClaim, type TaskCollectionRef } from "../../src/tasks"; +import { advisoryFail } from "../../src/tasks/collection/advisory-write-back"; +import { createFakeSequencerState } from "../helpers"; + +it("a noRetry fail that commits and then throws reaches the caller", async () => { + const state = createFakeSequencerState<{ tasks: Record }>({ tasks: {} }); + const inner = createStateBackedTaskCollection({ collectionId: "tasks", state }); + await inner.addTask({ id: "t", goal: "work", maxAttempts: 3 }); + const claim = ticketForClaim("tasks", (await inner.claim("w"))!); + const commitsThenThrows = { + ...inner, + fail: async (...args: Parameters) => { + await inner.fail(...args); + throw new Error("onChange rejected after the commit"); + }, + } as TaskCollectionRef; + + await expect( + advisoryFail(commitsThenThrows, "t", "refused", { ifAllowed: true, claim, noRetry: true }), + ).rejects.toThrow("onChange rejected after the commit"); + expect(inner.get("t")?.status).toBe("errored"); +}); diff --git a/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts index 2558d2be7b..dabf0edbd6 100644 --- a/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts +++ b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts @@ -1,18 +1,22 @@ /** - * A worker that knows its failure is final says so by throwing - * `TaskEndsForGoodError`, and the error recorder ends the row on that attempt. - * - * Why it matters: only the claim gate or the drain records a row's ending - * (the FIX-2056 epic's ER-1). A worker that could not say "don't retry this" - * had two bad choices: let the recorder re-queue a run that will be refused - * the same way on every attempt, or write the ending itself and become a - * second ending writer. The marker removes both. + * The recorder ends a final failure on this attempt. An ordinary error still + * retries, and so does a final failure about another row: an inner board's + * "don't retry" must not cost the outer row the attempts its filer chose. */ import { describe, expect, it } from "vitest"; +import { handler, sequencer } from "@flow-state-dev/core"; import type { BlockContext } from "@flow-state-dev/core/types"; -import { runForTest } from "@flow-state-dev/testing"; +import { runForTest, testBlock } from "@flow-state-dev/testing"; +import { z } from "zod"; import { createRecordError, TaskEndsForGoodError } from "../../src/task-board/blocks/record-result"; -import { createStateBackedTaskCollection, ticketForClaim, type TaskCollectionRef } from "../../src/tasks"; +import { taskBoard, taskWorkerInputSchema } from "../../src/task-board"; +import { + createStateBackedTaskCollection, + isTerminalStatus, + ticketForClaim, + type TaskCollectionRef, + type TaskWorker, +} from "../../src/tasks"; import { createFakeSequencerState } from "../helpers"; /** A claimed row with attempts to spare, and the recorder's context holding its claim. */ @@ -35,7 +39,7 @@ describe("the error recorder and a failure no retry can change", () => { it("fails the row for good on this attempt, with the worker's reason", async () => { const { collection, ctx } = await claimedWithAttemptsLeft(); - await runForTest(recorder(collection), new TaskEndsForGoodError("refused: names no project"), ctx); + await runForTest(recorder(collection), new TaskEndsForGoodError("refused: names no project", { taskId: "t" }), ctx); expect(collection.get("t")).toMatchObject({ status: "errored", attempts: 1, error: "refused: names no project" }); }); @@ -45,7 +49,7 @@ describe("the error recorder and a failure no retry can change", () => { await runForTest( recorder(collection), - new TaskEndsForGoodError("refused: remote not allowed", { ending: "cancelled" }), + new TaskEndsForGoodError("refused: remote not allowed", { taskId: "t", ending: "cancelled" }), ctx, ); @@ -59,4 +63,92 @@ describe("the error recorder and a failure no retry can change", () => { expect(collection.get("t")?.status).toBe("pending"); }); + + it("treats a marker naming another row as an ordinary failure", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new TaskEndsForGoodError("an inner row's verdict", { taskId: "inner" }), ctx); + + expect(collection.get("t")?.status).toBe("pending"); + }); + + // A store that throws on a declined transition instead of honouring + // `ifAllowed` is the extension point the advisory seam exists for. A row + // someone else already cancelled must not turn the cancel arm's write into + // an error that escapes the recorder and abandons the board's siblings. + it("contains a declined cancel on a store that throws on an illegal transition", async () => { + const { collection: inner, ctx } = await claimedWithAttemptsLeft(); + const throwsWhenTerminal = + (verb: "fail" | "cancel") => + async (id: string, ...rest: unknown[]) => { + const row = inner.get(id); + if (row !== undefined && isTerminalStatus(row.status)) throw new Error(`illegal ${row.status} -> ${verb}`); + return (inner[verb] as (...args: unknown[]) => Promise)(id, ...rest); + }; + const collection = { ...inner, fail: throwsWhenTerminal("fail"), cancel: throwsWhenTerminal("cancel") } as unknown as TaskCollectionRef; + await inner.cancel("t", "a coordinator cancelled it"); + + await runForTest( + recorder(collection), + new TaskEndsForGoodError("refused", { taskId: "t", ending: "cancelled" }), + ctx, + ); + + expect(inner.get("t")).toMatchObject({ status: "cancelled", error: "a coordinator cancelled it" }); + }); +}); + +describe("a board nested inside a worker", () => { + /** + * The outer board's worker drains an inner board that fails (`onError: + * "fail"`), so the inner worker's error reaches the outer recorder. Returns + * how many times the outer worker ran. + */ + async function outerRunsWhenInnerThrows(innerError: (taskId: string) => Error): Promise { + let outerRuns = 0; + const innerWorker = handler({ + name: "inner-worker", + inputSchema: taskWorkerInputSchema, + outputSchema: z.null(), + execute: (input) => { + throw innerError(input.taskId); + }, + }) as TaskWorker; + const inner = taskBoard({ + name: "inner", + collection: { collectionId: "inner" }, + concurrency: 1, + dispatcher: "fifo", + workers: innerWorker, + initialTasks: [{ id: "inner-t", goal: "inner", maxAttempts: 1 }], + onError: "fail", + }); + const outerWorker = sequencer({ name: "outer-worker", inputSchema: taskWorkerInputSchema }) + .tap(() => { + outerRuns += 1; + }) + .tap(inner.drain) as unknown as TaskWorker; + const outer = taskBoard({ + name: "outer", + collection: { collectionId: "outer" }, + concurrency: 1, + dispatcher: "fifo", + workers: outerWorker, + initialTasks: [{ id: "outer-t", goal: "outer", maxAttempts: 3 }], + onError: "skip", + }); + + await testBlock(outer.drain, { input: undefined }); + return outerRuns; + } + + it("does not let the inner row's 'don't retry' end the outer row on its first attempt", async () => { + const runs = await outerRunsWhenInnerThrows((taskId) => new TaskEndsForGoodError("inner refused", { taskId })); + expect(runs).toBeGreaterThan(1); + }); + + it("control: a plain inner error gives the outer row its retry", async () => { + const runs = await outerRunsWhenInnerThrows(() => new Error("inner failed")); + expect(runs).toBeGreaterThan(1); + }); });