diff --git a/.changeset/task-ends-for-good.md b/.changeset/task-ends-for-good.md new file mode 100644 index 0000000000..6eb06afef5 --- /dev/null +++ b/.changeset/task-ends-for-good.md @@ -0,0 +1,8 @@ +--- +"@flow-state-dev/orchestration": patch +"@flow-state-dev/harness-manager": minor +--- + +A worker can now end its task on the current attempt without writing the ending itself: throw `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`) naming the task it holds (`taskId`), and the board's error recorder ends that row under the worker's claim, `errored` with no retry by default or `cancelled` with `{ ending: "cancelled" }`. A marker naming any other row, such as one rethrown by a board nested inside the worker, is an ordinary failure. + +Harness Manager's refused run uses it: the manager no longer fails or cancels its own row, and the board records the ending. The outcome is unchanged (`errored` on a board read from the claim, `cancelled` on the fixed board). `HarnessRunRefused` now extends `TaskEndsForGoodError`, and its constructor takes a second argument, `{ taskId, ending }` (FIX-2090). diff --git a/apps/docs/docs/orchestration/task-board.md b/apps/docs/docs/orchestration/task-board.md index ef7833c08b..7abb01999f 100644 --- a/apps/docs/docs/orchestration/task-board.md +++ b/apps/docs/docs/orchestration/task-board.md @@ -704,6 +704,15 @@ waits. - `maxTotalRetries` (default `50`) — how many failure retries the board may authorize in total, across every task. See [Bounding the retries](#bounding-the-retries). - `maxIterations` — safety cap on how many times a single worker loops back to claim again, not a cap across the board. Default `10000`. +Some failures would come back the same on every attempt, such as a run refused because its task names nothing it may work on. Retrying those only spends the attempts. A worker that knows this throws `TaskEndsForGoodError`, exported from `@flow-state-dev/orchestration/task-board`, naming the task it is working on. The board ends that task on this attempt, with the error's message as its reason. It ends `errored` by default, or `cancelled` if you construct the error with `{ ending: "cancelled" }`. Dependents and filed work are cascaded as for any other ending. The error only counts for the task it names. If a board inside your worker rethrows one about its own task, your task treats it as an ordinary failure and keeps its retries. + +```ts +import { TaskEndsForGoodError } from "@flow-state-dev/orchestration/task-board"; + +// in a worker: input.taskId is the task this worker is running +throw new TaskEndsForGoodError("the task names no project", { taskId: input.taskId }); +``` + `onError` reaches an [assignee that dispatches](#assignees-that-dispatch) too, where there is no board run left to fail. `"fail"` fails the dispatch run that worker is running in, so it reports `failed`. `"skip"` leaves it reporting `completed`, with the error on the task as usual. See [What `status` tells you](../server/background-work#what-status-tells-you). A worker's result is not always the last word on its task. A coordinator can cancel the task while the worker runs. The worker can mark the task done itself partway through. The claim can expire and another worker can pick the task up. In each case the worker comes back with a result for a task that has already moved on. diff --git a/packages/harness-manager/src/manager.ts b/packages/harness-manager/src/manager.ts index daf3ff9d2a..83c4fc422e 100644 --- a/packages/harness-manager/src/manager.ts +++ b/packages/harness-manager/src/manager.ts @@ -48,7 +48,7 @@ import type { HarnessResolver, HarnessSessionHook, } from "@flow-state-dev/core/types"; -import { taskWorkerInputSchema } from "@flow-state-dev/orchestration/task-board"; +import { TaskEndsForGoodError, taskWorkerInputSchema } from "@flow-state-dev/orchestration/task-board"; import { getOrCreateTaskCollection, hasFrozenLedgerAssignee, @@ -751,18 +751,12 @@ export class HarnessAttemptFailed extends Error { /** * The run's workspace refused it: its source answered `refused`, or the host - * would not provision what the source named. - * - * Not a failed attempt. The same source and host give the same answer next - * time, so a retry would spend every attempt on one refusal. The manager - * settles the row with this message as its reason, under the attempt's own - * claim, and the board's recorder then has nothing to re-queue: `cancelled` - * on the fixed board, `errored` on a board read from the claim, where the - * conversation that filed the row reports it. + * would not provision what the source named. The same answer next time, so + * the recorder ends the row on this attempt instead of retrying. */ -export class HarnessRunRefused extends Error { - constructor(message: string) { - super(message); +export class HarnessRunRefused extends TaskEndsForGoodError { + constructor(message: string, options: { taskId: string; ending: "errored" | "cancelled" }) { + super(message, options); this.name = "HarnessRunRefused"; } } @@ -884,6 +878,7 @@ function taskPayload( `[harness-manager] task ${input.taskId} names only part of a feature payload: ` + `${parsed.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}. ` + `Feature work names both an issue and a phase.`, + { taskId: input.taskId, ending: "errored" }, ); } return undefined; @@ -936,6 +931,15 @@ function harnessCtxState(ctx: { return ctx.sequencer?.state as ManagerState | undefined; } +/** This run's refusal of the row it holds. Claim board: `errored` (a cancel there reaches nobody). Fixed board: `cancelled`. */ +function runRefused(ctx: { sequencer?: { state?: unknown } | undefined }, message: string): HarnessRunRefused { + const state = harnessCtxState(ctx); + return new HarnessRunRefused(message, { + taskId: state?.taskId ?? "", + ending: state?.claim != null ? "errored" : "cancelled", + }); +} + /** Read the manager's state, or throw rather than derive a wrong directory. */ function managerState(state: ManagerState | undefined): ManagerState { if (state?.workspacePath == null) { @@ -1707,7 +1711,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo } catch (cause) { if (cause instanceof WorkspaceRefusedError) { if (holding) await recordPlace(ctx, before!, "refused").catch(() => undefined); - throw new HarnessRunRefused(refusedMessage("host", cause)); + throw runRefused(ctx, refusedMessage("host", cause)); } if (cause instanceof HeldWorkMismatchError) await parkOnMismatch(ctx, cause, before); throw cause; @@ -1927,7 +1931,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo held: recorded?.held ?? null, }); const answer = pinnedAnswer(await host.source(ctx), pinned); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer)); + if (answer.kind === "refused") throw runRefused(ctx, refusedMessage("source", answer)); const workspacePath = host.locate(answer, { place: placeFor(location) }).cwd; await ctx.sequencer!.patchState({ workspacePath }); @@ -2124,7 +2128,7 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo state.branch, ); const answer = pinnedAnswer(await host.source(ctx), state); - if (answer.kind === "refused") throw new HarnessRunRefused(refusedMessage("source", answer)); + if (answer.kind === "refused") throw runRefused(ctx, refusedMessage("source", answer)); if (host.locate(answer, { place: placeFor(location) }).cwd !== state.workspacePath) { throw new HarnessAttemptFailed( `[harness-manager] the run's workspace source changed between opening the run and ` + @@ -2583,27 +2587,8 @@ export function harnessManager(options: ManagerOptions): HarnessManager | TaskWo { outcome: "failed", reason }, ); } - // **A refused run is settled, not retried** (see `HarnessRunRefused`). - // Under this attempt's own claim, so a superseded attempt cannot cancel - // a row its replacement holds. Best effort, like the write above: the - // error being unwound is what the caller gets either way. - // - // On a board read from the claim the row fails for good instead: a - // cancelled row there reaches nobody, and an `errored` one is what the - // conversation that filed it reports. The fixed board keeps the cancel. - if (error instanceof HarnessRunRefused && state?.taskId != null && state.attempt != null) { - try { - const tasks = await runBoard(ctx as BlockContext); - const row = tasks.get(state.taskId); - if (row !== undefined) { - const claim = { ...ticketForClaim(tasks.collectionId, row, tasks.partition), attempt: state.attempt }; - if (state.claim != null) await tasks.fail(state.taskId, reason, { claim, noRetry: true }); - else await tasks.cancel(state.taskId, reason, { claim }); - } - } catch { - // Left to the board's recorder, which re-queues it as before. - } - } + // A refusal's task row is ended by the recorder, not retried (see + // `HarnessRunRefused`); this handler updates only the run record above. throw error; }, }); diff --git a/packages/harness-manager/test/claim-board-refused.spec.ts b/packages/harness-manager/test/claim-board-refused.spec.ts index f173e88c3f..4408b9ee5a 100644 --- a/packages/harness-manager/test/claim-board-refused.spec.ts +++ b/packages/harness-manager/test/claim-board-refused.spec.ts @@ -50,6 +50,17 @@ describe("a refused run on a board read from the claim", () => { expect(row.error).toContain("the task names no project"); expect(refusing.seen).toHaveLength(0); }, 60_000); + + it("cancels a task that depends on the refused row, as any failed task's dependent is", async () => { + const lab = claimHost({ mode: "claim" }); + await lab.seed({ ledgerId: CONVERSATIONS, partition: "conv-a", id: "t1", goal: "half a feature", input: { issue: "FIX-1" }, maxAttempts: 2 }); + await lab.seed({ ledgerId: CONVERSATIONS, partition: "conv-a", id: "t2", goal: "after it", input: {}, deps: ["t1"] }); + + await lab.runAttempt(CONVERSATIONS, "t1", "conv-a"); + + expect(await lab.row(CONVERSATIONS, "t1", "conv-a")).toMatchObject({ status: "errored", attempts: 1 }); + expect((await lab.row(CONVERSATIONS, "t2", "conv-a")).status).toBe("cancelled"); + }, 60_000); }); describe("a refused run on a fixed board", () => { diff --git a/packages/harness-manager/test/claim-host.ts b/packages/harness-manager/test/claim-host.ts index c0cd865b29..ef11e75777 100644 --- a/packages/harness-manager/test/claim-host.ts +++ b/packages/harness-manager/test/claim-host.ts @@ -227,6 +227,7 @@ export function claimHost(options: HostOptions) { goal: z.string(), input: z.unknown().optional(), maxAttempts: z.number().optional(), + deps: z.array(z.string()).optional(), }), outputSchema: z.unknown(), uses: [ledgerResources], @@ -239,6 +240,7 @@ export function claimHost(options: HostOptions) { goal: input.goal, assignee: ASSIGNEE, ...(input.input !== undefined ? { input: input.input } : {}), + ...(input.deps !== undefined ? { deps: input.deps } : {}), maxAttempts: input.maxAttempts ?? 2, }); } diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index 418b200b30..e6d0e2f5d1 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -410,6 +410,15 @@ reports `terminationReason: "retry-budget-exhausted"` alongside `counts.retries` the limit in force. See the [Task board guide](https://flow-state.dev/docs/orchestration/task-board). +**A failure no retry can change.** A worker that knows the next attempt would end the +same way throws `TaskEndsForGoodError` (from `@flow-state-dev/orchestration/task-board`) +instead of writing the ending itself, naming the task it holds (`{ taskId }`). The +board's error recorder then ends that row on this attempt, under the worker's claim, +with the error's message as its reason: `errored` with attempts left unspent (`fail` +with `noRetry`), or `cancelled` when constructed with `{ taskId, ending: "cancelled" }`. +A marker naming another row (one rethrown by a board nested inside the worker) and any +other error take the ordinary retry route. + **When the board cannot record a result.** Saving a result is a commit followed by a change announcement, and the announcement can fail on its own. The board's two recorders correlate every result write (`beginTaskWrite` / `didWriteLand`, above) and diff --git a/packages/orchestration/src/task-board/blocks/record-result.ts b/packages/orchestration/src/task-board/blocks/record-result.ts index d7e11242c3..aff60aecef 100644 --- a/packages/orchestration/src/task-board/blocks/record-result.ts +++ b/packages/orchestration/src/task-board/blocks/record-result.ts @@ -13,7 +13,9 @@ * sequencer. Receives the caught error as input, reads * `currentClaim` from worker state, calls `collection.fail`. Honors * `onError`: `"skip"` swallows the error after writing the failure; - * `"fail"` rethrows so the parent forEach rejects. + * `"fail"` rethrows so the parent forEach rejects. A + * {@link TaskEndsForGoodError} ends the row on this attempt instead of + * re-queueing it: `fail` with `noRetry`, or `cancel`. * * Both write-backs are **advisory** (FIX-951): they pass `ifAllowed` and the * worker's `claim`, so a result that arrives after the task was settled by @@ -73,6 +75,7 @@ import { type TaskWriteToken, } from "../../tasks"; import { + advisoryCancel, advisoryComplete, advisoryFail, } from "../../tasks/collection/advisory-write-back"; @@ -208,6 +211,27 @@ function readTaskQuietly( } } +/** + * Thrown when the next attempt would end the same way. The recorder ends the + * row on this attempt, under the worker's claim, so the worker writes no + * ending of its own. `ending` defaults to `errored`. + * + * It speaks only for `taskId`, the row its thrower holds: a recorder holding + * another row (a board nested inside the worker rethrows its worker's error on + * `onError: "fail"`) reads it as an ordinary failure. + */ +export class TaskEndsForGoodError extends Error { + readonly taskId: string; + readonly ending: "errored" | "cancelled"; + + constructor(message: string, options: { taskId: string; ending?: "errored" | "cancelled" }) { + super(message); + this.name = "TaskEndsForGoodError"; + this.taskId = options.taskId; + this.ending = options.ending ?? "errored"; + } +} + function messageOf(err: unknown): string { return err instanceof Error ? err.message : String(err); } @@ -279,6 +303,8 @@ async function writeRethrowingIfNothingSaved(params: { recorder: RecorderKind; runId: RecorderFailureWiring["runId"]; attempt: (write: TaskWriteToken) => Promise; + /** The row's failure is final, so a release ends it rather than re-queueing it. */ + noRetry?: boolean; }): Promise { const { ctx, collection, claim, recorder, attempt } = params; const write = beginTaskWrite(readTaskQuietly(collection, claim.taskId)); @@ -304,7 +330,7 @@ async function writeRethrowingIfNothingSaved(params: { landed === true ? "committed" : "undetermined"; if (verdict === "undetermined") { - await releaseRow(collection, claim, err); + await releaseRow(collection, claim, err, params.noRetry === true); } const runId = params.runId?.(ctx); @@ -331,7 +357,8 @@ async function writeRethrowingIfNothingSaved(params: { * * One fenced write serves both halves of the uncertainty: if the original write * did commit, the claim fence declines this one harmlessly; if it did not, this - * settles the row (or re-queues it, where retries remain). + * settles the row (or re-queues it, where retries remain). A row whose + * worker said its failure is final (`noRetry`) is not re-queued here either. * * Best-effort by design. Nothing is hidden by a failure here — the report still * goes out and the run still fails — and letting a failed *recovery* displace @@ -340,14 +367,15 @@ async function writeRethrowingIfNothingSaved(params: { async function releaseRow( collection: TaskCollectionRef, claim: TaskClaimTicket, - err: unknown + err: unknown, + noRetry: boolean ): Promise { try { await advisoryFail( collection, claim.taskId, `task board could not confirm its own write: ${messageOf(err)}`, - { ifAllowed: true, claim, refuseWhenParked: true } + { ifAllowed: true, claim, refuseWhenParked: true, ...(noRetry ? { noRetry: true } : {}) } ); } catch { /* see above */ @@ -494,6 +522,11 @@ export function createRecordError(options: RecordErrorOptions) { execute: async (error: unknown, ctx) => { const claim = ctx.sequencer!.state.currentClaim; const message = error instanceof Error ? error.message : String(error); + const forGood = + error instanceof TaskEndsForGoodError && claim !== undefined && error.taskId === claim.taskId + ? error.ending + : undefined; + const recorded = forGood === "cancelled" ? ("cancelled" as const) : ("errored" as const); // A report that could not be DELIVERED is fatal wherever it happens, and // is never written to the row or swallowed by `onError`. The deferral @@ -556,13 +589,22 @@ export function createRecordError(options: RecordErrorOptions) { claim, recorder: "fail", runId: wiring.runId, + ...(forGood !== undefined ? { noRetry: true } : {}), attempt: (write) => - advisoryFail(collection, claim.taskId, message, { - ifAllowed: true, - claim, - refuseWhenParked: true, - write, - }), + forGood === "cancelled" + ? advisoryCancel(collection, claim.taskId, message, { + ifAllowed: true, + claim, + refuseWhenParked: true, + write, + }) + : advisoryFail(collection, claim.taskId, message, { + ifAllowed: true, + claim, + refuseWhenParked: true, + write, + ...(forGood === "errored" ? { noRetry: true } : {}), + }), }); } await ctx.sequencer!.patchState({ currentClaim: undefined, currentLedger: undefined }); @@ -586,13 +628,13 @@ export function createRecordError(options: RecordErrorOptions) { // `onError`'s throw here instead would reject the fan-out and abandon // every task that had not started, which is the defect this branch // exists to remove. - return { recorded: "errored" as const, error: message }; + return { recorded, error: message }; } if (onError === "fail") { throw error instanceof Error ? error : new Error(message); } - return { recorded: "errored" as const, error: message }; + return { recorded, error: message }; }, }); } diff --git a/packages/orchestration/src/task-board/index.ts b/packages/orchestration/src/task-board/index.ts index 5f98113890..2f073ab1b6 100644 --- a/packages/orchestration/src/task-board/index.ts +++ b/packages/orchestration/src/task-board/index.ts @@ -210,6 +210,7 @@ export type { BuildWorkerStepOptions } from "./blocks/worker-step"; export { createRecordSuccess, createRecordError, + TaskEndsForGoodError, } from "./blocks/record-result"; export type { RecordSuccessOptions, diff --git a/packages/orchestration/src/tasks/collection/advisory-write-back.ts b/packages/orchestration/src/tasks/collection/advisory-write-back.ts index fd096c89e6..94720af37f 100644 --- a/packages/orchestration/src/tasks/collection/advisory-write-back.ts +++ b/packages/orchestration/src/tasks/collection/advisory-write-back.ts @@ -99,6 +99,7 @@ import type { TaskChangeKind } from "./change-event"; import type { Task, TaskStatus } from "../schema/task"; import { routeFailure, sumGrantedRetries, transitionDeclineReason } from "./internal"; import type { + FailOptions, TaskCollectionRef, TaskTransitionOptions, TaskWriteOutcome, @@ -112,6 +113,9 @@ import type { * status the write could have produced when that prediction was overtakeable, * so clause (b) can decline to call such a row somebody else's work. */ +/** The write-backs this seam wraps. */ +type AdvisoryVerb = "complete" | "fail" | "cancel"; + interface WriteTarget { status: TaskStatus; kind: TaskChangeKind; @@ -151,9 +155,9 @@ export async function advisoryFail( ref: TaskCollectionRef, taskId: string, error: string, - options: TaskTransitionOptions + options: FailOptions ): Promise { - const before = observe(ref, taskId, "fail"); + const before = observe(ref, taskId, "fail", options.noRetry === true); try { return await ref.fail(taskId, error, options); } catch (err) { @@ -161,6 +165,29 @@ export async function advisoryFail( } } +/** + * Cancel a task, containing the throw a non-conforming store raises where a + * conforming one would have declined. + * + * A drop-in for `ref.cancel(id, reason, options)`, for a recorder ending the + * row its worker holds (a worker's `TaskEndsForGoodError` asking for a cancel). + * A row someone else already ended must not turn into a throw that escapes the + * recorder and abandons the board's siblings, any more than it does on `fail`. + */ +export async function advisoryCancel( + ref: TaskCollectionRef, + taskId: string, + reason: string, + options: TaskTransitionOptions +): Promise { + const before = observe(ref, taskId, "cancel"); + try { + return await ref.cancel(taskId, reason, options); + } catch (err) { + return containOrRethrow(ref, taskId, "cancel", before, options, err); + } +} + /** * Everything about the call that has to be known BEFORE the write. * @@ -187,12 +214,14 @@ interface CallBasis { function observe( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail" + verb: AdvisoryVerb, + noRetry = false ): CallBasis { return { task: readQuietly(ref, taskId), at: readClock(ref), - grantedRetries: readGrantedRetries(ref, verb), + // A `noRetry` fail never takes the retry route, so the budget can't decide it. + grantedRetries: noRetry ? undefined : readGrantedRetries(ref, verb), }; } @@ -210,7 +239,7 @@ function observe( */ function readGrantedRetries( ref: TaskCollectionRef, - verb: "complete" | "fail" + verb: AdvisoryVerb ): number | undefined { if (verb !== "fail" || ref.maxTotalRetries == null) return undefined; try { @@ -273,22 +302,26 @@ function readClock(ref: TaskCollectionRef): number { /** * Where a write-back was headed, resolved the way the backings resolve it. * - * `complete` is fixed. `fail` is not: with attempts left it is a *soft* fail + * `complete` and `cancel` are fixed. `fail` is not: with attempts left it is a *soft* fail * that re-pends the task, and only {@link routeFailure} — the same function * both backings call inside their atomic write, against the same budget the ref * reports — knows which. Resolving it here rather than at the call sites is * what keeps two callers from carrying two copies of that decision. * * Runs only on the error path, so the retry-budget sum it may walk is never - * paid by a write that succeeded. + * paid by a write that succeeded. `noRetry` is terminal in both backings, + * whatever attempts are left. */ function resolveTarget( ref: TaskCollectionRef, - verb: "complete" | "fail", + verb: AdvisoryVerb, basis: Task, - grantedRetries: number | undefined + grantedRetries: number | undefined, + noRetry: boolean ): WriteTarget { if (verb === "complete") return { status: "completed", kind: "completed" }; + if (verb === "cancel") return { status: "cancelled", kind: "cancelled" }; + if (noRetry) return { status: "errored", kind: "errored" }; const routing = routeFailure( basis, () => grantedRetries ?? sumGrantedRetries(ref.list()), @@ -323,9 +356,9 @@ function resolveTarget( function containOrRethrow( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail", + verb: AdvisoryVerb, before: CallBasis, - options: TaskTransitionOptions, + options: FailOptions, err: unknown ): TaskWriteOutcome { let contained: TaskWriteOutcome | undefined; @@ -350,9 +383,9 @@ function containOrRethrow( function attribute( ref: TaskCollectionRef, taskId: string, - verb: "complete" | "fail", + verb: AdvisoryVerb, before: CallBasis, - options: TaskTransitionOptions + options: FailOptions ): TaskWriteOutcome | undefined { const snapshot = before.task; const current = readQuietly(ref, taskId); @@ -362,7 +395,7 @@ function attribute( // advisory decline at all. if (basis === undefined) return undefined; - const target = resolveTarget(ref, verb, basis, before.grantedRetries); + const target = resolveTarget(ref, verb, basis, before.grantedRetries, options.noRetry === true); // (a) The task was already declinable when this call ran. // diff --git a/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts b/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts new file mode 100644 index 0000000000..598450e99c --- /dev/null +++ b/packages/orchestration/test/collection/advisory-fail-no-retry.test.ts @@ -0,0 +1,32 @@ +/** + * `advisoryFail` judges a throw against where the write was headed. A + * `noRetry` fail is headed for `errored` whatever attempts are left, so a + * store that commits it and then throws has failed after the commit: that is + * reported to the caller, never contained as somebody else's settlement. + * + * Predicting the retry route instead (`pending`) would read the committed + * `errored` row as a row someone else ended, and swallow the failure. + */ +import { expect, it } from "vitest"; +import { createStateBackedTaskCollection, ticketForClaim, type TaskCollectionRef } from "../../src/tasks"; +import { advisoryFail } from "../../src/tasks/collection/advisory-write-back"; +import { createFakeSequencerState } from "../helpers"; + +it("a noRetry fail that commits and then throws reaches the caller", async () => { + const state = createFakeSequencerState<{ tasks: Record }>({ tasks: {} }); + const inner = createStateBackedTaskCollection({ collectionId: "tasks", state }); + await inner.addTask({ id: "t", goal: "work", maxAttempts: 3 }); + const claim = ticketForClaim("tasks", (await inner.claim("w"))!); + const commitsThenThrows = { + ...inner, + fail: async (...args: Parameters) => { + await inner.fail(...args); + throw new Error("onChange rejected after the commit"); + }, + } as TaskCollectionRef; + + await expect( + advisoryFail(commitsThenThrows, "t", "refused", { ifAllowed: true, claim, noRetry: true }), + ).rejects.toThrow("onChange rejected after the commit"); + expect(inner.get("t")?.status).toBe("errored"); +}); diff --git a/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts new file mode 100644 index 0000000000..dabf0edbd6 --- /dev/null +++ b/packages/orchestration/test/task-board/record-error-ends-for-good.test.ts @@ -0,0 +1,154 @@ +/** + * The recorder ends a final failure on this attempt. An ordinary error still + * retries, and so does a final failure about another row: an inner board's + * "don't retry" must not cost the outer row the attempts its filer chose. + */ +import { describe, expect, it } from "vitest"; +import { handler, sequencer } from "@flow-state-dev/core"; +import type { BlockContext } from "@flow-state-dev/core/types"; +import { runForTest, testBlock } from "@flow-state-dev/testing"; +import { z } from "zod"; +import { createRecordError, TaskEndsForGoodError } from "../../src/task-board/blocks/record-result"; +import { taskBoard, taskWorkerInputSchema } from "../../src/task-board"; +import { + createStateBackedTaskCollection, + isTerminalStatus, + ticketForClaim, + type TaskCollectionRef, + type TaskWorker, +} from "../../src/tasks"; +import { createFakeSequencerState } from "../helpers"; + +/** A claimed row with attempts to spare, and the recorder's context holding its claim. */ +async function claimedWithAttemptsLeft(): Promise<{ collection: TaskCollectionRef; ctx: BlockContext }> { + const state = createFakeSequencerState<{ tasks: Record }>({ tasks: {} }); + const collection = createStateBackedTaskCollection({ collectionId: "tasks", state }); + await collection.addTask({ id: "t", goal: "work", maxAttempts: 3 }); + const task = (await collection.claim("w"))!; + const bodyState = createFakeSequencerState<{ currentClaim?: unknown }>({ + currentClaim: ticketForClaim("tasks", task), + }); + return { collection, ctx: { sequencer: bodyState } as unknown as BlockContext }; +} + +function recorder(collection: TaskCollectionRef) { + return createRecordError({ name: "ends-for-good", collection: async () => collection, onError: "skip" }); +} + +describe("the error recorder and a failure no retry can change", () => { + it("fails the row for good on this attempt, with the worker's reason", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new TaskEndsForGoodError("refused: names no project", { taskId: "t" }), ctx); + + expect(collection.get("t")).toMatchObject({ status: "errored", attempts: 1, error: "refused: names no project" }); + }); + + it("cancels the row instead when the worker asks for a cancel", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest( + recorder(collection), + new TaskEndsForGoodError("refused: remote not allowed", { taskId: "t", ending: "cancelled" }), + ctx, + ); + + expect(collection.get("t")).toMatchObject({ status: "cancelled", attempts: 1 }); + }); + + it("still re-queues an ordinary failure with attempts left", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new Error("flaky"), ctx); + + expect(collection.get("t")?.status).toBe("pending"); + }); + + it("treats a marker naming another row as an ordinary failure", async () => { + const { collection, ctx } = await claimedWithAttemptsLeft(); + + await runForTest(recorder(collection), new TaskEndsForGoodError("an inner row's verdict", { taskId: "inner" }), ctx); + + expect(collection.get("t")?.status).toBe("pending"); + }); + + // A store that throws on a declined transition instead of honouring + // `ifAllowed` is the extension point the advisory seam exists for. A row + // someone else already cancelled must not turn the cancel arm's write into + // an error that escapes the recorder and abandons the board's siblings. + it("contains a declined cancel on a store that throws on an illegal transition", async () => { + const { collection: inner, ctx } = await claimedWithAttemptsLeft(); + const throwsWhenTerminal = + (verb: "fail" | "cancel") => + async (id: string, ...rest: unknown[]) => { + const row = inner.get(id); + if (row !== undefined && isTerminalStatus(row.status)) throw new Error(`illegal ${row.status} -> ${verb}`); + return (inner[verb] as (...args: unknown[]) => Promise)(id, ...rest); + }; + const collection = { ...inner, fail: throwsWhenTerminal("fail"), cancel: throwsWhenTerminal("cancel") } as unknown as TaskCollectionRef; + await inner.cancel("t", "a coordinator cancelled it"); + + await runForTest( + recorder(collection), + new TaskEndsForGoodError("refused", { taskId: "t", ending: "cancelled" }), + ctx, + ); + + expect(inner.get("t")).toMatchObject({ status: "cancelled", error: "a coordinator cancelled it" }); + }); +}); + +describe("a board nested inside a worker", () => { + /** + * The outer board's worker drains an inner board that fails (`onError: + * "fail"`), so the inner worker's error reaches the outer recorder. Returns + * how many times the outer worker ran. + */ + async function outerRunsWhenInnerThrows(innerError: (taskId: string) => Error): Promise { + let outerRuns = 0; + const innerWorker = handler({ + name: "inner-worker", + inputSchema: taskWorkerInputSchema, + outputSchema: z.null(), + execute: (input) => { + throw innerError(input.taskId); + }, + }) as TaskWorker; + const inner = taskBoard({ + name: "inner", + collection: { collectionId: "inner" }, + concurrency: 1, + dispatcher: "fifo", + workers: innerWorker, + initialTasks: [{ id: "inner-t", goal: "inner", maxAttempts: 1 }], + onError: "fail", + }); + const outerWorker = sequencer({ name: "outer-worker", inputSchema: taskWorkerInputSchema }) + .tap(() => { + outerRuns += 1; + }) + .tap(inner.drain) as unknown as TaskWorker; + const outer = taskBoard({ + name: "outer", + collection: { collectionId: "outer" }, + concurrency: 1, + dispatcher: "fifo", + workers: outerWorker, + initialTasks: [{ id: "outer-t", goal: "outer", maxAttempts: 3 }], + onError: "skip", + }); + + await testBlock(outer.drain, { input: undefined }); + return outerRuns; + } + + it("does not let the inner row's 'don't retry' end the outer row on its first attempt", async () => { + const runs = await outerRunsWhenInnerThrows((taskId) => new TaskEndsForGoodError("inner refused", { taskId })); + expect(runs).toBeGreaterThan(1); + }); + + it("control: a plain inner error gives the outer row its retry", async () => { + const runs = await outerRunsWhenInnerThrows(() => new Error("inner failed")); + expect(runs).toBeGreaterThan(1); + }); +});